ink to text for groupings

13 files changed, 366 insertions, 32 deletions

diff --git a/src/client/documents/Documents.ts b/src/client/documents/Documents.ts
index 9bcd23aa0..884cc781d 100644
--- a/src/client/documents/Documents.ts
+++ b/src/client/documents/Documents.ts
@@ -4,7 +4,7 @@ import { DateField } from "../../fields/DateField";
 import { Doc, DocListCast, DocListCastAsync, Field, HeightSym, Initializing, Opt, updateCachedAcls, WidthSym } from "../../fields/Doc";
 import { Id } from "../../fields/FieldSymbols";
 import { HtmlField } from "../../fields/HtmlField";
-import { InkField } from "../../fields/InkField";
+import { InkField, PointData } from "../../fields/InkField";
 import { List } from "../../fields/List";
 import { ProxyField } from "../../fields/Proxy";
 import { RichTextField } from "../../fields/RichTextField";
@@ -738,7 +738,7 @@ export namespace Docs {
             return linkDoc;
         }
 
-        export function InkDocument(color: string, tool: string, strokeWidth: number, strokeBezier: string, fillColor: string, arrowStart: string, arrowEnd: string, dash: string, points: { X: number, Y: number }[], options: DocumentOptions = {}) {
+        export function InkDocument(color: string, tool: string, strokeWidth: number, strokeBezier: string, fillColor: string, arrowStart: string, arrowEnd: string, dash: string, points: PointData[], options: DocumentOptions = {}) {
             const I = new Doc();
             I[Initializing] = true;
             I.type = DocumentType.INK;
@@ -1161,6 +1161,7 @@ export namespace DocUtils {
             created = Docs.Create.AudioDocument((field).url.href, resolved);
             layout = AudioBox.LayoutString;
         } else if (field instanceof InkField) {
+            console.log("Documents " + field.inkData)
             created = Docs.Create.InkDocument(ActiveInkColor(), CurrentUserUtils.SelectedTool, ActiveInkWidth(), ActiveInkBezierApprox(), ActiveFillColor(), ActiveArrowStart(), ActiveArrowEnd(), ActiveDash(), (field).inkData, resolved);
             layout = InkingStroke.LayoutString;
         } else if (field instanceof List && field[0] instanceof Doc) {
diff --git a/src/client/views/GestureOverlay.tsx b/src/client/views/GestureOverlay.tsx
index 50dca0a99..16a9cfaeb 100644
--- a/src/client/views/GestureOverlay.tsx
+++ b/src/client/views/GestureOverlay.tsx
@@ -41,6 +41,7 @@ export class GestureOverlay extends Touchable {
     @observable private _menuY: number = -300;
     @observable private _pointerY?: number;
     @observable private _points: { X: number, Y: number }[] = [];
+    @observable private _timeStamps: number[] = [];
     @observable private _strokes: InkData[] = [];
     @observable private _palette?: JSX.Element;
     @observable private _clipboardDoc?: JSX.Element;
@@ -59,6 +60,7 @@ export class GestureOverlay extends Touchable {
     private pointerIdentifier?: number;
     private _hands: Map<number, React.Touch[]> = new Map<number, React.Touch[]>();
     private _holdTimer: NodeJS.Timeout | undefined;
+    private _strokeStartTime: number = 0;
 
     protected _multiTouchDisposer?: InteractionUtils.MultiTouchEventDisposer;
 
@@ -418,6 +420,7 @@ export class GestureOverlay extends Touchable {
 
             this._strokes = [];
             this._points = [];
+            this._timeStamps = [];
             this._possibilities = [];
             document.removeEventListener("touchend", this.handleHandUp);
         }
@@ -493,6 +496,8 @@ export class GestureOverlay extends Touchable {
     onPointerDown = (e: React.PointerEvent) => {
         if (InteractionUtils.IsType(e, InteractionUtils.PENTYPE) || [InkTool.Highlighter, InkTool.Pen].includes(CurrentUserUtils.SelectedTool)) {
             this._points.push({ X: e.clientX, Y: e.clientY });
+            this._timeStamps.push(0);
+            this._strokeStartTime = new Date().getTime();
             setupMoveUpEvents(this, e, this.onPointerMove, this.onPointerUp, emptyFunction);
             // if (CurrentUserUtils.SelectedTool === InkTool.Highlighter) SetActiveInkColor("rgba(245, 230, 95, 0.75)");
         }
@@ -501,6 +506,7 @@ export class GestureOverlay extends Touchable {
     @action
     onPointerMove = (e: PointerEvent) => {
         this._points.push({ X: e.clientX, Y: e.clientY });
+        this._timeStamps.push(new Date().getTime() - this._strokeStartTime);
 
         if (this._points.length > 1) {
             const B = this.svgBounds;
@@ -547,6 +553,7 @@ export class GestureOverlay extends Touchable {
             const points = this._points.map(p => ({ X: p.X - B.left, Y: p.Y - B.top }));
             //push first points to so interactionUtil knows pointer is up
             this._points.push({ X: this._points[0].X, Y: this._points[0].Y });
+            this._timeStamps.push(this._timeStamps[0]);
 
             const initialPoint = this._points[0.];
             const xInGlass = initialPoint.X > (this._thumbX ?? Number.MAX_SAFE_INTEGER) && initialPoint.X < (this._thumbX ?? Number.MAX_SAFE_INTEGER) + (this.height);
@@ -558,6 +565,7 @@ export class GestureOverlay extends Touchable {
                     case ToolglassTools.InkToText:
                         this._strokes.push(new Array(...this._points));
                         this._points = [];
+                        this._timeStamps = [];
                         CognitiveServices.Inking.Appliers.InterpretStrokes(this._strokes).then((results) => {
                             const wordResults = results.filter((r: any) => r.category === "line");
                             const possibilities: string[] = [];
@@ -581,6 +589,7 @@ export class GestureOverlay extends Touchable {
                     case ToolglassTools.IgnoreGesture:
                         this.dispatchGesture(GestureUtils.Gestures.Stroke);
                         this._points = [];
+                        this._timeStamps = [];
                         break;
                 }
             }
@@ -589,6 +598,7 @@ export class GestureOverlay extends Touchable {
                 this.makePolygon(this.InkShape, false);
                 this.dispatchGesture(GestureUtils.Gestures.Stroke);
                 this._points = [];
+                this._timeStamps = [];
                 if (!CollectionFreeFormViewChrome.Instance?._keepPrimitiveMode) {
                     this.InkShape = "";
                     Doc.UserDoc().activeInkTool = InkTool.None;
@@ -635,9 +645,11 @@ export class GestureOverlay extends Touchable {
                     this.dispatchGesture(GestureUtils.Gestures.Stroke);
                 }
                 this._points = [];
+                this._timeStamps = [];
             }
         } else {
             this._points = [];
+            this._timeStamps = [];
         }
         CollectionFreeFormViewChrome.Instance?.primCreated();
     }
@@ -686,6 +698,7 @@ export class GestureOverlay extends Touchable {
             }
         }
         this._points = [];
+        this._timeStamps = [];
         switch (shape) {
             //must push an extra point in the end so InteractionUtils knows pointer is up.
             //must be (points[0].X,points[0]-1)
@@ -808,7 +821,8 @@ export class GestureOverlay extends Touchable {
                         points: stroke ?? this._points,
                         gesture: gesture as any,
                         bounds: this.getBounds(stroke ?? this._points),
-                        text: data
+                        text: data,
+                        times: this._timeStamps
                     }
                 }
             )
diff --git a/src/client/views/InkTranscription.scss b/src/client/views/InkTranscription.scss
new file mode 100644
index 000000000..bbb0a1afa
--- /dev/null
+++ b/src/client/views/InkTranscription.scss
@@ -0,0 +1,5 @@
+.ink-transcription {
+    .error-msg {
+        display: none !important;
+    }
+}
\ No newline at end of file
diff --git a/src/client/views/InkTranscription.tsx b/src/client/views/InkTranscription.tsx
new file mode 100644
index 000000000..950f622dd
--- /dev/null
+++ b/src/client/views/InkTranscription.tsx
@@ -0,0 +1,188 @@
+import { timeStamp } from 'console';
+import * as iink from 'iink-js';
+import { action, observable } from 'mobx';
+import * as React from 'react';
+import { Doc } from '../../fields/Doc';
+import { InkData, InkField } from "../../fields/InkField";
+import { Cast, NumCast, StrCast } from '../../fields/Types';
+import { Docs } from "../documents/Documents";
+import './InkTranscription.scss';
+
+export class InkTranscription extends React.Component {
+    static Instance: InkTranscription;
+
+    @observable _mathRegister: any;
+    @observable _mathRef: any;
+    @observable _textRegister: any;
+    @observable _textRef: any;
+    private addDocument?: (doc: Doc | Doc[]) => boolean;
+    private bounds: { x: number, y: number, width: number, height: number } = { x: 0, y: 0, width: 0, height: 0 };
+
+    constructor(props: Readonly<{}>) {
+        super(props);
+
+        InkTranscription.Instance = this;
+    }
+
+    componentWillUnmount() {
+        this._mathRef.removeEventListener('exported', (e: any) => this.exportInk(e, this._mathRef));
+        this._textRef.removeEventListener('exported', (e: any) => this.exportInk(e, this._textRef));
+
+        this._mathRef.removeEventListener('changed', (e: any) => this.callExport(this._mathRef));
+        this._textRef.removeEventListener('changed', (e: any) => this.callExport(this._textRef));
+    }
+
+    @action
+    setMathRef = (r: any) => {
+        if (!this._mathRegister) {
+            this._mathRegister = r ? iink.register(r, {
+                recognitionParams: {
+                    type: 'MATH',
+                    protocol: 'WEBSOCKET',
+                    server: {
+                        host: 'cloud.myscript.com',
+                        applicationKey: '7277ec34-0c2e-4ee1-9757-ccb657e3f89f',
+                        hmacKey: 'f5cb18f2-1f95-4ddb-96ac-3f7c888dffc1',
+                        websocket: {
+                            pingEnabled: false,
+                            autoReconnect: true
+                        }
+                    },
+                    iink: {
+                        math: {
+                            mimeTypes: ['application/x-latex', 'application/vnd.myscript.jiix']
+                        },
+                        export: {
+                            jiix: {
+                                strokes: true
+                            }
+                        }
+                    }
+                },
+                triggers: {
+                    exportContent: 'DEMAND'
+                }
+            }) : null;
+        }
+
+        r.addEventListener('exported', (e: any) => this.exportInk(e, this._mathRef));
+        r.addEventListener('changed', (e: any) => this.callExport(this._mathRef));
+
+        return this._mathRef = r;
+    }
+
+    @action
+    setTextRef = (r: any) => {
+        if (!this._textRegister) {
+            this._textRegister = r ? iink.register(r, {
+                recognitionParams: {
+                    type: 'TEXT',
+                    protocol: 'WEBSOCKET',
+                    server: {
+                        host: 'cloud.myscript.com',
+                        applicationKey: '7277ec34-0c2e-4ee1-9757-ccb657e3f89f',
+                        hmacKey: 'f5cb18f2-1f95-4ddb-96ac-3f7c888dffc1',
+                        websocket: {
+                            pingEnabled: false,
+                            autoReconnect: true
+                        }
+                    },
+                    iink: {
+                        text: {
+                            mimeTypes: ['text/plain']
+                        },
+                        export: {
+                            jiix: {
+                                strokes: true
+                            }
+                        }
+                    }
+                },
+                triggers: {
+                    exportContent: 'DEMAND'
+                }
+            }) : null;
+        }
+
+        r.addEventListener('exported', (e: any) => this.exportInk(e, this._textRef));
+        r.addEventListener('changed', (e: any) => this.callExport(this._textRef));
+
+        return this._textRef = r;
+    }
+
+    transcribeInk = (inkdocs: Doc[], math: boolean, bounds: { x: number, y: number, width: number, height: number }, addDocument?: (doc: Doc | Doc[]) => boolean) => {
+        const strokes: InkData[] = [];
+        inkdocs.filter(i => Cast(i.data, InkField)).forEach(i => {
+            // TODO: interpolate missing times stamps
+            const d = Cast(i.data, InkField, null);
+            const left = Math.min(...d?.inkData.map(pd => pd.X) ?? [0]);
+            const top = Math.min(...d?.inkData.map(pd => pd.Y) ?? [0]);
+            strokes.push(d.inkData.map(pd => ({ X: pd.X + NumCast(i.x) - left, Y: pd.Y + NumCast(i.y) - top, time: pd.time })));
+        });
+
+        this.addDocument = addDocument;
+        this.bounds = bounds;
+
+        const pointerData = { "events": strokes.map(stroke => this.inkJSON(stroke)) };
+        // console.log(JSON.stringify(pointerData));
+        const processGestures = false;
+
+        if (math) {
+            this._mathRef.editor.pointerEvents(pointerData, processGestures);
+        }
+        else {
+            this._textRef.editor.pointerEvents(pointerData, processGestures);
+        }
+    }
+
+    callExport(ref: any) {
+        if (ref.editor.canExport) {
+            ref.editor.export_();
+        }
+    }
+
+    inkJSON = (stroke: InkData) => {
+        return {
+            "pointerType": "PEN",
+            "pointerId": 1,
+            "x": stroke.map(point => point.X),
+            "y": stroke.map(point => point.Y),
+            "t": stroke.map(point => point.time),
+            "p": new Array(stroke.length).fill(1.0)
+        };
+    }
+
+    exportInk = (e: any, ref: any) => {
+        const exports = e.detail.exports;
+        if (exports) {
+            if (exports['application/x-latex']) {
+                const latex = exports['application/x-latex'];
+                console.log(latex);
+
+                this.addDocument?.(Docs.Create.EquationDocument({ title: latex, x: this.bounds.width, y: 0, _width: this.bounds.width, _height: this.bounds.height }));
+            }
+            else if (exports['text/plain']) {
+                const text = exports['text/plain'];
+                console.log(text);
+                this.addDocument?.(Docs.Create.TextDocument(text, { title: text, x: this.bounds.width, y: 0, _width: this.bounds.width, _height: this.bounds.height }));
+            }
+
+            ref.editor.clear();
+        }
+    }
+
+    render() {
+        return (
+            <div className="ink-transcription">
+                <div className='math-editor'
+                    ref={this.setMathRef}
+                    touch-action="none">
+                </div>
+                <div className='text-editor'
+                    ref={this.setTextRef}
+                    touch-action="none">
+                </div>
+            </div>
+        )
+    }
+}
\ No newline at end of file
diff --git a/src/client/views/MainView.tsx b/src/client/views/MainView.tsx
index 8c0795881..84c1037dd 100644
--- a/src/client/views/MainView.tsx
+++ b/src/client/views/MainView.tsx
@@ -44,6 +44,7 @@ import { GestureOverlay } from './GestureOverlay';
 import { DASHBOARD_SELECTOR_HEIGHT, LEFT_MENU_WIDTH } from './global/globalCssVariables.scss';
 import { Colors } from './global/globalEnums';
 import { KeyManager } from './GlobalKeyHandler';
+import { InkTranscription } from './InkTranscription';
 import { LightboxView } from './LightboxView';
 import { LinkMenu } from './linking/LinkMenu';
 import "./MainView.scss";
@@ -184,7 +185,7 @@ export class MainView extends React.Component {
         fa.faArrowAltCircleDown, fa.faArrowAltCircleUp, fa.faArrowAltCircleLeft, fa.faArrowAltCircleRight, fa.faStopCircle, fa.faCheckCircle, fa.faGripVertical,
         fa.faSortUp, fa.faSortDown, fa.faTable, fa.faTh, fa.faThList, fa.faProjectDiagram, fa.faSignature, fa.faColumns, fa.faChevronCircleUp, fa.faUpload, fa.faBorderAll,
         fa.faBraille, fa.faChalkboard, fa.faPencilAlt, fa.faEyeSlash, fa.faSmile, fa.faIndent, fa.faOutdent, fa.faChartBar, fa.faBan, fa.faPhoneSlash, fa.faGripLines,
-        fa.faSave, fa.faBookmark, fa.faList, fa.faListOl, fa.faFolderPlus, fa.faLightbulb, fa.faBookOpen, fa.faMapMarkerAlt]);
+        fa.faSave, fa.faBookmark, fa.faList, fa.faListOl, fa.faFolderPlus, fa.faLightbulb, fa.faBookOpen, fa.faMapMarkerAlt, fa.faSquareRootAlt]);
         this.initAuthenticationRouters();
     }
 
@@ -636,6 +637,7 @@ export class MainView extends React.Component {
             <OverlayView />
             <TimelineMenu />
             <RichTextMenu />
+            <InkTranscription />
             {this.snapLines}
             <div className="mainView-webRef" ref={this.makeWebRef} />
             <LightboxView PanelWidth={this._windowWidth} PanelHeight={this._windowHeight} maxBorder={[200, 50]} />
diff --git a/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx b/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx
index e2ea81392..7d60387de 100644
--- a/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx
+++ b/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx
@@ -50,6 +50,7 @@ import { CollectionFreeFormRemoteCursors } from "./CollectionFreeFormRemoteCurso
 import "./CollectionFreeFormView.scss";
 import { MarqueeView } from "./MarqueeView";
 import React = require("react");
+import { InkTranscription } from "../../InkTranscription";
 
 export const panZoomSchema = createSchema({
     _panX: "number",
@@ -492,7 +493,9 @@ export class CollectionFreeFormView extends CollectionSubView<Partial<collection
             case GestureUtils.Gestures.Stroke:
                 const points = ge.points;
                 const B = this.getTransform().transformBounds(ge.bounds.left, ge.bounds.top, ge.bounds.width, ge.bounds.height);
-                const inkDoc = Docs.Create.InkDocument(ActiveInkColor(), CurrentUserUtils.SelectedTool, ActiveInkWidth(), ActiveInkBezierApprox(), ActiveFillColor(), ActiveArrowStart(), ActiveArrowEnd(), ActiveDash(), points,
+                const times = ge.times;
+                const strokes: PointData[] = !times || times.length == 0 ? points : points.map((pt, i) => { return { X: pt.X, Y: pt.Y, time: times[i] } });
+                const inkDoc = Docs.Create.InkDocument(ActiveInkColor(), CurrentUserUtils.SelectedTool, ActiveInkWidth(), ActiveInkBezierApprox(), ActiveFillColor(), ActiveArrowStart(), ActiveArrowEnd(), ActiveDash(), strokes,
                     { title: "ink stroke", x: B.x - ActiveInkWidth() / 2, y: B.y - ActiveInkWidth() / 2, _width: B.width + ActiveInkWidth(), _height: B.height + ActiveInkWidth() });
                 this.addDocument(inkDoc);
                 e.stopPropagation();
@@ -1477,6 +1480,11 @@ export class CollectionFreeFormView extends CollectionSubView<Partial<collection
         appearanceItems.push({ description: `${this.fitToContent ? "Make Zoomable" : "Scale to Window"}`, event: () => this.Document._fitToBox = !this.fitToContent, icon: !this.fitToContent ? "expand-arrows-alt" : "compress-arrows-alt" });
         this.props.ContainingCollectionView &&
             appearanceItems.push({ description: "Ungroup collection", event: this.promoteCollection, icon: "table" });
+
+        this.props.Document._isGroup && this.childDocs.filter(s => s.type === DocumentType.INK).length > 0 && appearanceItems.push({ description: "Ink to text", event: () => this.transcribeStrokes(false), icon: "font" });
+
+        this.props.Document._isGroup && this.childDocs.filter(s => s.type === DocumentType.INK).length > 0 && appearanceItems.push({ description: "Ink to math", event: () => this.transcribeStrokes(true), icon: "square-root-alt" });
+
         !Doc.UserDoc().noviceMode ? appearanceItems.push({ description: "Arrange contents in grid", event: this.layoutDocsInGrid, icon: "table" }) : null;
         !appearance && ContextMenu.Instance.addItem({ description: "Appearance...", subitems: appearanceItems, icon: "eye" });
 
@@ -1536,6 +1544,15 @@ export class CollectionFreeFormView extends CollectionSubView<Partial<collection
         input.click();
     }
 
+    @undoBatch
+    @action
+    transcribeStrokes = (math: boolean) => {
+        if (this.props.Document._isGroup) {
+            const inkdocs = this.childDocs.filter(s => s.type === DocumentType.INK);
+            InkTranscription.Instance.transcribeInk(inkdocs, math, { x: NumCast(this.layoutDoc.x) ?? 0, y: NumCast(this.layoutDoc.y) ?? 0, width: NumCast(this.layoutDoc._width) ?? 0, height: NumCast(this.layoutDoc._height) ?? 0 }, this.addDocument);
+        }
+    }
+
     @action
     setupDragLines = (snapToDraggedDoc: boolean = false) => {
         const activeDocs = this.getActiveDocuments();
diff --git a/src/client/views/collections/collectionFreeForm/MarqueeOptionsMenu.tsx b/src/client/views/collections/collectionFreeForm/MarqueeOptionsMenu.tsx
index 1f59f9732..8a8b528f6 100644
--- a/src/client/views/collections/collectionFreeForm/MarqueeOptionsMenu.tsx
+++ b/src/client/views/collections/collectionFreeForm/MarqueeOptionsMenu.tsx
@@ -14,7 +14,6 @@ export class MarqueeOptionsMenu extends AntimodeMenu<AntimodeMenuProps> {
     public createCollection: (e: KeyboardEvent | React.PointerEvent | undefined, group?: boolean) => void = unimplementedFunction;
     public delete: (e: KeyboardEvent | React.PointerEvent | undefined) => void = unimplementedFunction;
     public summarize: (e: KeyboardEvent | React.PointerEvent | undefined) => void = unimplementedFunction;
-    public inkToText: (e: KeyboardEvent | React.PointerEvent | undefined) => void = unimplementedFunction;
     public showMarquee: () => void = unimplementedFunction;
     public hideMarquee: () => void = unimplementedFunction;
     public pinWithView: (e: KeyboardEvent | React.PointerEvent | undefined) => void = unimplementedFunction;
@@ -64,16 +63,6 @@ export class MarqueeOptionsMenu extends AntimodeMenu<AntimodeMenuProps> {
                 </button>
             </Tooltip>,
         ];
-        if (false && !SelectionManager.Views().some(v => v.props.Document.type !== DocumentType.INK)) {
-            buttons.push(
-                <Tooltip key="inkToText" title={<div className="dash-tooltip">Change to Text</div>} placement="bottom">
-                    <button
-                        className="antimodeMenu-button"
-                        onPointerDown={this.inkToText}>
-                        <FontAwesomeIcon icon="font" size="lg" />
-                    </button>
-                </Tooltip>);
-        }
         return this.getElement(buttons);
     }
 }
\ No newline at end of file
diff --git a/src/client/views/collections/collectionFreeForm/MarqueeView.tsx b/src/client/views/collections/collectionFreeForm/MarqueeView.tsx
index b10b0912f..a828cd981 100644
--- a/src/client/views/collections/collectionFreeForm/MarqueeView.tsx
+++ b/src/client/views/collections/collectionFreeForm/MarqueeView.tsx
@@ -1,14 +1,15 @@
 import { action, computed, observable } from "mobx";
 import { observer } from "mobx-react";
-import { AclAugment, AclAdmin, AclEdit, DataSym, Doc, Opt } from "../../../../fields/Doc";
+import { AclAdmin, AclAugment, AclEdit, DataSym, Doc, Opt } from "../../../../fields/Doc";
 import { Id } from "../../../../fields/FieldSymbols";
 import { InkData, InkField, InkTool } from "../../../../fields/InkField";
 import { List } from "../../../../fields/List";
 import { RichTextField } from "../../../../fields/RichTextField";
 import { SchemaHeaderField } from "../../../../fields/SchemaHeaderField";
 import { Cast, FieldValue, NumCast, StrCast } from "../../../../fields/Types";
+import { ImageField } from "../../../../fields/URLField";
 import { GetEffectiveAcl } from "../../../../fields/util";
-import { Utils, intersectRect, returnFalse } from "../../../../Utils";
+import { intersectRect, returnFalse, Utils } from "../../../../Utils";
 import { CognitiveServices } from "../../../cognitive_services/CognitiveServices";
 import { Docs, DocumentOptions, DocUtils } from "../../../documents/Documents";
 import { DocumentType } from "../../../documents/DocumentTypes";
@@ -21,18 +22,17 @@ import { ContextMenu } from "../../ContextMenu";
 import { FormattedTextBox } from "../../nodes/formattedText/FormattedTextBox";
 import { PresBox } from "../../nodes/trails/PresBox";
 import { PresMovement } from "../../nodes/trails/PresEnums";
+import { VideoBox } from "../../nodes/VideoBox";
+import { pasteImageBitmap } from "../../nodes/WebBoxRenderer";
 import { PreviewCursor } from "../../PreviewCursor";
+import { StyleLayers } from "../../StyleProvider";
 import { CollectionDockingView } from "../CollectionDockingView";
 import { SubCollectionViewProps } from "../CollectionSubView";
 import { CollectionView } from "../CollectionView";
+import { TreeView } from "../TreeView";
 import { MarqueeOptionsMenu } from "./MarqueeOptionsMenu";
 import "./MarqueeView.scss";
 import React = require("react");
-import { StyleLayers } from "../../StyleProvider";
-import { TreeView } from "../TreeView";
-import { VideoBox } from "../../nodes/VideoBox";
-import { ImageField, WebField } from "../../../../fields/URLField";
-import { pasteImageBitmap } from "../../nodes/WebBoxRenderer";
 
 interface MarqueeViewProps {
     getContainerTransform: () => Transform;
@@ -265,7 +265,7 @@ export class MarqueeView extends React.Component<SubCollectionViewProps & Marque
             MarqueeOptionsMenu.Instance.createCollection = this.collection;
             MarqueeOptionsMenu.Instance.delete = this.delete;
             MarqueeOptionsMenu.Instance.summarize = this.summary;
-            MarqueeOptionsMenu.Instance.inkToText = this.syntaxHighlight;
+            // MarqueeOptionsMenu.Instance.inkToText = this.syntaxHighlight;
             MarqueeOptionsMenu.Instance.showMarquee = this.showMarquee;
             MarqueeOptionsMenu.Instance.hideMarquee = this.hideMarquee;
             MarqueeOptionsMenu.Instance.jumpTo(e.clientX, e.clientY);
diff --git a/src/client/views/nodes/EquationBox.scss b/src/client/views/nodes/EquationBox.scss
index 6c9d53d10..229a1a485 100644
--- a/src/client/views/nodes/EquationBox.scss
+++ b/src/client/views/nodes/EquationBox.scss
@@ -1,3 +1,39 @@
+@import "../global/globalCssVariables.scss";
+
 .equationBox-cont {
     transform-origin: top left;
+    overflow: visible;
+    width: 100%;
+    height: 100%;
+}
+
+.button {
+    position: absolute;
+    display: flex;
+    justify-content: center;
+    align-items: center;
+    top: 0;
+    right: 0;
+    width: 20px;
+    height: 20px;
+    border-radius: 5px;
+    background: $dark-gray;
+    color: white;
+
+    svg {
+        width: 12px;
+        height: 12px;
+    }
+}
+
+.ink-editor {
+    top: 20px;
+    min-width: 500px;
+    min-height: 300px;
+    background: $light-gray;
+    pointer-events: all;
+
+    button {
+        float: right;
+    }
 }
\ No newline at end of file
diff --git a/src/client/views/nodes/EquationBox.tsx b/src/client/views/nodes/EquationBox.tsx
index c170f9867..c0c4fab09 100644
--- a/src/client/views/nodes/EquationBox.tsx
+++ b/src/client/views/nodes/EquationBox.tsx
@@ -1,5 +1,7 @@
+import { FontAwesomeIcon } from "@fortawesome/react-fontawesome";
 import EquationEditor from 'equation-editor-react';
-import { action, reaction } from 'mobx';
+import * as iink from 'iink-js';
+import { action, observable, reaction } from 'mobx';
 import { observer } from 'mobx-react';
 import * as React from 'react';
 import { WidthSym } from '../../../fields/Doc';
@@ -12,12 +14,16 @@ import { LightboxView } from '../LightboxView';
 import './EquationBox.scss';
 import { FieldView, FieldViewProps } from './FieldView';
 
-
 @observer
 export class EquationBox extends ViewBoxBaseComponent<FieldViewProps>() {
     public static LayoutString(fieldKey: string) { return FieldView.LayoutString(EquationBox, fieldKey); }
     public static SelectOnLoad: string = "";
+
     _ref: React.RefObject<EquationEditor> = React.createRef();
+    @observable _inkOpen = false;
+    @observable _inkEditor: any;
+    @observable _inkRef: any;
+
     componentDidMount() {
         if (EquationBox.SelectOnLoad === this.rootDoc[Id] && (!LightboxView.LightboxDoc || LightboxView.IsLightboxDocView(this.props.docViewPath()))) {
             this.props.select(false);
@@ -39,7 +45,11 @@ export class EquationBox extends ViewBoxBaseComponent<FieldViewProps>() {
                 }
             }, { fireImmediately: true });
     }
-    plot: any;
+
+    componentWillUnmount() {
+        this._inkRef.removeEventListener('exported', this.exportInk);
+    }
+
     @action
     keyPressed = (e: KeyboardEvent) => {
         const _height = Number(getComputedStyle(this._ref.current!.element.current).height.replace("px", ""));
@@ -65,6 +75,7 @@ export class EquationBox extends ViewBoxBaseComponent<FieldViewProps>() {
         }
         if (e.key === "Backspace" && !this.dataDoc.text) this.props.removeDocument?.(this.rootDoc);
     }
+
     onChange = (str: string) => {
         this.dataDoc.text = str;
         const style = this._ref.current && getComputedStyle(this._ref.current.element.current);
@@ -75,6 +86,58 @@ export class EquationBox extends ViewBoxBaseComponent<FieldViewProps>() {
             this.layoutDoc._height = Math.max(25, _height);
         }
     }
+
+    @action
+    toggleInk = (e: React.PointerEvent) => {
+        e.stopPropagation();
+
+        this._inkOpen = !this._inkOpen;
+
+        if (!this._inkEditor) {
+            this._inkEditor = this._inkRef ? iink.register(this._inkRef, {
+                recognitionParams: {
+                    type: 'MATH',
+                    protocol: 'WEBSOCKET',
+                    server: {
+                        host: 'cloud.myscript.com',
+                        applicationKey: '7277ec34-0c2e-4ee1-9757-ccb657e3f89f',
+                        hmacKey: 'f5cb18f2-1f95-4ddb-96ac-3f7c888dffc1',
+                    },
+                    iink: {
+                        math: {
+                            mimeTypes: ['application/x-latex', 'application/vnd.myscript.jiix']
+                        },
+                        export: {
+                            jiix: {
+                                strokes: true
+                            }
+                        }
+                    }
+                }
+            }) : null;
+
+            this._inkRef.addEventListener('exported', this.exportInk)
+        }
+    }
+
+    convertInk = (e: React.MouseEvent) => {
+        this._inkRef.editor.convert();
+    }
+
+    clearInk = (e: React.MouseEvent) => {
+        this._inkRef.editor.clear();
+        this.onChange("");
+    }
+
+    exportInk = (e: any) => {
+        const exports = e.detail.exports;
+        if (exports && exports['application/x-latex']) {
+            this.onChange(exports['application/x-latex']);
+            console.log(JSON.parse(exports['application/vnd.myscript.jiix']).expressions[0].items[0]);
+        }
+    }
+
+
     render() {
         TraceMobx();
         const scale = (this.props.scaling?.() || 1) * NumCast(this.layoutDoc._viewScale, 1);
@@ -82,18 +145,34 @@ export class EquationBox extends ViewBoxBaseComponent<FieldViewProps>() {
             onPointerDown={e => !e.ctrlKey && e.stopPropagation()}
             style={{
                 transform: `scale(${scale})`,
-                width: `${100 / scale}%`,
+                width: `calc(${100 / scale}% + 25px)`,
                 height: `${100 / scale}%`,
                 pointerEvents: !this.props.isSelected() ? "none" : undefined,
             }}
-            onKeyDown={e => e.stopPropagation()}
-        >
+            onKeyDown={e => e.stopPropagation()}>
+
             <EquationEditor ref={this._ref}
                 value={this.dataDoc.text || "x"}
                 spaceBehavesLikeTab={true}
                 onChange={this.onChange}
                 autoCommands="pi theta sqrt sum prod alpha beta gamma rho"
                 autoOperatorNames="sin cos tan" />
+
+            <div className="button"
+                style={{ display: this.props.isContentActive() ? "flex" : "none" }}
+                onPointerDown={this.toggleInk}>
+                <FontAwesomeIcon icon="pencil-alt" />
+            </div>
+
+            <div className='ink-editor'
+                ref={action((r: any) => this._inkRef = r)}
+                id="editor" onPointerDown={(e) => e.stopPropagation()}
+                touch-action="none"
+                style={{ display: this._inkOpen && this.props.isContentActive() ? "block" : "none" }}>
+                <button onClick={this.convertInk}>convert</button>
+                <button onClick={this.clearInk}>clear</button>
+            </div>
+
         </div>);
     }
 }
\ No newline at end of file
diff --git a/src/fields/InkField.ts b/src/fields/InkField.ts
index 31024e805..48271d3d3 100644
--- a/src/fields/InkField.ts
+++ b/src/fields/InkField.ts
@@ -19,6 +19,7 @@ export enum InkTool {
 export interface PointData {
     X: number;
     Y: number;
+    time?: number;
 }
 
 export type Segment = Array<Bezier>;
@@ -83,7 +84,7 @@ export class InkField extends ObjectField {
     }
 
     [ToScriptString]() {
-        return "new InkField([" + this.inkData.map(i => `{X: ${i.X}, Y: ${i.Y}} `) + "])";
+        return "new InkField([" + this.inkData.map(i => `{X: ${i.X}, Y: ${i.Y}, time: ${i.time || 0}} `) + "])";
     }
     [ToString]() {
         return "InkField";
diff --git a/src/pen-gestures/GestureUtils.ts b/src/pen-gestures/GestureUtils.ts
index 65f2bf80c..cf59cb3c6 100644
--- a/src/pen-gestures/GestureUtils.ts
+++ b/src/pen-gestures/GestureUtils.ts
@@ -8,7 +8,8 @@ export namespace GestureUtils {
             readonly gesture: Gestures,
             readonly points: PointData[],
             readonly bounds: Rect,
-            readonly text?: any
+            readonly text?: any,
+            readonly times?: number[]
         ) { }
     }
 
diff --git a/src/typings/index.d.ts b/src/typings/index.d.ts
index 068ac2159..bc7727426 100644
--- a/src/typings/index.d.ts
+++ b/src/typings/index.d.ts
@@ -8,6 +8,7 @@ declare module 'webrtc-adapter';
 declare module 'bezier-curve';
 declare module 'fit-curve';
 declare module 'react-audio-waveform';
+declare module 'iink-js';
 
 declare module 'reveal';
 declare module 'react-reveal';