merged and getting started on mapping algo

7 files changed, 45 insertions, 96 deletions

diff --git a/src/client/documents/Documents.ts b/src/client/documents/Documents.ts
index 884cc781d..aee9ef49d 100644
--- a/src/client/documents/Documents.ts
+++ b/src/client/documents/Documents.ts
@@ -1161,7 +1161,6 @@ export namespace DocUtils {
             created = Docs.Create.AudioDocument((field).url.href, resolved);
             layout = AudioBox.LayoutString;
         } else if (field instanceof InkField) {
-            console.log("Documents " + field.inkData)
             created = Docs.Create.InkDocument(ActiveInkColor(), CurrentUserUtils.SelectedTool, ActiveInkWidth(), ActiveInkBezierApprox(), ActiveFillColor(), ActiveArrowStart(), ActiveArrowEnd(), ActiveDash(), (field).inkData, resolved);
             layout = InkingStroke.LayoutString;
         } else if (field instanceof List && field[0] instanceof Doc) {
diff --git a/src/client/views/GestureOverlay.tsx b/src/client/views/GestureOverlay.tsx
index e689cd43c..13fa62048 100644
--- a/src/client/views/GestureOverlay.tsx
+++ b/src/client/views/GestureOverlay.tsx
@@ -42,7 +42,6 @@ export class GestureOverlay extends Touchable {
     @observable private _menuY: number = -300;
     @observable private _pointerY?: number;
     @observable private _points: { X: number, Y: number }[] = [];
-    @observable private _timeStamps: number[] = [];
     @observable private _strokes: InkData[] = [];
     @observable private _palette?: JSX.Element;
     @observable private _clipboardDoc?: JSX.Element;
@@ -61,7 +60,6 @@ export class GestureOverlay extends Touchable {
     private pointerIdentifier?: number;
     private _hands: Map<number, React.Touch[]> = new Map<number, React.Touch[]>();
     private _holdTimer: NodeJS.Timeout | undefined;
-    private _strokeStartTime: number = 0;
 
     protected _multiTouchDisposer?: InteractionUtils.MultiTouchEventDisposer;
 
@@ -299,7 +297,7 @@ export class GestureOverlay extends Touchable {
             pointer = fingers.reduce((a, v) => a.clientX < v.clientX || v.identifier === thumb.identifier ? a : v);
         }
         else {
-            console.log("not hand");
+            // console.log("not hand");
         }
         this.pointerIdentifier = pointer?.identifier;
 
@@ -425,7 +423,6 @@ export class GestureOverlay extends Touchable {
 
             this._strokes = [];
             this._points = [];
-            this._timeStamps = [];
             this._possibilities = [];
             document.removeEventListener("touchend", this.handleHandUp);
         }
@@ -509,8 +506,6 @@ export class GestureOverlay extends Touchable {
                 CurrentUserUtils.SelectedTool = InkTool.Write;
             }
             this._points.push({ X: e.clientX, Y: e.clientY });
-            this._timeStamps.push(0);
-            this._strokeStartTime = new Date().getTime();
             setupMoveUpEvents(this, e, this.onPointerMove, this.onPointerUp, emptyFunction);
             // if (CurrentUserUtils.SelectedTool === InkTool.Highlighter) SetActiveInkColor("rgba(245, 230, 95, 0.75)");
         }
@@ -519,7 +514,6 @@ export class GestureOverlay extends Touchable {
     @action
     onPointerMove = (e: PointerEvent) => {
         this._points.push({ X: e.clientX, Y: e.clientY });
-        this._timeStamps.push(new Date().getTime() - this._strokeStartTime);
 
         if (this._points.length > 1) {
             const B = this.svgBounds;
@@ -566,7 +560,6 @@ export class GestureOverlay extends Touchable {
             const points = this._points.map(p => ({ X: p.X - B.left, Y: p.Y - B.top }));
             //push first points to so interactionUtil knows pointer is up
             this._points.push({ X: this._points[0].X, Y: this._points[0].Y });
-            this._timeStamps.push(this._timeStamps[0]);
 
             const initialPoint = this._points[0.];
             const xInGlass = initialPoint.X > (this._thumbX ?? Number.MAX_SAFE_INTEGER) && initialPoint.X < (this._thumbX ?? Number.MAX_SAFE_INTEGER) + (this.height);
@@ -578,7 +571,6 @@ export class GestureOverlay extends Touchable {
                     case ToolglassTools.InkToText:
                         this._strokes.push(new Array(...this._points));
                         this._points = [];
-                        this._timeStamps = [];
                         CognitiveServices.Inking.Appliers.InterpretStrokes(this._strokes).then((results) => {
                             const wordResults = results.filter((r: any) => r.category === "line");
                             const possibilities: string[] = [];
@@ -602,7 +594,6 @@ export class GestureOverlay extends Touchable {
                     case ToolglassTools.IgnoreGesture:
                         this.dispatchGesture(GestureUtils.Gestures.Stroke);
                         this._points = [];
-                        this._timeStamps = [];
                         break;
                 }
             }
@@ -611,7 +602,6 @@ export class GestureOverlay extends Touchable {
                 this.makePolygon(this.InkShape, false);
                 this.dispatchGesture(GestureUtils.Gestures.Stroke);
                 this._points = [];
-                this._timeStamps = [];
                 if (!CollectionFreeFormViewChrome.Instance?._keepPrimitiveMode) {
                     this.InkShape = "";
                     Doc.UserDoc().activeInkTool = InkTool.None;
@@ -638,7 +628,7 @@ export class GestureOverlay extends Touchable {
                 if (!actionPerformed) {
                     const newPoints = this._points.reduce((p, pts) => { p.push([pts.X, pts.Y]); return p; }, [] as number[][]);
                     newPoints.pop();
-                    console.log("getting to bezier math");
+                    // console.log("getting to bezier math");
                     const controlPoints: { X: number, Y: number }[] = [];
 
                     const bezierCurves = fitCurve(newPoints, 10);
@@ -649,21 +639,19 @@ export class GestureOverlay extends Touchable {
                         controlPoints.push({ X: curve[2][0], Y: curve[2][1] });
                         controlPoints.push({ X: curve[3][0], Y: curve[3][1] });
 
-                    }   
+                    }
                     const dist = Math.sqrt((controlPoints[0].X - controlPoints.lastElement().X) * (controlPoints[0].X - controlPoints.lastElement().X) +
                         (controlPoints[0].Y - controlPoints.lastElement().Y) * (controlPoints[0].Y - controlPoints.lastElement().Y));
                     if (controlPoints.length > 4 && dist < 10) controlPoints[controlPoints.length - 1] = controlPoints[0];
-                        this._points = controlPoints;
+                    this._points = controlPoints;
                     this.dispatchGesture(GestureUtils.Gestures.Stroke);
                     // TODO: nda - check inks to group here
                     checkInksToGroup();
                 }
                 this._points = [];
-                this._timeStamps = [];
             }
         } else {
             this._points = [];
-            this._timeStamps = [];
         }
         CollectionFreeFormViewChrome.Instance?.primCreated();
     }
@@ -712,7 +700,6 @@ export class GestureOverlay extends Touchable {
             }
         }
         this._points = [];
-        this._timeStamps = [];
         switch (shape) {
             //must push an extra point in the end so InteractionUtils knows pointer is up.
             //must be (points[0].X,points[0]-1)
@@ -836,7 +823,6 @@ export class GestureOverlay extends Touchable {
                         gesture: gesture as any,
                         bounds: this.getBounds(stroke ?? this._points),
                         text: data,
-                        times: this._timeStamps
                     }
                 }
             )
diff --git a/src/client/views/InkTranscription.tsx b/src/client/views/InkTranscription.tsx
index 3b464db10..c276e6e63 100644
--- a/src/client/views/InkTranscription.tsx
+++ b/src/client/views/InkTranscription.tsx
@@ -1,13 +1,13 @@
-import { timeStamp } from 'console';
 import * as iink from 'iink-js';
 import { action, observable } from 'mobx';
 import * as React from 'react';
-import { Doc } from '../../fields/Doc';
+import { Doc, DocListCast } from '../../fields/Doc';
 import { InkData, InkField } from "../../fields/InkField";
-import { Cast, NumCast, StrCast } from '../../fields/Types';
-import { aggregateBounds } from '../../Utils';
+import { Cast, NumCast } from '../../fields/Types';
 import { DocumentType } from "../documents/DocumentTypes";
 import './InkTranscription.scss';
+import { aggregateBounds, Utils } from '../../Utils';
+import { timesSeries } from 'async';
 
 export class InkTranscription extends React.Component {
     static Instance: InkTranscription;
@@ -21,6 +21,7 @@ export class InkTranscription extends React.Component {
     private wordsBoundMapping: any;
     private inkDocs: Doc[];
     private ffView: any;
+    private containingLayout?: Doc;
 
     constructor(props: Readonly<{}>) {
         super(props);
@@ -105,26 +106,28 @@ export class InkTranscription extends React.Component {
         return this._textRef = r;
     }
 
-    transcribeInk = (groupDoc: Doc | undefined, inkDocs: Doc[], math: boolean, ffView?: any) => {
+    transcribeInk = (groupDoc: Doc | undefined, containingLayout: Doc, inkDocs: Doc[], math: boolean) => {
         if (!groupDoc) return;
         this.inkDocs = inkDocs;
-        this.ffView = ffView;
 
         const validInks = inkDocs.filter(s => s.type === DocumentType.INK);
 
         const strokes: InkData[] = [];
+        const times: number[] = [];
+        console.log(validInks);
         validInks.filter(i => Cast(i.data, InkField)).forEach(i => {
-            // TODO: interpolate missing times stamps
             const d = Cast(i.data, InkField, null);
             const left = Math.min(...d?.inkData.map(pd => pd.X) ?? [0]);
             const top = Math.min(...d?.inkData.map(pd => pd.Y) ?? [0]);
-            strokes.push(d.inkData.map(pd => ({ X: pd.X + NumCast(i.x) - left, Y: pd.Y + NumCast(i.y) - top, time: pd.time })));
+            strokes.push(d.inkData.map(pd => ({ X: pd.X + NumCast(i.x) - left, Y: pd.Y + NumCast(i.y) - top })));
+            times.push(NumCast(i.creationDate, 1));
         });
 
         this.currGroup = groupDoc;
+        this.containingLayout = containingLayout;
 
-        const pointerData = { "events": strokes.map(stroke => this.inkJSON(stroke)) };
-        // console.log(JSON.stringify(pointerData));
+        const pointerData = { "events": strokes.map((stroke, i) => this.inkJSON(stroke, times[i])) };
+        console.log(JSON.stringify(pointerData));
         const processGestures = false;
 
         if (math) {
@@ -135,13 +138,17 @@ export class InkTranscription extends React.Component {
         }
     }
 
-    inkJSON = (stroke: InkData) => {
+    inkJSON = (stroke: InkData, time: number) => {
+        const scale = NumCast(this.containingLayout?._viewScale, 1);
+        const panX = NumCast(this.containingLayout?._panX);
+        const panY = NumCast(this.containingLayout?._panY);
+
         return {
             "pointerType": "PEN",
             "pointerId": 1,
-            "x": stroke.map(point => point.X),
-            "y": stroke.map(point => point.Y),
-            "t": stroke.map(point => point.time),
+            "x": stroke.map(point => (point.X - panX) * scale),
+            "y": stroke.map(point => (point.Y - panY) * scale),
+            "t": new Array(stroke.length).fill(time),
             "p": new Array(stroke.length).fill(1.0)
         };
     }
@@ -163,7 +170,7 @@ export class InkTranscription extends React.Component {
         if (exports) {
             if (exports['application/x-latex']) {
                 const latex = exports['application/x-latex'];
-                console.log(latex);
+                // console.log(latex);
 
                 if (this.currGroup) {
                     this.currGroup.text = latex;
@@ -175,58 +182,18 @@ export class InkTranscription extends React.Component {
             else if (exports['text/plain']) {
                 if (exports['application/vnd.myscript.jiix']) {
                     this.lastJiix = JSON.parse(exports['application/vnd.myscript.jiix']);
-                    // console.log("lastJiix:", this.lastJiix);
-                    // TODO: nda - go through all the words and look at x,y values and figure out the selection
-                    this.lastJiix.words.forEach((word: any) => {
-                        // map each word to a new map                            
-                        this.wordsBoundMapping[word] = new Map<any, any>();
-                        if (word.items) {
-                            word.items.map((item: any) => {
-                                this.wordsBoundMapping.set(word, {xList: item.X, yList: item.Y, width: this.calcBounds(item.X), height: this.calcBounds(item.Y)});
-                            })
-                        }
-                    })
-                    // make groups for each of the words
-                    // loop through this.wordsBoundMapping
-
-                    this.wordsBoundMapping.forEach((value: any, key: any) => {
-                        const xList = value.xList;
-                        const yList = value.yList;
-                        const width = value.width;
-                        const height = value.height;
-                        const bounds: { x: number, y: number, width?: number, height?: number }[] = []
-                        // loop through xList
-                        for (let i = 0; i < xList.length; i++) {
-                            const x = xList[i];
-                            const y = yList[i];
-                            const newBounds = { x: x, y: y };
-                            bounds.push(newBounds);
-                        }
-                        const aggregBounds = aggregateBounds(bounds, 0, 0);
-
-                        const marqViewRef = this.ffView?._marqueeViewRef.current;
-
-                        // set the vals for bounds in marqueeView
-                        if (marqViewRef) {
-                            marqViewRef._downX = aggregBounds.x;
-                            marqViewRef._downY = aggregBounds.y;
-                            marqViewRef._lastX = aggregBounds.r;
-                            marqViewRef._lastY = aggregBounds.b;
-                        }
-                    
-                    });
+                    // map timestamp to strokes
+                    // we can iterate through docs and for each doc index into the map by timestamp
 
-                    // console.log(this.wordsBoundMapping);
+                    // final mapping should be word to inkDoc
                 }
                 const text = exports['text/plain'];
-                console.log(text);
+                // console.log(text);
 
                 if (this.currGroup) {
                     console.log("curr grouping");
                     this.currGroup.text = text;
-                    this.currGroup.title = text;
-                    console.log(this.lastJiix);
-                    console.log(this.currGroup);
+                    this.currGroup.title = text.split("\n")[0];
                 }
 
                 ref.editor.clear();
diff --git a/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx b/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx
index 7c11c252a..3a5911fc9 100644
--- a/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx
+++ b/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx
@@ -517,9 +517,7 @@ export class CollectionFreeFormView extends CollectionSubView<Partial<collection
             case GestureUtils.Gestures.Stroke:
                 const points = ge.points;
                 const B = this.getTransform().transformBounds(ge.bounds.left, ge.bounds.top, ge.bounds.width, ge.bounds.height);
-                const times = ge.times;
-                const strokes: PointData[] = !times || times.length == 0 ? points : points.map((pt, i) => { return { X: pt.X, Y: pt.Y, time: times[i] } });
-                const inkDoc = Docs.Create.InkDocument(ActiveInkColor(), CurrentUserUtils.SelectedTool, ActiveInkWidth(), ActiveInkBezierApprox(), ActiveFillColor(), ActiveArrowStart(), ActiveArrowEnd(), ActiveDash(), strokes,
+                const inkDoc = Docs.Create.InkDocument(ActiveInkColor(), CurrentUserUtils.SelectedTool, ActiveInkWidth(), ActiveInkBezierApprox(), ActiveFillColor(), ActiveArrowStart(), ActiveArrowEnd(), ActiveDash(), points,
                     { title: "ink stroke", x: B.x - ActiveInkWidth() / 2, y: B.y - ActiveInkWidth() / 2, _width: B.width + ActiveInkWidth(), _height: B.height + ActiveInkWidth() });
                 if (CurrentUserUtils.SelectedTool === InkTool.Write) {
                     this.unprocessedDocs.push(inkDoc);
@@ -1580,13 +1578,14 @@ export class CollectionFreeFormView extends CollectionSubView<Partial<collection
     @action
     transcribeStrokes = (math: boolean) => {
         if (this.props.Document._isGroup && this.props.Document.text) {
-            const text = StrCast(this.props.Document.text);
+            if (!math) {
+                const text = StrCast(this.props.Document.text);
 
-            const lines = text.split("\n");
-            const width = 30 + 7 * Math.max(...lines.map(line => line.length));
-            const height = 30 + 15 * lines.length;
+                const lines = text.split("\n");
+                const height = 30 + 15 * lines.length;
 
-            this.props.ContainingCollectionView?.addDocument(Docs.Create.TextDocument(text, { title: text, x: NumCast(this.layoutDoc.x) + NumCast(this.layoutDoc._width) + 20, y: NumCast(this.layoutDoc.y), _width: width, _height: height }));
+                this.props.ContainingCollectionView?.addDocument(Docs.Create.TextDocument(text, { title: lines[0], x: NumCast(this.layoutDoc.x) + NumCast(this.layoutDoc._width) + 20, y: NumCast(this.layoutDoc.y), _width: 200, _height: height }));
+            }
         }
     }
 
diff --git a/src/client/views/nodes/button/FontIconBox.tsx b/src/client/views/nodes/button/FontIconBox.tsx
index 85130e69e..1c2a22226 100644
--- a/src/client/views/nodes/button/FontIconBox.tsx
+++ b/src/client/views/nodes/button/FontIconBox.tsx
@@ -262,7 +262,7 @@ export class FontIconBox extends DocComponent<ButtonProps>() {
                     "Comic Sans MS", "Tahoma", "Impact", "Crimson Text"];
             }
         } catch (e) {
-            console.log(e);
+            // console.log(e);
         }
 
         // Get items to place into the list
@@ -699,13 +699,13 @@ ScriptingGlobals.add(function toggleItalic(checkResult?: boolean) {
 
 
 export function checkInksToGroup() {
-    console.log("getting here to inks group");
+    // console.log("getting here to inks group");
     if (CurrentUserUtils.SelectedTool === InkTool.Write) {
         CollectionFreeFormView.collectionsWithUnprocessedInk.forEach(ffView => {
             // TODO: nda - will probably want to go through ffView unprocessed docs and then see if any of the inksToGroup docs are in it and only use those
             // find all inkDocs in ffView.unprocessedDocs that are within 200 pixels of each other
             const inksToGroup = ffView.unprocessedDocs.filter(inkDoc => {
-                console.log(inkDoc.x, inkDoc.y);
+                // console.log(inkDoc.x, inkDoc.y);
             });
         });
     }
@@ -767,7 +767,7 @@ export function createInkGroup(inksToGroup?: Doc[]) {
             // TODO: nda - will probably need to go through and only remove the unprocessed selected docs
             ffView.unprocessedDocs = [];
 
-            InkTranscription.Instance.transcribeInk(newCollection, selected, false, ffView);
+            InkTranscription.Instance.transcribeInk(newCollection, ffView.layoutDoc, selected, false);
         });
     }
     CollectionFreeFormView.collectionsWithUnprocessedInk.clear();
@@ -799,7 +799,7 @@ ScriptingGlobals.add(function setActiveInkTool(tool: string, checkResult?: boole
         if (Doc.UserDoc().activeInkTool === tool && !GestureOverlay.Instance.InkShape) {
             Doc.UserDoc().activeInkTool = InkTool.None;
         } else if (tool == "write") {
-            console.log("write mode selected - create groupDoc here!", tool)
+            // console.log("write mode selected - create groupDoc here!", tool)
             Doc.UserDoc().activeInkTool = tool;
             GestureOverlay.Instance.InkShape = "";
         } else {
diff --git a/src/fields/InkField.ts b/src/fields/InkField.ts
index 6d24c6cbc..31fa6dfac 100644
--- a/src/fields/InkField.ts
+++ b/src/fields/InkField.ts
@@ -20,7 +20,6 @@ export enum InkTool {
 export interface PointData {
     X: number;
     Y: number;
-    time?: number;
 }
 
 export type Segment = Array<Bezier>;
@@ -85,7 +84,7 @@ export class InkField extends ObjectField {
     }
 
     [ToScriptString]() {
-        return "new InkField([" + this.inkData.map(i => `{X: ${i.X}, Y: ${i.Y}, time: ${i.time || 0}} `) + "])";
+        return "new InkField([" + this.inkData.map(i => `{X: ${i.X}, Y: ${i.Y}`) + "])";
     }
     [ToString]() {
         return "InkField";
diff --git a/src/pen-gestures/GestureUtils.ts b/src/pen-gestures/GestureUtils.ts
index cf59cb3c6..65f2bf80c 100644
--- a/src/pen-gestures/GestureUtils.ts
+++ b/src/pen-gestures/GestureUtils.ts
@@ -8,8 +8,7 @@ export namespace GestureUtils {
             readonly gesture: Gestures,
             readonly points: PointData[],
             readonly bounds: Rect,
-            readonly text?: any,
-            readonly times?: number[]
+            readonly text?: any
         ) { }
     }