From fb9f08295c10c35c11b647d00a3c830867dc3f7d Mon Sep 17 00:00:00 2001
From: Stanley Yip <stanley_yip@brown.edu>
Date: Tue, 3 Dec 2019 18:54:41 -0500
Subject: nothing new, infrastructure stuffs

---
 src/pen-gestures/GestureUtils.ts | 35 +++++++++++++++++++++++++++++++++++
 1 file changed, 35 insertions(+)
 create mode 100644 src/pen-gestures/GestureUtils.ts

(limited to 'src/pen-gestures/GestureUtils.ts')

diff --git a/src/pen-gestures/GestureUtils.ts b/src/pen-gestures/GestureUtils.ts
new file mode 100644
index 000000000..caaa0736d
--- /dev/null
+++ b/src/pen-gestures/GestureUtils.ts
@@ -0,0 +1,35 @@
+import { NDollarRecognizer } from "./ndollar";
+import { Type } from "typescript";
+import { InkField } from "../new_fields/InkField";
+import { Docs } from "../client/documents/Documents";
+import { Doc } from "../new_fields/Doc";
+
+export namespace GestureUtils {
+    namespace GestureDataTypes {
+        export type BoxData = Doc[];
+    }
+
+    export const GestureRecognizer = new NDollarRecognizer(false);
+
+    export enum Gestures {
+        Box = "box"
+    }
+
+    export function GestureOptions(name: Gestures, gestureData: any): (() => any)[] {
+        switch (name) {
+            case Gestures.Box:
+                if (gestureData as GestureDataTypes.BoxData) {
+                    return BoxOptions(gestureData as GestureDataTypes.BoxData);
+                }
+                break;
+        }
+        throw new Error("This means that you're trying to do something with the gesture that hasn't been defined yet. Define it in GestureUtils.ts");
+    }
+
+    function BoxOptions(gestureData: GestureDataTypes.BoxData): (() => any)[] {
+        if (gestureData instanceof Doc[]) {
+            return [() => Docs.Create.FreeformDocument(gestureData as Doc[], {})];
+        }
+        return [];
+    }
+}
\ No newline at end of file
-- 
cgit v1.2.3-70-g09d2


From b3b81c8400ce2a92df1ae8c0db9b422cf4a2cf4c Mon Sep 17 00:00:00 2001
From: yipstanley <stanley_yip@brown.edu>
Date: Sat, 7 Dec 2019 14:48:43 -0500
Subject: i think this is better now

---
 src/client/util/InteractionUtils.ts |  2 +-
 src/client/views/Touchable.tsx      | 33 ++++++++++++++-------------------
 src/pen-gestures/GestureUtils.ts    |  8 ++++----
 3 files changed, 19 insertions(+), 24 deletions(-)

(limited to 'src/pen-gestures/GestureUtils.ts')

diff --git a/src/client/util/InteractionUtils.ts b/src/client/util/InteractionUtils.ts
index c24c70bca..7390890c6 100644
--- a/src/client/util/InteractionUtils.ts
+++ b/src/client/util/InteractionUtils.ts
@@ -53,7 +53,7 @@ export namespace InteractionUtils {
      * @param oldPoint2 - previous point 2
      */
     export function Pinching(pt1: React.Touch, pt2: React.Touch, oldPoint1: React.Touch, oldPoint2: React.Touch): number {
-        let threshold = 4;
+        let threshold = 3;
         let oldDist = TwoPointEuclidist(oldPoint1, oldPoint2);
         let newDist = TwoPointEuclidist(pt1, pt2);
 
diff --git a/src/client/views/Touchable.tsx b/src/client/views/Touchable.tsx
index d0e3db8a5..5dcfec768 100644
--- a/src/client/views/Touchable.tsx
+++ b/src/client/views/Touchable.tsx
@@ -26,7 +26,8 @@ export abstract class Touchable<T = {}> extends React.Component<T> {
         }
 
         if (this.prevPoints.size) {
-            switch (e.targetTouches.length) {
+            console.log(e.targetTouches.length);
+            switch (this.prevPoints.size) {
                 case 1:
                     this.handle1PointerDown(e);
                     break;
@@ -67,35 +68,29 @@ export abstract class Touchable<T = {}> extends React.Component<T> {
                 if (this.prevPoints.has(pt.identifier)) {
                     this.prevPoints.set(pt.identifier, pt);
                 }
-                else {
-                    this.prevPoints.set(pt.identifier, pt);
-                }
             }
         }
     }
 
     @action
     protected onTouchEnd = (e: TouchEvent): void => {
-        console.log(InteractionUtils.GetMyTargetTouches(e, this.prevPoints).length + " up");
-        this._touchDrag = false;
-        e.stopPropagation();
-
+        // console.log(InteractionUtils.GetMyTargetTouches(e, this.prevPoints).length + " up");
         // remove all the touches associated with the event
-        for (let i = 0; i < e.targetTouches.length; i++) {
-            let pt = e.targetTouches.item(i);
-            if (pt) {
-                if (this.prevPoints.has(pt.identifier)) {
-                    console.log("delete");
-                    this.prevPoints.delete(pt.identifier);
-                }
+        for (let i = 0; i < e.changedTouches.length; i++) {
+            let pt = e.changedTouches.item(i);
+            if (pt && this.prevPoints.has(pt.identifier)) {
+                this.prevPoints.delete(pt.identifier);
             }
         }
+        this._touchDrag = false;
+        e.stopPropagation();
 
-        if (e.targetTouches.length === 0) {
-            this.prevPoints.clear();
-        }
 
-        if (this.prevPoints.size === 0 && e.targetTouches.length === 0) {
+        // if (e.targetTouches.length === 0) {
+        //     this.prevPoints.clear();
+        // }
+
+        if (this.prevPoints.size === 0) {
             this.cleanUpInteractions();
         }
     }
diff --git a/src/pen-gestures/GestureUtils.ts b/src/pen-gestures/GestureUtils.ts
index caaa0736d..7a14452c8 100644
--- a/src/pen-gestures/GestureUtils.ts
+++ b/src/pen-gestures/GestureUtils.ts
@@ -6,15 +6,15 @@ import { Doc } from "../new_fields/Doc";
 
 export namespace GestureUtils {
     namespace GestureDataTypes {
-        export type BoxData = Doc[];
+        export type BoxData = Array<Doc>;
     }
 
-    export const GestureRecognizer = new NDollarRecognizer(false);
-
     export enum Gestures {
         Box = "box"
     }
 
+    export const GestureRecognizer = new NDollarRecognizer(false);
+
     export function GestureOptions(name: Gestures, gestureData: any): (() => any)[] {
         switch (name) {
             case Gestures.Box:
@@ -27,7 +27,7 @@ export namespace GestureUtils {
     }
 
     function BoxOptions(gestureData: GestureDataTypes.BoxData): (() => any)[] {
-        if (gestureData instanceof Doc[]) {
+        if (gestureData instanceof Array) {
             return [() => Docs.Create.FreeformDocument(gestureData as Doc[], {})];
         }
         return [];
-- 
cgit v1.2.3-70-g09d2


From 766e47cd144be6f0bcbb5787dbd174aa1bba3690 Mon Sep 17 00:00:00 2001
From: Stanley Yip <stanley_yip@brown.edu>
Date: Sat, 7 Dec 2019 16:53:43 -0500
Subject: box !!!!

---
 src/client/util/InteractionUtils.ts                |  2 +-
 src/client/views/Touchable.tsx                     |  4 ++--
 .../collectionFreeForm/CollectionFreeFormView.tsx  | 23 ++++++++++++++++++----
 src/client/views/nodes/DocumentView.tsx            |  8 ++++----
 src/pen-gestures/GestureUtils.ts                   | 16 ++++-----------
 src/pen-gestures/ndollar.ts                        |  2 +-
 6 files changed, 31 insertions(+), 24 deletions(-)

(limited to 'src/pen-gestures/GestureUtils.ts')

diff --git a/src/client/util/InteractionUtils.ts b/src/client/util/InteractionUtils.ts
index 7390890c6..7c386abc2 100644
--- a/src/client/util/InteractionUtils.ts
+++ b/src/client/util/InteractionUtils.ts
@@ -53,7 +53,7 @@ export namespace InteractionUtils {
      * @param oldPoint2 - previous point 2
      */
     export function Pinching(pt1: React.Touch, pt2: React.Touch, oldPoint1: React.Touch, oldPoint2: React.Touch): number {
-        let threshold = 3;
+        let threshold = window.devicePixelRatio;
         let oldDist = TwoPointEuclidist(oldPoint1, oldPoint2);
         let newDist = TwoPointEuclidist(pt1, pt2);
 
diff --git a/src/client/views/Touchable.tsx b/src/client/views/Touchable.tsx
index 5dcfec768..a647dec04 100644
--- a/src/client/views/Touchable.tsx
+++ b/src/client/views/Touchable.tsx
@@ -18,9 +18,9 @@ export abstract class Touchable<T = {}> extends React.Component<T> {
     protected onTouchStart = (e: React.TouchEvent): void => {
         for (let i = 0; i < e.targetTouches.length; i++) {
             let pt: any = e.targetTouches.item(i);
-            // pen is also a touch, but with a radius of 0.5 (at least with the surface pens). i doubt anyone's fingers are 2 pixels wide,
+            // pen is also a touch, but with a radius of 0.5 (at least with the surface pens).
             // and this seems to be the only way of differentiating pen and touch on touch events
-            if (pt.radiusX > 2 && pt.radiusY > 2) {
+            if (pt.radiusX > 0.5 && pt.radiusY > 0.5) {
                 this.prevPoints.set(pt.identifier, pt);
             }
         }
diff --git a/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx b/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx
index 6dd472c84..bec0ace5a 100644
--- a/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx
+++ b/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx
@@ -340,8 +340,23 @@ export class CollectionFreeFormView extends CollectionSubView(PanZoomDocument) {
             let points = this._points.map(p => ({ X: p.X - B.left, Y: p.Y - B.top }));
 
             let result = GestureUtils.GestureRecognizer.Recognize(new Array(points));
-            if (result) {
-                console.log(result.Name);
+            if (result && result.Score > 0.75) {
+                switch (result.Name) {
+                    case GestureUtils.Gestures.Box:
+                        let bounds = { x: Math.min(...this._points.map(p => p.X)), r: Math.max(...this._points.map(p => p.X)), y: Math.min(...this._points.map(p => p.y)), b: Math.max(...this._points.map(p => p.Y)) };
+                        let sel = this.getActiveDocuments().filter(doc => {
+                            let l = NumCast(doc.x);
+                            let r = l + doc[WidthSym]();
+                            let t = NumCast(doc.y);
+                            let b = t + doc[HeightSym]();
+                            doc.x = l - B.left - B.width / 2;
+                            doc.y = t - B.top - B.height / 2;
+                            return !(bounds.x > r || bounds.r < l || bounds.y > b || bounds.b < t);
+                        });
+                        this.addDocument(Docs.Create.FreeformDocument(sel, { x: B.left, y: B.top, width: B.width, height: B.height, panX: 0, panY: 0 }));
+                        sel.forEach(d => this.props.removeDocument(d));
+                        break;
+                }
                 this._points = [];
             }
             else {
@@ -474,8 +489,8 @@ export class CollectionFreeFormView extends CollectionSubView(PanZoomDocument) {
                         let rawDelta = (dir * (d1 + d2));
 
                         // this floors and ceils the delta value to prevent jitteriness
-                        let delta = Math.sign(rawDelta) * Math.min(Math.abs(rawDelta), 16);
-                        this.zoom(centerX, centerY, delta);
+                        let delta = Math.sign(rawDelta) * Math.min(Math.abs(rawDelta), 8);
+                        this.zoom(centerX, centerY, delta * window.devicePixelRatio);
                         this.prevPoints.set(pt1.identifier, pt1);
                         this.prevPoints.set(pt2.identifier, pt2);
                     }
diff --git a/src/client/views/nodes/DocumentView.tsx b/src/client/views/nodes/DocumentView.tsx
index 38c46b258..dfeadb8c0 100644
--- a/src/client/views/nodes/DocumentView.tsx
+++ b/src/client/views/nodes/DocumentView.tsx
@@ -142,7 +142,7 @@ export class DocumentView extends DocComponent<DocumentViewProps, Document>(Docu
             (Math.abs(e.clientX - this._downX) < Utils.DRAG_THRESHOLD && Math.abs(e.clientY - this._downY) < Utils.DRAG_THRESHOLD)) {
             e.stopPropagation();
             let preventDefault = true;
-            if (this._doubleTap && this.props.renderDepth && !this.onClickHandler ?.script) { // disable double-click to show full screen for things that have an on click behavior since clicking them twice can be misinterpreted as a double click
+            if (this._doubleTap && this.props.renderDepth && !this.onClickHandler?.script) { // disable double-click to show full screen for things that have an on click behavior since clicking them twice can be misinterpreted as a double click
                 let fullScreenAlias = Doc.MakeAlias(this.props.Document);
                 if (StrCast(fullScreenAlias.layoutKey) !== "layoutCustom" && fullScreenAlias.layoutCustom !== undefined) {
                     fullScreenAlias.layoutKey = "layoutCustom";
@@ -362,7 +362,7 @@ export class DocumentView extends DocComponent<DocumentViewProps, Document>(Docu
     @undoBatch
     @action
     setCustomView = (custom: boolean): void => {
-        if (this.props.ContainingCollectionView ?.props.DataDoc || this.props.ContainingCollectionView ?.props.Document.isTemplateDoc) {
+        if (this.props.ContainingCollectionView?.props.DataDoc || this.props.ContainingCollectionView?.props.Document.isTemplateDoc) {
             Doc.MakeMetadataFieldTemplate(this.props.Document, this.props.ContainingCollectionView.props.Document);
         } else {
             custom ? DocumentView.makeCustomViewClicked(this.props.Document, this.props.DataDoc) : DocumentView.makeNativeViewClicked(this.props.Document);
@@ -687,10 +687,10 @@ export class DocumentView extends DocComponent<DocumentViewProps, Document>(Docu
         return <div className={`documentView-node${this.topMost ? "-topmost" : ""}`} ref={this._mainCont}
             onDrop={this.onDrop} onContextMenu={this.onContextMenu} onPointerDown={this.onPointerDown} onClick={this.onClick}
             onPointerEnter={e => {
-                console.log("Brush" + this.props.Document.title);
+                // console.log("Brush" + this.props.Document.title);
                 Doc.BrushDoc(this.props.Document);
             }} onPointerLeave={e => {
-                console.log("UnBrush" + this.props.Document.title);
+                // console.log("UnBrush"     + this.props.Document.title);
                 Doc.UnBrushDoc(this.props.Document);
 
             }}
diff --git a/src/pen-gestures/GestureUtils.ts b/src/pen-gestures/GestureUtils.ts
index 7a14452c8..955dad5b4 100644
--- a/src/pen-gestures/GestureUtils.ts
+++ b/src/pen-gestures/GestureUtils.ts
@@ -2,7 +2,9 @@ import { NDollarRecognizer } from "./ndollar";
 import { Type } from "typescript";
 import { InkField } from "../new_fields/InkField";
 import { Docs } from "../client/documents/Documents";
-import { Doc } from "../new_fields/Doc";
+import { Doc, WidthSym, HeightSym } from "../new_fields/Doc";
+import { NumCast } from "../new_fields/Types";
+import { CollectionFreeFormView } from "../client/views/collections/collectionFreeForm/CollectionFreeFormView";
 
 export namespace GestureUtils {
     namespace GestureDataTypes {
@@ -15,21 +17,11 @@ export namespace GestureUtils {
 
     export const GestureRecognizer = new NDollarRecognizer(false);
 
-    export function GestureOptions(name: Gestures, gestureData: any): (() => any)[] {
+    export function GestureOptions(name: string, gestureData?: any): (params: {}) => any {
         switch (name) {
             case Gestures.Box:
-                if (gestureData as GestureDataTypes.BoxData) {
-                    return BoxOptions(gestureData as GestureDataTypes.BoxData);
-                }
                 break;
         }
         throw new Error("This means that you're trying to do something with the gesture that hasn't been defined yet. Define it in GestureUtils.ts");
     }
-
-    function BoxOptions(gestureData: GestureDataTypes.BoxData): (() => any)[] {
-        if (gestureData instanceof Array) {
-            return [() => Docs.Create.FreeformDocument(gestureData as Doc[], {})];
-        }
-        return [];
-    }
 }
\ No newline at end of file
diff --git a/src/pen-gestures/ndollar.ts b/src/pen-gestures/ndollar.ts
index 5d4384202..8c8e079a4 100644
--- a/src/pen-gestures/ndollar.ts
+++ b/src/pen-gestures/ndollar.ts
@@ -168,7 +168,7 @@ export class NDollarRecognizer {
         //
         this.Multistrokes = new Array(NumMultistrokes);
         this.Multistrokes[0] = new Multistroke(GestureUtils.Gestures.Box, useBoundedRotationInvariance, new Array(
-            new Array(new Point(30, 146), new Point(30, 222), new Point(106, 225), new Point(106, 146))
+            new Array(new Point(30, 146), new Point(30, 222), new Point(106, 225), new Point(106, 146), new Point(30, 146))
         ));
 
         //
-- 
cgit v1.2.3-70-g09d2


From 3a2aaf5ffec2a568bb749c6ac0df8367a150af39 Mon Sep 17 00:00:00 2001
From: yipstanley <stanley_yip@brown.edu>
Date: Sun, 8 Dec 2019 14:46:46 -0500
Subject: links!

---
 .../collectionFreeForm/CollectionFreeFormView.tsx  | 40 +++++++++++++++++++---
 src/pen-gestures/GestureUtils.ts                   |  3 +-
 src/pen-gestures/ndollar.ts                        |  8 ++---
 3 files changed, 41 insertions(+), 10 deletions(-)

(limited to 'src/pen-gestures/GestureUtils.ts')

diff --git a/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx b/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx
index bec0ace5a..5a7644d05 100644
--- a/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx
+++ b/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx
@@ -13,7 +13,7 @@ import { BoolCast, Cast, DateCast, NumCast, StrCast } from "../../../../new_fiel
 import { CurrentUserUtils } from "../../../../server/authentication/models/current_user_utils";
 import { aggregateBounds, emptyFunction, intersectRect, returnOne, Utils } from "../../../../Utils";
 import { DocServer } from "../../../DocServer";
-import { Docs } from "../../../documents/Documents";
+import { Docs, DocUtils } from "../../../documents/Documents";
 import { DocumentType } from "../../../documents/DocumentTypes";
 import { DocumentManager } from "../../../util/DocumentManager";
 import { DragManager } from "../../../util/DragManager";
@@ -42,6 +42,7 @@ import React = require("react");
 import { computedFn, keepAlive } from "mobx-utils";
 import { TraceMobx } from "../../../../new_fields/util";
 import { GestureUtils } from "../../../../pen-gestures/GestureUtils";
+import { LinkManager } from "../../../util/LinkManager";
 
 library.add(faEye as any, faTable, faPaintBrush, faExpandArrowsAlt, faCompressArrowsAlt, faCompass, faUpload, faBraille, faChalkboard, faFileUpload);
 
@@ -340,7 +341,8 @@ export class CollectionFreeFormView extends CollectionSubView(PanZoomDocument) {
             let points = this._points.map(p => ({ X: p.X - B.left, Y: p.Y - B.top }));
 
             let result = GestureUtils.GestureRecognizer.Recognize(new Array(points));
-            if (result && result.Score > 0.75) {
+            let actionPerformed = false;
+            if (result && result.Score > 0.7) {
                 switch (result.Name) {
                     case GestureUtils.Gestures.Box:
                         let bounds = { x: Math.min(...this._points.map(p => p.X)), r: Math.max(...this._points.map(p => p.X)), y: Math.min(...this._points.map(p => p.y)), b: Math.max(...this._points.map(p => p.Y)) };
@@ -355,11 +357,39 @@ export class CollectionFreeFormView extends CollectionSubView(PanZoomDocument) {
                         });
                         this.addDocument(Docs.Create.FreeformDocument(sel, { x: B.left, y: B.top, width: B.width, height: B.height, panX: 0, panY: 0 }));
                         sel.forEach(d => this.props.removeDocument(d));
+                        actionPerformed = true;
+                        break;
+                    case GestureUtils.Gestures.Line:
+                        let ep1 = this._points[0];
+                        let ep2 = this._points[this._points.length - 1];
+                        let d1: Doc | undefined;
+                        let d2: Doc | undefined;
+                        this.getActiveDocuments().map(doc => {
+                            let l = NumCast(doc.x);
+                            let r = l + doc[WidthSym]();
+                            let t = NumCast(doc.y);
+                            let b = t + doc[HeightSym]();
+                            if (!d1 && l < ep1.X && r > ep1.X && t < ep1.Y && b > ep1.Y) {
+                                d1 = doc;
+                            }
+                            else if (!d2 && l < ep2.X && r > ep2.X && t < ep2.Y && b > ep2.Y) {
+                                d2 = doc;
+                            }
+                        });
+                        if (d1 && d2) {
+                            if (!LinkManager.Instance.doesLinkExist(d1, d2)) {
+                                DocUtils.MakeLink({ doc: d1 }, { doc: d2 });
+                                actionPerformed = true;
+                            }
+                        }
                         break;
                 }
-                this._points = [];
+                if (actionPerformed) {
+                    this._points = [];
+                }
             }
-            else {
+
+            if (!actionPerformed) {
                 let inkDoc = Docs.Create.InkDocument(InkingControl.Instance.selectedColor, InkingControl.Instance.selectedTool, parseInt(InkingControl.Instance.selectedWidth), points, { width: B.width, height: B.height, x: B.left, y: B.top });
                 this.addDocument(inkDoc);
                 this._points = [];
@@ -880,7 +910,7 @@ export class CollectionFreeFormView extends CollectionSubView(PanZoomDocument) {
         let B = this.svgBounds;
 
         return (
-            <svg width={B.width} height={B.height} style={{ transform: `translate(${B.left}px, ${B.top}px)`, position: "relative", zIndex: 30000 }}>
+            <svg width={B.width} height={B.height} style={{ transform: `translate(${B.left}px, ${B.top}px)`, position: "absolute", zIndex: 30000 }}>
                 {CreatePolyline(this._points, B.left, B.top)}
             </svg>
         );
diff --git a/src/pen-gestures/GestureUtils.ts b/src/pen-gestures/GestureUtils.ts
index 955dad5b4..59a85b66b 100644
--- a/src/pen-gestures/GestureUtils.ts
+++ b/src/pen-gestures/GestureUtils.ts
@@ -12,7 +12,8 @@ export namespace GestureUtils {
     }
 
     export enum Gestures {
-        Box = "box"
+        Box = "box",
+        Line = "line"
     }
 
     export const GestureRecognizer = new NDollarRecognizer(false);
diff --git a/src/pen-gestures/ndollar.ts b/src/pen-gestures/ndollar.ts
index 8c8e079a4..12c2b25bb 100644
--- a/src/pen-gestures/ndollar.ts
+++ b/src/pen-gestures/ndollar.ts
@@ -142,7 +142,7 @@ export class Result {
 //
 // NDollarRecognizer constants
 //
-const NumMultistrokes = 1;
+const NumMultistrokes = 2;
 const NumPoints = 96;
 const SquareSize = 250.0;
 const OneDThreshold = 0.25; // customize to desired gesture set (usually 0.20 - 0.35)
@@ -170,6 +170,9 @@ export class NDollarRecognizer {
         this.Multistrokes[0] = new Multistroke(GestureUtils.Gestures.Box, useBoundedRotationInvariance, new Array(
             new Array(new Point(30, 146), new Point(30, 222), new Point(106, 225), new Point(106, 146), new Point(30, 146))
         ));
+        this.Multistrokes[1] = new Multistroke(GestureUtils.Gestures.Line, useBoundedRotationInvariance, new Array(
+            new Array(new Point(12, 347), new Point(119, 347))
+        ));
 
         //
         // PREDEFINED STROKES
@@ -210,9 +213,6 @@ export class NDollarRecognizer {
         //     new Array(new Point(526, 142), new Point(526, 204)),
         //     new Array(new Point(526, 221))
         // ));
-        // this.Multistrokes[8] = new Multistroke("line", useBoundedRotationInvariance, new Array(
-        //     new Array(new Point(12, 347), new Point(119, 347))
-        // ));
         // this.Multistrokes[9] = new Multistroke("five-point star", useBoundedRotationInvariance, new Array(
         //     new Array(new Point(177, 396), new Point(223, 299), new Point(262, 396), new Point(168, 332), new Point(278, 332), new Point(184, 397))
         // ));
-- 
cgit v1.2.3-70-g09d2


From cef6852d597ce67637466afb36c3498dc84211f6 Mon Sep 17 00:00:00 2001
From: Stanley Yip <stanley_yip@brown.edu>
Date: Wed, 8 Jan 2020 19:48:28 -0500
Subject: gestures are now overlayed and can span collections/panes!

---
 src/client/documents/Documents.ts                  |   3 +-
 src/client/views/CollectionLinearView.tsx          |   4 +-
 src/client/views/GestureOverlay.scss               |   8 +
 src/client/views/GestureOverlay.tsx                | 159 +++++++++++++++++++
 src/client/views/InkSelectDecorations.scss         |   5 -
 src/client/views/InkSelectDecorations.tsx          |  55 -------
 src/client/views/InkingStroke.tsx                  |   6 +-
 src/client/views/MainView.scss                     |   7 +-
 src/client/views/MainView.tsx                      |   7 +-
 .../views/collections/CollectionStackingView.tsx   |   6 +-
 src/client/views/collections/CollectionSubView.tsx |  13 +-
 .../views/collections/CollectionTreeView.scss      |   6 +-
 .../collectionFreeForm/CollectionFreeFormView.tsx  | 171 +++++++--------------
 src/client/views/nodes/DocumentView.tsx            |  19 ++-
 src/pen-gestures/GestureUtils.ts                   |  28 +++-
 15 files changed, 299 insertions(+), 198 deletions(-)
 create mode 100644 src/client/views/GestureOverlay.scss
 create mode 100644 src/client/views/GestureOverlay.tsx
 delete mode 100644 src/client/views/InkSelectDecorations.scss
 delete mode 100644 src/client/views/InkSelectDecorations.tsx

(limited to 'src/pen-gestures/GestureUtils.ts')

diff --git a/src/client/documents/Documents.ts b/src/client/documents/Documents.ts
index e149963b9..1853be529 100644
--- a/src/client/documents/Documents.ts
+++ b/src/client/documents/Documents.ts
@@ -29,7 +29,7 @@ import { listSpec } from "../../new_fields/Schema";
 import { DocServer } from "../DocServer";
 import { dropActionType } from "../util/DragManager";
 import { DateField } from "../../new_fields/DateField";
-import { UndoManager } from "../util/UndoManager";
+import { UndoManager, undoBatch } from "../util/UndoManager";
 import { YoutubeBox } from "../apis/youtube/YoutubeBox";
 import { CollectionDockingView } from "../views/collections/CollectionDockingView";
 import { LinkManager } from "../util/LinkManager";
@@ -730,6 +730,7 @@ export namespace DocUtils {
         });
     }
 
+    @undoBatch
     export function MakeLink(source: { doc: Doc, ctx?: Doc }, target: { doc: Doc, ctx?: Doc }, title: string = "", description: string = "", id?: string) {
         const sv = DocumentManager.Instance.getDocumentView(source.doc);
         if (sv && sv.props.ContainingCollectionDoc === target.doc) return;
diff --git a/src/client/views/CollectionLinearView.tsx b/src/client/views/CollectionLinearView.tsx
index 5ca861f71..5d6a58656 100644
--- a/src/client/views/CollectionLinearView.tsx
+++ b/src/client/views/CollectionLinearView.tsx
@@ -36,7 +36,7 @@ export class CollectionLinearView extends CollectionSubView(LinearDocument) {
             { fireImmediately: true }
         );
     }
-    protected createDropTarget = (ele: HTMLDivElement) => { //used for stacking and masonry view
+    protected createDropAndGestureTarget = (ele: HTMLDivElement) => { //used for stacking and masonry view
         this._dropDisposer && this._dropDisposer();
         if (ele) {
             this._dropDisposer = DragManager.MakeDropTarget(ele, this.drop.bind(this));
@@ -55,7 +55,7 @@ export class CollectionLinearView extends CollectionSubView(LinearDocument) {
     render() {
         const guid = Utils.GenerateGuid();
         return <div className="collectionLinearView-outer">
-            <div className="collectionLinearView" ref={this.createDropTarget} >
+            <div className="collectionLinearView" ref={this.createDropAndGestureTarget} >
                 <input id={`${guid}`} type="checkbox" checked={BoolCast(this.props.Document.isExpanded)} ref={this.addMenuToggle}
                     onChange={action((e: any) => this.props.Document.isExpanded = this.addMenuToggle.current!.checked)} />
                 <label htmlFor={`${guid}`} style={{ marginTop: "auto", marginBottom: "auto", background: StrCast(this.props.Document.backgroundColor, "black") === StrCast(this.props.Document.color, "white") ? "black" : StrCast(this.props.Document.backgroundColor, "black") }} title="Close Menu"><p>+</p></label>
diff --git a/src/client/views/GestureOverlay.scss b/src/client/views/GestureOverlay.scss
new file mode 100644
index 000000000..cbc1b6e7d
--- /dev/null
+++ b/src/client/views/GestureOverlay.scss
@@ -0,0 +1,8 @@
+.gestureOverlay-cont {
+    width: 100vw;
+    height: 100vw;
+    position: absolute;
+    top: 0;
+    left: 0;
+    touch-action: none;
+}
\ No newline at end of file
diff --git a/src/client/views/GestureOverlay.tsx b/src/client/views/GestureOverlay.tsx
new file mode 100644
index 000000000..c88e3f7ae
--- /dev/null
+++ b/src/client/views/GestureOverlay.tsx
@@ -0,0 +1,159 @@
+import React = require("react");
+import { Touchable } from "./Touchable";
+import { observer } from "mobx-react";
+import "./GestureOverlay.scss"
+import { computed, observable, action } from "mobx";
+import { CreatePolyline } from "./InkingStroke";
+import { GestureUtils } from "../../pen-gestures/GestureUtils";
+import { InteractionUtils } from "../util/InteractionUtils";
+import { InkingControl } from "./InkingControl";
+import { InkTool } from "../../new_fields/InkField";
+import { Doc } from "../../new_fields/Doc";
+import { LinkManager } from "../util/LinkManager";
+import { DocUtils } from "../documents/Documents";
+import { undoBatch } from "../util/UndoManager";
+
+@observer
+export default class GestureOverlay extends Touchable {
+    static Instance: GestureOverlay;
+
+    @observable private _points: { X: number, Y: number }[] = [];
+
+    private _d1: Doc | undefined;
+
+    constructor(props: Readonly<{}>) {
+        super(props);
+
+        GestureOverlay.Instance = this;
+    }
+
+    @action
+    onPointerDown = (e: React.PointerEvent) => {
+        if (InteractionUtils.IsType(e, InteractionUtils.PENTYPE) || (InkingControl.Instance.selectedTool === InkTool.Highlighter || InkingControl.Instance.selectedTool === InkTool.Pen)) {
+            this._points.push({ X: e.clientX, Y: e.clientY });
+            e.stopPropagation();
+            e.preventDefault();
+
+            document.removeEventListener("pointermove", this.onPointerMove);
+            document.removeEventListener("pointerup", this.onPointerUp);
+            document.addEventListener("pointermove", this.onPointerMove);
+            document.addEventListener("pointerup", this.onPointerUp);
+        }
+    }
+
+    @action
+    onPointerMove = (e: PointerEvent) => {
+        this._points.push({ X: e.clientX, Y: e.clientY });
+        e.stopPropagation();
+        e.preventDefault();
+    }
+
+    @action
+    onPointerUp = (e: PointerEvent) => {
+        if (this._points.length > 1) {
+            const B = this.svgBounds;
+            const points = this._points.map(p => ({ X: p.X - B.left, Y: p.Y - B.top }));
+
+            const result = GestureUtils.GestureRecognizer.Recognize(new Array(points));
+            let actionPerformed = false;
+            if (result && result.Score > 0.7) {
+                switch (result.Name) {
+                    case GestureUtils.Gestures.Box:
+                        const target = document.elementFromPoint(this._points[0].X, this._points[0].Y);
+                        target?.dispatchEvent(new CustomEvent<GestureUtils.GestureEvent>("dashOnGesture",
+                            {
+                                bubbles: true,
+                                detail: {
+                                    points: this._points,
+                                    gesture: GestureUtils.Gestures.Box,
+                                    bounds: B
+                                }
+                            }));
+                        actionPerformed = true;
+                        break;
+                    case GestureUtils.Gestures.Line:
+                        const ep1 = this._points[0];
+                        const ep2 = this._points[this._points.length - 1];
+                        const target1 = document.elementFromPoint(ep1.X, ep1.Y);
+                        const target2 = document.elementFromPoint(ep2.X, ep2.Y);
+                        const callback = (doc: Doc) => {
+                            if (!this._d1) {
+                                this._d1 = doc;
+                            }
+                            else if (this._d1 !== doc && !LinkManager.Instance.doesLinkExist(this._d1, doc)) {
+                                DocUtils.MakeLink({ doc: this._d1 }, { doc: doc });
+                                actionPerformed = true;
+                            }
+                        }
+                        const ge = new CustomEvent<GestureUtils.GestureEvent>("dashOnGesture",
+                            {
+                                bubbles: true,
+                                detail: {
+                                    points: this._points,
+                                    gesture: GestureUtils.Gestures.Line,
+                                    bounds: B,
+                                    callbackFn: callback
+                                }
+                            })
+                        target1?.dispatchEvent(ge);
+                        target2?.dispatchEvent(ge);
+                        break;
+                }
+                if (actionPerformed) {
+                    this._points = [];
+                }
+            }
+
+            if (!actionPerformed) {
+                const target = document.elementFromPoint(this._points[0].X, this._points[0].Y);
+                target?.dispatchEvent(
+                    new CustomEvent<GestureUtils.GestureEvent>("dashOnGesture",
+                        {
+                            bubbles: true,
+                            detail: {
+                                points: this._points,
+                                gesture: GestureUtils.Gestures.Stroke,
+                                bounds: B
+                            }
+                        }
+                    )
+                )
+                this._points = [];
+            }
+        }
+        document.removeEventListener("pointermove", this.onPointerMove);
+        document.removeEventListener("pointerup", this.onPointerUp);
+    }
+
+    @computed get svgBounds() {
+        const xs = this._points.map(p => p.X);
+        const ys = this._points.map(p => p.Y);
+        const right = Math.max(...xs);
+        const left = Math.min(...xs);
+        const bottom = Math.max(...ys);
+        const top = Math.min(...ys);
+        return { right: right, left: left, bottom: bottom, top: top, width: right - left, height: bottom - top };
+    }
+
+    @computed get currentStroke() {
+        if (this._points.length <= 1) {
+            return (null);
+        }
+
+        const B = this.svgBounds;
+
+        return (
+            <svg width={B.width} height={B.height} style={{ transform: `translate(${B.left}px, ${B.top}px)`, pointerEvents: "none", position: "absolute", zIndex: 30000 }}>
+                {CreatePolyline(this._points, B.left, B.top)}
+            </svg>
+        );
+    }
+
+    render() {
+        return (
+            <div className="gestureOverlay-cont" onPointerDown={this.onPointerDown}>
+                {this.props.children}
+                {this.currentStroke}
+            </div>);
+    }
+}
\ No newline at end of file
diff --git a/src/client/views/InkSelectDecorations.scss b/src/client/views/InkSelectDecorations.scss
deleted file mode 100644
index daff58fd6..000000000
--- a/src/client/views/InkSelectDecorations.scss
+++ /dev/null
@@ -1,5 +0,0 @@
-.inkSelectDecorations {
-    position: absolute;
-    border: black 1px solid;
-    z-index: 9001;
-}
\ No newline at end of file
diff --git a/src/client/views/InkSelectDecorations.tsx b/src/client/views/InkSelectDecorations.tsx
deleted file mode 100644
index 3ad50762d..000000000
--- a/src/client/views/InkSelectDecorations.tsx
+++ /dev/null
@@ -1,55 +0,0 @@
-import React = require("react");
-import { Touchable } from "./Touchable";
-import { PointData } from "../../new_fields/InkField";
-import { observer } from "mobx-react";
-import { computed, observable, action, runInAction } from "mobx";
-import "./InkSelectDecorations.scss";
-
-@observer
-export default class InkSelectDecorations extends Touchable {
-    static Instance: InkSelectDecorations;
-
-    @observable private _selectedInkNodes: Map<any, any> = new Map();
-
-    constructor(props: Readonly<{}>) {
-        super(props);
-
-        InkSelectDecorations.Instance = this;
-    }
-
-    @action
-    public SetSelected = (inkNodes: Map<any, any>, keepOld: boolean = false) => {
-        if (!keepOld) {
-            this._selectedInkNodes = new Map();
-        }
-        inkNodes.forEach((value: any, key: any) => {
-            runInAction(() => this._selectedInkNodes.set(key, value));
-        });
-    }
-
-    @computed
-    get Bounds(): { x: number, y: number, b: number, r: number } {
-        const left = Number.MAX_VALUE;
-        const top = Number.MAX_VALUE;
-        const right = -Number.MAX_VALUE;
-        const bottom = -Number.MAX_VALUE;
-        this._selectedInkNodes.forEach((value: PointData, key: string) => {
-            // value.pathData.map(val => {
-            //     left = Math.min(val.x, left);
-            //     top = Math.min(val.y, top);
-            //     right = Math.max(val.x, right);
-            //     bottom = Math.max(val.y, bottom);
-            // });
-        });
-        return { x: left, y: top, b: bottom, r: right };
-    }
-
-    render() {
-        const bounds = this.Bounds;
-        return <div style={{
-            top: bounds.y, left: bounds.x,
-            height: bounds.b - bounds.y,
-            width: bounds.r - bounds.x
-        }} />;
-    }
-}
\ No newline at end of file
diff --git a/src/client/views/InkingStroke.tsx b/src/client/views/InkingStroke.tsx
index a413eebc9..8b346d5d9 100644
--- a/src/client/views/InkingStroke.tsx
+++ b/src/client/views/InkingStroke.tsx
@@ -9,6 +9,7 @@ import { InkingControl } from "./InkingControl";
 import "./InkingStroke.scss";
 import { FieldView, FieldViewProps } from "./nodes/FieldView";
 import React = require("react");
+import { TraceMobx } from "../../new_fields/util";
 
 type InkDocument = makeInterface<[typeof documentSchema]>;
 const InkDocument = makeInterface(documentSchema);
@@ -35,6 +36,7 @@ export class InkingStroke extends DocExtendableComponent<FieldViewProps, InkDocu
     @computed get PanelHeight() { return this.props.PanelHeight(); }
 
     render() {
+        TraceMobx();
         const data: InkData = Cast(this.Document.data, InkField)?.inkData ?? [];
         const xs = data.map(p => p.X);
         const ys = data.map(p => p.Y);
@@ -42,7 +44,7 @@ export class InkingStroke extends DocExtendableComponent<FieldViewProps, InkDocu
         const top = Math.min(...ys);
         const right = Math.max(...xs);
         const bottom = Math.max(...ys);
-        const points = CreatePolyline(data, 0, 0, this.Document.color, this.Document.strokeWidth);
+        const points = CreatePolyline(data, left, top, this.Document.color, this.Document.strokeWidth);
         const width = right - left;
         const height = bottom - top;
         const scaleX = this.PanelWidth / width;
@@ -50,7 +52,7 @@ export class InkingStroke extends DocExtendableComponent<FieldViewProps, InkDocu
         return (
             <svg width={width} height={height} style={{
                 transformOrigin: "top left",
-                transform: `translate(${left}px, ${top}px) scale(${scaleX}, ${scaleY})`,
+                transform: `scale(${scaleX}, ${scaleY})`,
                 mixBlendMode: this.Document.tool === InkTool.Highlighter ? "multiply" : "unset",
                 pointerEvents: "all"
             }}>
diff --git a/src/client/views/MainView.scss b/src/client/views/MainView.scss
index 4c8c95529..2a1cc3303 100644
--- a/src/client/views/MainView.scss
+++ b/src/client/views/MainView.scss
@@ -9,8 +9,8 @@
 
 .mainContent-div {
     position: relative;
-    width:100%;
-    height:100%;
+    width: 100%;
+    height: 100%;
 }
 
 // add nodes menu. Note that the + button is actually an input label, not an actual button.
@@ -28,6 +28,7 @@
     top: 0;
     left: 0;
     z-index: 1;
+    touch-action: none;
 }
 
 .mainView-mainContent {
@@ -66,7 +67,7 @@
 
 .mainView-libraryFlyout {
     height: 100%;
-    width:100%;
+    width: 100%;
     position: absolute;
     display: flex;
     flex-direction: column;
diff --git a/src/client/views/MainView.tsx b/src/client/views/MainView.tsx
index a1196ee1c..14c1803c0 100644
--- a/src/client/views/MainView.tsx
+++ b/src/client/views/MainView.tsx
@@ -36,7 +36,7 @@ import { OverlayView } from './OverlayView';
 import PDFMenu from './pdf/PDFMenu';
 import { PreviewCursor } from './PreviewCursor';
 import MarqueeOptionsMenu from './collections/collectionFreeForm/MarqueeOptionsMenu';
-import InkSelectDecorations from './InkSelectDecorations';
+import GestureOverlay from './GestureOverlay';
 import { Scripting } from '../util/Scripting';
 import { AudioBox } from './nodes/AudioBox';
 import { TraceMobx } from '../../new_fields/util';
@@ -510,8 +510,9 @@ export class MainView extends React.Component {
             <SharingManager />
             <GoogleAuthenticationManager />
             <DocumentDecorations />
-            <InkSelectDecorations />
-            {this.mainContent}
+            <GestureOverlay>
+                {this.mainContent}
+            </GestureOverlay>
             <PreviewCursor />
             <ContextMenu />
             <PDFMenu />
diff --git a/src/client/views/collections/CollectionStackingView.tsx b/src/client/views/collections/CollectionStackingView.tsx
index e71e11b48..992820fc7 100644
--- a/src/client/views/collections/CollectionStackingView.tsx
+++ b/src/client/views/collections/CollectionStackingView.tsx
@@ -153,7 +153,7 @@ export class CollectionStackingView extends CollectionSubView(doc => doc) {
     }
     createRef = (ele: HTMLDivElement | null) => {
         this._masonryGridRef = ele;
-        this.createDropTarget(ele!); //so the whole grid is the drop target?
+        this.createDropAndGestureTarget(ele!); //so the whole grid is the drop target?
     }
 
     overlays = (doc: Doc) => {
@@ -309,7 +309,7 @@ export class CollectionStackingView extends CollectionSubView(doc => doc) {
             docList={docList}
             parent={this}
             type={type}
-            createDropTarget={this.createDropTarget}
+            createDropTarget={this.createDropAndGestureTarget}
             screenToLocalTransform={this.props.ScreenToLocalTransform}
         />;
     }
@@ -342,7 +342,7 @@ export class CollectionStackingView extends CollectionSubView(doc => doc) {
             docList={docList}
             parent={this}
             type={type}
-            createDropTarget={this.createDropTarget}
+            createDropTarget={this.createDropAndGestureTarget}
             screenToLocalTransform={this.props.ScreenToLocalTransform}
             setDocHeight={this.setDocHeight}
         />;
diff --git a/src/client/views/collections/CollectionSubView.tsx b/src/client/views/collections/CollectionSubView.tsx
index 062521690..73dc7edc6 100644
--- a/src/client/views/collections/CollectionSubView.tsx
+++ b/src/client/views/collections/CollectionSubView.tsx
@@ -23,6 +23,7 @@ import { basename } from 'path';
 import { GooglePhotos } from "../../apis/google_docs/GooglePhotosClientUtils";
 import { ImageUtils } from "../../util/Import & Export/ImageUtils";
 import { Networking } from "../../Network";
+import { GestureUtils } from "../../../pen-gestures/GestureUtils";
 
 export interface CollectionViewProps extends FieldViewProps {
     addDocument: (document: Doc) => boolean;
@@ -47,15 +48,18 @@ export interface SubCollectionViewProps extends CollectionViewProps {
 export function CollectionSubView<T>(schemaCtor: (doc: Doc) => T) {
     class CollectionSubView extends DocComponent<SubCollectionViewProps, T>(schemaCtor) {
         private dropDisposer?: DragManager.DragDropDisposer;
+        private gestureDisposer?: GestureUtils.GestureEventDisposer;
         private _childLayoutDisposer?: IReactionDisposer;
-        protected createDropTarget = (ele: HTMLDivElement) => { //used for stacking and masonry view
+        protected createDropAndGestureTarget = (ele: HTMLDivElement) => { //used for stacking and masonry view
             this.dropDisposer && this.dropDisposer();
+            this.gestureDisposer && this.gestureDisposer();
             if (ele) {
                 this.dropDisposer = DragManager.MakeDropTarget(ele, this.drop.bind(this));
+                this.gestureDisposer = GestureUtils.MakeGestureTarget(ele, this.onGesture.bind(this));
             }
         }
         protected CreateDropTarget(ele: HTMLDivElement) { //used in schema view
-            this.createDropTarget(ele);
+            this.createDropAndGestureTarget(ele);
         }
 
         componentDidMount() {
@@ -129,6 +133,11 @@ export function CollectionSubView<T>(schemaCtor: (doc: Doc) => T) {
             }
         }
 
+        @undoBatch
+        protected onGesture(e: Event, ge: GestureUtils.GestureEvent) {
+
+        }
+
         @undoBatch
         @action
         protected drop(e: Event, de: DragManager.DropEvent): boolean {
diff --git a/src/client/views/collections/CollectionTreeView.scss b/src/client/views/collections/CollectionTreeView.scss
index 0b9dc2eb2..2fa6813d7 100644
--- a/src/client/views/collections/CollectionTreeView.scss
+++ b/src/client/views/collections/CollectionTreeView.scss
@@ -7,9 +7,9 @@
     border-radius: inherit;
     box-sizing: border-box;
     height: 100%;
-    width:100%;
+    width: 100%;
     position: relative;
-    top:0;
+    top: 0;
     padding-left: 10px;
     padding-right: 10px;
     background: $light-color-secondary;
@@ -17,6 +17,7 @@
     overflow: auto;
     user-select: none;
     cursor: default;
+    touch-action: pan-y;
 
     ul {
         list-style: none;
@@ -115,6 +116,7 @@
 .treeViewItem-header {
     border: transparent 1px solid;
     display: flex;
+
     .editableView-container-editing-oneLine {
         min-width: 15px;
     }
diff --git a/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx b/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx
index eb5a074bb..9656debf3 100644
--- a/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx
+++ b/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx
@@ -272,11 +272,12 @@ export class CollectionFreeFormView extends CollectionSubView(PanZoomDocument) {
         return clusterColor;
     }
 
-    @observable private _points: { X: number, Y: number }[] = [];
 
     @action
     onPointerDown = (e: React.PointerEvent): void => {
-        if (e.nativeEvent.cancelBubble) return;
+        if (e.nativeEvent.cancelBubble || InteractionUtils.IsType(e, InteractionUtils.PENTYPE) || (InkingControl.Instance.selectedTool === InkTool.Highlighter || InkingControl.Instance.selectedTool === InkTool.Pen)) {
+            return;
+        }
         this._hitCluster = this.props.Document.useClusters ? this.pickCluster(this.getTransform().transformPoint(e.clientX, e.clientY)) !== -1 : false;
         if (e.button === 0 && !e.shiftKey && !e.altKey && !e.ctrlKey && this.props.active(true)) {
             document.removeEventListener("pointermove", this.onPointerMove);
@@ -284,14 +285,14 @@ export class CollectionFreeFormView extends CollectionSubView(PanZoomDocument) {
             document.addEventListener("pointermove", this.onPointerMove);
             document.addEventListener("pointerup", this.onPointerUp);
             // if physically using a pen or we're in pen or highlighter mode
-            if (InteractionUtils.IsType(e, InteractionUtils.PENTYPE) || (InkingControl.Instance.selectedTool === InkTool.Highlighter || InkingControl.Instance.selectedTool === InkTool.Pen)) {
-                e.stopPropagation();
-                e.preventDefault();
-                const point = this.getTransform().transformPoint(e.pageX, e.pageY);
-                this._points.push({ X: point[0], Y: point[1] });
-            }
+            // if (InteractionUtils.IsType(e, InteractionUtils.PENTYPE) || (InkingControl.Instance.selectedTool === InkTool.Highlighter || InkingControl.Instance.selectedTool === InkTool.Pen)) {
+            //     e.stopPropagation();
+            //     e.preventDefault();
+            //     const point = this.getTransform().transformPoint(e.pageX, e.pageY);
+            //     this._points.push({ X: point[0], Y: point[1] });
+            // }
             // if not using a pen and in no ink mode
-            else if (InkingControl.Instance.selectedTool === InkTool.None) {
+            if (InkingControl.Instance.selectedTool === InkTool.None) {
                 this._lastX = e.pageX;
                 this._lastY = e.pageY;
             }
@@ -332,13 +333,13 @@ export class CollectionFreeFormView extends CollectionSubView(PanZoomDocument) {
                 document.addEventListener("touchmove", this.onTouch);
                 document.removeEventListener("touchend", this.onTouchEnd);
                 document.addEventListener("touchend", this.onTouchEnd);
-                if (InkingControl.Instance.selectedTool === InkTool.Highlighter || InkingControl.Instance.selectedTool === InkTool.Pen) {
-                    e.stopPropagation();
-                    e.preventDefault();
-                    const point = this.getTransform().transformPoint(pt.pageX, pt.pageY);
-                    this._points.push({ X: point[0], Y: point[1] });
-                }
-                else if (InkingControl.Instance.selectedTool === InkTool.None) {
+                // if (InkingControl.Instance.selectedTool === InkTool.Highlighter || InkingControl.Instance.selectedTool === InkTool.Pen) {
+                //     e.stopPropagation();
+                //     e.preventDefault();
+                //     const point = this.getTransform().transformPoint(pt.pageX, pt.pageY);
+                //     this._points.push({ X: point[0], Y: point[1] });
+                // }
+                if (InkingControl.Instance.selectedTool === InkTool.None) {
                     this._lastX = pt.pageX;
                     this._lastY = pt.pageY;
                     e.stopPropagation();
@@ -352,72 +353,44 @@ export class CollectionFreeFormView extends CollectionSubView(PanZoomDocument) {
         }
     }
 
-    @action
-    onPointerUp = (e: PointerEvent): void => {
-        if (InteractionUtils.IsType(e, InteractionUtils.TOUCHTYPE) && this._points.length <= 1) return;
-
-        if (this._points.length > 1) {
-            const B = this.svgBounds;
-            const points = this._points.map(p => ({ X: p.X - B.left, Y: p.Y - B.top }));
-
-            const result = GestureUtils.GestureRecognizer.Recognize(new Array(points));
-            let actionPerformed = false;
-            if (result && result.Score > 0.7) {
-                switch (result.Name) {
-                    case GestureUtils.Gestures.Box:
-                        const bounds = { x: Math.min(...this._points.map(p => p.X)), r: Math.max(...this._points.map(p => p.X)), y: Math.min(...this._points.map(p => p.Y)), b: Math.max(...this._points.map(p => p.Y)) };
-                        const sel = this.getActiveDocuments().filter(doc => {
-                            const l = NumCast(doc.x);
-                            const r = l + doc[WidthSym]();
-                            const t = NumCast(doc.y);
-                            const b = t + doc[HeightSym]();
-                            const pass = !(bounds.x > r || bounds.r < l || bounds.y > b || bounds.b < t);
-                            if (pass) {
-                                doc.x = l - B.left - B.width / 2;
-                                doc.y = t - B.top - B.height / 2;
-                            }
-                            return pass;
-                        });
-                        this.addDocument(Docs.Create.FreeformDocument(sel, { x: B.left, y: B.top, width: B.width, height: B.height, panX: 0, panY: 0 }));
-                        sel.forEach(d => this.props.removeDocument(d));
-                        actionPerformed = true;
-                        break;
-                    case GestureUtils.Gestures.Line:
-                        const ep1 = this._points[0];
-                        const ep2 = this._points[this._points.length - 1];
-                        let d1: Doc | undefined;
-                        let d2: Doc | undefined;
-                        this.getActiveDocuments().map(doc => {
-                            const l = NumCast(doc.x);
-                            const r = l + doc[WidthSym]();
-                            const t = NumCast(doc.y);
-                            const b = t + doc[HeightSym]();
-                            if (!d1 && l < ep1.X && r > ep1.X && t < ep1.Y && b > ep1.Y) {
-                                d1 = doc;
-                            }
-                            else if (!d2 && l < ep2.X && r > ep2.X && t < ep2.Y && b > ep2.Y) {
-                                d2 = doc;
-                            }
-                        });
-                        if (d1 && d2) {
-                            if (!LinkManager.Instance.doesLinkExist(d1, d2)) {
-                                DocUtils.MakeLink({ doc: d1 }, { doc: d2 });
-                                actionPerformed = true;
-                            }
-                        }
-                        break;
-                }
-                if (actionPerformed) {
-                    this._points = [];
-                }
-            }
-
-            if (!actionPerformed) {
-                const inkDoc = Docs.Create.InkDocument(InkingControl.Instance.selectedColor, InkingControl.Instance.selectedTool, parseInt(InkingControl.Instance.selectedWidth), points, { width: B.width, height: B.height, x: B.left, y: B.top });
+    @undoBatch
+    onGesture = (e: Event, ge: GestureUtils.GestureEvent) => {
+        switch (ge.gesture) {
+            case GestureUtils.Gestures.Stroke:
+                const points = ge.points;
+                const B = this.getTransform().transformBounds(ge.bounds.left, ge.bounds.top, ge.bounds.width, ge.bounds.height);
+                const inkDoc = Docs.Create.InkDocument(InkingControl.Instance.selectedColor, InkingControl.Instance.selectedTool, parseInt(InkingControl.Instance.selectedWidth), points, { x: B.x, y: B.y, width: B.width, height: B.height });
                 this.addDocument(inkDoc);
-                this._points = [];
-            }
+                e.stopPropagation();
+                break;
+            case GestureUtils.Gestures.Box:
+                const lt = this.getTransform().transformPoint(Math.min(...ge.points.map(p => p.X)), Math.min(...ge.points.map(p => p.Y)));
+                const rb = this.getTransform().transformPoint(Math.max(...ge.points.map(p => p.X)), Math.max(...ge.points.map(p => p.Y)));
+                const bounds = { x: lt[0], r: rb[0], y: lt[1], b: rb[1] };
+                const bWidth = bounds.r - bounds.x;
+                const bHeight = bounds.b - bounds.y;
+                const sel = this.getActiveDocuments().filter(doc => {
+                    const l = NumCast(doc.x);
+                    const r = l + doc[WidthSym]();
+                    const t = NumCast(doc.y);
+                    const b = t + doc[HeightSym]();
+                    const pass = !(bounds.x > r || bounds.r < l || bounds.y > b || bounds.b < t);
+                    if (pass) {
+                        doc.x = l - bounds.x - bWidth / 2;
+                        doc.y = t - bounds.y - bHeight / 2;
+                    }
+                    return pass;
+                });
+                this.addDocument(Docs.Create.FreeformDocument(sel, { x: bounds.x, y: bounds.y, width: bWidth, height: bHeight, panX: 0, panY: 0 }));
+                sel.forEach(d => this.props.removeDocument(d));
+                break;
+
         }
+    }
+
+    @action
+    onPointerUp = (e: PointerEvent): void => {
+        if (InteractionUtils.IsType(e, InteractionUtils.TOUCHTYPE)) return;
 
         document.removeEventListener("pointermove", this.onPointerMove);
         document.removeEventListener("pointerup", this.onPointerUp);
@@ -473,11 +446,7 @@ export class CollectionFreeFormView extends CollectionSubView(PanZoomDocument) {
         }
         if (!e.cancelBubble) {
             const selectedTool = InkingControl.Instance.selectedTool;
-            if (selectedTool === InkTool.Highlighter || selectedTool === InkTool.Pen || InteractionUtils.IsType(e, InteractionUtils.PENTYPE)) {
-                const point = this.getTransform().transformPoint(e.clientX, e.clientY);
-                this._points.push({ X: point[0], Y: point[1] });
-            }
-            else if (selectedTool === InkTool.None) {
+            if (selectedTool === InkTool.None) {
                 if (this._hitCluster && this.tryDragCluster(e)) {
                     e.stopPropagation(); // doesn't actually stop propagation since all our listeners are listening to events on 'document'  however it does mark the event as cancelBubble=true which we test for in the move event handlers
                     e.preventDefault();
@@ -508,10 +477,6 @@ export class CollectionFreeFormView extends CollectionSubView(PanZoomDocument) {
                     }
                     this.pan(pt);
                 }
-                else if (InkingControl.Instance.selectedTool !== InkTool.Eraser && InkingControl.Instance.selectedTool !== InkTool.Scrubber) {
-                    const point = this.getTransform().transformPoint(pt.clientX, pt.clientY);
-                    this._points.push({ X: point[0], Y: point[1] });
-                }
             }
             e.stopPropagation();
             e.preventDefault();
@@ -942,34 +907,10 @@ export class CollectionFreeFormView extends CollectionSubView(PanZoomDocument) {
         ];
     }
 
-    @computed get svgBounds() {
-        const xs = this._points.map(p => p.X);
-        const ys = this._points.map(p => p.Y);
-        const right = Math.max(...xs);
-        const left = Math.min(...xs);
-        const bottom = Math.max(...ys);
-        const top = Math.min(...ys);
-        return { right: right, left: left, bottom: bottom, top: top, width: right - left, height: bottom - top };
-    }
-
-    @computed get currentStroke() {
-        if (this._points.length <= 1) {
-            return (null);
-        }
-
-        const B = this.svgBounds;
-
-        return (
-            <svg width={B.width} height={B.height} style={{ transform: `translate(${B.left}px, ${B.top}px)`, position: "absolute", zIndex: 30000 }}>
-                {CreatePolyline(this._points, B.left, B.top)}
-            </svg>
-        );
-    }
-
     children = () => {
         const eles: JSX.Element[] = [];
         this.extensionDoc && (eles.push(...this.childViews()));
-        this.currentStroke && (eles.push(this.currentStroke));
+        // this.currentStroke && (eles.push(this.currentStroke));
         eles.push(<CollectionFreeFormRemoteCursors {...this.props} key="remoteCursors" />);
         return eles;
     }
@@ -998,7 +939,7 @@ export class CollectionFreeFormView extends CollectionSubView(PanZoomDocument) {
         // otherwise, they are stored in fieldKey.  All annotations to this document are stored in the extension document
         if (!this.extensionDoc) return (null);
         // let lodarea = this.Document[WidthSym]() * this.Document[HeightSym]() / this.props.ScreenToLocalTransform().Scale / this.props.ScreenToLocalTransform().Scale;
-        return <div className={"collectionfreeformview-container"} ref={this.createDropTarget} onWheel={this.onPointerWheel}//pointerEvents: SelectionManager.GetIsDragging() ? "all" : undefined,
+        return <div className={"collectionfreeformview-container"} ref={this.createDropAndGestureTarget} onWheel={this.onPointerWheel}//pointerEvents: SelectionManager.GetIsDragging() ? "all" : undefined,
             style={{ pointerEvents: SelectionManager.GetIsDragging() ? "all" : undefined, height: this.isAnnotationOverlay ? (this.props.Document.scrollHeight ? this.Document.scrollHeight : "100%") : this.props.PanelHeight() }}
             onPointerDown={this.onPointerDown} onPointerMove={this.onCursorMove} onDrop={this.onDrop.bind(this)} onContextMenu={this.onContextMenu} onTouchStart={this.onTouchStart}>
             {!this.Document.LODdisable && !this.props.active() && !this.props.isAnnotationOverlay && !this.props.annotationsKey && this.props.renderDepth > 0 ? // && this.props.CollectionView && lodarea < NumCast(this.Document.LODarea, 100000) ?
diff --git a/src/client/views/nodes/DocumentView.tsx b/src/client/views/nodes/DocumentView.tsx
index a14f69f71..b544319d7 100644
--- a/src/client/views/nodes/DocumentView.tsx
+++ b/src/client/views/nodes/DocumentView.tsx
@@ -45,6 +45,7 @@ import { InkTool } from '../../../new_fields/InkField';
 import { TraceMobx } from '../../../new_fields/util';
 import { List } from '../../../new_fields/List';
 import { FormattedTextBoxComment } from './FormattedTextBoxComment';
+import { GestureUtils } from '../../../pen-gestures/GestureUtils';
 
 library.add(fa.faEdit, fa.faTrash, fa.faShare, fa.faDownload, fa.faExpandArrowsAlt, fa.faCompressArrowsAlt, fa.faLayerGroup, fa.faExternalLinkAlt, fa.faAlignCenter, fa.faCaretSquareRight,
     fa.faSquare, fa.faConciergeBell, fa.faWindowRestore, fa.faFolder, fa.faMapPin, fa.faLink, fa.faFingerprint, fa.faCrosshairs, fa.faDesktop, fa.faUnlock, fa.faLock, fa.faLaptopCode, fa.faMale,
@@ -94,6 +95,7 @@ export class DocumentView extends DocComponent<DocumentViewProps, Document>(Docu
     private _hitTemplateDrag = false;
     private _mainCont = React.createRef<HTMLDivElement>();
     private _dropDisposer?: DragManager.DragDropDisposer;
+    private _gestureEventDisposer?: GestureUtils.GestureEventDisposer;
     private _titleRef = React.createRef<EditableView>();
 
     public get displayName() { return "DocumentView(" + this.props.Document.title + ")"; } // this makes mobx trace() statements more descriptive
@@ -107,6 +109,7 @@ export class DocumentView extends DocComponent<DocumentViewProps, Document>(Docu
     @action
     componentDidMount() {
         this._mainCont.current && (this._dropDisposer = DragManager.MakeDropTarget(this._mainCont.current, this.drop.bind(this)));
+        this._mainCont.current && (this._gestureEventDisposer = GestureUtils.MakeGestureTarget(this._mainCont.current, this.onGesture.bind(this)));
 
         !this.props.dontRegisterView && DocumentManager.Instance.DocumentViews.push(this);
     }
@@ -365,10 +368,11 @@ export class DocumentView extends DocComponent<DocumentViewProps, Document>(Docu
         // console.log(e.button)
         // console.log(e.nativeEvent)
         // continue if the event hasn't been canceled AND we are using a moues or this is has an onClick or onDragStart function (meaning it is a button document)
+        if (InteractionUtils.IsType(e, InteractionUtils.PENTYPE) || (InkingControl.Instance.selectedTool === InkTool.Highlighter || InkingControl.Instance.selectedTool === InkTool.Pen)) {
+            return;
+        }
         if (!InteractionUtils.IsType(e, InteractionUtils.MOUSETYPE)) {
-            if (!InteractionUtils.IsType(e, InteractionUtils.PENTYPE)) {
-                e.stopPropagation();
-            }
+            e.stopPropagation();
             return;
         }
         if ((!e.nativeEvent.cancelBubble || this.Document.onClick || this.Document.onDragStart)) {
@@ -424,6 +428,15 @@ export class DocumentView extends DocComponent<DocumentViewProps, Document>(Docu
         this._lastTap = Date.now();
     }
 
+    onGesture = (e: Event, ge: GestureUtils.GestureEvent) => {
+        switch (ge.gesture) {
+            case GestureUtils.Gestures.Line:
+                ge.callbackFn && ge.callbackFn(this.props.Document);
+                e.stopPropagation();
+                break;
+        }
+    }
+
     @undoBatch
     deleteClicked = (): void => { SelectionManager.DeselectAll(); this.props.removeDocument && this.props.removeDocument(this.props.Document); }
 
diff --git a/src/pen-gestures/GestureUtils.ts b/src/pen-gestures/GestureUtils.ts
index 59a85b66b..4edcfa623 100644
--- a/src/pen-gestures/GestureUtils.ts
+++ b/src/pen-gestures/GestureUtils.ts
@@ -1,19 +1,43 @@
 import { NDollarRecognizer } from "./ndollar";
 import { Type } from "typescript";
-import { InkField } from "../new_fields/InkField";
+import { InkField, PointData } from "../new_fields/InkField";
 import { Docs } from "../client/documents/Documents";
 import { Doc, WidthSym, HeightSym } from "../new_fields/Doc";
 import { NumCast } from "../new_fields/Types";
 import { CollectionFreeFormView } from "../client/views/collections/collectionFreeForm/CollectionFreeFormView";
+import { Rect } from "react-measure";
 
 export namespace GestureUtils {
     namespace GestureDataTypes {
         export type BoxData = Array<Doc>;
     }
 
+    export class GestureEvent {
+        constructor(
+            readonly gesture: Gestures,
+            readonly points: PointData[],
+            readonly bounds: Rect,
+            readonly callbackFn?: Function
+        ) { }
+    }
+
+    export interface GestureEventDisposer { (): void; }
+
+    export function MakeGestureTarget(
+        element: HTMLElement,
+        func: (e: Event, ge: GestureEvent) => void
+    ): GestureEventDisposer {
+        const handler = (e: Event) => func(e, (e as CustomEvent<GestureEvent>).detail);
+        element.addEventListener("dashOnGesture", handler);
+        return () => {
+            element.removeEventListener("dashOnGesture", handler);
+        }
+    }
+
     export enum Gestures {
         Box = "box",
-        Line = "line"
+        Line = "line",
+        Stroke = "stroke"
     }
 
     export const GestureRecognizer = new NDollarRecognizer(false);
-- 
cgit v1.2.3-70-g09d2


From 36eed6cc3a3387a1f7755a848aea4e19e2645e14 Mon Sep 17 00:00:00 2001
From: Stanley Yip <stanley_yip@brown.edu>
Date: Thu, 9 Jan 2020 19:55:52 -0500
Subject: some stuff that ive played around with today. mobile interface, five
 fingers, other stuff

---
 package-lock.json                                  | 27 +++++++++
 package.json                                       |  1 +
 src/client/views/GestureOverlay.scss               |  2 +-
 src/client/views/GestureOverlay.tsx                | 69 +++++++++++++---------
 src/client/views/InkingControl.tsx                 | 21 ++++---
 src/client/views/MainView.tsx                      |  4 +-
 src/client/views/Palette.scss                      | 18 ++++++
 src/client/views/Palette.tsx                       | 25 ++++++++
 src/client/views/Touchable.tsx                     | 18 ++++++
 .../views/collections/CollectionSchemaView.scss    | 14 +++--
 .../collectionFreeForm/CollectionFreeFormView.tsx  | 54 +++++++++++++++++
 src/client/views/nodes/DocumentView.tsx            | 17 +-----
 src/mobile/ImageUpload.tsx                         | 31 +++++++---
 src/mobile/MobileInterface.tsx                     | 69 ++++++++++++++++++++++
 src/new_fields/InkField.ts                         |  3 +-
 src/pen-gestures/GestureUtils.ts                   |  3 +-
 .../authentication/models/current_user_utils.ts    | 26 ++++++++
 17 files changed, 335 insertions(+), 67 deletions(-)
 create mode 100644 src/client/views/Palette.scss
 create mode 100644 src/client/views/Palette.tsx
 create mode 100644 src/mobile/MobileInterface.tsx

(limited to 'src/pen-gestures/GestureUtils.ts')

diff --git a/package-lock.json b/package-lock.json
index d807d75a1..244b2e7e0 100644
--- a/package-lock.json
+++ b/package-lock.json
@@ -2040,6 +2040,11 @@
             "resolved": "https://registry.npmjs.org/atob/-/atob-2.1.2.tgz",
             "integrity": "sha512-Wm6ukoaOGJi/73p/cl2GvLjTI5JM1k/O14isD73YML8StrH/7/lRFgmg8nICZgD3bZZvjwCGxtMOD3wWNAu8cg=="
         },
+        "avl": {
+            "version": "1.4.4",
+            "resolved": "https://registry.npmjs.org/avl/-/avl-1.4.4.tgz",
+            "integrity": "sha512-vMjF4jhT2N0naO1fsBTA/lZ2AB70ECBv41na7oBe8B5PCgMVvNrWG+VRKMlbG5Mk5UGbMxzUD9fXK/IBsmmO5A=="
+        },
         "awesome-typescript-loader": {
             "version": "5.2.1",
             "resolved": "https://registry.npmjs.org/awesome-typescript-loader/-/awesome-typescript-loader-5.2.1.tgz",
@@ -2314,6 +2319,14 @@
                 "tweetnacl": "^0.14.3"
             }
         },
+        "bentley-ottman-sweepline": {
+            "version": "0.1.4",
+            "resolved": "https://registry.npmjs.org/bentley-ottman-sweepline/-/bentley-ottman-sweepline-0.1.4.tgz",
+            "integrity": "sha512-Qptw9uoNLM0zLiJfA+9PqFrF5b6pMiJNyoWgqONq4/wVml4NC2N8S4RXYXn4xXR4JEBhHF+euVwwML7D9X1LqA==",
+            "requires": {
+                "avl": "^1.4.0"
+            }
+        },
         "better-assert": {
             "version": "1.0.2",
             "resolved": "https://registry.npmjs.org/better-assert/-/better-assert-1.0.2.tgz",
@@ -4316,6 +4329,14 @@
                 "minimalistic-crypto-utils": "^1.0.0"
             }
         },
+        "emit-logger": {
+            "version": "github:chocolateboy/emit-logger#b9d25a2d939e42f29c940861e9648bd0fb810070",
+            "from": "github:chocolateboy/emit-logger#better-emitter-name",
+            "requires": {
+                "chalk": "^1.1.1",
+                "moment": "^2.10.6"
+            }
+        },
         "emoji-regex": {
             "version": "7.0.3",
             "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-7.0.3.tgz",
@@ -7065,6 +7086,7 @@
                 "babel-runtime": "^6.22.0",
                 "bluebird": "^3.4.7",
                 "boolify-string": "^2.0.2",
+                "emit-logger": "github:chocolateboy/emit-logger#better-emitter-name",
                 "lodash": "^4.17.4",
                 "semver": "^5.0.3",
                 "source-map-support": "^0.5.9"
@@ -8545,6 +8567,11 @@
                 }
             }
         },
+        "moment": {
+            "version": "2.24.0",
+            "resolved": "https://registry.npmjs.org/moment/-/moment-2.24.0.tgz",
+            "integrity": "sha512-bV7f+6l2QigeBBZSM/6yTNq4P2fNpSWj/0e7jQcy87A8e7o2nAfP/34/2ky5Vw4B9S446EtIhodAzkFCcR4dQg=="
+        },
         "mongodb": {
             "version": "3.2.7",
             "resolved": "https://registry.npmjs.org/mongodb/-/mongodb-3.2.7.tgz",
diff --git a/package.json b/package.json
index 9bd2273f8..2a72fac80 100644
--- a/package.json
+++ b/package.json
@@ -121,6 +121,7 @@
         "async": "^2.6.2",
         "babel-runtime": "^6.26.0",
         "bcrypt-nodejs": "0.0.3",
+        "bentley-ottman-sweepline": "^0.1.4",
         "bluebird": "^3.5.3",
         "body-parser": "^1.18.3",
         "bootstrap": "^4.3.1",
diff --git a/src/client/views/GestureOverlay.scss b/src/client/views/GestureOverlay.scss
index cbc1b6e7d..31601efd4 100644
--- a/src/client/views/GestureOverlay.scss
+++ b/src/client/views/GestureOverlay.scss
@@ -1,6 +1,6 @@
 .gestureOverlay-cont {
     width: 100vw;
-    height: 100vw;
+    height: 100vh;
     position: absolute;
     top: 0;
     left: 0;
diff --git a/src/client/views/GestureOverlay.tsx b/src/client/views/GestureOverlay.tsx
index c88e3f7ae..848927912 100644
--- a/src/client/views/GestureOverlay.tsx
+++ b/src/client/views/GestureOverlay.tsx
@@ -43,9 +43,43 @@ export default class GestureOverlay extends Touchable {
 
     @action
     onPointerMove = (e: PointerEvent) => {
-        this._points.push({ X: e.clientX, Y: e.clientY });
-        e.stopPropagation();
-        e.preventDefault();
+        if (InteractionUtils.IsType(e, InteractionUtils.PENTYPE) || (InkingControl.Instance.selectedTool === InkTool.Highlighter || InkingControl.Instance.selectedTool === InkTool.Pen)) {
+            this._points.push({ X: e.clientX, Y: e.clientY });
+            e.stopPropagation();
+            e.preventDefault();
+        }
+    }
+
+    handleLineGesture = (): boolean => {
+        let actionPerformed = false;
+        const B = this.svgBounds;
+        const ep1 = this._points[0];
+        const ep2 = this._points[this._points.length - 1];
+
+        const target1 = document.elementFromPoint(ep1.X, ep1.Y);
+        const target2 = document.elementFromPoint(ep2.X, ep2.Y);
+        const callback = (doc: Doc) => {
+            if (!this._d1) {
+                this._d1 = doc;
+            }
+            else if (this._d1 !== doc && !LinkManager.Instance.doesLinkExist(this._d1, doc)) {
+                DocUtils.MakeLink({ doc: this._d1 }, { doc: doc });
+                actionPerformed = true;
+            }
+        }
+        const ge = new CustomEvent<GestureUtils.GestureEvent>("dashOnGesture",
+            {
+                bubbles: true,
+                detail: {
+                    points: this._points,
+                    gesture: GestureUtils.Gestures.Line,
+                    bounds: B,
+                    callbackFn: callback
+                }
+            });
+        target1?.dispatchEvent(ge);
+        target2?.dispatchEvent(ge);
+        return actionPerformed;
     }
 
     @action
@@ -72,31 +106,10 @@ export default class GestureOverlay extends Touchable {
                         actionPerformed = true;
                         break;
                     case GestureUtils.Gestures.Line:
-                        const ep1 = this._points[0];
-                        const ep2 = this._points[this._points.length - 1];
-                        const target1 = document.elementFromPoint(ep1.X, ep1.Y);
-                        const target2 = document.elementFromPoint(ep2.X, ep2.Y);
-                        const callback = (doc: Doc) => {
-                            if (!this._d1) {
-                                this._d1 = doc;
-                            }
-                            else if (this._d1 !== doc && !LinkManager.Instance.doesLinkExist(this._d1, doc)) {
-                                DocUtils.MakeLink({ doc: this._d1 }, { doc: doc });
-                                actionPerformed = true;
-                            }
-                        }
-                        const ge = new CustomEvent<GestureUtils.GestureEvent>("dashOnGesture",
-                            {
-                                bubbles: true,
-                                detail: {
-                                    points: this._points,
-                                    gesture: GestureUtils.Gestures.Line,
-                                    bounds: B,
-                                    callbackFn: callback
-                                }
-                            })
-                        target1?.dispatchEvent(ge);
-                        target2?.dispatchEvent(ge);
+                        actionPerformed = this.handleLineGesture();
+                        break;
+                    case GestureUtils.Gestures.Scribble:
+                        console.log("scribble");
                         break;
                 }
                 if (actionPerformed) {
diff --git a/src/client/views/InkingControl.tsx b/src/client/views/InkingControl.tsx
index e33f193b8..243123352 100644
--- a/src/client/views/InkingControl.tsx
+++ b/src/client/views/InkingControl.tsx
@@ -4,19 +4,18 @@ import { Doc } from "../../new_fields/Doc";
 import { InkTool } from "../../new_fields/InkField";
 import { List } from "../../new_fields/List";
 import { listSpec } from "../../new_fields/Schema";
-import { Cast, NumCast, StrCast } from "../../new_fields/Types";
+import { Cast, NumCast, StrCast, FieldValue } from "../../new_fields/Types";
 import { Utils } from "../../Utils";
 import { Scripting } from "../util/Scripting";
 import { SelectionManager } from "../util/SelectionManager";
 import { undoBatch, UndoManager } from "../util/UndoManager";
 import { CurrentUserUtils } from "../../server/authentication/models/current_user_utils";
 
-
 export class InkingControl {
     @observable static Instance: InkingControl;
-    @observable private _selectedTool: InkTool = InkTool.None;
-    @observable private _selectedColor: string = "rgb(244, 67, 54)";
-    @observable private _selectedWidth: string = "5";
+    @computed private get _selectedTool(): InkTool { return FieldValue(NumCast(CurrentUserUtils.UserDocument.inkTool)) ?? InkTool.None; }
+    @computed private get _selectedColor(): string { return FieldValue(StrCast(CurrentUserUtils.UserDocument.inkColor)) ?? "rgb(244, 67, 54)"; }
+    @computed private get _selectedWidth(): string { return FieldValue(StrCast(CurrentUserUtils.UserDocument.inkWidth)) ?? "5"; }
     @observable public _open: boolean = false;
 
     constructor() {
@@ -24,7 +23,8 @@ export class InkingControl {
     }
 
     switchTool = action((tool: InkTool): void => {
-        this._selectedTool = tool;
+        // this._selectedTool = tool;
+        CurrentUserUtils.UserDocument.inkTool = tool;
     });
     decimalToHexString(number: number) {
         if (number < 0) {
@@ -36,7 +36,7 @@ export class InkingControl {
 
     @undoBatch
     switchColor = action((color: ColorState): void => {
-        this._selectedColor = color.hex + (color.rgb.a !== undefined ? this.decimalToHexString(Math.round(color.rgb.a * 255)) : "ff");
+        CurrentUserUtils.UserDocument.inkColor = color.hex + (color.rgb.a !== undefined ? this.decimalToHexString(Math.round(color.rgb.a * 255)) : "ff");
 
         if (InkingControl.Instance.selectedTool === InkTool.None) {
             const selected = SelectionManager.SelectedDocuments();
@@ -99,7 +99,8 @@ export class InkingControl {
     });
     @action
     switchWidth = (width: string): void => {
-        this._selectedWidth = width;
+        // this._selectedWidth = width;
+        CurrentUserUtils.UserDocument.inkWidth = width;
     }
 
     @computed
@@ -114,7 +115,8 @@ export class InkingControl {
 
     @action
     updateSelectedColor(value: string) {
-        this._selectedColor = value;
+        // this._selectedColor = value;
+        CurrentUserUtils.UserDocument.inkColor = value;
     }
 
     @computed
@@ -127,6 +129,7 @@ Scripting.addGlobal(function activatePen(pen: any, width: any, color: any) { Ink
 Scripting.addGlobal(function activateBrush(pen: any, width: any, color: any) { InkingControl.Instance.switchTool(pen ? InkTool.Highlighter : InkTool.None); InkingControl.Instance.switchWidth(width); InkingControl.Instance.updateSelectedColor(color); });
 Scripting.addGlobal(function activateEraser(pen: any) { return InkingControl.Instance.switchTool(pen ? InkTool.Eraser : InkTool.None); });
 Scripting.addGlobal(function activateScrubber(pen: any) { return InkingControl.Instance.switchTool(pen ? InkTool.Scrubber : InkTool.None); });
+Scripting.addGlobal(function activateStamp(pen: any) { return InkingControl.Instance.switchTool(pen ? InkTool.Stamp : InkTool.None); });
 Scripting.addGlobal(function deactivateInk() { return InkingControl.Instance.switchTool(InkTool.None); });
 Scripting.addGlobal(function setInkWidth(width: any) { return InkingControl.Instance.switchWidth(width); });
 Scripting.addGlobal(function setInkColor(color: any) { return InkingControl.Instance.updateSelectedColor(color); });
\ No newline at end of file
diff --git a/src/client/views/MainView.tsx b/src/client/views/MainView.tsx
index 14c1803c0..f463ca306 100644
--- a/src/client/views/MainView.tsx
+++ b/src/client/views/MainView.tsx
@@ -1,7 +1,7 @@
 import { library } from '@fortawesome/fontawesome-svg-core';
 import {
     faArrowDown, faArrowUp, faBolt, faCaretUp, faCat, faCheck, faChevronRight, faClone, faCloudUploadAlt, faCommentAlt, faCut, faEllipsisV, faExclamation, faFilePdf, faFilm, faFont, faGlobeAsia, faLongArrowAltRight,
-    faMusic, faObjectGroup, faPause, faMousePointer, faPenNib, faFileAudio, faPen, faEraser, faPlay, faPortrait, faRedoAlt, faThumbtack, faTree, faTv, faUndoAlt, faHighlighter, faMicrophone, faCompressArrowsAlt
+    faMusic, faObjectGroup, faPause, faMousePointer, faPenNib, faFileAudio, faPen, faEraser, faPlay, faPortrait, faRedoAlt, faThumbtack, faTree, faTv, faUndoAlt, faHighlighter, faMicrophone, faCompressArrowsAlt, faPhone, faStamp
 } from '@fortawesome/free-solid-svg-icons';
 import { FontAwesomeIcon } from '@fortawesome/react-fontawesome';
 import { action, computed, configure, observable, reaction, runInAction } from 'mobx';
@@ -135,6 +135,8 @@ export class MainView extends React.Component {
         library.add(faChevronRight);
         library.add(faEllipsisV);
         library.add(faMusic);
+        library.add(faPhone);
+        library.add(faStamp);
         this.initEventListeners();
         this.initAuthenticationRouters();
     }
diff --git a/src/client/views/Palette.scss b/src/client/views/Palette.scss
new file mode 100644
index 000000000..60004c81f
--- /dev/null
+++ b/src/client/views/Palette.scss
@@ -0,0 +1,18 @@
+.palette-container {
+    .palette-thumb {
+        width: 300px;
+        height: 300px;
+        touch-action: pan-x;
+        overflow: scroll;
+
+        .palette-thumbContent {
+            width: 100%;
+            height: 100%;
+
+            .palette-button {
+                width: 100px;
+                height: 100px;
+            }
+        }
+    }
+}
\ No newline at end of file
diff --git a/src/client/views/Palette.tsx b/src/client/views/Palette.tsx
new file mode 100644
index 000000000..390b7e2c2
--- /dev/null
+++ b/src/client/views/Palette.tsx
@@ -0,0 +1,25 @@
+import * as React from "react";
+import "./Palette.scss";
+import { PointData } from "../../new_fields/InkField";
+
+export interface PaletteProps {
+    x: number;
+    y: number;
+    thumb: number[];
+}
+
+export default class Palette extends React.Component<PaletteProps> {
+    render() {
+        return (
+            <div className="palette-container" style={{ transform: `translate(${this.props.x}px, ${this.props.y}px)` }}>
+                <div className="palette-thumb" style={{ transform: `translate(${this.props.thumb[0] - this.props.x}px, ${this.props.thumb[1] - this.props.y}px)` }}>
+                    <div className="palette-thumbContent">
+                        <div className="palette-button" style={{ background: "green" }} onPointerDown={() => console.log("hi")}>1</div>
+                        <div className="palette-button" style={{ background: "red" }}>2</div>
+                        <div className="palette-button" style={{ background: "blue" }}>3</div>
+                    </div>
+                </div>
+            </div>
+        );
+    }
+}
\ No newline at end of file
diff --git a/src/client/views/Touchable.tsx b/src/client/views/Touchable.tsx
index 251cd41e5..3eb66ff72 100644
--- a/src/client/views/Touchable.tsx
+++ b/src/client/views/Touchable.tsx
@@ -1,6 +1,7 @@
 import * as React from 'react';
 import { action } from 'mobx';
 import { InteractionUtils } from '../util/InteractionUtils';
+import { SelectionManager } from '../util/SelectionManager';
 
 const HOLD_DURATION = 1000;
 
@@ -34,11 +35,19 @@ export abstract class Touchable<T = {}> extends React.Component<T> {
                 case 1:
                     this.handle1PointerDown(e);
                     e.persist();
+                    if (this.holdTimer) {
+                        clearTimeout(this.holdTimer)
+                        this.holdTimer = undefined;
+                    }
                     this.holdTimer = setTimeout(() => this.handle1PointerHoldStart(e), HOLD_DURATION);
+                    console.log(this.holdTimer);
                     break;
                 case 2:
                     this.handle2PointersDown(e);
                     break;
+                case 5:
+                    this.handleHandDown(e);
+                    break;
             }
         }
     }
@@ -54,7 +63,9 @@ export abstract class Touchable<T = {}> extends React.Component<T> {
         if (!InteractionUtils.IsDragging(this.prevPoints, myTouches, 5) && !this._touchDrag) return;
         this._touchDrag = true;
         if (this.holdTimer) {
+            console.log("clear");
             clearTimeout(this.holdTimer);
+            this.holdTimer = undefined;
         }
         switch (myTouches.length) {
             case 1:
@@ -88,7 +99,9 @@ export abstract class Touchable<T = {}> extends React.Component<T> {
             }
         }
         if (this.holdTimer) {
+            console.log("clear");
             clearTimeout(this.holdTimer);
+            this.holdTimer = undefined;
         }
         this._touchDrag = false;
         e.stopPropagation();
@@ -137,4 +150,9 @@ export abstract class Touchable<T = {}> extends React.Component<T> {
         e.stopPropagation();
         e.preventDefault();
     }
+
+    handleHandDown = (e: React.TouchEvent) => {
+        e.stopPropagation();
+        e.preventDefault();
+    }
 }
\ No newline at end of file
diff --git a/src/client/views/collections/CollectionSchemaView.scss b/src/client/views/collections/CollectionSchemaView.scss
index cb95dcbbc..8b3d332af 100644
--- a/src/client/views/collections/CollectionSchemaView.scss
+++ b/src/client/views/collections/CollectionSchemaView.scss
@@ -15,6 +15,11 @@
     display: flex;
     justify-content: space-between;
     flex-wrap: nowrap;
+    touch-action: none;
+
+    div {
+        touch-action: none;
+    }
 
 
     .collectionSchemaView-tableContainer {
@@ -49,7 +54,7 @@
     .rt-table {
         height: 100%;
         display: -webkit-inline-box;
-        direction: ltr; 
+        direction: ltr;
         overflow: visible;
     }
 
@@ -202,7 +207,7 @@ button.add-column {
         border-bottom: 1px solid lightgray;
 
         &:first-child {
-            padding-top : 0;
+            padding-top: 0;
         }
 
         &:last-child {
@@ -231,7 +236,7 @@ button.add-column {
         transition: background-color 0.2s;
 
         &:hover {
-            background-color: $light-color-secondary; 
+            background-color: $light-color-secondary;
         }
 
         &.active {
@@ -267,7 +272,7 @@ button.add-column {
             overflow-y: scroll;
             position: absolute;
             top: 28px;
-            box-shadow: 0 10px 16px rgba(0,0,0,0.1);
+            box-shadow: 0 10px 16px rgba(0, 0, 0, 0.1);
 
             .key-option {
                 background-color: $light-color;
@@ -380,6 +385,7 @@ button.add-column {
 
     &.editing {
         padding: 0;
+
         input {
             outline: 0;
             border: none;
diff --git a/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx b/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx
index 9656debf3..d7cccc036 100644
--- a/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx
+++ b/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx
@@ -44,6 +44,7 @@ import { TraceMobx } from "../../../../new_fields/util";
 import { GestureUtils } from "../../../../pen-gestures/GestureUtils";
 import { LinkManager } from "../../../util/LinkManager";
 import { CognitiveServices } from "../../../cognitive_services/CognitiveServices";
+import Palette from "../../Palette";
 
 library.add(faEye as any, faTable, faPaintBrush, faExpandArrowsAlt, faCompressArrowsAlt, faCompass, faUpload, faBraille, faChalkboard, faFileUpload);
 
@@ -444,6 +445,9 @@ export class CollectionFreeFormView extends CollectionSubView(PanZoomDocument) {
             }
             return;
         }
+        if (InteractionUtils.IsType(e, InteractionUtils.PENTYPE)) {
+            return;
+        }
         if (!e.cancelBubble) {
             const selectedTool = InkingControl.Instance.selectedTool;
             if (selectedTool === InkTool.None) {
@@ -844,6 +848,53 @@ export class CollectionFreeFormView extends CollectionSubView(PanZoomDocument) {
         CognitiveServices.Inking.Appliers.ConcatenateHandwriting(this.dataDoc, ["inkAnalysis", "handwriting"], inkData);
     }
 
+    private thumbIdentifier?: number;
+    private hand?: React.Touch[];
+
+    @action
+    handleHandDown = (e: React.TouchEvent) => {
+        const fingers = InteractionUtils.GetMyTargetTouches(e, this.prevPoints);
+        this.hand = fingers;
+        const thumb = fingers.reduce((a, v) => a.clientY > v.clientY ? a : v, fingers[0]);
+        this.thumbIdentifier = thumb?.identifier;
+        const others = fingers.filter(f => f !== thumb);
+        const minX = Math.min(...others.map(f => f.clientX));
+        const minY = Math.min(...others.map(f => f.clientY));
+        const t = this.getTransform().transformPoint(minX, minY);
+        const th = this.getTransform().transformPoint(thumb.clientX, thumb.clientY);
+        this._palette = <Palette x={t[0]} y={t[1]} thumb={th} />;
+
+        document.removeEventListener("touchmove", this.onTouch);
+        document.removeEventListener("touchmove", this.handleHandMove);
+        document.addEventListener("touchmove", this.handleHandMove);
+        document.removeEventListener("touchend", this.handleHandUp);
+        document.addEventListener("touchend", this.handleHandUp);
+    }
+
+    @action
+    handleHandMove = (e: TouchEvent) => {
+        for (let i = 0; i < e.changedTouches.length; i++) {
+            const pt = e.changedTouches.item(i);
+            if (pt?.identifier === this.thumbIdentifier) {
+            }
+        }
+    }
+
+    @action
+    handleHandUp = (e: TouchEvent) => {
+        console.log(e.changedTouches.length);
+        this.onTouchEnd(e);
+        if (this.prevPoints.size < 3) {
+            if (this.hand) {
+                for (const h of this.hand) {
+                    this.prevPoints.has(h.identifier) && this.prevPoints.delete(h.identifier);
+                }
+            }
+            this._palette = undefined;
+            document.removeEventListener("touchend", this.handleHandUp);
+        }
+    }
+
     onContextMenu = (e: React.MouseEvent) => {
         const layoutItems: ContextMenuProps[] = [];
 
@@ -907,9 +958,12 @@ export class CollectionFreeFormView extends CollectionSubView(PanZoomDocument) {
         ];
     }
 
+    @observable private _palette?: JSX.Element;
+
     children = () => {
         const eles: JSX.Element[] = [];
         this.extensionDoc && (eles.push(...this.childViews()));
+        this._palette && (eles.push(this._palette));
         // this.currentStroke && (eles.push(this.currentStroke));
         eles.push(<CollectionFreeFormRemoteCursors {...this.props} key="remoteCursors" />);
         return eles;
diff --git a/src/client/views/nodes/DocumentView.tsx b/src/client/views/nodes/DocumentView.tsx
index 7d6b8bf01..415d0e2ef 100644
--- a/src/client/views/nodes/DocumentView.tsx
+++ b/src/client/views/nodes/DocumentView.tsx
@@ -256,7 +256,6 @@ export class DocumentView extends DocComponent<DocumentViewProps, Document>(Docu
             document.removeEventListener("touchend", this.onTouchEnd);
             document.addEventListener("touchend", this.onTouchEnd);
             if ((e.nativeEvent as any).formattedHandled) e.stopPropagation();
-            console.log("down")
         }
     }
 
@@ -327,6 +326,7 @@ export class DocumentView extends DocComponent<DocumentViewProps, Document>(Docu
                 const fixedAspect = e.ctrlKey || (!layoutDoc.ignoreAspect && nwidth && nheight);
                 if (fixedAspect && e.ctrlKey && layoutDoc.ignoreAspect) {
                     layoutDoc.ignoreAspect = false;
+
                     layoutDoc.nativeWidth = nwidth = layoutDoc.width || 0;
                     layoutDoc.nativeHeight = nheight = layoutDoc.height || 0;
                 }
@@ -357,8 +357,6 @@ export class DocumentView extends DocComponent<DocumentViewProps, Document>(Docu
                     dH && layoutDoc.autoHeight && (layoutDoc.autoHeight = false);
                 }
             }
-            // let newWidth = Math.max(Math.abs(oldPoint1!.clientX - oldPoint2!.clientX), Math.abs(pt1.clientX - pt2.clientX))
-            // this.props.Document.width = newWidth;
             e.stopPropagation();
             e.preventDefault();
         }
@@ -374,19 +372,7 @@ export class DocumentView extends DocComponent<DocumentViewProps, Document>(Docu
             }
             return;
         }
-        // if (!InteractionUtils.IsType(e, InteractionUtils.MOUSETYPE)) {
-        //     e.stopPropagation();
-        //     return;
-        // }
         if (!e.nativeEvent.cancelBubble || this.Document.onClick || this.Document.onDragStart) {
-            // if ((e.nativeEvent.cancelBubble && (e.button === 0 || InteractionUtils.IsType(e, InteractionUtils.TOUCHTYPE)))
-            //     // return if we're inking, and not selecting a button document
-            //     || (InkingControl.Instance.selectedTool !== InkTool.None && !this.Document.onClick)
-            //     // return if using pen or eraser
-            //     || InteractionUtils.IsType(e, InteractionUtils.PENTYPE) || InteractionUtils.IsType(e, InteractionUtils.ERASERTYPE)) {
-            //     return;
-            // }
-
             this._downX = e.clientX;
             this._downY = e.clientY;
             this._hitTemplateDrag = false;
@@ -408,6 +394,7 @@ export class DocumentView extends DocComponent<DocumentViewProps, Document>(Docu
 
     onPointerMove = (e: PointerEvent): void => {
         if ((e as any).formattedHandled) { e.stopPropagation(); return; }
+        if ((InteractionUtils.IsType(e, InteractionUtils.PENTYPE) || InkingControl.Instance.selectedTool === InkTool.Highlighter || InkingControl.Instance.selectedTool === InkTool.Pen)) return;
         if (e.cancelBubble && this.active) {
             document.removeEventListener("pointermove", this.onPointerMove); // stop listening to pointerMove if something else has stopPropagated it (e.g., the MarqueeView)
         }
diff --git a/src/mobile/ImageUpload.tsx b/src/mobile/ImageUpload.tsx
index ec698b151..0b0280519 100644
--- a/src/mobile/ImageUpload.tsx
+++ b/src/mobile/ImageUpload.tsx
@@ -11,6 +11,8 @@ import { List } from '../new_fields/List';
 import { observer } from 'mobx-react';
 import { observable } from 'mobx';
 import { Utils } from '../Utils';
+import MobileInterface from './MobileInterface';
+import { CurrentUserUtils } from '../server/authentication/models/current_user_utils';
 
 
 
@@ -104,10 +106,25 @@ class Uploader extends React.Component {
 }
 
 
-DocServer.init(window.location.protocol, window.location.hostname, 4321, "image upload");
-
-ReactDOM.render((
-    <Uploader />
-),
-    document.getElementById('root')
-);
\ No newline at end of file
+// DocServer.init(window.location.protocol, window.location.hostname, 4321, "image upload");
+(async () => {
+    const info = await CurrentUserUtils.loadCurrentUser();
+    DocServer.init(window.location.protocol, window.location.hostname, 4321, info.email + "mobile");
+    await Docs.Prototypes.initialize();
+    if (info.id !== "__guest__") {
+        // a guest will not have an id registered
+        await CurrentUserUtils.loadUserDocument(info);
+    }
+    document.getElementById('root')!.addEventListener('wheel', event => {
+        if (event.ctrlKey) {
+            event.preventDefault();
+        }
+    }, true);
+    ReactDOM.render((
+        // <Uploader />
+        <MobileInterface />
+    ),
+        document.getElementById('root')
+    );
+}
+)();
\ No newline at end of file
diff --git a/src/mobile/MobileInterface.tsx b/src/mobile/MobileInterface.tsx
new file mode 100644
index 000000000..ea2fc917f
--- /dev/null
+++ b/src/mobile/MobileInterface.tsx
@@ -0,0 +1,69 @@
+import React = require('react');
+import { observer } from 'mobx-react';
+import { computed, action } from 'mobx';
+import { CurrentUserUtils } from '../server/authentication/models/current_user_utils';
+import { FieldValue, Cast } from '../new_fields/Types';
+import { Doc } from '../new_fields/Doc';
+import { Docs } from '../client/documents/Documents';
+import { CollectionView } from '../client/views/collections/CollectionView';
+import { DocumentView } from '../client/views/nodes/DocumentView';
+import { emptyPath, emptyFunction, returnFalse, returnOne, returnEmptyString, returnTrue } from '../Utils';
+import { Transform } from '../client/util/Transform';
+import { library } from '@fortawesome/fontawesome-svg-core';
+import { faPenNib, faHighlighter, faEraser, faMousePointer } from '@fortawesome/free-solid-svg-icons';
+
+@observer
+export default class MobileInterface extends React.Component {
+    @computed private get userDoc() { return CurrentUserUtils.UserDocument; }
+    @computed private get mainContainer() { return this.userDoc ? FieldValue(Cast(this.userDoc.activeMobile, Doc)) : CurrentUserUtils.GuestMobile; }
+
+    @action
+    componentDidMount = () => {
+        library.add(...[faPenNib, faHighlighter, faEraser, faMousePointer]);
+
+        if (this.userDoc && !this.mainContainer) {
+            const doc = CurrentUserUtils.setupMobileDoc(this.userDoc);
+            this.userDoc.activeMobile = doc;
+        }
+    }
+
+    @computed
+    get mainContent() {
+        if (this.mainContainer) {
+            return <DocumentView
+                Document={this.mainContainer}
+                DataDoc={undefined}
+                LibraryPath={emptyPath}
+                addDocument={undefined}
+                addDocTab={returnFalse}
+                pinToPres={emptyFunction}
+                removeDocument={undefined}
+                ruleProvider={undefined}
+                onClick={undefined}
+                ScreenToLocalTransform={Transform.Identity}
+                ContentScaling={returnOne}
+                PanelWidth={() => window.screen.width}
+                PanelHeight={() => window.screen.height}
+                renderDepth={0}
+                focus={emptyFunction}
+                backgroundColor={returnEmptyString}
+                parentActive={returnTrue}
+                whenActiveChanged={emptyFunction}
+                bringToFront={emptyFunction}
+                ContainingCollectionView={undefined}
+                ContainingCollectionDoc={undefined}
+                zoomToScale={emptyFunction}
+                getScale={returnOne}>
+            </DocumentView>;
+        }
+        return "hello";
+    }
+
+    render() {
+        return (
+            <div className="mobile-container">
+                {this.mainContent}
+            </div>
+        );
+    }
+}
\ No newline at end of file
diff --git a/src/new_fields/InkField.ts b/src/new_fields/InkField.ts
index 83d631958..20dbf4b17 100644
--- a/src/new_fields/InkField.ts
+++ b/src/new_fields/InkField.ts
@@ -9,7 +9,8 @@ export enum InkTool {
     Pen,
     Highlighter,
     Eraser,
-    Scrubber
+    Scrubber,
+    Stamp
 }
 
 export interface PointData {
diff --git a/src/pen-gestures/GestureUtils.ts b/src/pen-gestures/GestureUtils.ts
index 4edcfa623..062604458 100644
--- a/src/pen-gestures/GestureUtils.ts
+++ b/src/pen-gestures/GestureUtils.ts
@@ -37,7 +37,8 @@ export namespace GestureUtils {
     export enum Gestures {
         Box = "box",
         Line = "line",
-        Stroke = "stroke"
+        Stroke = "stroke",
+        Scribble = "scribble"
     }
 
     export const GestureRecognizer = new NDollarRecognizer(false);
diff --git a/src/server/authentication/models/current_user_utils.ts b/src/server/authentication/models/current_user_utils.ts
index 220c37e2b..71dd34e68 100644
--- a/src/server/authentication/models/current_user_utils.ts
+++ b/src/server/authentication/models/current_user_utils.ts
@@ -28,6 +28,7 @@ export class CurrentUserUtils {
 
     @observable public static GuestTarget: Doc | undefined;
     @observable public static GuestWorkspace: Doc | undefined;
+    @observable public static GuestMobile: Doc | undefined;
 
     // a default set of note types .. not being used yet...
     static setupNoteTypes(doc: Doc) {
@@ -55,8 +56,10 @@ export class CurrentUserUtils {
             { title: "clickable button", icon: "bolt", ignoreClick: true, drag: 'Docs.Create.ButtonDocument({ width: 150, height: 50, title: "Button" })' },
             { title: "presentation", icon: "tv", ignoreClick: true, drag: 'Doc.UserDoc().curPresentation = Docs.Create.PresDocument(new List<Doc>(), { width: 200, height: 500, title: "a presentation trail" })' },
             { title: "import folder", icon: "cloud-upload-alt", ignoreClick: true, drag: 'Docs.Create.DirectoryImportDocument({ title: "Directory Import", width: 400, height: 400 })' },
+            { title: "mobile view", icon: "phone", ignoreClick: true, drag: 'Doc.UserDoc().activeMobile' },
             { title: "use pen", icon: "pen-nib", click: 'activatePen(this.activePen.pen = sameDocs(this.activePen.pen, this) ? undefined : this,2, this.backgroundColor)', backgroundColor: "blue", ischecked: `sameDocs(this.activePen.pen,  this)`, activePen: doc },
             { title: "use highlighter", icon: "highlighter", click: 'activateBrush(this.activePen.pen = sameDocs(this.activePen.pen, this) ? undefined : this,20,this.backgroundColor)', backgroundColor: "yellow", ischecked: `sameDocs(this.activePen.pen, this)`, activePen: doc },
+            { title: "use stamp", icon: "stamp", click: 'activateStamp(this.activePen.pen = sameDocs(this.activePen.pen, this) ? undefined : this)', backgroundColor: "orange", ischecked: `sameDocs(this.activePen.pen, this)`, activePen: doc },
             { title: "use eraser", icon: "eraser", click: 'activateEraser(this.activePen.pen = sameDocs(this.activePen.pen, this) ? undefined : this);', ischecked: `sameDocs(this.activePen.pen, this)`, backgroundColor: "pink", activePen: doc },
             { title: "use scrubber", icon: "eraser", click: 'activateScrubber(this.activePen.pen = sameDocs(this.activePen.pen, this) ? undefined : this);', ischecked: `sameDocs(this.activePen.pen, this)`, backgroundColor: "green", activePen: doc },
             { title: "use drag", icon: "mouse-pointer", click: 'deactivateInk();this.activePen.pen = this;', ischecked: `sameDocs(this.activePen.pen, this)`, backgroundColor: "white", activePen: doc },
@@ -90,6 +93,29 @@ export class CurrentUserUtils {
         }
     }
 
+    static setupMobileButtons(doc: Doc, buttons?: string[]) {
+        const docProtoData: { title: string, icon: string, drag?: string, ignoreClick?: boolean, click?: string, ischecked?: string, activePen?: Doc, backgroundColor?: string, dragFactory?: Doc }[] = [
+            { title: "record", icon: "microphone", ignoreClick: true, click: "FILL" },
+            { title: "use pen", icon: "pen-nib", click: 'activatePen(this.activePen.pen = sameDocs(this.activePen.pen, this) ? undefined : this,2, this.backgroundColor)', backgroundColor: "blue", ischecked: `sameDocs(this.activePen.pen,  this)`, activePen: doc },
+            { title: "use highlighter", icon: "highlighter", click: 'activateBrush(this.activePen.pen = sameDocs(this.activePen.pen, this) ? undefined : this,20,this.backgroundColor)', backgroundColor: "yellow", ischecked: `sameDocs(this.activePen.pen, this)`, activePen: doc },
+            { title: "use eraser", icon: "eraser", click: 'activateEraser(this.activePen.pen = sameDocs(this.activePen.pen, this) ? undefined : this);', ischecked: `sameDocs(this.activePen.pen, this)`, backgroundColor: "pink", activePen: doc },
+            { title: "use scrubber", icon: "eraser", click: 'activateScrubber(this.activePen.pen = sameDocs(this.activePen.pen, this) ? undefined : this);', ischecked: `sameDocs(this.activePen.pen, this)`, backgroundColor: "green", activePen: doc },
+            { title: "use drag", icon: "mouse-pointer", click: 'deactivateInk();this.activePen.pen = this;', ischecked: `sameDocs(this.activePen.pen, this)`, backgroundColor: "white", activePen: doc },
+        ];
+        return docProtoData.filter(d => !buttons || !buttons.includes(d.title)).map(data => Docs.Create.FontIconDocument({
+            nativeWidth: 100, nativeHeight: 100, width: 100, height: 100, dropAction: data.click ? "copy" : undefined, title: data.title, icon: data.icon, ignoreClick: data.ignoreClick,
+            onDragStart: data.drag ? ScriptField.MakeFunction(data.drag) : undefined, onClick: data.click ? ScriptField.MakeScript(data.click) : undefined,
+            ischecked: data.ischecked ? ComputedField.MakeFunction(data.ischecked) : undefined, activePen: data.activePen,
+            backgroundColor: data.backgroundColor, removeDropProperties: new List<string>(["dropAction"]), dragFactory: data.dragFactory,
+        }));
+    }
+
+    static setupMobileDoc(userDoc: Doc) {
+        return userDoc.activeMoble ?? Docs.Create.MasonryDocument(CurrentUserUtils.setupMobileButtons(userDoc), {
+            columnWidth: 100, ignoreClick: true, lockedPosition: true, chromeStatus: "disabled", title: "buttons", autoHeight: true, yMargin: 5
+        });
+    }
+
     // setup the Creator button which will display the creator panel.  This panel will include the drag creators and the color picker.  when clicked, this panel will be displayed in the target container (ie, sidebarContainer)  
     static setupToolsPanel(sidebarContainer: Doc, doc: Doc) {
         // setup a masonry view of all he creators
-- 
cgit v1.2.3-70-g09d2


From de0c00ff0bb1f58a0736da24acc984f5a090d009 Mon Sep 17 00:00:00 2001
From: Stanley Yip <stanley_yip@brown.edu>
Date: Tue, 14 Jan 2020 18:35:28 -0500
Subject: ok we now use custom events for touch. im struggling with figuring
 out how to ignore a hand lol

---
 src/client/util/InteractionUtils.ts                |  50 ++++++-
 src/client/views/CollectionLinearView.tsx          |   4 +-
 src/client/views/GestureOverlay.tsx                | 160 +++++++++++++++++++--
 src/client/views/Touchable.tsx                     | 114 ++++++++++-----
 .../views/collections/CollectionStackingView.tsx   |   6 +-
 src/client/views/collections/CollectionSubView.tsx |   8 +-
 .../collectionFreeForm/CollectionFreeFormView.tsx  |  80 ++++++-----
 src/client/views/nodes/DocumentView.tsx            |  32 +++--
 src/pen-gestures/GestureUtils.ts                   |   2 +-
 9 files changed, 344 insertions(+), 112 deletions(-)

(limited to 'src/pen-gestures/GestureUtils.ts')

diff --git a/src/client/util/InteractionUtils.ts b/src/client/util/InteractionUtils.ts
index 76b43da3c..63ee2fb92 100644
--- a/src/client/util/InteractionUtils.ts
+++ b/src/client/util/InteractionUtils.ts
@@ -8,13 +8,53 @@ export namespace InteractionUtils {
     const REACT_POINTER_PEN_BUTTON = 0;
     const ERASER_BUTTON = 5;
 
+    export class MultiTouchEvent<T extends React.TouchEvent | TouchEvent> {
+        constructor(
+            readonly fingers: number,
+            readonly points: T extends React.TouchEvent ? React.TouchList : TouchList,
+            readonly touchEvent: T extends React.TouchEvent ? React.TouchEvent : TouchEvent
+        ) { }
+    }
+
+    export interface MultiTouchEventDisposer { (): void; }
+
+    export function MakeMultiTouchTarget(
+        element: HTMLElement,
+        startFunc: (e: Event, me: MultiTouchEvent<React.TouchEvent>) => void,
+    ): MultiTouchEventDisposer {
+        const onMultiTouchStartHandler = (e: Event) => startFunc(e, (e as CustomEvent<MultiTouchEvent<React.TouchEvent>>).detail);
+        // const onMultiTouchMoveHandler = moveFunc ? (e: Event) => moveFunc(e, (e as CustomEvent<MultiTouchEvent<TouchEvent>>).detail) : undefined;
+        // const onMultiTouchEndHandler = endFunc ? (e: Event) => endFunc(e, (e as CustomEvent<MultiTouchEvent<TouchEvent>>).detail) : undefined;
+        element.addEventListener("dashOnTouchStart", onMultiTouchStartHandler);
+        // if (onMultiTouchMoveHandler) {
+        //     element.addEventListener("dashOnTouchMove", onMultiTouchMoveHandler);
+        // }
+        // if (onMultiTouchEndHandler) {
+        //     element.addEventListener("dashOnTouchEnd", onMultiTouchEndHandler);
+        // }
+        return () => {
+            element.removeEventListener("dashOnTouchStart", onMultiTouchStartHandler);
+            // if (onMultiTouchMoveHandler) {
+            //     element.removeEventListener("dashOnTouchMove", onMultiTouchMoveHandler);
+            // }
+            // if (onMultiTouchEndHandler) {
+            //     element.removeEventListener("dashOnTouchend", onMultiTouchEndHandler);
+            // }
+        };
+    }
+
     export function GetMyTargetTouches(e: TouchEvent | React.TouchEvent, prevPoints: Map<number, React.Touch>, ignorePen: boolean): React.Touch[] {
         const myTouches = new Array<React.Touch>();
-        for (let i = 0; i < e.targetTouches.length; i++) {
-            const pt: any = e.targetTouches.item(i);
-            if (pt && prevPoints.has(pt.identifier)) {
-                if (ignorePen || (pt.radiusX > 1 && pt.radiusY > 1)) {
-                    myTouches.push(pt);
+        for (let i = 0; i < e.touches.length; i++) {
+            const pt: any = e.touches.item(i);
+            if (ignorePen || (pt.radiusX > 1 && pt.radiusY > 1)) {
+                for (let j = 0; j < e.targetTouches.length; j++) {
+                    const tPt = e.targetTouches.item(j);
+                    if (tPt?.screenX === pt?.screenX && tPt?.screenY === pt?.screenY) {
+                        if (pt && prevPoints.has(pt.identifier)) {
+                            myTouches.push(pt);
+                        }
+                    }
                 }
             }
         }
diff --git a/src/client/views/CollectionLinearView.tsx b/src/client/views/CollectionLinearView.tsx
index 5d6a58656..38f2decdd 100644
--- a/src/client/views/CollectionLinearView.tsx
+++ b/src/client/views/CollectionLinearView.tsx
@@ -36,7 +36,7 @@ export class CollectionLinearView extends CollectionSubView(LinearDocument) {
             { fireImmediately: true }
         );
     }
-    protected createDropAndGestureTarget = (ele: HTMLDivElement) => { //used for stacking and masonry view
+    protected createDashEventsTarget = (ele: HTMLDivElement) => { //used for stacking and masonry view
         this._dropDisposer && this._dropDisposer();
         if (ele) {
             this._dropDisposer = DragManager.MakeDropTarget(ele, this.drop.bind(this));
@@ -55,7 +55,7 @@ export class CollectionLinearView extends CollectionSubView(LinearDocument) {
     render() {
         const guid = Utils.GenerateGuid();
         return <div className="collectionLinearView-outer">
-            <div className="collectionLinearView" ref={this.createDropAndGestureTarget} >
+            <div className="collectionLinearView" ref={this.createDashEventsTarget} >
                 <input id={`${guid}`} type="checkbox" checked={BoolCast(this.props.Document.isExpanded)} ref={this.addMenuToggle}
                     onChange={action((e: any) => this.props.Document.isExpanded = this.addMenuToggle.current!.checked)} />
                 <label htmlFor={`${guid}`} style={{ marginTop: "auto", marginBottom: "auto", background: StrCast(this.props.Document.backgroundColor, "black") === StrCast(this.props.Document.color, "white") ? "black" : StrCast(this.props.Document.backgroundColor, "black") }} title="Close Menu"><p>+</p></label>
diff --git a/src/client/views/GestureOverlay.tsx b/src/client/views/GestureOverlay.tsx
index 830c06f1f..a5fba362c 100644
--- a/src/client/views/GestureOverlay.tsx
+++ b/src/client/views/GestureOverlay.tsx
@@ -1,7 +1,7 @@
 import React = require("react");
 import { Touchable } from "./Touchable";
 import { observer } from "mobx-react";
-import "./GestureOverlay.scss"
+import "./GestureOverlay.scss";
 import { computed, observable, action, runInAction } from "mobx";
 import { CreatePolyline } from "./InkingStroke";
 import { GestureUtils } from "../../pen-gestures/GestureUtils";
@@ -16,6 +16,7 @@ import { Scripting } from "../util/Scripting";
 import { FieldValue, Cast } from "../../new_fields/Types";
 import { CurrentUserUtils } from "../../server/authentication/models/current_user_utils";
 import Palette from "./Palette";
+import { Utils } from "../../Utils";
 
 @observer
 export default class GestureOverlay extends Touchable {
@@ -28,6 +29,9 @@ export default class GestureOverlay extends Touchable {
 
     private _d1: Doc | undefined;
     private thumbIdentifier?: number;
+    private _hands: (React.Touch[])[] = [];
+
+    protected multiTouchDisposer?: InteractionUtils.MultiTouchEventDisposer;
 
     constructor(props: Readonly<{}>) {
         super(props);
@@ -35,11 +39,143 @@ export default class GestureOverlay extends Touchable {
         GestureOverlay.Instance = this;
     }
 
-    @action
+    getNewTouches(e: React.TouchEvent | TouchEvent) {
+        const ntt: (React.Touch | Touch)[] = [];
+        const nct: (React.Touch | Touch)[] = [];
+        const nt: (React.Touch | Touch)[] = [];
+        this._hands.forEach((hand) => {
+            for (let i = 0; i < e.targetTouches.length; i++) {
+                const pt = e.targetTouches.item(i);
+                if (pt && !hand.some((finger) => finger.screenX === pt?.screenX && finger.screenY === pt.screenY)) {
+                    ntt.push(pt);
+                }
+            }
+
+            for (let i = 0; i < e.changedTouches.length; i++) {
+                const pt = e.changedTouches.item(i);
+                if (pt && !hand.some((finger: React.Touch) => finger.screenX === pt?.screenX && finger.screenY === pt.screenY)) {
+                    nct.push(pt);
+                }
+            }
+
+            for (let i = 0; i < e.changedTouches.length; i++) {
+                const pt = e.touches.item(i);
+                if (pt && !hand.some((finger: React.Touch) => finger.screenX === pt?.screenX && finger.screenY === pt.screenY)) {
+                    nt.push(pt);
+                }
+            }
+        });
+        return { ntt, nct, nt };
+    }
+
+    onReactTouchStart = (te: React.TouchEvent) => {
+        const actualPts: React.Touch[] = [];
+        for (let i = 0; i < te.touches.length; i++) {
+            const pt: any = te.touches.item(i);
+            actualPts.push(pt);
+            // pen is also a touch, but with a radius of 0.5 (at least with the surface pens)
+            // and this seems to be the only way of differentiating pen and touch on touch events
+            if (pt.radiusX > 1 && pt.radiusY > 1) {
+                // if (typeof pt.identifier !== "string") {
+                //     pt.identifier = Utils.GenerateGuid();
+                // }
+                this.prevPoints.set(pt.identifier, pt);
+            }
+        }
+
+        const ptsToDelete: number[] = [];
+        this.prevPoints.forEach(pt => {
+            if (!actualPts.includes(pt)) {
+                ptsToDelete.push(pt.identifier);
+            }
+        });
+
+        ptsToDelete.forEach(pt => this.prevPoints.delete(pt));
+
+        if (this.prevPoints.size && this.prevPoints.size < 5) {
+            const nts: any = this.getNewTouches(te);
+            const target = document.elementFromPoint(te.changedTouches.item(0).clientX, te.changedTouches.item(0).clientY);
+            target?.dispatchEvent(
+                new CustomEvent<InteractionUtils.MultiTouchEvent<React.TouchEvent>>("dashOnTouchStart",
+                    {
+                        bubbles: true,
+                        detail: {
+                            fingers: this.prevPoints.size,
+                            points: te.targetTouches,
+                            touchEvent: new React.TouchEvent("start", {
+                                targetTouches: nts.ntt,
+                                touches: nts.nt,
+                                changedTouches: nts.nct
+                            });
+                        }
+                    }
+                )
+            );
+            document.removeEventListener("touchmove", this.onReactTouchMove);
+            document.removeEventListener("touchend", this.onReactTouchEnd);
+            document.addEventListener("touchmove", this.onReactTouchMove);
+            document.addEventListener("touchend", this.onReactTouchEnd);
+        }
+        else {
+            this.handleHandDown(te);
+            document.removeEventListener("touchmove", this.onReactTouchMove);
+            document.removeEventListener("touchend", this.onReactTouchEnd);
+        }
+    }
+
+    onReactTouchMove = (e: TouchEvent) => {
+        const nts: any = this.getNewTouches(e);
+        document.dispatchEvent(
+            new CustomEvent<InteractionUtils.MultiTouchEvent<TouchEvent>>("dashOnTouchMove",
+                {
+                    bubbles: true,
+                    detail: {
+                        fingers: this.prevPoints.size,
+                        points: e.touches,
+                        touchEvent: new TouchEvent("move", {
+                            targetTouches: nts.ntt,
+                            changedTouches: nts.nct,
+                            touches: nts.nt,
+                        })
+                    }
+                })
+        );
+    }
+
+    onReactTouchEnd = (e: TouchEvent) => {
+        document.dispatchEvent(
+            new CustomEvent<InteractionUtils.MultiTouchEvent<TouchEvent>>("dashOnTouchEnd",
+                {
+                    bubbles: true,
+                    detail: {
+                        fingers: this.prevPoints.size,
+                        points: e.touches,
+                        touchEvent: e
+                    }
+                })
+        );
+        for (let i = 0; i < e.changedTouches.length; i++) {
+            const pt = e.changedTouches.item(i);
+            if (pt) {
+                if (this.prevPoints.has(pt.identifier)) {
+                    this.prevPoints.delete(pt.identifier);
+                }
+            }
+        }
+
+        if (this.prevPoints.size === 0) {
+            document.removeEventListener("touchmove", this.onReactTouchMove);
+            document.removeEventListener("touchend", this.onReactTouchEnd);
+        }
+        e.stopPropagation();
+    }
+
     handleHandDown = (e: React.TouchEvent) => {
         const fingers = InteractionUtils.GetMyTargetTouches(e, this.prevPoints, true);
         const thumb = fingers.reduce((a, v) => a.clientY > v.clientY ? a : v, fingers[0]);
         this.thumbIdentifier = thumb?.identifier;
+        fingers.forEach((f) => this.prevPoints.delete(f.identifier));
+        this._hands.push(fingers);
         const others = fingers.filter(f => f !== thumb);
         const minX = Math.min(...others.map(f => f.clientX));
         const minY = Math.min(...others.map(f => f.clientY));
@@ -48,10 +184,12 @@ export default class GestureOverlay extends Touchable {
 
         const thumbDoc = FieldValue(Cast(CurrentUserUtils.setupThumbDoc(CurrentUserUtils.UserDocument), Doc));
         if (thumbDoc) {
-            this._palette = <Palette x={minX} y={minY} thumb={[thumb.clientX, thumb.clientY]} thumbDoc={thumbDoc} />;
+            runInAction(() => {
+                this._palette = <Palette x={minX} y={minY} thumb={[thumb.clientX, thumb.clientY]} thumbDoc={thumbDoc} />;
+            });
         }
 
-        document.removeEventListener("touchmove", this.onTouch);
+        this.removeMoveListeners();
         document.removeEventListener("touchmove", this.handleHandMove);
         document.addEventListener("touchmove", this.handleHandMove);
         document.removeEventListener("touchend", this.handleHandUp);
@@ -69,11 +207,9 @@ export default class GestureOverlay extends Touchable {
 
     @action
     handleHandUp = (e: TouchEvent) => {
-        this.onTouchEnd(e);
-        if (this.prevPoints.size < 3) {
-            this._palette = undefined;
-            document.removeEventListener("touchend", this.handleHandUp);
-        }
+        // this.onTouchEnd(e);
+        this._palette = undefined;
+        document.removeEventListener("touchend", this.handleHandUp);
     }
 
     @action
@@ -115,7 +251,7 @@ export default class GestureOverlay extends Touchable {
                 DocUtils.MakeLink({ doc: this._d1 }, { doc: doc });
                 actionPerformed = true;
             }
-        }
+        };
         const ge = new CustomEvent<GestureUtils.GestureEvent>("dashOnGesture",
             {
                 bubbles: true,
@@ -181,7 +317,7 @@ export default class GestureOverlay extends Touchable {
                             }
                         }
                     )
-                )
+                );
                 this._points = [];
             }
         }
@@ -215,7 +351,7 @@ export default class GestureOverlay extends Touchable {
 
     render() {
         return (
-            <div className="gestureOverlay-cont" onPointerDown={this.onPointerDown} onTouchStart={this.onTouchStart}>
+            <div className="gestureOverlay-cont" onPointerDown={this.onPointerDown} onTouchStart={this.onReactTouchStart}>
                 {this.props.children}
                 {this._palette}
                 {this.currentStroke}
diff --git a/src/client/views/Touchable.tsx b/src/client/views/Touchable.tsx
index 2a7599fbf..a71015b05 100644
--- a/src/client/views/Touchable.tsx
+++ b/src/client/views/Touchable.tsx
@@ -8,8 +8,11 @@ const HOLD_DURATION = 1000;
 
 export abstract class Touchable<T = {}> extends React.Component<T> {
     //private holdTimer: NodeJS.Timeout | undefined;
-    holdTimer: NodeJS.Timeout | undefined;
+    private holdTimer: NodeJS.Timeout | undefined;
+    private moveDisposer?: InteractionUtils.MultiTouchEventDisposer;
+    private endDisposer?: InteractionUtils.MultiTouchEventDisposer;
 
+    protected abstract multiTouchDisposer?: InteractionUtils.MultiTouchEventDisposer;
     protected _touchDrag: boolean = false;
     protected prevPoints: Map<number, React.Touch> = new Map<number, React.Touch>();
 
@@ -22,16 +25,27 @@ export abstract class Touchable<T = {}> extends React.Component<T> {
      * When a touch even starts, we keep track of each touch that is associated with that event
      */
     @action
-    protected onTouchStart = (e: React.TouchEvent): void => {
+    protected onTouchStart = (e: Event, me: InteractionUtils.MultiTouchEvent<React.TouchEvent>): void => {
         const actualPts: React.Touch[] = [];
-        for (let i = 0; i < e.targetTouches.length; i++) {
-            const pt: any = e.targetTouches.item(i);
+        const te = me.touchEvent;
+        // loop through all touches on screen
+        for (let i = 0; i < te.touches.length; i++) {
+            const pt: any = te.touches.item(i);
             actualPts.push(pt);
-            // pen is also a touch, but with a radius of 0.5 (at least with the surface pens)
-            // and this seems to be the only way of differentiating pen and touch on touch events
-            if (pt.radiusX > 1 && pt.radiusY > 1) {
+            if (this.prevPoints.has(pt.identifier)) {
                 this.prevPoints.set(pt.identifier, pt);
             }
+            // only add the ones that are targeted on "this" element, but with the identifier that the screen touch gives
+            for (let j = 0; j < te.changedTouches.length; j++) {
+                const tPt = te.changedTouches.item(j);
+                if (pt.clientX === tPt.clientX && pt.clientY === tPt.clientY) {
+                    // pen is also a touch, but with a radius of 0.5 (at least with the surface pens)
+                    // and this seems to be the only way of differentiating pen and touch on touch events
+                    if (pt.radiusX > 1 && pt.radiusY > 1) {
+                        this.prevPoints.set(pt.identifier, pt);
+                    }
+                }
+            }
         }
 
         const ptsToDelete: number[] = [];
@@ -41,26 +55,30 @@ export abstract class Touchable<T = {}> extends React.Component<T> {
             }
         });
 
+        // console.log(ptsToDelete.length);
         ptsToDelete.forEach(pt => this.prevPoints.delete(pt));
 
+        // console.log(this.prevPoints.size);
         if (this.prevPoints.size) {
             switch (this.prevPoints.size) {
                 case 1:
-                    this.handle1PointerDown(e);
-                    e.persist();
+                    this.handle1PointerDown(te);
+                    te.persist();
                     // if (this.holdTimer) {
                     //     clearTimeout(this.holdTimer)
                     //     this.holdTimer = undefined;
                     // }
-                    this.holdTimer = setTimeout(() => this.handle1PointerHoldStart(e), HOLD_DURATION);
+                    this.holdTimer = setTimeout(() => this.handle1PointerHoldStart(te), HOLD_DURATION);
+                    // e.stopPropagation();
                     // console.log(this.holdTimer);
                     break;
                 case 2:
-                    this.handle2PointersDown(e);
-                    break;
-                case 5:
-                    this.handleHandDown(e);
+                    this.handle2PointersDown(te);
+                    // e.stopPropagation();
                     break;
+                // case 5:
+                //     this.handleHandDown(te);
+                //     break;
             }
         }
     }
@@ -69,28 +87,30 @@ export abstract class Touchable<T = {}> extends React.Component<T> {
     * Handle touch move event
     */
     @action
-    protected onTouch = (e: TouchEvent): void => {
-        const myTouches = InteractionUtils.GetMyTargetTouches(e, this.prevPoints, true);
+    protected onTouch = (e: Event, me: InteractionUtils.MultiTouchEvent<TouchEvent>): void => {
+        const te = me.touchEvent;
+        const myTouches = InteractionUtils.GetMyTargetTouches(te, this.prevPoints, true);
 
         // if we're not actually moving a lot, don't consider it as dragging yet
         if (!InteractionUtils.IsDragging(this.prevPoints, myTouches, 5) && !this._touchDrag) return;
         this._touchDrag = true;
         if (this.holdTimer) {
-            console.log("CLEAR")
+            console.log("CLEAR");
             clearTimeout(this.holdTimer);
             // this.holdTimer = undefined;
         }
+        // console.log(myTouches.length);
         switch (myTouches.length) {
             case 1:
-                this.handle1PointerMove(e);
+                this.handle1PointerMove(te);
                 break;
             case 2:
-                this.handle2PointersMove(e);
+                this.handle2PointersMove(te);
                 break;
         }
 
-        for (let i = 0; i < e.targetTouches.length; i++) {
-            const pt = e.targetTouches.item(i);
+        for (let i = 0; i < te.touches.length; i++) {
+            const pt = te.touches.item(i);
             if (pt) {
                 if (this.prevPoints.has(pt.identifier)) {
                     this.prevPoints.set(pt.identifier, pt);
@@ -100,11 +120,12 @@ export abstract class Touchable<T = {}> extends React.Component<T> {
     }
 
     @action
-    protected onTouchEnd = (e: TouchEvent): void => {
+    protected onTouchEnd = (e: Event, me: InteractionUtils.MultiTouchEvent<TouchEvent>): void => {
         // console.log(InteractionUtils.GetMyTargetTouches(e, this.prevPoints).length + " up");
         // remove all the touches associated with the event
-        for (let i = 0; i < e.changedTouches.length; i++) {
-            const pt = e.changedTouches.item(i);
+        const te = me.touchEvent;
+        for (let i = 0; i < te.changedTouches.length; i++) {
+            const pt = te.changedTouches.item(i);
             if (pt) {
                 if (this.prevPoints.has(pt.identifier)) {
                     this.prevPoints.delete(pt.identifier);
@@ -116,7 +137,7 @@ export abstract class Touchable<T = {}> extends React.Component<T> {
             console.log("clear");
         }
         this._touchDrag = false;
-        e.stopPropagation();
+        te.stopPropagation();
 
 
         // if (e.targetTouches.length === 0) {
@@ -126,11 +147,12 @@ export abstract class Touchable<T = {}> extends React.Component<T> {
         if (this.prevPoints.size === 0) {
             this.cleanUpInteractions();
         }
+        e.stopPropagation();
     }
 
     cleanUpInteractions = (): void => {
-        document.removeEventListener("touchmove", this.onTouch);
-        document.removeEventListener("touchend", this.onTouchEnd);
+        this.removeMoveListeners();
+        this.removeEndListeners();
     }
 
     handle1PointerMove = (e: TouchEvent): any => {
@@ -144,23 +166,43 @@ export abstract class Touchable<T = {}> extends React.Component<T> {
     }
 
     handle1PointerDown = (e: React.TouchEvent): any => {
-        document.removeEventListener("touchmove", this.onTouch);
-        document.addEventListener("touchmove", this.onTouch);
-        document.removeEventListener("touchend", this.onTouchEnd);
-        document.addEventListener("touchend", this.onTouchEnd);
+        this.removeMoveListeners();
+        this.addMoveListeners();
+        this.removeEndListeners();
+        this.addEndListeners();
     }
 
     handle2PointersDown = (e: React.TouchEvent): any => {
-        document.removeEventListener("touchmove", this.onTouch);
-        document.addEventListener("touchmove", this.onTouch);
-        document.removeEventListener("touchend", this.onTouchEnd);
-        document.addEventListener("touchend", this.onTouchEnd);
+        this.removeMoveListeners();
+        this.addMoveListeners();
+        this.removeEndListeners();
+        this.addEndListeners();
     }
 
     handle1PointerHoldStart = (e: React.TouchEvent): any => {
         e.stopPropagation();
         e.preventDefault();
-        document.removeEventListener("touchmove", this.onTouch);
+        this.removeMoveListeners();
+    }
+
+    addMoveListeners = () => {
+        const handler = (e: Event) => this.onTouch(e, (e as CustomEvent<InteractionUtils.MultiTouchEvent<TouchEvent>>).detail);
+        document.addEventListener("dashOnTouchMove", handler);
+        this.moveDisposer = () => document.removeEventListener("dashOnTouchMove", handler);
+    }
+
+    removeMoveListeners = () => {
+        this.moveDisposer && this.moveDisposer();
+    }
+
+    addEndListeners = () => {
+        const handler = (e: Event) => this.onTouchEnd(e, (e as CustomEvent<InteractionUtils.MultiTouchEvent<TouchEvent>>).detail);
+        document.addEventListener("dashOnTouchEnd", handler);
+        this.endDisposer = () => document.removeEventListener("dashOnTouchEnd", handler);
+    }
+
+    removeEndListeners = () => {
+        this.endDisposer && this.endDisposer();
     }
 
     handleHandDown = (e: React.TouchEvent) => {
diff --git a/src/client/views/collections/CollectionStackingView.tsx b/src/client/views/collections/CollectionStackingView.tsx
index 992820fc7..b8423af20 100644
--- a/src/client/views/collections/CollectionStackingView.tsx
+++ b/src/client/views/collections/CollectionStackingView.tsx
@@ -153,7 +153,7 @@ export class CollectionStackingView extends CollectionSubView(doc => doc) {
     }
     createRef = (ele: HTMLDivElement | null) => {
         this._masonryGridRef = ele;
-        this.createDropAndGestureTarget(ele!); //so the whole grid is the drop target?
+        this.createDashEventsTarget(ele!); //so the whole grid is the drop target?
     }
 
     overlays = (doc: Doc) => {
@@ -309,7 +309,7 @@ export class CollectionStackingView extends CollectionSubView(doc => doc) {
             docList={docList}
             parent={this}
             type={type}
-            createDropTarget={this.createDropAndGestureTarget}
+            createDropTarget={this.createDashEventsTarget}
             screenToLocalTransform={this.props.ScreenToLocalTransform}
         />;
     }
@@ -342,7 +342,7 @@ export class CollectionStackingView extends CollectionSubView(doc => doc) {
             docList={docList}
             parent={this}
             type={type}
-            createDropTarget={this.createDropAndGestureTarget}
+            createDropTarget={this.createDashEventsTarget}
             screenToLocalTransform={this.props.ScreenToLocalTransform}
             setDocHeight={this.setDocHeight}
         />;
diff --git a/src/client/views/collections/CollectionSubView.tsx b/src/client/views/collections/CollectionSubView.tsx
index 73dc7edc6..4133956fc 100644
--- a/src/client/views/collections/CollectionSubView.tsx
+++ b/src/client/views/collections/CollectionSubView.tsx
@@ -24,6 +24,7 @@ import { GooglePhotos } from "../../apis/google_docs/GooglePhotosClientUtils";
 import { ImageUtils } from "../../util/Import & Export/ImageUtils";
 import { Networking } from "../../Network";
 import { GestureUtils } from "../../../pen-gestures/GestureUtils";
+import { InteractionUtils } from "../../util/InteractionUtils";
 
 export interface CollectionViewProps extends FieldViewProps {
     addDocument: (document: Doc) => boolean;
@@ -49,17 +50,20 @@ export function CollectionSubView<T>(schemaCtor: (doc: Doc) => T) {
     class CollectionSubView extends DocComponent<SubCollectionViewProps, T>(schemaCtor) {
         private dropDisposer?: DragManager.DragDropDisposer;
         private gestureDisposer?: GestureUtils.GestureEventDisposer;
+        protected multiTouchDisposer?: InteractionUtils.MultiTouchEventDisposer;
         private _childLayoutDisposer?: IReactionDisposer;
-        protected createDropAndGestureTarget = (ele: HTMLDivElement) => { //used for stacking and masonry view
+        protected createDashEventsTarget = (ele: HTMLDivElement) => { //used for stacking and masonry view
             this.dropDisposer && this.dropDisposer();
             this.gestureDisposer && this.gestureDisposer();
+            this.multiTouchDisposer && this.multiTouchDisposer();
             if (ele) {
                 this.dropDisposer = DragManager.MakeDropTarget(ele, this.drop.bind(this));
                 this.gestureDisposer = GestureUtils.MakeGestureTarget(ele, this.onGesture.bind(this));
+                this.multiTouchDisposer = InteractionUtils.MakeMultiTouchTarget(ele, this.onTouchStart.bind(this));
             }
         }
         protected CreateDropTarget(ele: HTMLDivElement) { //used in schema view
-            this.createDropAndGestureTarget(ele);
+            this.createDashEventsTarget(ele);
         }
 
         componentDidMount() {
diff --git a/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx b/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx
index 84945c6e6..3894e9d63 100644
--- a/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx
+++ b/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx
@@ -326,29 +326,30 @@ export class CollectionFreeFormView extends CollectionSubView(PanZoomDocument) {
 
     @action
     handle1PointerDown = (e: React.TouchEvent) => {
-        const pt = e.targetTouches.item(0);
-        if (pt) {
-            this._hitCluster = this.props.Document.useCluster ? this.pickCluster(this.getTransform().transformPoint(pt.clientX, pt.clientY)) !== -1 : false;
-            if (!e.shiftKey && !e.altKey && !e.ctrlKey && this.props.active(true)) {
-                document.removeEventListener("touchmove", this.onTouch);
-                document.addEventListener("touchmove", this.onTouch);
-                document.removeEventListener("touchend", this.onTouchEnd);
-                document.addEventListener("touchend", this.onTouchEnd);
-                // if (InkingControl.Instance.selectedTool === InkTool.Highlighter || InkingControl.Instance.selectedTool === InkTool.Pen) {
-                //     e.stopPropagation();
-                //     e.preventDefault();
-                //     const point = this.getTransform().transformPoint(pt.pageX, pt.pageY);
-                //     this._points.push({ X: point[0], Y: point[1] });
-                // }
-                if (InkingControl.Instance.selectedTool === InkTool.None) {
-                    this._lastX = pt.pageX;
-                    this._lastY = pt.pageY;
-                    e.stopPropagation();
-                    e.preventDefault();
-                }
-                else {
-                    e.stopPropagation();
-                    e.preventDefault();
+        if (!e.nativeEvent.cancelBubble) {
+            // const myTouches = InteractionUtils.GetMyTargetTouches(e, this.prevPoints, true);
+            const pt = e.changedTouches.item(0);
+            if (pt) {
+                this._hitCluster = this.props.Document.useCluster ? this.pickCluster(this.getTransform().transformPoint(pt.clientX, pt.clientY)) !== -1 : false;
+                if (!e.shiftKey && !e.altKey && !e.ctrlKey && this.props.active(true)) {
+                    this.removeMoveListeners();
+                    this.addMoveListeners();
+                    this.removeEndListeners();
+                    this.addEndListeners();
+                    // if (InkingControl.Instance.selectedTool === InkTool.Highlighter || InkingControl.Instance.selectedTool === InkTool.Pen) {
+                    //     e.stopPropagation();
+                    //     e.preventDefault();
+                    //     const point = this.getTransform().transformPoint(pt.pageX, pt.pageY);
+                    //     this._points.push({ X: point[0], Y: point[1] });
+                    // }
+                    if (InkingControl.Instance.selectedTool === InkTool.None) {
+                        this._lastX = pt.pageX;
+                        this._lastY = pt.pageY;
+                        e.preventDefault();
+                    }
+                    else {
+                        e.preventDefault();
+                    }
                 }
             }
         }
@@ -395,8 +396,8 @@ export class CollectionFreeFormView extends CollectionSubView(PanZoomDocument) {
 
         document.removeEventListener("pointermove", this.onPointerMove);
         document.removeEventListener("pointerup", this.onPointerUp);
-        document.removeEventListener("touchmove", this.onTouch);
-        document.removeEventListener("touchend", this.onTouchEnd);
+        this.removeMoveListeners();
+        this.removeEndListeners();
     }
 
     @action
@@ -482,7 +483,7 @@ export class CollectionFreeFormView extends CollectionSubView(PanZoomDocument) {
                     this.pan(pt);
                 }
             }
-            e.stopPropagation();
+            // e.stopPropagation();
             e.preventDefault();
         }
     }
@@ -527,7 +528,7 @@ export class CollectionFreeFormView extends CollectionSubView(PanZoomDocument) {
                     }
                 }
             }
-            e.stopPropagation();
+            // e.stopPropagation();
             e.preventDefault();
         }
     }
@@ -535,18 +536,21 @@ export class CollectionFreeFormView extends CollectionSubView(PanZoomDocument) {
     @action
     handle2PointersDown = (e: React.TouchEvent) => {
         if (!e.nativeEvent.cancelBubble && this.props.active(true)) {
-            const pt1: React.Touch | null = e.targetTouches.item(0);
-            const pt2: React.Touch | null = e.targetTouches.item(1);
-            if (!pt1 || !pt2) return;
+            // const pt1: React.Touch | null = e.targetTouches.item(0);
+            // const pt2: React.Touch | null = e.targetTouches.item(1);
+            // // if (!pt1 || !pt2) return;
+            const myTouches = InteractionUtils.GetMyTargetTouches(e, this.prevPoints, true);
+            const pt1 = myTouches[0];
+            const pt2 = myTouches[1];
 
             const centerX = Math.min(pt1.clientX, pt2.clientX) + Math.abs(pt2.clientX - pt1.clientX) / 2;
             const centerY = Math.min(pt1.clientY, pt2.clientY) + Math.abs(pt2.clientY - pt1.clientY) / 2;
             this._lastX = centerX;
             this._lastY = centerY;
-            document.removeEventListener("touchmove", this.onTouch);
-            document.addEventListener("touchmove", this.onTouch);
-            document.removeEventListener("touchend", this.onTouchEnd);
-            document.addEventListener("touchend", this.onTouchEnd);
+            this.removeMoveListeners();
+            this.addMoveListeners();
+            this.removeEndListeners();
+            this.addEndListeners();
             e.stopPropagation();
         }
     }
@@ -554,8 +558,8 @@ export class CollectionFreeFormView extends CollectionSubView(PanZoomDocument) {
     cleanUpInteractions = () => {
         document.removeEventListener("pointermove", this.onPointerMove);
         document.removeEventListener("pointerup", this.onPointerUp);
-        document.removeEventListener("touchmove", this.onTouch);
-        document.removeEventListener("touchend", this.onTouchEnd);
+        this.removeMoveListeners();
+        this.removeEndListeners();
     }
 
     @action
@@ -989,9 +993,9 @@ export class CollectionFreeFormView extends CollectionSubView(PanZoomDocument) {
         // otherwise, they are stored in fieldKey.  All annotations to this document are stored in the extension document
         if (!this.extensionDoc) return (null);
         // let lodarea = this.Document[WidthSym]() * this.Document[HeightSym]() / this.props.ScreenToLocalTransform().Scale / this.props.ScreenToLocalTransform().Scale;
-        return <div className={"collectionfreeformview-container"} ref={this.createDropAndGestureTarget} onWheel={this.onPointerWheel}//pointerEvents: SelectionManager.GetIsDragging() ? "all" : undefined,
+        return <div className={"collectionfreeformview-container"} ref={this.createDashEventsTarget} onWheel={this.onPointerWheel}//pointerEvents: SelectionManager.GetIsDragging() ? "all" : undefined,
             style={{ pointerEvents: SelectionManager.GetIsDragging() ? "all" : undefined, height: this.isAnnotationOverlay ? (this.props.Document.scrollHeight ? this.Document.scrollHeight : "100%") : this.props.PanelHeight() }}
-            onPointerDown={this.onPointerDown} onPointerMove={this.onCursorMove} onDrop={this.onDrop.bind(this)} onContextMenu={this.onContextMenu} onTouchStart={this.onTouchStart}>
+            onPointerDown={this.onPointerDown} onPointerMove={this.onCursorMove} onDrop={this.onDrop.bind(this)} onContextMenu={this.onContextMenu}>
             {!this.Document.LODdisable && !this.props.active() && !this.props.isAnnotationOverlay && !this.props.annotationsKey && this.props.renderDepth > 0 ? // && this.props.CollectionView && lodarea < NumCast(this.Document.LODarea, 100000) ?
                 this.placeholder : this.marqueeView}
             <CollectionFreeFormOverlayView elements={this.elementFunc} />
diff --git a/src/client/views/nodes/DocumentView.tsx b/src/client/views/nodes/DocumentView.tsx
index 2e0ae09ba..b33bebe7d 100644
--- a/src/client/views/nodes/DocumentView.tsx
+++ b/src/client/views/nodes/DocumentView.tsx
@@ -103,6 +103,8 @@ export class DocumentView extends DocComponent<DocumentViewProps, Document>(Docu
     private _gestureEventDisposer?: GestureUtils.GestureEventDisposer;
     private _titleRef = React.createRef<EditableView>();
 
+    protected multiTouchDisposer?: InteractionUtils.MultiTouchEventDisposer;
+
     public get displayName() { return "DocumentView(" + this.props.Document.title + ")"; } // this makes mobx trace() statements more descriptive
     public get ContentDiv() { return this._mainCont.current; }
     @computed get active() { return SelectionManager.IsSelected(this, true) || this.props.parentActive(true); }
@@ -177,6 +179,7 @@ export class DocumentView extends DocComponent<DocumentViewProps, Document>(Docu
     componentDidMount() {
         this._mainCont.current && (this._dropDisposer = DragManager.MakeDropTarget(this._mainCont.current, this.drop.bind(this)));
         this._mainCont.current && (this._gestureEventDisposer = GestureUtils.MakeGestureTarget(this._mainCont.current, this.onGesture.bind(this)));
+        this._mainCont.current && (this.multiTouchDisposer = InteractionUtils.MakeMultiTouchTarget(this._mainCont.current, this.onTouchStart.bind(this)));
 
         !this.props.dontRegisterView && DocumentManager.Instance.DocumentViews.push(this);
     }
@@ -184,7 +187,11 @@ export class DocumentView extends DocComponent<DocumentViewProps, Document>(Docu
     @action
     componentDidUpdate() {
         this._dropDisposer && this._dropDisposer();
+        this._gestureEventDisposer && this._gestureEventDisposer();
+        this.multiTouchDisposer && this.multiTouchDisposer();
         this._mainCont.current && (this._dropDisposer = DragManager.MakeDropTarget(this._mainCont.current, this.drop.bind(this)));
+        this._mainCont.current && (this._gestureEventDisposer = GestureUtils.MakeGestureTarget(this._mainCont.current, this.onGesture.bind(this)));
+        this._mainCont.current && (this.multiTouchDisposer = InteractionUtils.MakeMultiTouchTarget(this._mainCont.current, this.onTouchStart.bind(this)));
     }
 
     @action
@@ -319,25 +326,24 @@ export class DocumentView extends DocComponent<DocumentViewProps, Document>(Docu
                 }
             }
             if ((this.active || this.Document.onDragStart || this.Document.onClick) && !e.ctrlKey && !this.Document.lockedPosition && !this.Document.inOverlay) e.stopPropagation();
-            document.removeEventListener("touchmove", this.onTouch);
-            document.addEventListener("touchmove", this.onTouch);
-            document.removeEventListener("touchend", this.onTouchEnd);
-            document.addEventListener("touchend", this.onTouchEnd);
-            if ((e.nativeEvent as any).formattedHandled) e.stopPropagation();
+            this.removeMoveListeners();
+            this.addMoveListeners();
+            this.removeEndListeners();
+            this.addEndListeners();
+            e.stopPropagation();
         }
     }
 
     handle1PointerMove = (e: TouchEvent) => {
         if ((e as any).formattedHandled) { e.stopPropagation; return; }
         if (e.cancelBubble && this.active) {
-            document.removeEventListener("touchmove", this.onTouch);
+            this.removeMoveListeners();
         }
         else if (!e.cancelBubble && (SelectionManager.IsSelected(this, true) || this.props.parentActive(true) || this.Document.onDragStart || this.Document.onClick) && !this.Document.lockedPosition && !this.Document.inOverlay) {
             const touch = InteractionUtils.GetMyTargetTouches(e, this.prevPoints, true)[0];
             if (Math.abs(this._downX - touch.clientX) > 3 || Math.abs(this._downY - touch.clientY) > 3) {
                 if (!e.altKey && (!this.topMost || this.Document.onDragStart || this.Document.onClick)) {
-                    document.removeEventListener("touchmove", this.onTouch);
-                    document.removeEventListener("touchend", this.onTouchEnd);
+                    this.cleanUpInteractions();
                     this.startDragging(this._downX, this._downY, this.Document.dropAction ? this.Document.dropAction as any : e.ctrlKey || e.altKey ? "alias" : undefined, this._hitTemplateDrag);
                 }
             }
@@ -352,10 +358,10 @@ export class DocumentView extends DocComponent<DocumentViewProps, Document>(Docu
             e.stopPropagation();
             e.preventDefault();
 
-            document.removeEventListener("touchmove", this.onTouch);
-            document.addEventListener("touchmove", this.onTouch);
-            document.removeEventListener("touchend", this.onTouchEnd);
-            document.addEventListener("touchend", this.onTouchEnd);
+            this.removeMoveListeners();
+            this.addMoveListeners();
+            this.removeEndListeners();
+            this.addEndListeners();
         }
     }
 
@@ -970,7 +976,7 @@ export class DocumentView extends DocComponent<DocumentViewProps, Document>(Docu
                 width: animwidth,
                 height: animheight,
                 opacity: this.Document.opacity
-            }} onTouchStart={this.onTouchStart}>
+            }}>
             {this.innards}
         </div>;
     }
diff --git a/src/pen-gestures/GestureUtils.ts b/src/pen-gestures/GestureUtils.ts
index 062604458..4b5ad6684 100644
--- a/src/pen-gestures/GestureUtils.ts
+++ b/src/pen-gestures/GestureUtils.ts
@@ -31,7 +31,7 @@ export namespace GestureUtils {
         element.addEventListener("dashOnGesture", handler);
         return () => {
             element.removeEventListener("dashOnGesture", handler);
-        }
+        };
     }
 
     export enum Gestures {
-- 
cgit v1.2.3-70-g09d2


From 1007cfb325f2dcddc4365538e4b354d06eb85f2f Mon Sep 17 00:00:00 2001
From: Stanley Yip <stanley_yip@brown.edu>
Date: Tue, 4 Feb 2020 18:12:47 -0500
Subject: ok so the toolglass is working, but it's super slow... i'll fix that
 later lol

---
 src/client/views/GestureOverlay.scss               |  13 +-
 src/client/views/GestureOverlay.tsx                | 151 ++++++++++++---------
 .../collectionFreeForm/CollectionFreeFormView.tsx  |   6 +-
 src/pen-gestures/GestureUtils.ts                   |  11 +-
 .../authentication/models/current_user_utils.ts    |   2 +-
 5 files changed, 109 insertions(+), 74 deletions(-)

(limited to 'src/pen-gestures/GestureUtils.ts')

diff --git a/src/client/views/GestureOverlay.scss b/src/client/views/GestureOverlay.scss
index 60b53c528..f425c438e 100644
--- a/src/client/views/GestureOverlay.scss
+++ b/src/client/views/GestureOverlay.scss
@@ -16,11 +16,13 @@
 .inkToTextDoc-cont {
     position: absolute;
     width: 300px;
-    height: 300px;
     overflow: hidden;
 
     .inkToTextDoc-scroller {
         overflow: visible;
+        position: absolute;
+        width: 100%;
+        left: -24px;
 
         .collectionView {
             overflow: visible;
@@ -30,6 +32,15 @@
             }
         }
     }
+
+    .shadow {
+        width: 100%;
+        height: calc(100% - 25px);
+        position: absolute;
+        top: 25px;
+        background-color: black;
+        opacity: 0.2;
+    }
 }
 
 .filter-cont {
diff --git a/src/client/views/GestureOverlay.tsx b/src/client/views/GestureOverlay.tsx
index 0c2e9e1bc..0fd6f3dba 100644
--- a/src/client/views/GestureOverlay.tsx
+++ b/src/client/views/GestureOverlay.tsx
@@ -2,7 +2,7 @@ import React = require("react");
 import { Touchable } from "./Touchable";
 import { observer } from "mobx-react";
 import "./GestureOverlay.scss";
-import { computed, observable, action, runInAction, IReactionDisposer, reaction, flow } from "mobx";
+import { computed, observable, action, runInAction, IReactionDisposer, reaction, flow, trace } from "mobx";
 import { GestureUtils } from "../../pen-gestures/GestureUtils";
 import { InteractionUtils } from "../util/InteractionUtils";
 import { InkingControl } from "./InkingControl";
@@ -25,6 +25,8 @@ import htmlToImage from "html-to-image";
 import { ScriptField } from "../../new_fields/ScriptField";
 import { listSpec } from "../../new_fields/Schema";
 import { List } from "../../new_fields/List";
+import { CollectionViewType } from "./collections/CollectionView";
+import InkCanvas from "./InkCanvas";
 
 @observer
 export default class GestureOverlay extends Touchable {
@@ -63,6 +65,11 @@ export default class GestureOverlay extends Touchable {
         GestureOverlay.Instance = this;
     }
 
+    componentDidMount = () => {
+        this._thumbDoc = FieldValue(Cast(CurrentUserUtils.setupThumbDoc(CurrentUserUtils.UserDocument), Doc));
+        this._inkToTextDoc = FieldValue(Cast(this._thumbDoc?.inkToTextDoc, Doc));
+    }
+
     getNewTouches(e: React.TouchEvent | TouchEvent) {
         const ntt: (React.Touch | Touch)[] = Array.from(e.targetTouches);
         const nct: (React.Touch | Touch)[] = Array.from(e.changedTouches);
@@ -313,14 +320,23 @@ export default class GestureOverlay extends Touchable {
             this._palette = undefined;
             this.thumbIdentifier = undefined;
             this._thumbDoc = undefined;
-            this._strokes.forEach(s => {
-                this.dispatchGesture(GestureUtils.Gestures.Stroke, s);
-            });
-            this._strokes = [];
-            if (NumCast(this._inkToTextDoc?.selectedIndex) > 0) {
-                const selectedButton = this._possibilities[NumCast(this._inkToTextDoc?.selectedIndex) - 1];
-                Cast(selectedButton?.proto?.onClick, ScriptField)?.script.run({ this: selectedButton.proto }, console.log);
+
+            let scriptWorked = false;
+            if (NumCast(this._inkToTextDoc?.selectedIndex) > -1) {
+                const selectedButton = this._possibilities[NumCast(this._inkToTextDoc?.selectedIndex)];
+                if (Cast(selectedButton?.proto?.onClick, ScriptField)?.script.run({ this: selectedButton }, console.log).success) {
+                    scriptWorked = true;
+                    console.log("success");
+                };
+                console.log(Cast(selectedButton?.proto?.onClick, ScriptField)?.script);
+            }
+
+            if (!scriptWorked) {
+                this._strokes.forEach(s => {
+                    this.dispatchGesture(GestureUtils.Gestures.Stroke, s);
+                });
             }
+            this._strokes = [];
             this._possibilities = [];
             document.removeEventListener("touchend", this.handleHandUp);
         }
@@ -409,14 +425,14 @@ export default class GestureOverlay extends Touchable {
                         const l = Math.min(this.svgBounds.left, ...this._strokes.map(s => this.getBounds(s).left));
                         const t = Math.min(this.svgBounds.top, ...this._strokes.map(s => this.getBounds(s).top));
                         runInAction(() => {
-                            console.log(possibilities);
                             const buttons = possibilities.map(p => Docs.Create.ButtonDocument({
-                                _height: r - l, _width: 25, backgroundColor: "lightgrey", color: "rgb(34, 34, 34)", x: l, y: t,
+                                _height: 25, _width: r - l, backgroundColor: "lightgrey", color: "rgb(34, 34, 34)", x: l, y: t, _viewType: CollectionViewType.Linear,
+                                isExpanded: true,
                                 title: p, fontSize: 10, letterSpacing: "0px", textTransform: "unset", boxShadow: ".5px .5px 0px rgb(34, 34, 34)",
-                                onClick: ScriptField.MakeScript('Docs.Create.TextDocument(this.title, {_width: 200, _height: 35, x: this.x, y: this.y})')
+                                onClick: ScriptField.MakeScript('createText(this.proto.title, this.x, this.y)')
                             }));
-                            if (this._inkToTextDoc) {
-                                this._inkToTextDoc.data = new List<Doc>(buttons);
+                            if (this._inkToTextDoc && this._thumbDoc) {
+                                this._inkToTextDoc = this._thumbDoc.inkToTextDoc = Docs.Create.LinearDocument(buttons, { _width: 300, _height: 25, _autoHeight: true, _chromeStatus: "disabled", isExpanded: true, flexDirection: "column" });
                                 this._inkToTextDoc.selectedIndex = 0;
                             }
                             this._possibilities = buttons;
@@ -459,7 +475,7 @@ export default class GestureOverlay extends Touchable {
         document.removeEventListener("pointerup", this.onPointerUp);
     }
 
-    dispatchGesture = (gesture: GestureUtils.Gestures, stroke?: InkData) => {
+    dispatchGesture = (gesture: GestureUtils.Gestures, stroke?: InkData, data?: any) => {
         const target = document.elementFromPoint((stroke ?? this._points)[0].X, (stroke ?? this._points)[0].Y);
         target?.dispatchEvent(
             new CustomEvent<GestureUtils.GestureEvent>("dashOnGesture",
@@ -468,7 +484,8 @@ export default class GestureOverlay extends Touchable {
                     detail: {
                         points: stroke ?? this._points,
                         gesture: gesture,
-                        bounds: this.getBounds(stroke ?? this._points)
+                        bounds: this.getBounds(stroke ?? this._points),
+                        text: data
                     }
                 }
             )
@@ -489,27 +506,21 @@ export default class GestureOverlay extends Touchable {
         return this.getBounds(this._points);
     }
 
-    @computed get currentStrokes() {
+    @computed get elements() {
         const B = this.svgBounds;
+        return [
+            this.props.children,
+            this._palette,
 
-        return (
             [this._strokes.map(l => {
                 const b = this.getBounds(l);
                 return <svg width={b.width} height={b.height} style={{ transform: `translate(${b.left}px, ${b.top}px)`, pointerEvents: "none", position: "absolute", zIndex: 30000 }}>
-                    {InteractionUtils.CreatePolyline(l, b.left, b.top, this.Color, this.Width)}
+                    {InteractionUtils.CreatePolyline(l, b.left, b.top, GestureOverlay.Instance.Color, GestureOverlay.Instance.Width)}
                 </svg>;
             }),
             this._points.length <= 1 ? (null) : <svg width={B.width} height={B.height} style={{ transform: `translate(${B.left}px, ${B.top}px)`, pointerEvents: "none", position: "absolute", zIndex: 30000 }}>
-                {InteractionUtils.CreatePolyline(this._points, B.left, B.top, this.Color, this.Width)}
+                {InteractionUtils.CreatePolyline(this._points, B.left, B.top, GestureOverlay.Instance.Color, GestureOverlay.Instance.Width)}
             </svg>]
-        );
-    }
-
-    @computed get elements() {
-        return [
-            this.props.children,
-            this._palette,
-            this.currentStrokes
         ];
     }
 
@@ -548,53 +559,59 @@ export default class GestureOverlay extends Touchable {
     }
 
     @computed
+    private get suggestionContent() {
+        const b = Math.max(this.svgBounds.bottom, ...this._strokes.map(s => this.getBounds(s).bottom));
+        const r = Math.max(this.svgBounds.right, ...this._strokes.map(s => this.getBounds(s).right));
+        const l = Math.min(this.svgBounds.left, ...this._strokes.map(s => this.getBounds(s).left));
+        const t = Math.min(this.svgBounds.top, ...this._strokes.map(s => this.getBounds(s).top));
+        return (
+            <div className="inkToTextDoc-cont" style={{
+                transform: `translate(${l}px, ${b}px)`,
+                width: r - l,
+                height: this._possibilities.length * 25
+            }}>
+                <div className="inkToTextDoc-scroller" style={{ transform: `translate(0, ${-NumCast(this._inkToTextDoc!.selectedIndex) * 25}px)` }}>
+                    <DocumentView
+                        Document={this._inkToTextDoc!}
+                        DataDoc={undefined}
+                        LibraryPath={emptyPath}
+                        addDocument={undefined}
+                        addDocTab={returnFalse}
+                        pinToPres={emptyFunction}
+                        onClick={undefined}
+                        removeDocument={undefined}
+                        ScreenToLocalTransform={() => new Transform(-l, -(b + NumCast(this._inkToTextDoc?.selectedIndex, 0) * 25), 1)}
+                        ContentScaling={returnOne}
+                        PanelWidth={() => 300}
+                        PanelHeight={() => 300}
+                        renderDepth={0}
+                        backgroundColor={returnEmptyString}
+                        focus={emptyFunction}
+                        parentActive={returnTrue}
+                        whenActiveChanged={emptyFunction}
+                        bringToFront={emptyFunction}
+                        ContainingCollectionView={undefined}
+                        ContainingCollectionDoc={undefined}
+                        zoomToScale={emptyFunction}
+                        getScale={returnOne}
+                    />;
+                    </div>
+                <div className="shadow" style={{ height: `calc(100% - 25px - ${NumCast(this._inkToTextDoc!.selectedIndex) * 25}px)` }}>
+                </div>
+            </div>)
+    }
+
     private get inkToTextSuggestions() {
-        console.log(this._possibilities.length);
         if (this._inkToTextDoc && this._possibilities.length) {
-            const b = Math.max(this.svgBounds.bottom, ...this._strokes.map(s => this.getBounds(s).bottom));
-            const r = Math.max(this.svgBounds.right, ...this._strokes.map(s => this.getBounds(s).right));
-            const l = Math.min(this.svgBounds.left, ...this._strokes.map(s => this.getBounds(s).left));
-            const t = Math.min(this.svgBounds.top, ...this._strokes.map(s => this.getBounds(s).top));
-
             return (
-                <div className="inkToTextDoc-cont" style={{
-                    transform: `translate(${l}px, ${b}px)`,
-                    width: r - l,
-                    height: 25,
-                }}>
-                    <div className="inkToTextDoc-scroller" style={{ transform: `translate(0, ${-NumCast(this._inkToTextDoc.selectedIndex) * 25}px)` }}>
-                        <DocumentView
-                            Document={this._inkToTextDoc}
-                            DataDoc={undefined}
-                            LibraryPath={emptyPath}
-                            addDocument={undefined}
-                            addDocTab={returnFalse}
-                            pinToPres={emptyFunction}
-                            onClick={undefined}
-                            removeDocument={undefined}
-                            ScreenToLocalTransform={() => new Transform(-l, -(b + NumCast(this._inkToTextDoc?.selectedIndex, 0) * 25), 1)}
-                            ContentScaling={returnOne}
-                            PanelWidth={() => 300}
-                            PanelHeight={() => 300}
-                            renderDepth={0}
-                            backgroundColor={returnEmptyString}
-                            focus={emptyFunction}
-                            parentActive={returnTrue}
-                            whenActiveChanged={emptyFunction}
-                            bringToFront={emptyFunction}
-                            ContainingCollectionView={undefined}
-                            ContainingCollectionDoc={undefined}
-                            zoomToScale={emptyFunction}
-                            getScale={returnOne}
-                        />;
-                    </div>
-                </div>
+                this.suggestionContent
             );
         }
         return null;
     }
 
     render() {
+        trace();
         return (
             <div className="gestureOverlay-cont" onPointerDown={this.onPointerDown} onTouchStart={this.onReactTouchStart}>
                 {this.elements}
@@ -647,4 +664,8 @@ Scripting.addGlobal(function resetPen() {
         GestureOverlay.Instance.Color = GestureOverlay.Instance.SavedColor ?? "rgb(244, 67, 54)";
         GestureOverlay.Instance.Width = GestureOverlay.Instance.SavedWidth ?? 5;
     });
+});
+Scripting.addGlobal(function createText(text: any, x: any, y: any) {
+    console.log("creating");
+    GestureOverlay.Instance.dispatchGesture("text", [{ X: x, Y: y }], text);
 });
\ No newline at end of file
diff --git a/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx b/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx
index 41ef8c2a6..17139d9d9 100644
--- a/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx
+++ b/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx
@@ -412,7 +412,11 @@ export class CollectionFreeFormView extends CollectionSubView(PanZoomDocument) {
                 this.addDocument(Docs.Create.FreeformDocument(sel, { title: "nested collection", x: bounds.x, y: bounds.y, _width: bWidth, _height: bHeight, _panX: 0, _panY: 0 }));
                 sel.forEach(d => this.props.removeDocument(d));
                 break;
-
+            case GestureUtils.Gestures.Text:
+                if (ge.text) {
+                    const B = this.getTransform().transformPoint(ge.points[0].X, ge.points[0].Y);
+                    this.addDocument(Docs.Create.TextDocument(ge.text, { title: ge.text, x: B[0], y: B[1] }));
+                }
         }
     }
 
diff --git a/src/pen-gestures/GestureUtils.ts b/src/pen-gestures/GestureUtils.ts
index 4b5ad6684..4e3493c1c 100644
--- a/src/pen-gestures/GestureUtils.ts
+++ b/src/pen-gestures/GestureUtils.ts
@@ -6,18 +6,16 @@ import { Doc, WidthSym, HeightSym } from "../new_fields/Doc";
 import { NumCast } from "../new_fields/Types";
 import { CollectionFreeFormView } from "../client/views/collections/collectionFreeForm/CollectionFreeFormView";
 import { Rect } from "react-measure";
+import { Scripting } from "../client/util/Scripting";
 
 export namespace GestureUtils {
-    namespace GestureDataTypes {
-        export type BoxData = Array<Doc>;
-    }
-
     export class GestureEvent {
         constructor(
             readonly gesture: Gestures,
             readonly points: PointData[],
             readonly bounds: Rect,
-            readonly callbackFn?: Function
+            readonly callbackFn?: Function,
+            readonly text?: any
         ) { }
     }
 
@@ -38,7 +36,8 @@ export namespace GestureUtils {
         Box = "box",
         Line = "line",
         Stroke = "stroke",
-        Scribble = "scribble"
+        Scribble = "scribble",
+        Text = "text"
     }
 
     export const GestureRecognizer = new NDollarRecognizer(false);
diff --git a/src/server/authentication/models/current_user_utils.ts b/src/server/authentication/models/current_user_utils.ts
index 6c37380f3..b0ea2f9ad 100644
--- a/src/server/authentication/models/current_user_utils.ts
+++ b/src/server/authentication/models/current_user_utils.ts
@@ -137,7 +137,7 @@ export class CurrentUserUtils {
             thumbDoc.inkToTextDoc = Docs.Create.LinearDocument([], { _width: 300, _height: 25, _autoHeight: true, _chromeStatus: "disabled", isExpanded: true, flexDirection: "column" });
             userDoc.thumbDoc = thumbDoc;
         }
-        return userDoc.thumbDoc;
+        return Cast(userDoc.thumbDoc, Doc);
     }
 
     static setupMobileDoc(userDoc: Doc) {
-- 
cgit v1.2.3-70-g09d2


From 593016303351f365d6ae13413b72d77495ea6a03 Mon Sep 17 00:00:00 2001
From: Stanley Yip <stanley_yip@brown.edu>
Date: Sat, 8 Feb 2020 14:35:20 -0500
Subject: syntax highlighting :)

---
 src/client/documents/Documents.ts                  |  1 +
 src/client/views/GestureOverlay.tsx                | 22 +++++--
 .../collectionFreeForm/CollectionFreeFormView.tsx  | 74 +++++++++++++++++++++-
 .../collections/collectionFreeForm/MarqueeView.tsx | 24 +++++--
 src/pen-gestures/GestureUtils.ts                   |  2 +
 src/pen-gestures/ndollar.ts                        | 11 +++-
 6 files changed, 119 insertions(+), 15 deletions(-)

(limited to 'src/pen-gestures/GestureUtils.ts')

diff --git a/src/client/documents/Documents.ts b/src/client/documents/Documents.ts
index 1c13eb079..81a6ff802 100644
--- a/src/client/documents/Documents.ts
+++ b/src/client/documents/Documents.ts
@@ -139,6 +139,7 @@ export interface DocumentOptions {
     letterSpacing?: string; // is linear view expanded
     flexDirection?: "unset" | "row" | "column" | "row-reverse" | "column-reverse";
     selectedIndex?: number;
+    syntaxColor?: string; // can be applied to text for syntax highlighting all matches in the text
 }
 
 class EmptyBox {
diff --git a/src/client/views/GestureOverlay.tsx b/src/client/views/GestureOverlay.tsx
index e8262af1b..0b77b4b86 100644
--- a/src/client/views/GestureOverlay.tsx
+++ b/src/client/views/GestureOverlay.tsx
@@ -50,7 +50,7 @@ export default class GestureOverlay extends Touchable {
     @observable private _clipboardDoc?: JSX.Element;
     @observable private _possibilities: JSX.Element[] = [];
 
-    @computed private get height(): number { return Math.max(this._pointerY && this._thumbY ? this._thumbY - this._pointerY : 300, 300); }
+    @computed private get height(): number { return 2 * Math.max(this._pointerY && this._thumbY ? this._thumbY - this._pointerY : 300, 300); }
     @computed private get showBounds() { return this.Tool !== ToolglassTools.None; }
 
     private _d1: Doc | undefined;
@@ -408,8 +408,8 @@ export default class GestureOverlay extends Touchable {
             const points = this._points.map(p => ({ X: p.X - B.left, Y: p.Y - B.top }));
 
             const initialPoint = this._points[0.];
-            const xInGlass = initialPoint.X > (this._thumbX ?? Number.MAX_SAFE_INTEGER) && initialPoint.X < (this._thumbX ?? Number.MAX_SAFE_INTEGER) + this.height;
-            const yInGlass = initialPoint.Y > (this._thumbY ?? Number.MAX_SAFE_INTEGER) - this.height && initialPoint.Y < (this._thumbY ?? Number.MAX_SAFE_INTEGER);
+            const xInGlass = initialPoint.X > (this._thumbX ?? Number.MAX_SAFE_INTEGER) && initialPoint.X < (this._thumbX ?? Number.MAX_SAFE_INTEGER) + (this.height);
+            const yInGlass = initialPoint.Y > (this._thumbY ?? Number.MAX_SAFE_INTEGER) - (this.height) && initialPoint.Y < (this._thumbY ?? Number.MAX_SAFE_INTEGER);
 
             if (this.Tool !== ToolglassTools.None && xInGlass && yInGlass) {
                 switch (this.Tool) {
@@ -450,6 +450,14 @@ export default class GestureOverlay extends Touchable {
                             this.dispatchGesture(GestureUtils.Gestures.Box);
                             actionPerformed = true;
                             break;
+                        case GestureUtils.Gestures.StartBracket:
+                            this.dispatchGesture(GestureUtils.Gestures.StartBracket);
+                            actionPerformed = true;
+                            break;
+                        case GestureUtils.Gestures.EndBracket:
+                            this.dispatchGesture(GestureUtils.Gestures.EndBracket);
+                            actionPerformed = true;
+                            break;
                         case GestureUtils.Gestures.Line:
                             actionPerformed = this.handleLineGesture();
                             break;
@@ -562,8 +570,8 @@ export default class GestureOverlay extends Touchable {
 
                 <div className="clipboardDoc-cont" style={{
                     transform: `translate(${this._thumbX}px, ${(this._thumbY ?? 0) - this.height}px)`,
-                    height: this.height * 2,
-                    width: this.height * 2,
+                    height: this.height,
+                    width: this.height,
                     pointerEvents: this._clipboardDoc ? "unset" : "none",
                     touchAction: this._clipboardDoc ? "unset" : "none",
                 }}>
@@ -571,8 +579,8 @@ export default class GestureOverlay extends Touchable {
                 </div>
                 <div className="filter-cont" style={{
                     transform: `translate(${this._thumbX}px, ${(this._thumbY ?? 0) - this.height}px)`,
-                    height: this.height * 2,
-                    width: this.height * 2,
+                    height: this.height,
+                    width: this.height,
                     pointerEvents: "none",
                     touchAction: "none",
                     display: this.showBounds ? "unset" : "none",
diff --git a/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx b/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx
index 2b160f2ff..7b73050ab 100644
--- a/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx
+++ b/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx
@@ -9,7 +9,7 @@ import { Id } from "../../../../new_fields/FieldSymbols";
 import { InkTool, InkField, InkData } from "../../../../new_fields/InkField";
 import { createSchema, makeInterface } from "../../../../new_fields/Schema";
 import { ScriptField } from "../../../../new_fields/ScriptField";
-import { BoolCast, Cast, DateCast, NumCast, StrCast, ScriptCast } from "../../../../new_fields/Types";
+import { BoolCast, Cast, DateCast, NumCast, StrCast, ScriptCast, FieldValue } from "../../../../new_fields/Types";
 import { CurrentUserUtils } from "../../../../server/authentication/models/current_user_utils";
 import { aggregateBounds, emptyFunction, intersectRect, returnOne, Utils } from "../../../../Utils";
 import { DocServer } from "../../../DocServer";
@@ -42,6 +42,8 @@ import { computedFn } from "mobx-utils";
 import { TraceMobx } from "../../../../new_fields/util";
 import { GestureUtils } from "../../../../pen-gestures/GestureUtils";
 import { CognitiveServices } from "../../../cognitive_services/CognitiveServices";
+import { RichTextField } from "../../../../new_fields/RichTextField";
+import { List } from "../../../../new_fields/List";
 
 library.add(faEye as any, faTable, faPaintBrush, faExpandArrowsAlt, faCompressArrowsAlt, faCompass, faUpload, faBraille, faChalkboard, faFileUpload);
 
@@ -70,6 +72,9 @@ const PanZoomDocument = makeInterface(panZoomSchema, documentSchema, positionSch
 export class CollectionFreeFormView extends CollectionSubView(PanZoomDocument) {
     private _lastX: number = 0;
     private _lastY: number = 0;
+    private _inkToTextStartX: number | undefined;
+    private _inkToTextStartY: number | undefined;
+    private _wordPalette: Map<string, string> = new Map<string, string>();
     private _clusterDistance: number = 75;
     private _hitCluster = false;
     private _layoutComputeReaction: IReactionDisposer | undefined;
@@ -411,11 +416,78 @@ export class CollectionFreeFormView extends CollectionSubView(PanZoomDocument) {
                 });
                 this.addDocument(Docs.Create.FreeformDocument(sel, { title: "nested collection", x: bounds.x, y: bounds.y, _width: bWidth, _height: bHeight, _panX: 0, _panY: 0 }));
                 sel.forEach(d => this.props.removeDocument(d));
+                e.stopPropagation();
+                break;
+            case GestureUtils.Gestures.StartBracket:
+                const start = this.getTransform().transformPoint(Math.min(...ge.points.map(p => p.X)), Math.min(...ge.points.map(p => p.Y)));
+                this._inkToTextStartX = start[0];
+                this._inkToTextStartY = start[1];
+                console.log("start");
+                break;
+            case GestureUtils.Gestures.EndBracket:
+                console.log("end");
+                if (this._inkToTextStartX && this._inkToTextStartY) {
+                    const end = this.getTransform().transformPoint(Math.max(...ge.points.map(p => p.X)), Math.max(...ge.points.map(p => p.Y)));
+                    const setDocs = this.getActiveDocuments().filter(s => s.proto?.type === "text" && s.color);
+                    const sets = setDocs.map((sd) => {
+                        return Cast(sd.data, RichTextField)?.Text as string;
+                    });
+                    if (sets.length && sets[0]) {
+                        this._wordPalette.clear();
+                        const colors = setDocs.map(sd => FieldValue(sd.color) as string);
+                        sets.forEach((st: string, i: number) => {
+                            const words = st.split(",");
+                            words.forEach(word => {
+                                this._wordPalette.set(word, colors[i]);
+                            });
+                        });
+                    }
+                    const inks = this.getActiveDocuments().filter(doc => {
+                        if (doc.type === "ink") {
+                            const l = NumCast(doc.x);
+                            const r = l + doc[WidthSym]();
+                            const t = NumCast(doc.y);
+                            const b = t + doc[HeightSym]();
+                            const pass = !(this._inkToTextStartX! > r || end[0] < l || this._inkToTextStartY! > b || end[1] < t);
+                            return pass;
+                        }
+                        return false;
+                    });
+                    const inkFields = inks.map(i => Cast(i.data, InkField));
+                    CognitiveServices.Inking.Appliers.InterpretStrokes(inkFields.filter(i => i instanceof InkField).map(i => i!.inkData)).then((results) => {
+                        const wordResults = results.filter((r: any) => r.category === "inkWord");
+                        console.log(wordResults);
+                        for (const word of wordResults) {
+                            const indices: number[] = word.strokeIds;
+                            indices.forEach(i => {
+                                const otherInks: Doc[] = [];
+                                indices.forEach(i2 => i2 !== i && otherInks.push(inks[i2]));
+                                inks[i].relatedInks = new List<Doc>(otherInks);
+                                const uniqueColors: string[] = [];
+                                Array.from(this._wordPalette.values()).forEach(c => uniqueColors.indexOf(c) === -1 && uniqueColors.push(c));
+                                inks[i].alternativeColors = new List<string>(uniqueColors);
+                                if (this._wordPalette.has(word.recognizedText)) {
+                                    inks[i].color = this._wordPalette.get(word.recognizedText);
+                                }
+                                else {
+                                    for (const alt of word.alternates) {
+                                        if (this._wordPalette.has(alt.recognizedString)) {
+                                            inks[i].color = this._wordPalette.get(alt.recognizedString);
+                                            break;
+                                        }
+                                    }
+                                }
+                            });
+                        }
+                    });
+                    this._inkToTextStartX = end[0];
+                }
                 break;
             case GestureUtils.Gestures.Text:
                 if (ge.text) {
                     const B = this.getTransform().transformPoint(ge.points[0].X, ge.points[0].Y);
                     this.addDocument(Docs.Create.TextDocument(ge.text, { title: ge.text, x: B[0], y: B[1] }));
+                    e.stopPropagation();
                 }
         }
     }
diff --git a/src/client/views/collections/collectionFreeForm/MarqueeView.tsx b/src/client/views/collections/collectionFreeForm/MarqueeView.tsx
index 112df8f05..19a71012a 100644
--- a/src/client/views/collections/collectionFreeForm/MarqueeView.tsx
+++ b/src/client/views/collections/collectionFreeForm/MarqueeView.tsx
@@ -20,6 +20,7 @@ import React = require("react");
 import MarqueeOptionsMenu from "./MarqueeOptionsMenu";
 import { SubCollectionViewProps } from "../CollectionSubView";
 import { CognitiveServices } from "../../../cognitive_services/CognitiveServices";
+import { RichTextField } from "../../../../new_fields/RichTextField";
 
 interface MarqueeViewProps {
     getContainerTransform: () => Transform;
@@ -368,18 +369,29 @@ export class MarqueeView extends React.Component<SubCollectionViewProps & Marque
         const selected = this.marqueeSelect(false);
         if (e instanceof KeyboardEvent ? e.key === "i" : true) {
             const inks = selected.filter(s => s.proto?.type === "ink");
-            const sets = selected.filter(s => s.proto?.type === "text")
-            const inkFields = inks.map(i => FieldValue(Cast(i.data, InkField)));
+            const setDocs = selected.filter(s => s.proto?.type === "text" && s.color);
+            const sets = setDocs.map((sd) => {
+                return Cast(sd.data, RichTextField)?.Text as string;
+            });
+            const colors = setDocs.map(sd => FieldValue(sd.color) as string);
+            const wordToColor = new Map<string, string>();
+            console.log(sets);
+            sets.forEach((st: string, i: number) => {
+                const words = st.split(",");
+                words.forEach(word => {
+                    wordToColor.set(word, colors[i]);
+                });
+            });
+            const inkFields = inks.map(i => Cast(i.data, InkField));
             CognitiveServices.Inking.Appliers.InterpretStrokes(inkFields.filter(i => i instanceof InkField).map(i => i!.inkData)).then((results) => {
                 const wordResults = results.filter((r: any) => r.category === "inkWord");
                 console.log(wordResults);
                 for (const word of wordResults) {
                     const indices: number[] = word.strokeIds;
-                    const r = Math.floor(Math.random() * 256);
-                    const g = Math.floor(Math.random() * 256);
-                    const b = Math.floor(Math.random() * 256);
                     indices.forEach(i => {
-                        inks[i].color = `rgb(${r}, ${g}, ${b})`;
+                        if (wordToColor.has(word.recognizedText)) {
+                            inks[i].color = wordToColor.get(word.recognizedText);
+                        }
                     })
                 }
             });
diff --git a/src/pen-gestures/GestureUtils.ts b/src/pen-gestures/GestureUtils.ts
index 4e3493c1c..f14c573c3 100644
--- a/src/pen-gestures/GestureUtils.ts
+++ b/src/pen-gestures/GestureUtils.ts
@@ -35,6 +35,8 @@ export namespace GestureUtils {
     export enum Gestures {
         Box = "box",
         Line = "line",
+        StartBracket = "startbracket",
+        EndBracket = "endbracket",
         Stroke = "stroke",
         Scribble = "scribble",
         Text = "text"
diff --git a/src/pen-gestures/ndollar.ts b/src/pen-gestures/ndollar.ts
index 9e15ada2d..643d58591 100644
--- a/src/pen-gestures/ndollar.ts
+++ b/src/pen-gestures/ndollar.ts
@@ -142,7 +142,7 @@ export class Result {
 //
 // NDollarRecognizer constants
 //
-const NumMultistrokes = 2;
+const NumMultistrokes = 4;
 const NumPoints = 96;
 const SquareSize = 250.0;
 const OneDThreshold = 0.25; // customize to desired gesture set (usually 0.20 - 0.35)
@@ -178,6 +178,15 @@ export class NDollarRecognizer {
         this.Multistrokes[1] = new Multistroke(GestureUtils.Gestures.Line, useBoundedRotationInvariance, new Array(
             new Array(new Point(12, 347), new Point(119, 347))
         ));
+        this.Multistrokes[2] = new Multistroke(GestureUtils.Gestures.StartBracket, useBoundedRotationInvariance, new Array(
+            // new Array(new Point(145, 20), new Point(30, 21), new Point(34, 150))
+            new Array(new Point(31, 25), new Point(145, 20), new Point(31, 25), new Point(34, 150))
+        ));
+        this.Multistrokes[3] = new Multistroke(GestureUtils.Gestures.EndBracket, useBoundedRotationInvariance, new Array(
+            // new Array(new Point(150, 21), new Point(149, 150), new Point(26, 152))
+            // new Array(new Point(150, 150), new Point(150, 0), new Point(150, 150), new Point(0, 150))
+            new Array(new Point(10, 100), new Point(50, 12), new Point(100, 103))
+        ));
 
         //
         // PREDEFINED STROKES
-- 
cgit v1.2.3-70-g09d2