aboutsummaryrefslogtreecommitdiff
path: root/src/client/views/GestureOverlay.tsx
diff options
context:
space:
mode:
Diffstat (limited to 'src/client/views/GestureOverlay.tsx')
-rw-r--r--src/client/views/GestureOverlay.tsx537
1 files changed, 537 insertions, 0 deletions
diff --git a/src/client/views/GestureOverlay.tsx b/src/client/views/GestureOverlay.tsx
new file mode 100644
index 000000000..84d089b47
--- /dev/null
+++ b/src/client/views/GestureOverlay.tsx
@@ -0,0 +1,537 @@
+import React = require("react");
+import { Touchable } from "./Touchable";
+import { observer } from "mobx-react";
+import "./GestureOverlay.scss";
+import { computed, observable, action, runInAction, IReactionDisposer, reaction } from "mobx";
+import { GestureUtils } from "../../pen-gestures/GestureUtils";
+import { InteractionUtils } from "../util/InteractionUtils";
+import { InkingControl } from "./InkingControl";
+import { InkTool } from "../../new_fields/InkField";
+import { Doc } from "../../new_fields/Doc";
+import { LinkManager } from "../util/LinkManager";
+import { DocUtils } from "../documents/Documents";
+import { undoBatch } from "../util/UndoManager";
+import { Scripting } from "../util/Scripting";
+import { FieldValue, Cast, NumCast, BoolCast } from "../../new_fields/Types";
+import { CurrentUserUtils } from "../../server/authentication/models/current_user_utils";
+import Palette from "./Palette";
+import { Utils, emptyPath, emptyFunction, returnFalse, returnOne, returnEmptyString, returnTrue, numberRange } from "../../Utils";
+import { DocumentView } from "./nodes/DocumentView";
+import { Transform } from "../util/Transform";
+import { DocumentContentsView } from "./nodes/DocumentContentsView";
+
+@observer
+export default class GestureOverlay extends Touchable {
+ static Instance: GestureOverlay;
+
+ @observable public Color: string = "rgb(244, 67, 54)";
+ @observable public Width: number = 5;
+ @observable public SavedColor?: string;
+ @observable public SavedWidth?: number;
+ @observable public Tool: ToolglassTools = ToolglassTools.None;
+
+ @observable private _thumbX?: number;
+ @observable private _thumbY?: number;
+ @observable private _pointerY?: number;
+ @observable private _points: { X: number, Y: number }[] = [];
+ @observable private _palette?: JSX.Element;
+ @observable private _clipboardDoc?: JSX.Element;
+
+ @computed private get height(): number { return Math.max(this._pointerY && this._thumbY ? this._thumbY - this._pointerY : 300, 300); }
+ @computed private get showBounds() { return this.Tool !== ToolglassTools.None; }
+
+ private _d1: Doc | undefined;
+ private _thumbDoc: Doc | undefined;
+ private thumbIdentifier?: number;
+ private pointerIdentifier?: number;
+ private _hands: Map<number, React.Touch[]> = new Map<number, React.Touch[]>();
+
+ protected multiTouchDisposer?: InteractionUtils.MultiTouchEventDisposer;
+
+ constructor(props: Readonly<{}>) {
+ super(props);
+
+ GestureOverlay.Instance = this;
+ }
+
+ getNewTouches(e: React.TouchEvent | TouchEvent) {
+ const ntt: (React.Touch | Touch)[] = Array.from(e.targetTouches);
+ const nct: (React.Touch | Touch)[] = Array.from(e.changedTouches);
+ const nt: (React.Touch | Touch)[] = Array.from(e.touches);
+ this._hands.forEach((hand) => {
+ for (let i = 0; i < e.targetTouches.length; i++) {
+ const pt = e.targetTouches.item(i);
+ if (pt && hand.some((finger) => finger.screenX === pt.screenX && finger.screenY === pt.screenY)) {
+ ntt.splice(ntt.indexOf(pt), 1);
+ }
+ }
+
+ for (let i = 0; i < e.changedTouches.length; i++) {
+ const pt = e.changedTouches.item(i);
+ if (pt && hand.some((finger) => finger.screenX === pt.screenX && finger.screenY === pt.screenY)) {
+ nct.splice(nct.indexOf(pt), 1);
+ }
+ }
+
+ for (let i = 0; i < e.touches.length; i++) {
+ const pt = e.touches.item(i);
+ if (pt && hand.some((finger) => finger.screenX === pt.screenX && finger.screenY === pt.screenY)) {
+ nt.splice(nt.indexOf(pt), 1);
+ }
+ }
+ });
+ return { ntt, nct, nt };
+ }
+
+ onReactTouchStart = (te: React.TouchEvent) => {
+ const actualPts: React.Touch[] = [];
+ for (let i = 0; i < te.touches.length; i++) {
+ const pt: any = te.touches.item(i);
+ actualPts.push(pt);
+ // pen is also a touch, but with a radius of 0.5 (at least with the surface pens)
+ // and this seems to be the only way of differentiating pen and touch on touch events
+ if (pt.radiusX > 1 && pt.radiusY > 1) {
+ // if (typeof pt.identifier !== "string") {
+ // pt.identifier = Utils.GenerateGuid();
+ // }
+ this.prevPoints.set(pt.identifier, pt);
+ }
+ }
+
+ const ptsToDelete: number[] = [];
+ this.prevPoints.forEach(pt => {
+ if (!actualPts.includes(pt)) {
+ ptsToDelete.push(pt.identifier);
+ }
+ });
+
+ ptsToDelete.forEach(pt => this.prevPoints.delete(pt));
+ const nts = this.getNewTouches(te);
+ console.log(nts.nt.length);
+
+ if (nts.nt.length < 5) {
+ const target = document.elementFromPoint(te.changedTouches.item(0).clientX, te.changedTouches.item(0).clientY);
+ target?.dispatchEvent(
+ new CustomEvent<InteractionUtils.MultiTouchEvent<React.TouchEvent>>("dashOnTouchStart",
+ {
+ bubbles: true,
+ detail: {
+ fingers: this.prevPoints.size,
+ targetTouches: nts.ntt,
+ touches: nts.nt,
+ changedTouches: nts.nct,
+ touchEvent: te
+ }
+ }
+ )
+ );
+ document.removeEventListener("touchmove", this.onReactTouchMove);
+ document.removeEventListener("touchend", this.onReactTouchEnd);
+ document.addEventListener("touchmove", this.onReactTouchMove);
+ document.addEventListener("touchend", this.onReactTouchEnd);
+ }
+ else {
+ this.handleHandDown(te);
+ document.removeEventListener("touchmove", this.onReactTouchMove);
+ document.removeEventListener("touchend", this.onReactTouchEnd);
+ }
+ }
+
+ onReactTouchMove = (e: TouchEvent) => {
+ const nts: any = this.getNewTouches(e);
+ document.dispatchEvent(
+ new CustomEvent<InteractionUtils.MultiTouchEvent<TouchEvent>>("dashOnTouchMove",
+ {
+ bubbles: true,
+ detail: {
+ fingers: this.prevPoints.size,
+ targetTouches: nts.ntt,
+ touches: nts.nt,
+ changedTouches: nts.nct,
+ touchEvent: e
+ }
+ })
+ );
+ }
+
+ onReactTouchEnd = (e: TouchEvent) => {
+ const nts: any = this.getNewTouches(e);
+ document.dispatchEvent(
+ new CustomEvent<InteractionUtils.MultiTouchEvent<TouchEvent>>("dashOnTouchEnd",
+ {
+ bubbles: true,
+ detail: {
+ fingers: this.prevPoints.size,
+ targetTouches: nts.ntt,
+ touches: nts.nt,
+ changedTouches: nts.nct,
+ touchEvent: e
+ }
+ })
+ );
+ for (let i = 0; i < e.changedTouches.length; i++) {
+ const pt = e.changedTouches.item(i);
+ if (pt) {
+ if (this.prevPoints.has(pt.identifier)) {
+ this.prevPoints.delete(pt.identifier);
+ }
+ }
+ }
+
+ if (this.prevPoints.size === 0) {
+ document.removeEventListener("touchmove", this.onReactTouchMove);
+ document.removeEventListener("touchend", this.onReactTouchEnd);
+ }
+ e.stopPropagation();
+ }
+
+ handleHandDown = async (e: React.TouchEvent) => {
+ const fingers = new Array<React.Touch>();
+ for (let i = 0; i < e.touches.length; i++) {
+ const pt: any = e.touches.item(i);
+ if (pt.radiusX > 1 && pt.radiusY > 1) {
+ for (let j = 0; j < e.targetTouches.length; j++) {
+ const tPt = e.targetTouches.item(j);
+ if (tPt?.screenX === pt?.screenX && tPt?.screenY === pt?.screenY) {
+ if (pt && this.prevPoints.has(pt.identifier)) {
+ fingers.push(pt);
+ }
+ }
+ }
+ }
+ }
+ const thumb = fingers.reduce((a, v) => a.clientY > v.clientY ? a : v, fingers[0]);
+ const rightMost = Math.max(...fingers.map(f => f.clientX));
+ const leftMost = Math.min(...fingers.map(f => f.clientX));
+ let pointer: React.Touch | undefined;
+ // left hand
+ if (thumb.clientX === rightMost) {
+ pointer = fingers.reduce((a, v) => a.clientX > v.clientX || v.identifier === thumb.identifier ? a : v);
+ }
+ // right hand
+ else if (thumb.clientX === leftMost) {
+ pointer = fingers.reduce((a, v) => a.clientX < v.clientX || v.identifier === thumb.identifier ? a : v);
+ }
+ else {
+ console.log("not hand");
+ }
+ this.pointerIdentifier = pointer?.identifier;
+ runInAction(() => this._pointerY = pointer?.clientY);
+ if (thumb.identifier === this.thumbIdentifier) {
+ this._thumbX = thumb.clientX;
+ this._thumbY = thumb.clientY;
+ this._hands.set(thumb.identifier, fingers);
+ return;
+ }
+ this.thumbIdentifier = thumb?.identifier;
+ this._hands.set(thumb.identifier, fingers);
+ const others = fingers.filter(f => f !== thumb);
+ const minX = Math.min(...others.map(f => f.clientX));
+ const minY = Math.min(...others.map(f => f.clientY));
+
+ const thumbDoc = await Cast(CurrentUserUtils.setupThumbDoc(CurrentUserUtils.UserDocument), Doc);
+ if (thumbDoc) {
+ runInAction(() => {
+ this._thumbDoc = thumbDoc;
+ this._thumbX = thumb.clientX;
+ this._thumbY = thumb.clientY;
+ this._palette = <Palette x={minX} y={minY} thumb={[thumb.clientX, thumb.clientY]} thumbDoc={thumbDoc} />;
+ });
+ }
+
+ this.removeMoveListeners();
+ document.removeEventListener("touchmove", this.handleHandMove);
+ document.addEventListener("touchmove", this.handleHandMove);
+ document.removeEventListener("touchend", this.handleHandUp);
+ document.addEventListener("touchend", this.handleHandUp);
+ }
+
+ @action
+ handleHandMove = (e: TouchEvent) => {
+ const fingers = new Array<React.Touch>();
+ for (let i = 0; i < e.touches.length; i++) {
+ const pt: any = e.touches.item(i);
+ if (pt.radiusX > 1 && pt.radiusY > 1) {
+ for (let j = 0; j < e.targetTouches.length; j++) {
+ const tPt = e.targetTouches.item(j);
+ if (tPt?.screenX === pt?.screenX && tPt?.screenY === pt?.screenY) {
+ if (pt && this.prevPoints.has(pt.identifier)) {
+ this._hands.forEach(hand => hand.some(f => {
+ if (f.identifier === pt.identifier) {
+ fingers.push(pt);
+ }
+ }));
+ }
+ }
+ }
+ }
+ }
+ const thumb = fingers.reduce((a, v) => a.clientY > v.clientY ? a : v, fingers[0]);
+ if (thumb?.identifier && thumb?.identifier === this.thumbIdentifier) {
+ this._hands.set(thumb.identifier, fingers);
+ }
+
+ for (let i = 0; i < e.changedTouches.length; i++) {
+ const pt = e.changedTouches.item(i);
+ if (pt && pt.identifier === this.thumbIdentifier && this._thumbX && this._thumbDoc) {
+ if (Math.abs(pt.clientX - this._thumbX) > 20) {
+ this._thumbDoc.selectedIndex = Math.max(0, NumCast(this._thumbDoc.selectedIndex) - Math.sign(pt.clientX - this._thumbX));
+ this._thumbX = pt.clientX;
+ }
+ }
+ if (pt && pt.identifier === this.pointerIdentifier) {
+ this._pointerY = pt.clientY;
+ }
+ }
+ }
+
+ @action
+ handleHandUp = (e: TouchEvent) => {
+ if (e.touches.length < 3) {
+ // this.onTouchEnd(e);
+ if (this.thumbIdentifier) this._hands.delete(this.thumbIdentifier);
+ this._palette = undefined;
+ this.thumbIdentifier = undefined;
+ this._thumbDoc = undefined;
+ document.removeEventListener("touchend", this.handleHandUp);
+ }
+ }
+
+ @action
+ onPointerDown = (e: React.PointerEvent) => {
+ if (InteractionUtils.IsType(e, InteractionUtils.PENTYPE) || (InkingControl.Instance.selectedTool === InkTool.Highlighter || InkingControl.Instance.selectedTool === InkTool.Pen)) {
+ this._points.push({ X: e.clientX, Y: e.clientY });
+ e.stopPropagation();
+ e.preventDefault();
+
+ document.removeEventListener("pointermove", this.onPointerMove);
+ document.removeEventListener("pointerup", this.onPointerUp);
+ document.addEventListener("pointermove", this.onPointerMove);
+ document.addEventListener("pointerup", this.onPointerUp);
+ }
+ }
+
+ @action
+ onPointerMove = (e: PointerEvent) => {
+ if (InteractionUtils.IsType(e, InteractionUtils.PENTYPE) || (InkingControl.Instance.selectedTool === InkTool.Highlighter || InkingControl.Instance.selectedTool === InkTool.Pen)) {
+ this._points.push({ X: e.clientX, Y: e.clientY });
+ e.stopPropagation();
+ e.preventDefault();
+ }
+ }
+
+ handleLineGesture = (): boolean => {
+ let actionPerformed = false;
+ const B = this.svgBounds;
+ const ep1 = this._points[0];
+ const ep2 = this._points[this._points.length - 1];
+
+ const target1 = document.elementFromPoint(ep1.X, ep1.Y);
+ const target2 = document.elementFromPoint(ep2.X, ep2.Y);
+ const callback = (doc: Doc) => {
+ if (!this._d1) {
+ this._d1 = doc;
+ }
+ else if (this._d1 !== doc && !LinkManager.Instance.doesLinkExist(this._d1, doc)) {
+ DocUtils.MakeLink({ doc: this._d1 }, { doc: doc });
+ actionPerformed = true;
+ }
+ };
+ const ge = new CustomEvent<GestureUtils.GestureEvent>("dashOnGesture",
+ {
+ bubbles: true,
+ detail: {
+ points: this._points,
+ gesture: GestureUtils.Gestures.Line,
+ bounds: B,
+ callbackFn: callback
+ }
+ });
+ target1?.dispatchEvent(ge);
+ target2?.dispatchEvent(ge);
+ return actionPerformed;
+ }
+
+ @action
+ onPointerUp = (e: PointerEvent) => {
+ if (this._points.length > 1) {
+ const B = this.svgBounds;
+ const points = this._points.map(p => ({ X: p.X - B.left, Y: p.Y - B.top }));
+
+ const xInGlass = points[0].X > (this._thumbX ?? Number.MAX_SAFE_INTEGER) && points[0].X < (this._thumbX ?? Number.MAX_SAFE_INTEGER) + this.height;
+ const yInGlass = points[0].Y > (this._thumbY ?? Number.MAX_SAFE_INTEGER) - this.height && points[0].Y < (this._thumbY ?? Number.MAX_SAFE_INTEGER);
+
+ if (this.Tool !== ToolglassTools.None && xInGlass && yInGlass) {
+ switch (this.Tool) {
+ case ToolglassTools.InkToText:
+ break;
+ }
+ }
+ else {
+ const result = GestureUtils.GestureRecognizer.Recognize(new Array(points));
+ let actionPerformed = false;
+ if (result && result.Score > 0.7) {
+ switch (result.Name) {
+ case GestureUtils.Gestures.Box:
+ const target = document.elementFromPoint(this._points[0].X, this._points[0].Y);
+ target?.dispatchEvent(new CustomEvent<GestureUtils.GestureEvent>("dashOnGesture",
+ {
+ bubbles: true,
+ detail: {
+ points: this._points,
+ gesture: GestureUtils.Gestures.Box,
+ bounds: B
+ }
+ }));
+ actionPerformed = true;
+ break;
+ case GestureUtils.Gestures.Line:
+ actionPerformed = this.handleLineGesture();
+ break;
+ case GestureUtils.Gestures.Scribble:
+ console.log("scribble");
+ break;
+ }
+ if (actionPerformed) {
+ this._points = [];
+ }
+ }
+
+ if (!actionPerformed) {
+ const target = document.elementFromPoint(this._points[0].X, this._points[0].Y);
+ target?.dispatchEvent(
+ new CustomEvent<GestureUtils.GestureEvent>("dashOnGesture",
+ {
+ bubbles: true,
+ detail: {
+ points: this._points,
+ gesture: GestureUtils.Gestures.Stroke,
+ bounds: B
+ }
+ }
+ )
+ );
+ this._points = [];
+ }
+ }
+ }
+ document.removeEventListener("pointermove", this.onPointerMove);
+ document.removeEventListener("pointerup", this.onPointerUp);
+ }
+
+ @computed get svgBounds() {
+ const xs = this._points.map(p => p.X);
+ const ys = this._points.map(p => p.Y);
+ const right = Math.max(...xs);
+ const left = Math.min(...xs);
+ const bottom = Math.max(...ys);
+ const top = Math.min(...ys);
+ return { right: right, left: left, bottom: bottom, top: top, width: right - left, height: bottom - top };
+ }
+
+ @computed get currentStroke() {
+ if (this._points.length <= 1) {
+ return (null);
+ }
+
+ const B = this.svgBounds;
+
+ return (
+ <svg width={B.width} height={B.height} style={{ transform: `translate(${B.left}px, ${B.top}px)`, pointerEvents: "none", position: "absolute", zIndex: 30000 }}>
+ {InteractionUtils.CreatePolyline(this._points, B.left, B.top, this.Color, this.Width)}
+ </svg>
+ );
+ }
+
+ @computed get elements() {
+ return [
+ this.props.children,
+ this._palette,
+ this.currentStroke
+ ];
+ }
+
+ @action
+ public openFloatingDoc = (doc: Doc) => {
+ this._clipboardDoc =
+ <DocumentView
+ Document={doc}
+ DataDoc={undefined}
+ LibraryPath={emptyPath}
+ addDocument={undefined}
+ addDocTab={returnFalse}
+ pinToPres={emptyFunction}
+ onClick={undefined}
+ ruleProvider={undefined}
+ removeDocument={undefined}
+ ScreenToLocalTransform={() => new Transform(-(this._thumbX ?? 0), -(this._thumbY ?? 0) + this.height, 1)}
+ ContentScaling={returnOne}
+ PanelWidth={() => 300}
+ PanelHeight={() => 300}
+ renderDepth={0}
+ backgroundColor={returnEmptyString}
+ focus={emptyFunction}
+ parentActive={returnTrue}
+ whenActiveChanged={emptyFunction}
+ bringToFront={emptyFunction}
+ ContainingCollectionView={undefined}
+ ContainingCollectionDoc={undefined}
+ zoomToScale={emptyFunction}
+ getScale={returnOne}
+ />;
+ }
+
+ @action
+ public closeFloatingDoc = () => {
+ this._clipboardDoc = undefined;
+ }
+
+ render() {
+ return (
+ <div className="gestureOverlay-cont" onPointerDown={this.onPointerDown} onTouchStart={this.onReactTouchStart}>
+ {this.elements}
+ <div className="clipboardDoc-cont" style={{
+ transform: `translate(${this._thumbX}px, ${(this._thumbY ?? 0) - this.height}px)`,
+ height: this.height,
+ width: this.height,
+ pointerEvents: this._clipboardDoc ? "unset" : "none",
+ touchAction: this._clipboardDoc ? "unset" : "none",
+ }}>
+ {this._clipboardDoc}
+ </div>
+ <div className="filter-cont" style={{
+ transform: `translate(${this._thumbX}px, ${(this._thumbY ?? 0) - this.height}px)`,
+ height: this.height,
+ width: this.height,
+ pointerEvents: "none",
+ touchAction: "none",
+ display: this.showBounds ? "unset" : "none",
+ }}>
+ </div>
+ </div >);
+ }
+}
+
+export enum ToolglassTools {
+ InkToText = "inktotext",
+ None = "none",
+}
+
+Scripting.addGlobal("GestureOverlay", GestureOverlay);
+Scripting.addGlobal(function setToolglass(tool: any) {
+ runInAction(() => GestureOverlay.Instance.Tool = tool);
+});
+Scripting.addGlobal(function setPen(width: any, color: any) {
+ runInAction(() => {
+ GestureOverlay.Instance.SavedColor = GestureOverlay.Instance.Color;
+ GestureOverlay.Instance.Color = color;
+ GestureOverlay.Instance.SavedWidth = GestureOverlay.Instance.Width;
+ GestureOverlay.Instance.Width = width;
+ });
+});
+Scripting.addGlobal(function resetPen() {
+ runInAction(() => {
+ GestureOverlay.Instance.Color = GestureOverlay.Instance.SavedColor ?? "rgb(244, 67, 54)";
+ GestureOverlay.Instance.Width = GestureOverlay.Instance.SavedWidth ?? 5;
+ });
+}); \ No newline at end of file