diff options
Diffstat (limited to 'src/client/views/GestureOverlay.tsx')
| -rw-r--r-- | src/client/views/GestureOverlay.tsx | 319 |
1 files changed, 243 insertions, 76 deletions
diff --git a/src/client/views/GestureOverlay.tsx b/src/client/views/GestureOverlay.tsx index befd19f6e..5fddaec9a 100644 --- a/src/client/views/GestureOverlay.tsx +++ b/src/client/views/GestureOverlay.tsx @@ -3,34 +3,40 @@ import { action, computed, makeObservable, observable, runInAction } from 'mobx' import { observer } from 'mobx-react'; import * as React from 'react'; import { returnEmptyFilter, returnEmptyString, returnFalse, setupMoveUpEvents } from '../../ClientUtils'; -import { emptyFunction } from '../../Utils'; +import { emptyFunction, intersectRect } from '../../Utils'; import { Doc, Opt, returnEmptyDoclist } from '../../fields/Doc'; import { InkData, InkField, InkTool } from '../../fields/InkField'; import { NumCast } from '../../fields/Types'; +import { Gestures } from '../../pen-gestures/GestureTypes'; +import { GestureUtils } from '../../pen-gestures/GestureUtils'; +import { Result } from '../../pen-gestures/ndollar'; +import { DocumentType } from '../documents/DocumentTypes'; +import { Docs } from '../documents/Documents'; +import { InteractionUtils } from '../util/InteractionUtils'; +import { ScriptingGlobals } from '../util/ScriptingGlobals'; +import { Transform } from '../util/Transform'; +import { undoable } from '../util/UndoManager'; +import './GestureOverlay.scss'; +import { InkingStroke } from './InkingStroke'; +import { ObservableReactComponent } from './ObservableReactComponent'; +import { returnEmptyDocViewList } from './StyleProvider'; +import { CollectionFreeFormView } from './collections/collectionFreeForm'; import { ActiveArrowEnd, ActiveArrowScale, ActiveArrowStart, ActiveDash, + ActiveFillColor, ActiveInkBezierApprox, ActiveInkColor, ActiveInkWidth, + DocumentView, SetActiveArrowStart, SetActiveDash, SetActiveFillColor, SetActiveInkColor, SetActiveInkWidth, } from './nodes/DocumentView'; -import { Gestures } from '../../pen-gestures/GestureTypes'; -import { GestureUtils } from '../../pen-gestures/GestureUtils'; -import { InteractionUtils } from '../util/InteractionUtils'; -import { ScriptingGlobals } from '../util/ScriptingGlobals'; -import { Transform } from '../util/Transform'; -import './GestureOverlay.scss'; -import { ObservableReactComponent } from './ObservableReactComponent'; -import { returnEmptyDocViewList } from './StyleProvider'; -import { ActiveFillColor, DocumentView } from './nodes/DocumentView'; - export enum ToolglassTools { InkToText = 'inktotext', IgnoreGesture = 'ignoregesture', @@ -41,6 +47,10 @@ interface GestureOverlayProps { isActive: boolean; } @observer +/** + * class for gestures. will determine if what the user drew is a gesture, and will transform the ink stroke into the shape the user + * drew or perform the gesture's action + */ export class GestureOverlay extends ObservableReactComponent<React.PropsWithChildren<GestureOverlayProps>> { // eslint-disable-next-line no-use-before-define static Instance: GestureOverlay; @@ -70,10 +80,6 @@ export class GestureOverlay extends ObservableReactComponent<React.PropsWithChil } private _overlayRef = React.createRef<HTMLDivElement>(); - private _d1: Doc | undefined; - private _inkToTextDoc: Doc | undefined; - private thumbIdentifier?: number; - private pointerIdentifier?: number; constructor(props: GestureOverlayProps) { super(props); @@ -88,7 +94,6 @@ export class GestureOverlay extends ObservableReactComponent<React.PropsWithChil componentDidMount() { GestureOverlay.Instance = this; } - @action onPointerDown = (e: React.PointerEvent) => { if (!(e.target as HTMLElement)?.className?.toString().startsWith('lm_')) { @@ -127,78 +132,241 @@ export class GestureOverlay extends ObservableReactComponent<React.PropsWithChil // SetActiveArrowEnd('none'); } } + /** + * If what the user drew is a scribble, this returns the documents that were scribbled over + * I changed it so it doesnt use triangles. It will modify an intersect array, with its length being + * how many sharp cusps there are. The first index will have a boolean that is based on if there is an + * intersection in the first 1/length percent of the stroke. The second index will be if there is an intersection + * in the 2nd 1/length percent of the stroke. This array will be used in determineIfScribble(). + * @param ffview freeform view where scribble is drawn + * @param scribbleStroke scribble stroke in screen space coordinats + * @returns array of documents scribbled over + */ + isScribble = (ffView: CollectionFreeFormView, cuspArray: { X: number; Y: number }[], scribbleStroke: { X: number; Y: number }[]) => { + const intersectArray = cuspArray.map(() => false); + const scribbleBounds = InkField.getBounds(scribbleStroke); + const docsToDelete = ffView.childDocs + .map(doc => DocumentView.getDocumentView(doc)) + .filter(dv => dv?.ComponentView instanceof InkingStroke) + .map(dv => dv?.ComponentView as InkingStroke) + .filter(otherInk => { + const otherScreenPts = otherInk.inkScaledData?.().inkData.map(otherInk.ptToScreen); + if (intersectRect(InkField.getBounds(otherScreenPts), scribbleBounds)) { + const intersects = this.findInkIntersections(scribbleStroke, otherScreenPts).map(intersect => { + const percentage = intersect.split('/')[0]; + intersectArray[Math.floor(Number(percentage) * cuspArray.length)] = true; + }); + return intersects.length > 0; + } + }); + return !this.determineIfScribble(intersectArray) ? undefined : + [ ...docsToDelete.map(stroke => stroke.Document), + // bcz: NOTE: docsInBoundingBox test should be replaced with a docsInConvexHull test + ...this.docsInBoundingBox({ topLeft : ffView.ScreenToContentsXf().transformPoint(scribbleBounds.left, scribbleBounds.top), + bottomRight: ffView.ScreenToContentsXf().transformPoint(scribbleBounds.right,scribbleBounds.bottom)}, + ffView.childDocs.filter(doc => !docsToDelete.map(s => s.Document).includes(doc)) )]; // prettier-ignore + }; + /** + * Returns all docs in array that overlap bounds. Note that the bounds should be given in screen space coordinates. + * @param boundingBox screen space bounding box + * @param childDocs array of docs to test against bounding box + * @returns list of docs that overlap rect + */ + docsInBoundingBox = (boundingBox: { topLeft: number[]; bottomRight: number[] }, childDocs: Doc[]): Doc[] => { + const rect = { left: boundingBox.topLeft[0], top: boundingBox.topLeft[1], width: boundingBox.bottomRight[0] - boundingBox.topLeft[0], height: boundingBox.bottomRight[1] - boundingBox.topLeft[1] }; + return childDocs.filter(doc => intersectRect(rect, { left: NumCast(doc.x), top: NumCast(doc.y), width: NumCast(doc._width), height: NumCast(doc._height) })); + }; + /** + * Determines if what the array of cusp/intersection data corresponds to a scribble. + * true if there are at least 4 cusps and either: + * 1) the initial and final quarters of the array contain objects + * 2) or half of the cusps contain objects + * @param intersectArray array of booleans coresponding to which scribble sections (regions separated by a cusp) contain Docs + * @returns + */ + determineIfScribble = (intersectArray: boolean[]) => { + const quarterArrayLength = Math.ceil(intersectArray.length / 3.9); // use 3.9 instead of 4 to work better with strokes with only 4 cusps + const { start, end } = intersectArray.reduce((res, val, i) => ({ // test for scribbles at start and end of scribble stroke + start: res.start || (val && i <= quarterArrayLength), + end: res.end || (val && i >= intersectArray.length - quarterArrayLength) + }), { start: false, end: false }); // prettier-ignore + + const percentCuspsWithContent = intersectArray.filter(value => value).length / intersectArray.length; + return intersectArray.length > 3 && (percentCuspsWithContent >= 0.5 || (start && end)); + }; + /** + * determines if inks intersect + * @param line is pointData + * @param triangle triangle with 3 points + * @returns will return an array, with its lenght being equal to how many intersections there are betweent the 2 strokes. + * each item in the array will contain a number between 0-1 or a number 0-1 seperated by a comma. If one of the curves is a line, then + * then there will just be a number that reprents how far that intersection is along the scribble. For example, + * .1 means that the intersection occurs 10% into the scribble, so near the beginning of it. but if they are both curves, then + * it will return two numbers, one for each curve, seperated by a comma. Sometimes, the percentage it returns is inaccurate, + * espcially in the beginning and end parts of the stroke. dont know why. hope this makes sense + */ + findInkIntersections = (scribble: InkData, inkStroke: InkData): string[] => { + const intersectArray: string[] = []; + const scribbleBounds = InkField.getBounds(scribble); + for (let i = 0; i < scribble.length - 3; i += 4) { // for each segment of scribble + for (let j = 0; j < inkStroke.length - 3; j += 4) { // for each segment of ink stroke + const scribbleSeg = InkField.Segment(scribble, i); + const strokeSeg = InkField.Segment(inkStroke, j); + const strokeBounds = InkField.getBounds(strokeSeg.points.map(pt => ({ X: pt.x, Y: pt.y }))); + if (intersectRect(scribbleBounds, strokeBounds)) { + const result = InkField.bintersects(scribbleSeg, strokeSeg)[0]; + if (result !== undefined) { + intersectArray.push(result.toString()); + } + } + } // prettier-ignore + } // prettier-ignore + return intersectArray; + }; + dryInk = () => { + const newPoints = this._points.reduce((p, pts) => { + p.push([pts.X, pts.Y]); + return p; + }, [] as number[][]); + newPoints.pop(); + const controlPoints: { X: number; Y: number }[] = []; + + const bezierCurves = fitCurve.default(newPoints, 10); + Array.from(bezierCurves).forEach(curve => { + controlPoints.push({ X: curve[0][0], Y: curve[0][1] }); + controlPoints.push({ X: curve[1][0], Y: curve[1][1] }); + controlPoints.push({ X: curve[2][0], Y: curve[2][1] }); + controlPoints.push({ X: curve[3][0], Y: curve[3][1] }); + }); + const dist = Math.sqrt((controlPoints[0].X - controlPoints.lastElement().X) * (controlPoints[0].X - controlPoints.lastElement().X) + (controlPoints[0].Y - controlPoints.lastElement().Y) * (controlPoints[0].Y - controlPoints.lastElement().Y)); + if (controlPoints.length > 4 && dist < 10) controlPoints[controlPoints.length - 1] = controlPoints[0]; + this._points.length = 0; + this._points.push(...controlPoints); + this.dispatchGesture(Gestures.Stroke); + }; @action onPointerUp = () => { + const ffView = DocumentView.DownDocView?.ComponentView instanceof CollectionFreeFormView && DocumentView.DownDocView.ComponentView; DocumentView.DownDocView = undefined; if (this._points.length > 1) { const B = this.svgBounds; const points = this._points.map(p => ({ X: p.X - B.left, Y: p.Y - B.top })); - + const { Name, Score } = + (this.InkShape + ? new Result(this.InkShape, 1, Date.now) + : Doc.UserDoc().recognizeGestures && points.length > 2 + ? GestureUtils.GestureRecognizer.Recognize([points]) + : undefined) ?? + new Result(Gestures.Stroke, 1, Date.now); // prettier-ignore + + const cuspArray = this.getCusps(points); // if any of the shape is activated in the CollectionFreeFormViewChrome - if (this.InkShape) { - GestureOverlay.makeBezierPolygon(this._points, this.InkShape, false); - this.dispatchGesture(this.InkShape); - this.primCreated(); - } - // if we're not drawing in a toolglass try to recognize as gesture - else { - // need to decide when to turn gestures back on - const result = points.length > 2 && GestureUtils.GestureRecognizer.Recognize([points]); - let actionPerformed = false; - if (Doc.UserDoc().recognizeGestures && result && result.Score > 0.7) { - switch (result.Name) { - case Gestures.Line: - case Gestures.Triangle: - case Gestures.Rectangle: - case Gestures.Circle: - GestureOverlay.makeBezierPolygon(this._points, result.Name, true); - actionPerformed = this.dispatchGesture(result.Name); - break; - case Gestures.Scribble: - console.log('scribble'); - break; - case Gestures.RightAngle: - console.log('RightAngle'); - break; - default: { - /* empty */ - } - } + // need to decide when to turn gestures back on + const actionPerformed = ((name: Gestures) => { + switch (name) { + case Gestures.Line: + if (cuspArray.length > 2) return undefined; + // eslint-disable-next-line no-fallthrough + case Gestures.Triangle: + case Gestures.Rectangle: + case Gestures.Circle: + this.makeBezierPolygon(this._points, Name, true); + return this.dispatchGesture(name); + case Gestures.RightAngle: + return ffView && this.convertToText(ffView).length > 0; + default: } - - // if no gesture (or if the gesture was unsuccessful), "dry" the stroke into an ink document - if (!actionPerformed) { - const newPoints = this._points.reduce((p, pts) => { - p.push([pts.X, pts.Y]); - return p; - }, [] as number[][]); - newPoints.pop(); - const controlPoints: { X: number; Y: number }[] = []; - - const bezierCurves = fitCurve.default(newPoints, 10); - Array.from(bezierCurves).forEach(curve => { - controlPoints.push({ X: curve[0][0], Y: curve[0][1] }); - controlPoints.push({ X: curve[1][0], Y: curve[1][1] }); - controlPoints.push({ X: curve[2][0], Y: curve[2][1] }); - controlPoints.push({ X: curve[3][0], Y: curve[3][1] }); - }); - const dist = Math.sqrt( - (controlPoints[0].X - controlPoints.lastElement().X) * (controlPoints[0].X - controlPoints.lastElement().X) + (controlPoints[0].Y - controlPoints.lastElement().Y) * (controlPoints[0].Y - controlPoints.lastElement().Y) - ); - // eslint-disable-next-line prefer-destructuring - if (controlPoints.length > 4 && dist < 10) controlPoints[controlPoints.length - 1] = controlPoints[0]; - this._points.length = 0; - this._points.push(...controlPoints); - this.dispatchGesture(Gestures.Stroke); + })(Score < 0.7 ? Gestures.Stroke : (Name as Gestures)); + // if no gesture (or if the gesture was unsuccessful), "dry" the stroke into an ink document + + if (!actionPerformed) { + const scribbledOver = ffView && this.isScribble(ffView, cuspArray, this._points); + if (scribbledOver) { + undoable(() => ffView.removeDocument(scribbledOver), 'scribble erase')(); + } else { + this.dryInk(); } } } this._points.length = 0; }; - - public static makeBezierPolygon = (points: { X: number; Y: number }[], shape: string, gesture: boolean) => { - const xs = points.map(p => p.X); - const ys = points.map(p => p.Y); + /** + * used in the rightAngle gesture to convert handwriting into text. will only work on collections + * TODO: make it work on individual ink docs. + */ + convertToText = (ffView: CollectionFreeFormView) => { + let minX = 999999999; + let maxX = -999999999; + let minY = 999999999; + let maxY = -999999999; + const textDocs: Doc[] = []; + ffView.childDocs + .filter(doc => doc.type === DocumentType.COL) + .forEach(doc => { + if (typeof doc.width === 'number' && typeof doc.height === 'number' && typeof doc.x === 'number' && typeof doc.y === 'number') { + const bounds = DocumentView.getDocumentView(doc)?.getBounds; + if (bounds) { + if (intersectRect({ ...bounds, width: bounds.right - bounds.left, height: bounds.bottom - bounds.top }, InkField.getBounds(this._points))) { + if (doc.x < minX) { + minX = doc.x; + } + if (doc.x > maxX) { + maxX = doc.x; + } + if (doc.y < minY) { + minY = doc.y; + } + if (doc.y + doc.height > maxY) { + maxY = doc.y + doc.height; + } + const newDoc = Docs.Create.TextDocument(doc.transcription as string, { title: '', x: doc.x as number, y: minY }); + newDoc.height = doc.height; + newDoc.width = doc.width; + if (ffView.addDocument && ffView.removeDocument) { + ffView.addDocument(newDoc); + ffView.removeDocument(doc); + } + textDocs.push(newDoc); + } + } + } + }); + return textDocs; + }; + /** + * Returns array of coordinates corresponding to the sharp cusps in an input stroke + * @param points array of X,Y stroke coordinates + * @returns array containing the coordinates of the sharp cusps + */ + getCusps(points: InkData) { + const arrayOfPoints: { X: number; Y: number }[] = []; + arrayOfPoints.push(points[0]); + for (let i = 0; i < points.length - 2; i++) { + const point1 = points[i]; + const point2 = points[i + 1]; + const point3 = points[i + 2]; + if (this.find_angle(point1, point2, point3) < 90) { + // NOTE: this is not an accurate way to find cusps -- it is highly dependent on sampling rate and doesn't work well with slowly drawn scribbles + arrayOfPoints.push(point2); + } + } + arrayOfPoints.push(points[points.length - 1]); + return arrayOfPoints; + } + /** + * takes in three points and then determines the angle of the points. used to determine if the cusp + * is sharp enoug + * @returns + */ + find_angle(A: { X: number; Y: number }, B: { X: number; Y: number }, C: { X: number; Y: number }) { + const AB = Math.sqrt(Math.pow(B.X - A.X, 2) + Math.pow(B.Y - A.Y, 2)); + const BC = Math.sqrt(Math.pow(B.X - C.X, 2) + Math.pow(B.Y - C.Y, 2)); + const AC = Math.sqrt(Math.pow(C.X - A.X, 2) + Math.pow(C.Y - A.Y, 2)); + return Math.acos((BC * BC + AB * AB - AC * AC) / (2 * BC * AB)) * (180 / Math.PI); + } + makeBezierPolygon = (points: { X: number; Y: number }[], shape: string, gesture: boolean) => { + const xs = this._points.map(p => p.X); + const ys = this._points.map(p => p.Y); let right = Math.max(...xs); let left = Math.min(...xs); let bottom = Math.max(...ys); @@ -394,7 +562,6 @@ export class GestureOverlay extends ObservableReactComponent<React.PropsWithChil this._strokes.map((l, i) => { const b = { left: -20000, right: 20000, top: -20000, bottom: 20000, width: 40000, height: 40000 }; // this.getBounds(l, true); return ( - // eslint-disable-next-line react/no-array-index-key <svg key={i} width={b.width} height={b.height} style={{ top: 0, left: 0, transform: `translate(${b.left}px, ${b.top}px)`, pointerEvents: 'none', position: 'absolute', zIndex: 30000, overflow: 'visible' }}> {InteractionUtils.CreatePolyline( l, |