diff options
Diffstat (limited to 'src/client/views/GestureOverlay.tsx')
| -rw-r--r-- | src/client/views/GestureOverlay.tsx | 392 |
1 files changed, 324 insertions, 68 deletions
diff --git a/src/client/views/GestureOverlay.tsx b/src/client/views/GestureOverlay.tsx index 3a2738c3b..1cd6de55c 100644 --- a/src/client/views/GestureOverlay.tsx +++ b/src/client/views/GestureOverlay.tsx @@ -1,4 +1,5 @@ import * as fitCurve from 'fit-curve'; +import { Bezier } from 'bezier-js'; import { action, computed, makeObservable, observable, runInAction } from 'mobx'; import { observer } from 'mobx-react'; import * as React from 'react'; @@ -7,30 +8,36 @@ import { emptyFunction } from '../../Utils'; import { Doc, Opt, returnEmptyDoclist } from '../../fields/Doc'; import { InkData, InkField, InkTool } from '../../fields/InkField'; import { NumCast } from '../../fields/Types'; +import { Gestures } from '../../pen-gestures/GestureTypes'; +import { GestureUtils } from '../../pen-gestures/GestureUtils'; +import { Result } from '../../pen-gestures/ndollar'; +import { DocumentType } from '../documents/DocumentTypes'; +import { Docs } from '../documents/Documents'; +import { InteractionUtils } from '../util/InteractionUtils'; +import { ScriptingGlobals } from '../util/ScriptingGlobals'; +import { Transform } from '../util/Transform'; +import './GestureOverlay.scss'; +import { InkingStroke } from './InkingStroke'; +import { ObservableReactComponent } from './ObservableReactComponent'; +import { returnEmptyDocViewList } from './StyleProvider'; +import { CollectionFreeFormView } from './collections/collectionFreeForm'; import { ActiveArrowEnd, ActiveArrowScale, ActiveArrowStart, ActiveDash, + ActiveFillColor, ActiveInkBezierApprox, ActiveInkColor, ActiveInkWidth, + DocumentView, SetActiveArrowStart, SetActiveDash, SetActiveFillColor, SetActiveInkColor, SetActiveInkWidth, } from './nodes/DocumentView'; -import { Gestures } from '../../pen-gestures/GestureTypes'; -import { GestureUtils } from '../../pen-gestures/GestureUtils'; -import { InteractionUtils } from '../util/InteractionUtils'; -import { ScriptingGlobals } from '../util/ScriptingGlobals'; -import { Transform } from '../util/Transform'; -import './GestureOverlay.scss'; -import { ObservableReactComponent } from './ObservableReactComponent'; -import { returnEmptyDocViewList } from './StyleProvider'; -import { ActiveFillColor, DocumentView } from './nodes/DocumentView'; - +import { docs } from 'googleapis/build/src/apis/docs'; export enum ToolglassTools { InkToText = 'inktotext', IgnoreGesture = 'ignoregesture', @@ -41,10 +48,12 @@ interface GestureOverlayProps { isActive: boolean; } @observer +/** + * class for gestures. will determine if what the user drew is a gesture, and will transform the ink stroke into the shape the user + * drew or perform the gesture's action + */ export class GestureOverlay extends ObservableReactComponent<React.PropsWithChildren<GestureOverlayProps>> { - // eslint-disable-next-line no-use-before-define static Instance: GestureOverlay; - // eslint-disable-next-line no-use-before-define static Instances: GestureOverlay[] = []; @observable public InkShape: Opt<Gestures> = undefined; @@ -88,7 +97,6 @@ export class GestureOverlay extends ObservableReactComponent<React.PropsWithChil componentDidMount() { GestureOverlay.Instance = this; } - @action onPointerDown = (e: React.PointerEvent) => { if (!(e.target as HTMLElement)?.className?.toString().startsWith('lm_')) { @@ -127,70 +135,322 @@ export class GestureOverlay extends ObservableReactComponent<React.PropsWithChil // SetActiveArrowEnd('none'); } } + /** + * this method returns if what the user drew is a scribble. if it is, it will determine what documents need + * to be deleted and then it will delete them. + * I changed it so it doesnt use triangles. It will modify an intersect array, with its length being + * how many sharp cusps there are. The first index will have a boolean that is based on if there is an + * intersection in the first 1/length percent of the stroke. The second index will be if there is an intersection + * in the 2nd 1/length percent of the stroke. This array will be used in determineInScribble(). + * @returns + */ + isScribble(inkData: InkData) { + const ffView = DocumentView.allViews().find(view => view.ComponentView instanceof CollectionFreeFormView); + console.log(inkData.map(ink => ({ x: ink.X, y: ink.Y }))); + let intersectArray: boolean[] = []; + const cuspArray = this.getCusps(inkData); + console.log(cuspArray.length); + for (let i = 0; i < cuspArray.length; i++) { + intersectArray[i] = false; + } + const docsToDelete: Doc[] = []; + let childDocs = (ffView?.ComponentView as CollectionFreeFormView).childDocs.slice(0, -1); + childDocs.filter(doc => doc.type === 'ink').map(doc => DocumentView.getDocumentView(doc, DocumentView.getDocumentView(doc))); + if ((ffView?.ComponentView as CollectionFreeFormView).childDocs) { + for (const doc of childDocs) { + const otherInk = DocumentView.getDocumentView(doc)?.ComponentView as InkingStroke; + const { inkData: otherInkData } = otherInk?.inkScaledData() ?? { inkData: [] }; + const otherScreenPts = otherInkData.map(point => otherInk.ptToScreen(point)); + if (this.isRectangleOverlap(this.getExtremeCoordinates(otherScreenPts), this.getExtremeCoordinates(inkData))) { + const intersects = this.doInksIntersect(inkData, otherScreenPts); + intersects.forEach(intersect => { + let percentage = ''; + if (intersect.includes('/')) { + const leftOfSlash = intersect.split('/')[0]; + percentage = leftOfSlash; + } else { + percentage = intersect; + } + intersectArray[Math.floor((percentage as unknown as number) * cuspArray.length)] = true; + const docsInBoundingBox = this.docsInBoundingBox(doc, childDocs); + childDocs = childDocs.filter(doc => !docsInBoundingBox.includes(doc)); + docsToDelete.push(...docsInBoundingBox); + docsToDelete.push(doc); + }); + } + } + console.log(intersectArray); + if (intersectArray.length > 3 && this.determineIfScribble(intersectArray)) { + const uniqueArray = Array.from(new Set(docsToDelete)); + console.log(uniqueArray.length); + console.log('is a scribble'); + docsToDelete.forEach(doc => { + ffView?.ComponentView?.removeDocument?.(doc); + }); + this._points = []; + return true; + } + } + return false; + } + /** + * this will return all the docs overlapping with the maindocs bounding box + * @param mainDoc the bounding box of this doc will be used + * @param childDocs the array of all docs in collection + * @returns + */ + docsInBoundingBox(mainDoc: Doc, childDocs: Doc[]): Doc[] { + return childDocs.filter( + doc => + typeof doc.x === 'number' && + typeof doc.y === 'number' && + typeof doc.width === 'number' && + typeof doc.height === 'number' && + typeof mainDoc.x === 'number' && + typeof mainDoc.y === 'number' && + typeof mainDoc.width === 'number' && + typeof mainDoc.height === 'number' && + doc.x < mainDoc.x + mainDoc.width && + doc.x + doc.width > mainDoc.x && + doc.y < mainDoc.y + mainDoc.height && + doc.y + doc.height > mainDoc.y + ); + } + /** + * this method determines if what the user drew is a scribble based on certain criteria. + * @param cuspBooleanArray will take in an array of booleans tht represent what sections(seperated by a cusp) in the scribble + * has an object in it. + * @returns + */ + determineIfScribble(cuspBooleanArray: boolean[]) { + if (!cuspBooleanArray) { + return false; + } + const quarterArrayLength = Math.ceil((cuspBooleanArray.length - 2) * 0.3); + let hasObjectInFirstAndLast25 = true; + for (let i = 0; i < quarterArrayLength; i++) { + if (cuspBooleanArray[i] == false || cuspBooleanArray[cuspBooleanArray.length - 1 - i] == false) { + hasObjectInFirstAndLast25 = false; + } + } + const trueCount = cuspBooleanArray.filter(value => value).length; + const percentageTrues = trueCount / cuspBooleanArray.length; + return percentageTrues >= 0.5 || hasObjectInFirstAndLast25; + } + /** + * determines if two rectangles are overlapping each other + * @param rect1 the rectangle object has has a minX,maxX,minY, and maxY + * @param rect2 + * @returns + */ + // eslint-disable-next-line @typescript-eslint/no-explicit-any + isRectangleOverlap(rect1: any, rect2: any): boolean { + const noOverlap = rect1.maxX < rect2.minX || rect1.minX > rect2.maxX || rect1.maxY < rect2.minY || rect1.minY > rect2.maxY; + + return !noOverlap; + } + /** + * determines if inks intersect + * @param line is pointData + * @param triangle triangle with 3 points + * @returns will return an array, with its lenght being equal to how many intersections there are betweent the 2 strokes. + * each item in the array will contain a number between 0-1 or a number 0-1 seperated by a comma. If one of the curves is a line, then + * then there will just be a number that reprents how far that intersection is along the scribble. For example, + * .1 means that the intersection occurs 10% into the scribble, so near the beginning of it. but if they are both curves, then + * it will return two numbers, one for each curve, seperated by a comma. Sometimes, the percentage it returns is inaccurate, + * espcially in the beginning and end parts of the stroke. dont know why. hope this makes sense + */ + // eslint-disable-next-line @typescript-eslint/no-explicit-any + doInksIntersect(scribble: InkData, inkStroke: InkData): string[] { + let intersectArray: string[] = []; + const extremeScribble = this.getExtremeCoordinates(scribble); + const ffView = DocumentView.allViews().find(view => view.ComponentView instanceof CollectionFreeFormView); + if (ffView && ffView.ComponentView instanceof CollectionFreeFormView) { + for (let i = 0; i < scribble.length - 3; i += 4) { + // iterate over each segment of bezier curve + for (let j = 0; j < inkStroke.length - 3; j += 4) { + const scribbleCurve: Bezier = InkField.Segment(scribble, i); + const strokeCurve: Bezier = InkField.Segment(inkStroke, j); + const points = strokeCurve.points.map(point => ({ + X: point.x, + Y: point.y, + })); + if (ffView && ffView.ComponentView instanceof CollectionFreeFormView && this.isRectangleOverlap(extremeScribble, this.getExtremeCoordinates(points))) { + const result = (ffView.ComponentView as CollectionFreeFormView).bintersects(scribbleCurve, strokeCurve)[0]; + if (result !== undefined) { + intersectArray.push(result.toString()); + } + } + } + } + } + return intersectArray; + } + dryInk = () => { + const newPoints = this._points.reduce((p, pts) => { + p.push([pts.X, pts.Y]); + return p; + }, [] as number[][]); + newPoints.pop(); + const controlPoints: { X: number; Y: number }[] = []; + + const bezierCurves = fitCurve.default(newPoints, 10); + Array.from(bezierCurves).forEach(curve => { + controlPoints.push({ X: curve[0][0], Y: curve[0][1] }); + controlPoints.push({ X: curve[1][0], Y: curve[1][1] }); + controlPoints.push({ X: curve[2][0], Y: curve[2][1] }); + controlPoints.push({ X: curve[3][0], Y: curve[3][1] }); + }); + const dist = Math.sqrt((controlPoints[0].X - controlPoints.lastElement().X) * (controlPoints[0].X - controlPoints.lastElement().X) + (controlPoints[0].Y - controlPoints.lastElement().Y) * (controlPoints[0].Y - controlPoints.lastElement().Y)); + if (controlPoints.length > 4 && dist < 10) controlPoints[controlPoints.length - 1] = controlPoints[0]; + this._points.length = 0; + this._points.push(...controlPoints); + this.dispatchGesture(Gestures.Stroke); + }; @action onPointerUp = () => { DocumentView.DownDocView = undefined; if (this._points.length > 1) { const B = this.svgBounds; const points = this._points.map(p => ({ X: p.X - B.left, Y: p.Y - B.top })); - + const { Name, Score } = + (this.InkShape + ? new Result(this.InkShape, 1, Date.now) + : Doc.UserDoc().recognizeGestures && points.length > 2 + ? GestureUtils.GestureRecognizer.Recognize([points]) + : undefined) ?? + new Result(Gestures.Stroke, 1, Date.now); // prettier-ignore // if any of the shape is activated in the CollectionFreeFormViewChrome - if (this.InkShape) { - this.makeBezierPolygon(this.InkShape, false); - this.dispatchGesture(this.InkShape); - this.primCreated(); - } - // if we're not drawing in a toolglass try to recognize as gesture - else { - // need to decide when to turn gestures back on - const result = points.length > 2 && GestureUtils.GestureRecognizer.Recognize([points]); - let actionPerformed = false; - if (Doc.UserDoc().recognizeGestures && result && result.Score > 0.7) { - switch (result.Name) { - case Gestures.Line: - case Gestures.Triangle: - case Gestures.Rectangle: - case Gestures.Circle: - this.makeBezierPolygon(result.Name, true); - actionPerformed = this.dispatchGesture(result.Name); - break; - case Gestures.Scribble: - console.log('scribble'); - break; - default: - } - } - - // if no gesture (or if the gesture was unsuccessful), "dry" the stroke into an ink document - if (!actionPerformed) { - const newPoints = this._points.reduce((p, pts) => { - p.push([pts.X, pts.Y]); - return p; - }, [] as number[][]); - newPoints.pop(); - const controlPoints: { X: number; Y: number }[] = []; - - const bezierCurves = fitCurve.default(newPoints, 10); - Array.from(bezierCurves).forEach(curve => { - controlPoints.push({ X: curve[0][0], Y: curve[0][1] }); - controlPoints.push({ X: curve[1][0], Y: curve[1][1] }); - controlPoints.push({ X: curve[2][0], Y: curve[2][1] }); - controlPoints.push({ X: curve[3][0], Y: curve[3][1] }); - }); - const dist = Math.sqrt( - (controlPoints[0].X - controlPoints.lastElement().X) * (controlPoints[0].X - controlPoints.lastElement().X) + (controlPoints[0].Y - controlPoints.lastElement().Y) * (controlPoints[0].Y - controlPoints.lastElement().Y) - ); - // eslint-disable-next-line prefer-destructuring - if (controlPoints.length > 4 && dist < 10) controlPoints[controlPoints.length - 1] = controlPoints[0]; - this._points.length = 0; - this._points.push(...controlPoints); - this.dispatchGesture(Gestures.Stroke); + // need to decide when to turn gestures back on + const actionPerformed = ((name: Gestures) => { + switch (name) { + case Gestures.Line: + case Gestures.Triangle: + case Gestures.Rectangle: + case Gestures.Circle: + this.makeBezierPolygon(Name, true); + return this.dispatchGesture(name); + case Gestures.RightAngle: + return this.convertToText().length > 0; + default: } + })(Score < 0.7 ? Gestures.Stroke : (Name as Gestures)); + // if no gesture (or if the gesture was unsuccessful), "dry" the stroke into an ink document + if (!actionPerformed) { + this.dryInk(); + setTimeout(() => { + const ffView = DocumentView.allViews().find(view => view.ComponentView instanceof CollectionFreeFormView); + const scribbleInk = (ffView?.ComponentView as CollectionFreeFormView).childDocs[(ffView?.ComponentView as CollectionFreeFormView).childDocs.length - 1]; + if (this.isScribble((DocumentView.getDocumentView(scribbleInk)?.ComponentView as InkingStroke).inkScaledData().inkData)) { + ffView?.ComponentView?.removeDocument?.(scribbleInk); + } + }, 1); } } this._points.length = 0; }; + /** + * used in the rightAngle gesture to convert handwriting into text. will only work on collections + * TODO: make it work on individual ink docs. + */ + convertToText() { + const ffView = DocumentView.allViews().find(view => view.ComponentView instanceof CollectionFreeFormView); + let minX = 999999999; + let maxX = -999999999; + let minY = 999999999; + let maxY = -999999999; + const textDocs: Doc[] = []; + (ffView?.ComponentView as CollectionFreeFormView).childDocs + .filter(doc => doc.type === DocumentType.COL) + .forEach(doc => { + if (typeof doc.width === 'number' && typeof doc.height === 'number' && typeof doc.x === 'number' && typeof doc.y === 'number') { + const bounds = DocumentView.getDocumentView(doc)?.getBounds; + if (bounds) { + const rect1 = { minX: bounds.left, maxX: bounds.right, minY: bounds.top, maxY: bounds.bottom }; + if (this.isRectangleOverlap(rect1, this.getExtremeCoordinates(this._points))) { + if (doc.x < minX) { + minX = doc.x; + } + if (doc.x > maxX) { + maxX = doc.x; + } + if (doc.y < minY) { + minY = doc.y; + } + if (doc.y + doc.height > maxY) { + maxY = doc.y + doc.height; + } + const newDoc = Docs.Create.TextDocument(doc.transcription as string, { title: '', x: doc.x as number, y: minY }); + newDoc.height = doc.height; + newDoc.width = doc.width; + if (ffView?.ComponentView?.addDocument && ffView?.ComponentView?.removeDocument) { + ffView.ComponentView.addDocument(newDoc); + ffView.ComponentView.removeDocument(doc); + } + textDocs.push(newDoc); + } + } + } + }); + return textDocs; + } + /** + * used to determine how many cusps and where the cusps are in order + * @returns will return an array containing the coordinates of the sharp cusps + */ + getCusps(points: InkData) { + const arrayOfPoints: { X: number; Y: number }[] = []; + arrayOfPoints.push(points[0]); + for (let i = 0; i < points.length - 2; i++) { + const point1 = points[i]; + const point2 = points[i + 1]; + const point3 = points[i + 2]; + if (this.find_angle(point1, point2, point3) < 90) { + arrayOfPoints.push(point2); + } + } + arrayOfPoints.push(points[points.length - 1]); + return arrayOfPoints; + } + /** + * will look through an array of point data and return the coordinates of the smallest box that can fit all the points + * @returns the minX,maxX,minY,maxY of the box + */ + getExtremeCoordinates(points: { X: number; Y: number }[]) { + const coordinates = points; + if (coordinates.length === 0) { + throw new Error('Coordinates array is empty'); + } + let minX = coordinates[0].X; + let maxX = coordinates[0].X; + let minY = coordinates[0].Y; + let maxY = coordinates[0].Y; + + coordinates.forEach(coord => { + if (coord.X < minX) minX = coord.X; + if (coord.X > maxX) maxX = coord.X; + if (coord.Y < minY) minY = coord.Y; + if (coord.Y > maxY) maxY = coord.Y; + }); + return { + minX, + maxX, + minY, + maxY, + }; + } + /** + * takes in three points and then determines the angle of the points. used to determine if the cusp + * is sharp enoug + * @returns + */ + find_angle(A: { X: number; Y: number }, B: { X: number; Y: number }, C: { X: number; Y: number }) { + const AB = Math.sqrt(Math.pow(B.X - A.X, 2) + Math.pow(B.Y - A.Y, 2)); + const BC = Math.sqrt(Math.pow(B.X - C.X, 2) + Math.pow(B.Y - C.Y, 2)); + const AC = Math.sqrt(Math.pow(C.X - A.X, 2) + Math.pow(C.Y - A.Y, 2)); + return Math.acos((BC * BC + AB * AB - AC * AC) / (2 * BC * AB)) * (180 / Math.PI); + } makeBezierPolygon = (shape: string, gesture: boolean) => { const xs = this._points.map(p => p.X); const ys = this._points.map(p => p.Y); @@ -389,7 +649,6 @@ export class GestureOverlay extends ObservableReactComponent<React.PropsWithChil this._strokes.map((l, i) => { const b = { left: -20000, right: 20000, top: -20000, bottom: 20000, width: 40000, height: 40000 }; // this.getBounds(l, true); return ( - // eslint-disable-next-line react/no-array-index-key <svg key={i} width={b.width} height={b.height} style={{ top: 0, left: 0, transform: `translate(${b.left}px, ${b.top}px)`, pointerEvents: 'none', position: 'absolute', zIndex: 30000, overflow: 'visible' }}> {InteractionUtils.CreatePolyline( l, @@ -511,7 +770,6 @@ export class GestureOverlay extends ObservableReactComponent<React.PropsWithChil } ScriptingGlobals.add('GestureOverlay', GestureOverlay); -// eslint-disable-next-line prefer-arrow-callback ScriptingGlobals.add(function setPen(width: string, color: string, fill: string, arrowStart: string, arrowEnd: string, dash: string) { runInAction(() => { GestureOverlay.Instance.SavedColor = ActiveInkColor(); @@ -524,7 +782,6 @@ ScriptingGlobals.add(function setPen(width: string, color: string, fill: string, SetActiveDash(dash); }); }); -// eslint-disable-next-line prefer-arrow-callback ScriptingGlobals.add(function resetPen() { runInAction(() => { SetActiveInkColor(GestureOverlay.Instance.SavedColor ?? 'rgb(0, 0, 0)'); @@ -532,7 +789,6 @@ ScriptingGlobals.add(function resetPen() { }); }, 'resets the pen tool'); ScriptingGlobals.add( - // eslint-disable-next-line prefer-arrow-callback function createText(text: string, X: number, Y: number) { GestureOverlay.Instance.dispatchGesture(Gestures.Text, [{ X, Y }], text); }, |