From 0c33bc8033c9877abbe6e4074a687559bc4948d0 Mon Sep 17 00:00:00 2001 From: eleanor-park Date: Tue, 23 Apr 2024 15:16:06 -0400 Subject: erase multiple segments bug --- src/client/views/MainView.tsx | 3 +++ 1 file changed, 3 insertions(+) (limited to 'src/client/views/MainView.tsx') diff --git a/src/client/views/MainView.tsx b/src/client/views/MainView.tsx index 58b8d255a..56db0c488 100644 --- a/src/client/views/MainView.tsx +++ b/src/client/views/MainView.tsx @@ -358,6 +358,9 @@ export class MainView extends ObservableReactComponent<{}> { fa.faCut, fa.faEllipsisV, fa.faEraser, + fa.faDeleteLeft, + fa.faXmarksLines, + fa.faCircleXmark, fa.faExclamation, fa.faFileAlt, fa.faFileAudio, -- cgit v1.2.3-70-g09d2 From 4a01680bd22a0652dbdd0da5c3a7167ca8117440 Mon Sep 17 00:00:00 2001 From: eleanor-park Date: Wed, 8 May 2024 22:22:29 -0400 Subject: close curve and bounding box fixes --- src/client/util/CurrentUserUtils.ts | 10 +- src/client/views/MainView.tsx | 1 + .../collectionFreeForm/CollectionFreeFormView.tsx | 347 ++++++++++++--------- src/client/views/global/globalScripts.ts | 10 +- 4 files changed, 214 insertions(+), 154 deletions(-) (limited to 'src/client/views/MainView.tsx') diff --git a/src/client/util/CurrentUserUtils.ts b/src/client/util/CurrentUserUtils.ts index b1673ff1c..3ee1b42aa 100644 --- a/src/client/util/CurrentUserUtils.ts +++ b/src/client/util/CurrentUserUtils.ts @@ -752,13 +752,13 @@ pie title Minerals in my tap water return [ { title: "Pen", toolTip: "Pen (Ctrl+P)", btnType: ButtonType.ToggleButton, icon: "pen-nib", toolType: "pen", scripts: {onClick:'{ return setActiveTool(this.toolType, false, _readOnly_);}' }}, { title: "Write", toolTip: "Write (Ctrl+Shift+P)", btnType: ButtonType.ToggleButton, icon: "pen", toolType: "write", scripts: {onClick:'{ return setActiveTool(this.toolType, false, _readOnly_);}' }, funcs: {hidden:"IsNoviceMode()" }}, - { title: "Eraser", toolTip: "Eraser (Ctrl+E)", btnType: ButtonType.MultiToggleButton, toolType:"eraser", scripts: {onClick: '{ return setActiveTool(this.toolType, false, _readOnly_);}'}, + { title: "Eraser", toolTip: "Eraser (Ctrl+E)", btnType: ButtonType.MultiToggleButton, scripts: {onClick: '{ return setActiveTool(this.toolType, false, _readOnly_);}'}, funcs: {toolType:"activeEraserTool()"}, subMenu: [ { title: "Stroke", toolTip: "Stroke Erase", btnType: ButtonType.ToggleButton, icon: "eraser", toolType:InkTool.StrokeEraser, ignoreClick: true, scripts: {onClick: '{ return setActiveTool(this.toolType, false, _readOnly_);}'} }, - { title: "Segment", toolTip: "Segment Erase", btnType: ButtonType.ToggleButton, icon: "xmarks-lines",toolType:InkTool.SegmentEraser,ignoreClick: true, scripts: {onClick: '{ return setActiveTool(this.toolType, false, _readOnly_);}'} }, + { title: "Segment", toolTip: "Segment Erase", btnType: ButtonType.ToggleButton, icon: "xmark",toolType:InkTool.SegmentEraser,ignoreClick: true, scripts: {onClick: '{ return setActiveTool(this.toolType, false, _readOnly_);}'} }, { title: "Radius", toolTip: "Radius Erase", btnType: ButtonType.ToggleButton, icon: "circle-xmark",toolType:InkTool.RadiusEraser, ignoreClick: true, scripts: {onClick: '{ return setActiveTool(this.toolType, false, _readOnly_);}'} }, - ]}, - { title: "Eraser Width", toolTip: "Eraser Width", btnType: ButtonType.NumberSliderButton, toolType: "eraserWidth", ignoreClick: true, scripts: {script: '{ return setInkProperty(this.toolType, value, _readOnly_);}'}, numBtnMin: 1, funcs: {hidden:"isRadiusEraser()" }}, + ]}, + { title: "Eraser Width", toolTip: "Eraser Width", btnType: ButtonType.NumberSliderButton, toolType: "eraserWidth", ignoreClick: true, scripts: {script: '{ return setInkProperty(this.toolType, value, _readOnly_);}'}, numBtnMin: 1, funcs: {hidden:"NotRadiusEraser()"}}, { title: "Circle", toolTip: "Circle (double tap to lock mode)", btnType: ButtonType.ToggleButton, icon: "circle", toolType:GestureUtils.Gestures.Circle, scripts: {onClick:`{ return setActiveTool(this.toolType, false, _readOnly_);}`, onDoubleClick:`{ return setActiveTool(this.toolType, true, _readOnly_);}`} }, { title: "Square", toolTip: "Square (double tap to lock mode)", btnType: ButtonType.ToggleButton, icon: "square", toolType:GestureUtils.Gestures.Rectangle, scripts: {onClick:`{ return setActiveTool(this.toolType, false, _readOnly_);}`, onDoubleClick:`{ return setActiveTool(this.toolType, true, _readOnly_);}`} }, { title: "Line", toolTip: "Line (double tap to lock mode)", btnType: ButtonType.ToggleButton, icon: "minus", toolType:GestureUtils.Gestures.Line, scripts: {onClick:`{ return setActiveTool(this.toolType, false, _readOnly_);}`, onDoubleClick:`{ return setActiveTool(this.toolType, true, _readOnly_);}`} }, @@ -1119,7 +1119,7 @@ pie title Minerals in my tap water ScriptingGlobals.add(function MySharedDocs() { return Doc.MySharedDocs; }, "document containing all shared Docs"); ScriptingGlobals.add(function IsExploreMode() { return SnappingManager.ExploreMode; }, "is Dash in exploration mode"); ScriptingGlobals.add(function IsNoviceMode() { return Doc.noviceMode; }, "is Dash in novice mode"); -ScriptingGlobals.add(function isRadiusEraser() { return !(Doc.ActiveTool === InkTool.RadiusEraser); }, "is the eraser selected"); +ScriptingGlobals.add(function NotRadiusEraser() { return Doc.ActiveTool !== InkTool.RadiusEraser; }, "is the active tool anything but the radius eraser"); ScriptingGlobals.add(function toggleComicMode() { Doc.UserDoc().renderStyle = Doc.UserDoc().renderStyle === "comic" ? undefined : "comic"; }, "switches between comic and normal document rendering"); ScriptingGlobals.add(function importDocument() { return CurrentUserUtils.importDocument(); }, "imports files from device directly into the import sidebar"); ScriptingGlobals.add(function setInkToolDefaults() { Doc.ActiveTool = InkTool.None; }); diff --git a/src/client/views/MainView.tsx b/src/client/views/MainView.tsx index 56db0c488..d0b3221b4 100644 --- a/src/client/views/MainView.tsx +++ b/src/client/views/MainView.tsx @@ -361,6 +361,7 @@ export class MainView extends ObservableReactComponent<{}> { fa.faDeleteLeft, fa.faXmarksLines, fa.faCircleXmark, + fa.faXmark, fa.faExclamation, fa.faFileAlt, fa.faFileAudio, diff --git a/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx b/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx index eedee0b18..2e174be30 100644 --- a/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx +++ b/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx @@ -1,5 +1,7 @@ +import { inside } from '@turf/turf'; import { Bezier } from 'bezier-js'; import { Colors } from 'browndash-components'; +import { validationResult } from 'express-validator'; import { action, computed, IReactionDisposer, makeObservable, observable, reaction, runInAction } from 'mobx'; import { observer } from 'mobx-react'; import { computedFn } from 'mobx-utils'; @@ -35,7 +37,7 @@ import { Timeline } from '../../animationtimeline/Timeline'; import { ContextMenu } from '../../ContextMenu'; import { GestureOverlay } from '../../GestureOverlay'; import { CtrlKey } from '../../GlobalKeyHandler'; -import { ActiveInkWidth, InkingStroke, SetActiveInkColor, SetActiveInkWidth } from '../../InkingStroke'; +import { ActiveEraserWidth, ActiveInkWidth, InkingStroke, SetActiveInkColor, SetActiveInkWidth } from '../../InkingStroke'; import { LightboxView } from '../../LightboxView'; import { CollectionFreeFormDocumentView } from '../../nodes/CollectionFreeFormDocumentView'; import { SchemaCSVPopUp } from '../../nodes/DataVizBox/SchemaCSVPopUp'; @@ -618,11 +620,15 @@ export class CollectionFreeFormView extends CollectionSubView { @@ -750,10 +755,7 @@ export class CollectionFreeFormView extends CollectionSubView this.forceStrokeGesture( e, @@ -764,12 +766,44 @@ export class CollectionFreeFormView extends CollectionSubView this._eraserLock--); } // Lower ink opacity to give the user a visual indicator of deletion. - intersect.inkView.layoutDoc.opacity = 0.2; + intersect.inkView.layoutDoc.opacity = 0; intersect.inkView.layoutDoc.dontIntersect = true; } }); return false; }; + + @action + onRadiusEraserMove = (e: PointerEvent, down: number[], delta: number[]) => { + const currPoint = { X: e.clientX, Y: e.clientY }; + this._eraserPts.push([currPoint.X, currPoint.Y]); + this._eraserPts = this._eraserPts.slice(Math.max(0, this._eraserPts.length - 5)); + if (this._eraserLock) return false; // bcz: should be fixed by putting it on a queue to be processed after the last eraser movement is processed. + const strokeMap: Map = this.getRadiusEraserIntersections({ X: currPoint.X - delta[0], Y: currPoint.Y - delta[1] }, currPoint); + strokeMap.forEach((intersects, stroke) => { + if (!this._deleteList.includes(stroke)) { + this._deleteList.push(stroke); + SetActiveInkWidth(StrCast(stroke.Document.stroke_width?.toString()) || '1'); + SetActiveInkColor(StrCast(stroke.Document.color?.toString()) || 'black'); + this._eraserLock++; + // create a new curve by appending all curves of the current segment together in order to render a single new stroke. + const segments = this.radiusErase(stroke, intersects.sort()); + segments?.forEach(segment => + this.forceStrokeGesture( + e, + GestureUtils.Gestures.Stroke, + segment.reduce((data, curve) => [...data, ...curve.points.map(p => stroke.ComponentView?.ptToScreen?.({ X: p.x, Y: p.y }) ?? { X: 0, Y: 0 })], [] as PointData[]) + ) + ); + setTimeout(() => this._eraserLock--); + } + // Lower ink opacity to give the user a visual indicator of deletion. + stroke.layoutDoc.opacity = 0; + stroke.layoutDoc.dontIntersect = true; + }); + return false; + }; + forceStrokeGesture = (e: PointerEvent, gesture: GestureUtils.Gestures, points: InkData, text?: any) => { this.onGesture(e, new GestureUtils.GestureEvent(gesture, points, GestureOverlay.getBounds(points), text)); }; @@ -788,7 +822,7 @@ export class CollectionFreeFormView extends CollectionSubView { - const radius = ActiveEraserWidth() / 2 + 3; // reduce values to avoid extreme radii + const radius = ActiveEraserWidth() + 3; // add 3 to avoid eraser being too thin const c = 0.551915024494; // circle tangent length to side ratio const movement = { x: endInkCoordsIn.X - startInkCoordsIn.X, y: endInkCoordsIn.Y - startInkCoordsIn.Y }; const moveLen = Math.sqrt(movement.x ** 2 + movement.y ** 2); @@ -836,6 +870,39 @@ export class CollectionFreeFormView extends CollectionSubView { + var isInside = false; + if (!isNaN(eraserOutline[0].X) && !isNaN(eraserOutline[0].Y)) { + var minX = eraserOutline[0].X, maxX = eraserOutline[0].X; + var minY = eraserOutline[0].Y, maxY = eraserOutline[0].Y; + for (var i = 1; i < eraserOutline.length; i++) { + const currPoint: {X: number, Y: number} = eraserOutline[i]; + minX = Math.min(currPoint.X, minX); + maxX = Math.max(currPoint.X, maxX); + minY = Math.min(currPoint.Y, minY); + maxY = Math.max(currPoint.Y, maxY); + } + + if (point.X < minX || point.X > maxX || point.Y < minY || point.Y > maxY) { + return false; + } + + for (var i = 0, j = eraserOutline.length - 1; i < eraserOutline.length; j = i, i++) { + if ((eraserOutline[i].Y > point.Y) != (eraserOutline[j].Y > point.Y) && + point.X < (eraserOutline[j].X - eraserOutline[i].X) * (point.Y - eraserOutline[i].Y) / (eraserOutline[j].Y - eraserOutline[i].Y) + eraserOutline[i].X ) { + isInside = !isInside; + } + } + } + return isInside; + }; + /** * Determines if the Eraser tool has intersected with an ink stroke in the current freeform collection. * @returns an array of tuples containing the intersected ink DocumentView and the t-value where it was intersected @@ -880,94 +947,141 @@ export class CollectionFreeFormView extends CollectionSubView { + const eraserRadius = ActiveEraserWidth() + 3; + const eraserMin = { X: Math.min(lastPoint.X, currPoint.X) - eraserRadius, Y: Math.min(lastPoint.Y, currPoint.Y) - eraserRadius }; + const eraserMax = { X: Math.max(lastPoint.X, currPoint.X) + eraserRadius, Y: Math.max(lastPoint.Y, currPoint.Y) + eraserRadius}; + const strokeToTVals = new Map(); + const intersectingStrokes = this.childDocs + .map(doc => DocumentManager.Instance.getDocumentView(doc, this.DocumentView?.())) + .filter(inkView => inkView?.ComponentView instanceof InkingStroke) // filter to all inking strokes + .map(inkView => ({ inkViewBounds: inkView!.getBounds, inkStroke: inkView!.ComponentView as InkingStroke, inkView: inkView! })) + .filter( + ({ inkViewBounds }) => + inkViewBounds && // bounding box of eraser segment and ink stroke overlap + eraserMin.X <= inkViewBounds.right && + eraserMin.Y <= inkViewBounds.bottom && + eraserMax.X >= inkViewBounds.left && + eraserMax.Y >= inkViewBounds.top + ); + console.log("itersectnig strokes", intersectingStrokes); + intersectingStrokes.forEach(({ inkStroke, inkView }) => { + const { inkData } = inkStroke.inkScaledData(); + const prevPointInkSpace = inkStroke.ptFromScreen(lastPoint); + const currPointInkSpace = inkStroke.ptFromScreen(currPoint); + const eraserInkData = this.createEraserOutline(prevPointInkSpace, currPointInkSpace).inkData; + // add the ends of the stroke in as "intersections" + if (this.insideEraserOutline(eraserInkData, inkData[0])) { + strokeToTVals.set(inkView, [0]); + } + if (this.insideEraserOutline(eraserInkData, inkData[inkData.length - 1])) { + const inkList = strokeToTVals.get(inkView); + if (inkList !== undefined) { + inkList.push(Math.floor(inkData.length / 4) + 1); + } else { + strokeToTVals.set(inkView, [Math.floor(inkData.length / 4) + 1]); + } + } + for (var i = 0; i < inkData.length - 3; i += 4) { + // iterate over each segment of bezier curve + for (var j = 0; j < eraserInkData.length - 3; j += 4) { + const intersectCurve: Bezier = InkField.Segment(inkData, i); // other curve + const eraserCurve: Bezier = InkField.Segment(eraserInkData, j); // eraser curve + this.bintersects(intersectCurve, eraserCurve).forEach((val: string | number, k: number) => { + // Converting the Bezier.js Split type to a t-value number. + const t = +val.toString().split('/')[0]; + if (k % 2 === 0) { + // here, add to the map + const inkList = strokeToTVals.get(inkView); + if (inkList !== undefined) { + const tValOffset = ActiveEraserWidth() / 1030; // to prevent tVals from being added when too close, but scaled by eraser width + const inList = inkList.some(val => Math.abs(val - (t + Math.floor(i / 4))) <= tValOffset); + if (!inList) { + inkList.push(t + Math.floor(i / 4)); + } + } else { + strokeToTVals.set(inkView, [t + Math.floor(i / 4)]); + } + } + }); + } + } + }); + console.log("strokeToTVals", strokeToTVals); + return strokeToTVals; + }; + + /** + * Splits the passed in ink stroke at the intersection t values. Generally operates in pairs of t values, where + * the first t value is the start of the erased portion and the following t value is the end. + * @param ink the ink stroke DocumentView to split + * @param tVals all the t values to split the ink stroke at + * @returns a list of the new segments with the erased part removed + */ @action - radiusErase = (ink: DocumentView, startPt: { X: number; Y: number }, screenEraserPt: { X: number; Y: number }): Segment[] => { + radiusErase = (ink: DocumentView, tVals: number[]): Segment[] => { const segments: Segment[] = []; - const startInkCoords = ink.ComponentView?.ptFromScreen?.(startPt); - const inkCoords = ink.ComponentView?.ptFromScreen?.(screenEraserPt); // coordinates in ink space - if (!inkCoords || !startInkCoords) return []; - - var eraseSegment: Segment = []; // for eraser visualization const inkStroke = ink?.ComponentView as InkingStroke; - - const eraserStroke: InkData = inkStroke.splitByEraser(startInkCoords, inkCoords).inkData; - const strokeToTVals: Map = new Map(); - for (var i = 0; i < eraserStroke.length - 3; i += 4) { - eraseSegment.push(InkField.Segment(eraserStroke, i)); // for eraser visualization - this.getOtherInkIntersections(ink, i, eraserStroke, strokeToTVals); + const { inkData } = inkStroke.inkScaledData(); + var currSegment: Segment = []; + if (tVals.length % 2 !== 0) { // should always have even tVals + for (var i = 0; i < inkData.length - 3; i +=4) { + currSegment.push(InkField.Segment(inkData, i)); + } + if (currSegment.length > 0) { + segments.push(currSegment); + return segments; + } } - strokeToTVals.forEach((tVals, inkStroke) => { - var segment1: Segment = []; - var segment2: Segment = []; - const { inkData } = inkStroke.inkScaledData(); - tVals.sort(); - console.log('TVALS', inkStroke, tVals); - var hasSplit = false; - var continueErasing = false; - - // below is curve splitting logic - if (tVals.length) { - for (var i = 0; i < inkData.length - 3; i += 4) { - const inkSegment: Bezier = InkField.Segment(inkData, i); - const currCurveT = Math.floor(i / 4); + var continueErasing = false; + var firstSegment: Segment = []; + // early return if nothing to split on + if (tVals.length === 0 || (tVals.length === 1 && tVals[0] === 0)) { + return segments; + } - if (tVals.length === 2) { - if (tVals[0] > currCurveT && tVals[0] < currCurveT + 1) { - segment1.push(inkSegment.split(0, tVals[0] - currCurveT)); + for (var i = 0; i < inkData.length - 3; i += 4) { + const currCurveT = Math.floor(i/4); + const inkBezier: Bezier = InkField.Segment(inkData, i); + const segmentTs = tVals.filter(t => t >= currCurveT && t < currCurveT + 1); + + if (segmentTs.length > 0) { + for (var j = 0; j < segmentTs.length; j++) { + // if the first end of the segment is within the eraser + if (segmentTs[j] === 0 ) { + continueErasing = true; + } else if (segmentTs[j] === Math.floor(inkData.length / 4) + 1) { + break; + }else { + if (!continueErasing) { + currSegment.push(inkBezier.split(0, segmentTs[j] - currCurveT)); continueErasing = true; - if (tVals[1] > currCurveT && tVals[1] < currCurveT + 1) { - segment2.push(inkSegment.split(tVals[1] - currCurveT, 1)); - continueErasing = false; - hasSplit = true; - } - } else if (tVals[1] > currCurveT && tVals[1] < currCurveT + 1) { - segment2.push(inkSegment.split(tVals[1] - currCurveT, 1)); - continueErasing = false; - hasSplit = true; - } else if (!continueErasing) { - if (hasSplit) { - segment2.push(inkSegment); - } else { - segment1.push(inkSegment); - } - } - } else if (tVals.length === 1) { - if (tVals[0] > currCurveT && tVals[0] < currCurveT + 1) { - // this heuristic for determine which segment to keep is not quite right even though it will work most of the time. - // We should really store the eraser intersection normal in getOtherInkIntersections, - // and then test its dot product with the tangent of the stroke at the intersection point. - // if the dot product is positive, then erase the first part of the stroke, otherwise the second. - const leftDist = Utils.ptDistance({ x: inkCoords.X, y: inkCoords.Y }, inkSegment.points.lastElement()); - const rightDist = Utils.ptDistance({ x: inkCoords.X, y: inkCoords.Y }, inkSegment.points[0]); - const splits = inkSegment.split(tVals[0] - currCurveT); - if (leftDist < rightDist) { - // if it's on the first end - segment1.push(splits.left); - hasSplit = true; - } else { - segment1.push(splits.right); - hasSplit = true; - } } else { - if (tVals[0] < Math.floor(inkData.length / 4) - tVals[0] && hasSplit) { - segment1.push(inkSegment); - } else if (tVals[0] >= Math.floor(inkData.length / 4) - tVals[0] && !hasSplit) { - segment1.push(inkSegment); + continueErasing = false; + if (currSegment.length > 0) { + segments.push(currSegment); + if (firstSegment.length === 0) { + firstSegment = currSegment; + } + currSegment = []; } + currSegment.push(inkBezier.split(segmentTs[j] - currCurveT, 1)); } } } + } else { + if (!continueErasing) { // push the bezier piece if not in the eraser circle + currSegment.push(inkBezier); + } } - if (segment1.length && (Math.abs(segment1[0].points[0].x - segment1[0].points.lastElement().x) > 0.5 || Math.abs(segment1[0].points[0].y - segment1[0].points.lastElement().y) > 0.5)) { - segments.push(segment1); - } - if (segment2.length && (Math.abs(segment2[0].points[0].x - segment2[0].points.lastElement().x) > 0.5 || Math.abs(segment2[0].points[0].y - segment2[0].points.lastElement().y) > 0.5)) { - segments.push(segment2); + } + if (currSegment.length > 0) { + if (InkingStroke.IsClosed(inkData)) { + currSegment = currSegment.concat(firstSegment); } - }); - - segments.push(eraseSegment); // for eraser visualization + segments.push(currSegment); + } return segments; }; @@ -993,7 +1107,7 @@ export class CollectionFreeFormView extends CollectionSubView tVal + Math.floor(i / 4)); + currIntersects = currIntersects.filter(tVal => tVal > 0 && tVal < 1).map(tVal => tVal + Math.floor(i / 4)); if (currIntersects.length) { intersections = [...intersections, ...currIntersects]; for (var j = 0; j < currIntersects.length; j++) { @@ -1002,19 +1116,9 @@ export class CollectionFreeFormView extends CollectionSubView (value > 0.0001 && value < Math.floor(inkData.length / 4) ? index : -1)).filter(index => index !== -1); - - // Filter intersections and segmentIndexes based on validIndices - intersections = indices.map(index => intersections[index]); - segmentIndexes = indices.map(index => segmentIndexes[index]); - // intersections = intersections.slice(1, intersections.length ); // take the 0 intersection out - // segmentIndexes = segmentIndexes.slice(1, segmentIndexes.length); // same for indexes } if (intersections.length) { @@ -1068,7 +1172,7 @@ export class CollectionFreeFormView extends CollectionSubView 0 && segment2.length > 0) { + segment2 = segment2.concat(segment1); + segment1 = []; + } + // push 1 or both segments if they are not empty if (segment1.length && (Math.abs(segment1[0].points[0].x - segment1[0].points.lastElement().x) > 0.5 || Math.abs(segment1[0].points[0].y - segment1[0].points.lastElement().y) > 0.5)) { segments.push(segment1); @@ -1203,46 +1310,6 @@ export class CollectionFreeFormView extends CollectionSubView): Map => { - this.childDocs - .filter(doc => doc.type === DocumentType.INK && !doc.dontIntersect) - .forEach(doc => { - // InkingStroke of other ink strokes - const otherInk = otherInkDocView.ComponentView as InkingStroke; - // ink Data of other ink strokes - const { inkData: otherInkData } = otherInk?.inkScaledData() ?? { inkData: [] }; - for (var j = 0; j < otherInkData.length - 3; j += 4) { - const curve: Bezier = InkField.Segment(points, i); // eraser curve - const otherCurve: Bezier = InkField.Segment(otherInkData, j); // other curve - this.bintersects(otherCurve, curve).forEach((val: string | number, k: number) => { - // Converting the Bezier.js Split type to a t-value number. - const t = +val.toString().split('/')[0]; - if (k % 2 === 0) { - // here, add to the map - const inkList = strokeToTVals.get(otherInk); - if (inkList !== undefined) { - const inList = inkList.some(val => Math.abs(val - (t + Math.floor(j / 4))) <= 0.01); - if (!inList) { - inkList.push(t + Math.floor(j / 4)); - } - } else { - strokeToTVals.set(otherInk, [t + Math.floor(j / 4)]); - } - } - }); - } - }); - return strokeToTVals; - }; - @action zoom = (pointX: number, pointY: number, deltaY: number): void => { if (this.Document.isGroup || this.Document[(this._props.viewField ?? '_') + 'freeform_noZoom']) return; diff --git a/src/client/views/global/globalScripts.ts b/src/client/views/global/globalScripts.ts index 5c5f8de03..19741e2e0 100644 --- a/src/client/views/global/globalScripts.ts +++ b/src/client/views/global/globalScripts.ts @@ -341,10 +341,7 @@ function setActiveTool(tool: InkTool | GestureUtils.Gestures, keepPrim: boolean, Doc.UserDoc().activeEraserTool = tool; } // pen or eraser - if (Doc.ActiveTool === tool && tool === InkTool.Eraser) { - Doc.ActiveTool = InkTool.SegmentEraser; - console.log("erase click twice") - } else if (Doc.ActiveTool === tool && !GestureOverlay.Instance.InkShape && !keepPrim) { + if (Doc.ActiveTool === tool && !GestureOverlay.Instance.InkShape && !keepPrim) { Doc.ActiveTool = InkTool.None; } else { Doc.ActiveTool = tool as any; @@ -362,11 +359,6 @@ ScriptingGlobals.add(function activeEraserTool() { return StrCast(Doc.UserDoc().activeEraserTool, InkTool.StrokeEraser); }, 'returns the current eraser tool'); -// ScriptingGlobals.add(function setEraserProperty(option: 'eraseWidth', value: any, checkResult?: boolean) { -// setInk: (doc: Doc) => (doc[DocData].eraserWidth = NumCast(value)) -// InkingStroke.setEraserRadius(NumCast(value)); -// }) - // toggle: Set overlay status of selected document ScriptingGlobals.add(function setInkProperty(option: 'inkMask' | 'labels' | 'fillColor' | 'strokeWidth' | 'strokeColor' | 'eraserWidth', value: any, checkResult?: boolean) { const selected = SelectionManager.Docs.lastElement() ?? Doc.UserDoc(); -- cgit v1.2.3-70-g09d2 From 47e3e54543b2b9a613d0029435794d2265e2a952 Mon Sep 17 00:00:00 2001 From: IEatChili Date: Thu, 16 May 2024 16:49:55 -0700 Subject: feat: added image sorting --- package-lock.json | 41 +++++++ package.json | 2 + src/Utils.ts | 18 +++ src/client/apis/gpt/GPT.ts | 62 +++++++++- src/client/views/MainView.tsx | 2 + .../collectionFreeForm/ImageLabelHandler.scss | 44 +++++++ .../collectionFreeForm/ImageLabelHandler.tsx | 120 +++++++++++++++++++ .../collectionFreeForm/MarqueeOptionsMenu.tsx | 3 + .../collections/collectionFreeForm/MarqueeView.tsx | 129 +++++++++++++++++++-- 9 files changed, 412 insertions(+), 9 deletions(-) create mode 100644 src/client/views/collections/collectionFreeForm/ImageLabelHandler.scss create mode 100644 src/client/views/collections/collectionFreeForm/ImageLabelHandler.tsx (limited to 'src/client/views/MainView.tsx') diff --git a/package-lock.json b/package-lock.json index c417b7193..be51d2ca1 100644 --- a/package-lock.json +++ b/package-lock.json @@ -72,6 +72,7 @@ "class-transformer": "^0.5.1", "color": "^4.2.3", "colors": "^1.4.0", + "compute-cosine-similarity": "^1.1.0", "connect-flash": "^0.1.1", "connect-mongo": "^5.1.0", "cookie-parser": "^1.4.6", @@ -117,6 +118,7 @@ "image-data-uri": "^2.0.1", "image-size": "^1.0.2", "image-size-stream": "^1.1.0", + "is-plain-obj": "^4.1.0", "jimp": "^0.22.10", "jpeg-autorotate": "^9.0.0", "jquery": "^3.7.1", @@ -14725,6 +14727,35 @@ "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==", "dev": true }, + "node_modules/compute-cosine-similarity": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/compute-cosine-similarity/-/compute-cosine-similarity-1.1.0.tgz", + "integrity": "sha512-FXhNx0ILLjGi9Z9+lglLzM12+0uoTnYkHm7GiadXDAr0HGVLm25OivUS1B/LPkbzzvlcXz/1EvWg9ZYyJSdhTw==", + "dependencies": { + "compute-dot": "^1.1.0", + "compute-l2norm": "^1.1.0", + "validate.io-array": "^1.0.5", + "validate.io-function": "^1.0.2" + } + }, + "node_modules/compute-dot": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/compute-dot/-/compute-dot-1.1.0.tgz", + "integrity": "sha512-L5Ocet4DdMrXboss13K59OK23GXjiSia7+7Ukc7q4Bl+RVpIXK2W9IHMbWDZkh+JUEvJAwOKRaJDiFUa1LTnJg==", + "dependencies": { + "validate.io-array": "^1.0.3", + "validate.io-function": "^1.0.2" + } + }, + "node_modules/compute-l2norm": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/compute-l2norm/-/compute-l2norm-1.1.0.tgz", + "integrity": "sha512-6EHh1Elj90eU28SXi+h2PLnTQvZmkkHWySpoFz+WOlVNLz3DQoC4ISUHSV9n5jMxPHtKGJ01F4uu2PsXBB8sSg==", + "dependencies": { + "validate.io-array": "^1.0.3", + "validate.io-function": "^1.0.2" + } + }, "node_modules/concat-map": { "version": "0.0.1", "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz", @@ -33352,6 +33383,16 @@ "resolved": "https://registry.npmjs.org/valid-url/-/valid-url-1.0.9.tgz", "integrity": "sha512-QQDsV8OnSf5Uc30CKSwG9lnhMPe6exHtTXLRYX8uMwKENy640pU+2BgBL0LRbDh/eYRahNCS7aewCx0wf3NYVA==" }, + "node_modules/validate.io-array": { + "version": "1.0.6", + "resolved": "https://registry.npmjs.org/validate.io-array/-/validate.io-array-1.0.6.tgz", + "integrity": "sha512-DeOy7CnPEziggrOO5CZhVKJw6S3Yi7e9e65R1Nl/RTN1vTQKnzjfvks0/8kQ40FP/dsjRAOd4hxmJ7uLa6vxkg==" + }, + "node_modules/validate.io-function": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/validate.io-function/-/validate.io-function-1.0.2.tgz", + "integrity": "sha512-LlFybRJEriSuBnUhQyG5bwglhh50EpTL2ul23MPIuR1odjO7XaMLFV8vHGwp7AZciFxtYOeiSCT5st+XSPONiQ==" + }, "node_modules/validator": { "version": "13.11.0", "resolved": "https://registry.npmjs.org/validator/-/validator-13.11.0.tgz", diff --git a/package.json b/package.json index 65b06f65c..fc8365746 100644 --- a/package.json +++ b/package.json @@ -155,6 +155,7 @@ "class-transformer": "^0.5.1", "color": "^4.2.3", "colors": "^1.4.0", + "compute-cosine-similarity": "^1.1.0", "connect-flash": "^0.1.1", "connect-mongo": "^5.1.0", "cookie-parser": "^1.4.6", @@ -200,6 +201,7 @@ "image-data-uri": "^2.0.1", "image-size": "^1.0.2", "image-size-stream": "^1.1.0", + "is-plain-obj": "^4.1.0", "jimp": "^0.22.10", "jpeg-autorotate": "^9.0.0", "jquery": "^3.7.1", diff --git a/src/Utils.ts b/src/Utils.ts index 38325a463..0353a2ff7 100644 --- a/src/Utils.ts +++ b/src/Utils.ts @@ -925,3 +925,21 @@ export function dateRangeStrToDates(dateStr: string) { return [new Date(fromYear, fromMonth, fromDay), new Date(toYear, toMonth, toDay)]; } + +export async function convertImageToBase64(url: string): Promise { + try { + const response = await fetch(url); // Fetch the image + if (!response.ok) throw new Error('Network response was not ok'); + const blob = await response.blob(); // Convert response to Blob + + return new Promise((resolve, reject) => { + const reader = new FileReader(); + reader.readAsDataURL(blob); // Read blob as DataURL (Base64) + reader.onloadend = () => resolve(reader.result as string); // Resolve promise with Base64 string + reader.onerror = error => reject(error); // Reject promise on error + }); + } catch (error) { + console.error('Error:', error); + throw error; // Rethrow the error after logging it + } +} diff --git a/src/client/apis/gpt/GPT.ts b/src/client/apis/gpt/GPT.ts index fb51278ae..4240c07b8 100644 --- a/src/client/apis/gpt/GPT.ts +++ b/src/client/apis/gpt/GPT.ts @@ -68,4 +68,64 @@ const gptImageCall = async (prompt: string, n?: number) => { } }; -export { gptAPICall, gptImageCall, GPTCallType }; +const gptGetEmbedding = async (src: string): Promise => { + try { + const configuration: ClientOptions = { + apiKey: process.env.OPENAI_KEY, + dangerouslyAllowBrowser: true, + }; + const openai = new OpenAI(configuration); + const embeddingResponse = await openai.embeddings.create({ + model: 'text-embedding-3-large', + input: [src], + encoding_format: 'float', + dimensions: 256, + }); + + // Assume the embeddingResponse structure is correct; adjust based on actual API response + const embedding = embeddingResponse.data[0].embedding; + return embedding; + } catch (err) { + console.log(err); + return []; + } +}; + +const gptImageLabel = async (src: string): Promise => { + try { + const configuration: ClientOptions = { + apiKey: process.env.OPENAI_KEY, + dangerouslyAllowBrowser: true, + }; + + const openai = new OpenAI(configuration); + const response = await openai.chat.completions.create({ + model: 'gpt-4-vision-preview', + messages: [ + { + role: 'user', + content: [ + { type: 'text', text: 'Give three labels to describe this image.' }, + { + type: 'image_url', + image_url: { + url: `${src}`, + detail: 'low', + }, + }, + ], + }, + ], + }); + if (response.choices[0].message.content) { + return response.choices[0].message.content; + } else { + return 'Missing labels'; + } + } catch (err) { + console.log(err); + return 'Error connecting with API'; + } +}; + +export { gptAPICall, gptImageCall, GPTCallType, gptImageLabel, gptGetEmbedding }; diff --git a/src/client/views/MainView.tsx b/src/client/views/MainView.tsx index 207db2c99..f5d0539c2 100644 --- a/src/client/views/MainView.tsx +++ b/src/client/views/MainView.tsx @@ -72,6 +72,7 @@ import { PresBox } from './nodes/trails'; import { AnchorMenu } from './pdf/AnchorMenu'; import { GPTPopup } from './pdf/GPTPopup/GPTPopup'; import { TopBar } from './topbar/TopBar'; +import { ImageLabelHandler } from './collections/collectionFreeForm/ImageLabelHandler'; const { default: { LEFT_MENU_WIDTH, TOPBAR_HEIGHT } } = require('./global/globalCssVariables.module.scss'); // prettier-ignore const _global = (window /* browser */ || global) /* node */ as any; @@ -1030,6 +1031,7 @@ export class MainView extends ObservableReactComponent<{}> { + diff --git a/src/client/views/collections/collectionFreeForm/ImageLabelHandler.scss b/src/client/views/collections/collectionFreeForm/ImageLabelHandler.scss new file mode 100644 index 000000000..e7413bf8e --- /dev/null +++ b/src/client/views/collections/collectionFreeForm/ImageLabelHandler.scss @@ -0,0 +1,44 @@ +#label-handler { + display: flex; + flex-direction: column; + align-items: center; + + > div:first-child { + display: flex; // Puts the input and button on the same row + align-items: center; // Vertically centers items in the flex container + + input { + color: black; + } + + .IconButton { + margin-left: 8px; // Adds space between the input and the icon button + width: 19px; + } + } + + > div:not(:first-of-type) { + display: flex; + flex-direction: column; + align-items: center; // Centers the content vertically in the flex container + width: 100%; + + > div { + display: flex; + justify-content: space-between; // Puts the content and delete button on opposite ends + align-items: center; + width: 100%; + margin-top: 8px; // Adds space between label rows + + p { + text-align: center; // Centers the text of the paragraph + flex-grow: 1; // Allows the paragraph to grow and occupy the available space + } + + .IconButton { + // Styling for the delete button + margin-left: auto; // Pushes the button to the far right + } + } + } +} diff --git a/src/client/views/collections/collectionFreeForm/ImageLabelHandler.tsx b/src/client/views/collections/collectionFreeForm/ImageLabelHandler.tsx new file mode 100644 index 000000000..46bc3d946 --- /dev/null +++ b/src/client/views/collections/collectionFreeForm/ImageLabelHandler.tsx @@ -0,0 +1,120 @@ +import { FontAwesomeIcon } from '@fortawesome/react-fontawesome'; +import { IconButton } from 'browndash-components'; +import { action, makeObservable, observable } from 'mobx'; +import { observer } from 'mobx-react'; +import React from 'react'; +import { SettingsManager } from '../../../util/SettingsManager'; +import { ObservableReactComponent } from '../../ObservableReactComponent'; +import { MarqueeOptionsMenu } from './MarqueeOptionsMenu'; +import './ImageLabelHandler.scss'; + +@observer +export class ImageLabelHandler extends ObservableReactComponent<{}> { + static Instance: ImageLabelHandler; + + @observable _display: boolean = false; + @observable _pageX: number = 0; + @observable _pageY: number = 0; + @observable _yRelativeToTop: boolean = true; + @observable _currentLabel: string = ''; + @observable _labelGroups: string[] = []; + + constructor(props: any) { + super(props); + makeObservable(this); + ImageLabelHandler.Instance = this; + console.log('Instantiated label handler!'); + } + + @action + displayLabelHandler = (x: number, y: number) => { + this._pageX = x; + this._pageY = y; + this._display = true; + this._labelGroups = []; + }; + + @action + hideLabelhandler = () => { + this._display = false; + this._labelGroups = []; + }; + + @action + addLabel = (label: string) => { + label = label.toUpperCase().trim(); + if (label.length > 0) { + if (!this._labelGroups.includes(label)) { + this._labelGroups = [...this._labelGroups, label]; + } + } + }; + + @action + removeLabel = (label: string) => { + label = label.toUpperCase(); + this._labelGroups = this._labelGroups.filter(group => group !== label); + }; + + @action + groupImages = () => { + MarqueeOptionsMenu.Instance.groupImages(); + this._display = false; + }; + + render() { + if (this._display) { + return ( +
+
+ } color={MarqueeOptionsMenu.Instance.userColor} style={{ width: '19px' }} /> + + { + const input = document.getElementById('new-label') as HTMLInputElement; + const newLabel = input.value; + this.addLabel(newLabel); + this._currentLabel = ''; + input.value = ''; + }} + icon={} + color={MarqueeOptionsMenu.Instance.userColor} + style={{ width: '19px' }} + /> + } color={MarqueeOptionsMenu.Instance.userColor} style={{ width: '19px' }} /> +
+
+ {this._labelGroups.map(group => { + return ( +
+

{group}

+ { + this.removeLabel(group); + }} + icon={'x'} + color={MarqueeOptionsMenu.Instance.userColor} + style={{ width: '19px' }} + /> +
+ ); + })} +
+
+ ); + } else { + return <>; + } + } +} diff --git a/src/client/views/collections/collectionFreeForm/MarqueeOptionsMenu.tsx b/src/client/views/collections/collectionFreeForm/MarqueeOptionsMenu.tsx index 79cc534dc..414858aee 100644 --- a/src/client/views/collections/collectionFreeForm/MarqueeOptionsMenu.tsx +++ b/src/client/views/collections/collectionFreeForm/MarqueeOptionsMenu.tsx @@ -17,6 +17,8 @@ export class MarqueeOptionsMenu extends AntimodeMenu { public showMarquee: () => void = unimplementedFunction; public hideMarquee: () => void = unimplementedFunction; public pinWithView: (e: KeyboardEvent | React.PointerEvent | undefined) => void = unimplementedFunction; + public classifyImages: (e: React.MouseEvent | undefined) => void = unimplementedFunction; + public groupImages: () => void = unimplementedFunction; public isShown = () => this._opacity > 0; constructor(props: any) { super(props); @@ -37,6 +39,7 @@ export class MarqueeOptionsMenu extends AntimodeMenu { } color={this.userColor} /> } color={this.userColor} /> } color={this.userColor} /> + } color={this.userColor} /> ); return this.getElement(buttons); diff --git a/src/client/views/collections/collectionFreeForm/MarqueeView.tsx b/src/client/views/collections/collectionFreeForm/MarqueeView.tsx index 6b3a56b0b..0918ae293 100644 --- a/src/client/views/collections/collectionFreeForm/MarqueeView.tsx +++ b/src/client/views/collections/collectionFreeForm/MarqueeView.tsx @@ -1,15 +1,15 @@ import { action, computed, makeObservable, observable } from 'mobx'; import { observer } from 'mobx-react'; import * as React from 'react'; -import { Utils, intersectRect, lightOrDark, returnFalse } from '../../../../Utils'; -import { Doc, Opt } from '../../../../fields/Doc'; +import { Utils, intersectRect, lightOrDark, returnFalse, convertImageToBase64 } from '../../../../Utils'; +import { Doc, FieldResult, NumListCast, Opt } from '../../../../fields/Doc'; import { AclAdmin, AclAugment, AclEdit, DocData } from '../../../../fields/DocSymbols'; import { Id } from '../../../../fields/FieldSymbols'; import { InkData, InkField, InkTool } from '../../../../fields/InkField'; import { List } from '../../../../fields/List'; import { RichTextField } from '../../../../fields/RichTextField'; import { Cast, FieldValue, NumCast, StrCast } from '../../../../fields/Types'; -import { ImageField } from '../../../../fields/URLField'; +import { ImageField, URLField } from '../../../../fields/URLField'; import { GetEffectiveAcl } from '../../../../fields/util'; import { CognitiveServices } from '../../../cognitive_services/CognitiveServices'; import { DocumentType } from '../../../documents/DocumentTypes'; @@ -28,6 +28,10 @@ import { FormattedTextBox } from '../../nodes/formattedText/FormattedTextBox'; import { SubCollectionViewProps } from '../CollectionSubView'; import { MarqueeOptionsMenu } from './MarqueeOptionsMenu'; import './MarqueeView.scss'; +import { ObjectField } from '../../../../fields/ObjectField'; +import { gptGetEmbedding, gptImageLabel } from '../../../apis/gpt/GPT'; +import { ImageLabelHandler } from './ImageLabelHandler'; +import { listSpec } from '../../../../fields/Schema'; interface MarqueeViewProps { getContainerTransform: () => Transform; getTransform: () => Transform; @@ -64,11 +68,13 @@ export class MarqueeView extends ObservableReactComponent { + const selected = this.marqueeSelect(false, DocumentType.IMG); + this._selectedDocs = selected; + + const imagePromises = selected.map(doc => { + let href = (doc['data'] as URLField).url.href; + let hrefParts = href.split('.'); + let hrefComplete = `${hrefParts[0]}_o.${hrefParts[1]}`; + return convertImageToBase64(hrefComplete).then(hrefBase64 => { + return gptImageLabel(hrefBase64).then(response => { + console.log(response); + const labels = response.split('\n'); + console.log(labels); + doc.image_labels = new List(Array.from(labels!)); + return Promise.all(labels!.map(label => gptGetEmbedding(label))).then(embeddings => { + return { doc, embeddings }; + }); + }); + }); + }); + + let docsAndEmbeddings = await Promise.all(imagePromises); + + for (const docAndEmbedding of docsAndEmbeddings) { + if (Array.isArray(docAndEmbedding.embeddings)) { + let doc = docAndEmbedding.doc; + for (let i = 0; i < 3; i++) { + doc[`label_embedding_${i + 1}`] = new List(docAndEmbedding.embeddings[i]); + } + } + } + + if (e) { + ImageLabelHandler.Instance.displayLabelHandler(e.pageX, e.pageY); + } + }); + + /** + * Groups images to most similar labels. + */ + @undoBatch + groupImages = action(async () => { + const labelGroups: string[] = ImageLabelHandler.Instance._labelGroups; + const labelToCollection: Map = new Map(); + const labelToEmbedding: Map = new Map(); + var similarity = require('compute-cosine-similarity'); + + // Create new collections associated with each label and get the embeddings for the labels. + for (const label of labelGroups) { + const newCollection = this.getCollection([], undefined, false); + newCollection._freeform_panX = this.Bounds.left + this.Bounds.width / 2; + newCollection._freeform_panY = this.Bounds.top + this.Bounds.height / 2; + labelToCollection.set(label, newCollection); + this._props.addDocument?.(newCollection); + const labelEmbedding = await gptGetEmbedding(label); + if (Array.isArray(labelEmbedding)) { + labelToEmbedding.set(label, labelEmbedding); + } + } + + // For each image, loop through the labels, and calculate similarity. Associate it with the + // most similar one. + this._selectedDocs.forEach(doc => { + let mostSimilarLabel: string | undefined; + let maxSimilarity: number = 0; + const embeddingAsList1 = NumListCast(doc.label_embedding_1); + const embeddingAsList2 = NumListCast(doc.label_embedding_2); + const embeddingAsList3 = NumListCast(doc.label_embedding_3); + + labelGroups.forEach(label => { + let curSimilarity1 = similarity(labelToEmbedding.get(label)!, Array.from(embeddingAsList1)); + let curSimilarity2 = similarity(labelToEmbedding.get(label)!, Array.from(embeddingAsList2)); + let curSimilarity3 = similarity(labelToEmbedding.get(label)!, Array.from(embeddingAsList3)); + let maxCurSimilarity = Math.max(curSimilarity1, curSimilarity2, curSimilarity3); + if (maxCurSimilarity >= 0.3 && maxCurSimilarity > maxSimilarity) { + mostSimilarLabel = label; + maxSimilarity = maxCurSimilarity; + } + + console.log('Doc with labels ' + doc.image_labels + 'has similarity score ' + maxCurSimilarity + ' to ' + mostSimilarLabel); + }); + + if (mostSimilarLabel) { + Doc.AddDocToList(labelToCollection.get(mostSimilarLabel)!, undefined, doc); + this._props.removeDocument?.(doc); + } + }); + }); + @undoBatch syntaxHighlight = action((e: KeyboardEvent | React.PointerEvent | undefined) => { const selected = this.marqueeSelect(false); @@ -574,7 +676,10 @@ export class MarqueeView extends ObservableReactComponent { const layoutDoc = Doc.Layout(doc); @@ -584,11 +689,19 @@ export class MarqueeView extends ObservableReactComponent !doc.z && !doc._lockedPosition) - .map(selectFunc); + if (docType) { + this._props + .activeDocuments() + .filter(doc => !doc.z && !doc._lockedPosition && doc['type'] === docType) + .map(selectFunc); + } else { + this._props + .activeDocuments() + .filter(doc => !doc.z && !doc._lockedPosition) + .map(selectFunc); + } if (!selection.length && selectBackgrounds) this._props .activeDocuments() -- cgit v1.2.3-70-g09d2