From 0c33bc8033c9877abbe6e4074a687559bc4948d0 Mon Sep 17 00:00:00 2001
From: eleanor-park <eleanor_park@brown.edu>
Date: Tue, 23 Apr 2024 15:16:06 -0400
Subject: erase multiple segments bug

---
 src/client/views/MainView.tsx | 3 +++
 1 file changed, 3 insertions(+)

(limited to 'src/client/views/MainView.tsx')

diff --git a/src/client/views/MainView.tsx b/src/client/views/MainView.tsx
index 58b8d255a..56db0c488 100644
--- a/src/client/views/MainView.tsx
+++ b/src/client/views/MainView.tsx
@@ -358,6 +358,9 @@ export class MainView extends ObservableReactComponent<{}> {
                 fa.faCut,
                 fa.faEllipsisV,
                 fa.faEraser,
+                fa.faDeleteLeft,
+                fa.faXmarksLines,
+                fa.faCircleXmark,
                 fa.faExclamation,
                 fa.faFileAlt,
                 fa.faFileAudio,
-- 
cgit v1.2.3-70-g09d2


From 4a01680bd22a0652dbdd0da5c3a7167ca8117440 Mon Sep 17 00:00:00 2001
From: eleanor-park <eleanor_park@brown.edu>
Date: Wed, 8 May 2024 22:22:29 -0400
Subject: close curve and bounding box fixes

---
 src/client/util/CurrentUserUtils.ts                |  10 +-
 src/client/views/MainView.tsx                      |   1 +
 .../collectionFreeForm/CollectionFreeFormView.tsx  | 347 ++++++++++++---------
 src/client/views/global/globalScripts.ts           |  10 +-
 4 files changed, 214 insertions(+), 154 deletions(-)

(limited to 'src/client/views/MainView.tsx')

diff --git a/src/client/util/CurrentUserUtils.ts b/src/client/util/CurrentUserUtils.ts
index b1673ff1c..3ee1b42aa 100644
--- a/src/client/util/CurrentUserUtils.ts
+++ b/src/client/util/CurrentUserUtils.ts
@@ -752,13 +752,13 @@ pie title Minerals in my tap water
         return [
             { title: "Pen",    toolTip: "Pen (Ctrl+P)",                     btnType: ButtonType.ToggleButton, icon: "pen-nib",    toolType: "pen",                            scripts: {onClick:'{ return setActiveTool(this.toolType, false, _readOnly_);}' }},
             { title: "Write",  toolTip: "Write (Ctrl+Shift+P)",             btnType: ButtonType.ToggleButton, icon: "pen",        toolType: "write",                          scripts: {onClick:'{ return setActiveTool(this.toolType, false, _readOnly_);}' }, funcs: {hidden:"IsNoviceMode()" }},
-            { title: "Eraser", toolTip: "Eraser (Ctrl+E)",                  btnType: ButtonType.MultiToggleButton,                toolType:"eraser",                          scripts: {onClick: '{ return setActiveTool(this.toolType, false, _readOnly_);}'},
+            { title: "Eraser", toolTip: "Eraser (Ctrl+E)",                  btnType: ButtonType.MultiToggleButton,                                                            scripts: {onClick: '{ return setActiveTool(this.toolType, false, _readOnly_);}'}, funcs: {toolType:"activeEraserTool()"},
               subMenu: [
                     { title: "Stroke",   toolTip: "Stroke Erase",           btnType: ButtonType.ToggleButton, icon: "eraser",      toolType:InkTool.StrokeEraser, ignoreClick: true, scripts: {onClick: '{ return setActiveTool(this.toolType, false, _readOnly_);}'} },
-                    { title: "Segment",  toolTip: "Segment Erase",          btnType: ButtonType.ToggleButton, icon: "xmarks-lines",toolType:InkTool.SegmentEraser,ignoreClick: true, scripts: {onClick: '{ return setActiveTool(this.toolType, false, _readOnly_);}'} },
+                    { title: "Segment",  toolTip: "Segment Erase",          btnType: ButtonType.ToggleButton, icon: "xmark",toolType:InkTool.SegmentEraser,ignoreClick: true, scripts: {onClick: '{ return setActiveTool(this.toolType, false, _readOnly_);}'} },
                     { title: "Radius",   toolTip: "Radius Erase",           btnType: ButtonType.ToggleButton, icon: "circle-xmark",toolType:InkTool.RadiusEraser, ignoreClick: true, scripts: {onClick: '{ return setActiveTool(this.toolType, false, _readOnly_);}'} },
-            ]},  
-            { title: "Eraser Width",  toolTip: "Eraser Width",              btnType: ButtonType.NumberSliderButton,               toolType: "eraserWidth", ignoreClick: true, scripts: {script: '{ return setInkProperty(this.toolType, value, _readOnly_);}'}, numBtnMin: 1, funcs: {hidden:"isRadiusEraser()" }},
+            ]},
+            { title: "Eraser Width",  toolTip: "Eraser Width",              btnType: ButtonType.NumberSliderButton,               toolType: "eraserWidth", ignoreClick: true, scripts: {script: '{ return setInkProperty(this.toolType, value, _readOnly_);}'}, numBtnMin: 1, funcs: {hidden:"NotRadiusEraser()"}},
             { title: "Circle", toolTip: "Circle (double tap to lock mode)", btnType: ButtonType.ToggleButton, icon: "circle",     toolType:GestureUtils.Gestures.Circle,      scripts: {onClick:`{ return setActiveTool(this.toolType, false, _readOnly_);}`, onDoubleClick:`{ return setActiveTool(this.toolType, true, _readOnly_);}`} },
             { title: "Square", toolTip: "Square (double tap to lock mode)", btnType: ButtonType.ToggleButton, icon: "square",     toolType:GestureUtils.Gestures.Rectangle,   scripts: {onClick:`{ return setActiveTool(this.toolType, false, _readOnly_);}`, onDoubleClick:`{ return setActiveTool(this.toolType, true, _readOnly_);}`} },
             { title: "Line",   toolTip: "Line (double tap to lock mode)",   btnType: ButtonType.ToggleButton, icon: "minus",      toolType:GestureUtils.Gestures.Line,        scripts: {onClick:`{ return setActiveTool(this.toolType, false, _readOnly_);}`, onDoubleClick:`{ return setActiveTool(this.toolType, true, _readOnly_);}`} },
@@ -1119,7 +1119,7 @@ pie title Minerals in my tap water
 ScriptingGlobals.add(function MySharedDocs() { return Doc.MySharedDocs; }, "document containing all shared Docs");
 ScriptingGlobals.add(function IsExploreMode() { return SnappingManager.ExploreMode; }, "is Dash in exploration mode");
 ScriptingGlobals.add(function IsNoviceMode() { return Doc.noviceMode; }, "is Dash in novice mode");
-ScriptingGlobals.add(function isRadiusEraser() { return !(Doc.ActiveTool === InkTool.RadiusEraser); }, "is the eraser selected");
+ScriptingGlobals.add(function NotRadiusEraser() { return Doc.ActiveTool !== InkTool.RadiusEraser; }, "is the active tool anything but the radius eraser");
 ScriptingGlobals.add(function toggleComicMode() { Doc.UserDoc().renderStyle = Doc.UserDoc().renderStyle === "comic" ? undefined : "comic"; }, "switches between comic and normal document rendering");
 ScriptingGlobals.add(function importDocument() { return CurrentUserUtils.importDocument(); }, "imports files from device directly into the import sidebar");
 ScriptingGlobals.add(function setInkToolDefaults() { Doc.ActiveTool = InkTool.None; });
diff --git a/src/client/views/MainView.tsx b/src/client/views/MainView.tsx
index 56db0c488..d0b3221b4 100644
--- a/src/client/views/MainView.tsx
+++ b/src/client/views/MainView.tsx
@@ -361,6 +361,7 @@ export class MainView extends ObservableReactComponent<{}> {
                 fa.faDeleteLeft,
                 fa.faXmarksLines,
                 fa.faCircleXmark,
+                fa.faXmark,
                 fa.faExclamation,
                 fa.faFileAlt,
                 fa.faFileAudio,
diff --git a/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx b/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx
index eedee0b18..2e174be30 100644
--- a/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx
+++ b/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx
@@ -1,5 +1,7 @@
+import { inside } from '@turf/turf';
 import { Bezier } from 'bezier-js';
 import { Colors } from 'browndash-components';
+import { validationResult } from 'express-validator';
 import { action, computed, IReactionDisposer, makeObservable, observable, reaction, runInAction } from 'mobx';
 import { observer } from 'mobx-react';
 import { computedFn } from 'mobx-utils';
@@ -35,7 +37,7 @@ import { Timeline } from '../../animationtimeline/Timeline';
 import { ContextMenu } from '../../ContextMenu';
 import { GestureOverlay } from '../../GestureOverlay';
 import { CtrlKey } from '../../GlobalKeyHandler';
-import { ActiveInkWidth, InkingStroke, SetActiveInkColor, SetActiveInkWidth } from '../../InkingStroke';
+import { ActiveEraserWidth, ActiveInkWidth, InkingStroke, SetActiveInkColor, SetActiveInkWidth } from '../../InkingStroke';
 import { LightboxView } from '../../LightboxView';
 import { CollectionFreeFormDocumentView } from '../../nodes/CollectionFreeFormDocumentView';
 import { SchemaCSVPopUp } from '../../nodes/DataVizBox/SchemaCSVPopUp';
@@ -618,11 +620,15 @@ export class CollectionFreeFormView extends CollectionSubView<Partial<collection
                         break; // the GestureOverlay handles ink stroke input -- either as gestures, or drying as ink strokes that are added to document views
                     case InkTool.StrokeEraser:
                     case InkTool.SegmentEraser:
-                    case InkTool.RadiusEraser:
                         this._batch = UndoManager.StartBatch('collectionErase');
                         this._eraserPts.length = 0;
                         setupMoveUpEvents(this, e, this.onEraserMove, this.onEraserUp, emptyFunction);
                         break;
+                    case InkTool.RadiusEraser:
+                        this._batch = UndoManager.StartBatch('collectionErase');
+                        this._eraserPts.length = 0;
+                        setupMoveUpEvents(this, e, this.onRadiusEraserMove, this.onEraserUp, emptyFunction);
+                        break;
                     case InkTool.None:
                         if (!(this._props.layoutEngine?.() || StrCast(this.layoutDoc._layoutEngine))) {
                             this._hitCluster = this.pickCluster(this.screenToFreeformContentsXf.transformPoint(e.clientX, e.clientY));
@@ -734,7 +740,6 @@ export class CollectionFreeFormView extends CollectionSubView<Partial<collection
      * Erases strokes by intersecting them with an invisible "eraser stroke".
      *  By default this iterates through all intersected ink strokes, determines their segmentation, draws back the non-intersected segments,
      * and deletes the original stroke.
-     *  However, if Shift is held, then no segmentation is done -- instead any intersected stroke is deleted in its entirety.
      */
     @action
     onEraserMove = (e: PointerEvent, down: number[], delta: number[]) => {
@@ -750,10 +755,7 @@ export class CollectionFreeFormView extends CollectionSubView<Partial<collection
                 // create a new curve by appending all curves of the current segment together in order to render a single new stroke.
                 if (Doc.ActiveTool !== InkTool.StrokeEraser) {
                     this._eraserLock++;
-                    const segments =
-                        Doc.ActiveTool === InkTool.SegmentEraser
-                            ? this.segmentErase(intersect.inkView, intersect.t) // intersect.t is where the eraser intersected the ink stroke - want to remove the segment that starts at the intersection just before this t value and goes to the one just after it
-                            : this.radiusErase(intersect.inkView, { X: this._eraserPts[0][0], Y: this._eraserPts[0][1] }, currPoint);
+                    const segments = this.segmentErase(intersect.inkView, intersect.t); // intersect.t is where the eraser intersected the ink stroke - want to remove the segment that starts at the intersection just before this t value and goes to the one just after it
                     segments?.forEach(segment =>
                         this.forceStrokeGesture(
                             e,
@@ -764,12 +766,44 @@ export class CollectionFreeFormView extends CollectionSubView<Partial<collection
                     setTimeout(() => this._eraserLock--);
                 }
                 // Lower ink opacity to give the user a visual indicator of deletion.
-                intersect.inkView.layoutDoc.opacity = 0.2;
+                intersect.inkView.layoutDoc.opacity = 0;
                 intersect.inkView.layoutDoc.dontIntersect = true;
             }
         });
         return false;
     };
+
+    @action
+    onRadiusEraserMove = (e: PointerEvent, down: number[], delta: number[]) => {
+        const currPoint = { X: e.clientX, Y: e.clientY };
+        this._eraserPts.push([currPoint.X, currPoint.Y]);
+        this._eraserPts = this._eraserPts.slice(Math.max(0, this._eraserPts.length - 5));
+        if (this._eraserLock) return false; // bcz: should be fixed by putting it on a queue to be processed after the last eraser movement is processed.
+        const strokeMap: Map<DocumentView, number[]> = this.getRadiusEraserIntersections({ X: currPoint.X - delta[0], Y: currPoint.Y - delta[1] }, currPoint);
+        strokeMap.forEach((intersects, stroke) => {
+            if (!this._deleteList.includes(stroke)) {
+                this._deleteList.push(stroke);
+                SetActiveInkWidth(StrCast(stroke.Document.stroke_width?.toString()) || '1');
+                SetActiveInkColor(StrCast(stroke.Document.color?.toString()) || 'black');
+                this._eraserLock++;
+                // create a new curve by appending all curves of the current segment together in order to render a single new stroke.
+                const segments = this.radiusErase(stroke, intersects.sort());
+                segments?.forEach(segment =>
+                    this.forceStrokeGesture(
+                        e,
+                        GestureUtils.Gestures.Stroke,
+                        segment.reduce((data, curve) => [...data, ...curve.points.map(p => stroke.ComponentView?.ptToScreen?.({ X: p.x, Y: p.y }) ?? { X: 0, Y: 0 })], [] as PointData[])
+                    )
+                );
+                setTimeout(() => this._eraserLock--);
+            }
+            // Lower ink opacity to give the user a visual indicator of deletion.
+            stroke.layoutDoc.opacity = 0;
+            stroke.layoutDoc.dontIntersect = true;
+        });
+        return false;
+    };
+
     forceStrokeGesture = (e: PointerEvent, gesture: GestureUtils.Gestures, points: InkData, text?: any) => {
         this.onGesture(e, new GestureUtils.GestureEvent(gesture, points, GestureOverlay.getBounds(points), text));
     };
@@ -788,7 +822,7 @@ export class CollectionFreeFormView extends CollectionSubView<Partial<collection
     };
 
     createEraserOutline = (startInkCoordsIn: { X: number; Y: number }, endInkCoordsIn: { X: number; Y: number }) => {
-        const radius = ActiveEraserWidth() / 2 + 3; // reduce values to avoid extreme radii
+        const radius = ActiveEraserWidth() + 3; // add 3 to avoid eraser being too thin
         const c = 0.551915024494; // circle tangent length to side ratio
         const movement = { x: endInkCoordsIn.X - startInkCoordsIn.X, y: endInkCoordsIn.Y - startInkCoordsIn.Y };
         const moveLen = Math.sqrt(movement.x ** 2 + movement.y ** 2);
@@ -836,6 +870,39 @@ export class CollectionFreeFormView extends CollectionSubView<Partial<collection
         ]);
     };
 
+    /**
+     * Ray-tracing algorithm to determine whether a point is inside the eraser outline
+     * @param eraserOutline 
+     * @param point 
+     * @returns 
+     */
+    insideEraserOutline = (eraserOutline: InkData, point: { X: number; Y: number }) => {
+        var isInside = false;
+        if (!isNaN(eraserOutline[0].X) && !isNaN(eraserOutline[0].Y)) {
+            var minX = eraserOutline[0].X, maxX = eraserOutline[0].X;
+            var minY = eraserOutline[0].Y, maxY = eraserOutline[0].Y;
+            for (var i = 1; i < eraserOutline.length; i++) {
+                const currPoint: {X: number, Y: number} = eraserOutline[i];
+                minX = Math.min(currPoint.X, minX);
+                maxX = Math.max(currPoint.X, maxX);
+                minY = Math.min(currPoint.Y, minY);
+                maxY = Math.max(currPoint.Y, maxY);
+            }
+
+            if (point.X < minX || point.X > maxX || point.Y < minY || point.Y > maxY) {
+                return false;
+            }
+
+            for (var i = 0, j = eraserOutline.length - 1; i < eraserOutline.length; j = i, i++) {
+                if ((eraserOutline[i].Y > point.Y) != (eraserOutline[j].Y > point.Y) &&
+                    point.X < (eraserOutline[j].X - eraserOutline[i].X) * (point.Y - eraserOutline[i].Y) / (eraserOutline[j].Y - eraserOutline[i].Y) + eraserOutline[i].X ) {
+                    isInside = !isInside;
+                }
+            }
+        }
+        return isInside;
+    };
+
     /**
      * Determines if the Eraser tool has intersected with an ink stroke in the current freeform collection.
      * @returns an array of tuples containing the intersected ink DocumentView and the t-value where it was intersected
@@ -880,94 +947,141 @@ export class CollectionFreeFormView extends CollectionSubView<Partial<collection
             );
     };
 
+    getRadiusEraserIntersections = (lastPoint: { X: number; Y: number }, currPoint: { X: number; Y: number }) => {
+        const eraserRadius = ActiveEraserWidth() + 3;
+        const eraserMin = { X: Math.min(lastPoint.X, currPoint.X) - eraserRadius, Y: Math.min(lastPoint.Y, currPoint.Y) - eraserRadius };
+        const eraserMax = { X: Math.max(lastPoint.X, currPoint.X) + eraserRadius, Y: Math.max(lastPoint.Y, currPoint.Y) + eraserRadius};
+        const strokeToTVals = new Map<DocumentView, number[]>();
+        const intersectingStrokes = this.childDocs
+            .map(doc => DocumentManager.Instance.getDocumentView(doc, this.DocumentView?.()))
+            .filter(inkView => inkView?.ComponentView instanceof InkingStroke) // filter to all inking strokes
+            .map(inkView => ({ inkViewBounds: inkView!.getBounds, inkStroke: inkView!.ComponentView as InkingStroke, inkView: inkView! }))
+            .filter(
+                ({ inkViewBounds }) =>
+                    inkViewBounds && // bounding box of eraser segment and ink stroke overlap
+                    eraserMin.X <= inkViewBounds.right &&
+                    eraserMin.Y <= inkViewBounds.bottom &&
+                    eraserMax.X >= inkViewBounds.left &&
+                    eraserMax.Y >= inkViewBounds.top
+            );
+        console.log("itersectnig strokes", intersectingStrokes);
+        intersectingStrokes.forEach(({ inkStroke, inkView }) => {
+            const { inkData } = inkStroke.inkScaledData();
+            const prevPointInkSpace = inkStroke.ptFromScreen(lastPoint);
+            const currPointInkSpace = inkStroke.ptFromScreen(currPoint);
+            const eraserInkData = this.createEraserOutline(prevPointInkSpace, currPointInkSpace).inkData;
+            // add the ends of the stroke in as "intersections"
+            if (this.insideEraserOutline(eraserInkData, inkData[0])) {
+                strokeToTVals.set(inkView, [0]);
+            }
+            if (this.insideEraserOutline(eraserInkData, inkData[inkData.length - 1])) {
+                const inkList = strokeToTVals.get(inkView);
+                if (inkList !== undefined) {
+                    inkList.push(Math.floor(inkData.length / 4) + 1);
+                } else {
+                    strokeToTVals.set(inkView, [Math.floor(inkData.length / 4) + 1]);
+                }
+            }
+            for (var i = 0; i < inkData.length - 3; i += 4) {
+                // iterate over each segment of bezier curve
+                for (var j = 0; j < eraserInkData.length - 3; j += 4) {
+                    const intersectCurve: Bezier = InkField.Segment(inkData, i); // other curve
+                    const eraserCurve: Bezier = InkField.Segment(eraserInkData, j); // eraser curve
+                    this.bintersects(intersectCurve, eraserCurve).forEach((val: string | number, k: number) => {
+                        // Converting the Bezier.js Split type to a t-value number.
+                        const t = +val.toString().split('/')[0];
+                        if (k % 2 === 0) {
+                            // here, add to the map
+                            const inkList = strokeToTVals.get(inkView);
+                            if (inkList !== undefined) {
+                                const tValOffset = ActiveEraserWidth() / 1030; // to prevent tVals from being added when too close, but scaled by eraser width
+                                const inList = inkList.some(val => Math.abs(val - (t + Math.floor(i / 4))) <= tValOffset);
+                                if (!inList) {
+                                    inkList.push(t + Math.floor(i / 4));
+                                }
+                            } else {
+                                strokeToTVals.set(inkView, [t + Math.floor(i / 4)]);
+                            }
+                        }
+                    });
+                }
+            }
+        });
+        console.log("strokeToTVals", strokeToTVals);
+        return strokeToTVals;
+    };
+
+    /**
+     * Splits the passed in ink stroke at the intersection t values. Generally operates in pairs of t values, where
+     * the first t value is the start of the erased portion and the following t value is the end.
+     * @param ink the ink stroke DocumentView to split
+     * @param tVals all the t values to split the ink stroke at
+     * @returns a list of the new segments with the erased part removed
+     */
     @action
-    radiusErase = (ink: DocumentView, startPt: { X: number; Y: number }, screenEraserPt: { X: number; Y: number }): Segment[] => {
+    radiusErase = (ink: DocumentView, tVals: number[]): Segment[] => {
         const segments: Segment[] = [];
-        const startInkCoords = ink.ComponentView?.ptFromScreen?.(startPt);
-        const inkCoords = ink.ComponentView?.ptFromScreen?.(screenEraserPt); // coordinates in ink space
-        if (!inkCoords || !startInkCoords) return [];
-
-        var eraseSegment: Segment = []; // for eraser visualization
         const inkStroke = ink?.ComponentView as InkingStroke;
-
-        const eraserStroke: InkData = inkStroke.splitByEraser(startInkCoords, inkCoords).inkData;
-        const strokeToTVals: Map<InkingStroke, number[]> = new Map<InkingStroke, number[]>();
-        for (var i = 0; i < eraserStroke.length - 3; i += 4) {
-            eraseSegment.push(InkField.Segment(eraserStroke, i)); // for eraser visualization
-            this.getOtherInkIntersections(ink, i, eraserStroke, strokeToTVals);
+        const { inkData } = inkStroke.inkScaledData();
+        var currSegment: Segment = [];
+        if (tVals.length % 2 !== 0) { // should always have even tVals
+            for (var i = 0; i < inkData.length - 3; i +=4) {
+                currSegment.push(InkField.Segment(inkData, i));
+            }
+            if (currSegment.length > 0) {
+                segments.push(currSegment);
+                return segments;
+            }
         }
-        strokeToTVals.forEach((tVals, inkStroke) => {
-            var segment1: Segment = [];
-            var segment2: Segment = [];
-            const { inkData } = inkStroke.inkScaledData();
-            tVals.sort();
-            console.log('TVALS', inkStroke, tVals);
 
-            var hasSplit = false;
-            var continueErasing = false;
-
-            // below is curve splitting logic
-            if (tVals.length) {
-                for (var i = 0; i < inkData.length - 3; i += 4) {
-                    const inkSegment: Bezier = InkField.Segment(inkData, i);
-                    const currCurveT = Math.floor(i / 4);
+        var continueErasing = false;
+        var firstSegment: Segment = [];
+        // early return if nothing to split on
+        if (tVals.length === 0 || (tVals.length === 1 && tVals[0] === 0)) {
+            return segments;
+        }
 
-                    if (tVals.length === 2) {
-                        if (tVals[0] > currCurveT && tVals[0] < currCurveT + 1) {
-                            segment1.push(inkSegment.split(0, tVals[0] - currCurveT));
+        for (var i = 0; i < inkData.length - 3; i += 4) {
+            const currCurveT = Math.floor(i/4);
+            const inkBezier: Bezier = InkField.Segment(inkData, i);
+            const segmentTs = tVals.filter(t => t >= currCurveT && t < currCurveT + 1);
+
+            if (segmentTs.length > 0) {
+                for (var j = 0; j < segmentTs.length; j++) {
+                    // if the first end of the segment is within the eraser
+                    if (segmentTs[j] === 0 ) {
+                        continueErasing = true;
+                    } else if (segmentTs[j] === Math.floor(inkData.length / 4) + 1) {
+                        break;
+                    }else {
+                        if (!continueErasing) {
+                            currSegment.push(inkBezier.split(0, segmentTs[j] - currCurveT));
                             continueErasing = true;
-                            if (tVals[1] > currCurveT && tVals[1] < currCurveT + 1) {
-                                segment2.push(inkSegment.split(tVals[1] - currCurveT, 1));
-                                continueErasing = false;
-                                hasSplit = true;
-                            }
-                        } else if (tVals[1] > currCurveT && tVals[1] < currCurveT + 1) {
-                            segment2.push(inkSegment.split(tVals[1] - currCurveT, 1));
-                            continueErasing = false;
-                            hasSplit = true;
-                        } else if (!continueErasing) {
-                            if (hasSplit) {
-                                segment2.push(inkSegment);
-                            } else {
-                                segment1.push(inkSegment);
-                            }
-                        }
-                    } else if (tVals.length === 1) {
-                        if (tVals[0] > currCurveT && tVals[0] < currCurveT + 1) {
-                            // this heuristic for determine which segment to keep is not quite right even though it will work most of the time.
-                            // We should really store the eraser intersection normal in getOtherInkIntersections,
-                            // and then test its dot product with the tangent of the stroke at the intersection point.
-                            // if the dot product is positive, then erase the first part of the stroke, otherwise the second.
-                            const leftDist = Utils.ptDistance({ x: inkCoords.X, y: inkCoords.Y }, inkSegment.points.lastElement());
-                            const rightDist = Utils.ptDistance({ x: inkCoords.X, y: inkCoords.Y }, inkSegment.points[0]);
-                            const splits = inkSegment.split(tVals[0] - currCurveT);
-                            if (leftDist < rightDist) {
-                                // if it's on the first end
-                                segment1.push(splits.left);
-                                hasSplit = true;
-                            } else {
-                                segment1.push(splits.right);
-                                hasSplit = true;
-                            }
                         } else {
-                            if (tVals[0] < Math.floor(inkData.length / 4) - tVals[0] && hasSplit) {
-                                segment1.push(inkSegment);
-                            } else if (tVals[0] >= Math.floor(inkData.length / 4) - tVals[0] && !hasSplit) {
-                                segment1.push(inkSegment);
+                            continueErasing = false;
+                            if (currSegment.length > 0) {
+                                segments.push(currSegment);
+                                if (firstSegment.length === 0) {
+                                    firstSegment = currSegment;
+                                }
+                                currSegment = [];
                             }
+                            currSegment.push(inkBezier.split(segmentTs[j] - currCurveT, 1));
                         }
                     }
                 }
+            } else {
+                if (!continueErasing) { // push the bezier piece if not in the eraser circle
+                    currSegment.push(inkBezier);
+                }
             }
-            if (segment1.length && (Math.abs(segment1[0].points[0].x - segment1[0].points.lastElement().x) > 0.5 || Math.abs(segment1[0].points[0].y - segment1[0].points.lastElement().y) > 0.5)) {
-                segments.push(segment1);
-            }
-            if (segment2.length && (Math.abs(segment2[0].points[0].x - segment2[0].points.lastElement().x) > 0.5 || Math.abs(segment2[0].points[0].y - segment2[0].points.lastElement().y) > 0.5)) {
-                segments.push(segment2);
+        }
+        if (currSegment.length > 0) {
+            if (InkingStroke.IsClosed(inkData)) {
+                currSegment = currSegment.concat(firstSegment);
             }
-        });
-
-        segments.push(eraseSegment); // for eraser visualization
+            segments.push(currSegment);
+        }
         return segments;
     };
 
@@ -993,7 +1107,7 @@ export class CollectionFreeFormView extends CollectionSubView<Partial<collection
             const inkSegment: Bezier = InkField.Segment(inkData, i);
             var currIntersects = this.getInkIntersections(i, ink, inkSegment).sort();
             // get current segments intersections (if any) and add the curve index
-            currIntersects = currIntersects.map(tVal => tVal + Math.floor(i / 4));
+            currIntersects = currIntersects.filter(tVal => tVal > 0 && tVal < 1).map(tVal => tVal + Math.floor(i / 4));
             if (currIntersects.length) {
                 intersections = [...intersections, ...currIntersects];
                 for (var j = 0; j < currIntersects.length; j++) {
@@ -1002,19 +1116,9 @@ export class CollectionFreeFormView extends CollectionSubView<Partial<collection
             }
         }
 
-        // for closed curve bookkeeping
         var isClosedCurve = false;
-        if (intersections[0] === 0) {
+        if (InkingStroke.IsClosed(inkData)) {
             isClosedCurve = true;
-
-            // now reduce intersections list to the actual intersections because closed curves have additional ones with itself
-            const indices = intersections.map((value, index) => (value > 0.0001 && value < Math.floor(inkData.length / 4) ? index : -1)).filter(index => index !== -1);
-
-            // Filter intersections and segmentIndexes based on validIndices
-            intersections = indices.map(index => intersections[index]);
-            segmentIndexes = indices.map(index => segmentIndexes[index]);
-            // intersections = intersections.slice(1, intersections.length ); // take the 0 intersection out
-            // segmentIndexes = segmentIndexes.slice(1, segmentIndexes.length); // same for indexes
         }
 
         if (intersections.length) {
@@ -1068,7 +1172,7 @@ export class CollectionFreeFormView extends CollectionSubView<Partial<collection
                         }
                     }
                 } else if (splitSegment1 === -1) {
-                    // case where left end is erased
+                    // case where first end is erased
                     if (currCurveT === splitSegment2) {
                         segment2.push(inkSegment.split(intersections[closestTs[1]] - currCurveT, 1));
                         hasSplit = true;
@@ -1081,15 +1185,13 @@ export class CollectionFreeFormView extends CollectionSubView<Partial<collection
                         }
                     }
                 } else {
-                    // case where right end is erased
+                    // case where last end is erased
                     if (currCurveT === segmentIndexes[0] && isClosedCurve) {
                         segment1.push(inkSegment.split(intersections[0] - currCurveT, 1));
                         hasSplit = true;
                     } else if (currCurveT === splitSegment1) {
                         segment1.push(inkSegment.split(0, intersections[closestTs[0]] - currCurveT));
-                        hasSplit = false; // although this is not technically true, we set to false so the segments after don't get added
-                        // } else if (isClosedCurve && currCurveT === segmentIndexes[1]) {
-                        //     segment1.push(inkSegment.split(intersections[0] - currCurveT, 1));
+                        hasSplit = true; 
                     } else {
                         if ((isClosedCurve && hasSplit) || (!isClosedCurve && !hasSplit)) {
                             segment1.push(inkSegment);
@@ -1099,6 +1201,11 @@ export class CollectionFreeFormView extends CollectionSubView<Partial<collection
             }
         }
 
+        if (isClosedCurve && segment1.length > 0 && segment2.length > 0) {
+            segment2 = segment2.concat(segment1);
+            segment1 = [];
+        }
+
         // push 1 or both segments if they are not empty
         if (segment1.length && (Math.abs(segment1[0].points[0].x - segment1[0].points.lastElement().x) > 0.5 || Math.abs(segment1[0].points[0].y - segment1[0].points.lastElement().y) > 0.5)) {
             segments.push(segment1);
@@ -1203,46 +1310,6 @@ export class CollectionFreeFormView extends CollectionSubView<Partial<collection
         return tVals;
     };
 
-    /**
-     * For radius erase, a function to find an eraser's intersections with a single ink stroke
-     * @param otherInkDocVIew the ink document that is being erased
-     * @param i the index of the eraser's segment
-     * @param points the eraser's ink data points
-     * @param strokeToTVals Map of InkingStroke to the tVals of its intersections with the eraser
-     * @returns
-     */
-    getOtherInkIntersections = (otherInkDocView: DocumentView, i: number, points: InkData, strokeToTVals: Map<InkingStroke, number[]>): Map<InkingStroke, number[]> => {
-        this.childDocs
-            .filter(doc => doc.type === DocumentType.INK && !doc.dontIntersect)
-            .forEach(doc => {
-                // InkingStroke of other ink strokes
-                const otherInk = otherInkDocView.ComponentView as InkingStroke;
-                // ink Data of other ink strokes
-                const { inkData: otherInkData } = otherInk?.inkScaledData() ?? { inkData: [] };
-                for (var j = 0; j < otherInkData.length - 3; j += 4) {
-                    const curve: Bezier = InkField.Segment(points, i); // eraser curve
-                    const otherCurve: Bezier = InkField.Segment(otherInkData, j); // other curve
-                    this.bintersects(otherCurve, curve).forEach((val: string | number, k: number) => {
-                        // Converting the Bezier.js Split type to a t-value number.
-                        const t = +val.toString().split('/')[0];
-                        if (k % 2 === 0) {
-                            // here, add to the map
-                            const inkList = strokeToTVals.get(otherInk);
-                            if (inkList !== undefined) {
-                                const inList = inkList.some(val => Math.abs(val - (t + Math.floor(j / 4))) <= 0.01);
-                                if (!inList) {
-                                    inkList.push(t + Math.floor(j / 4));
-                                }
-                            } else {
-                                strokeToTVals.set(otherInk, [t + Math.floor(j / 4)]);
-                            }
-                        }
-                    });
-                }
-            });
-        return strokeToTVals;
-    };
-
     @action
     zoom = (pointX: number, pointY: number, deltaY: number): void => {
         if (this.Document.isGroup || this.Document[(this._props.viewField ?? '_') + 'freeform_noZoom']) return;
diff --git a/src/client/views/global/globalScripts.ts b/src/client/views/global/globalScripts.ts
index 5c5f8de03..19741e2e0 100644
--- a/src/client/views/global/globalScripts.ts
+++ b/src/client/views/global/globalScripts.ts
@@ -341,10 +341,7 @@ function setActiveTool(tool: InkTool | GestureUtils.Gestures, keepPrim: boolean,
                 Doc.UserDoc().activeEraserTool = tool;
             }
             // pen or eraser
-            if (Doc.ActiveTool === tool && tool === InkTool.Eraser) {
-                Doc.ActiveTool = InkTool.SegmentEraser;
-                console.log("erase click twice")
-            } else if (Doc.ActiveTool === tool && !GestureOverlay.Instance.InkShape && !keepPrim) {
+            if (Doc.ActiveTool === tool && !GestureOverlay.Instance.InkShape && !keepPrim) {
                 Doc.ActiveTool = InkTool.None;
             } else {
                 Doc.ActiveTool = tool as any;
@@ -362,11 +359,6 @@ ScriptingGlobals.add(function activeEraserTool() {
     return StrCast(Doc.UserDoc().activeEraserTool, InkTool.StrokeEraser);
 }, 'returns the current eraser tool');
 
-// ScriptingGlobals.add(function setEraserProperty(option: 'eraseWidth', value: any, checkResult?: boolean) {
-//     setInk: (doc: Doc) => (doc[DocData].eraserWidth = NumCast(value))
-//     InkingStroke.setEraserRadius(NumCast(value));
-// })
-
 // toggle: Set overlay status of selected document
 ScriptingGlobals.add(function setInkProperty(option: 'inkMask' | 'labels' | 'fillColor' | 'strokeWidth' | 'strokeColor' | 'eraserWidth', value: any, checkResult?: boolean) {
     const selected = SelectionManager.Docs.lastElement() ?? Doc.UserDoc();
-- 
cgit v1.2.3-70-g09d2


From 47e3e54543b2b9a613d0029435794d2265e2a952 Mon Sep 17 00:00:00 2001
From: IEatChili <nanunguyen99@gmail.com>
Date: Thu, 16 May 2024 16:49:55 -0700
Subject: feat: added image sorting

---
 package-lock.json                                  |  41 +++++++
 package.json                                       |   2 +
 src/Utils.ts                                       |  18 +++
 src/client/apis/gpt/GPT.ts                         |  62 +++++++++-
 src/client/views/MainView.tsx                      |   2 +
 .../collectionFreeForm/ImageLabelHandler.scss      |  44 +++++++
 .../collectionFreeForm/ImageLabelHandler.tsx       | 120 +++++++++++++++++++
 .../collectionFreeForm/MarqueeOptionsMenu.tsx      |   3 +
 .../collections/collectionFreeForm/MarqueeView.tsx | 129 +++++++++++++++++++--
 9 files changed, 412 insertions(+), 9 deletions(-)
 create mode 100644 src/client/views/collections/collectionFreeForm/ImageLabelHandler.scss
 create mode 100644 src/client/views/collections/collectionFreeForm/ImageLabelHandler.tsx

(limited to 'src/client/views/MainView.tsx')

diff --git a/package-lock.json b/package-lock.json
index c417b7193..be51d2ca1 100644
--- a/package-lock.json
+++ b/package-lock.json
@@ -72,6 +72,7 @@
                 "class-transformer": "^0.5.1",
                 "color": "^4.2.3",
                 "colors": "^1.4.0",
+                "compute-cosine-similarity": "^1.1.0",
                 "connect-flash": "^0.1.1",
                 "connect-mongo": "^5.1.0",
                 "cookie-parser": "^1.4.6",
@@ -117,6 +118,7 @@
                 "image-data-uri": "^2.0.1",
                 "image-size": "^1.0.2",
                 "image-size-stream": "^1.1.0",
+                "is-plain-obj": "^4.1.0",
                 "jimp": "^0.22.10",
                 "jpeg-autorotate": "^9.0.0",
                 "jquery": "^3.7.1",
@@ -14725,6 +14727,35 @@
             "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==",
             "dev": true
         },
+        "node_modules/compute-cosine-similarity": {
+            "version": "1.1.0",
+            "resolved": "https://registry.npmjs.org/compute-cosine-similarity/-/compute-cosine-similarity-1.1.0.tgz",
+            "integrity": "sha512-FXhNx0ILLjGi9Z9+lglLzM12+0uoTnYkHm7GiadXDAr0HGVLm25OivUS1B/LPkbzzvlcXz/1EvWg9ZYyJSdhTw==",
+            "dependencies": {
+                "compute-dot": "^1.1.0",
+                "compute-l2norm": "^1.1.0",
+                "validate.io-array": "^1.0.5",
+                "validate.io-function": "^1.0.2"
+            }
+        },
+        "node_modules/compute-dot": {
+            "version": "1.1.0",
+            "resolved": "https://registry.npmjs.org/compute-dot/-/compute-dot-1.1.0.tgz",
+            "integrity": "sha512-L5Ocet4DdMrXboss13K59OK23GXjiSia7+7Ukc7q4Bl+RVpIXK2W9IHMbWDZkh+JUEvJAwOKRaJDiFUa1LTnJg==",
+            "dependencies": {
+                "validate.io-array": "^1.0.3",
+                "validate.io-function": "^1.0.2"
+            }
+        },
+        "node_modules/compute-l2norm": {
+            "version": "1.1.0",
+            "resolved": "https://registry.npmjs.org/compute-l2norm/-/compute-l2norm-1.1.0.tgz",
+            "integrity": "sha512-6EHh1Elj90eU28SXi+h2PLnTQvZmkkHWySpoFz+WOlVNLz3DQoC4ISUHSV9n5jMxPHtKGJ01F4uu2PsXBB8sSg==",
+            "dependencies": {
+                "validate.io-array": "^1.0.3",
+                "validate.io-function": "^1.0.2"
+            }
+        },
         "node_modules/concat-map": {
             "version": "0.0.1",
             "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz",
@@ -33352,6 +33383,16 @@
             "resolved": "https://registry.npmjs.org/valid-url/-/valid-url-1.0.9.tgz",
             "integrity": "sha512-QQDsV8OnSf5Uc30CKSwG9lnhMPe6exHtTXLRYX8uMwKENy640pU+2BgBL0LRbDh/eYRahNCS7aewCx0wf3NYVA=="
         },
+        "node_modules/validate.io-array": {
+            "version": "1.0.6",
+            "resolved": "https://registry.npmjs.org/validate.io-array/-/validate.io-array-1.0.6.tgz",
+            "integrity": "sha512-DeOy7CnPEziggrOO5CZhVKJw6S3Yi7e9e65R1Nl/RTN1vTQKnzjfvks0/8kQ40FP/dsjRAOd4hxmJ7uLa6vxkg=="
+        },
+        "node_modules/validate.io-function": {
+            "version": "1.0.2",
+            "resolved": "https://registry.npmjs.org/validate.io-function/-/validate.io-function-1.0.2.tgz",
+            "integrity": "sha512-LlFybRJEriSuBnUhQyG5bwglhh50EpTL2ul23MPIuR1odjO7XaMLFV8vHGwp7AZciFxtYOeiSCT5st+XSPONiQ=="
+        },
         "node_modules/validator": {
             "version": "13.11.0",
             "resolved": "https://registry.npmjs.org/validator/-/validator-13.11.0.tgz",
diff --git a/package.json b/package.json
index 65b06f65c..fc8365746 100644
--- a/package.json
+++ b/package.json
@@ -155,6 +155,7 @@
         "class-transformer": "^0.5.1",
         "color": "^4.2.3",
         "colors": "^1.4.0",
+        "compute-cosine-similarity": "^1.1.0",
         "connect-flash": "^0.1.1",
         "connect-mongo": "^5.1.0",
         "cookie-parser": "^1.4.6",
@@ -200,6 +201,7 @@
         "image-data-uri": "^2.0.1",
         "image-size": "^1.0.2",
         "image-size-stream": "^1.1.0",
+        "is-plain-obj": "^4.1.0",
         "jimp": "^0.22.10",
         "jpeg-autorotate": "^9.0.0",
         "jquery": "^3.7.1",
diff --git a/src/Utils.ts b/src/Utils.ts
index 38325a463..0353a2ff7 100644
--- a/src/Utils.ts
+++ b/src/Utils.ts
@@ -925,3 +925,21 @@ export function dateRangeStrToDates(dateStr: string) {
 
     return [new Date(fromYear, fromMonth, fromDay), new Date(toYear, toMonth, toDay)];
 }
+
+export async function convertImageToBase64(url: string): Promise<string> {
+    try {
+        const response = await fetch(url); // Fetch the image
+        if (!response.ok) throw new Error('Network response was not ok');
+        const blob = await response.blob(); // Convert response to Blob
+
+        return new Promise((resolve, reject) => {
+            const reader = new FileReader();
+            reader.readAsDataURL(blob); // Read blob as DataURL (Base64)
+            reader.onloadend = () => resolve(reader.result as string); // Resolve promise with Base64 string
+            reader.onerror = error => reject(error); // Reject promise on error
+        });
+    } catch (error) {
+        console.error('Error:', error);
+        throw error; // Rethrow the error after logging it
+    }
+}
diff --git a/src/client/apis/gpt/GPT.ts b/src/client/apis/gpt/GPT.ts
index fb51278ae..4240c07b8 100644
--- a/src/client/apis/gpt/GPT.ts
+++ b/src/client/apis/gpt/GPT.ts
@@ -68,4 +68,64 @@ const gptImageCall = async (prompt: string, n?: number) => {
     }
 };
 
-export { gptAPICall, gptImageCall, GPTCallType };
+const gptGetEmbedding = async (src: string): Promise<number[]> => {
+    try {
+        const configuration: ClientOptions = {
+            apiKey: process.env.OPENAI_KEY,
+            dangerouslyAllowBrowser: true,
+        };
+        const openai = new OpenAI(configuration);
+        const embeddingResponse = await openai.embeddings.create({
+            model: 'text-embedding-3-large',
+            input: [src],
+            encoding_format: 'float',
+            dimensions: 256,
+        });
+
+        // Assume the embeddingResponse structure is correct; adjust based on actual API response
+        const embedding = embeddingResponse.data[0].embedding;
+        return embedding;
+    } catch (err) {
+        console.log(err);
+        return [];
+    }
+};
+
+const gptImageLabel = async (src: string): Promise<string> => {
+    try {
+        const configuration: ClientOptions = {
+            apiKey: process.env.OPENAI_KEY,
+            dangerouslyAllowBrowser: true,
+        };
+
+        const openai = new OpenAI(configuration);
+        const response = await openai.chat.completions.create({
+            model: 'gpt-4-vision-preview',
+            messages: [
+                {
+                    role: 'user',
+                    content: [
+                        { type: 'text', text: 'Give three labels to describe this image.' },
+                        {
+                            type: 'image_url',
+                            image_url: {
+                                url: `${src}`,
+                                detail: 'low',
+                            },
+                        },
+                    ],
+                },
+            ],
+        });
+        if (response.choices[0].message.content) {
+            return response.choices[0].message.content;
+        } else {
+            return 'Missing labels';
+        }
+    } catch (err) {
+        console.log(err);
+        return 'Error connecting with API';
+    }
+};
+
+export { gptAPICall, gptImageCall, GPTCallType, gptImageLabel, gptGetEmbedding };
diff --git a/src/client/views/MainView.tsx b/src/client/views/MainView.tsx
index 207db2c99..f5d0539c2 100644
--- a/src/client/views/MainView.tsx
+++ b/src/client/views/MainView.tsx
@@ -72,6 +72,7 @@ import { PresBox } from './nodes/trails';
 import { AnchorMenu } from './pdf/AnchorMenu';
 import { GPTPopup } from './pdf/GPTPopup/GPTPopup';
 import { TopBar } from './topbar/TopBar';
+import { ImageLabelHandler } from './collections/collectionFreeForm/ImageLabelHandler';
 const { default: { LEFT_MENU_WIDTH, TOPBAR_HEIGHT } } = require('./global/globalCssVariables.module.scss'); // prettier-ignore
 const _global = (window /* browser */ || global) /* node */ as any;
 
@@ -1030,6 +1031,7 @@ export class MainView extends ObservableReactComponent<{}> {
                 <PreviewCursor />
                 <TaskCompletionBox />
                 <ContextMenu />
+                <ImageLabelHandler />
                 <AnchorMenu />
                 <MapAnchorMenu />
                 <DirectionsAnchorMenu />
diff --git a/src/client/views/collections/collectionFreeForm/ImageLabelHandler.scss b/src/client/views/collections/collectionFreeForm/ImageLabelHandler.scss
new file mode 100644
index 000000000..e7413bf8e
--- /dev/null
+++ b/src/client/views/collections/collectionFreeForm/ImageLabelHandler.scss
@@ -0,0 +1,44 @@
+#label-handler {
+    display: flex;
+    flex-direction: column;
+    align-items: center;
+
+    > div:first-child {
+        display: flex; // Puts the input and button on the same row
+        align-items: center; // Vertically centers items in the flex container
+
+        input {
+            color: black;
+        }
+
+        .IconButton {
+            margin-left: 8px; // Adds space between the input and the icon button
+            width: 19px;
+        }
+    }
+
+    > div:not(:first-of-type) {
+        display: flex;
+        flex-direction: column;
+        align-items: center; // Centers the content vertically in the flex container
+        width: 100%;
+
+        > div {
+            display: flex;
+            justify-content: space-between; // Puts the content and delete button on opposite ends
+            align-items: center;
+            width: 100%;
+            margin-top: 8px; // Adds space between label rows
+
+            p {
+                text-align: center; // Centers the text of the paragraph
+                flex-grow: 1; // Allows the paragraph to grow and occupy the available space
+            }
+
+            .IconButton {
+                // Styling for the delete button
+                margin-left: auto; // Pushes the button to the far right
+            }
+        }
+    }
+}
diff --git a/src/client/views/collections/collectionFreeForm/ImageLabelHandler.tsx b/src/client/views/collections/collectionFreeForm/ImageLabelHandler.tsx
new file mode 100644
index 000000000..46bc3d946
--- /dev/null
+++ b/src/client/views/collections/collectionFreeForm/ImageLabelHandler.tsx
@@ -0,0 +1,120 @@
+import { FontAwesomeIcon } from '@fortawesome/react-fontawesome';
+import { IconButton } from 'browndash-components';
+import { action, makeObservable, observable } from 'mobx';
+import { observer } from 'mobx-react';
+import React from 'react';
+import { SettingsManager } from '../../../util/SettingsManager';
+import { ObservableReactComponent } from '../../ObservableReactComponent';
+import { MarqueeOptionsMenu } from './MarqueeOptionsMenu';
+import './ImageLabelHandler.scss';
+
+@observer
+export class ImageLabelHandler extends ObservableReactComponent<{}> {
+    static Instance: ImageLabelHandler;
+
+    @observable _display: boolean = false;
+    @observable _pageX: number = 0;
+    @observable _pageY: number = 0;
+    @observable _yRelativeToTop: boolean = true;
+    @observable _currentLabel: string = '';
+    @observable _labelGroups: string[] = [];
+
+    constructor(props: any) {
+        super(props);
+        makeObservable(this);
+        ImageLabelHandler.Instance = this;
+        console.log('Instantiated label handler!');
+    }
+
+    @action
+    displayLabelHandler = (x: number, y: number) => {
+        this._pageX = x;
+        this._pageY = y;
+        this._display = true;
+        this._labelGroups = [];
+    };
+
+    @action
+    hideLabelhandler = () => {
+        this._display = false;
+        this._labelGroups = [];
+    };
+
+    @action
+    addLabel = (label: string) => {
+        label = label.toUpperCase().trim();
+        if (label.length > 0) {
+            if (!this._labelGroups.includes(label)) {
+                this._labelGroups = [...this._labelGroups, label];
+            }
+        }
+    };
+
+    @action
+    removeLabel = (label: string) => {
+        label = label.toUpperCase();
+        this._labelGroups = this._labelGroups.filter(group => group !== label);
+    };
+
+    @action
+    groupImages = () => {
+        MarqueeOptionsMenu.Instance.groupImages();
+        this._display = false;
+    };
+
+    render() {
+        if (this._display) {
+            return (
+                <div
+                    id="label-handler"
+                    className="contextMenu-cont"
+                    style={{
+                        display: this._display ? '' : 'none',
+                        left: this._pageX,
+                        ...(this._yRelativeToTop ? { top: Math.max(0, this._pageY) } : { bottom: this._pageY }),
+                        background: SettingsManager.userBackgroundColor,
+                        color: SettingsManager.userColor,
+                    }}>
+                    <div>
+                        <IconButton tooltip={'Cancel'} onPointerDown={this.hideLabelhandler} icon={<FontAwesomeIcon icon="eye-slash" />} color={MarqueeOptionsMenu.Instance.userColor} style={{ width: '19px' }} />
+                        <input aria-label="label-input" id="new-label" type="text" style={{ color: 'black' }} />
+                        <IconButton
+                            tooltip={'Add Label'}
+                            onPointerDown={() => {
+                                const input = document.getElementById('new-label') as HTMLInputElement;
+                                const newLabel = input.value;
+                                this.addLabel(newLabel);
+                                this._currentLabel = '';
+                                input.value = '';
+                            }}
+                            icon={<FontAwesomeIcon icon="plus" />}
+                            color={MarqueeOptionsMenu.Instance.userColor}
+                            style={{ width: '19px' }}
+                        />
+                        <IconButton tooltip={'Group Images'} onPointerDown={this.groupImages} icon={<FontAwesomeIcon icon="object-group" />} color={MarqueeOptionsMenu.Instance.userColor} style={{ width: '19px' }} />
+                    </div>
+                    <div>
+                        {this._labelGroups.map(group => {
+                            return (
+                                <div>
+                                    <p>{group}</p>
+                                    <IconButton
+                                        tooltip={'Remove Label'}
+                                        onPointerDown={() => {
+                                            this.removeLabel(group);
+                                        }}
+                                        icon={'x'}
+                                        color={MarqueeOptionsMenu.Instance.userColor}
+                                        style={{ width: '19px' }}
+                                    />
+                                </div>
+                            );
+                        })}
+                    </div>
+                </div>
+            );
+        } else {
+            return <></>;
+        }
+    }
+}
diff --git a/src/client/views/collections/collectionFreeForm/MarqueeOptionsMenu.tsx b/src/client/views/collections/collectionFreeForm/MarqueeOptionsMenu.tsx
index 79cc534dc..414858aee 100644
--- a/src/client/views/collections/collectionFreeForm/MarqueeOptionsMenu.tsx
+++ b/src/client/views/collections/collectionFreeForm/MarqueeOptionsMenu.tsx
@@ -17,6 +17,8 @@ export class MarqueeOptionsMenu extends AntimodeMenu<AntimodeMenuProps> {
     public showMarquee: () => void = unimplementedFunction;
     public hideMarquee: () => void = unimplementedFunction;
     public pinWithView: (e: KeyboardEvent | React.PointerEvent | undefined) => void = unimplementedFunction;
+    public classifyImages: (e: React.MouseEvent | undefined) => void = unimplementedFunction;
+    public groupImages: () => void = unimplementedFunction;
     public isShown = () => this._opacity > 0;
     constructor(props: any) {
         super(props);
@@ -37,6 +39,7 @@ export class MarqueeOptionsMenu extends AntimodeMenu<AntimodeMenuProps> {
                 <IconButton tooltip={'Summarize Documents'} onPointerDown={this.summarize} icon={<FontAwesomeIcon icon="compress-arrows-alt" />} color={this.userColor} />
                 <IconButton tooltip={'Delete Documents'} onPointerDown={this.delete} icon={<FontAwesomeIcon icon="trash-alt" />} color={this.userColor} />
                 <IconButton tooltip={'Pin selected region'} onPointerDown={this.pinWithView} icon={<FontAwesomeIcon icon="map-pin" />} color={this.userColor} />
+                <IconButton tooltip={'Classify Images'} onPointerDown={this.classifyImages} icon={<FontAwesomeIcon icon="object-group" />} color={this.userColor} />
             </>
         );
         return this.getElement(buttons);
diff --git a/src/client/views/collections/collectionFreeForm/MarqueeView.tsx b/src/client/views/collections/collectionFreeForm/MarqueeView.tsx
index 6b3a56b0b..0918ae293 100644
--- a/src/client/views/collections/collectionFreeForm/MarqueeView.tsx
+++ b/src/client/views/collections/collectionFreeForm/MarqueeView.tsx
@@ -1,15 +1,15 @@
 import { action, computed, makeObservable, observable } from 'mobx';
 import { observer } from 'mobx-react';
 import * as React from 'react';
-import { Utils, intersectRect, lightOrDark, returnFalse } from '../../../../Utils';
-import { Doc, Opt } from '../../../../fields/Doc';
+import { Utils, intersectRect, lightOrDark, returnFalse, convertImageToBase64 } from '../../../../Utils';
+import { Doc, FieldResult, NumListCast, Opt } from '../../../../fields/Doc';
 import { AclAdmin, AclAugment, AclEdit, DocData } from '../../../../fields/DocSymbols';
 import { Id } from '../../../../fields/FieldSymbols';
 import { InkData, InkField, InkTool } from '../../../../fields/InkField';
 import { List } from '../../../../fields/List';
 import { RichTextField } from '../../../../fields/RichTextField';
 import { Cast, FieldValue, NumCast, StrCast } from '../../../../fields/Types';
-import { ImageField } from '../../../../fields/URLField';
+import { ImageField, URLField } from '../../../../fields/URLField';
 import { GetEffectiveAcl } from '../../../../fields/util';
 import { CognitiveServices } from '../../../cognitive_services/CognitiveServices';
 import { DocumentType } from '../../../documents/DocumentTypes';
@@ -28,6 +28,10 @@ import { FormattedTextBox } from '../../nodes/formattedText/FormattedTextBox';
 import { SubCollectionViewProps } from '../CollectionSubView';
 import { MarqueeOptionsMenu } from './MarqueeOptionsMenu';
 import './MarqueeView.scss';
+import { ObjectField } from '../../../../fields/ObjectField';
+import { gptGetEmbedding, gptImageLabel } from '../../../apis/gpt/GPT';
+import { ImageLabelHandler } from './ImageLabelHandler';
+import { listSpec } from '../../../../fields/Schema';
 interface MarqueeViewProps {
     getContainerTransform: () => Transform;
     getTransform: () => Transform;
@@ -64,11 +68,13 @@ export class MarqueeView extends ObservableReactComponent<SubCollectionViewProps
     }
 
     private _commandExecuted = false;
+    private _selectedDocs: Doc[] = [];
     @observable _lastX: number = 0;
     @observable _lastY: number = 0;
     @observable _downX: number = 0;
     @observable _downY: number = 0;
     @observable _visible: boolean = false; // selection rentangle for marquee selection/free hand lasso is visible
+    @observable _labelsVisibile: boolean = false;
     @observable _lassoPts: [number, number][] = [];
     @observable _lassoFreehand: boolean = false;
 
@@ -270,6 +276,8 @@ export class MarqueeView extends ObservableReactComponent<SubCollectionViewProps
             MarqueeOptionsMenu.Instance.hideMarquee = this.hideMarquee;
             MarqueeOptionsMenu.Instance.jumpTo(e.clientX, e.clientY);
             MarqueeOptionsMenu.Instance.pinWithView = this.pinWithView;
+            MarqueeOptionsMenu.Instance.classifyImages = this.classifyImages;
+            MarqueeOptionsMenu.Instance.groupImages = this.groupImages;
             document.addEventListener('pointerdown', hideMarquee, true);
             document.addEventListener('wheel', hideMarquee, true);
         } else {
@@ -416,6 +424,100 @@ export class MarqueeView extends ObservableReactComponent<SubCollectionViewProps
         this.hideMarquee();
     });
 
+    /**
+     * Classifies images and assigns the labels as document fields.
+     * TODO: Turn into lists of labels instead of individual fields.
+     */
+    @undoBatch
+    classifyImages = action(async (e: React.MouseEvent | undefined) => {
+        const selected = this.marqueeSelect(false, DocumentType.IMG);
+        this._selectedDocs = selected;
+
+        const imagePromises = selected.map(doc => {
+            let href = (doc['data'] as URLField).url.href;
+            let hrefParts = href.split('.');
+            let hrefComplete = `${hrefParts[0]}_o.${hrefParts[1]}`;
+            return convertImageToBase64(hrefComplete).then(hrefBase64 => {
+                return gptImageLabel(hrefBase64).then(response => {
+                    console.log(response);
+                    const labels = response.split('\n');
+                    console.log(labels);
+                    doc.image_labels = new List<string>(Array.from(labels!));
+                    return Promise.all(labels!.map(label => gptGetEmbedding(label))).then(embeddings => {
+                        return { doc, embeddings };
+                    });
+                });
+            });
+        });
+
+        let docsAndEmbeddings = await Promise.all(imagePromises);
+
+        for (const docAndEmbedding of docsAndEmbeddings) {
+            if (Array.isArray(docAndEmbedding.embeddings)) {
+                let doc = docAndEmbedding.doc;
+                for (let i = 0; i < 3; i++) {
+                    doc[`label_embedding_${i + 1}`] = new List<number>(docAndEmbedding.embeddings[i]);
+                }
+            }
+        }
+
+        if (e) {
+            ImageLabelHandler.Instance.displayLabelHandler(e.pageX, e.pageY);
+        }
+    });
+
+    /**
+     * Groups images to most similar labels.
+     */
+    @undoBatch
+    groupImages = action(async () => {
+        const labelGroups: string[] = ImageLabelHandler.Instance._labelGroups;
+        const labelToCollection: Map<string, Doc> = new Map();
+        const labelToEmbedding: Map<string, number[]> = new Map();
+        var similarity = require('compute-cosine-similarity');
+
+        // Create new collections associated with each label and get the embeddings for the labels.
+        for (const label of labelGroups) {
+            const newCollection = this.getCollection([], undefined, false);
+            newCollection._freeform_panX = this.Bounds.left + this.Bounds.width / 2;
+            newCollection._freeform_panY = this.Bounds.top + this.Bounds.height / 2;
+            labelToCollection.set(label, newCollection);
+            this._props.addDocument?.(newCollection);
+            const labelEmbedding = await gptGetEmbedding(label);
+            if (Array.isArray(labelEmbedding)) {
+                labelToEmbedding.set(label, labelEmbedding);
+            }
+        }
+
+        // For each image, loop through the labels, and calculate similarity. Associate it with the
+        // most similar one.
+        this._selectedDocs.forEach(doc => {
+            let mostSimilarLabel: string | undefined;
+            let maxSimilarity: number = 0;
+            const embeddingAsList1 = NumListCast(doc.label_embedding_1);
+            const embeddingAsList2 = NumListCast(doc.label_embedding_2);
+            const embeddingAsList3 = NumListCast(doc.label_embedding_3);
+
+            labelGroups.forEach(label => {
+                let curSimilarity1 = similarity(labelToEmbedding.get(label)!, Array.from(embeddingAsList1));
+                let curSimilarity2 = similarity(labelToEmbedding.get(label)!, Array.from(embeddingAsList2));
+                let curSimilarity3 = similarity(labelToEmbedding.get(label)!, Array.from(embeddingAsList3));
+                let maxCurSimilarity = Math.max(curSimilarity1, curSimilarity2, curSimilarity3);
+                if (maxCurSimilarity >= 0.3 && maxCurSimilarity > maxSimilarity) {
+                    mostSimilarLabel = label;
+                    maxSimilarity = maxCurSimilarity;
+                }
+
+                console.log('Doc with labels ' + doc.image_labels + 'has similarity score ' + maxCurSimilarity + ' to ' + mostSimilarLabel);
+            });
+
+            if (mostSimilarLabel) {
+                Doc.AddDocToList(labelToCollection.get(mostSimilarLabel)!, undefined, doc);
+                this._props.removeDocument?.(doc);
+            }
+        });
+    });
+
     @undoBatch
     syntaxHighlight = action((e: KeyboardEvent | React.PointerEvent | undefined) => {
         const selected = this.marqueeSelect(false);
@@ -574,7 +676,10 @@ export class MarqueeView extends ObservableReactComponent<SubCollectionViewProps
         return false;
     }
 
-    marqueeSelect(selectBackgrounds: boolean = false) {
+    /**
+     * When this is called, returns the list of documents that have been selected by the marquee box.
+     */
+    marqueeSelect(selectBackgrounds: boolean = false, docType: DocumentType | undefined = undefined) {
         const selection: Doc[] = [];
         const selectFunc = (doc: Doc) => {
             const layoutDoc = Doc.Layout(doc);
@@ -584,11 +689,19 @@ export class MarqueeView extends ObservableReactComponent<SubCollectionViewProps
             } else {
                 (this.touchesLine(bounds) || this.boundingShape(bounds)) && selection.push(doc);
             }
+            console.log(doc['type']);
         };
-        this._props
-            .activeDocuments()
-            .filter(doc => !doc.z && !doc._lockedPosition)
-            .map(selectFunc);
+        if (docType) {
+            this._props
+                .activeDocuments()
+                .filter(doc => !doc.z && !doc._lockedPosition && doc['type'] === docType)
+                .map(selectFunc);
+        } else {
+            this._props
+                .activeDocuments()
+                .filter(doc => !doc.z && !doc._lockedPosition)
+                .map(selectFunc);
+        }
         if (!selection.length && selectBackgrounds)
             this._props
                 .activeDocuments()
-- 
cgit v1.2.3-70-g09d2

