aboutsummaryrefslogtreecommitdiff
path: root/src
diff options
context:
space:
mode:
Diffstat (limited to 'src')
-rw-r--r--src/.DS_Storebin10244 -> 10244 bytes
-rw-r--r--src/Utils.ts3
-rw-r--r--src/client/apis/gpt/GPT.ts32
-rw-r--r--src/client/documents/Documents.ts2
-rw-r--r--src/client/util/CurrentUserUtils.ts2
-rw-r--r--src/client/util/Scripting.ts4
-rw-r--r--src/client/views/GestureOverlay.tsx308
-rw-r--r--src/client/views/InkTranscription.scss5
-rw-r--r--src/client/views/InkTranscription.tsx765
-rw-r--r--src/client/views/InkingStroke.tsx8
-rw-r--r--src/client/views/MainView.tsx2
-rw-r--r--src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx33
-rw-r--r--src/client/views/global/globalScripts.ts6
-rw-r--r--src/client/views/nodes/DiagramBox.scss4
-rw-r--r--src/client/views/nodes/DiagramBox.tsx23
-rw-r--r--src/fields/InkField.ts20
-rw-r--r--src/pen-gestures/GestureTypes.ts2
-rw-r--r--src/pen-gestures/ndollar.ts28
18 files changed, 782 insertions, 465 deletions
diff --git a/src/.DS_Store b/src/.DS_Store
index 426a2ee90..b91453587 100644
--- a/src/.DS_Store
+++ b/src/.DS_Store
Binary files differ
diff --git a/src/Utils.ts b/src/Utils.ts
index 0590c6930..724725c23 100644
--- a/src/Utils.ts
+++ b/src/Utils.ts
@@ -1,4 +1,3 @@
-/* eslint-disable @typescript-eslint/no-namespace */
import * as uuid from 'uuid';
export function clamp(n: number, lower: number, upper: number) {
@@ -205,7 +204,6 @@ export function intersectRect(r1: { left: number; top: number; width: number; he
}
export function stringHash(s?: string) {
- // eslint-disable-next-line no-bitwise
return !s ? undefined : Math.abs(s.split('').reduce((a, b) => (n => n & n)((a << 5) - a + b.charCodeAt(0)), 0));
}
@@ -254,6 +252,7 @@ export namespace JSONUtils {
try {
results = JSON.parse(source);
} catch (e) {
+ console.log('JSONparse error: ', e);
results = source;
}
return results;
diff --git a/src/client/apis/gpt/GPT.ts b/src/client/apis/gpt/GPT.ts
index 5be9d84ff..258b79b02 100644
--- a/src/client/apis/gpt/GPT.ts
+++ b/src/client/apis/gpt/GPT.ts
@@ -182,5 +182,35 @@ const gptImageLabel = async (src: string): Promise<string> => {
return 'Error connecting with API';
}
};
+const gptHandwriting = async (src: string): Promise<string> => {
+ try {
+ const response = await openai.chat.completions.create({
+ model: 'gpt-4o',
+ temperature: 0,
+ messages: [
+ {
+ role: 'user',
+ content: [
+ { type: 'text', text: 'What is this does this handwriting say. Only return the text' },
+ {
+ type: 'image_url',
+ image_url: {
+ url: `${src}`,
+ detail: 'low',
+ },
+ },
+ ],
+ },
+ ],
+ });
+ if (response.choices[0].message.content) {
+ return response.choices[0].message.content;
+ }
+ return 'Missing labels';
+ } catch (err) {
+ console.log(err);
+ return 'Error connecting with API';
+ }
+};
-export { gptAPICall, gptImageCall, GPTCallType, gptImageLabel, gptGetEmbedding };
+export { gptAPICall, gptImageCall, GPTCallType, gptImageLabel, gptGetEmbedding, gptHandwriting };
diff --git a/src/client/documents/Documents.ts b/src/client/documents/Documents.ts
index 00edaddcc..ce40a89c0 100644
--- a/src/client/documents/Documents.ts
+++ b/src/client/documents/Documents.ts
@@ -775,7 +775,7 @@ export namespace Docs {
export function ComparisonDocument(text: string, options: DocumentOptions = { title: 'Comparison Box' }) {
return InstanceFromProto(Prototypes.get(DocumentType.COMPARISON), text, options);
}
- export function DiagramDocument(options: DocumentOptions = { title: 'bruh box' }) {
+ export function DiagramDocument(options: DocumentOptions = { title: '' }) {
return InstanceFromProto(Prototypes.get(DocumentType.DIAGRAM), undefined, options);
}
diff --git a/src/client/util/CurrentUserUtils.ts b/src/client/util/CurrentUserUtils.ts
index 4d952d408..a340ea4e3 100644
--- a/src/client/util/CurrentUserUtils.ts
+++ b/src/client/util/CurrentUserUtils.ts
@@ -393,7 +393,7 @@ pie title Minerals in my tap water
{ toolTip: "Tap or drag to create a plotly node", title: "Plotly", icon: "rocket", dragFactory: doc.emptyPlotly as Doc, clickFactory: DocCast(doc.emptyMermaids)},
{ toolTip: "Tap or drag to create a physics simulation",title: "Simulation", icon: "rocket",dragFactory: doc.emptySimulation as Doc, clickFactory: DocCast(doc.emptySimulation), funcs: { hidden: "IsNoviceMode()"}},
{ toolTip: "Tap or drag to create a note board", title: "Notes", icon: "book", dragFactory: doc.emptyNoteboard as Doc, clickFactory: DocCast(doc.emptyNoteboard)},
- { toolTip: "Tap or drag to create an iamge", title: "Image", icon: "image", dragFactory: doc.emptyImage as Doc, clickFactory: DocCast(doc.emptyImage)},
+ { toolTip: "Tap or drag to create an image", title: "Image", icon: "image", dragFactory: doc.emptyImage as Doc, clickFactory: DocCast(doc.emptyImage)},
{ toolTip: "Tap or drag to create a collection", title: "Col", icon: "folder", dragFactory: doc.emptyCollection as Doc, clickFactory: DocCast(doc.emptyTab)},
{ toolTip: "Tap or drag to create a webpage", title: "Web", icon: "globe-asia", dragFactory: doc.emptyWebpage as Doc, clickFactory: DocCast(doc.emptyWebpage)},
{ toolTip: "Tap or drag to create a comparison box", title: "Compare", icon: "columns", dragFactory: doc.emptyComparison as Doc, clickFactory: DocCast(doc.emptyComparison)},
diff --git a/src/client/util/Scripting.ts b/src/client/util/Scripting.ts
index c63d3d7cb..68dab8f99 100644
--- a/src/client/util/Scripting.ts
+++ b/src/client/util/Scripting.ts
@@ -1,7 +1,7 @@
// export const ts = (window as any).ts;
// import * as typescriptlib from '!!raw-loader!../../../node_modules/typescript/lib/lib.d.ts'
// import * as typescriptes5 from '!!raw-loader!../../../node_modules/typescript/lib/lib.es5.d.ts'
-import typescriptlib from 'type_decls.d';
+//import typescriptlib from 'type_decls.d';
import * as ts from 'typescript';
import { Doc, FieldType } from '../../fields/Doc';
import { RefField } from '../../fields/RefField';
@@ -248,7 +248,7 @@ export function CompileScript(script: string, options: ScriptOptions = {}): Comp
const funcScript = `(function(${paramString})${reqTypes} { ${body} })`;
host.writeFile('file.ts', funcScript);
- if (typecheck) host.writeFile('node_modules/typescript/lib/lib.d.ts', typescriptlib);
+ //if (typecheck) host.writeFile('node_modules/typescript/lib/lib.d.ts', typescriptlib);
const program = ts.createProgram(['file.ts'], {}, host);
const testResult = program.emit();
const outputText = host.readFile('file.js');
diff --git a/src/client/views/GestureOverlay.tsx b/src/client/views/GestureOverlay.tsx
index 3a2738c3b..e9e12fc3b 100644
--- a/src/client/views/GestureOverlay.tsx
+++ b/src/client/views/GestureOverlay.tsx
@@ -3,34 +3,40 @@ import { action, computed, makeObservable, observable, runInAction } from 'mobx'
import { observer } from 'mobx-react';
import * as React from 'react';
import { returnEmptyFilter, returnEmptyString, returnFalse, setupMoveUpEvents } from '../../ClientUtils';
-import { emptyFunction } from '../../Utils';
+import { emptyFunction, intersectRect } from '../../Utils';
import { Doc, Opt, returnEmptyDoclist } from '../../fields/Doc';
import { InkData, InkField, InkTool } from '../../fields/InkField';
import { NumCast } from '../../fields/Types';
+import { Gestures } from '../../pen-gestures/GestureTypes';
+import { GestureUtils } from '../../pen-gestures/GestureUtils';
+import { Result } from '../../pen-gestures/ndollar';
+import { DocumentType } from '../documents/DocumentTypes';
+import { Docs } from '../documents/Documents';
+import { InteractionUtils } from '../util/InteractionUtils';
+import { ScriptingGlobals } from '../util/ScriptingGlobals';
+import { Transform } from '../util/Transform';
+import { undoable } from '../util/UndoManager';
+import './GestureOverlay.scss';
+import { InkingStroke } from './InkingStroke';
+import { ObservableReactComponent } from './ObservableReactComponent';
+import { returnEmptyDocViewList } from './StyleProvider';
+import { CollectionFreeFormView } from './collections/collectionFreeForm';
import {
ActiveArrowEnd,
ActiveArrowScale,
ActiveArrowStart,
ActiveDash,
+ ActiveFillColor,
ActiveInkBezierApprox,
ActiveInkColor,
ActiveInkWidth,
+ DocumentView,
SetActiveArrowStart,
SetActiveDash,
SetActiveFillColor,
SetActiveInkColor,
SetActiveInkWidth,
} from './nodes/DocumentView';
-import { Gestures } from '../../pen-gestures/GestureTypes';
-import { GestureUtils } from '../../pen-gestures/GestureUtils';
-import { InteractionUtils } from '../util/InteractionUtils';
-import { ScriptingGlobals } from '../util/ScriptingGlobals';
-import { Transform } from '../util/Transform';
-import './GestureOverlay.scss';
-import { ObservableReactComponent } from './ObservableReactComponent';
-import { returnEmptyDocViewList } from './StyleProvider';
-import { ActiveFillColor, DocumentView } from './nodes/DocumentView';
-
export enum ToolglassTools {
InkToText = 'inktotext',
IgnoreGesture = 'ignoregesture',
@@ -41,6 +47,10 @@ interface GestureOverlayProps {
isActive: boolean;
}
@observer
+/**
+ * class for gestures. will determine if what the user drew is a gesture, and will transform the ink stroke into the shape the user
+ * drew or perform the gesture's action
+ */
export class GestureOverlay extends ObservableReactComponent<React.PropsWithChildren<GestureOverlayProps>> {
// eslint-disable-next-line no-use-before-define
static Instance: GestureOverlay;
@@ -70,10 +80,6 @@ export class GestureOverlay extends ObservableReactComponent<React.PropsWithChil
}
private _overlayRef = React.createRef<HTMLDivElement>();
- private _d1: Doc | undefined;
- private _inkToTextDoc: Doc | undefined;
- private thumbIdentifier?: number;
- private pointerIdentifier?: number;
constructor(props: GestureOverlayProps) {
super(props);
@@ -88,7 +94,6 @@ export class GestureOverlay extends ObservableReactComponent<React.PropsWithChil
componentDidMount() {
GestureOverlay.Instance = this;
}
-
@action
onPointerDown = (e: React.PointerEvent) => {
if (!(e.target as HTMLElement)?.className?.toString().startsWith('lm_')) {
@@ -127,70 +132,238 @@ export class GestureOverlay extends ObservableReactComponent<React.PropsWithChil
// SetActiveArrowEnd('none');
}
}
+ /**
+ * If what the user drew is a scribble, this returns the documents that were scribbled over
+ * I changed it so it doesnt use triangles. It will modify an intersect array, with its length being
+ * how many sharp cusps there are. The first index will have a boolean that is based on if there is an
+ * intersection in the first 1/length percent of the stroke. The second index will be if there is an intersection
+ * in the 2nd 1/length percent of the stroke. This array will be used in determineIfScribble().
+ * @param ffview freeform view where scribble is drawn
+ * @param scribbleStroke scribble stroke in screen space coordinats
+ * @returns array of documents scribbled over
+ */
+ isScribble = (ffView: CollectionFreeFormView, cuspArray: { X: number; Y: number }[], scribbleStroke: { X: number; Y: number }[]) => {
+ const intersectArray = cuspArray.map(() => false);
+ const scribbleBounds = InkField.getBounds(scribbleStroke);
+ const docsToDelete = ffView.childDocs
+ .map(doc => DocumentView.getDocumentView(doc))
+ .filter(dv => dv?.ComponentView instanceof InkingStroke)
+ .map(dv => dv?.ComponentView as InkingStroke)
+ .filter(otherInk => {
+ const otherScreenPts = otherInk.inkScaledData?.().inkData.map(otherInk.ptToScreen);
+ if (intersectRect(InkField.getBounds(otherScreenPts), scribbleBounds)) {
+ const intersects = this.findInkIntersections(scribbleStroke, otherScreenPts).map(intersect => {
+ const percentage = intersect.split('/')[0];
+ intersectArray[Math.floor(Number(percentage) * cuspArray.length)] = true;
+ });
+ return intersects.length > 0;
+ }
+ });
+ return !this.determineIfScribble(intersectArray) ? undefined :
+ [ ...docsToDelete.map(stroke => stroke.Document),
+ // bcz: NOTE: docsInBoundingBox test should be replaced with a docsInConvexHull test
+ ...this.docsInBoundingBox({ topLeft : ffView.ScreenToContentsXf().transformPoint(scribbleBounds.left, scribbleBounds.top),
+ bottomRight: ffView.ScreenToContentsXf().transformPoint(scribbleBounds.right,scribbleBounds.bottom)},
+ ffView.childDocs.filter(doc => !docsToDelete.map(s => s.Document).includes(doc)) )]; // prettier-ignore
+ };
+ /**
+ * Returns all docs in array that overlap bounds. Note that the bounds should be given in screen space coordinates.
+ * @param boundingBox screen space bounding box
+ * @param childDocs array of docs to test against bounding box
+ * @returns list of docs that overlap rect
+ */
+ docsInBoundingBox = (boundingBox: { topLeft: number[]; bottomRight: number[] }, childDocs: Doc[]): Doc[] => {
+ const rect = { left: boundingBox.topLeft[0], top: boundingBox.topLeft[1], width: boundingBox.bottomRight[0] - boundingBox.topLeft[0], height: boundingBox.bottomRight[1] - boundingBox.topLeft[1] };
+ return childDocs.filter(doc => intersectRect(rect, { left: NumCast(doc.x), top: NumCast(doc.y), width: NumCast(doc._width), height: NumCast(doc._height) }));
+ };
+ /**
+ * Determines if what the array of cusp/intersection data corresponds to a scribble.
+ * true if there are at least 4 cusps and either:
+ * 1) the initial and final quarters of the array contain objects
+ * 2) or half of the cusps contain objects
+ * @param intersectArray array of booleans coresponding to which scribble sections (regions separated by a cusp) contain Docs
+ * @returns
+ */
+ determineIfScribble = (intersectArray: boolean[]) => {
+ const quarterArrayLength = Math.ceil(intersectArray.length / 3.9); // use 3.9 instead of 4 to work better with strokes with only 4 cusps
+ const { start, end } = intersectArray.reduce((res, val, i) => ({ // test for scribbles at start and end of scribble stroke
+ start: res.start || (val && i <= quarterArrayLength),
+ end: res.end || (val && i >= intersectArray.length - quarterArrayLength)
+ }), { start: false, end: false }); // prettier-ignore
+
+ const percentCuspsWithContent = intersectArray.filter(value => value).length / intersectArray.length;
+ return intersectArray.length > 3 && (percentCuspsWithContent >= 0.5 || (start && end));
+ };
+ /**
+ * determines if inks intersect
+ * @param line is pointData
+ * @param triangle triangle with 3 points
+ * @returns will return an array, with its lenght being equal to how many intersections there are betweent the 2 strokes.
+ * each item in the array will contain a number between 0-1 or a number 0-1 seperated by a comma. If one of the curves is a line, then
+ * then there will just be a number that reprents how far that intersection is along the scribble. For example,
+ * .1 means that the intersection occurs 10% into the scribble, so near the beginning of it. but if they are both curves, then
+ * it will return two numbers, one for each curve, seperated by a comma. Sometimes, the percentage it returns is inaccurate,
+ * espcially in the beginning and end parts of the stroke. dont know why. hope this makes sense
+ */
+ findInkIntersections = (scribble: InkData, inkStroke: InkData): string[] => {
+ const intersectArray: string[] = [];
+ const scribbleBounds = InkField.getBounds(scribble);
+ for (let i = 0; i < scribble.length - 3; i += 4) { // for each segment of scribble
+ for (let j = 0; j < inkStroke.length - 3; j += 4) { // for each segment of ink stroke
+ const scribbleSeg = InkField.Segment(scribble, i);
+ const strokeSeg = InkField.Segment(inkStroke, j);
+ const strokeBounds = InkField.getBounds(strokeSeg.points.map(pt => ({ X: pt.x, Y: pt.y })));
+ if (intersectRect(scribbleBounds, strokeBounds)) {
+ const result = InkField.bintersects(scribbleSeg, strokeSeg)[0];
+ if (result !== undefined) {
+ intersectArray.push(result.toString());
+ }
+ }
+ } // prettier-ignore
+ } // prettier-ignore
+ return intersectArray;
+ };
+ dryInk = () => {
+ const newPoints = this._points.reduce((p, pts) => {
+ p.push([pts.X, pts.Y]);
+ return p;
+ }, [] as number[][]);
+ newPoints.pop();
+ const controlPoints: { X: number; Y: number }[] = [];
+
+ const bezierCurves = fitCurve.default(newPoints, 10);
+ Array.from(bezierCurves).forEach(curve => {
+ controlPoints.push({ X: curve[0][0], Y: curve[0][1] });
+ controlPoints.push({ X: curve[1][0], Y: curve[1][1] });
+ controlPoints.push({ X: curve[2][0], Y: curve[2][1] });
+ controlPoints.push({ X: curve[3][0], Y: curve[3][1] });
+ });
+ const dist = Math.sqrt((controlPoints[0].X - controlPoints.lastElement().X) * (controlPoints[0].X - controlPoints.lastElement().X) + (controlPoints[0].Y - controlPoints.lastElement().Y) * (controlPoints[0].Y - controlPoints.lastElement().Y));
+ if (controlPoints.length > 4 && dist < 10) controlPoints[controlPoints.length - 1] = controlPoints[0];
+ this._points.length = 0;
+ this._points.push(...controlPoints);
+ this.dispatchGesture(Gestures.Stroke);
+ };
@action
onPointerUp = () => {
+ const ffView = DocumentView.DownDocView?.ComponentView instanceof CollectionFreeFormView && DocumentView.DownDocView.ComponentView;
DocumentView.DownDocView = undefined;
if (this._points.length > 1) {
const B = this.svgBounds;
const points = this._points.map(p => ({ X: p.X - B.left, Y: p.Y - B.top }));
-
+ const { Name, Score } =
+ (this.InkShape
+ ? new Result(this.InkShape, 1, Date.now)
+ : Doc.UserDoc().recognizeGestures && points.length > 2
+ ? GestureUtils.GestureRecognizer.Recognize([points])
+ : undefined) ??
+ new Result(Gestures.Stroke, 1, Date.now); // prettier-ignore
+
+ const cuspArray = this.getCusps(points);
// if any of the shape is activated in the CollectionFreeFormViewChrome
- if (this.InkShape) {
- this.makeBezierPolygon(this.InkShape, false);
- this.dispatchGesture(this.InkShape);
- this.primCreated();
- }
- // if we're not drawing in a toolglass try to recognize as gesture
- else {
- // need to decide when to turn gestures back on
- const result = points.length > 2 && GestureUtils.GestureRecognizer.Recognize([points]);
- let actionPerformed = false;
- if (Doc.UserDoc().recognizeGestures && result && result.Score > 0.7) {
- switch (result.Name) {
- case Gestures.Line:
- case Gestures.Triangle:
- case Gestures.Rectangle:
- case Gestures.Circle:
- this.makeBezierPolygon(result.Name, true);
- actionPerformed = this.dispatchGesture(result.Name);
- break;
- case Gestures.Scribble:
- console.log('scribble');
- break;
- default:
- }
+ // need to decide when to turn gestures back on
+ const actionPerformed = ((name: Gestures) => {
+ switch (name) {
+ case Gestures.Line:
+ if (cuspArray.length > 2) return undefined;
+ // eslint-disable-next-line no-fallthrough
+ case Gestures.Triangle:
+ case Gestures.Rectangle:
+ case Gestures.Circle:
+ this.makeBezierPolygon(Name, true);
+ return this.dispatchGesture(name);
+ case Gestures.RightAngle:
+ return ffView && this.convertToText(ffView).length > 0;
+ default:
}
-
- // if no gesture (or if the gesture was unsuccessful), "dry" the stroke into an ink document
- if (!actionPerformed) {
- const newPoints = this._points.reduce((p, pts) => {
- p.push([pts.X, pts.Y]);
- return p;
- }, [] as number[][]);
- newPoints.pop();
- const controlPoints: { X: number; Y: number }[] = [];
-
- const bezierCurves = fitCurve.default(newPoints, 10);
- Array.from(bezierCurves).forEach(curve => {
- controlPoints.push({ X: curve[0][0], Y: curve[0][1] });
- controlPoints.push({ X: curve[1][0], Y: curve[1][1] });
- controlPoints.push({ X: curve[2][0], Y: curve[2][1] });
- controlPoints.push({ X: curve[3][0], Y: curve[3][1] });
- });
- const dist = Math.sqrt(
- (controlPoints[0].X - controlPoints.lastElement().X) * (controlPoints[0].X - controlPoints.lastElement().X) + (controlPoints[0].Y - controlPoints.lastElement().Y) * (controlPoints[0].Y - controlPoints.lastElement().Y)
- );
- // eslint-disable-next-line prefer-destructuring
- if (controlPoints.length > 4 && dist < 10) controlPoints[controlPoints.length - 1] = controlPoints[0];
- this._points.length = 0;
- this._points.push(...controlPoints);
- this.dispatchGesture(Gestures.Stroke);
+ })(Score < 0.7 ? Gestures.Stroke : (Name as Gestures));
+ // if no gesture (or if the gesture was unsuccessful), "dry" the stroke into an ink document
+
+ if (!actionPerformed) {
+ const scribbledOver = ffView && this.isScribble(ffView, cuspArray, this._points);
+ if (scribbledOver) {
+ undoable(() => ffView.removeDocument(scribbledOver), 'scribble erase')();
+ } else {
+ this.dryInk();
}
}
}
this._points.length = 0;
};
-
+ /**
+ * used in the rightAngle gesture to convert handwriting into text. will only work on collections
+ * TODO: make it work on individual ink docs.
+ */
+ convertToText = (ffView: CollectionFreeFormView) => {
+ let minX = 999999999;
+ let maxX = -999999999;
+ let minY = 999999999;
+ let maxY = -999999999;
+ const textDocs: Doc[] = [];
+ ffView.childDocs
+ .filter(doc => doc.type === DocumentType.COL)
+ .forEach(doc => {
+ if (typeof doc.width === 'number' && typeof doc.height === 'number' && typeof doc.x === 'number' && typeof doc.y === 'number') {
+ const bounds = DocumentView.getDocumentView(doc)?.getBounds;
+ if (bounds) {
+ if (intersectRect({ ...bounds, width: bounds.right - bounds.left, height: bounds.bottom - bounds.top }, InkField.getBounds(this._points))) {
+ if (doc.x < minX) {
+ minX = doc.x;
+ }
+ if (doc.x > maxX) {
+ maxX = doc.x;
+ }
+ if (doc.y < minY) {
+ minY = doc.y;
+ }
+ if (doc.y + doc.height > maxY) {
+ maxY = doc.y + doc.height;
+ }
+ const newDoc = Docs.Create.TextDocument(doc.transcription as string, { title: '', x: doc.x as number, y: minY });
+ newDoc.height = doc.height;
+ newDoc.width = doc.width;
+ if (ffView.addDocument && ffView.removeDocument) {
+ ffView.addDocument(newDoc);
+ ffView.removeDocument(doc);
+ }
+ textDocs.push(newDoc);
+ }
+ }
+ }
+ });
+ return textDocs;
+ };
+ /**
+ * Returns array of coordinates corresponding to the sharp cusps in an input stroke
+ * @param points array of X,Y stroke coordinates
+ * @returns array containing the coordinates of the sharp cusps
+ */
+ getCusps(points: InkData) {
+ const arrayOfPoints: { X: number; Y: number }[] = [];
+ arrayOfPoints.push(points[0]);
+ for (let i = 0; i < points.length - 2; i++) {
+ const point1 = points[i];
+ const point2 = points[i + 1];
+ const point3 = points[i + 2];
+ if (this.find_angle(point1, point2, point3) < 90) {
+ // NOTE: this is not an accurate way to find cusps -- it is highly dependent on sampling rate and doesn't work well with slowly drawn scribbles
+ arrayOfPoints.push(point2);
+ }
+ }
+ arrayOfPoints.push(points[points.length - 1]);
+ return arrayOfPoints;
+ }
+ /**
+ * takes in three points and then determines the angle of the points. used to determine if the cusp
+ * is sharp enoug
+ * @returns
+ */
+ find_angle(A: { X: number; Y: number }, B: { X: number; Y: number }, C: { X: number; Y: number }) {
+ const AB = Math.sqrt(Math.pow(B.X - A.X, 2) + Math.pow(B.Y - A.Y, 2));
+ const BC = Math.sqrt(Math.pow(B.X - C.X, 2) + Math.pow(B.Y - C.Y, 2));
+ const AC = Math.sqrt(Math.pow(C.X - A.X, 2) + Math.pow(C.Y - A.Y, 2));
+ return Math.acos((BC * BC + AB * AB - AC * AC) / (2 * BC * AB)) * (180 / Math.PI);
+ }
makeBezierPolygon = (shape: string, gesture: boolean) => {
const xs = this._points.map(p => p.X);
const ys = this._points.map(p => p.Y);
@@ -389,7 +562,6 @@ export class GestureOverlay extends ObservableReactComponent<React.PropsWithChil
this._strokes.map((l, i) => {
const b = { left: -20000, right: 20000, top: -20000, bottom: 20000, width: 40000, height: 40000 }; // this.getBounds(l, true);
return (
- // eslint-disable-next-line react/no-array-index-key
<svg key={i} width={b.width} height={b.height} style={{ top: 0, left: 0, transform: `translate(${b.left}px, ${b.top}px)`, pointerEvents: 'none', position: 'absolute', zIndex: 30000, overflow: 'visible' }}>
{InteractionUtils.CreatePolyline(
l,
diff --git a/src/client/views/InkTranscription.scss b/src/client/views/InkTranscription.scss
index bbb0a1afa..c77117ccc 100644
--- a/src/client/views/InkTranscription.scss
+++ b/src/client/views/InkTranscription.scss
@@ -2,4 +2,7 @@
.error-msg {
display: none !important;
}
-} \ No newline at end of file
+ .ms-editor {
+ top: 1000px;
+ }
+}
diff --git a/src/client/views/InkTranscription.tsx b/src/client/views/InkTranscription.tsx
index 33db72960..e5f47c1c3 100644
--- a/src/client/views/InkTranscription.tsx
+++ b/src/client/views/InkTranscription.tsx
@@ -1,350 +1,415 @@
-// import * as iink from 'iink-js';
-// import { action, observable } from 'mobx';
-// import * as React from 'react';
-// import { Doc, DocListCast } from '../../fields/Doc';
-// import { InkData, InkField, InkTool } from '../../fields/InkField';
-// import { Cast, DateCast, NumCast } from '../../fields/Types';
-// import { aggregateBounds } from '../../Utils';
-// import { DocumentType } from '../documents/DocumentTypes';
-// import { CollectionFreeFormView } from './collections/collectionFreeForm';
-// import { InkingStroke } from './InkingStroke';
-// import './InkTranscription.scss';
-
-// /**
-// * Class component that handles inking in writing mode
-// */
-// export class InkTranscription extends React.Component {
-// static Instance: InkTranscription;
-
-// @observable _mathRegister: any= undefined;
-// @observable _mathRef: any= undefined;
-// @observable _textRegister: any= undefined;
-// @observable _textRef: any= undefined;
-// private lastJiix: any;
-// private currGroup?: Doc;
-
-// constructor(props: Readonly<{}>) {
-// super(props);
-
-// InkTranscription.Instance = this;
-// }
-
-// componentWillUnmount() {
-// this._mathRef.removeEventListener('exported', (e: any) => this.exportInk(e, this._mathRef));
-// this._textRef.removeEventListener('exported', (e: any) => this.exportInk(e, this._textRef));
-// }
-
-// @action
-// setMathRef = (r: any) => {
-// if (!this._mathRegister) {
-// this._mathRegister = r
-// ? iink.register(r, {
-// recognitionParams: {
-// type: 'MATH',
-// protocol: 'WEBSOCKET',
-// server: {
-// host: 'cloud.myscript.com',
-// applicationKey: process.env.IINKJS_APP,
-// hmacKey: process.env.IINKJS_HMAC,
-// websocket: {
-// pingEnabled: false,
-// autoReconnect: true,
-// },
-// },
-// iink: {
-// math: {
-// mimeTypes: ['application/x-latex', 'application/vnd.myscript.jiix'],
-// },
-// export: {
-// jiix: {
-// strokes: true,
-// },
-// },
-// },
-// },
-// })
-// : null;
-// }
-
-// r?.addEventListener('exported', (e: any) => this.exportInk(e, this._mathRef));
-
-// return (this._mathRef = r);
-// };
-
-// @action
-// setTextRef = (r: any) => {
-// if (!this._textRegister) {
-// this._textRegister = r
-// ? iink.register(r, {
-// recognitionParams: {
-// type: 'TEXT',
-// protocol: 'WEBSOCKET',
-// server: {
-// host: 'cloud.myscript.com',
-// applicationKey: '7277ec34-0c2e-4ee1-9757-ccb657e3f89f',
-// hmacKey: 'f5cb18f2-1f95-4ddb-96ac-3f7c888dffc1',
-// websocket: {
-// pingEnabled: false,
-// autoReconnect: true,
-// },
-// },
-// iink: {
-// text: {
-// mimeTypes: ['text/plain'],
-// },
-// export: {
-// jiix: {
-// strokes: true,
-// },
-// },
-// },
-// },
-// })
-// : null;
-// }
-
-// r?.addEventListener('exported', (e: any) => this.exportInk(e, this._textRef));
-
-// return (this._textRef = r);
-// };
-
-// /**
-// * Handles processing Dash Doc data for ink transcription.
-// *
-// * @param groupDoc the group which contains the ink strokes we want to transcribe
-// * @param inkDocs the ink docs contained within the selected group
-// * @param math boolean whether to do math transcription or not
-// */
-// transcribeInk = (groupDoc: Doc | undefined, inkDocs: Doc[], math: boolean) => {
-// if (!groupDoc) return;
-// const validInks = inkDocs.filter(s => s.type === DocumentType.INK);
-
-// const strokes: InkData[] = [];
-// const times: number[] = [];
-// validInks
-// .filter(i => Cast(i[Doc.LayoutFieldKey(i)], InkField))
-// .forEach(i => {
-// const d = Cast(i[Doc.LayoutFieldKey(i)], InkField, null);
-// const inkStroke = DocumentManager.Instance.getDocumentView(i)?.ComponentView as InkingStroke;
-// strokes.push(d.inkData.map(pd => inkStroke.ptToScreen({ X: pd.X, Y: pd.Y })));
-// times.push(DateCast(i.author_date).getDate().getTime());
-// });
-
-// this.currGroup = groupDoc;
-
-// const pointerData = { events: strokes.map((stroke, i) => this.inkJSON(stroke, times[i])) };
-// const processGestures = false;
-
-// if (math) {
-// this._mathRef.editor.pointerEvents(pointerData, processGestures);
-// } else {
-// this._textRef.editor.pointerEvents(pointerData, processGestures);
-// }
-// };
-
-// /**
-// * Converts the Dash Ink Data to JSON.
-// *
-// * @param stroke The dash ink data
-// * @param time the time of the stroke
-// * @returns json object representation of ink data
-// */
-// inkJSON = (stroke: InkData, time: number) => {
-// return {
-// pointerType: 'PEN',
-// pointerId: 1,
-// x: stroke.map(point => point.X),
-// y: stroke.map(point => point.Y),
-// t: new Array(stroke.length).fill(time),
-// p: new Array(stroke.length).fill(1.0),
-// };
-// };
-
-// /**
-// * Creates subgroups for each word for the whole text transcription
-// * @param wordInkDocMap the mapping of words to ink strokes (Ink Docs)
-// */
-// subgroupsTranscriptions = (wordInkDocMap: Map<string, Doc[]>) => {
-// // iterate through the keys of wordInkDocMap
-// wordInkDocMap.forEach(async (inkDocs: Doc[], word: string) => {
-// const selected = inkDocs.slice();
-// if (!selected) {
-// return;
-// }
-// const ctx = await Cast(selected[0].embedContainer, Doc);
-// if (!ctx) {
-// return;
-// }
-// const docView: CollectionFreeFormView = DocumentManager.Instance.getDocumentView(ctx)?.ComponentView as CollectionFreeFormView;
-
-// if (!docView) return;
-// const marqViewRef = docView._marqueeViewRef.current;
-// if (!marqViewRef) return;
-// this.groupInkDocs(selected, docView, word);
-// });
-// };
-
-// /**
-// * Event listener function for when the 'exported' event is heard.
-// *
-// * @param e the event objects
-// * @param ref the ref to the editor
-// */
-// exportInk = (e: any, ref: any) => {
-// const exports = e.detail.exports;
-// if (exports) {
-// if (exports['application/x-latex']) {
-// const latex = exports['application/x-latex'];
-// if (this.currGroup) {
-// this.currGroup.text = latex;
-// this.currGroup.title = latex;
-// }
-
-// ref.editor.clear();
-// } else if (exports['text/plain']) {
-// if (exports['application/vnd.myscript.jiix']) {
-// this.lastJiix = JSON.parse(exports['application/vnd.myscript.jiix']);
-// // map timestamp to strokes
-// const timestampWord = new Map<number, string>();
-// this.lastJiix.words.map((word: any) => {
-// if (word.items) {
-// word.items.forEach((i: { id: string; timestamp: string; X: Array<number>; Y: Array<number>; F: Array<number> }) => {
-// const ms = Date.parse(i.timestamp);
-// timestampWord.set(ms, word.label);
-// });
-// }
-// });
-
-// const wordInkDocMap = new Map<string, Doc[]>();
-// if (this.currGroup) {
-// const docList = DocListCast(this.currGroup.data);
-// docList.forEach((inkDoc: Doc) => {
-// // just having the times match up and be a unique value (actual timestamp doesn't matter)
-// const ms = DateCast(inkDoc.author_date).getDate().getTime() + 14400000;
-// const word = timestampWord.get(ms);
-// if (!word) {
-// return;
-// }
-// const entry = wordInkDocMap.get(word);
-// if (entry) {
-// entry.push(inkDoc);
-// wordInkDocMap.set(word, entry);
-// } else {
-// const newEntry = [inkDoc];
-// wordInkDocMap.set(word, newEntry);
-// }
-// });
-// if (this.lastJiix.words.length > 1) this.subgroupsTranscriptions(wordInkDocMap);
-// }
-// }
-// const text = exports['text/plain'];
-
-// if (this.currGroup) {
-// this.currGroup.text = text; // transcription text
-// this.currGroup.icon_fieldKey = 'transcription'; // use the transcription icon template when iconifying
-// this.currGroup.title = text.split('\n')[0];
-// }
-
-// ref.editor.clear();
-// }
-// }
-// };
-
-// /**
-// * Creates the ink grouping once the user leaves the writing mode.
-// */
-// createInkGroup() {
-// // TODO nda - if document being added to is a inkGrouping then we can just add to that group
-// if (Doc.ActiveTool === InkTool.Write) {
-// CollectionFreeFormView.collectionsWithUnprocessedInk.forEach(ffView => {
-// // TODO: nda - will probably want to go through ffView unprocessed docs and then see if any of the inksToGroup docs are in it and only use those
-// const selected = ffView.unprocessedDocs;
-// const newCollection = this.groupInkDocs(
-// selected.filter(doc => doc.embedContainer),
-// ffView
-// );
-// ffView.unprocessedDocs = [];
-
-// InkTranscription.Instance.transcribeInk(newCollection, selected, false);
-// });
-// }
-// CollectionFreeFormView.collectionsWithUnprocessedInk.clear();
-// }
-
-// /**
-// * Creates the groupings for a given list of ink docs on a specific doc view
-// * @param selected: the list of ink docs to create a grouping of
-// * @param docView: the view in which we want the grouping to be created
-// * @param word: optional param if the group we are creating is a word (subgrouping individual words)
-// * @returns a new collection Doc or undefined if the grouping fails
-// */
-// groupInkDocs(selected: Doc[], docView: CollectionFreeFormView, word?: string): Doc | undefined {
-// const bounds: { x: number; y: number; width?: number; height?: number }[] = [];
-
-// // calculate the necessary bounds from the selected ink docs
-// selected.map(
-// action(d => {
-// const x = NumCast(d.x);
-// const y = NumCast(d.y);
-// const width = NumCast(d._width);
-// const height = NumCast(d._height);
-// bounds.push({ x, y, width, height });
-// })
-// );
-
-// // calculate the aggregated bounds
-// const aggregBounds = aggregateBounds(bounds, 0, 0);
-// const marqViewRef = docView._marqueeViewRef.current;
-
-// // set the vals for bounds in marqueeView
-// if (marqViewRef) {
-// marqViewRef._downX = aggregBounds.x;
-// marqViewRef._downY = aggregBounds.y;
-// marqViewRef._lastX = aggregBounds.r;
-// marqViewRef._lastY = aggregBounds.b;
-// }
-
-// // map through all the selected ink strokes and create the groupings
-// selected.map(
-// action(d => {
-// const dx = NumCast(d.x);
-// const dy = NumCast(d.y);
-// delete d.x;
-// delete d.y;
-// delete d.activeFrame;
-// delete d._timecodeToShow; // bcz: this should be automatic somehow.. along with any other properties that were logically associated with the original collection
-// delete d._timecodeToHide; // bcz: this should be automatic somehow.. along with any other properties that were logically associated with the original collection
-// // calculate pos based on bounds
-// if (marqViewRef?.Bounds) {
-// d.x = dx - marqViewRef.Bounds.left - marqViewRef.Bounds.width / 2;
-// d.y = dy - marqViewRef.Bounds.top - marqViewRef.Bounds.height / 2;
-// }
-// return d;
-// })
-// );
-// docView.props.removeDocument?.(selected);
-// // Gets a collection based on the selected nodes using a marquee view ref
-// const newCollection = marqViewRef?.getCollection(selected, undefined, true);
-// if (newCollection) {
-// newCollection.width = NumCast(newCollection._width);
-// newCollection.height = NumCast(newCollection._height);
-// // if the grouping we are creating is an individual word
-// if (word) {
-// newCollection.title = word;
-// }
-// }
-
-// // nda - bug: when deleting a stroke before leaving writing mode, delete the stroke from unprocessed ink docs
-// newCollection && docView.props.addDocument?.(newCollection);
-// return newCollection;
-// }
-
-// render() {
-// return (
-// <div className="ink-transcription">
-// <div className="math-editor" ref={this.setMathRef} touch-action="none"></div>
-// <div className="text-editor" ref={this.setTextRef} touch-action="none"></div>
-// </div>
-// );
-// }
-// }
+import * as iink from 'iink-ts';
+import { action, observable } from 'mobx';
+import * as React from 'react';
+import { Doc, DocListCast } from '../../fields/Doc';
+import { InkData, InkField, InkTool } from '../../fields/InkField';
+import { Cast, DateCast, ImageCast, NumCast } from '../../fields/Types';
+import { aggregateBounds } from '../../Utils';
+import { DocumentType } from '../documents/DocumentTypes';
+import { CollectionFreeFormView } from './collections/collectionFreeForm';
+import { InkingStroke } from './InkingStroke';
+import './InkTranscription.scss';
+import { Docs } from '../documents/Documents';
+import { DocumentView } from './nodes/DocumentView';
+import { ImageField } from '../../fields/URLField';
+import { gptHandwriting } from '../apis/gpt/GPT';
+import { URLField } from '../../fields/URLField';
+/**
+ * Class component that handles inking in writing mode
+ */
+export class InkTranscription extends React.Component {
+ static Instance: InkTranscription;
+
+ // eslint-disable-next-line @typescript-eslint/no-explicit-any
+ @observable _mathRegister: any = undefined;
+ // eslint-disable-next-line @typescript-eslint/no-explicit-any
+ @observable _mathRef: any = undefined;
+ // eslint-disable-next-line @typescript-eslint/no-explicit-any
+ @observable _textRegister: any = undefined;
+ // eslint-disable-next-line @typescript-eslint/no-explicit-any
+ @observable _textRef: any = undefined;
+ // eslint-disable-next-line @typescript-eslint/no-explicit-any
+ @observable iinkEditor: any = undefined;
+ // eslint-disable-next-line @typescript-eslint/no-explicit-any
+ private lastJiix: any;
+ private currGroup?: Doc;
+ private collectionFreeForm?: CollectionFreeFormView;
+
+ constructor(props: Readonly<object>) {
+ super(props);
+
+ InkTranscription.Instance = this;
+ }
+ @action
+ // eslint-disable-next-line @typescript-eslint/no-explicit-any
+ setMathRef = async (r: any) => {
+ if (!this._textRegister && r) {
+ const options = {
+ configuration: {
+ server: {
+ scheme: 'https',
+ host: 'cloud.myscript.com',
+ applicationKey: 'c0901093-5ac5-4454-8e64-0def0f13f2ca',
+ hmacKey: 'f6465cca-1856-4492-a6a4-e2395841be2f',
+ protocol: 'WEBSOCKET',
+ },
+ recognition: {
+ type: 'TEXT',
+ },
+ },
+ };
+ // eslint-disable-next-line @typescript-eslint/no-explicit-any
+ const editor = new iink.Editor(r, options as any);
+
+ await editor.initialize();
+
+ this._textRegister = r;
+ // eslint-disable-next-line @typescript-eslint/no-explicit-any
+ r?.addEventListener('exported', (e: any) => this.exportInk(e, this._textRef));
+
+ return (this._textRef = r);
+ }
+ };
+ @action
+ // eslint-disable-next-line @typescript-eslint/no-explicit-any
+ setTextRef = async (r: any) => {
+ if (!this._textRegister && r) {
+ const options = {
+ configuration: {
+ server: {
+ scheme: 'https',
+ host: 'cloud.myscript.com',
+ applicationKey: 'c0901093-5ac5-4454-8e64-0def0f13f2ca',
+ hmacKey: 'f6465cca-1856-4492-a6a4-e2395841be2f',
+ protocol: 'WEBSOCKET',
+ },
+ recognition: {
+ type: 'TEXT',
+ },
+ },
+ };
+ // eslint-disable-next-line @typescript-eslint/no-explicit-any
+ const editor = new iink.Editor(r, options as any);
+
+ await editor.initialize();
+ this.iinkEditor = editor;
+ this._textRegister = r;
+ // eslint-disable-next-line @typescript-eslint/no-explicit-any
+ r?.addEventListener('exported', (e: any) => this.exportInk(e, this._textRef));
+
+ return (this._textRef = r);
+ }
+ };
+
+ /**
+ * Handles processing Dash Doc data for ink transcription.
+ *
+ * @param groupDoc the group which contains the ink strokes we want to transcribe
+ * @param inkDocs the ink docs contained within the selected group
+ * @param math boolean whether to do math transcription or not
+ */
+ transcribeInk = (groupDoc: Doc | undefined, inkDocs: Doc[], math: boolean) => {
+ if (!groupDoc) return;
+ const validInks = inkDocs.filter(s => s.type === DocumentType.INK);
+
+ const strokes: InkData[] = [];
+
+ const times: number[] = [];
+ validInks
+ .filter(i => Cast(i[Doc.LayoutFieldKey(i)], InkField))
+ .forEach(i => {
+ const d = Cast(i[Doc.LayoutFieldKey(i)], InkField, null);
+ const inkStroke = DocumentView.getDocumentView(i)?.ComponentView as InkingStroke;
+ strokes.push(d.inkData.map(pd => inkStroke.ptToScreen({ X: pd.X, Y: pd.Y })));
+ times.push(DateCast(i.author_date).getDate().getTime());
+ });
+ this.currGroup = groupDoc;
+ const pointerData = strokes.map((stroke, i) => this.inkJSON(stroke, times[i]));
+ if (math) {
+ this.iinkEditor.importPointEvents(pointerData);
+ } else {
+ this.iinkEditor.importPointEvents(pointerData);
+ }
+ };
+ convertPointsToString(points: InkData[]): string {
+ return points[0].map(point => `new Point(${point.X}, ${point.Y})`).join(',');
+ }
+ convertPointsToString2(points: InkData[]): string {
+ return points[0].map(point => `(${point.X},${point.Y})`).join(',');
+ }
+
+ /**
+ * Converts the Dash Ink Data to JSON.
+ *
+ * @param stroke The dash ink data
+ * @param time the time of the stroke
+ * @returns json object representation of ink data
+ */
+ inkJSON = (stroke: InkData, time: number) => {
+ interface strokeData {
+ x: number;
+ y: number;
+ t: number;
+ p: number;
+ }
+ const strokeObjects: strokeData[] = [];
+ stroke.forEach(point => {
+ const tempObject: strokeData = {
+ x: point.X,
+ y: point.Y,
+ t: time,
+ p: 1.0,
+ };
+ strokeObjects.push(tempObject);
+ });
+ return {
+ pointerType: 'PEN',
+ pointerId: 1,
+ pointers: strokeObjects,
+ };
+ };
+
+ /**
+ * Creates subgroups for each word for the whole text transcription
+ * @param wordInkDocMap the mapping of words to ink strokes (Ink Docs)
+ */
+ subgroupsTranscriptions = (wordInkDocMap: Map<string, Doc[]>) => {
+ // iterate through the keys of wordInkDocMap
+ wordInkDocMap.forEach(async (inkDocs: Doc[], word: string) => {
+ const selected = inkDocs.slice();
+ if (!selected) {
+ return;
+ }
+ const ctx = await Cast(selected[0].embedContainer, Doc);
+ if (!ctx) {
+ return;
+ }
+ const docView: CollectionFreeFormView = DocumentView.getDocumentView(ctx)?.ComponentView as CollectionFreeFormView;
+ // DocumentManager.Instance.getDocumentView(ctx)?.ComponentView as CollectionFreeFormView;
+
+ if (!docView) return;
+ const marqViewRef = docView._marqueeViewRef.current;
+ if (!marqViewRef) return;
+ this.groupInkDocs(selected, docView, word);
+ });
+ };
+
+ /**
+ * Event listener function for when the 'exported' event is heard.
+ *
+ * @param e the event objects
+ * @param ref the ref to the editor
+ */
+ // eslint-disable-next-line @typescript-eslint/no-explicit-any
+ exportInk = async (e: any, ref: any) => {
+ const exports = e.detail['application/vnd.myscript.jiix'];
+ if (exports) {
+ if (exports['type'] == 'Math') {
+ const latex = exports['application/x-latex'];
+ if (this.currGroup) {
+ this.currGroup.text = latex;
+ this.currGroup.title = latex;
+ }
+
+ ref.editor.clear();
+ } else if (exports['type'] == 'Text') {
+ if (exports['application/vnd.myscript.jiix']) {
+ this.lastJiix = JSON.parse(exports['application/vnd.myscript.jiix']);
+ // map timestamp to strokes
+ const timestampWord = new Map<number, string>();
+ // eslint-disable-next-line @typescript-eslint/no-explicit-any
+ this.lastJiix.words.map((word: any) => {
+ if (word.items) {
+ word.items.forEach((i: { id: string; timestamp: string; X: Array<number>; Y: Array<number>; F: Array<number> }) => {
+ const ms = Date.parse(i.timestamp);
+ timestampWord.set(ms, word.label);
+ });
+ }
+ });
+
+ const wordInkDocMap = new Map<string, Doc[]>();
+ if (this.currGroup) {
+ const docList = DocListCast(this.currGroup.data);
+ docList.forEach((inkDoc: Doc) => {
+ // just having the times match up and be a unique value (actual timestamp doesn't matter)
+ const ms = DateCast(inkDoc.author_date).getDate().getTime() + 14400000;
+ const word = timestampWord.get(ms);
+ if (!word) {
+ return;
+ }
+ const entry = wordInkDocMap.get(word);
+ if (entry) {
+ entry.push(inkDoc);
+ wordInkDocMap.set(word, entry);
+ } else {
+ const newEntry = [inkDoc];
+ wordInkDocMap.set(word, newEntry);
+ }
+ });
+ if (this.lastJiix.words.length > 1) this.subgroupsTranscriptions(wordInkDocMap);
+ }
+ }
+ const text = exports['label'];
+
+ if (this.currGroup && text) {
+ DocumentView.getDocumentView(this.currGroup)?.ComponentView?.updateIcon?.();
+ const image = await this.getIcon();
+ const { href } = (image as URLField).url;
+ const hrefParts = href.split('.');
+ const hrefComplete = `${hrefParts[0]}_o.${hrefParts[1]}`;
+ let response;
+ try {
+ const hrefBase64 = await this.imageUrlToBase64(hrefComplete);
+ response = await gptHandwriting(hrefBase64);
+ } catch {
+ console.error('Error getting image');
+ }
+ const textBoxText = 'iink: ' + text + '\n' + '\n' + 'ChatGPT: ' + response;
+ this.currGroup.transcription = response;
+ this.currGroup.title = response;
+ if (!this.currGroup.hasTextBox) {
+ const newDoc = Docs.Create.TextDocument(textBoxText, { title: '', x: this.currGroup.x as number, y: (this.currGroup.y as number) + (this.currGroup.height as number) });
+ newDoc.height = 200;
+ this.collectionFreeForm?.addDocument(newDoc);
+ this.currGroup.hasTextBox = true;
+ }
+ ref.editor.clear();
+ }
+ }
+ }
+ };
+ /**
+ * gets the icon of the collection that was just made
+ * @returns the image of the collection
+ */
+ async getIcon() {
+ const docView = DocumentView.getDocumentView(this.currGroup);
+ if (docView) {
+ docView.ComponentView?.updateIcon?.();
+ return new Promise<ImageField | undefined>(res => setTimeout(() => res(ImageCast(docView.Document.icon)), 1000));
+ }
+ return undefined;
+ }
+ /**
+ * converts the image to base url formate
+ * @param imageUrl imageurl taken from the collection icon
+ */
+ imageUrlToBase64 = async (imageUrl: string): Promise<string> => {
+ try {
+ const response = await fetch(imageUrl);
+ const blob = await response.blob();
+
+ return new Promise((resolve, reject) => {
+ const reader = new FileReader();
+ reader.readAsDataURL(blob);
+ reader.onloadend = () => resolve(reader.result as string);
+ reader.onerror = error => reject(error);
+ });
+ } catch (error) {
+ console.error('Error:', error);
+ throw error;
+ }
+ };
+
+ /**
+ * Creates the ink grouping once the user leaves the writing mode.
+ */
+ createInkGroup() {
+ // TODO nda - if document being added to is a inkGrouping then we can just add to that group
+ if (Doc.ActiveTool === InkTool.Write) {
+ CollectionFreeFormView.collectionsWithUnprocessedInk.forEach(ffView => {
+ // TODO: nda - will probably want to go through ffView unprocessed docs and then see if any of the inksToGroup docs are in it and only use those
+ const selected = ffView.unprocessedDocs;
+ const newCollection = this.groupInkDocs(
+ selected.filter(doc => doc.embedContainer),
+ ffView
+ );
+ ffView.unprocessedDocs = [];
+
+ InkTranscription.Instance.transcribeInk(newCollection, selected, false);
+ });
+ }
+ CollectionFreeFormView.collectionsWithUnprocessedInk.clear();
+ }
+
+ /**
+ * Creates the groupings for a given list of ink docs on a specific doc view
+ * @param selected: the list of ink docs to create a grouping of
+ * @param docView: the view in which we want the grouping to be created
+ * @param word: optional param if the group we are creating is a word (subgrouping individual words)
+ * @returns a new collection Doc or undefined if the grouping fails
+ */
+ groupInkDocs(selected: Doc[], docView: CollectionFreeFormView, word?: string): Doc | undefined {
+ this.collectionFreeForm = docView;
+ const bounds: { x: number; y: number; width?: number; height?: number }[] = [];
+
+ // calculate the necessary bounds from the selected ink docs
+ selected.map(
+ action(d => {
+ const x = NumCast(d.x);
+ const y = NumCast(d.y);
+ const width = NumCast(d._width);
+ const height = NumCast(d._height);
+ bounds.push({ x, y, width, height });
+ })
+ );
+
+ // calculate the aggregated bounds
+ const aggregBounds = aggregateBounds(bounds, 0, 0);
+ const marqViewRef = docView._marqueeViewRef.current;
+
+ // set the vals for bounds in marqueeView
+ if (marqViewRef) {
+ marqViewRef._downX = aggregBounds.x;
+ marqViewRef._downY = aggregBounds.y;
+ marqViewRef._lastX = aggregBounds.r;
+ marqViewRef._lastY = aggregBounds.b;
+ }
+
+ // map through all the selected ink strokes and create the groupings
+ selected.map(
+ action(d => {
+ const dx = NumCast(d.x);
+ const dy = NumCast(d.y);
+ delete d.x;
+ delete d.y;
+ delete d.activeFrame;
+ delete d._timecodeToShow; // bcz: this should be automatic somehow.. along with any other properties that were logically associated with the original collection
+ delete d._timecodeToHide; // bcz: this should be automatic somehow.. along with any other properties that were logically associated with the original collection
+ // calculate pos based on bounds
+ if (marqViewRef?.Bounds) {
+ d.x = dx - marqViewRef.Bounds.left - marqViewRef.Bounds.width / 2;
+ d.y = dy - marqViewRef.Bounds.top - marqViewRef.Bounds.height / 2;
+ }
+ return d;
+ })
+ );
+ docView.props.removeDocument?.(selected);
+ // Gets a collection based on the selected nodes using a marquee view ref
+ const newCollection = marqViewRef?.getCollection(selected, undefined, true);
+ if (newCollection) {
+ newCollection.width = NumCast(newCollection._width);
+ newCollection.height = NumCast(newCollection._height);
+ // if the grouping we are creating is an individual word
+ if (word) {
+ newCollection.title = word;
+ }
+ }
+
+ // nda - bug: when deleting a stroke before leaving writing mode, delete the stroke from unprocessed ink docs
+ if (newCollection) {
+ docView.props.addDocument?.(newCollection);
+ newCollection.hasTextBox = false;
+ }
+ return newCollection;
+ }
+
+ render() {
+ return (
+ <div className="ink-transcription">
+ <div className="math-editor" ref={this.setMathRef}></div>
+ <div className="text-editor" ref={this.setTextRef}></div>
+ </div>
+ );
+ }
+}
diff --git a/src/client/views/InkingStroke.tsx b/src/client/views/InkingStroke.tsx
index 498042938..177400882 100644
--- a/src/client/views/InkingStroke.tsx
+++ b/src/client/views/InkingStroke.tsx
@@ -49,7 +49,7 @@ import { PinDocView, PinProps } from './PinFuncs';
import { StyleProp } from './StyleProp';
import { ViewBoxInterface } from './ViewBoxInterface';
-// eslint-disable-next-line @typescript-eslint/no-var-requires
+// eslint-disable-next-line @typescript-eslint/no-var-requires, @typescript-eslint/no-require-imports
const { INK_MASK_SIZE } = require('./global/globalCssVariables.module.scss'); // prettier-ignore
@observer
@@ -107,10 +107,10 @@ export class InkingStroke extends ViewBoxAnnotatableComponent<FieldViewProps>()
* analyzes the ink stroke and saves the analysis of the stroke to the 'inkAnalysis' field,
* and the recognized words to the 'handwriting'
*/
- analyzeStrokes() {
- const data: InkData = Cast(this.dataDoc[this.fieldKey], InkField)?.inkData ?? [];
+ analyzeStrokes = () => {
+ const data: InkData = this.inkScaledData().inkData ?? [];
CognitiveServices.Inking.Appliers.ConcatenateHandwriting(this.dataDoc, ['inkAnalysis', 'handwriting'], [data]);
- }
+ };
/**
* Toggles whether the ink stroke is displayed as an overlay mask or as a regular stroke.
diff --git a/src/client/views/MainView.tsx b/src/client/views/MainView.tsx
index 8b8f85dfb..5285e67f6 100644
--- a/src/client/views/MainView.tsx
+++ b/src/client/views/MainView.tsx
@@ -74,6 +74,7 @@ import { PresBox } from './nodes/trails';
import { AnchorMenu } from './pdf/AnchorMenu';
import { GPTPopup } from './pdf/GPTPopup/GPTPopup';
import { TopBar } from './topbar/TopBar';
+import { InkTranscription } from './InkTranscription';
// eslint-disable-next-line @typescript-eslint/no-var-requires, @typescript-eslint/no-require-imports
const { LEFT_MENU_WIDTH, TOPBAR_HEIGHT } = require('./global/globalCssVariables.module.scss'); // prettier-ignore
@@ -1087,6 +1088,7 @@ export class MainView extends ObservableReactComponent<object> {
<MarqueeOptionsMenu />
<TimelineMenu />
<RichTextMenu />
+ <InkTranscription />
{this.snapLines}
<LightboxView key="lightbox" PanelWidth={this._windowWidth} addSplit={CollectionDockingView.AddSplit} PanelHeight={this._windowHeight} maxBorder={this.lightboxMaxBorder} />
<GPTPopup key="gptpopup" />
diff --git a/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx b/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx
index f106eba26..36772e8ec 100644
--- a/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx
+++ b/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx
@@ -889,13 +889,12 @@ export class CollectionFreeFormView extends CollectionSubView<Partial<collection
strokeToTVals.set(inkView, [Math.floor(inkData.length / 4) + 1]);
}
}
-
for (let i = 0; i < inkData.length - 3; i += 4) {
// iterate over each segment of bezier curve
for (let j = 0; j < eraserInkData.length - 3; j += 4) {
const intersectCurve: Bezier = InkField.Segment(inkData, i); // other curve
const eraserCurve: Bezier = InkField.Segment(eraserInkData, j); // eraser curve
- this.bintersects(intersectCurve, eraserCurve).forEach((val: string | number, k: number) => {
+ InkField.bintersects(intersectCurve, eraserCurve).forEach((val: string | number, k: number) => {
// Converting the Bezier.js Split type to a t-value number.
const t = +val.toString().split('/')[0];
if (k % 2 === 0) {
@@ -1169,26 +1168,6 @@ export class CollectionFreeFormView extends CollectionSubView<Partial<collection
return this.getClosestTs(tVals, excludeT, startIndex, mid - 1);
};
- // for some reason bezier.js doesn't handle the case of intersecting a linear curve, so we wrap the intersection
- // call in a test for linearity
- bintersects = (curve: Bezier, otherCurve: Bezier) => {
- // eslint-disable-next-line @typescript-eslint/no-explicit-any
- if ((curve as any)._linear) {
- // bezier.js doesn't intersect properly if the curve is actually a line -- so get intersect other curve against this line, then figure out the t coordinates of the intersection on this line
- const intersections = otherCurve.lineIntersects({ p1: curve.points[0], p2: curve.points[3] });
- if (intersections.length) {
- const intPt = otherCurve.get(intersections[0]);
- const intT = curve.project(intPt).t;
- return intT ? [intT] : [];
- }
- }
- // eslint-disable-next-line @typescript-eslint/no-explicit-any
- if ((otherCurve as any)._linear) {
- return curve.lineIntersects({ p1: otherCurve.points[0], p2: otherCurve.points[3] });
- }
- return curve.intersects(otherCurve);
- };
-
/**
* Determines all possible intersections of the current curve of the intersected ink stroke with all other curves of all
* ink strokes in the current collection.
@@ -1220,7 +1199,7 @@ export class CollectionFreeFormView extends CollectionSubView<Partial<collection
if (apt.d !== undefined && apt.d < 1 && apt.t !== undefined && !tVals.includes(apt.t)) {
tVals.push(apt.t);
}
- this.bintersects(curve, otherCurve).forEach((val: string | number, ival: number) => {
+ InkField.bintersects(curve, otherCurve).forEach((val: string | number, ival: number) => {
// Converting the Bezier.js Split type to a t-value number.
const t = +val.toString().split('/')[0];
if (ival % 2 === 0 && !tVals.includes(t)) tVals.push(t); // bcz: Hack! don't know why but intersection points are doubled from bezier.js (but not identical).
@@ -1945,6 +1924,14 @@ export class CollectionFreeFormView extends CollectionSubView<Partial<collection
!options && ContextMenu.Instance.addItem({ description: 'Options...', subitems: optionItems, icon: 'eye' });
const mores = ContextMenu.Instance.findByDescription('More...');
const moreItems = mores?.subitems ?? [];
+ moreItems.push({
+ description: 'recognize all ink',
+ event: () => {
+ this.unprocessedDocs.push(...this.childDocs.filter(doc => doc.type === DocumentType.INK));
+ CollectionFreeFormView.collectionsWithUnprocessedInk.add(this);
+ },
+ icon: 'pen',
+ });
!mores && ContextMenu.Instance.addItem({ description: 'More...', subitems: moreItems, icon: 'eye' });
};
diff --git a/src/client/views/global/globalScripts.ts b/src/client/views/global/globalScripts.ts
index 588073568..7f6c726d6 100644
--- a/src/client/views/global/globalScripts.ts
+++ b/src/client/views/global/globalScripts.ts
@@ -15,6 +15,7 @@ import { ScriptingGlobals } from '../../util/ScriptingGlobals';
import { SnappingManager } from '../../util/SnappingManager';
import { UndoManager, undoable } from '../../util/UndoManager';
import { GestureOverlay } from '../GestureOverlay';
+import { InkTranscription } from '../InkTranscription';
import { InkingStroke } from '../InkingStroke';
import { MainView } from '../MainView';
import { PropertiesView } from '../PropertiesView';
@@ -386,6 +387,7 @@ ScriptingGlobals.add(function toggleCharStyle(charStyle: attrname, checkResult?:
export function createInkGroup(/* inksToGroup?: Doc[], isSubGroup?: boolean */) {
// TODO nda - if document being added to is a inkGrouping then we can just add to that group
if (Doc.ActiveTool === InkTool.Write) {
+ console.log('create inking group ');
CollectionFreeFormView.collectionsWithUnprocessedInk.forEach(ffView => {
// TODO: nda - will probably want to go through ffView unprocessed docs and then see if any of the inksToGroup docs are in it and only use those
const selected = ffView.unprocessedDocs;
@@ -443,14 +445,14 @@ export function createInkGroup(/* inksToGroup?: Doc[], isSubGroup?: boolean */)
// TODO: nda - will probably need to go through and only remove the unprocessed selected docs
ffView.unprocessedDocs = [];
- // InkTranscription.Instance.transcribeInk(newCollection, selected, false);
+ InkTranscription.Instance.transcribeInk(newCollection, selected, false);
});
}
CollectionFreeFormView.collectionsWithUnprocessedInk.clear();
}
function setActiveTool(tool: InkTool | Gestures, keepPrim: boolean, checkResult?: boolean) {
- // InkTranscription.Instance?.createInkGroup();
+ InkTranscription.Instance?.createInkGroup();
if (checkResult) {
return (Doc.ActiveTool === tool && !GestureOverlay.Instance?.InkShape) || GestureOverlay.Instance?.InkShape === tool
? GestureOverlay.Instance?.KeepPrimitiveMode || ![Gestures.Circle, Gestures.Line, Gestures.Rectangle].includes(tool as Gestures)
diff --git a/src/client/views/nodes/DiagramBox.scss b/src/client/views/nodes/DiagramBox.scss
index 323638bff..8a7863c14 100644
--- a/src/client/views/nodes/DiagramBox.scss
+++ b/src/client/views/nodes/DiagramBox.scss
@@ -1,5 +1,3 @@
-$searchbarHeight: 50px;
-
.DIYNodeBox {
width: 100%;
height: 100%;
@@ -23,7 +21,6 @@ $searchbarHeight: 50px;
justify-content: center;
align-items: center;
width: 100%;
- height: $searchbarHeight;
padding: 10px;
input[type='text'] {
@@ -42,7 +39,6 @@ $searchbarHeight: 50px;
justify-content: center;
align-items: center;
width: 100%;
- height: calc(100% - $searchbarHeight);
.diagramBox {
flex: 1;
display: flex;
diff --git a/src/client/views/nodes/DiagramBox.tsx b/src/client/views/nodes/DiagramBox.tsx
index 36deb2d8d..0d755fdbe 100644
--- a/src/client/views/nodes/DiagramBox.tsx
+++ b/src/client/views/nodes/DiagramBox.tsx
@@ -18,7 +18,9 @@ import { InkingStroke } from '../InkingStroke';
import './DiagramBox.scss';
import { FieldView, FieldViewProps } from './FieldView';
import { FormattedTextBox } from './formattedText/FormattedTextBox';
-
+/**
+ * this is a class for the diagram box doc type that can be found in the tools section of the side bar
+ */
@observer
export class DiagramBox extends ViewBoxAnnotatableComponent<FieldViewProps>() {
public static LayoutString(fieldKey: string) {
@@ -50,7 +52,9 @@ export class DiagramBox extends ViewBoxAnnotatableComponent<FieldViewProps>() {
mermaid.initialize({
securityLevel: 'loose',
startOnLoad: true,
- flowchart: { useMaxWidth: true, htmlLabels: true, curve: 'cardinal' },
+ darkMode: true,
+ flowchart: { useMaxWidth: false, htmlLabels: true, curve: 'cardinal' },
+ gantt: { useMaxWidth: true, useWidth: 2000 },
});
// when a new doc/text/ink/shape is created in the freeform view, this generates the corresponding mermaid diagram code
reaction(
@@ -59,14 +63,21 @@ export class DiagramBox extends ViewBoxAnnotatableComponent<FieldViewProps>() {
{ fireImmediately: true }
);
}
+ /**
+ * helper method for renderMermaidAsync
+ * @param str string containing the mermaid code
+ * @returns
+ */
renderMermaid = (str: string) => {
try {
return mermaid.render('graph' + Date.now(), str);
- } catch (error) {
+ } catch {
return { svg: '', bindFunctions: undefined };
}
};
-
+ /**
+ * will update the div containing the mermaid diagram to render the new mermaidCode
+ */
renderMermaidAsync = async (mermaidCode: string, dashDiv: HTMLDivElement) => {
try {
const { svg, bindFunctions } = await this.renderMermaid(mermaidCode);
@@ -97,7 +108,9 @@ export class DiagramBox extends ViewBoxAnnotatableComponent<FieldViewProps>() {
res
);
}, 'set mermaid code');
-
+ /**
+ * will generate mermaid code with GPT based on what the user requested
+ */
generateMermaidCode = action(() => {
this._generating = true;
const prompt = 'Write this in mermaid code and only give me the mermaid code: ' + this._inputValue;
diff --git a/src/fields/InkField.ts b/src/fields/InkField.ts
index 32abf0076..ad524f73f 100644
--- a/src/fields/InkField.ts
+++ b/src/fields/InkField.ts
@@ -102,6 +102,26 @@ export class InkField extends ObjectField {
const top = Math.min(...ys);
return { right, left, bottom, top, width: right - left, height: bottom - top };
}
+
+ // for some reason bezier.js doesn't handle the case of intersecting a linear curve, so we wrap the intersection
+ // call in a test for linearity
+ public static bintersects(curve: Bezier, otherCurve: Bezier) {
+ // eslint-disable-next-line @typescript-eslint/no-explicit-any
+ if ((curve as any)._linear) {
+ // bezier.js doesn't intersect properly if the curve is actually a line -- so get intersect other curve against this line, then figure out the t coordinates of the intersection on this line
+ const intersections = otherCurve.lineIntersects({ p1: curve.points[0], p2: curve.points[3] });
+ if (intersections.length) {
+ const intPt = otherCurve.get(intersections[0]);
+ const intT = curve.project(intPt).t;
+ return intT ? [intT] : [];
+ }
+ }
+ // eslint-disable-next-line @typescript-eslint/no-explicit-any
+ if ((otherCurve as any)._linear) {
+ return curve.lineIntersects({ p1: otherCurve.points[0], p2: otherCurve.points[3] });
+ }
+ return curve.intersects(otherCurve);
+ }
}
ScriptingGlobals.add('InkField', InkField);
diff --git a/src/pen-gestures/GestureTypes.ts b/src/pen-gestures/GestureTypes.ts
index d86562580..5a8e9bd97 100644
--- a/src/pen-gestures/GestureTypes.ts
+++ b/src/pen-gestures/GestureTypes.ts
@@ -1,12 +1,12 @@
export enum Gestures {
Line = 'line',
Stroke = 'stroke',
- Scribble = 'scribble',
Text = 'text',
Triangle = 'triangle',
Circle = 'circle',
Rectangle = 'rectangle',
Arrow = 'arrow',
+ RightAngle = 'rightangle',
}
// Defines a point in an ink as a pair of x- and y-coordinates.
diff --git a/src/pen-gestures/ndollar.ts b/src/pen-gestures/ndollar.ts
index ff7f7310b..04262b61f 100644
--- a/src/pen-gestures/ndollar.ts
+++ b/src/pen-gestures/ndollar.ts
@@ -209,6 +209,7 @@ export class NDollarRecognizer {
])
)
);
+ this.Multistrokes.push(new Multistroke(Gestures.Rectangle, useBoundedRotationInvariance, new Array([new Point(30, 143), new Point(106, 146), new Point(106, 225), new Point(30, 222), new Point(30, 146)])));
this.Multistrokes.push(new Multistroke(Gestures.Line, useBoundedRotationInvariance, [[new Point(12, 347), new Point(119, 347)]]));
this.Multistrokes.push(
new Multistroke(
@@ -219,6 +220,13 @@ export class NDollarRecognizer {
);
this.Multistrokes.push(
new Multistroke(
+ Gestures.Triangle, // equilateral
+ useBoundedRotationInvariance,
+ new Array([new Point(42, 100), new Point(140, 102), new Point(100, 200), new Point(40, 100)])
+ )
+ );
+ this.Multistrokes.push(
+ new Multistroke(
Gestures.Circle,
useBoundedRotationInvariance,
new Array([
@@ -236,6 +244,26 @@ export class NDollarRecognizer {
])
)
);
+ this.Multistrokes.push(
+ new Multistroke(
+ Gestures.Circle,
+ useBoundedRotationInvariance,
+ new Array([
+ new Point(201, 250),
+ new Point(160, 230),
+ new Point(151, 210),
+ new Point(151, 190),
+ new Point(160, 170),
+ new Point(200, 150),
+ new Point(240, 170),
+ new Point(248, 190),
+ new Point(248, 210),
+ new Point(240, 230),
+ new Point(200, 250),
+ ])
+ )
+ );
+ this.Multistrokes.push(new Multistroke(Gestures.RightAngle, useBoundedRotationInvariance, new Array([new Point(0, 0), new Point(0, 100), new Point(200, 100)])));
NumMultistrokes = this.Multistrokes.length; // NumMultistrokes flags the end of the non user-defined gstures strokes
//
// PREDEFINED STROKES
generated by cgit v1.2.3-70-g09d2 (git 2.46.0) at 2025-10-06 18:52:49 +0000