aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
Diffstat
-rw-r--r--package.json1
-rw-r--r--src/.DS_Storebin10244 -> 14340 bytes
-rw-r--r--src/client/apis/gpt/GPT.ts32
-rw-r--r--src/client/documents/Documents.ts2
-rw-r--r--src/client/util/CurrentUserUtils.ts2
-rw-r--r--src/client/util/Scripting.ts4
-rw-r--r--src/client/views/GestureOverlay.tsx400
-rw-r--r--src/client/views/InkTranscription.scss5
-rw-r--r--src/client/views/InkTranscription.tsx765
-rw-r--r--src/client/views/InkingStroke.tsx4
-rw-r--r--src/client/views/MainView.tsx2
-rw-r--r--src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx8
-rw-r--r--src/client/views/global/globalScripts.ts6
-rw-r--r--src/client/views/nodes/DiagramBox.scss4
-rw-r--r--src/client/views/nodes/DiagramBox.tsx23
-rw-r--r--src/pen-gestures/GestureTypes.ts2
-rw-r--r--src/pen-gestures/ndollar.ts28
17 files changed, 850 insertions, 438 deletions
diff --git a/package.json b/package.json
index 967549197..b8a23d0b5 100644
--- a/package.json
+++ b/package.json
@@ -209,6 +209,7 @@
"https": "^1.0.0",
"https-browserify": "^1.0.0",
"i": "^0.3.7",
+ "iink-ts": "^1.0.5",
"image-data-uri": "^2.0.1",
"image-size": "^1.0.2",
"image-size-stream": "^1.1.0",
diff --git a/src/.DS_Store b/src/.DS_Store
index 426a2ee90..19eea5fce 100644
--- a/src/.DS_Store
+++ b/src/.DS_Store
Binary files differ
diff --git a/src/client/apis/gpt/GPT.ts b/src/client/apis/gpt/GPT.ts
index 5be9d84ff..258b79b02 100644
--- a/src/client/apis/gpt/GPT.ts
+++ b/src/client/apis/gpt/GPT.ts
@@ -182,5 +182,35 @@ const gptImageLabel = async (src: string): Promise<string> => {
return 'Error connecting with API';
}
};
+const gptHandwriting = async (src: string): Promise<string> => {
+ try {
+ const response = await openai.chat.completions.create({
+ model: 'gpt-4o',
+ temperature: 0,
+ messages: [
+ {
+ role: 'user',
+ content: [
+ { type: 'text', text: 'What is this does this handwriting say. Only return the text' },
+ {
+ type: 'image_url',
+ image_url: {
+ url: `${src}`,
+ detail: 'low',
+ },
+ },
+ ],
+ },
+ ],
+ });
+ if (response.choices[0].message.content) {
+ return response.choices[0].message.content;
+ }
+ return 'Missing labels';
+ } catch (err) {
+ console.log(err);
+ return 'Error connecting with API';
+ }
+};
-export { gptAPICall, gptImageCall, GPTCallType, gptImageLabel, gptGetEmbedding };
+export { gptAPICall, gptImageCall, GPTCallType, gptImageLabel, gptGetEmbedding, gptHandwriting };
diff --git a/src/client/documents/Documents.ts b/src/client/documents/Documents.ts
index aac15059e..5ecded6a5 100644
--- a/src/client/documents/Documents.ts
+++ b/src/client/documents/Documents.ts
@@ -775,7 +775,7 @@ export namespace Docs {
export function ComparisonDocument(text: string, options: DocumentOptions = { title: 'Comparison Box' }) {
return InstanceFromProto(Prototypes.get(DocumentType.COMPARISON), text, options);
}
- export function DiagramDocument(options: DocumentOptions = { title: 'bruh box' }) {
+ export function DiagramDocument(options: DocumentOptions = { title: '' }) {
return InstanceFromProto(Prototypes.get(DocumentType.DIAGRAM), undefined, options);
}
diff --git a/src/client/util/CurrentUserUtils.ts b/src/client/util/CurrentUserUtils.ts
index 21df0fd3b..698e4e315 100644
--- a/src/client/util/CurrentUserUtils.ts
+++ b/src/client/util/CurrentUserUtils.ts
@@ -393,7 +393,7 @@ pie title Minerals in my tap water
{ toolTip: "Tap or drag to create a plotly node", title: "Plotly", icon: "rocket", dragFactory: doc.emptyPlotly as Doc, clickFactory: DocCast(doc.emptyMermaids)},
{ toolTip: "Tap or drag to create a physics simulation",title: "Simulation", icon: "rocket",dragFactory: doc.emptySimulation as Doc, clickFactory: DocCast(doc.emptySimulation), funcs: { hidden: "IsNoviceMode()"}},
{ toolTip: "Tap or drag to create a note board", title: "Notes", icon: "book", dragFactory: doc.emptyNoteboard as Doc, clickFactory: DocCast(doc.emptyNoteboard)},
- { toolTip: "Tap or drag to create an iamge", title: "Image", icon: "image", dragFactory: doc.emptyImage as Doc, clickFactory: DocCast(doc.emptyImage)},
+ { toolTip: "Tap or drag to create an image", title: "Image", icon: "image", dragFactory: doc.emptyImage as Doc, clickFactory: DocCast(doc.emptyImage)},
{ toolTip: "Tap or drag to create a collection", title: "Col", icon: "folder", dragFactory: doc.emptyCollection as Doc, clickFactory: DocCast(doc.emptyTab)},
{ toolTip: "Tap or drag to create a webpage", title: "Web", icon: "globe-asia", dragFactory: doc.emptyWebpage as Doc, clickFactory: DocCast(doc.emptyWebpage)},
{ toolTip: "Tap or drag to create a comparison box", title: "Compare", icon: "columns", dragFactory: doc.emptyComparison as Doc, clickFactory: DocCast(doc.emptyComparison)},
diff --git a/src/client/util/Scripting.ts b/src/client/util/Scripting.ts
index c63d3d7cb..68dab8f99 100644
--- a/src/client/util/Scripting.ts
+++ b/src/client/util/Scripting.ts
@@ -1,7 +1,7 @@
// export const ts = (window as any).ts;
// import * as typescriptlib from '!!raw-loader!../../../node_modules/typescript/lib/lib.d.ts'
// import * as typescriptes5 from '!!raw-loader!../../../node_modules/typescript/lib/lib.es5.d.ts'
-import typescriptlib from 'type_decls.d';
+//import typescriptlib from 'type_decls.d';
import * as ts from 'typescript';
import { Doc, FieldType } from '../../fields/Doc';
import { RefField } from '../../fields/RefField';
@@ -248,7 +248,7 @@ export function CompileScript(script: string, options: ScriptOptions = {}): Comp
const funcScript = `(function(${paramString})${reqTypes} { ${body} })`;
host.writeFile('file.ts', funcScript);
- if (typecheck) host.writeFile('node_modules/typescript/lib/lib.d.ts', typescriptlib);
+ //if (typecheck) host.writeFile('node_modules/typescript/lib/lib.d.ts', typescriptlib);
const program = ts.createProgram(['file.ts'], {}, host);
const testResult = program.emit();
const outputText = host.readFile('file.js');
diff --git a/src/client/views/GestureOverlay.tsx b/src/client/views/GestureOverlay.tsx
index 3a2738c3b..935eeb251 100644
--- a/src/client/views/GestureOverlay.tsx
+++ b/src/client/views/GestureOverlay.tsx
@@ -7,30 +7,35 @@ import { emptyFunction } from '../../Utils';
import { Doc, Opt, returnEmptyDoclist } from '../../fields/Doc';
import { InkData, InkField, InkTool } from '../../fields/InkField';
import { NumCast } from '../../fields/Types';
+import { Gestures } from '../../pen-gestures/GestureTypes';
+import { GestureUtils } from '../../pen-gestures/GestureUtils';
+import { Result } from '../../pen-gestures/ndollar';
+import { DocumentType } from '../documents/DocumentTypes';
+import { Docs } from '../documents/Documents';
+import { InteractionUtils } from '../util/InteractionUtils';
+import { ScriptingGlobals } from '../util/ScriptingGlobals';
+import { Transform } from '../util/Transform';
+import './GestureOverlay.scss';
+import { InkingStroke } from './InkingStroke';
+import { ObservableReactComponent } from './ObservableReactComponent';
+import { returnEmptyDocViewList } from './StyleProvider';
+import { CollectionFreeFormView } from './collections/collectionFreeForm';
import {
ActiveArrowEnd,
ActiveArrowScale,
ActiveArrowStart,
ActiveDash,
+ ActiveFillColor,
ActiveInkBezierApprox,
ActiveInkColor,
ActiveInkWidth,
+ DocumentView,
SetActiveArrowStart,
SetActiveDash,
SetActiveFillColor,
SetActiveInkColor,
SetActiveInkWidth,
} from './nodes/DocumentView';
-import { Gestures } from '../../pen-gestures/GestureTypes';
-import { GestureUtils } from '../../pen-gestures/GestureUtils';
-import { InteractionUtils } from '../util/InteractionUtils';
-import { ScriptingGlobals } from '../util/ScriptingGlobals';
-import { Transform } from '../util/Transform';
-import './GestureOverlay.scss';
-import { ObservableReactComponent } from './ObservableReactComponent';
-import { returnEmptyDocViewList } from './StyleProvider';
-import { ActiveFillColor, DocumentView } from './nodes/DocumentView';
-
export enum ToolglassTools {
InkToText = 'inktotext',
IgnoreGesture = 'ignoregesture',
@@ -41,10 +46,12 @@ interface GestureOverlayProps {
isActive: boolean;
}
@observer
+/**
+ * class for gestures. will determine if what the user drew is a gesture, and will transform the ink stroke into the shape the user
+ * drew or perform the gesture's action
+ */
export class GestureOverlay extends ObservableReactComponent<React.PropsWithChildren<GestureOverlayProps>> {
- // eslint-disable-next-line no-use-before-define
static Instance: GestureOverlay;
- // eslint-disable-next-line no-use-before-define
static Instances: GestureOverlay[] = [];
@observable public InkShape: Opt<Gestures> = undefined;
@@ -127,70 +134,331 @@ export class GestureOverlay extends ObservableReactComponent<React.PropsWithChil
// SetActiveArrowEnd('none');
}
}
+ /**
+ * this method returns if what the user drew is a scribble. if it is, it will determine what documents need
+ * to be deleted and then it will delete them.
+ * @returns
+ */
+ isScribble() {
+ const ffView = DocumentView.allViews().find(view => view.ComponentView instanceof CollectionFreeFormView);
+ const cuspArray = this.getCusps();
+ const cuspBooleanArray: boolean[] = [];
+ const docsToDelete: Doc[] = [];
+ const childDocs = (ffView?.ComponentView as CollectionFreeFormView).childDocs;
+ childDocs.filter(doc => doc.type === 'ink').map(doc => DocumentView.getDocumentView(doc, DocumentView.getDocumentView(doc)));
+ if ((ffView?.ComponentView as CollectionFreeFormView).childDocs) {
+ //how many cusps the scribble hsa
+ if (cuspArray.length > 4) {
+ for (let i = 0; i < cuspArray.length - 2; i++) {
+ let hasDocInTriangle = false;
+ for (const doc of childDocs) {
+ const point1 = cuspArray[i];
+ const point2 = cuspArray[i + 1];
+ const point3 = cuspArray[i + 2];
+ const triangleObject = { p1: { X: point1.X, Y: point1.Y }, p2: { X: point2.X, Y: point2.Y }, p3: { X: point3.X, Y: point3.Y } };
+ const otherInk = DocumentView.getDocumentView(doc)?.ComponentView as InkingStroke;
+ if (otherInk instanceof InkingStroke) {
+ const { inkData: otherInkData } = otherInk?.inkScaledData() ?? { inkData: [] };
+ const otherScreenPts = otherInkData.map(point => otherInk.ptToScreen(point));
+ if (doc.title === 'line') {
+ if (this.doesLineIntersectTriangle(otherScreenPts, triangleObject)) {
+ docsToDelete.push(doc);
+ hasDocInTriangle = true;
+ cuspBooleanArray.push(true);
+ }
+ } else {
+ if (this.isAnyPointInTriangle(triangleObject, otherScreenPts)) {
+ docsToDelete.push(doc);
+ hasDocInTriangle = true;
+ cuspBooleanArray.push(true);
+ }
+ }
+ }
+ }
+ cuspBooleanArray.push(hasDocInTriangle);
+ }
+ if (this.determineIfScribble(cuspBooleanArray)) {
+ docsToDelete.forEach(doc => {
+ ffView?.ComponentView?.removeDocument?.(doc);
+ });
+ this._points = [];
+ return true;
+ }
+ }
+ }
+ return false;
+ }
+ /**
+ * this method determines if what the user drew is a scribble based on certain criteria.
+ * @param cuspBooleanArray will take in an array of booleans tht represent what triangles in the scribble
+ * has an object in it. the 1st triangle is the 0th index, 2nd triangle is the 1st index and so on.
+ * @returns
+ */
+ determineIfScribble(cuspBooleanArray: boolean[]) {
+ if (!cuspBooleanArray) {
+ return false;
+ }
+ const quarterArrayLength = Math.ceil((cuspBooleanArray.length - 2) * 0.25);
+ let hasObjectInFirstAndLast25 = true;
+ for (let i = 0; i < quarterArrayLength; i++) {
+ if (cuspBooleanArray[i] == false || cuspBooleanArray[cuspBooleanArray.length - 1 - i] == false) {
+ hasObjectInFirstAndLast25 = false;
+ }
+ }
+ const trueCount = cuspBooleanArray.filter(value => value).length;
+ const percentageTrues = trueCount / cuspBooleanArray.length;
+ return percentageTrues > 0.65 || hasObjectInFirstAndLast25;
+ }
+ /**
+ * determines if two rectangles are overlapping each other
+ * @param rect1 the rectangle object has has a minX,maxX,minY, and maxY
+ * @param rect2
+ * @returns
+ */
+ // eslint-disable-next-line @typescript-eslint/no-explicit-any
+ isRectangleOverlap(rect1: any, rect2: any): boolean {
+ const noOverlap = rect1.maxX < rect2.minX || rect1.minX > rect2.maxX || rect1.maxY < rect2.minY || rect1.minY > rect2.maxY;
+
+ return !noOverlap;
+ }
+ /**
+ * determines if there is a point in a triangle used to determine what triangles in the scribble have an object
+ * @param pt the point in question
+ * @param triangle the triangle with 3 points
+ * @returns true or false if point is in triangle
+ */
+ isPointInTriangle(pt: { X: number; Y: number }, triangle: { p1: { X: number; Y: number }; p2: { X: number; Y: number }; p3: { X: number; Y: number } }): boolean {
+ const area = (v1: { X: number; Y: number }, v2: { X: number; Y: number }, v3: { X: number; Y: number }) => Math.abs((v1.X * (v2.Y - v3.Y) + v2.X * (v3.Y - v1.Y) + v3.X * (v1.Y - v2.Y)) / 2.0);
+
+ const A = area(triangle.p1, triangle.p2, triangle.p3);
+
+ const A1 = area(pt, triangle.p2, triangle.p3);
+ const A2 = area(triangle.p1, pt, triangle.p3);
+ const A3 = area(triangle.p1, triangle.p2, pt);
+
+ return A === A1 + A2 + A3;
+ }
+ /**
+ * determines if any points in an array are in a triangle
+ * @param triangle the triangle with 3 points
+ * @param points the point in question
+ * @returns true or false if point is in triangle
+ */
+ // eslint-disable-next-line @typescript-eslint/no-explicit-any
+ isAnyPointInTriangle(triangle: any, points: any[]): boolean {
+ for (const point of points) {
+ if (this.isPointInTriangle(point, triangle)) {
+ return true;
+ }
+ }
+ return false;
+ }
+ /**
+ * determines if a line intersects a triangle. used for scribble gesture since the line doesnt have a lot
+ * of points across is so isPointInTriangle will not work for it.
+ * @param line is pointData
+ * @param triangle triangle with 3 points
+ * @returns true or false if it intersects
+ */
+ // eslint-disable-next-line @typescript-eslint/no-explicit-any
+ doesLineIntersectTriangle(line: any, triangle: any): boolean {
+ const edges = [
+ { start: triangle.p1, end: triangle.p2 },
+ { start: triangle.p2, end: triangle.p3 },
+ { start: triangle.p3, end: triangle.p1 },
+ ];
+
+ for (const edge of edges) {
+ if (this.doLinesIntersect(line, edge)) {
+ return true;
+ }
+ }
+
+ return false;
+ }
+ /**
+ * used in doesLineIntersectTriangle, splits up the triangle into 3 lines and runs this method
+ * individually
+ * @param line1 point data where 0th index is the coordinate of beginnning of line and last index is the coordinate of end of limne
+ * @param line2 same thing
+ * @returns
+ */
+ // eslint-disable-next-line @typescript-eslint/no-explicit-any
+ doLinesIntersect(line1: any, line2: any): boolean {
+ const A = line1[0];
+ const B = line1[line1.length - 1];
+ const { start: C, end: D } = line2;
+ const denominator = (B.X - A.X) * (D.Y - C.Y) - (B.Y - A.Y) * (D.X - C.X);
+ if (denominator === 0) return false;
+
+ const numerator1 = (A.Y - C.Y) * (D.X - C.X) - (A.X - C.X) * (D.Y - C.Y);
+ const numerator2 = (A.Y - C.Y) * (B.X - A.X) - (A.X - C.X) * (B.Y - A.Y);
+
+ const r = numerator1 / denominator;
+ const s = numerator2 / denominator;
+
+ return r >= 0 && r <= 1 && s >= 0 && s <= 1;
+ }
+
+ dryInk = () => {
+ const newPoints = this._points.reduce((p, pts) => {
+ p.push([pts.X, pts.Y]);
+ return p;
+ }, [] as number[][]);
+ newPoints.pop();
+ const controlPoints: { X: number; Y: number }[] = [];
+
+ const bezierCurves = fitCurve.default(newPoints, 10);
+ Array.from(bezierCurves).forEach(curve => {
+ controlPoints.push({ X: curve[0][0], Y: curve[0][1] });
+ controlPoints.push({ X: curve[1][0], Y: curve[1][1] });
+ controlPoints.push({ X: curve[2][0], Y: curve[2][1] });
+ controlPoints.push({ X: curve[3][0], Y: curve[3][1] });
+ });
+ const dist = Math.sqrt((controlPoints[0].X - controlPoints.lastElement().X) * (controlPoints[0].X - controlPoints.lastElement().X) + (controlPoints[0].Y - controlPoints.lastElement().Y) * (controlPoints[0].Y - controlPoints.lastElement().Y));
+ if (controlPoints.length > 4 && dist < 10) controlPoints[controlPoints.length - 1] = controlPoints[0];
+ this._points.length = 0;
+ this._points.push(...controlPoints);
+ this.dispatchGesture(Gestures.Stroke);
+ };
@action
onPointerUp = () => {
DocumentView.DownDocView = undefined;
if (this._points.length > 1) {
const B = this.svgBounds;
const points = this._points.map(p => ({ X: p.X - B.left, Y: p.Y - B.top }));
-
+ const { Name, Score } =
+ (this.InkShape
+ ? new Result(this.InkShape, 1, Date.now)
+ : Doc.UserDoc().recognizeGestures && points.length > 2
+ ? GestureUtils.GestureRecognizer.Recognize([points])
+ : undefined) ??
+ new Result(Gestures.Stroke, 1, Date.now); // prettier-ignore
// if any of the shape is activated in the CollectionFreeFormViewChrome
- if (this.InkShape) {
- this.makeBezierPolygon(this.InkShape, false);
- this.dispatchGesture(this.InkShape);
- this.primCreated();
- }
- // if we're not drawing in a toolglass try to recognize as gesture
- else {
- // need to decide when to turn gestures back on
- const result = points.length > 2 && GestureUtils.GestureRecognizer.Recognize([points]);
- let actionPerformed = false;
- if (Doc.UserDoc().recognizeGestures && result && result.Score > 0.7) {
- switch (result.Name) {
- case Gestures.Line:
- case Gestures.Triangle:
- case Gestures.Rectangle:
- case Gestures.Circle:
- this.makeBezierPolygon(result.Name, true);
- actionPerformed = this.dispatchGesture(result.Name);
- break;
- case Gestures.Scribble:
- console.log('scribble');
- break;
- default:
- }
- }
-
- // if no gesture (or if the gesture was unsuccessful), "dry" the stroke into an ink document
- if (!actionPerformed) {
- const newPoints = this._points.reduce((p, pts) => {
- p.push([pts.X, pts.Y]);
- return p;
- }, [] as number[][]);
- newPoints.pop();
- const controlPoints: { X: number; Y: number }[] = [];
-
- const bezierCurves = fitCurve.default(newPoints, 10);
- Array.from(bezierCurves).forEach(curve => {
- controlPoints.push({ X: curve[0][0], Y: curve[0][1] });
- controlPoints.push({ X: curve[1][0], Y: curve[1][1] });
- controlPoints.push({ X: curve[2][0], Y: curve[2][1] });
- controlPoints.push({ X: curve[3][0], Y: curve[3][1] });
- });
- const dist = Math.sqrt(
- (controlPoints[0].X - controlPoints.lastElement().X) * (controlPoints[0].X - controlPoints.lastElement().X) + (controlPoints[0].Y - controlPoints.lastElement().Y) * (controlPoints[0].Y - controlPoints.lastElement().Y)
- );
- // eslint-disable-next-line prefer-destructuring
- if (controlPoints.length > 4 && dist < 10) controlPoints[controlPoints.length - 1] = controlPoints[0];
- this._points.length = 0;
- this._points.push(...controlPoints);
- this.dispatchGesture(Gestures.Stroke);
+ // need to decide when to turn gestures back on
+ const actionPerformed = ((name: Gestures) => {
+ switch (name) {
+ case Gestures.Line:
+ case Gestures.Triangle:
+ case Gestures.Rectangle:
+ case Gestures.Circle:
+ this.makeBezierPolygon(Name, true);
+ return this.dispatchGesture(name);
+ case Gestures.RightAngle:
+ return this.convertToText().length > 0;
+ default:
+ case Gestures.Stroke:
+ return this.isScribble();
}
- }
+ })(Score < 0.7 ? Gestures.Stroke : (Name as Gestures));
+ // if no gesture (or if the gesture was unsuccessful), "dry" the stroke into an ink document
+ if (!actionPerformed) this.dryInk();
}
this._points.length = 0;
};
+ /**
+ * used in the rightAngle gesture to convert handwriting into text. will only work on collections
+ * TODO: make it work on individual ink docs.
+ */
+ convertToText() {
+ const ffView = DocumentView.allViews().find(view => view.ComponentView instanceof CollectionFreeFormView);
+ let minX = 999999999;
+ let maxX = -999999999;
+ let minY = 999999999;
+ let maxY = -999999999;
+ const textDocs: Doc[] = [];
+ (ffView?.ComponentView as CollectionFreeFormView).childDocs
+ .filter(doc => doc.type === DocumentType.COL)
+ .forEach(doc => {
+ if (typeof doc.width === 'number' && typeof doc.height === 'number' && typeof doc.x === 'number' && typeof doc.y === 'number') {
+ const bounds = DocumentView.getDocumentView(doc)?.getBounds;
+ if (bounds) {
+ const rect1 = { minX: bounds.left, maxX: bounds.right, minY: bounds.top, maxY: bounds.bottom };
+ if (this.isRectangleOverlap(rect1, this.getExtremeCoordinates())) {
+ if (doc.x < minX) {
+ minX = doc.x;
+ }
+ if (doc.x > maxX) {
+ maxX = doc.x;
+ }
+ if (doc.y < minY) {
+ minY = doc.y;
+ }
+ if (doc.y + doc.height > maxY) {
+ maxY = doc.y + doc.height;
+ }
+ const newDoc = Docs.Create.TextDocument(doc.transcription as string, { title: '', x: doc.x as number, y: minY });
+ newDoc.height = doc.height;
+ newDoc.width = doc.width;
+ if (ffView?.ComponentView?.addDocument && ffView?.ComponentView?.removeDocument) {
+ ffView.ComponentView.addDocument(newDoc);
+ ffView.ComponentView.removeDocument(doc);
+ }
+ textDocs.push(newDoc);
+ }
+ }
+ }
+ });
+ return textDocs;
+ }
+ /**
+ * used to determine how many cusps and where the cusps are in order
+ * @returns will return an array containing the coordinates of the sharp cusps
+ */
+ getCusps() {
+ const points = this._points.map(p => ({ X: p.X, Y: p.Y }));
+ const arrayOfPoints: { X: number; Y: number }[] = [];
+ arrayOfPoints.push(points[0]);
+ for (let i = 0; i < points.length - 2; i++) {
+ const point1 = points[i];
+ const point2 = points[i + 1];
+ const point3 = points[i + 2];
+ if (this.find_angle(point1, point2, point3) < 90) {
+ arrayOfPoints.push(point2);
+ }
+ }
+ arrayOfPoints.push(points[points.length - 1]);
+ return arrayOfPoints;
+ }
+ /**
+ * will look through an array of point data and return the coordinates of the smallest box that can fit all the points
+ * @returns the minX,maxX,minY,maxY of the box
+ */
+ getExtremeCoordinates() {
+ const coordinates = this._points;
+ if (coordinates.length === 0) {
+ throw new Error('Coordinates array is empty');
+ }
+ let minX = coordinates[0].X;
+ let maxX = coordinates[0].X;
+ let minY = coordinates[0].Y;
+ let maxY = coordinates[0].Y;
+
+ coordinates.forEach(coord => {
+ if (coord.X < minX) minX = coord.X;
+ if (coord.X > maxX) maxX = coord.X;
+ if (coord.Y < minY) minY = coord.Y;
+ if (coord.Y > maxY) maxY = coord.Y;
+ });
+ return {
+ minX,
+ maxX,
+ minY,
+ maxY,
+ };
+ }
+ /**
+ * takes in three points and then determines the angle of the points. used to determine if the cusp
+ * is sharp enoug
+ * @returns
+ */
+ find_angle(A: { X: number; Y: number }, B: { X: number; Y: number }, C: { X: number; Y: number }) {
+ const AB = Math.sqrt(Math.pow(B.X - A.X, 2) + Math.pow(B.Y - A.Y, 2));
+ const BC = Math.sqrt(Math.pow(B.X - C.X, 2) + Math.pow(B.Y - C.Y, 2));
+ const AC = Math.sqrt(Math.pow(C.X - A.X, 2) + Math.pow(C.Y - A.Y, 2));
+ return Math.acos((BC * BC + AB * AB - AC * AC) / (2 * BC * AB)) * (180 / Math.PI);
+ }
makeBezierPolygon = (shape: string, gesture: boolean) => {
const xs = this._points.map(p => p.X);
const ys = this._points.map(p => p.Y);
@@ -389,7 +657,6 @@ export class GestureOverlay extends ObservableReactComponent<React.PropsWithChil
this._strokes.map((l, i) => {
const b = { left: -20000, right: 20000, top: -20000, bottom: 20000, width: 40000, height: 40000 }; // this.getBounds(l, true);
return (
- // eslint-disable-next-line react/no-array-index-key
<svg key={i} width={b.width} height={b.height} style={{ top: 0, left: 0, transform: `translate(${b.left}px, ${b.top}px)`, pointerEvents: 'none', position: 'absolute', zIndex: 30000, overflow: 'visible' }}>
{InteractionUtils.CreatePolyline(
l,
@@ -511,7 +778,6 @@ export class GestureOverlay extends ObservableReactComponent<React.PropsWithChil
}
ScriptingGlobals.add('GestureOverlay', GestureOverlay);
-// eslint-disable-next-line prefer-arrow-callback
ScriptingGlobals.add(function setPen(width: string, color: string, fill: string, arrowStart: string, arrowEnd: string, dash: string) {
runInAction(() => {
GestureOverlay.Instance.SavedColor = ActiveInkColor();
@@ -524,7 +790,6 @@ ScriptingGlobals.add(function setPen(width: string, color: string, fill: string,
SetActiveDash(dash);
});
});
-// eslint-disable-next-line prefer-arrow-callback
ScriptingGlobals.add(function resetPen() {
runInAction(() => {
SetActiveInkColor(GestureOverlay.Instance.SavedColor ?? 'rgb(0, 0, 0)');
@@ -532,7 +797,6 @@ ScriptingGlobals.add(function resetPen() {
});
}, 'resets the pen tool');
ScriptingGlobals.add(
- // eslint-disable-next-line prefer-arrow-callback
function createText(text: string, X: number, Y: number) {
GestureOverlay.Instance.dispatchGesture(Gestures.Text, [{ X, Y }], text);
},
diff --git a/src/client/views/InkTranscription.scss b/src/client/views/InkTranscription.scss
index bbb0a1afa..c77117ccc 100644
--- a/src/client/views/InkTranscription.scss
+++ b/src/client/views/InkTranscription.scss
@@ -2,4 +2,7 @@
.error-msg {
display: none !important;
}
-} \ No newline at end of file
+ .ms-editor {
+ top: 1000px;
+ }
+}
diff --git a/src/client/views/InkTranscription.tsx b/src/client/views/InkTranscription.tsx
index 33db72960..e5f47c1c3 100644
--- a/src/client/views/InkTranscription.tsx
+++ b/src/client/views/InkTranscription.tsx
@@ -1,350 +1,415 @@
-// import * as iink from 'iink-js';
-// import { action, observable } from 'mobx';
-// import * as React from 'react';
-// import { Doc, DocListCast } from '../../fields/Doc';
-// import { InkData, InkField, InkTool } from '../../fields/InkField';
-// import { Cast, DateCast, NumCast } from '../../fields/Types';
-// import { aggregateBounds } from '../../Utils';
-// import { DocumentType } from '../documents/DocumentTypes';
-// import { CollectionFreeFormView } from './collections/collectionFreeForm';
-// import { InkingStroke } from './InkingStroke';
-// import './InkTranscription.scss';
-
-// /**
-// * Class component that handles inking in writing mode
-// */
-// export class InkTranscription extends React.Component {
-// static Instance: InkTranscription;
-
-// @observable _mathRegister: any= undefined;
-// @observable _mathRef: any= undefined;
-// @observable _textRegister: any= undefined;
-// @observable _textRef: any= undefined;
-// private lastJiix: any;
-// private currGroup?: Doc;
-
-// constructor(props: Readonly<{}>) {
-// super(props);
-
-// InkTranscription.Instance = this;
-// }
-
-// componentWillUnmount() {
-// this._mathRef.removeEventListener('exported', (e: any) => this.exportInk(e, this._mathRef));
-// this._textRef.removeEventListener('exported', (e: any) => this.exportInk(e, this._textRef));
-// }
-
-// @action
-// setMathRef = (r: any) => {
-// if (!this._mathRegister) {
-// this._mathRegister = r
-// ? iink.register(r, {
-// recognitionParams: {
-// type: 'MATH',
-// protocol: 'WEBSOCKET',
-// server: {
-// host: 'cloud.myscript.com',
-// applicationKey: process.env.IINKJS_APP,
-// hmacKey: process.env.IINKJS_HMAC,
-// websocket: {
-// pingEnabled: false,
-// autoReconnect: true,
-// },
-// },
-// iink: {
-// math: {
-// mimeTypes: ['application/x-latex', 'application/vnd.myscript.jiix'],
-// },
-// export: {
-// jiix: {
-// strokes: true,
-// },
-// },
-// },
-// },
-// })
-// : null;
-// }
-
-// r?.addEventListener('exported', (e: any) => this.exportInk(e, this._mathRef));
-
-// return (this._mathRef = r);
-// };
-
-// @action
-// setTextRef = (r: any) => {
-// if (!this._textRegister) {
-// this._textRegister = r
-// ? iink.register(r, {
-// recognitionParams: {
-// type: 'TEXT',
-// protocol: 'WEBSOCKET',
-// server: {
-// host: 'cloud.myscript.com',
-// applicationKey: '7277ec34-0c2e-4ee1-9757-ccb657e3f89f',
-// hmacKey: 'f5cb18f2-1f95-4ddb-96ac-3f7c888dffc1',
-// websocket: {
-// pingEnabled: false,
-// autoReconnect: true,
-// },
-// },
-// iink: {
-// text: {
-// mimeTypes: ['text/plain'],
-// },
-// export: {
-// jiix: {
-// strokes: true,
-// },
-// },
-// },
-// },
-// })
-// : null;
-// }
-
-// r?.addEventListener('exported', (e: any) => this.exportInk(e, this._textRef));
-
-// return (this._textRef = r);
-// };
-
-// /**
-// * Handles processing Dash Doc data for ink transcription.
-// *
-// * @param groupDoc the group which contains the ink strokes we want to transcribe
-// * @param inkDocs the ink docs contained within the selected group
-// * @param math boolean whether to do math transcription or not
-// */
-// transcribeInk = (groupDoc: Doc | undefined, inkDocs: Doc[], math: boolean) => {
-// if (!groupDoc) return;
-// const validInks = inkDocs.filter(s => s.type === DocumentType.INK);
-
-// const strokes: InkData[] = [];
-// const times: number[] = [];
-// validInks
-// .filter(i => Cast(i[Doc.LayoutFieldKey(i)], InkField))
-// .forEach(i => {
-// const d = Cast(i[Doc.LayoutFieldKey(i)], InkField, null);
-// const inkStroke = DocumentManager.Instance.getDocumentView(i)?.ComponentView as InkingStroke;
-// strokes.push(d.inkData.map(pd => inkStroke.ptToScreen({ X: pd.X, Y: pd.Y })));
-// times.push(DateCast(i.author_date).getDate().getTime());
-// });
-
-// this.currGroup = groupDoc;
-
-// const pointerData = { events: strokes.map((stroke, i) => this.inkJSON(stroke, times[i])) };
-// const processGestures = false;
-
-// if (math) {
-// this._mathRef.editor.pointerEvents(pointerData, processGestures);
-// } else {
-// this._textRef.editor.pointerEvents(pointerData, processGestures);
-// }
-// };
-
-// /**
-// * Converts the Dash Ink Data to JSON.
-// *
-// * @param stroke The dash ink data
-// * @param time the time of the stroke
-// * @returns json object representation of ink data
-// */
-// inkJSON = (stroke: InkData, time: number) => {
-// return {
-// pointerType: 'PEN',
-// pointerId: 1,
-// x: stroke.map(point => point.X),
-// y: stroke.map(point => point.Y),
-// t: new Array(stroke.length).fill(time),
-// p: new Array(stroke.length).fill(1.0),
-// };
-// };
-
-// /**
-// * Creates subgroups for each word for the whole text transcription
-// * @param wordInkDocMap the mapping of words to ink strokes (Ink Docs)
-// */
-// subgroupsTranscriptions = (wordInkDocMap: Map<string, Doc[]>) => {
-// // iterate through the keys of wordInkDocMap
-// wordInkDocMap.forEach(async (inkDocs: Doc[], word: string) => {
-// const selected = inkDocs.slice();
-// if (!selected) {
-// return;
-// }
-// const ctx = await Cast(selected[0].embedContainer, Doc);
-// if (!ctx) {
-// return;
-// }
-// const docView: CollectionFreeFormView = DocumentManager.Instance.getDocumentView(ctx)?.ComponentView as CollectionFreeFormView;
-
-// if (!docView) return;
-// const marqViewRef = docView._marqueeViewRef.current;
-// if (!marqViewRef) return;
-// this.groupInkDocs(selected, docView, word);
-// });
-// };
-
-// /**
-// * Event listener function for when the 'exported' event is heard.
-// *
-// * @param e the event objects
-// * @param ref the ref to the editor
-// */
-// exportInk = (e: any, ref: any) => {
-// const exports = e.detail.exports;
-// if (exports) {
-// if (exports['application/x-latex']) {
-// const latex = exports['application/x-latex'];
-// if (this.currGroup) {
-// this.currGroup.text = latex;
-// this.currGroup.title = latex;
-// }
-
-// ref.editor.clear();
-// } else if (exports['text/plain']) {
-// if (exports['application/vnd.myscript.jiix']) {
-// this.lastJiix = JSON.parse(exports['application/vnd.myscript.jiix']);
-// // map timestamp to strokes
-// const timestampWord = new Map<number, string>();
-// this.lastJiix.words.map((word: any) => {
-// if (word.items) {
-// word.items.forEach((i: { id: string; timestamp: string; X: Array<number>; Y: Array<number>; F: Array<number> }) => {
-// const ms = Date.parse(i.timestamp);
-// timestampWord.set(ms, word.label);
-// });
-// }
-// });
-
-// const wordInkDocMap = new Map<string, Doc[]>();
-// if (this.currGroup) {
-// const docList = DocListCast(this.currGroup.data);
-// docList.forEach((inkDoc: Doc) => {
-// // just having the times match up and be a unique value (actual timestamp doesn't matter)
-// const ms = DateCast(inkDoc.author_date).getDate().getTime() + 14400000;
-// const word = timestampWord.get(ms);
-// if (!word) {
-// return;
-// }
-// const entry = wordInkDocMap.get(word);
-// if (entry) {
-// entry.push(inkDoc);
-// wordInkDocMap.set(word, entry);
-// } else {
-// const newEntry = [inkDoc];
-// wordInkDocMap.set(word, newEntry);
-// }
-// });
-// if (this.lastJiix.words.length > 1) this.subgroupsTranscriptions(wordInkDocMap);
-// }
-// }
-// const text = exports['text/plain'];
-
-// if (this.currGroup) {
-// this.currGroup.text = text; // transcription text
-// this.currGroup.icon_fieldKey = 'transcription'; // use the transcription icon template when iconifying
-// this.currGroup.title = text.split('\n')[0];
-// }
-
-// ref.editor.clear();
-// }
-// }
-// };
-
-// /**
-// * Creates the ink grouping once the user leaves the writing mode.
-// */
-// createInkGroup() {
-// // TODO nda - if document being added to is a inkGrouping then we can just add to that group
-// if (Doc.ActiveTool === InkTool.Write) {
-// CollectionFreeFormView.collectionsWithUnprocessedInk.forEach(ffView => {
-// // TODO: nda - will probably want to go through ffView unprocessed docs and then see if any of the inksToGroup docs are in it and only use those
-// const selected = ffView.unprocessedDocs;
-// const newCollection = this.groupInkDocs(
-// selected.filter(doc => doc.embedContainer),
-// ffView
-// );
-// ffView.unprocessedDocs = [];
-
-// InkTranscription.Instance.transcribeInk(newCollection, selected, false);
-// });
-// }
-// CollectionFreeFormView.collectionsWithUnprocessedInk.clear();
-// }
-
-// /**
-// * Creates the groupings for a given list of ink docs on a specific doc view
-// * @param selected: the list of ink docs to create a grouping of
-// * @param docView: the view in which we want the grouping to be created
-// * @param word: optional param if the group we are creating is a word (subgrouping individual words)
-// * @returns a new collection Doc or undefined if the grouping fails
-// */
-// groupInkDocs(selected: Doc[], docView: CollectionFreeFormView, word?: string): Doc | undefined {
-// const bounds: { x: number; y: number; width?: number; height?: number }[] = [];
-
-// // calculate the necessary bounds from the selected ink docs
-// selected.map(
-// action(d => {
-// const x = NumCast(d.x);
-// const y = NumCast(d.y);
-// const width = NumCast(d._width);
-// const height = NumCast(d._height);
-// bounds.push({ x, y, width, height });
-// })
-// );
-
-// // calculate the aggregated bounds
-// const aggregBounds = aggregateBounds(bounds, 0, 0);
-// const marqViewRef = docView._marqueeViewRef.current;
-
-// // set the vals for bounds in marqueeView
-// if (marqViewRef) {
-// marqViewRef._downX = aggregBounds.x;
-// marqViewRef._downY = aggregBounds.y;
-// marqViewRef._lastX = aggregBounds.r;
-// marqViewRef._lastY = aggregBounds.b;
-// }
-
-// // map through all the selected ink strokes and create the groupings
-// selected.map(
-// action(d => {
-// const dx = NumCast(d.x);
-// const dy = NumCast(d.y);
-// delete d.x;
-// delete d.y;
-// delete d.activeFrame;
-// delete d._timecodeToShow; // bcz: this should be automatic somehow.. along with any other properties that were logically associated with the original collection
-// delete d._timecodeToHide; // bcz: this should be automatic somehow.. along with any other properties that were logically associated with the original collection
-// // calculate pos based on bounds
-// if (marqViewRef?.Bounds) {
-// d.x = dx - marqViewRef.Bounds.left - marqViewRef.Bounds.width / 2;
-// d.y = dy - marqViewRef.Bounds.top - marqViewRef.Bounds.height / 2;
-// }
-// return d;
-// })
-// );
-// docView.props.removeDocument?.(selected);
-// // Gets a collection based on the selected nodes using a marquee view ref
-// const newCollection = marqViewRef?.getCollection(selected, undefined, true);
-// if (newCollection) {
-// newCollection.width = NumCast(newCollection._width);
-// newCollection.height = NumCast(newCollection._height);
-// // if the grouping we are creating is an individual word
-// if (word) {
-// newCollection.title = word;
-// }
-// }
-
-// // nda - bug: when deleting a stroke before leaving writing mode, delete the stroke from unprocessed ink docs
-// newCollection && docView.props.addDocument?.(newCollection);
-// return newCollection;
-// }
-
-// render() {
-// return (
-// <div className="ink-transcription">
-// <div className="math-editor" ref={this.setMathRef} touch-action="none"></div>
-// <div className="text-editor" ref={this.setTextRef} touch-action="none"></div>
-// </div>
-// );
-// }
-// }
+import * as iink from 'iink-ts';
+import { action, observable } from 'mobx';
+import * as React from 'react';
+import { Doc, DocListCast } from '../../fields/Doc';
+import { InkData, InkField, InkTool } from '../../fields/InkField';
+import { Cast, DateCast, ImageCast, NumCast } from '../../fields/Types';
+import { aggregateBounds } from '../../Utils';
+import { DocumentType } from '../documents/DocumentTypes';
+import { CollectionFreeFormView } from './collections/collectionFreeForm';
+import { InkingStroke } from './InkingStroke';
+import './InkTranscription.scss';
+import { Docs } from '../documents/Documents';
+import { DocumentView } from './nodes/DocumentView';
+import { ImageField } from '../../fields/URLField';
+import { gptHandwriting } from '../apis/gpt/GPT';
+import { URLField } from '../../fields/URLField';
+/**
+ * Class component that handles inking in writing mode
+ */
+export class InkTranscription extends React.Component {
+ static Instance: InkTranscription;
+
+ // eslint-disable-next-line @typescript-eslint/no-explicit-any
+ @observable _mathRegister: any = undefined;
+ // eslint-disable-next-line @typescript-eslint/no-explicit-any
+ @observable _mathRef: any = undefined;
+ // eslint-disable-next-line @typescript-eslint/no-explicit-any
+ @observable _textRegister: any = undefined;
+ // eslint-disable-next-line @typescript-eslint/no-explicit-any
+ @observable _textRef: any = undefined;
+ // eslint-disable-next-line @typescript-eslint/no-explicit-any
+ @observable iinkEditor: any = undefined;
+ // eslint-disable-next-line @typescript-eslint/no-explicit-any
+ private lastJiix: any;
+ private currGroup?: Doc;
+ private collectionFreeForm?: CollectionFreeFormView;
+
+ constructor(props: Readonly<object>) {
+ super(props);
+
+ InkTranscription.Instance = this;
+ }
+ @action
+ // eslint-disable-next-line @typescript-eslint/no-explicit-any
+ setMathRef = async (r: any) => {
+ if (!this._textRegister && r) {
+ const options = {
+ configuration: {
+ server: {
+ scheme: 'https',
+ host: 'cloud.myscript.com',
+ applicationKey: 'c0901093-5ac5-4454-8e64-0def0f13f2ca',
+ hmacKey: 'f6465cca-1856-4492-a6a4-e2395841be2f',
+ protocol: 'WEBSOCKET',
+ },
+ recognition: {
+ type: 'TEXT',
+ },
+ },
+ };
+ // eslint-disable-next-line @typescript-eslint/no-explicit-any
+ const editor = new iink.Editor(r, options as any);
+
+ await editor.initialize();
+
+ this._textRegister = r;
+ // eslint-disable-next-line @typescript-eslint/no-explicit-any
+ r?.addEventListener('exported', (e: any) => this.exportInk(e, this._textRef));
+
+ return (this._textRef = r);
+ }
+ };
+ @action
+ // eslint-disable-next-line @typescript-eslint/no-explicit-any
+ setTextRef = async (r: any) => {
+ if (!this._textRegister && r) {
+ const options = {
+ configuration: {
+ server: {
+ scheme: 'https',
+ host: 'cloud.myscript.com',
+ applicationKey: 'c0901093-5ac5-4454-8e64-0def0f13f2ca',
+ hmacKey: 'f6465cca-1856-4492-a6a4-e2395841be2f',
+ protocol: 'WEBSOCKET',
+ },
+ recognition: {
+ type: 'TEXT',
+ },
+ },
+ };
+ // eslint-disable-next-line @typescript-eslint/no-explicit-any
+ const editor = new iink.Editor(r, options as any);
+
+ await editor.initialize();
+ this.iinkEditor = editor;
+ this._textRegister = r;
+ // eslint-disable-next-line @typescript-eslint/no-explicit-any
+ r?.addEventListener('exported', (e: any) => this.exportInk(e, this._textRef));
+
+ return (this._textRef = r);
+ }
+ };
+
+ /**
+ * Handles processing Dash Doc data for ink transcription.
+ *
+ * @param groupDoc the group which contains the ink strokes we want to transcribe
+ * @param inkDocs the ink docs contained within the selected group
+ * @param math boolean whether to do math transcription or not
+ */
+ transcribeInk = (groupDoc: Doc | undefined, inkDocs: Doc[], math: boolean) => {
+ if (!groupDoc) return;
+ const validInks = inkDocs.filter(s => s.type === DocumentType.INK);
+
+ const strokes: InkData[] = [];
+
+ const times: number[] = [];
+ validInks
+ .filter(i => Cast(i[Doc.LayoutFieldKey(i)], InkField))
+ .forEach(i => {
+ const d = Cast(i[Doc.LayoutFieldKey(i)], InkField, null);
+ const inkStroke = DocumentView.getDocumentView(i)?.ComponentView as InkingStroke;
+ strokes.push(d.inkData.map(pd => inkStroke.ptToScreen({ X: pd.X, Y: pd.Y })));
+ times.push(DateCast(i.author_date).getDate().getTime());
+ });
+ this.currGroup = groupDoc;
+ const pointerData = strokes.map((stroke, i) => this.inkJSON(stroke, times[i]));
+ if (math) {
+ this.iinkEditor.importPointEvents(pointerData);
+ } else {
+ this.iinkEditor.importPointEvents(pointerData);
+ }
+ };
+ convertPointsToString(points: InkData[]): string {
+ return points[0].map(point => `new Point(${point.X}, ${point.Y})`).join(',');
+ }
+ convertPointsToString2(points: InkData[]): string {
+ return points[0].map(point => `(${point.X},${point.Y})`).join(',');
+ }
+
+ /**
+ * Converts the Dash Ink Data to JSON.
+ *
+ * @param stroke The dash ink data
+ * @param time the time of the stroke
+ * @returns json object representation of ink data
+ */
+ inkJSON = (stroke: InkData, time: number) => {
+ interface strokeData {
+ x: number;
+ y: number;
+ t: number;
+ p: number;
+ }
+ const strokeObjects: strokeData[] = [];
+ stroke.forEach(point => {
+ const tempObject: strokeData = {
+ x: point.X,
+ y: point.Y,
+ t: time,
+ p: 1.0,
+ };
+ strokeObjects.push(tempObject);
+ });
+ return {
+ pointerType: 'PEN',
+ pointerId: 1,
+ pointers: strokeObjects,
+ };
+ };
+
+ /**
+ * Creates subgroups for each word for the whole text transcription
+ * @param wordInkDocMap the mapping of words to ink strokes (Ink Docs)
+ */
+ subgroupsTranscriptions = (wordInkDocMap: Map<string, Doc[]>) => {
+ // iterate through the keys of wordInkDocMap
+ wordInkDocMap.forEach(async (inkDocs: Doc[], word: string) => {
+ const selected = inkDocs.slice();
+ if (!selected) {
+ return;
+ }
+ const ctx = await Cast(selected[0].embedContainer, Doc);
+ if (!ctx) {
+ return;
+ }
+ const docView: CollectionFreeFormView = DocumentView.getDocumentView(ctx)?.ComponentView as CollectionFreeFormView;
+ // DocumentManager.Instance.getDocumentView(ctx)?.ComponentView as CollectionFreeFormView;
+
+ if (!docView) return;
+ const marqViewRef = docView._marqueeViewRef.current;
+ if (!marqViewRef) return;
+ this.groupInkDocs(selected, docView, word);
+ });
+ };
+
+ /**
+ * Event listener function for when the 'exported' event is heard.
+ *
+ * @param e the event objects
+ * @param ref the ref to the editor
+ */
+ // eslint-disable-next-line @typescript-eslint/no-explicit-any
+ exportInk = async (e: any, ref: any) => {
+ const exports = e.detail['application/vnd.myscript.jiix'];
+ if (exports) {
+ if (exports['type'] == 'Math') {
+ const latex = exports['application/x-latex'];
+ if (this.currGroup) {
+ this.currGroup.text = latex;
+ this.currGroup.title = latex;
+ }
+
+ ref.editor.clear();
+ } else if (exports['type'] == 'Text') {
+ if (exports['application/vnd.myscript.jiix']) {
+ this.lastJiix = JSON.parse(exports['application/vnd.myscript.jiix']);
+ // map timestamp to strokes
+ const timestampWord = new Map<number, string>();
+ // eslint-disable-next-line @typescript-eslint/no-explicit-any
+ this.lastJiix.words.map((word: any) => {
+ if (word.items) {
+ word.items.forEach((i: { id: string; timestamp: string; X: Array<number>; Y: Array<number>; F: Array<number> }) => {
+ const ms = Date.parse(i.timestamp);
+ timestampWord.set(ms, word.label);
+ });
+ }
+ });
+
+ const wordInkDocMap = new Map<string, Doc[]>();
+ if (this.currGroup) {
+ const docList = DocListCast(this.currGroup.data);
+ docList.forEach((inkDoc: Doc) => {
+ // just having the times match up and be a unique value (actual timestamp doesn't matter)
+ const ms = DateCast(inkDoc.author_date).getDate().getTime() + 14400000;
+ const word = timestampWord.get(ms);
+ if (!word) {
+ return;
+ }
+ const entry = wordInkDocMap.get(word);
+ if (entry) {
+ entry.push(inkDoc);
+ wordInkDocMap.set(word, entry);
+ } else {
+ const newEntry = [inkDoc];
+ wordInkDocMap.set(word, newEntry);
+ }
+ });
+ if (this.lastJiix.words.length > 1) this.subgroupsTranscriptions(wordInkDocMap);
+ }
+ }
+ const text = exports['label'];
+
+ if (this.currGroup && text) {
+ DocumentView.getDocumentView(this.currGroup)?.ComponentView?.updateIcon?.();
+ const image = await this.getIcon();
+ const { href } = (image as URLField).url;
+ const hrefParts = href.split('.');
+ const hrefComplete = `${hrefParts[0]}_o.${hrefParts[1]}`;
+ let response;
+ try {
+ const hrefBase64 = await this.imageUrlToBase64(hrefComplete);
+ response = await gptHandwriting(hrefBase64);
+ } catch {
+ console.error('Error getting image');
+ }
+ const textBoxText = 'iink: ' + text + '\n' + '\n' + 'ChatGPT: ' + response;
+ this.currGroup.transcription = response;
+ this.currGroup.title = response;
+ if (!this.currGroup.hasTextBox) {
+ const newDoc = Docs.Create.TextDocument(textBoxText, { title: '', x: this.currGroup.x as number, y: (this.currGroup.y as number) + (this.currGroup.height as number) });
+ newDoc.height = 200;
+ this.collectionFreeForm?.addDocument(newDoc);
+ this.currGroup.hasTextBox = true;
+ }
+ ref.editor.clear();
+ }
+ }
+ }
+ };
+ /**
+ * gets the icon of the collection that was just made
+ * @returns the image of the collection
+ */
+ async getIcon() {
+ const docView = DocumentView.getDocumentView(this.currGroup);
+ if (docView) {
+ docView.ComponentView?.updateIcon?.();
+ return new Promise<ImageField | undefined>(res => setTimeout(() => res(ImageCast(docView.Document.icon)), 1000));
+ }
+ return undefined;
+ }
+ /**
+ * converts the image to base url formate
+ * @param imageUrl imageurl taken from the collection icon
+ */
+ imageUrlToBase64 = async (imageUrl: string): Promise<string> => {
+ try {
+ const response = await fetch(imageUrl);
+ const blob = await response.blob();
+
+ return new Promise((resolve, reject) => {
+ const reader = new FileReader();
+ reader.readAsDataURL(blob);
+ reader.onloadend = () => resolve(reader.result as string);
+ reader.onerror = error => reject(error);
+ });
+ } catch (error) {
+ console.error('Error:', error);
+ throw error;
+ }
+ };
+
+ /**
+ * Creates the ink grouping once the user leaves the writing mode.
+ */
+ createInkGroup() {
+ // TODO nda - if document being added to is a inkGrouping then we can just add to that group
+ if (Doc.ActiveTool === InkTool.Write) {
+ CollectionFreeFormView.collectionsWithUnprocessedInk.forEach(ffView => {
+ // TODO: nda - will probably want to go through ffView unprocessed docs and then see if any of the inksToGroup docs are in it and only use those
+ const selected = ffView.unprocessedDocs;
+ const newCollection = this.groupInkDocs(
+ selected.filter(doc => doc.embedContainer),
+ ffView
+ );
+ ffView.unprocessedDocs = [];
+
+ InkTranscription.Instance.transcribeInk(newCollection, selected, false);
+ });
+ }
+ CollectionFreeFormView.collectionsWithUnprocessedInk.clear();
+ }
+
+ /**
+ * Creates the groupings for a given list of ink docs on a specific doc view
+ * @param selected: the list of ink docs to create a grouping of
+ * @param docView: the view in which we want the grouping to be created
+ * @param word: optional param if the group we are creating is a word (subgrouping individual words)
+ * @returns a new collection Doc or undefined if the grouping fails
+ */
+ groupInkDocs(selected: Doc[], docView: CollectionFreeFormView, word?: string): Doc | undefined {
+ this.collectionFreeForm = docView;
+ const bounds: { x: number; y: number; width?: number; height?: number }[] = [];
+
+ // calculate the necessary bounds from the selected ink docs
+ selected.map(
+ action(d => {
+ const x = NumCast(d.x);
+ const y = NumCast(d.y);
+ const width = NumCast(d._width);
+ const height = NumCast(d._height);
+ bounds.push({ x, y, width, height });
+ })
+ );
+
+ // calculate the aggregated bounds
+ const aggregBounds = aggregateBounds(bounds, 0, 0);
+ const marqViewRef = docView._marqueeViewRef.current;
+
+ // set the vals for bounds in marqueeView
+ if (marqViewRef) {
+ marqViewRef._downX = aggregBounds.x;
+ marqViewRef._downY = aggregBounds.y;
+ marqViewRef._lastX = aggregBounds.r;
+ marqViewRef._lastY = aggregBounds.b;
+ }
+
+ // map through all the selected ink strokes and create the groupings
+ selected.map(
+ action(d => {
+ const dx = NumCast(d.x);
+ const dy = NumCast(d.y);
+ delete d.x;
+ delete d.y;
+ delete d.activeFrame;
+ delete d._timecodeToShow; // bcz: this should be automatic somehow.. along with any other properties that were logically associated with the original collection
+ delete d._timecodeToHide; // bcz: this should be automatic somehow.. along with any other properties that were logically associated with the original collection
+ // calculate pos based on bounds
+ if (marqViewRef?.Bounds) {
+ d.x = dx - marqViewRef.Bounds.left - marqViewRef.Bounds.width / 2;
+ d.y = dy - marqViewRef.Bounds.top - marqViewRef.Bounds.height / 2;
+ }
+ return d;
+ })
+ );
+ docView.props.removeDocument?.(selected);
+ // Gets a collection based on the selected nodes using a marquee view ref
+ const newCollection = marqViewRef?.getCollection(selected, undefined, true);
+ if (newCollection) {
+ newCollection.width = NumCast(newCollection._width);
+ newCollection.height = NumCast(newCollection._height);
+ // if the grouping we are creating is an individual word
+ if (word) {
+ newCollection.title = word;
+ }
+ }
+
+ // nda - bug: when deleting a stroke before leaving writing mode, delete the stroke from unprocessed ink docs
+ if (newCollection) {
+ docView.props.addDocument?.(newCollection);
+ newCollection.hasTextBox = false;
+ }
+ return newCollection;
+ }
+
+ render() {
+ return (
+ <div className="ink-transcription">
+ <div className="math-editor" ref={this.setMathRef}></div>
+ <div className="text-editor" ref={this.setTextRef}></div>
+ </div>
+ );
+ }
+}
diff --git a/src/client/views/InkingStroke.tsx b/src/client/views/InkingStroke.tsx
index 2e82371cb..887e10027 100644
--- a/src/client/views/InkingStroke.tsx
+++ b/src/client/views/InkingStroke.tsx
@@ -107,8 +107,8 @@ export class InkingStroke extends ViewBoxAnnotatableComponent<FieldViewProps>()
* analyzes the ink stroke and saves the analysis of the stroke to the 'inkAnalysis' field,
* and the recognized words to the 'handwriting'
*/
- analyzeStrokes() {
- const data: InkData = Cast(this.dataDoc[this.fieldKey], InkField)?.inkData ?? [];
+ analyzeStrokes=()=> {
+ const data: InkData = this.inkScaledData().inkData ?? [];
CognitiveServices.Inking.Appliers.ConcatenateHandwriting(this.dataDoc, ['inkAnalysis', 'handwriting'], [data]);
}
diff --git a/src/client/views/MainView.tsx b/src/client/views/MainView.tsx
index 8b8f85dfb..5285e67f6 100644
--- a/src/client/views/MainView.tsx
+++ b/src/client/views/MainView.tsx
@@ -74,6 +74,7 @@ import { PresBox } from './nodes/trails';
import { AnchorMenu } from './pdf/AnchorMenu';
import { GPTPopup } from './pdf/GPTPopup/GPTPopup';
import { TopBar } from './topbar/TopBar';
+import { InkTranscription } from './InkTranscription';
// eslint-disable-next-line @typescript-eslint/no-var-requires, @typescript-eslint/no-require-imports
const { LEFT_MENU_WIDTH, TOPBAR_HEIGHT } = require('./global/globalCssVariables.module.scss'); // prettier-ignore
@@ -1087,6 +1088,7 @@ export class MainView extends ObservableReactComponent<object> {
<MarqueeOptionsMenu />
<TimelineMenu />
<RichTextMenu />
+ <InkTranscription />
{this.snapLines}
<LightboxView key="lightbox" PanelWidth={this._windowWidth} addSplit={CollectionDockingView.AddSplit} PanelHeight={this._windowHeight} maxBorder={this.lightboxMaxBorder} />
<GPTPopup key="gptpopup" />
diff --git a/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx b/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx
index f106eba26..e63c00f17 100644
--- a/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx
+++ b/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx
@@ -1945,6 +1945,14 @@ export class CollectionFreeFormView extends CollectionSubView<Partial<collection
!options && ContextMenu.Instance.addItem({ description: 'Options...', subitems: optionItems, icon: 'eye' });
const mores = ContextMenu.Instance.findByDescription('More...');
const moreItems = mores?.subitems ?? [];
+ moreItems.push({
+ description: 'recognize all ink',
+ event: () => {
+ this.unprocessedDocs.push(...this.childDocs.filter(doc => doc.type === DocumentType.INK));
+ CollectionFreeFormView.collectionsWithUnprocessedInk.add(this);
+ },
+ icon: 'pen',
+ });
!mores && ContextMenu.Instance.addItem({ description: 'More...', subitems: moreItems, icon: 'eye' });
};
diff --git a/src/client/views/global/globalScripts.ts b/src/client/views/global/globalScripts.ts
index f7a9689c9..0afb3635f 100644
--- a/src/client/views/global/globalScripts.ts
+++ b/src/client/views/global/globalScripts.ts
@@ -15,6 +15,7 @@ import { ScriptingGlobals } from '../../util/ScriptingGlobals';
import { SnappingManager } from '../../util/SnappingManager';
import { UndoManager, undoable } from '../../util/UndoManager';
import { GestureOverlay } from '../GestureOverlay';
+import { InkTranscription } from '../InkTranscription';
import { InkingStroke } from '../InkingStroke';
import { MainView } from '../MainView';
import { PropertiesView } from '../PropertiesView';
@@ -386,6 +387,7 @@ ScriptingGlobals.add(function toggleCharStyle(charStyle: attrname, checkResult?:
export function createInkGroup(/* inksToGroup?: Doc[], isSubGroup?: boolean */) {
// TODO nda - if document being added to is a inkGrouping then we can just add to that group
if (Doc.ActiveTool === InkTool.Write) {
+ console.log('create inking group ');
CollectionFreeFormView.collectionsWithUnprocessedInk.forEach(ffView => {
// TODO: nda - will probably want to go through ffView unprocessed docs and then see if any of the inksToGroup docs are in it and only use those
const selected = ffView.unprocessedDocs;
@@ -443,14 +445,14 @@ export function createInkGroup(/* inksToGroup?: Doc[], isSubGroup?: boolean */)
// TODO: nda - will probably need to go through and only remove the unprocessed selected docs
ffView.unprocessedDocs = [];
- // InkTranscription.Instance.transcribeInk(newCollection, selected, false);
+ InkTranscription.Instance.transcribeInk(newCollection, selected, false);
});
}
CollectionFreeFormView.collectionsWithUnprocessedInk.clear();
}
function setActiveTool(tool: InkTool | Gestures, keepPrim: boolean, checkResult?: boolean) {
- // InkTranscription.Instance?.createInkGroup();
+ InkTranscription.Instance?.createInkGroup();
if (checkResult) {
return (Doc.ActiveTool === tool && !GestureOverlay.Instance?.InkShape) || GestureOverlay.Instance?.InkShape === tool
? GestureOverlay.Instance?.KeepPrimitiveMode || ![Gestures.Circle, Gestures.Line, Gestures.Rectangle].includes(tool as Gestures)
diff --git a/src/client/views/nodes/DiagramBox.scss b/src/client/views/nodes/DiagramBox.scss
index 323638bff..8a7863c14 100644
--- a/src/client/views/nodes/DiagramBox.scss
+++ b/src/client/views/nodes/DiagramBox.scss
@@ -1,5 +1,3 @@
-$searchbarHeight: 50px;
-
.DIYNodeBox {
width: 100%;
height: 100%;
@@ -23,7 +21,6 @@ $searchbarHeight: 50px;
justify-content: center;
align-items: center;
width: 100%;
- height: $searchbarHeight;
padding: 10px;
input[type='text'] {
@@ -42,7 +39,6 @@ $searchbarHeight: 50px;
justify-content: center;
align-items: center;
width: 100%;
- height: calc(100% - $searchbarHeight);
.diagramBox {
flex: 1;
display: flex;
diff --git a/src/client/views/nodes/DiagramBox.tsx b/src/client/views/nodes/DiagramBox.tsx
index 36deb2d8d..0d755fdbe 100644
--- a/src/client/views/nodes/DiagramBox.tsx
+++ b/src/client/views/nodes/DiagramBox.tsx
@@ -18,7 +18,9 @@ import { InkingStroke } from '../InkingStroke';
import './DiagramBox.scss';
import { FieldView, FieldViewProps } from './FieldView';
import { FormattedTextBox } from './formattedText/FormattedTextBox';
-
+/**
+ * this is a class for the diagram box doc type that can be found in the tools section of the side bar
+ */
@observer
export class DiagramBox extends ViewBoxAnnotatableComponent<FieldViewProps>() {
public static LayoutString(fieldKey: string) {
@@ -50,7 +52,9 @@ export class DiagramBox extends ViewBoxAnnotatableComponent<FieldViewProps>() {
mermaid.initialize({
securityLevel: 'loose',
startOnLoad: true,
- flowchart: { useMaxWidth: true, htmlLabels: true, curve: 'cardinal' },
+ darkMode: true,
+ flowchart: { useMaxWidth: false, htmlLabels: true, curve: 'cardinal' },
+ gantt: { useMaxWidth: true, useWidth: 2000 },
});
// when a new doc/text/ink/shape is created in the freeform view, this generates the corresponding mermaid diagram code
reaction(
@@ -59,14 +63,21 @@ export class DiagramBox extends ViewBoxAnnotatableComponent<FieldViewProps>() {
{ fireImmediately: true }
);
}
+ /**
+ * helper method for renderMermaidAsync
+ * @param str string containing the mermaid code
+ * @returns
+ */
renderMermaid = (str: string) => {
try {
return mermaid.render('graph' + Date.now(), str);
- } catch (error) {
+ } catch {
return { svg: '', bindFunctions: undefined };
}
};
-
+ /**
+ * will update the div containing the mermaid diagram to render the new mermaidCode
+ */
renderMermaidAsync = async (mermaidCode: string, dashDiv: HTMLDivElement) => {
try {
const { svg, bindFunctions } = await this.renderMermaid(mermaidCode);
@@ -97,7 +108,9 @@ export class DiagramBox extends ViewBoxAnnotatableComponent<FieldViewProps>() {
res
);
}, 'set mermaid code');
-
+ /**
+ * will generate mermaid code with GPT based on what the user requested
+ */
generateMermaidCode = action(() => {
this._generating = true;
const prompt = 'Write this in mermaid code and only give me the mermaid code: ' + this._inputValue;
diff --git a/src/pen-gestures/GestureTypes.ts b/src/pen-gestures/GestureTypes.ts
index d86562580..5a8e9bd97 100644
--- a/src/pen-gestures/GestureTypes.ts
+++ b/src/pen-gestures/GestureTypes.ts
@@ -1,12 +1,12 @@
export enum Gestures {
Line = 'line',
Stroke = 'stroke',
- Scribble = 'scribble',
Text = 'text',
Triangle = 'triangle',
Circle = 'circle',
Rectangle = 'rectangle',
Arrow = 'arrow',
+ RightAngle = 'rightangle',
}
// Defines a point in an ink as a pair of x- and y-coordinates.
diff --git a/src/pen-gestures/ndollar.ts b/src/pen-gestures/ndollar.ts
index ff7f7310b..04262b61f 100644
--- a/src/pen-gestures/ndollar.ts
+++ b/src/pen-gestures/ndollar.ts
@@ -209,6 +209,7 @@ export class NDollarRecognizer {
])
)
);
+ this.Multistrokes.push(new Multistroke(Gestures.Rectangle, useBoundedRotationInvariance, new Array([new Point(30, 143), new Point(106, 146), new Point(106, 225), new Point(30, 222), new Point(30, 146)])));
this.Multistrokes.push(new Multistroke(Gestures.Line, useBoundedRotationInvariance, [[new Point(12, 347), new Point(119, 347)]]));
this.Multistrokes.push(
new Multistroke(
@@ -219,6 +220,13 @@ export class NDollarRecognizer {
);
this.Multistrokes.push(
new Multistroke(
+ Gestures.Triangle, // equilateral
+ useBoundedRotationInvariance,
+ new Array([new Point(42, 100), new Point(140, 102), new Point(100, 200), new Point(40, 100)])
+ )
+ );
+ this.Multistrokes.push(
+ new Multistroke(
Gestures.Circle,
useBoundedRotationInvariance,
new Array([
@@ -236,6 +244,26 @@ export class NDollarRecognizer {
])
)
);
+ this.Multistrokes.push(
+ new Multistroke(
+ Gestures.Circle,
+ useBoundedRotationInvariance,
+ new Array([
+ new Point(201, 250),
+ new Point(160, 230),
+ new Point(151, 210),
+ new Point(151, 190),
+ new Point(160, 170),
+ new Point(200, 150),
+ new Point(240, 170),
+ new Point(248, 190),
+ new Point(248, 210),
+ new Point(240, 230),
+ new Point(200, 250),
+ ])
+ )
+ );
+ this.Multistrokes.push(new Multistroke(Gestures.RightAngle, useBoundedRotationInvariance, new Array([new Point(0, 0), new Point(0, 100), new Point(200, 100)])));
NumMultistrokes = this.Multistrokes.length; // NumMultistrokes flags the end of the non user-defined gstures strokes
//
// PREDEFINED STROKES
generated by cgit v1.2.3-70-g09d2 (git 2.46.0) at 2025-10-06 11:21:00 +0000