fixed invalidations to not trigger creating new refs when ref= was assigned to an anonymous function. fixed scribble erase to not delete everything it overlaps, just things it intersects with or contains. fixed ink to have a Math mode and fixed math recognition myscript calls.

1 files changed, 94 insertions, 88 deletions

diff --git a/src/client/views/InkTranscription.tsx b/src/client/views/InkTranscription.tsx
index 6220033d4..32ca5f56b 100644
--- a/src/client/views/InkTranscription.tsx
+++ b/src/client/views/InkTranscription.tsx
@@ -32,7 +32,7 @@ export class InkTranscription extends React.Component {
     // eslint-disable-next-line @typescript-eslint/no-explicit-any
     @observable _iinkEditor: any = undefined;
     // eslint-disable-next-line @typescript-eslint/no-explicit-any
-    private lastJiix: any;
+    @observable _iinkMathEditor: any = undefined;
     private currGroup?: Doc;
     private collectionFreeForm?: CollectionFreeFormView;
 
@@ -44,7 +44,7 @@ export class InkTranscription extends React.Component {
     @action
     // eslint-disable-next-line @typescript-eslint/no-explicit-any
     setMathRef = async (r: any) => {
-        if (!this._textRegister && r) {
+        if (!this._mathRegister && r) {
             const options = {
                 configuration: {
                     server: {
@@ -55,19 +55,17 @@ export class InkTranscription extends React.Component {
                         protocol: 'WEBSOCKET',
                     },
                     recognition: {
-                        type: 'TEXT',
-                        lang: 'en_US',
-                        text: {
-                            mimeTypes: ['application/vnd.myscript.jiix'] as 'application/vnd.myscript.jiix'[],
+                        type: 'MATH',
+                        math: {
+                            mimeTypes: ['application/x-latex'] as unknown as 'application/vnd.myscript.jiix'[],
                         },
                     },
                 },
             };
-            await iink.Editor.load(r, 'INKV2', options);
-
-            this._textRegister = r;
+            this._iinkMathEditor = await iink.Editor.load(r, 'INKV2', options);
+            this._mathRegister = r;
             // eslint-disable-next-line @typescript-eslint/no-explicit-any
-            r?.addEventListener('exported', (e: any) => this.exportInk(e, this._textRef));
+            r?.addEventListener('exported', (e: any) => this.exportInk(e));
 
             return (this._textRef = r);
         }
@@ -97,12 +95,14 @@ export class InkTranscription extends React.Component {
             this._iinkEditor = await iink.Editor.load(r, 'INKV2', options);
             this._textRegister = r;
             // eslint-disable-next-line @typescript-eslint/no-explicit-any
-            r?.addEventListener('exported', (e: any) => this.exportInk(e, this._textRef));
+            r?.addEventListener('exported', (e: any) => this.exportInk(e));
 
             return (this._textRef = r);
         }
     };
 
+    _ffview: CollectionFreeFormView | undefined;
+
     /**
      * Handles processing Dash Doc data for ink transcription.
      *
@@ -110,7 +110,7 @@ export class InkTranscription extends React.Component {
      * @param inkDocs the ink docs contained within the selected group
      * @param math boolean whether to do math transcription or not
      */
-    transcribeInk = (groupDoc: Doc | undefined, inkDocs: Doc[], math: boolean) => {
+    transcribeInk = (ffview: CollectionFreeFormView, groupDoc: Doc | undefined, inkDocs: Doc[], math: boolean) => {
         if (!groupDoc) return;
         const validInks = inkDocs.filter(s => s.type === DocumentType.INK);
 
@@ -126,13 +126,10 @@ export class InkTranscription extends React.Component {
                 strokes.push(d.inkData.map(pd => inkStroke.ptToScreen({ X: pd.X, Y: pd.Y })));
                 times.push(authorTime);
             });
+        this._ffview = ffview;
         this.currGroup = groupDoc;
         const pointerData = strokes.map((stroke, i) => this.inkJSON(stroke, times[i]));
-        if (math) {
-            this._iinkEditor.importPointEvents(pointerData);
-        } else {
-            this._iinkEditor.importPointEvents(pointerData);
-        }
+        (math ? this._iinkMathEditor : this._iinkEditor).importPointEvents(pointerData);
     };
     convertPointsToString(points: InkData[]): string {
         return points[0].map(point => `new Point(${point.X}, ${point.Y})`).join(',');
@@ -204,83 +201,92 @@ export class InkTranscription extends React.Component {
      * @param ref the ref to the editor
      */
     // eslint-disable-next-line @typescript-eslint/no-explicit-any
-    exportInk = async (e: any, ref: any) => {
-        const exports = e.detail['application/vnd.myscript.jiix'];
-        if (exports) {
-            if (exports['type'] == 'Math') {
-                const latex = exports['application/x-latex'];
-                if (this.currGroup) {
-                    this.currGroup.text = latex;
-                    this.currGroup.title = latex;
-                }
+    exportInk = (e: { detail: { [key: string]: string } }) => {
+        const exports = e.detail;
+        if (exports['application/x-latex']) {
+            const latex = exports['application/x-latex'];
+            if (this.currGroup) {
+                this.currGroup.text = latex;
+                this.currGroup.title = latex;
+            }
 
-                ref.editor.clear();
-            } else if (exports['type'] == 'Text') {
-                if (exports['application/vnd.myscript.jiix']) {
-                    this.lastJiix = JSON.parse(exports['application/vnd.myscript.jiix']);
-                    // map timestamp to strokes
-                    const timestampWord = new Map<number, string>();
-                    // eslint-disable-next-line @typescript-eslint/no-explicit-any
-                    this.lastJiix.words.map((word: any) => {
-                        if (word.items) {
-                            word.items.forEach((i: { id: string; timestamp: string; X: Array<number>; Y: Array<number>; F: Array<number> }) => {
-                                const ms = Date.parse(i.timestamp);
-                                timestampWord.set(ms, word.label);
-                            });
-                        }
+            this._ffview?.addDocument(
+                Docs.Create.EquationDocument(latex, {
+                    title: '',
+                    x: this.currGroup?.x as number,
+                    y: (this.currGroup?.y as number) + (this.currGroup?.height as number),
+                    nativeHeight: 40,
+                    _height: 50,
+                    nativeWidth: 40,
+                    _width: 50,
+                })
+            );
+            // this.showRecogBox(latex as string);
+            this._iinkMathEditor.clear();
+        } else if (exports['application/vnd.myscript.jiix']) {
+            const lastJiix = exports['application/vnd.myscript.jiix'] as unknown as { words: string[]; label: string };
+            // map timestamp to strokes
+            const timestampWord = new Map<number, string>();
+            // eslint-disable-next-line @typescript-eslint/no-explicit-any
+            lastJiix.words.map((word: any) => {
+                if (word.items) {
+                    word.items.forEach((i: { id: string; timestamp: string; X: Array<number>; Y: Array<number>; F: Array<number> }) => {
+                        const ms = Date.parse(i.timestamp);
+                        timestampWord.set(ms, word.label);
                     });
-
-                    const wordInkDocMap = new Map<string, Doc[]>();
-                    if (this.currGroup) {
-                        const docList = DocListCast(this.currGroup.data);
-                        docList.forEach((inkDoc: Doc) => {
-                            // just having the times match up and be a unique value (actual timestamp doesn't matter)
-                            const ms = (DateCast(inkDoc.author_date)?.getDate().getTime() ?? 0) + 14400000;
-                            const word = timestampWord.get(ms);
-                            if (!word) {
-                                return;
-                            }
-                            const entry = wordInkDocMap.get(word);
-                            if (entry) {
-                                entry.push(inkDoc);
-                                wordInkDocMap.set(word, entry);
-                            } else {
-                                const newEntry = [inkDoc];
-                                wordInkDocMap.set(word, newEntry);
-                            }
-                        });
-                        if (this.lastJiix.words.length > 1) this.subgroupsTranscriptions(wordInkDocMap);
-                    }
                 }
-                const text = exports['label'];
+            });
 
-                if (this.currGroup && text) {
-                    DocumentView.getDocumentView(this.currGroup)?.ComponentView?.updateIcon?.();
-                    const image = await this.getIcon();
-                    const { href } = (image as URLField).url;
-                    const hrefParts = href.split('.');
-                    const hrefComplete = `${hrefParts[0]}_o.${hrefParts[1]}`;
-                    let response;
-                    try {
-                        const hrefBase64 = await imageUrlToBase64(hrefComplete);
-                        response = await gptHandwriting(hrefBase64);
-                    } catch {
-                        console.error('Error getting image');
-                    }
-                    const textBoxText = 'iink: ' + text + '\n' + '\n' + 'ChatGPT: ' + response;
-                    this.currGroup.transcription = response;
-                    this.currGroup.title = response;
-                    if (!this.currGroup.hasTextBox) {
-                        const newDoc = Docs.Create.TextDocument(textBoxText, { title: '', x: this.currGroup.x as number, y: (this.currGroup.y as number) + (this.currGroup.height as number) });
-                        newDoc.height = 200;
-                        this.collectionFreeForm?.addDocument(newDoc);
-                        this.currGroup.hasTextBox = true;
+            const wordInkDocMap = new Map<string, Doc[]>();
+            if (this.currGroup) {
+                DocListCast(this.currGroup.data).forEach((inkDoc: Doc) => {
+                    // just having the times match up and be a unique value (actual timestamp doesn't matter)
+                    const ms = (DateCast(inkDoc.author_date)?.getDate().getTime() ?? 0) + 14400000;
+                    const word = timestampWord.get(ms);
+                    if (word) {
+                        const entry = wordInkDocMap.get(word);
+                        if (entry) {
+                            entry.push(inkDoc);
+                            wordInkDocMap.set(word, entry);
+                        } else {
+                            const newEntry = [inkDoc];
+                            wordInkDocMap.set(word, newEntry);
+                        }
                     }
-                    ref.editor.clear();
-                }
+                });
+                if (lastJiix.words.length > 1) this.subgroupsTranscriptions(wordInkDocMap);
             }
+            this.showRecogBox(lastJiix.label);
+            this._iinkEditor.clear();
         }
     };
+    private showRecogBox(text: string) {
+        if (this.currGroup) {
+            let response;
+            // DocumentView.getDocumentView(this.currGroup)?.ComponentView?.updateIcon?.();
+            // const image = await this.getIcon();
+            // const { href } = (image as URLField).url;
+            // const hrefParts = href.split('.');
+            // const hrefComplete = `${hrefParts[0]}_o.${hrefParts[1]}`;
+            // try {
+            //     const hrefBase64 = await imageUrlToBase64(hrefComplete);
+            //     response = await gptHandwriting(hrefBase64);
+            // } catch {
+            //     console.error('Error getting image');
+            // }
+            const textBoxText = 'iink: ' + text + '\n' + '\n' + 'ChatGPT: ' + response;
+
+            this.currGroup.transcription = response;
+            this.currGroup.title = response;
+            if (!this.currGroup.hasTextBox) {
+                const newDoc = Docs.Create.TextDocument(textBoxText, { title: '', x: this.currGroup.x as number, y: (this.currGroup.y as number) + (this.currGroup.height as number) });
+                newDoc.height = 200;
+                this.collectionFreeForm?.addDocument(newDoc);
+                this.currGroup.hasTextBox = true;
+            }
+        }
+    }
+
     /**
      * gets the icon of the collection that was just made
      * @returns the image of the collection
@@ -299,7 +305,7 @@ export class InkTranscription extends React.Component {
      */
     createInkGroup() {
         // TODO nda - if document being added to is a inkGrouping then we can just add to that group
-        if (Doc.ActiveTool === InkTool.Ink && Doc.ActiveInk === InkInkTool.Write) {
+        if (Doc.ActiveTool === InkTool.Ink && [InkInkTool.Write, InkInkTool.Math].includes(Doc.ActiveInk)) {
             CollectionFreeFormView.collectionsWithUnprocessedInk.forEach(ffView => {
                 // TODO: nda - will probably want to go through ffView unprocessed docs and then see if any of the inksToGroup docs are in it and only use those
                 const selected = ffView.unprocessedDocs;
@@ -309,7 +315,7 @@ export class InkTranscription extends React.Component {
                 );
                 ffView.unprocessedDocs = [];
 
-                InkTranscription.Instance.transcribeInk(newCollection, selected, false);
+                InkTranscription.Instance.transcribeInk(ffView, newCollection, selected, Doc.ActiveInk === InkInkTool.Math);
             });
         }
         CollectionFreeFormView.collectionsWithUnprocessedInk.clear();