auto transcribes on grouping

3 files changed, 57 insertions, 55 deletions

diff --git a/src/client/views/InkTranscription.tsx b/src/client/views/InkTranscription.tsx
index 4957446dc..ea5d5a1ec 100644
--- a/src/client/views/InkTranscription.tsx
+++ b/src/client/views/InkTranscription.tsx
@@ -5,7 +5,7 @@ import * as React from 'react';
 import { Doc } from '../../fields/Doc';
 import { InkData, InkField } from "../../fields/InkField";
 import { Cast, NumCast, StrCast } from '../../fields/Types';
-import { Docs } from "../documents/Documents";
+import { DocumentType } from "../documents/DocumentTypes";
 import './InkTranscription.scss';
 
 export class InkTranscription extends React.Component {
@@ -15,8 +15,7 @@ export class InkTranscription extends React.Component {
     @observable _mathRef: any;
     @observable _textRegister: any;
     @observable _textRef: any;
-    private addDocument?: (doc: Doc | Doc[]) => boolean;
-    private bounds: { x: number, y: number, width: number, height: number } = { x: 0, y: 0, width: 0, height: 0 };
+    private currGroup?: Doc;
 
     constructor(props: Readonly<{}>) {
         super(props);
@@ -27,9 +26,6 @@ export class InkTranscription extends React.Component {
     componentWillUnmount() {
         this._mathRef.removeEventListener('exported', (e: any) => this.exportInk(e, this._mathRef));
         this._textRef.removeEventListener('exported', (e: any) => this.exportInk(e, this._textRef));
-
-        this._mathRef.removeEventListener('changed', (e: any) => this.callExport(this._mathRef));
-        this._textRef.removeEventListener('changed', (e: any) => this.callExport(this._textRef));
     }
 
     @action
@@ -58,15 +54,11 @@ export class InkTranscription extends React.Component {
                             }
                         }
                     }
-                },
-                triggers: {
-                    exportContent: 'DEMAND'
                 }
             }) : null;
         }
 
         r.addEventListener('exported', (e: any) => this.exportInk(e, this._mathRef));
-        r.addEventListener('changed', (e: any) => this.callExport(this._mathRef));
 
         return this._mathRef = r;
     }
@@ -97,22 +89,22 @@ export class InkTranscription extends React.Component {
                             }
                         }
                     }
-                },
-                triggers: {
-                    exportContent: 'DEMAND'
                 }
             }) : null;
         }
 
         r.addEventListener('exported', (e: any) => this.exportInk(e, this._textRef));
-        r.addEventListener('changed', (e: any) => this.callExport(this._textRef));
 
         return this._textRef = r;
     }
 
-    transcribeInk = (inkdocs: Doc[], math: boolean, bounds: { x: number, y: number, width: number, height: number }, addDocument?: (doc: Doc | Doc[]) => boolean) => {
+    transcribeInk = (groupDoc: Doc | undefined, inkDocs: Doc[], math: boolean) => {
+        if (!groupDoc) return;
+
+        const validInks = inkDocs.filter(s => s.type === DocumentType.INK);
+
         const strokes: InkData[] = [];
-        inkdocs.filter(i => Cast(i.data, InkField)).forEach(i => {
+        validInks.filter(i => Cast(i.data, InkField)).forEach(i => {
             // TODO: interpolate missing times stamps
             const d = Cast(i.data, InkField, null);
             const left = Math.min(...d?.inkData.map(pd => pd.X) ?? [0]);
@@ -120,8 +112,7 @@ export class InkTranscription extends React.Component {
             strokes.push(d.inkData.map(pd => ({ X: pd.X + NumCast(i.x) - left, Y: pd.Y + NumCast(i.y) - top, time: pd.time })));
         });
 
-        this.addDocument = addDocument;
-        this.bounds = bounds;
+        this.currGroup = groupDoc;
 
         const pointerData = { "events": strokes.map(stroke => this.inkJSON(stroke)) };
         // console.log(JSON.stringify(pointerData));
@@ -135,12 +126,6 @@ export class InkTranscription extends React.Component {
         }
     }
 
-    callExport(ref: any) {
-        if (ref.editor.canExport) {
-            ref.editor.export_();
-        }
-    }
-
     inkJSON = (stroke: InkData) => {
         return {
             "pointerType": "PEN",
@@ -159,15 +144,24 @@ export class InkTranscription extends React.Component {
                 const latex = exports['application/x-latex'];
                 console.log(latex);
 
-                this.addDocument?.(Docs.Create.EquationDocument({ title: latex, x: this.bounds.x + this.bounds.width, y: this.bounds.y, _width: this.bounds.width, _height: this.bounds.height }));
+                if (this.currGroup) {
+                    this.currGroup.text = latex;
+                    this.currGroup.title = latex;
+                }
+
+                ref.editor.clear();
             }
             else if (exports['text/plain']) {
                 const text = exports['text/plain'];
                 console.log(text);
-                this.addDocument?.(Docs.Create.TextDocument(text, { title: text, x: this.bounds.x + this.bounds.width, y: this.bounds.y, _width: this.bounds.width, _height: this.bounds.height }));
-            }
 
-            ref.editor.clear();
+                if (this.currGroup) {
+                    this.currGroup.text = text;
+                    this.currGroup.title = text;
+                }
+
+                ref.editor.clear();
+            }
         }
     }
 
diff --git a/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx b/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx
index c9465e7a4..023953658 100644
--- a/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx
+++ b/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx
@@ -98,7 +98,7 @@ export class CollectionFreeFormView extends CollectionSubView<Partial<collection
     private _cachedPool: Map<string, PoolData> = new Map();
     private _lastTap = 0;
     private _batch: UndoManager.Batch | undefined = undefined;
-    
+
     // private isWritingMode: boolean = true;
     // private writingModeDocs: Doc[] = []; 
 
@@ -461,10 +461,10 @@ export class CollectionFreeFormView extends CollectionSubView<Partial<collection
                 switch (CurrentUserUtils.SelectedTool) {
                     case InkTool.Highlighter:
                         break;
-                        // TODO: nda - this where we want to create the new "writingDoc" collection that we add strokes to
+                    // TODO: nda - this where we want to create the new "writingDoc" collection that we add strokes to
                     case InkTool.Write:
                         break;
-                    case InkTool.Pen: 
+                    case InkTool.Pen:
                         break; // the GestureOverlay handles ink stroke input -- either as gestures, or drying as ink strokes that are added to document views
                     case InkTool.Eraser:
                         document.addEventListener("pointermove", this.onEraserMove);
@@ -509,7 +509,7 @@ export class CollectionFreeFormView extends CollectionSubView<Partial<collection
         }
     }
 
-    public unprocessedDocs: Doc[] =[];
+    public unprocessedDocs: Doc[] = [];
     public static collectionsWithUnprocessedInk = new Set<CollectionFreeFormView>();
     @undoBatch
     onGesture = (e: Event, ge: GestureUtils.GestureEvent) => {
@@ -521,11 +521,11 @@ export class CollectionFreeFormView extends CollectionSubView<Partial<collection
                 const strokes: PointData[] = !times || times.length == 0 ? points : points.map((pt, i) => { return { X: pt.X, Y: pt.Y, time: times[i] } });
                 const inkDoc = Docs.Create.InkDocument(ActiveInkColor(), CurrentUserUtils.SelectedTool, ActiveInkWidth(), ActiveInkBezierApprox(), ActiveFillColor(), ActiveArrowStart(), ActiveArrowEnd(), ActiveDash(), strokes,
                     { title: "ink stroke", x: B.x - ActiveInkWidth() / 2, y: B.y - ActiveInkWidth() / 2, _width: B.width + ActiveInkWidth(), _height: B.height + ActiveInkWidth() });
-                    if (CurrentUserUtils.SelectedTool === InkTool.Write) {
-                        this.unprocessedDocs.push(inkDoc);
-                        CollectionFreeFormView.collectionsWithUnprocessedInk.add(this);
-                    }
-                    this.addDocument(inkDoc);
+                if (CurrentUserUtils.SelectedTool === InkTool.Write) {
+                    this.unprocessedDocs.push(inkDoc);
+                    CollectionFreeFormView.collectionsWithUnprocessedInk.add(this);
+                }
+                this.addDocument(inkDoc);
                 e.stopPropagation();
                 break;
             case GestureUtils.Gestures.Box:
@@ -1577,9 +1577,14 @@ export class CollectionFreeFormView extends CollectionSubView<Partial<collection
     @undoBatch
     @action
     transcribeStrokes = (math: boolean) => {
-        if (this.props.Document._isGroup) {
-            const inkdocs = this.childDocs.filter(s => s.type === DocumentType.INK);
-            InkTranscription.Instance.transcribeInk(inkdocs, math, { x: NumCast(this.layoutDoc.x) ?? 0, y: NumCast(this.layoutDoc.y) ?? 0, width: NumCast(this.layoutDoc._width) ?? 0, height: NumCast(this.layoutDoc._height) ?? 0 }, this.props.ContainingCollectionView?.addDocument);
+        if (this.props.Document._isGroup && this.props.Document.text) {
+            const text = StrCast(this.props.Document.text);
+
+            const lines = text.split("\n");
+            const width = 30 + 7 * Math.max(...lines.map(line => line.length));
+            const height = 30 + 15 * lines.length;
+
+            this.props.ContainingCollectionView?.addDocument(Docs.Create.TextDocument(text, { title: text, x: NumCast(this.layoutDoc.x) + NumCast(this.layoutDoc._width) + 20, y: NumCast(this.layoutDoc.y), _width: width, _height: height }));
         }
     }
 
diff --git a/src/client/views/nodes/button/FontIconBox.tsx b/src/client/views/nodes/button/FontIconBox.tsx
index 862c16050..fc358f106 100644
--- a/src/client/views/nodes/button/FontIconBox.tsx
+++ b/src/client/views/nodes/button/FontIconBox.tsx
@@ -26,6 +26,7 @@ import { EditableView } from '../../EditableView';
 import { GestureOverlay } from '../../GestureOverlay';
 import { Colors } from '../../global/globalEnums';
 import { ActiveFillColor, ActiveInkColor, ActiveInkWidth, SetActiveFillColor, SetActiveInkColor, SetActiveInkWidth } from '../../InkingStroke';
+import { InkTranscription } from '../../InkTranscription';
 import { StyleProp } from '../../StyleProvider';
 import { FieldView, FieldViewProps } from '.././FieldView';
 import { RichTextMenu } from '../formattedText/RichTextMenu';
@@ -702,9 +703,9 @@ export function checkInksToGroup() {
     if (CurrentUserUtils.SelectedTool === InkTool.Write) {
         CollectionFreeFormView.collectionsWithUnprocessedInk.forEach(ffView => {
             // TODO: nda - will probably want to go through ffView unprocessed docs and then see if any of the inksToGroup docs are in it and only use those
-           // find all inkDocs in ffView.unprocessedDocs that are within 200 pixels of each other
-              const inksToGroup = ffView.unprocessedDocs.filter(inkDoc => {
-                    console.log(inkDoc.x, inkDoc.y);
+            // find all inkDocs in ffView.unprocessedDocs that are within 200 pixels of each other
+            const inksToGroup = ffView.unprocessedDocs.filter(inkDoc => {
+                console.log(inkDoc.x, inkDoc.y);
             });
         });
     }
@@ -718,26 +719,26 @@ export function createInkGroup(inksToGroup?: Doc[]) {
             const selected = ffView.unprocessedDocs;
             // loop through selected an get the bound
             const bounds: { x: number, y: number, width?: number, height?: number }[] = []
-            
+
             selected.map(action(d => {
                 const x = NumCast(d.x);
                 const y = NumCast(d.y);
                 const width = d[WidthSym]();
                 const height = d[HeightSym]();
-                bounds.push({x, y, width, height});
+                bounds.push({ x, y, width, height });
             }))
-    
+
             const aggregBounds = aggregateBounds(bounds, 0, 0);
             const marqViewRef = ffView._marqueeViewRef.current;
-    
+
             // set the vals for bounds in marqueeView
             if (marqViewRef) {
                 marqViewRef._downX = aggregBounds.x;
                 marqViewRef._downY = aggregBounds.y;
                 marqViewRef._lastX = aggregBounds.r;
                 marqViewRef._lastY = aggregBounds.b;
-            }        
-    
+            }
+
             selected.map(action(d => {
                 const dx = NumCast(d.x);
                 const dy = NumCast(d.y);
@@ -756,11 +757,13 @@ export function createInkGroup(inksToGroup?: Doc[]) {
             ffView.props.removeDocument?.(selected);
             // TODO: nda - this is the code to actually get a new grouped collection
             const newCollection = marqViewRef?.getCollection(selected, undefined, [], true);
-    
+
             // nda - bug: when deleting a stroke before leaving writing mode, delete the stroke from unprocessed ink docs
             newCollection && ffView.props.addDocument?.(newCollection);
             // TODO: nda - will probably need to go through and only remove the unprocessed selected docs
             ffView.unprocessedDocs = [];
+
+            InkTranscription.Instance.transcribeInk(newCollection, selected, false);
         });
     }
     CollectionFreeFormView.collectionsWithUnprocessedInk.clear();
@@ -780,7 +783,7 @@ ScriptingGlobals.add(function setActiveInkTool(tool: string, checkResult?: boole
     //         const selected = ffView.unprocessedDocs;
     //         // loop through selected an get the bound
     //         const bounds: { x: number, y: number, width?: number, height?: number }[] = []
-            
+
     //         selected.map(action(d => {
     //             const x = NumCast(d.x);
     //             const y = NumCast(d.y);
@@ -788,10 +791,10 @@ ScriptingGlobals.add(function setActiveInkTool(tool: string, checkResult?: boole
     //             const height = d[HeightSym]();
     //             bounds.push({x, y, width, height});
     //         }))
-    
+
     //         const aggregBounds = aggregateBounds(bounds, 0, 0);
     //         const marqViewRef = ffView._marqueeViewRef.current;
-    
+
     //         // set the vals for bounds in marqueeView
     //         if (marqViewRef) {
     //             marqViewRef._downX = aggregBounds.x;
@@ -799,7 +802,7 @@ ScriptingGlobals.add(function setActiveInkTool(tool: string, checkResult?: boole
     //             marqViewRef._lastX = aggregBounds.r;
     //             marqViewRef._lastY = aggregBounds.b;
     //         }        
-    
+
     //         selected.map(action(d => {
     //             const dx = NumCast(d.x);
     //             const dy = NumCast(d.y);
@@ -818,7 +821,7 @@ ScriptingGlobals.add(function setActiveInkTool(tool: string, checkResult?: boole
     //         ffView.props.removeDocument?.(selected);
     //         // TODO: nda - this is the code to actually get a new grouped collection
     //         const newCollection = marqViewRef?.getCollection(selected, undefined, [], true);
-    
+
     //         // nda - bug: when deleting a stroke before leaving writing mode, delete the stroke from unprocessed ink docs
     //         newCollection && ffView.props.addDocument?.(newCollection);
     //         ffView.unprocessedDocs = [];