Merge branch 'inking-naafi-mehek' into master-cleanup

1 files changed, 312 insertions, 0 deletions

diff --git a/src/client/views/InkTranscription.tsx b/src/client/views/InkTranscription.tsx
new file mode 100644
index 000000000..5e0667bed
--- /dev/null
+++ b/src/client/views/InkTranscription.tsx
@@ -0,0 +1,312 @@
+import * as iink from 'iink-js';
+import { action, observable } from 'mobx';
+import * as React from 'react';
+import { Doc, DocListCast, HeightSym, WidthSym } from '../../fields/Doc';
+import { InkData, InkField } from "../../fields/InkField";
+import { Cast, DateCast, NumCast } from '../../fields/Types';
+import { DocumentType } from "../documents/DocumentTypes";
+import './InkTranscription.scss';
+import { aggregateBounds } from '../../Utils';
+import { CollectionFreeFormView } from './collections/collectionFreeForm';
+import { DocumentManager } from "../util/DocumentManager";
+import { InkingStroke } from './InkingStroke';
+
+
+export class InkTranscription extends React.Component {
+    static Instance: InkTranscription;
+
+    @observable _mathRegister: any;
+    @observable _mathRef: any;
+    @observable _textRegister: any;
+    @observable _textRef: any;
+    private lastJiix: any;
+    private currGroup?: Doc;
+    private containingLayout?: Doc;
+
+    constructor(props: Readonly<{}>) {
+        super(props);
+
+        InkTranscription.Instance = this;
+    }
+
+    componentWillUnmount() {
+        this._mathRef.removeEventListener('exported', (e: any) => this.exportInk(e, this._mathRef));
+        this._textRef.removeEventListener('exported', (e: any) => this.exportInk(e, this._textRef));
+    }
+
+    @action
+    setMathRef = (r: any) => {
+        if (!this._mathRegister) {
+            this._mathRegister = r ? iink.register(r, {
+                recognitionParams: {
+                    type: 'MATH',
+                    protocol: 'WEBSOCKET',
+                    server: {
+                        host: 'cloud.myscript.com',
+                        applicationKey: '7277ec34-0c2e-4ee1-9757-ccb657e3f89f',
+                        hmacKey: 'f5cb18f2-1f95-4ddb-96ac-3f7c888dffc1',
+                        websocket: {
+                            pingEnabled: false,
+                            autoReconnect: true
+                        }
+                    },
+                    iink: {
+                        math: {
+                            mimeTypes: ['application/x-latex', 'application/vnd.myscript.jiix']
+                        },
+                        export: {
+                            jiix: {
+                                strokes: true
+                            }
+                        }
+                    }
+                }
+            }) : null;
+        }
+
+        r.addEventListener('exported', (e: any) => this.exportInk(e, this._mathRef));
+
+        return this._mathRef = r;
+    }
+
+    @action
+    setTextRef = (r: any) => {
+        if (!this._textRegister) {
+            this._textRegister = r ? iink.register(r, {
+                recognitionParams: {
+                    type: 'TEXT',
+                    protocol: 'WEBSOCKET',
+                    server: {
+                        host: 'cloud.myscript.com',
+                        applicationKey: '7277ec34-0c2e-4ee1-9757-ccb657e3f89f',
+                        hmacKey: 'f5cb18f2-1f95-4ddb-96ac-3f7c888dffc1',
+                        websocket: {
+                            pingEnabled: false,
+                            autoReconnect: true
+                        }
+                    },
+                    iink: {
+                        text: {
+                            mimeTypes: ['text/plain']
+                        },
+                        export: {
+                            jiix: {
+                                strokes: true
+                            }
+                        }
+                    }
+                }
+            }) : null;
+        }
+
+        r.addEventListener('exported', (e: any) => this.exportInk(e, this._textRef));
+
+        return this._textRef = r;
+    }
+
+    transcribeInk = (groupDoc: Doc | undefined, containingLayout: Doc, inkDocs: Doc[], math: boolean, ffView?: CollectionFreeFormView) => {
+        if (!groupDoc) return;
+        const validInks = inkDocs.filter(s => s.type === DocumentType.INK);
+
+        const strokes: InkData[] = [];
+        const times: number[] = [];
+        // console.log(validInks);
+        validInks.filter(i => Cast(i.data, InkField)).forEach(i => {
+            const d = Cast(i.data, InkField, null);
+            // const left = Math.min(...d?.inkData.map(pd => pd.X) ?? [0]);
+            // const top = Math.min(...d?.inkData.map(pd => pd.Y) ?? [0]);
+            // strokes.push(d.inkData.map(pd => ({ X: pd.X + NumCast(i.x) - left, Y: pd.Y + NumCast(i.y) - top })));
+            const inkStroke = DocumentManager.Instance.getDocumentView(i)?.ComponentView as InkingStroke;
+            strokes.push(d.inkData.map(pd => (inkStroke.ptToScreen({ X: pd.X, Y: pd.Y }))));
+            times.push(DateCast(i.creationDate).getDate().getTime());
+        });
+
+        this.currGroup = groupDoc;
+        this.containingLayout = containingLayout;
+
+        const pointerData = { "events": strokes.map((stroke, i) => this.inkJSON(stroke, times[i])) };
+        // console.log(JSON.stringify(pointerData));
+        // console.log(pointerData);
+        const processGestures = false;
+
+        if (math) {
+            this._mathRef.editor.pointerEvents(pointerData, processGestures);
+        }
+        else {
+            this._textRef.editor.pointerEvents(pointerData, processGestures);
+        }
+    }
+
+    inkJSON = (stroke: InkData, time: number) => {
+        return {
+            "pointerType": "PEN",
+            "pointerId": 1,
+            "x": stroke.map(point => point.X),
+            "y": stroke.map(point => point.Y),
+            "t": new Array(stroke.length).fill(time),
+            "p": new Array(stroke.length).fill(1.0)
+        };
+    }
+
+    mmToPixel = (mm: number) => {
+        return ((96 * mm) / 25.4);
+    }
+
+    calcBounds = (coords: any) => {
+        // find max and min x values and subtract
+        const max = Math.max(...coords);
+        const min = Math.min(...coords);
+        return max - min;
+    }
+
+    subgroupsTranscriptions = (wordInkDocMap: Map<string, Doc[]>) => {
+        // loop through the words in wordInkDocMap
+        // for each word, get the inkDocs
+
+        // iterate through the keys of wordInkDocMap
+        wordInkDocMap.forEach(async (inkDocs: Doc[], word: string) => {
+            const selected = inkDocs.slice();
+            if (!selected) {
+                return;
+            }
+            // TODO: nda - probably have to cast this to an actual Doc
+            const ctx = await Cast(selected[0].context, Doc);
+            if (!ctx) {
+                return;
+            }
+            const docView: CollectionFreeFormView = DocumentManager.Instance.getDocumentView(ctx)?.ComponentView as CollectionFreeFormView;
+
+            if (!docView) return;
+            const marqViewRef = docView._marqueeViewRef.current;
+            if (!marqViewRef) return;
+            //  loop through selected an get the bound
+            const bounds: { x: number, y: number, width?: number, height?: number }[] = []
+
+            selected.map(action(d => {
+                const x = NumCast(d.x);
+                const y = NumCast(d.y);
+                const width = d[WidthSym]();
+                const height = d[HeightSym]();
+                bounds.push({ x, y, width, height });
+            }))
+
+            const aggregBounds = aggregateBounds(bounds, 0, 0);
+
+            if (marqViewRef) {
+                marqViewRef._downX = aggregBounds.x;
+                marqViewRef._downY = aggregBounds.y;
+                marqViewRef._lastX = aggregBounds.r;
+                marqViewRef._lastY = aggregBounds.b;
+            }
+
+            // set the vals for bounds in marqueeView
+
+            selected.map(action(d => {
+                const dx = NumCast(d.x);
+                const dy = NumCast(d.y);
+                delete d.x;
+                delete d.y;
+                delete d.activeFrame;
+                delete d._timecodeToShow;  // bcz: this should be automatic somehow.. along with any other properties that were logically associated with the original collection
+                delete d._timecodeToHide;  // bcz: this should be automatic somehow.. along with any other properties that were logically associated with the original collection
+                // calculate pos based on bounds
+                if (marqViewRef?.Bounds) {
+                    d.x = dx - marqViewRef.Bounds.left - marqViewRef.Bounds.width / 2;
+                    d.y = dy - marqViewRef.Bounds.top - marqViewRef.Bounds.height / 2;
+                }
+                return d;
+            }));
+
+            docView.props.removeDocument?.(selected);
+            const newCollection = marqViewRef?.getCollection(selected, undefined, [], true);
+            if (newCollection) {
+                newCollection.height = newCollection[HeightSym]();
+                newCollection.width = newCollection[WidthSym]();
+                newCollection.title = word;
+            }
+            // nda - bug: when deleting a stroke before leaving writing mode, delete the stroke from unprocessed ink docs
+            // console.log(newCollection);
+            newCollection && docView.props.addDocument?.(newCollection);
+        });
+    }
+
+    exportInk = (e: any, ref: any) => {
+        const exports = e.detail.exports;
+        // console.log(e);
+        if (exports) {
+            if (exports['application/x-latex']) {
+                const latex = exports['application/x-latex'];
+                // console.log(latex);
+
+                if (this.currGroup) {
+                    this.currGroup.text = latex;
+                    this.currGroup.title = latex;
+                }
+
+                ref.editor.clear();
+            }
+            else if (exports['text/plain']) {
+                if (exports['application/vnd.myscript.jiix']) {
+                    this.lastJiix = JSON.parse(exports['application/vnd.myscript.jiix']);
+                    // map timestamp to strokes
+                    const timestampWord = new Map<number, string>();
+                    this.lastJiix.words.map((word: any) => {
+                        if (word.items) {
+                            word.items.forEach((i: { id: string, timestamp: string, X: Array<number>, Y: Array<number>, F: Array<number> }) => {
+                                const ms = Date.parse(i.timestamp);
+                                timestampWord.set(ms, word.label);
+                            })
+                        }
+                    })
+
+                    const wordInkDocMap = new Map<string, Doc[]>();
+                    if (this.currGroup) {
+                        const docList = DocListCast(this.currGroup.data)
+                        docList.forEach((inkDoc: Doc) => {
+                            // just having the times match up and be a unique value (actual timestamp doesn't matter)
+                            const ms = DateCast(inkDoc.creationDate).getDate().getTime() + 14400000;
+                            const word = timestampWord.get(ms);
+                            if (!word) {
+                                return;
+                            }
+                            const entry = wordInkDocMap.get(word);
+                            if (entry) {
+                                entry.push(inkDoc);
+                                wordInkDocMap.set(word, entry);
+                            } else {
+                                const newEntry = [inkDoc];
+                                wordInkDocMap.set(word, newEntry);
+                            }
+                        });
+                        if (this.lastJiix.words.length > 1) this.subgroupsTranscriptions(wordInkDocMap);
+                    }
+                }
+                const text = exports['text/plain'];
+                // console.log(text);
+
+                if (this.currGroup) {
+                    // console.log("curr grouping");
+                    this.currGroup.transcription = text;
+                    this.currGroup.title = text.split("\n")[0];
+                }
+
+                ref.editor.clear();
+            }
+        }
+    }
+
+    render() {
+        return (
+            <div className="ink-transcription">
+                <div className='math-editor'
+                    ref={this.setMathRef}
+                    touch-action="none">
+                </div>
+                <div className='text-editor'
+                    ref={this.setTextRef}
+                    touch-action="none">
+                </div>
+            </div>
+        )
+    }
+}
\ No newline at end of file