import * as iink from 'iink-js'; import { action, observable } from 'mobx'; import * as React from 'react'; import { DataSym, Doc, DocListCast } from '../../fields/Doc'; import { InkData, InkField } from "../../fields/InkField"; import { Cast, DateCast, NumCast } from '../../fields/Types'; import { DocumentType } from "../documents/DocumentTypes"; import './InkTranscription.scss'; import { aggregateBounds, Utils } from '../../Utils'; import { timesSeries } from 'async'; export class InkTranscription extends React.Component { static Instance: InkTranscription; @observable _mathRegister: any; @observable _mathRef: any; @observable _textRegister: any; @observable _textRef: any; private lastJiix: any; private currGroup?: Doc; private wordsBoundMapping: any; private inkDocs: Doc[]; private ffView: any; private containingLayout?: Doc; constructor(props: Readonly<{}>) { super(props); InkTranscription.Instance = this; this.wordsBoundMapping = new Map(); this.inkDocs = new Array(); } componentWillUnmount() { this._mathRef.removeEventListener('exported', (e: any) => this.exportInk(e, this._mathRef)); this._textRef.removeEventListener('exported', (e: any) => this.exportInk(e, this._textRef)); } @action setMathRef = (r: any) => { if (!this._mathRegister) { this._mathRegister = r ? iink.register(r, { recognitionParams: { type: 'MATH', protocol: 'WEBSOCKET', server: { host: 'cloud.myscript.com', applicationKey: '7277ec34-0c2e-4ee1-9757-ccb657e3f89f', hmacKey: 'f5cb18f2-1f95-4ddb-96ac-3f7c888dffc1', websocket: { pingEnabled: false, autoReconnect: true } }, iink: { math: { mimeTypes: ['application/x-latex', 'application/vnd.myscript.jiix'] }, export: { jiix: { strokes: true } } } } }) : null; } r.addEventListener('exported', (e: any) => this.exportInk(e, this._mathRef)); return this._mathRef = r; } @action setTextRef = (r: any) => { if (!this._textRegister) { this._textRegister = r ? iink.register(r, { recognitionParams: { type: 'TEXT', protocol: 'WEBSOCKET', server: { host: 'cloud.myscript.com', applicationKey: '7277ec34-0c2e-4ee1-9757-ccb657e3f89f', hmacKey: 'f5cb18f2-1f95-4ddb-96ac-3f7c888dffc1', websocket: { pingEnabled: false, autoReconnect: true } }, iink: { text: { mimeTypes: ['text/plain'] }, export: { jiix: { strokes: true } } } } }) : null; } r.addEventListener('exported', (e: any) => this.exportInk(e, this._textRef)); return this._textRef = r; } transcribeInk = (groupDoc: Doc | undefined, containingLayout: Doc, inkDocs: Doc[], math: boolean) => { if (!groupDoc) return; this.inkDocs = inkDocs; const validInks = inkDocs.filter(s => s.type === DocumentType.INK); const strokes: InkData[] = []; const times: number[] = []; console.log(validInks); validInks.filter(i => Cast(i.data, InkField)).forEach(i => { const d = Cast(i.data, InkField, null); const left = Math.min(...d?.inkData.map(pd => pd.X) ?? [0]); const top = Math.min(...d?.inkData.map(pd => pd.Y) ?? [0]); strokes.push(d.inkData.map(pd => ({ X: pd.X + NumCast(i.x) - left, Y: pd.Y + NumCast(i.y) - top }))); times.push(DateCast(i.creationDate).getDate().getTime()); }); this.currGroup = groupDoc; this.containingLayout = containingLayout; const pointerData = { "events": strokes.map((stroke, i) => this.inkJSON(stroke, times[i])) }; // console.log(JSON.stringify(pointerData)); // console.log(pointerData); const processGestures = false; if (math) { this._mathRef.editor.pointerEvents(pointerData, processGestures); } else { this._textRef.editor.pointerEvents(pointerData, processGestures); } } inkJSON = (stroke: InkData, time: number) => { const scale = NumCast(this.containingLayout?._viewScale, 1); const panX = NumCast(this.containingLayout?._panX); const panY = NumCast(this.containingLayout?._panY); return { "pointerType": "PEN", "pointerId": 1, "x": stroke.map(point => (point.X - panX) * scale), "y": stroke.map(point => (point.Y - panY) * scale), "t": new Array(stroke.length).fill(time), "p": new Array(stroke.length).fill(1.0) }; } mmToPixel = (mm: number) => { return ((96 * mm) / 25.4); } calcBounds = (coords: any) => { // find max and min x values and subtract const max = Math.max(...coords); const min = Math.min(...coords); return max - min; } exportInk = (e: any, ref: any) => { const exports = e.detail.exports; console.log(e); if (exports) { if (exports['application/x-latex']) { const latex = exports['application/x-latex']; // console.log(latex); if (this.currGroup) { this.currGroup.text = latex; this.currGroup.title = latex; } ref.editor.clear(); } else if (exports['text/plain']) { if (exports['application/vnd.myscript.jiix']) { this.lastJiix = JSON.parse(exports['application/vnd.myscript.jiix']); // map timestamp to strokes const timestampWord = new Map(); this.lastJiix.words.map((word: any) => { word.items.map((i: any) => { const ms = Date.parse(i.timestamp); timestampWord.set(ms, word.label); }) }) const wordInkDocMap = new Map(); if (this.currGroup) { const docList = DocListCast(this.currGroup.data) docList.forEach((inkDoc: Doc) => { const ms = DateCast(inkDoc.creationDate).getDate().getTime(); const word = timestampWord.get(ms); if (!word) { return; } const entry = wordInkDocMap.get(word); if (entry) { entry.push(inkDoc); wordInkDocMap.set(word, entry); } else { const newEntry = [inkDoc]; wordInkDocMap.set(word, newEntry); } }); console.log("getting here"); console.log(wordInkDocMap); } // we can iterate through docs and for each doc index into the map by timestamp // final mapping should be word to inkDoc } const text = exports['text/plain']; // console.log(text); if (exports['application/vnd.myscript.jiix']) { console.log(JSON.parse(exports['application/vnd.myscript.jiix'])); } if (this.currGroup) { console.log("curr grouping"); this.currGroup.text = text; this.currGroup.title = text.split("\n")[0]; } ref.editor.clear(); } } } render() { return (
) } }