import * as iink from 'iink-ts'; import { action, observable } from 'mobx'; import * as React from 'react'; import { imageUrlToBase64 } from '../../ClientUtils'; import { aggregateBounds } from '../../Utils'; import { Doc, DocListCast } from '../../fields/Doc'; import { InkData, InkInkTool, InkTool } from '../../fields/InkField'; import { Cast, DateCast, ImageCast, InkCast, NumCast } from '../../fields/Types'; import { ImageField, URLField } from '../../fields/URLField'; import { gptHandwriting } from '../apis/gpt/GPT'; import { DocumentType } from '../documents/DocumentTypes'; import { Docs } from '../documents/Documents'; import './InkTranscription.scss'; import { InkingStroke } from './InkingStroke'; import { CollectionFreeFormView, MarqueeView } from './collections/collectionFreeForm'; import { DocumentView } from './nodes/DocumentView'; /** * Class component that handles inking in writing mode */ export class InkTranscription extends React.Component { // eslint-disable-next-line no-use-before-define static Instance: InkTranscription; // eslint-disable-next-line @typescript-eslint/no-explicit-any @observable _mathRegister: any = undefined; // eslint-disable-next-line @typescript-eslint/no-explicit-any @observable _mathRef: any = undefined; // eslint-disable-next-line @typescript-eslint/no-explicit-any @observable _textRegister: any = undefined; // eslint-disable-next-line @typescript-eslint/no-explicit-any @observable _textRef: any = undefined; // eslint-disable-next-line @typescript-eslint/no-explicit-any @observable _iinkEditor: any = undefined; // eslint-disable-next-line @typescript-eslint/no-explicit-any @observable _iinkMathEditor: any = undefined; private currGroup?: Doc; private collectionFreeForm?: CollectionFreeFormView; constructor(props: Readonly) { super(props); InkTranscription.Instance = this; } @action // eslint-disable-next-line @typescript-eslint/no-explicit-any setMathRef = async (r: any) => { if (!this._mathRegister && r) { const options = { configuration: { server: { scheme: 'https' as iink.TScheme, host: 'cloud.myscript.com', applicationKey: 'c0901093-5ac5-4454-8e64-0def0f13f2ca', hmacKey: 'f6465cca-1856-4492-a6a4-e2395841be2f', protocol: 'WEBSOCKET', }, recognition: { type: 'MATH', math: { mimeTypes: ['application/x-latex'] as unknown as 'application/vnd.myscript.jiix'[], }, }, }, }; this._iinkMathEditor = await iink.Editor.load(r, 'INKV2', options); this._mathRegister = r; // eslint-disable-next-line @typescript-eslint/no-explicit-any r?.addEventListener('exported', (e: any) => this.exportInk(e)); return (this._textRef = r); } }; @action // eslint-disable-next-line @typescript-eslint/no-explicit-any setTextRef = async (r: any) => { if (!this._textRegister && r) { const options = { configuration: { server: { scheme: 'https' as iink.TScheme, host: 'cloud.myscript.com', applicationKey: 'c0901093-5ac5-4454-8e64-0def0f13f2ca', hmacKey: 'f6465cca-1856-4492-a6a4-e2395841be2f', protocol: 'WEBSOCKET', }, recognition: { type: 'TEXT', lang: 'en_US', text: { mimeTypes: ['application/vnd.myscript.jiix'] as 'application/vnd.myscript.jiix'[], }, }, }, }; this._iinkEditor = await iink.Editor.load(r, 'INKV2', options); this._textRegister = r; // eslint-disable-next-line @typescript-eslint/no-explicit-any r?.addEventListener('exported', (e: any) => this.exportInk(e)); return (this._textRef = r); } }; _ffview: CollectionFreeFormView | undefined; /** * Handles processing Dash Doc data for ink transcription. * * @param groupDoc the group which contains the ink strokes we want to transcribe * @param inkDocs the ink docs contained within the selected group * @param math boolean whether to do math transcription or not */ transcribeInk = (ffview: CollectionFreeFormView, groupDoc: Doc | undefined, inkDocs: Doc[], math: boolean) => { if (!groupDoc) return; const validInks = inkDocs.filter(s => s.type === DocumentType.INK); const strokes: InkData[] = []; const times: number[] = []; validInks .filter(i => InkCast(i[Doc.LayoutDataKey(i)])) .forEach(i => { const d = InkCast(i[Doc.LayoutDataKey(i)])!; const authorTime = DateCast(i.author_date)?.getDate().getTime() ?? 0; const inkStroke = DocumentView.getDocumentView(i)?.ComponentView as InkingStroke; strokes.push(d.inkData.map(pd => inkStroke.ptToScreen({ X: pd.X, Y: pd.Y }))); times.push(authorTime); }); this._ffview = ffview; this.currGroup = groupDoc; const pointerData = strokes.map((stroke, i) => this.inkJSON(stroke, times[i])); (math ? this._iinkMathEditor : this._iinkEditor).importPointEvents(pointerData); }; convertPointsToString(points: InkData[]): string { return points[0].map(point => `new Point(${point.X}, ${point.Y})`).join(','); } convertPointsToString2(points: InkData[]): string { return points[0].map(point => `(${point.X},${point.Y})`).join(','); } /** * Converts the Dash Ink Data to JSON. * * @param stroke The dash ink data * @param time the time of the stroke * @returns json object representation of ink data */ inkJSON = (stroke: InkData, time: number) => { interface strokeData { x: number; y: number; t: number; p: number; } const strokeObjects: strokeData[] = []; stroke.forEach(point => { const tempObject: strokeData = { x: point.X, y: point.Y, t: time, p: 1.0, }; strokeObjects.push(tempObject); }); return { pointerType: 'PEN', pointerId: 1, pointers: strokeObjects, }; }; /** * Creates subgroups for each word for the whole text transcription * @param wordInkDocMap the mapping of words to ink strokes (Ink Docs) */ subgroupsTranscriptions = (wordInkDocMap: Map) => { // iterate through the keys of wordInkDocMap wordInkDocMap.forEach(async (inkDocs: Doc[], word: string) => { const selected = inkDocs.slice(); if (!selected) { return; } const ctx = await Cast(selected[0].embedContainer, Doc); if (!ctx) { return; } const docView: CollectionFreeFormView = DocumentView.getDocumentView(ctx)?.ComponentView as CollectionFreeFormView; // DocumentManager.Instance.getDocumentView(ctx)?.ComponentView as CollectionFreeFormView; if (!docView) return; const marqViewRef = docView._marqueeViewRef.current; if (!marqViewRef) return; this.groupInkDocs(selected, docView, word); }); }; /** * Event listener function for when the 'exported' event is heard. * * @param e the event objects * @param ref the ref to the editor */ // eslint-disable-next-line @typescript-eslint/no-explicit-any exportInk = (e: { detail: { [key: string]: string } }) => { const exports = e.detail; if (exports['application/x-latex']) { const latex = exports['application/x-latex']; if (this.currGroup) { this.currGroup.text = latex; this.currGroup.title = latex; } this._ffview?.addDocument( Docs.Create.EquationDocument(latex, { title: '', x: this.currGroup?.x as number, y: (this.currGroup?.y as number) + (this.currGroup?.height as number), nativeHeight: 40, _height: 50, nativeWidth: 40, _width: 50, }) ); // this.showRecogBox(latex as string); this._iinkMathEditor.clear(); } else if (exports['application/vnd.myscript.jiix']) { const lastJiix = exports['application/vnd.myscript.jiix'] as unknown as { words: string[]; label: string }; // map timestamp to strokes const timestampWord = new Map(); // eslint-disable-next-line @typescript-eslint/no-explicit-any lastJiix.words.map((word: any) => { if (word.items) { word.items.forEach((i: { id: string; timestamp: string; X: Array; Y: Array; F: Array }) => { const ms = Date.parse(i.timestamp); timestampWord.set(ms, word.label); }); } }); const wordInkDocMap = new Map(); if (this.currGroup) { DocListCast(this.currGroup.data).forEach((inkDoc: Doc) => { // just having the times match up and be a unique value (actual timestamp doesn't matter) const ms = (DateCast(inkDoc.author_date)?.getDate().getTime() ?? 0) + 14400000; const word = timestampWord.get(ms); if (word) { const entry = wordInkDocMap.get(word); if (entry) { entry.push(inkDoc); wordInkDocMap.set(word, entry); } else { const newEntry = [inkDoc]; wordInkDocMap.set(word, newEntry); } } }); if (lastJiix.words.length > 1) this.subgroupsTranscriptions(wordInkDocMap); } this.showRecogBox(lastJiix.label); this._iinkEditor.clear(); } }; private showRecogBox(text: string) { if (this.currGroup) { let response; // DocumentView.getDocumentView(this.currGroup)?.ComponentView?.updateIcon?.(); // const image = await this.getIcon(); // const { href } = (image as URLField).url; // const hrefParts = href.split('.'); // const hrefComplete = `${hrefParts[0]}_o.${hrefParts[1]}`; // try { // const hrefBase64 = await imageUrlToBase64(hrefComplete); // response = await gptHandwriting(hrefBase64); // } catch { // console.error('Error getting image'); // } const textBoxText = 'iink: ' + text + '\n' + '\n' + 'ChatGPT: ' + response; this.currGroup.transcription = response; this.currGroup.title = response; if (!this.currGroup.hasTextBox) { const newDoc = Docs.Create.TextDocument(textBoxText, { title: '', x: this.currGroup.x as number, y: (this.currGroup.y as number) + (this.currGroup.height as number) }); newDoc.height = 200; this.collectionFreeForm?.addDocument(newDoc); this.currGroup.hasTextBox = true; } } } /** * gets the icon of the collection that was just made * @returns the image of the collection */ async getIcon() { const docView = DocumentView.getDocumentView(this.currGroup); if (docView) { docView.ComponentView?.updateIcon?.(); return new Promise(res => setTimeout(() => res(ImageCast(docView.Document.icon)), 1000)); } return undefined; } /** * Creates the ink grouping once the user leaves the writing mode. */ createInkGroup() { // TODO nda - if document being added to is a inkGrouping then we can just add to that group if (Doc.ActiveTool === InkTool.Ink && [InkInkTool.Write, InkInkTool.Math].includes(Doc.ActiveInk)) { CollectionFreeFormView.collectionsWithUnprocessedInk.forEach(ffView => { // TODO: nda - will probably want to go through ffView unprocessed docs and then see if any of the inksToGroup docs are in it and only use those const selected = ffView.unprocessedDocs; const newCollection = this.groupInkDocs( selected.filter(doc => doc.embedContainer), ffView ); ffView.unprocessedDocs = []; InkTranscription.Instance.transcribeInk(ffView, newCollection, selected, Doc.ActiveInk === InkInkTool.Math); }); } CollectionFreeFormView.collectionsWithUnprocessedInk.clear(); } /** * Creates the groupings for a given list of ink docs on a specific doc view * @param selected: the list of ink docs to create a grouping of * @param docView: the view in which we want the grouping to be created * @param word: optional param if the group we are creating is a word (subgrouping individual words) * @returns a new collection Doc or undefined if the grouping fails */ groupInkDocs(selected: Doc[], docView: CollectionFreeFormView, word?: string): Doc | undefined { this.collectionFreeForm = docView; const bounds: { x: number; y: number; width?: number; height?: number }[] = []; // calculate the necessary bounds from the selected ink docs selected.forEach( action(d => { const x = NumCast(d.x); const y = NumCast(d.y); const width = NumCast(d._width); const height = NumCast(d._height); bounds.push({ x, y, width, height }); }) ); // calculate the aggregated bounds const aggregBounds = aggregateBounds(bounds, 0, 0); const marqViewRef = docView._marqueeViewRef.current; // set the vals for bounds in marqueeView if (marqViewRef) { marqViewRef._downX = aggregBounds.x; marqViewRef._downY = aggregBounds.y; marqViewRef._lastX = aggregBounds.r; marqViewRef._lastY = aggregBounds.b; } // map through all the selected ink strokes and create the groupings selected.forEach( action(d => { const dx = NumCast(d.x); const dy = NumCast(d.y); delete d.x; delete d.y; delete d.activeFrame; delete d._timecodeToShow; // bcz: this should be automatic somehow.. along with any other properties that were logically associated with the original collection delete d._timecodeToHide; // bcz: this should be automatic somehow.. along with any other properties that were logically associated with the original collection // calculate pos based on bounds if (marqViewRef?.Bounds) { d.x = dx - marqViewRef.Bounds.left - marqViewRef.Bounds.width / 2; d.y = dy - marqViewRef.Bounds.top - marqViewRef.Bounds.height / 2; } return d; }) ); docView.props.removeDocument?.(selected); // Gets a collection based on the selected nodes using a marquee view ref const newCollection = MarqueeView.getCollection(selected, undefined, true, marqViewRef?.Bounds ?? { top: 1, left: 1, width: 1, height: 1 }); // if the grouping we are creating is an individual word if (word) { newCollection.title = word; } // nda - bug: when deleting a stroke before leaving writing mode, delete the stroke from unprocessed ink docs docView.props.addDocument?.(newCollection); newCollection.hasTextBox = false; return newCollection; } render() { return (
); } }