aboutsummaryrefslogtreecommitdiff
path: root/src/client/views/InkTranscription.tsx
diff options
context:
space:
mode:
authorbobzel <zzzman@gmail.com>2023-12-21 14:55:48 -0500
committerbobzel <zzzman@gmail.com>2023-12-21 14:55:48 -0500
commit1caba64ee0f32ee8af79263cd4ef2a8bc5d5146e (patch)
tree0fa0e957d1f342fdc6ed4a4b43f5dddfddb1298a /src/client/views/InkTranscription.tsx
parent02eb7da95df283606d4275a22d9451cef371c3b5 (diff)
parent2691b951d96f2ce7652acbea9e340b61737b3b57 (diff)
Merge branch 'moreUpgrading' into dataViz-annotations
Diffstat (limited to 'src/client/views/InkTranscription.tsx')
-rw-r--r--src/client/views/InkTranscription.tsx700
1 files changed, 350 insertions, 350 deletions
diff --git a/src/client/views/InkTranscription.tsx b/src/client/views/InkTranscription.tsx
index 17136a737..b2ac208ca 100644
--- a/src/client/views/InkTranscription.tsx
+++ b/src/client/views/InkTranscription.tsx
@@ -1,350 +1,350 @@
-import * as iink from 'iink-js';
-import { action, observable } from 'mobx';
-import * as React from 'react';
-import { Doc, DocListCast } from '../../fields/Doc';
-import { InkData, InkField, InkTool } from '../../fields/InkField';
-import { Cast, DateCast, NumCast } from '../../fields/Types';
-import { aggregateBounds } from '../../Utils';
-import { DocumentType } from '../documents/DocumentTypes';
-import { DocumentManager } from '../util/DocumentManager';
-import { CollectionFreeFormView } from './collections/collectionFreeForm';
-import { InkingStroke } from './InkingStroke';
-import './InkTranscription.scss';
-
-/**
- * Class component that handles inking in writing mode
- */
-export class InkTranscription extends React.Component {
- static Instance: InkTranscription;
-
- @observable _mathRegister: any;
- @observable _mathRef: any;
- @observable _textRegister: any;
- @observable _textRef: any;
- private lastJiix: any;
- private currGroup?: Doc;
-
- constructor(props: Readonly<{}>) {
- super(props);
-
- InkTranscription.Instance = this;
- }
-
- componentWillUnmount() {
- this._mathRef.removeEventListener('exported', (e: any) => this.exportInk(e, this._mathRef));
- this._textRef.removeEventListener('exported', (e: any) => this.exportInk(e, this._textRef));
- }
-
- @action
- setMathRef = (r: any) => {
- if (!this._mathRegister) {
- this._mathRegister = r
- ? iink.register(r, {
- recognitionParams: {
- type: 'MATH',
- protocol: 'WEBSOCKET',
- server: {
- host: 'cloud.myscript.com',
- applicationKey: process.env.IINKJS_APP,
- hmacKey: process.env.IINKJS_HMAC,
- websocket: {
- pingEnabled: false,
- autoReconnect: true,
- },
- },
- iink: {
- math: {
- mimeTypes: ['application/x-latex', 'application/vnd.myscript.jiix'],
- },
- export: {
- jiix: {
- strokes: true,
- },
- },
- },
- },
- })
- : null;
- }
-
- r?.addEventListener('exported', (e: any) => this.exportInk(e, this._mathRef));
-
- return (this._mathRef = r);
- };
-
- @action
- setTextRef = (r: any) => {
- if (!this._textRegister) {
- this._textRegister = r
- ? iink.register(r, {
- recognitionParams: {
- type: 'TEXT',
- protocol: 'WEBSOCKET',
- server: {
- host: 'cloud.myscript.com',
- applicationKey: '7277ec34-0c2e-4ee1-9757-ccb657e3f89f',
- hmacKey: 'f5cb18f2-1f95-4ddb-96ac-3f7c888dffc1',
- websocket: {
- pingEnabled: false,
- autoReconnect: true,
- },
- },
- iink: {
- text: {
- mimeTypes: ['text/plain'],
- },
- export: {
- jiix: {
- strokes: true,
- },
- },
- },
- },
- })
- : null;
- }
-
- r?.addEventListener('exported', (e: any) => this.exportInk(e, this._textRef));
-
- return (this._textRef = r);
- };
-
- /**
- * Handles processing Dash Doc data for ink transcription.
- *
- * @param groupDoc the group which contains the ink strokes we want to transcribe
- * @param inkDocs the ink docs contained within the selected group
- * @param math boolean whether to do math transcription or not
- */
- transcribeInk = (groupDoc: Doc | undefined, inkDocs: Doc[], math: boolean) => {
- if (!groupDoc) return;
- const validInks = inkDocs.filter(s => s.type === DocumentType.INK);
-
- const strokes: InkData[] = [];
- const times: number[] = [];
- validInks
- .filter(i => Cast(i[Doc.LayoutFieldKey(i)], InkField))
- .forEach(i => {
- const d = Cast(i[Doc.LayoutFieldKey(i)], InkField, null);
- const inkStroke = DocumentManager.Instance.getDocumentView(i)?.ComponentView as InkingStroke;
- strokes.push(d.inkData.map(pd => inkStroke.ptToScreen({ X: pd.X, Y: pd.Y })));
- times.push(DateCast(i.author_date).getDate().getTime());
- });
-
- this.currGroup = groupDoc;
-
- const pointerData = { events: strokes.map((stroke, i) => this.inkJSON(stroke, times[i])) };
- const processGestures = false;
-
- if (math) {
- this._mathRef.editor.pointerEvents(pointerData, processGestures);
- } else {
- this._textRef.editor.pointerEvents(pointerData, processGestures);
- }
- };
-
- /**
- * Converts the Dash Ink Data to JSON.
- *
- * @param stroke The dash ink data
- * @param time the time of the stroke
- * @returns json object representation of ink data
- */
- inkJSON = (stroke: InkData, time: number) => {
- return {
- pointerType: 'PEN',
- pointerId: 1,
- x: stroke.map(point => point.X),
- y: stroke.map(point => point.Y),
- t: new Array(stroke.length).fill(time),
- p: new Array(stroke.length).fill(1.0),
- };
- };
-
- /**
- * Creates subgroups for each word for the whole text transcription
- * @param wordInkDocMap the mapping of words to ink strokes (Ink Docs)
- */
- subgroupsTranscriptions = (wordInkDocMap: Map<string, Doc[]>) => {
- // iterate through the keys of wordInkDocMap
- wordInkDocMap.forEach(async (inkDocs: Doc[], word: string) => {
- const selected = inkDocs.slice();
- if (!selected) {
- return;
- }
- const ctx = await Cast(selected[0].embedContainer, Doc);
- if (!ctx) {
- return;
- }
- const docView: CollectionFreeFormView = DocumentManager.Instance.getDocumentView(ctx)?.ComponentView as CollectionFreeFormView;
-
- if (!docView) return;
- const marqViewRef = docView._marqueeViewRef.current;
- if (!marqViewRef) return;
- this.groupInkDocs(selected, docView, word);
- });
- };
-
- /**
- * Event listener function for when the 'exported' event is heard.
- *
- * @param e the event objects
- * @param ref the ref to the editor
- */
- exportInk = (e: any, ref: any) => {
- const exports = e.detail.exports;
- if (exports) {
- if (exports['application/x-latex']) {
- const latex = exports['application/x-latex'];
- if (this.currGroup) {
- this.currGroup.text = latex;
- this.currGroup.title = latex;
- }
-
- ref.editor.clear();
- } else if (exports['text/plain']) {
- if (exports['application/vnd.myscript.jiix']) {
- this.lastJiix = JSON.parse(exports['application/vnd.myscript.jiix']);
- // map timestamp to strokes
- const timestampWord = new Map<number, string>();
- this.lastJiix.words.map((word: any) => {
- if (word.items) {
- word.items.forEach((i: { id: string; timestamp: string; X: Array<number>; Y: Array<number>; F: Array<number> }) => {
- const ms = Date.parse(i.timestamp);
- timestampWord.set(ms, word.label);
- });
- }
- });
-
- const wordInkDocMap = new Map<string, Doc[]>();
- if (this.currGroup) {
- const docList = DocListCast(this.currGroup.data);
- docList.forEach((inkDoc: Doc) => {
- // just having the times match up and be a unique value (actual timestamp doesn't matter)
- const ms = DateCast(inkDoc.author_date).getDate().getTime() + 14400000;
- const word = timestampWord.get(ms);
- if (!word) {
- return;
- }
- const entry = wordInkDocMap.get(word);
- if (entry) {
- entry.push(inkDoc);
- wordInkDocMap.set(word, entry);
- } else {
- const newEntry = [inkDoc];
- wordInkDocMap.set(word, newEntry);
- }
- });
- if (this.lastJiix.words.length > 1) this.subgroupsTranscriptions(wordInkDocMap);
- }
- }
- const text = exports['text/plain'];
-
- if (this.currGroup) {
- this.currGroup.transcription = text;
- this.currGroup.title = text.split('\n')[0];
- }
-
- ref.editor.clear();
- }
- }
- };
-
- /**
- * Creates the ink grouping once the user leaves the writing mode.
- */
- createInkGroup() {
- // TODO nda - if document being added to is a inkGrouping then we can just add to that group
- if (Doc.ActiveTool === InkTool.Write) {
- CollectionFreeFormView.collectionsWithUnprocessedInk.forEach(ffView => {
- // TODO: nda - will probably want to go through ffView unprocessed docs and then see if any of the inksToGroup docs are in it and only use those
- const selected = ffView.unprocessedDocs;
- const newCollection = this.groupInkDocs(
- selected.filter(doc => doc.embedContainer),
- ffView
- );
- ffView.unprocessedDocs = [];
-
- InkTranscription.Instance.transcribeInk(newCollection, selected, false);
- });
- }
- CollectionFreeFormView.collectionsWithUnprocessedInk.clear();
- }
-
- /**
- * Creates the groupings for a given list of ink docs on a specific doc view
- * @param selected: the list of ink docs to create a grouping of
- * @param docView: the view in which we want the grouping to be created
- * @param word: optional param if the group we are creating is a word (subgrouping individual words)
- * @returns a new collection Doc or undefined if the grouping fails
- */
- groupInkDocs(selected: Doc[], docView: CollectionFreeFormView, word?: string): Doc | undefined {
- const bounds: { x: number; y: number; width?: number; height?: number }[] = [];
-
- // calculate the necessary bounds from the selected ink docs
- selected.map(
- action(d => {
- const x = NumCast(d.x);
- const y = NumCast(d.y);
- const width = NumCast(d._width);
- const height = NumCast(d._height);
- bounds.push({ x, y, width, height });
- })
- );
-
- // calculate the aggregated bounds
- const aggregBounds = aggregateBounds(bounds, 0, 0);
- const marqViewRef = docView._marqueeViewRef.current;
-
- // set the vals for bounds in marqueeView
- if (marqViewRef) {
- marqViewRef._downX = aggregBounds.x;
- marqViewRef._downY = aggregBounds.y;
- marqViewRef._lastX = aggregBounds.r;
- marqViewRef._lastY = aggregBounds.b;
- }
-
- // map through all the selected ink strokes and create the groupings
- selected.map(
- action(d => {
- const dx = NumCast(d.x);
- const dy = NumCast(d.y);
- delete d.x;
- delete d.y;
- delete d.activeFrame;
- delete d._timecodeToShow; // bcz: this should be automatic somehow.. along with any other properties that were logically associated with the original collection
- delete d._timecodeToHide; // bcz: this should be automatic somehow.. along with any other properties that were logically associated with the original collection
- // calculate pos based on bounds
- if (marqViewRef?.Bounds) {
- d.x = dx - marqViewRef.Bounds.left - marqViewRef.Bounds.width / 2;
- d.y = dy - marqViewRef.Bounds.top - marqViewRef.Bounds.height / 2;
- }
- return d;
- })
- );
- docView.props.removeDocument?.(selected);
- // Gets a collection based on the selected nodes using a marquee view ref
- const newCollection = marqViewRef?.getCollection(selected, undefined, true);
- if (newCollection) {
- newCollection.width = NumCast(newCollection._width);
- newCollection.height = NumCast(newCollection._height);
- // if the grouping we are creating is an individual word
- if (word) {
- newCollection.title = word;
- }
- }
-
- // nda - bug: when deleting a stroke before leaving writing mode, delete the stroke from unprocessed ink docs
- newCollection && docView.props.addDocument?.(newCollection);
- return newCollection;
- }
-
- render() {
- return (
- <div className="ink-transcription">
- <div className="math-editor" ref={this.setMathRef} touch-action="none"></div>
- <div className="text-editor" ref={this.setTextRef} touch-action="none"></div>
- </div>
- );
- }
-}
+// import * as iink from 'iink-js';
+// import { action, observable } from 'mobx';
+// import * as React from 'react';
+// import { Doc, DocListCast } from '../../fields/Doc';
+// import { InkData, InkField, InkTool } from '../../fields/InkField';
+// import { Cast, DateCast, NumCast } from '../../fields/Types';
+// import { aggregateBounds } from '../../Utils';
+// import { DocumentType } from '../documents/DocumentTypes';
+// import { DocumentManager } from '../util/DocumentManager';
+// import { CollectionFreeFormView } from './collections/collectionFreeForm';
+// import { InkingStroke } from './InkingStroke';
+// import './InkTranscription.scss';
+
+// /**
+// * Class component that handles inking in writing mode
+// */
+// export class InkTranscription extends React.Component {
+// static Instance: InkTranscription;
+
+// @observable _mathRegister: any;
+// @observable _mathRef: any;
+// @observable _textRegister: any;
+// @observable _textRef: any;
+// private lastJiix: any;
+// private currGroup?: Doc;
+
+// constructor(props: Readonly<{}>) {
+// super(props);
+
+// InkTranscription.Instance = this;
+// }
+
+// componentWillUnmount() {
+// this._mathRef.removeEventListener('exported', (e: any) => this.exportInk(e, this._mathRef));
+// this._textRef.removeEventListener('exported', (e: any) => this.exportInk(e, this._textRef));
+// }
+
+// @action
+// setMathRef = (r: any) => {
+// if (!this._mathRegister) {
+// this._mathRegister = r
+// ? iink.register(r, {
+// recognitionParams: {
+// type: 'MATH',
+// protocol: 'WEBSOCKET',
+// server: {
+// host: 'cloud.myscript.com',
+// applicationKey: process.env.IINKJS_APP,
+// hmacKey: process.env.IINKJS_HMAC,
+// websocket: {
+// pingEnabled: false,
+// autoReconnect: true,
+// },
+// },
+// iink: {
+// math: {
+// mimeTypes: ['application/x-latex', 'application/vnd.myscript.jiix'],
+// },
+// export: {
+// jiix: {
+// strokes: true,
+// },
+// },
+// },
+// },
+// })
+// : null;
+// }
+
+// r?.addEventListener('exported', (e: any) => this.exportInk(e, this._mathRef));
+
+// return (this._mathRef = r);
+// };
+
+// @action
+// setTextRef = (r: any) => {
+// if (!this._textRegister) {
+// this._textRegister = r
+// ? iink.register(r, {
+// recognitionParams: {
+// type: 'TEXT',
+// protocol: 'WEBSOCKET',
+// server: {
+// host: 'cloud.myscript.com',
+// applicationKey: '7277ec34-0c2e-4ee1-9757-ccb657e3f89f',
+// hmacKey: 'f5cb18f2-1f95-4ddb-96ac-3f7c888dffc1',
+// websocket: {
+// pingEnabled: false,
+// autoReconnect: true,
+// },
+// },
+// iink: {
+// text: {
+// mimeTypes: ['text/plain'],
+// },
+// export: {
+// jiix: {
+// strokes: true,
+// },
+// },
+// },
+// },
+// })
+// : null;
+// }
+
+// r?.addEventListener('exported', (e: any) => this.exportInk(e, this._textRef));
+
+// return (this._textRef = r);
+// };
+
+// /**
+// * Handles processing Dash Doc data for ink transcription.
+// *
+// * @param groupDoc the group which contains the ink strokes we want to transcribe
+// * @param inkDocs the ink docs contained within the selected group
+// * @param math boolean whether to do math transcription or not
+// */
+// transcribeInk = (groupDoc: Doc | undefined, inkDocs: Doc[], math: boolean) => {
+// if (!groupDoc) return;
+// const validInks = inkDocs.filter(s => s.type === DocumentType.INK);
+
+// const strokes: InkData[] = [];
+// const times: number[] = [];
+// validInks
+// .filter(i => Cast(i[Doc.LayoutFieldKey(i)], InkField))
+// .forEach(i => {
+// const d = Cast(i[Doc.LayoutFieldKey(i)], InkField, null);
+// const inkStroke = DocumentManager.Instance.getDocumentView(i)?.ComponentView as InkingStroke;
+// strokes.push(d.inkData.map(pd => inkStroke.ptToScreen({ X: pd.X, Y: pd.Y })));
+// times.push(DateCast(i.author_date).getDate().getTime());
+// });
+
+// this.currGroup = groupDoc;
+
+// const pointerData = { events: strokes.map((stroke, i) => this.inkJSON(stroke, times[i])) };
+// const processGestures = false;
+
+// if (math) {
+// this._mathRef.editor.pointerEvents(pointerData, processGestures);
+// } else {
+// this._textRef.editor.pointerEvents(pointerData, processGestures);
+// }
+// };
+
+// /**
+// * Converts the Dash Ink Data to JSON.
+// *
+// * @param stroke The dash ink data
+// * @param time the time of the stroke
+// * @returns json object representation of ink data
+// */
+// inkJSON = (stroke: InkData, time: number) => {
+// return {
+// pointerType: 'PEN',
+// pointerId: 1,
+// x: stroke.map(point => point.X),
+// y: stroke.map(point => point.Y),
+// t: new Array(stroke.length).fill(time),
+// p: new Array(stroke.length).fill(1.0),
+// };
+// };
+
+// /**
+// * Creates subgroups for each word for the whole text transcription
+// * @param wordInkDocMap the mapping of words to ink strokes (Ink Docs)
+// */
+// subgroupsTranscriptions = (wordInkDocMap: Map<string, Doc[]>) => {
+// // iterate through the keys of wordInkDocMap
+// wordInkDocMap.forEach(async (inkDocs: Doc[], word: string) => {
+// const selected = inkDocs.slice();
+// if (!selected) {
+// return;
+// }
+// const ctx = await Cast(selected[0].embedContainer, Doc);
+// if (!ctx) {
+// return;
+// }
+// const docView: CollectionFreeFormView = DocumentManager.Instance.getDocumentView(ctx)?.ComponentView as CollectionFreeFormView;
+
+// if (!docView) return;
+// const marqViewRef = docView._marqueeViewRef.current;
+// if (!marqViewRef) return;
+// this.groupInkDocs(selected, docView, word);
+// });
+// };
+
+// /**
+// * Event listener function for when the 'exported' event is heard.
+// *
+// * @param e the event objects
+// * @param ref the ref to the editor
+// */
+// exportInk = (e: any, ref: any) => {
+// const exports = e.detail.exports;
+// if (exports) {
+// if (exports['application/x-latex']) {
+// const latex = exports['application/x-latex'];
+// if (this.currGroup) {
+// this.currGroup.text = latex;
+// this.currGroup.title = latex;
+// }
+
+// ref.editor.clear();
+// } else if (exports['text/plain']) {
+// if (exports['application/vnd.myscript.jiix']) {
+// this.lastJiix = JSON.parse(exports['application/vnd.myscript.jiix']);
+// // map timestamp to strokes
+// const timestampWord = new Map<number, string>();
+// this.lastJiix.words.map((word: any) => {
+// if (word.items) {
+// word.items.forEach((i: { id: string; timestamp: string; X: Array<number>; Y: Array<number>; F: Array<number> }) => {
+// const ms = Date.parse(i.timestamp);
+// timestampWord.set(ms, word.label);
+// });
+// }
+// });
+
+// const wordInkDocMap = new Map<string, Doc[]>();
+// if (this.currGroup) {
+// const docList = DocListCast(this.currGroup.data);
+// docList.forEach((inkDoc: Doc) => {
+// // just having the times match up and be a unique value (actual timestamp doesn't matter)
+// const ms = DateCast(inkDoc.author_date).getDate().getTime() + 14400000;
+// const word = timestampWord.get(ms);
+// if (!word) {
+// return;
+// }
+// const entry = wordInkDocMap.get(word);
+// if (entry) {
+// entry.push(inkDoc);
+// wordInkDocMap.set(word, entry);
+// } else {
+// const newEntry = [inkDoc];
+// wordInkDocMap.set(word, newEntry);
+// }
+// });
+// if (this.lastJiix.words.length > 1) this.subgroupsTranscriptions(wordInkDocMap);
+// }
+// }
+// const text = exports['text/plain'];
+
+// if (this.currGroup) {
+// this.currGroup.transcription = text;
+// this.currGroup.title = text.split('\n')[0];
+// }
+
+// ref.editor.clear();
+// }
+// }
+// };
+
+// /**
+// * Creates the ink grouping once the user leaves the writing mode.
+// */
+// createInkGroup() {
+// // TODO nda - if document being added to is a inkGrouping then we can just add to that group
+// if (Doc.ActiveTool === InkTool.Write) {
+// CollectionFreeFormView.collectionsWithUnprocessedInk.forEach(ffView => {
+// // TODO: nda - will probably want to go through ffView unprocessed docs and then see if any of the inksToGroup docs are in it and only use those
+// const selected = ffView.unprocessedDocs;
+// const newCollection = this.groupInkDocs(
+// selected.filter(doc => doc.embedContainer),
+// ffView
+// );
+// ffView.unprocessedDocs = [];
+
+// InkTranscription.Instance.transcribeInk(newCollection, selected, false);
+// });
+// }
+// CollectionFreeFormView.collectionsWithUnprocessedInk.clear();
+// }
+
+// /**
+// * Creates the groupings for a given list of ink docs on a specific doc view
+// * @param selected: the list of ink docs to create a grouping of
+// * @param docView: the view in which we want the grouping to be created
+// * @param word: optional param if the group we are creating is a word (subgrouping individual words)
+// * @returns a new collection Doc or undefined if the grouping fails
+// */
+// groupInkDocs(selected: Doc[], docView: CollectionFreeFormView, word?: string): Doc | undefined {
+// const bounds: { x: number; y: number; width?: number; height?: number }[] = [];
+
+// // calculate the necessary bounds from the selected ink docs
+// selected.map(
+// action(d => {
+// const x = NumCast(d.x);
+// const y = NumCast(d.y);
+// const width = NumCast(d._width);
+// const height = NumCast(d._height);
+// bounds.push({ x, y, width, height });
+// })
+// );
+
+// // calculate the aggregated bounds
+// const aggregBounds = aggregateBounds(bounds, 0, 0);
+// const marqViewRef = docView._marqueeViewRef.current;
+
+// // set the vals for bounds in marqueeView
+// if (marqViewRef) {
+// marqViewRef._downX = aggregBounds.x;
+// marqViewRef._downY = aggregBounds.y;
+// marqViewRef._lastX = aggregBounds.r;
+// marqViewRef._lastY = aggregBounds.b;
+// }
+
+// // map through all the selected ink strokes and create the groupings
+// selected.map(
+// action(d => {
+// const dx = NumCast(d.x);
+// const dy = NumCast(d.y);
+// delete d.x;
+// delete d.y;
+// delete d.activeFrame;
+// delete d._timecodeToShow; // bcz: this should be automatic somehow.. along with any other properties that were logically associated with the original collection
+// delete d._timecodeToHide; // bcz: this should be automatic somehow.. along with any other properties that were logically associated with the original collection
+// // calculate pos based on bounds
+// if (marqViewRef?.Bounds) {
+// d.x = dx - marqViewRef.Bounds.left - marqViewRef.Bounds.width / 2;
+// d.y = dy - marqViewRef.Bounds.top - marqViewRef.Bounds.height / 2;
+// }
+// return d;
+// })
+// );
+// docView.props.removeDocument?.(selected);
+// // Gets a collection based on the selected nodes using a marquee view ref
+// const newCollection = marqViewRef?.getCollection(selected, undefined, true);
+// if (newCollection) {
+// newCollection.width = NumCast(newCollection._width);
+// newCollection.height = NumCast(newCollection._height);
+// // if the grouping we are creating is an individual word
+// if (word) {
+// newCollection.title = word;
+// }
+// }
+
+// // nda - bug: when deleting a stroke before leaving writing mode, delete the stroke from unprocessed ink docs
+// newCollection && docView.props.addDocument?.(newCollection);
+// return newCollection;
+// }
+
+// render() {
+// return (
+// <div className="ink-transcription">
+// <div className="math-editor" ref={this.setMathRef} touch-action="none"></div>
+// <div className="text-editor" ref={this.setTextRef} touch-action="none"></div>
+// </div>
+// );
+// }
+// }