aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
Diffstat
-rw-r--r--package-lock.json14
-rw-r--r--package.json1
-rw-r--r--src/client/cognitive_services/CognitiveServices.ts11
-rw-r--r--src/client/documents/Documents.ts2
-rw-r--r--src/client/views/DocumentDecorations.tsx13
-rw-r--r--src/client/views/InkTranscription.tsx687
-rw-r--r--src/client/views/InkingStroke.tsx4
-rw-r--r--src/client/views/MainView.tsx2
-rw-r--r--src/client/views/global/globalScripts.ts8
-rw-r--r--src/client/views/nodes/DiagramBox.tsx32
-rw-r--r--src/client/views/nodes/ScreenshotBox.tsx2
11 files changed, 398 insertions, 378 deletions
diff --git a/package-lock.json b/package-lock.json
index b143dc5c1..9baf9b83a 100644
--- a/package-lock.json
+++ b/package-lock.json
@@ -126,6 +126,7 @@
"https": "^1.0.0",
"https-browserify": "^1.0.0",
"i": "^0.3.7",
+ "iink-ts": "^1.0.5",
"image-data-uri": "^2.0.1",
"image-size": "^1.0.2",
"image-size-stream": "^1.1.0",
@@ -21626,6 +21627,14 @@
"resolved": "https://registry.npmjs.org/ignore-by-default/-/ignore-by-default-1.0.1.tgz",
"integrity": "sha512-Ius2VYcGNk7T90CppJqcIkS5ooHUZyIQK+ClZfMfMNFEF9VSE73Fq+906u/CWu92x4gzZMWOwfFYckPObzdEbA=="
},
+ "node_modules/iink-ts": {
+ "version": "1.0.5",
+ "resolved": "https://registry.npmjs.org/iink-ts/-/iink-ts-1.0.5.tgz",
+ "integrity": "sha512-LAWWPvgcsLtouI9ExVijZbPGIWMfJtVCcuznbIDyboaPb+cHYsftcuJdjDU7TQwIpGP4hmcjNQA57XPCw4MK2A==",
+ "dependencies": {
+ "json-css": "^1.5.6"
+ }
+ },
"node_modules/image-data-uri": {
"version": "2.0.1",
"resolved": "https://registry.npmjs.org/image-data-uri/-/image-data-uri-2.0.1.tgz",
@@ -23231,6 +23240,11 @@
"resolved": "https://registry.npmjs.org/json-buffer/-/json-buffer-3.0.1.tgz",
"integrity": "sha512-4bV5BfR2mqfQTJm+V5tPPdf+ZpuhiIvTuAB5g8kcrXOZpTT/QwwVRWBywX1ozr6lEuPdbHxwaJlm9G6mI2sfSQ=="
},
+ "node_modules/json-css": {
+ "version": "1.5.6",
+ "resolved": "https://registry.npmjs.org/json-css/-/json-css-1.5.6.tgz",
+ "integrity": "sha512-B/0T0OxZH9tSb93tXV6VOYtXqrPz/Vgz2QrCT/4NXen8HGElYkYr9V+8IrSVTMj/ftxa8cG1kcu7f3iAMlaFlQ=="
+ },
"node_modules/json-parse-even-better-errors": {
"version": "2.3.1",
"resolved": "https://registry.npmjs.org/json-parse-even-better-errors/-/json-parse-even-better-errors-2.3.1.tgz",
diff --git a/package.json b/package.json
index 52f627b63..45c2458fb 100644
--- a/package.json
+++ b/package.json
@@ -211,6 +211,7 @@
"https": "^1.0.0",
"https-browserify": "^1.0.0",
"i": "^0.3.7",
+ "iink-ts": "^1.0.5",
"image-data-uri": "^2.0.1",
"image-size": "^1.0.2",
"image-size-stream": "^1.1.0",
diff --git a/src/client/cognitive_services/CognitiveServices.ts b/src/client/cognitive_services/CognitiveServices.ts
index 9808b6a01..9f46b8685 100644
--- a/src/client/cognitive_services/CognitiveServices.ts
+++ b/src/client/cognitive_services/CognitiveServices.ts
@@ -46,14 +46,15 @@ export enum Confidence {
export namespace CognitiveServices {
const ExecuteQuery = async <D>(service: Service, manager: APIManager<D>, data: D): Promise<any> => {
const apiKey = process.env[service.toUpperCase()];
- if (!apiKey) {
+ if (apiKey) {
+ console.log(data)
console.log(`No API key found for ${service}: ensure youe root directory has .env file with _CLIENT_${service.toUpperCase()}.`);
return undefined;
}
let results: any;
try {
- results = await manager.requester(apiKey, manager.converter(data), service).then(json => JSON.parse(json));
+ results = await manager.requester("has", manager.converter(data), service).then(json => JSON.parse(json));
} catch (e) {
throw e;
}
@@ -137,6 +138,12 @@ export namespace CognitiveServices {
points: points.map(({ X: x, Y: y }) => `${x},${y}`).join(','),
language: 'en-US',
}));
+ console.log(JSON.stringify({
+ version: 1,
+ language: 'en-US',
+ unit: 'mm',
+ strokes,
+ }))
return JSON.stringify({
version: 1,
language: 'en-US',
diff --git a/src/client/documents/Documents.ts b/src/client/documents/Documents.ts
index a67e6b4f6..c2e201f52 100644
--- a/src/client/documents/Documents.ts
+++ b/src/client/documents/Documents.ts
@@ -767,7 +767,7 @@ export namespace Docs {
export function ComparisonDocument(text: string, options: DocumentOptions = { title: 'Comparison Box' }) {
return InstanceFromProto(Prototypes.get(DocumentType.COMPARISON), text, options);
}
- export function DiagramDocument(options: DocumentOptions = { title: 'bruh box' }) {
+ export function DiagramDocument(options: DocumentOptions = { title: '' }) {
return InstanceFromProto(Prototypes.get(DocumentType.DIAGRAM), undefined, options);
}
diff --git a/src/client/views/DocumentDecorations.tsx b/src/client/views/DocumentDecorations.tsx
index 93c3e3338..96ce181f8 100644
--- a/src/client/views/DocumentDecorations.tsx
+++ b/src/client/views/DocumentDecorations.tsx
@@ -152,18 +152,25 @@ export class DocumentDecorations extends ObservableReactComponent<DocumentDecora
onContainerDown = (e: React.PointerEvent) => {
const effectiveLayoutAcl = GetEffectiveAcl(DocumentView.Selected()[0].Document);
if (effectiveLayoutAcl === AclAdmin || effectiveLayoutAcl === AclEdit || effectiveLayoutAcl === AclAugment) {
- setupMoveUpEvents(this, e, moveEv => this.onBackgroundMove(true, moveEv), emptyFunction, emptyFunction);
+ setupMoveUpEvents(this, e, moveEv => {this.onBackgroundMove(true, moveEv)
+ console.log("im being moved ink")
+ return false;
+ }, emptyFunction, emptyFunction);
e.stopPropagation();
}
};
onTitleDown = (e: React.PointerEvent) => {
+ console.log("im clicked")
const effectiveLayoutAcl = GetEffectiveAcl(DocumentView.SelectedDocs()[0]);
if (effectiveLayoutAcl === AclAdmin || effectiveLayoutAcl === AclEdit || effectiveLayoutAcl === AclAugment) {
setupMoveUpEvents(
this,
e,
- moveEv => this.onBackgroundMove(true, moveEv),
+ moveEv => {this.onBackgroundMove(true, moveEv)
+ console.log("im being dragged")
+ return false;
+ },
emptyFunction,
action(() => {
const selected = DocumentView.SelectedDocs().length === 1 ? DocumentView.SelectedDocs()[0] : undefined;
@@ -182,7 +189,7 @@ export class DocumentDecorations extends ObservableReactComponent<DocumentDecora
};
@action
onBackgroundMove = (dragTitle: boolean, e: PointerEvent): boolean => {
- const dragDocView = DocumentView.Selected()[0];
+ const dragDocView = DocumentView.Selected()[0];``
const effectiveLayoutAcl = GetEffectiveAcl(dragDocView.Document);
if (effectiveLayoutAcl !== AclAdmin && effectiveLayoutAcl !== AclEdit && effectiveLayoutAcl !== AclAugment) {
return false;
diff --git a/src/client/views/InkTranscription.tsx b/src/client/views/InkTranscription.tsx
index 1ed8de1be..495bb6b83 100644
--- a/src/client/views/InkTranscription.tsx
+++ b/src/client/views/InkTranscription.tsx
@@ -1,349 +1,338 @@
-// import * as iink from 'iink-js';
-// import { action, observable } from 'mobx';
-// import * as React from 'react';
-// import { Doc, DocListCast } from '../../fields/Doc';
-// import { InkData, InkField, InkTool } from '../../fields/InkField';
-// import { Cast, DateCast, NumCast } from '../../fields/Types';
-// import { aggregateBounds } from '../../Utils';
-// import { DocumentType } from '../documents/DocumentTypes';
-// import { CollectionFreeFormView } from './collections/collectionFreeForm';
-// import { InkingStroke } from './InkingStroke';
-// import './InkTranscription.scss';
-
-// /**
-// * Class component that handles inking in writing mode
-// */
-// export class InkTranscription extends React.Component {
-// static Instance: InkTranscription;
-
-// @observable _mathRegister: any= undefined;
-// @observable _mathRef: any= undefined;
-// @observable _textRegister: any= undefined;
-// @observable _textRef: any= undefined;
-// private lastJiix: any;
-// private currGroup?: Doc;
-
-// constructor(props: Readonly<{}>) {
-// super(props);
-
-// InkTranscription.Instance = this;
-// }
-
-// componentWillUnmount() {
-// this._mathRef.removeEventListener('exported', (e: any) => this.exportInk(e, this._mathRef));
-// this._textRef.removeEventListener('exported', (e: any) => this.exportInk(e, this._textRef));
-// }
-
-// @action
-// setMathRef = (r: any) => {
-// if (!this._mathRegister) {
-// this._mathRegister = r
-// ? iink.register(r, {
-// recognitionParams: {
-// type: 'MATH',
-// protocol: 'WEBSOCKET',
-// server: {
-// host: 'cloud.myscript.com',
-// applicationKey: process.env.IINKJS_APP,
-// hmacKey: process.env.IINKJS_HMAC,
-// websocket: {
-// pingEnabled: false,
-// autoReconnect: true,
-// },
-// },
-// iink: {
-// math: {
-// mimeTypes: ['application/x-latex', 'application/vnd.myscript.jiix'],
-// },
-// export: {
-// jiix: {
-// strokes: true,
-// },
-// },
-// },
-// },
-// })
-// : null;
-// }
-
-// r?.addEventListener('exported', (e: any) => this.exportInk(e, this._mathRef));
-
-// return (this._mathRef = r);
-// };
-
-// @action
-// setTextRef = (r: any) => {
-// if (!this._textRegister) {
-// this._textRegister = r
-// ? iink.register(r, {
-// recognitionParams: {
-// type: 'TEXT',
-// protocol: 'WEBSOCKET',
-// server: {
-// host: 'cloud.myscript.com',
-// applicationKey: '7277ec34-0c2e-4ee1-9757-ccb657e3f89f',
-// hmacKey: 'f5cb18f2-1f95-4ddb-96ac-3f7c888dffc1',
-// websocket: {
-// pingEnabled: false,
-// autoReconnect: true,
-// },
-// },
-// iink: {
-// text: {
-// mimeTypes: ['text/plain'],
-// },
-// export: {
-// jiix: {
-// strokes: true,
-// },
-// },
-// },
-// },
-// })
-// : null;
-// }
-
-// r?.addEventListener('exported', (e: any) => this.exportInk(e, this._textRef));
-
-// return (this._textRef = r);
-// };
-
-// /**
-// * Handles processing Dash Doc data for ink transcription.
-// *
-// * @param groupDoc the group which contains the ink strokes we want to transcribe
-// * @param inkDocs the ink docs contained within the selected group
-// * @param math boolean whether to do math transcription or not
-// */
-// transcribeInk = (groupDoc: Doc | undefined, inkDocs: Doc[], math: boolean) => {
-// if (!groupDoc) return;
-// const validInks = inkDocs.filter(s => s.type === DocumentType.INK);
-
-// const strokes: InkData[] = [];
-// const times: number[] = [];
-// validInks
-// .filter(i => Cast(i[Doc.LayoutFieldKey(i)], InkField))
-// .forEach(i => {
-// const d = Cast(i[Doc.LayoutFieldKey(i)], InkField, null);
-// const inkStroke = DocumentManager.Instance.getDocumentView(i)?.ComponentView as InkingStroke;
-// strokes.push(d.inkData.map(pd => inkStroke.ptToScreen({ X: pd.X, Y: pd.Y })));
-// times.push(DateCast(i.author_date).getDate().getTime());
-// });
-
-// this.currGroup = groupDoc;
-
-// const pointerData = { events: strokes.map((stroke, i) => this.inkJSON(stroke, times[i])) };
-// const processGestures = false;
-
-// if (math) {
-// this._mathRef.editor.pointerEvents(pointerData, processGestures);
-// } else {
-// this._textRef.editor.pointerEvents(pointerData, processGestures);
-// }
-// };
-
-// /**
-// * Converts the Dash Ink Data to JSON.
-// *
-// * @param stroke The dash ink data
-// * @param time the time of the stroke
-// * @returns json object representation of ink data
-// */
-// inkJSON = (stroke: InkData, time: number) => {
-// return {
-// pointerType: 'PEN',
-// pointerId: 1,
-// x: stroke.map(point => point.X),
-// y: stroke.map(point => point.Y),
-// t: new Array(stroke.length).fill(time),
-// p: new Array(stroke.length).fill(1.0),
-// };
-// };
-
-// /**
-// * Creates subgroups for each word for the whole text transcription
-// * @param wordInkDocMap the mapping of words to ink strokes (Ink Docs)
-// */
-// subgroupsTranscriptions = (wordInkDocMap: Map<string, Doc[]>) => {
-// // iterate through the keys of wordInkDocMap
-// wordInkDocMap.forEach(async (inkDocs: Doc[], word: string) => {
-// const selected = inkDocs.slice();
-// if (!selected) {
-// return;
-// }
-// const ctx = await Cast(selected[0].embedContainer, Doc);
-// if (!ctx) {
-// return;
-// }
-// const docView: CollectionFreeFormView = DocumentManager.Instance.getDocumentView(ctx)?.ComponentView as CollectionFreeFormView;
-
-// if (!docView) return;
-// const marqViewRef = docView._marqueeViewRef.current;
-// if (!marqViewRef) return;
-// this.groupInkDocs(selected, docView, word);
-// });
-// };
-
-// /**
-// * Event listener function for when the 'exported' event is heard.
-// *
-// * @param e the event objects
-// * @param ref the ref to the editor
-// */
-// exportInk = (e: any, ref: any) => {
-// const exports = e.detail.exports;
-// if (exports) {
-// if (exports['application/x-latex']) {
-// const latex = exports['application/x-latex'];
-// if (this.currGroup) {
-// this.currGroup.text = latex;
-// this.currGroup.title = latex;
-// }
-
-// ref.editor.clear();
-// } else if (exports['text/plain']) {
-// if (exports['application/vnd.myscript.jiix']) {
-// this.lastJiix = JSON.parse(exports['application/vnd.myscript.jiix']);
-// // map timestamp to strokes
-// const timestampWord = new Map<number, string>();
-// this.lastJiix.words.map((word: any) => {
-// if (word.items) {
-// word.items.forEach((i: { id: string; timestamp: string; X: Array<number>; Y: Array<number>; F: Array<number> }) => {
-// const ms = Date.parse(i.timestamp);
-// timestampWord.set(ms, word.label);
-// });
-// }
-// });
-
-// const wordInkDocMap = new Map<string, Doc[]>();
-// if (this.currGroup) {
-// const docList = DocListCast(this.currGroup.data);
-// docList.forEach((inkDoc: Doc) => {
-// // just having the times match up and be a unique value (actual timestamp doesn't matter)
-// const ms = DateCast(inkDoc.author_date).getDate().getTime() + 14400000;
-// const word = timestampWord.get(ms);
-// if (!word) {
-// return;
-// }
-// const entry = wordInkDocMap.get(word);
-// if (entry) {
-// entry.push(inkDoc);
-// wordInkDocMap.set(word, entry);
-// } else {
-// const newEntry = [inkDoc];
-// wordInkDocMap.set(word, newEntry);
-// }
-// });
-// if (this.lastJiix.words.length > 1) this.subgroupsTranscriptions(wordInkDocMap);
-// }
-// }
-// const text = exports['text/plain'];
-
-// if (this.currGroup) {
-// this.currGroup.transcription = text;
-// this.currGroup.title = text.split('\n')[0];
-// }
-
-// ref.editor.clear();
-// }
-// }
-// };
-
-// /**
-// * Creates the ink grouping once the user leaves the writing mode.
-// */
-// createInkGroup() {
-// // TODO nda - if document being added to is a inkGrouping then we can just add to that group
-// if (Doc.ActiveTool === InkTool.Write) {
-// CollectionFreeFormView.collectionsWithUnprocessedInk.forEach(ffView => {
-// // TODO: nda - will probably want to go through ffView unprocessed docs and then see if any of the inksToGroup docs are in it and only use those
-// const selected = ffView.unprocessedDocs;
-// const newCollection = this.groupInkDocs(
-// selected.filter(doc => doc.embedContainer),
-// ffView
-// );
-// ffView.unprocessedDocs = [];
-
-// InkTranscription.Instance.transcribeInk(newCollection, selected, false);
-// });
-// }
-// CollectionFreeFormView.collectionsWithUnprocessedInk.clear();
-// }
-
-// /**
-// * Creates the groupings for a given list of ink docs on a specific doc view
-// * @param selected: the list of ink docs to create a grouping of
-// * @param docView: the view in which we want the grouping to be created
-// * @param word: optional param if the group we are creating is a word (subgrouping individual words)
-// * @returns a new collection Doc or undefined if the grouping fails
-// */
-// groupInkDocs(selected: Doc[], docView: CollectionFreeFormView, word?: string): Doc | undefined {
-// const bounds: { x: number; y: number; width?: number; height?: number }[] = [];
-
-// // calculate the necessary bounds from the selected ink docs
-// selected.map(
-// action(d => {
-// const x = NumCast(d.x);
-// const y = NumCast(d.y);
-// const width = NumCast(d._width);
-// const height = NumCast(d._height);
-// bounds.push({ x, y, width, height });
-// })
-// );
-
-// // calculate the aggregated bounds
-// const aggregBounds = aggregateBounds(bounds, 0, 0);
-// const marqViewRef = docView._marqueeViewRef.current;
-
-// // set the vals for bounds in marqueeView
-// if (marqViewRef) {
-// marqViewRef._downX = aggregBounds.x;
-// marqViewRef._downY = aggregBounds.y;
-// marqViewRef._lastX = aggregBounds.r;
-// marqViewRef._lastY = aggregBounds.b;
-// }
-
-// // map through all the selected ink strokes and create the groupings
-// selected.map(
-// action(d => {
-// const dx = NumCast(d.x);
-// const dy = NumCast(d.y);
-// delete d.x;
-// delete d.y;
-// delete d.activeFrame;
-// delete d._timecodeToShow; // bcz: this should be automatic somehow.. along with any other properties that were logically associated with the original collection
-// delete d._timecodeToHide; // bcz: this should be automatic somehow.. along with any other properties that were logically associated with the original collection
-// // calculate pos based on bounds
-// if (marqViewRef?.Bounds) {
-// d.x = dx - marqViewRef.Bounds.left - marqViewRef.Bounds.width / 2;
-// d.y = dy - marqViewRef.Bounds.top - marqViewRef.Bounds.height / 2;
-// }
-// return d;
-// })
-// );
-// docView.props.removeDocument?.(selected);
-// // Gets a collection based on the selected nodes using a marquee view ref
-// const newCollection = marqViewRef?.getCollection(selected, undefined, true);
-// if (newCollection) {
-// newCollection.width = NumCast(newCollection._width);
-// newCollection.height = NumCast(newCollection._height);
-// // if the grouping we are creating is an individual word
-// if (word) {
-// newCollection.title = word;
-// }
-// }
-
-// // nda - bug: when deleting a stroke before leaving writing mode, delete the stroke from unprocessed ink docs
-// newCollection && docView.props.addDocument?.(newCollection);
-// return newCollection;
-// }
-
-// render() {
-// return (
-// <div className="ink-transcription">
-// <div className="math-editor" ref={this.setMathRef} touch-action="none"></div>
-// <div className="text-editor" ref={this.setTextRef} touch-action="none"></div>
-// </div>
-// );
-// }
-// }
+import * as iink from 'iink-ts';
+import {TProtocol,TSchene,TServerConfiguration,TConfiguration,TRecognitionConfiguration} from 'iink-ts';
+import { action, observable } from 'mobx';
+import * as React from 'react';
+import { Doc, DocListCast } from '../../fields/Doc';
+import { InkData, InkField, InkTool } from '../../fields/InkField';
+import { Cast, DateCast, NumCast } from '../../fields/Types';
+import { aggregateBounds } from '../../Utils';
+import { DocumentType } from '../documents/DocumentTypes';
+import { CollectionFreeFormView } from './collections/collectionFreeForm';
+import { InkingStroke } from './InkingStroke';
+import './InkTranscription.scss';
+import { DocumentView } from './nodes/DocumentView';
+
+/**
+ * Class component that handles inking in writing mode
+ */
+export class InkTranscription extends React.Component {
+ static Instance: InkTranscription;
+
+ @observable _mathRegister: any= undefined;
+ @observable _mathRef: any= undefined;
+ @observable _textRegister: any= undefined;
+ @observable _textRef: any= undefined;
+ private lastJiix: any;
+ private currGroup?: Doc;
+
+ constructor(props: Readonly<{}>) {
+ super(props);
+
+ InkTranscription.Instance = this;
+ }
+
+ componentWillUnmount() {
+ // this._mathRef.removeEventListener('exported', (e: any) => this.exportInk(e, this._mathRef));
+ // this._textRef.removeEventListener('exported', (e: any) => this.exportInk(e, this._textRef));
+ }
+
+ @action
+ setMathRef = async (r: any) => {
+ if (!this._textRegister && r) {
+ let editor;
+ const options = {
+ configuration: {
+ server: {
+ scheme: "https",
+ host: "cloud.myscript.com",
+ applicationKey: "c0901093-5ac5-4454-8e64-0def0f13f2ca",
+ hmacKey: "f6465cca-1856-4492-a6a4-e2395841be2f",
+ protocol: "WEBSOCKET"
+ },
+ recognition: {
+ type: "TEXT",
+ }
+ }
+ };
+
+ editor = new iink.Editor(r, options as any);
+
+ await editor.initialize();
+
+ this._textRegister = r;
+ r?.addEventListener('exported', (e: any) => this.exportInk(e, this._textRef));
+
+ return (this._textRef = r);
+ }
+ }
+ @action
+ setTextRef = async (r:any) => {
+ if (!this._textRegister && r) {
+ let editor;
+ const options = {
+ configuration: {
+ server: {
+ scheme: "https",
+ host: "cloud.myscript.com",
+ applicationKey: "c0901093-5ac5-4454-8e64-0def0f13f2ca",
+ hmacKey: "f6465cca-1856-4492-a6a4-e2395841be2f",
+ protocol: "WEBSOCKET"
+ },
+ recognition: {
+ type: "TEXT",
+ }
+ }
+ };
+
+ editor = new iink.Editor(r, options as any);
+
+ await editor.initialize();
+
+ this._textRegister = r;
+ r?.addEventListener('exported', (e: any) => this.exportInk(e, this._textRef));
+
+ return (this._textRef = r);
+ }
+
+ };
+
+ /**
+ * Handles processing Dash Doc data for ink transcription.
+ *
+ * @param groupDoc the group which contains the ink strokes we want to transcribe
+ * @param inkDocs the ink docs contained within the selected group
+ * @param math boolean whether to do math transcription or not
+ */
+ transcribeInk = (groupDoc: Doc | undefined, inkDocs: Doc[], math: boolean) => {
+ if (!groupDoc) return;
+ const validInks = inkDocs.filter(s => s.type === DocumentType.INK);
+
+ const strokes: InkData[] = [];
+ const times: number[] = [];
+ validInks
+ .filter(i => Cast(i[Doc.LayoutFieldKey(i)], InkField))
+ .forEach(i => {
+ const d = Cast(i[Doc.LayoutFieldKey(i)], InkField, null);
+ const inkStroke = DocumentView.getDocumentView(i)?.ComponentView as InkingStroke;
+ strokes.push(d.inkData.map(pd => inkStroke.ptToScreen({ X: pd.X, Y: pd.Y })));
+ times.push(DateCast(i.author_date).getDate().getTime());
+ });
+
+ this.currGroup = groupDoc;
+
+ const pointerData = { events: strokes.map((stroke, i) => this.inkJSON(stroke, times[i])) };
+ const processGestures = false;
+
+ if (math) {
+ this._mathRef.editor.pointerEvents(pointerData, processGestures);
+ } else {
+ this._textRef.editor.pointerEvents(pointerData, processGestures);
+ }
+ };
+
+ /**
+ * Converts the Dash Ink Data to JSON.
+ *
+ * @param stroke The dash ink data
+ * @param time the time of the stroke
+ * @returns json object representation of ink data
+ */
+ inkJSON = (stroke: InkData, time: number) => {
+ return {
+ pointerType: 'PEN',
+ pointerId: 1,
+ x: stroke.map(point => point.X),
+ y: stroke.map(point => point.Y),
+ t: new Array(stroke.length).fill(time),
+ p: new Array(stroke.length).fill(1.0),
+ };
+ };
+
+ /**
+ * Creates subgroups for each word for the whole text transcription
+ * @param wordInkDocMap the mapping of words to ink strokes (Ink Docs)
+ */
+ subgroupsTranscriptions = (wordInkDocMap: Map<string, Doc[]>) => {
+ // iterate through the keys of wordInkDocMap
+ wordInkDocMap.forEach(async (inkDocs: Doc[], word: string) => {
+ const selected = inkDocs.slice();
+ if (!selected) {
+ return;
+ }
+ const ctx = await Cast(selected[0].embedContainer, Doc);
+ if (!ctx) {
+ return;
+ }
+ const docView: CollectionFreeFormView =DocumentView.getDocumentView(ctx)?.ComponentView as CollectionFreeFormView;
+ // DocumentManager.Instance.getDocumentView(ctx)?.ComponentView as CollectionFreeFormView;
+
+ if (!docView) return;
+ const marqViewRef = docView._marqueeViewRef.current;
+ if (!marqViewRef) return;
+ this.groupInkDocs(selected, docView, word);
+ });
+ };
+
+ /**
+ * Event listener function for when the 'exported' event is heard.
+ *
+ * @param e the event objects
+ * @param ref the ref to the editor
+ */
+ exportInk = (e: any, ref: any) => {
+ const exports = e.detail.exports;
+ if (exports) {
+ if (exports['application/x-latex']) {
+ const latex = exports['application/x-latex'];
+ if (this.currGroup) {
+ this.currGroup.text = latex;
+ this.currGroup.title = latex;
+ }
+
+ ref.editor.clear();
+ } else if (exports['text/plain']) {
+ if (exports['application/vnd.myscript.jiix']) {
+ this.lastJiix = JSON.parse(exports['application/vnd.myscript.jiix']);
+ // map timestamp to strokes
+ const timestampWord = new Map<number, string>();
+ this.lastJiix.words.map((word: any) => {
+ if (word.items) {
+ word.items.forEach((i: { id: string; timestamp: string; X: Array<number>; Y: Array<number>; F: Array<number> }) => {
+ const ms = Date.parse(i.timestamp);
+ timestampWord.set(ms, word.label);
+ });
+ }
+ });
+
+ const wordInkDocMap = new Map<string, Doc[]>();
+ if (this.currGroup) {
+ const docList = DocListCast(this.currGroup.data);
+ docList.forEach((inkDoc: Doc) => {
+ // just having the times match up and be a unique value (actual timestamp doesn't matter)
+ const ms = DateCast(inkDoc.author_date).getDate().getTime() + 14400000;
+ const word = timestampWord.get(ms);
+ if (!word) {
+ return;
+ }
+ const entry = wordInkDocMap.get(word);
+ if (entry) {
+ entry.push(inkDoc);
+ wordInkDocMap.set(word, entry);
+ } else {
+ const newEntry = [inkDoc];
+ wordInkDocMap.set(word, newEntry);
+ }
+ });
+ if (this.lastJiix.words.length > 1) this.subgroupsTranscriptions(wordInkDocMap);
+ }
+ }
+ const text = exports['text/plain'];
+
+ if (this.currGroup) {
+ this.currGroup.transcription = text;
+ this.currGroup.title = text.split('\n')[0];
+ }
+
+ ref.editor.clear();
+ }
+ }
+ };
+
+ /**
+ * Creates the ink grouping once the user leaves the writing mode.
+ */
+ createInkGroup() {
+ // TODO nda - if document being added to is a inkGrouping then we can just add to that group
+ if (Doc.ActiveTool === InkTool.Write) {
+ CollectionFreeFormView.collectionsWithUnprocessedInk.forEach(ffView => {
+ // TODO: nda - will probably want to go through ffView unprocessed docs and then see if any of the inksToGroup docs are in it and only use those
+ const selected = ffView.unprocessedDocs;
+ const newCollection = this.groupInkDocs(
+ selected.filter(doc => doc.embedContainer),
+ ffView
+ );
+ ffView.unprocessedDocs = [];
+
+ InkTranscription.Instance.transcribeInk(newCollection, selected, false);
+ });
+ }
+ CollectionFreeFormView.collectionsWithUnprocessedInk.clear();
+ }
+
+ /**
+ * Creates the groupings for a given list of ink docs on a specific doc view
+ * @param selected: the list of ink docs to create a grouping of
+ * @param docView: the view in which we want the grouping to be created
+ * @param word: optional param if the group we are creating is a word (subgrouping individual words)
+ * @returns a new collection Doc or undefined if the grouping fails
+ */
+ groupInkDocs(selected: Doc[], docView: CollectionFreeFormView, word?: string): Doc | undefined {
+ const bounds: { x: number; y: number; width?: number; height?: number }[] = [];
+
+ // calculate the necessary bounds from the selected ink docs
+ selected.map(
+ action(d => {
+ const x = NumCast(d.x);
+ const y = NumCast(d.y);
+ const width = NumCast(d._width);
+ const height = NumCast(d._height);
+ bounds.push({ x, y, width, height });
+ })
+ );
+
+ // calculate the aggregated bounds
+ const aggregBounds = aggregateBounds(bounds, 0, 0);
+ const marqViewRef = docView._marqueeViewRef.current;
+
+ // set the vals for bounds in marqueeView
+ if (marqViewRef) {
+ marqViewRef._downX = aggregBounds.x;
+ marqViewRef._downY = aggregBounds.y;
+ marqViewRef._lastX = aggregBounds.r;
+ marqViewRef._lastY = aggregBounds.b;
+ }
+
+ // map through all the selected ink strokes and create the groupings
+ selected.map(
+ action(d => {
+ const dx = NumCast(d.x);
+ const dy = NumCast(d.y);
+ delete d.x;
+ delete d.y;
+ delete d.activeFrame;
+ delete d._timecodeToShow; // bcz: this should be automatic somehow.. along with any other properties that were logically associated with the original collection
+ delete d._timecodeToHide; // bcz: this should be automatic somehow.. along with any other properties that were logically associated with the original collection
+ // calculate pos based on bounds
+ if (marqViewRef?.Bounds) {
+ d.x = dx - marqViewRef.Bounds.left - marqViewRef.Bounds.width / 2;
+ d.y = dy - marqViewRef.Bounds.top - marqViewRef.Bounds.height / 2;
+ }
+ return d;
+ })
+ );
+ docView.props.removeDocument?.(selected);
+ // Gets a collection based on the selected nodes using a marquee view ref
+ const newCollection = marqViewRef?.getCollection(selected, undefined, true);
+ if (newCollection) {
+ newCollection.width = NumCast(newCollection._width);
+ newCollection.height = NumCast(newCollection._height);
+ // if the grouping we are creating is an individual word
+ if (word) {
+ newCollection.title = word;
+ }
+ }
+
+ // nda - bug: when deleting a stroke before leaving writing mode, delete the stroke from unprocessed ink docs
+ newCollection && docView.props.addDocument?.(newCollection);
+ return newCollection;
+ }
+
+ render() {
+ return (
+ <div className="ink-transcription">
+ <div className="math-editor" ref={this.setMathRef} touch-action="none"></div>
+ <div className="text-editor" ref={this.setTextRef} touch-action="none"></div>
+ </div>
+ );
+ }
+}
diff --git a/src/client/views/InkingStroke.tsx b/src/client/views/InkingStroke.tsx
index 55f28f415..ce1c07f2f 100644
--- a/src/client/views/InkingStroke.tsx
+++ b/src/client/views/InkingStroke.tsx
@@ -104,8 +104,8 @@ export class InkingStroke extends ViewBoxAnnotatableComponent<FieldViewProps>()
* analyzes the ink stroke and saves the analysis of the stroke to the 'inkAnalysis' field,
* and the recognized words to the 'handwriting'
*/
- analyzeStrokes() {
- const data: InkData = Cast(this.dataDoc[this.fieldKey], InkField)?.inkData ?? [];
+ analyzeStrokes=()=> {
+ const data: InkData = this.inkScaledData().inkData ?? [];
CognitiveServices.Inking.Appliers.ConcatenateHandwriting(this.dataDoc, ['inkAnalysis', 'handwriting'], [data]);
}
diff --git a/src/client/views/MainView.tsx b/src/client/views/MainView.tsx
index 31d88fb87..0dcf2fa79 100644
--- a/src/client/views/MainView.tsx
+++ b/src/client/views/MainView.tsx
@@ -76,6 +76,7 @@ import { PresBox } from './nodes/trails';
import { AnchorMenu } from './pdf/AnchorMenu';
import { GPTPopup } from './pdf/GPTPopup/GPTPopup';
import { TopBar } from './topbar/TopBar';
+import { InkTranscription } from './InkTranscription';
const { LEFT_MENU_WIDTH, TOPBAR_HEIGHT } = require('./global/globalCssVariables.module.scss'); // prettier-ignore
const _global = (window /* browser */ || global) /* node */ as any;
@@ -1096,6 +1097,7 @@ export class MainView extends ObservableReactComponent<{}> {
<MarqueeOptionsMenu />
<TimelineMenu />
<RichTextMenu />
+ <InkTranscription/>
{this.snapLines}
<LightboxView key="lightbox" PanelWidth={this._windowWidth} addSplit={CollectionDockingView.AddSplit} PanelHeight={this._windowHeight} maxBorder={this.lightboxMaxBorder} />
<GPTPopup key="gptpopup" />
diff --git a/src/client/views/global/globalScripts.ts b/src/client/views/global/globalScripts.ts
index 7730ed385..83c0865f6 100644
--- a/src/client/views/global/globalScripts.ts
+++ b/src/client/views/global/globalScripts.ts
@@ -38,8 +38,7 @@ import { WebBox } from '../nodes/WebBox';
import { RichTextMenu } from '../nodes/formattedText/RichTextMenu';
import { NumListCast } from '../../../fields/Doc';
import { List } from '../../../fields/List';
-
-// import { InkTranscription } from '../InkTranscription';
+import { InkTranscription } from '../InkTranscription';
// eslint-disable-next-line prefer-arrow-callback
ScriptingGlobals.add(function IsNoneSelected() {
@@ -367,6 +366,7 @@ ScriptingGlobals.add(function toggleCharStyle(charStyle: attrname, checkResult?:
export function createInkGroup(/* inksToGroup?: Doc[], isSubGroup?: boolean */) {
// TODO nda - if document being added to is a inkGrouping then we can just add to that group
if (Doc.ActiveTool === InkTool.Write) {
+ console.log("create inking group ")
CollectionFreeFormView.collectionsWithUnprocessedInk.forEach(ffView => {
// TODO: nda - will probably want to go through ffView unprocessed docs and then see if any of the inksToGroup docs are in it and only use those
const selected = ffView.unprocessedDocs;
@@ -424,14 +424,14 @@ export function createInkGroup(/* inksToGroup?: Doc[], isSubGroup?: boolean */)
// TODO: nda - will probably need to go through and only remove the unprocessed selected docs
ffView.unprocessedDocs = [];
- // InkTranscription.Instance.transcribeInk(newCollection, selected, false);
+ InkTranscription.Instance.transcribeInk(newCollection, selected, false);
});
}
CollectionFreeFormView.collectionsWithUnprocessedInk.clear();
}
function setActiveTool(tool: InkTool | Gestures, keepPrim: boolean, checkResult?: boolean) {
- // InkTranscription.Instance?.createInkGroup();
+ //InkTranscription.Instance?.createInkGroup();
if (checkResult) {
return (Doc.ActiveTool === tool && !GestureOverlay.Instance?.InkShape) || GestureOverlay.Instance?.InkShape === tool
? GestureOverlay.Instance?.KeepPrimitiveMode || ![Gestures.Circle, Gestures.Line, Gestures.Rectangle].includes(tool as Gestures)
diff --git a/src/client/views/nodes/DiagramBox.tsx b/src/client/views/nodes/DiagramBox.tsx
index 6ea574aab..fcc028aab 100644
--- a/src/client/views/nodes/DiagramBox.tsx
+++ b/src/client/views/nodes/DiagramBox.tsx
@@ -71,19 +71,19 @@ export class DiagramBox extends ViewBoxAnnotatableComponent<FieldViewProps>() {
this.renderMermaidAsync(this.Document.drawingMermaidCode);
}
// this is so that ever time a new doc, text node or ink node, is created, this.createMermaidCode will run which will create a save
- reaction(
- () => DocListCast(this.Document.data),
- () => this.lockInkStroke(),
- { fireImmediately: true }
- );
- reaction(
- () =>
- DocListCast(this.Document.data)
- .filter(doc => doc.type === 'rich text')
- .map(doc => (doc.text as RichTextField).Text),
- () => this.convertDrawingToMermaidCode(),
- { fireImmediately: true }
- );
+ // reaction(
+ // () => DocListCast(this.Document.data),
+ // () => this.lockInkStroke(),
+ // { fireImmediately: true }
+ // );
+ // reaction(
+ // () =>
+ // DocListCast(this.Document.data)
+ // .filter(doc => doc.type === 'rich text')
+ // .map(doc => (doc.text as RichTextField).Text),
+ // () => this.convertDrawingToMermaidCode(),
+ // { fireImmediately: true }
+ // );
// const rectangleXValues = computed(() =>
// DocListCast(this.Document.data)
// .filter(doc => doc.title === 'rectangle')
@@ -94,7 +94,7 @@ export class DiagramBox extends ViewBoxAnnotatableComponent<FieldViewProps>() {
// () => this.lockInkStroke(),
// { fireImmediately: true }
// );
- // this.lockInkStroke();
+ this.lockInkStroke();
}
componentDidUpdate = () => {
@@ -363,8 +363,8 @@ export class DiagramBox extends ViewBoxAnnotatableComponent<FieldViewProps>() {
DocumentManager.Instance.AddViewRenderedCb(this.Document, (docViewForYourCollection) => {
if (docViewForYourCollection && docViewForYourCollection.ComponentView) {
if (docViewForYourCollection.ComponentView.addDocument&&docViewForYourCollection.ComponentView.removeDocument) {
- // docViewForYourCollection.ComponentView?.removeDocument(inkToDelete)
- // docViewForYourCollection.ComponentView?.addDocument(newInkDoc);
+ docViewForYourCollection.ComponentView?.removeDocument(inkToDelete)
+ docViewForYourCollection.ComponentView?.addDocument(newInkDoc);
diff --git a/src/client/views/nodes/ScreenshotBox.tsx b/src/client/views/nodes/ScreenshotBox.tsx
index 3be50f5e6..e6590958b 100644
--- a/src/client/views/nodes/ScreenshotBox.tsx
+++ b/src/client/views/nodes/ScreenshotBox.tsx
@@ -149,7 +149,7 @@ export class ScreenshotBox extends ViewBoxAnnotatableComponent<FieldViewProps>()
componentDidMount() {
this.dataDoc.nativeWidth = this.dataDoc.nativeHeight = 0;
- this._props.setContentViewBox?.(this); // this tells the DocumentView that this ScreenshotBox is the "content" of the document. this allows the DocumentView to indirectly call getAnchor() on the AudioBox when making a link.
+ this._props.setContentViewBox?.(this); // this tells the DocumentView that this Box is the "content" of the document. this allows the DocumentView to indirectly call getAnchor() on the AudioBox when making a link.
// this.layoutDoc.videoWall && reaction(() => ({ width: this._props.PanelWidth(), height: this._props.PanelHeight() }),
// ({ width, height }) => {
// if (this._camera) {
generated by cgit v1.2.3-70-g09d2 (git 2.46.0) at 2025-10-06 17:23:12 +0000