From ab0e1ac6f5b21f2e10bdce428c882482a0f159b4 Mon Sep 17 00:00:00 2001 From: mehekj Date: Tue, 12 Apr 2022 18:09:03 -0400 Subject: ink to text for groupings --- src/client/views/InkTranscription.tsx | 188 ++++++++++++++++++++++++++++++++++ 1 file changed, 188 insertions(+) create mode 100644 src/client/views/InkTranscription.tsx (limited to 'src/client/views/InkTranscription.tsx') diff --git a/src/client/views/InkTranscription.tsx b/src/client/views/InkTranscription.tsx new file mode 100644 index 000000000..950f622dd --- /dev/null +++ b/src/client/views/InkTranscription.tsx @@ -0,0 +1,188 @@ +import { timeStamp } from 'console'; +import * as iink from 'iink-js'; +import { action, observable } from 'mobx'; +import * as React from 'react'; +import { Doc } from '../../fields/Doc'; +import { InkData, InkField } from "../../fields/InkField"; +import { Cast, NumCast, StrCast } from '../../fields/Types'; +import { Docs } from "../documents/Documents"; +import './InkTranscription.scss'; + +export class InkTranscription extends React.Component { + static Instance: InkTranscription; + + @observable _mathRegister: any; + @observable _mathRef: any; + @observable _textRegister: any; + @observable _textRef: any; + private addDocument?: (doc: Doc | Doc[]) => boolean; + private bounds: { x: number, y: number, width: number, height: number } = { x: 0, y: 0, width: 0, height: 0 }; + + constructor(props: Readonly<{}>) { + super(props); + + InkTranscription.Instance = this; + } + + componentWillUnmount() { + this._mathRef.removeEventListener('exported', (e: any) => this.exportInk(e, this._mathRef)); + this._textRef.removeEventListener('exported', (e: any) => this.exportInk(e, this._textRef)); + + this._mathRef.removeEventListener('changed', (e: any) => this.callExport(this._mathRef)); + this._textRef.removeEventListener('changed', (e: any) => this.callExport(this._textRef)); + } + + @action + setMathRef = (r: any) => { + if (!this._mathRegister) { + this._mathRegister = r ? iink.register(r, { + recognitionParams: { + type: 'MATH', + protocol: 'WEBSOCKET', + server: { + host: 'cloud.myscript.com', + applicationKey: '7277ec34-0c2e-4ee1-9757-ccb657e3f89f', + hmacKey: 'f5cb18f2-1f95-4ddb-96ac-3f7c888dffc1', + websocket: { + pingEnabled: false, + autoReconnect: true + } + }, + iink: { + math: { + mimeTypes: ['application/x-latex', 'application/vnd.myscript.jiix'] + }, + export: { + jiix: { + strokes: true + } + } + } + }, + triggers: { + exportContent: 'DEMAND' + } + }) : null; + } + + r.addEventListener('exported', (e: any) => this.exportInk(e, this._mathRef)); + r.addEventListener('changed', (e: any) => this.callExport(this._mathRef)); + + return this._mathRef = r; + } + + @action + setTextRef = (r: any) => { + if (!this._textRegister) { + this._textRegister = r ? iink.register(r, { + recognitionParams: { + type: 'TEXT', + protocol: 'WEBSOCKET', + server: { + host: 'cloud.myscript.com', + applicationKey: '7277ec34-0c2e-4ee1-9757-ccb657e3f89f', + hmacKey: 'f5cb18f2-1f95-4ddb-96ac-3f7c888dffc1', + websocket: { + pingEnabled: false, + autoReconnect: true + } + }, + iink: { + text: { + mimeTypes: ['text/plain'] + }, + export: { + jiix: { + strokes: true + } + } + } + }, + triggers: { + exportContent: 'DEMAND' + } + }) : null; + } + + r.addEventListener('exported', (e: any) => this.exportInk(e, this._textRef)); + r.addEventListener('changed', (e: any) => this.callExport(this._textRef)); + + return this._textRef = r; + } + + transcribeInk = (inkdocs: Doc[], math: boolean, bounds: { x: number, y: number, width: number, height: number }, addDocument?: (doc: Doc | Doc[]) => boolean) => { + const strokes: InkData[] = []; + inkdocs.filter(i => Cast(i.data, InkField)).forEach(i => { + // TODO: interpolate missing times stamps + const d = Cast(i.data, InkField, null); + const left = Math.min(...d?.inkData.map(pd => pd.X) ?? [0]); + const top = Math.min(...d?.inkData.map(pd => pd.Y) ?? [0]); + strokes.push(d.inkData.map(pd => ({ X: pd.X + NumCast(i.x) - left, Y: pd.Y + NumCast(i.y) - top, time: pd.time }))); + }); + + this.addDocument = addDocument; + this.bounds = bounds; + + const pointerData = { "events": strokes.map(stroke => this.inkJSON(stroke)) }; + // console.log(JSON.stringify(pointerData)); + const processGestures = false; + + if (math) { + this._mathRef.editor.pointerEvents(pointerData, processGestures); + } + else { + this._textRef.editor.pointerEvents(pointerData, processGestures); + } + } + + callExport(ref: any) { + if (ref.editor.canExport) { + ref.editor.export_(); + } + } + + inkJSON = (stroke: InkData) => { + return { + "pointerType": "PEN", + "pointerId": 1, + "x": stroke.map(point => point.X), + "y": stroke.map(point => point.Y), + "t": stroke.map(point => point.time), + "p": new Array(stroke.length).fill(1.0) + }; + } + + exportInk = (e: any, ref: any) => { + const exports = e.detail.exports; + if (exports) { + if (exports['application/x-latex']) { + const latex = exports['application/x-latex']; + console.log(latex); + + this.addDocument?.(Docs.Create.EquationDocument({ title: latex, x: this.bounds.width, y: 0, _width: this.bounds.width, _height: this.bounds.height })); + } + else if (exports['text/plain']) { + const text = exports['text/plain']; + console.log(text); + this.addDocument?.(Docs.Create.TextDocument(text, { title: text, x: this.bounds.width, y: 0, _width: this.bounds.width, _height: this.bounds.height })); + } + + ref.editor.clear(); + } + } + + render() { + return ( +
+
+
+
+
+
+ ) + } +} \ No newline at end of file -- cgit v1.2.3-70-g09d2 From 9a92361e9c585ed02c7d2561835d64ab736b96e3 Mon Sep 17 00:00:00 2001 From: mehekj Date: Wed, 13 Apr 2022 00:11:55 -0400 Subject: fixed ink transcription document location --- src/client/views/InkTranscription.tsx | 4 ++-- .../views/collections/collectionFreeForm/CollectionFreeFormView.tsx | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) (limited to 'src/client/views/InkTranscription.tsx') diff --git a/src/client/views/InkTranscription.tsx b/src/client/views/InkTranscription.tsx index 950f622dd..4957446dc 100644 --- a/src/client/views/InkTranscription.tsx +++ b/src/client/views/InkTranscription.tsx @@ -159,12 +159,12 @@ export class InkTranscription extends React.Component { const latex = exports['application/x-latex']; console.log(latex); - this.addDocument?.(Docs.Create.EquationDocument({ title: latex, x: this.bounds.width, y: 0, _width: this.bounds.width, _height: this.bounds.height })); + this.addDocument?.(Docs.Create.EquationDocument({ title: latex, x: this.bounds.x + this.bounds.width, y: this.bounds.y, _width: this.bounds.width, _height: this.bounds.height })); } else if (exports['text/plain']) { const text = exports['text/plain']; console.log(text); - this.addDocument?.(Docs.Create.TextDocument(text, { title: text, x: this.bounds.width, y: 0, _width: this.bounds.width, _height: this.bounds.height })); + this.addDocument?.(Docs.Create.TextDocument(text, { title: text, x: this.bounds.x + this.bounds.width, y: this.bounds.y, _width: this.bounds.width, _height: this.bounds.height })); } ref.editor.clear(); diff --git a/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx b/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx index 7d60387de..6bcaf4f83 100644 --- a/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx +++ b/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx @@ -1549,7 +1549,7 @@ export class CollectionFreeFormView extends CollectionSubView { if (this.props.Document._isGroup) { const inkdocs = this.childDocs.filter(s => s.type === DocumentType.INK); - InkTranscription.Instance.transcribeInk(inkdocs, math, { x: NumCast(this.layoutDoc.x) ?? 0, y: NumCast(this.layoutDoc.y) ?? 0, width: NumCast(this.layoutDoc._width) ?? 0, height: NumCast(this.layoutDoc._height) ?? 0 }, this.addDocument); + InkTranscription.Instance.transcribeInk(inkdocs, math, { x: NumCast(this.layoutDoc.x) ?? 0, y: NumCast(this.layoutDoc.y) ?? 0, width: NumCast(this.layoutDoc._width) ?? 0, height: NumCast(this.layoutDoc._height) ?? 0 }, this.props.ContainingCollectionView?.addDocument); } } -- cgit v1.2.3-70-g09d2 From ab5a0bd7cee9366dabf4ce40bde89b67730da2a5 Mon Sep 17 00:00:00 2001 From: mehekj Date: Thu, 14 Apr 2022 18:06:34 -0400 Subject: auto transcribes on grouping --- src/client/views/InkTranscription.tsx | 50 ++++++++++------------ .../collectionFreeForm/CollectionFreeFormView.tsx | 29 +++++++------ src/client/views/nodes/button/FontIconBox.tsx | 33 +++++++------- 3 files changed, 57 insertions(+), 55 deletions(-) (limited to 'src/client/views/InkTranscription.tsx') diff --git a/src/client/views/InkTranscription.tsx b/src/client/views/InkTranscription.tsx index 4957446dc..ea5d5a1ec 100644 --- a/src/client/views/InkTranscription.tsx +++ b/src/client/views/InkTranscription.tsx @@ -5,7 +5,7 @@ import * as React from 'react'; import { Doc } from '../../fields/Doc'; import { InkData, InkField } from "../../fields/InkField"; import { Cast, NumCast, StrCast } from '../../fields/Types'; -import { Docs } from "../documents/Documents"; +import { DocumentType } from "../documents/DocumentTypes"; import './InkTranscription.scss'; export class InkTranscription extends React.Component { @@ -15,8 +15,7 @@ export class InkTranscription extends React.Component { @observable _mathRef: any; @observable _textRegister: any; @observable _textRef: any; - private addDocument?: (doc: Doc | Doc[]) => boolean; - private bounds: { x: number, y: number, width: number, height: number } = { x: 0, y: 0, width: 0, height: 0 }; + private currGroup?: Doc; constructor(props: Readonly<{}>) { super(props); @@ -27,9 +26,6 @@ export class InkTranscription extends React.Component { componentWillUnmount() { this._mathRef.removeEventListener('exported', (e: any) => this.exportInk(e, this._mathRef)); this._textRef.removeEventListener('exported', (e: any) => this.exportInk(e, this._textRef)); - - this._mathRef.removeEventListener('changed', (e: any) => this.callExport(this._mathRef)); - this._textRef.removeEventListener('changed', (e: any) => this.callExport(this._textRef)); } @action @@ -58,15 +54,11 @@ export class InkTranscription extends React.Component { } } } - }, - triggers: { - exportContent: 'DEMAND' } }) : null; } r.addEventListener('exported', (e: any) => this.exportInk(e, this._mathRef)); - r.addEventListener('changed', (e: any) => this.callExport(this._mathRef)); return this._mathRef = r; } @@ -97,22 +89,22 @@ export class InkTranscription extends React.Component { } } } - }, - triggers: { - exportContent: 'DEMAND' } }) : null; } r.addEventListener('exported', (e: any) => this.exportInk(e, this._textRef)); - r.addEventListener('changed', (e: any) => this.callExport(this._textRef)); return this._textRef = r; } - transcribeInk = (inkdocs: Doc[], math: boolean, bounds: { x: number, y: number, width: number, height: number }, addDocument?: (doc: Doc | Doc[]) => boolean) => { + transcribeInk = (groupDoc: Doc | undefined, inkDocs: Doc[], math: boolean) => { + if (!groupDoc) return; + + const validInks = inkDocs.filter(s => s.type === DocumentType.INK); + const strokes: InkData[] = []; - inkdocs.filter(i => Cast(i.data, InkField)).forEach(i => { + validInks.filter(i => Cast(i.data, InkField)).forEach(i => { // TODO: interpolate missing times stamps const d = Cast(i.data, InkField, null); const left = Math.min(...d?.inkData.map(pd => pd.X) ?? [0]); @@ -120,8 +112,7 @@ export class InkTranscription extends React.Component { strokes.push(d.inkData.map(pd => ({ X: pd.X + NumCast(i.x) - left, Y: pd.Y + NumCast(i.y) - top, time: pd.time }))); }); - this.addDocument = addDocument; - this.bounds = bounds; + this.currGroup = groupDoc; const pointerData = { "events": strokes.map(stroke => this.inkJSON(stroke)) }; // console.log(JSON.stringify(pointerData)); @@ -135,12 +126,6 @@ export class InkTranscription extends React.Component { } } - callExport(ref: any) { - if (ref.editor.canExport) { - ref.editor.export_(); - } - } - inkJSON = (stroke: InkData) => { return { "pointerType": "PEN", @@ -159,15 +144,24 @@ export class InkTranscription extends React.Component { const latex = exports['application/x-latex']; console.log(latex); - this.addDocument?.(Docs.Create.EquationDocument({ title: latex, x: this.bounds.x + this.bounds.width, y: this.bounds.y, _width: this.bounds.width, _height: this.bounds.height })); + if (this.currGroup) { + this.currGroup.text = latex; + this.currGroup.title = latex; + } + + ref.editor.clear(); } else if (exports['text/plain']) { const text = exports['text/plain']; console.log(text); - this.addDocument?.(Docs.Create.TextDocument(text, { title: text, x: this.bounds.x + this.bounds.width, y: this.bounds.y, _width: this.bounds.width, _height: this.bounds.height })); - } - ref.editor.clear(); + if (this.currGroup) { + this.currGroup.text = text; + this.currGroup.title = text; + } + + ref.editor.clear(); + } } } diff --git a/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx b/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx index c9465e7a4..023953658 100644 --- a/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx +++ b/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx @@ -98,7 +98,7 @@ export class CollectionFreeFormView extends CollectionSubView = new Map(); private _lastTap = 0; private _batch: UndoManager.Batch | undefined = undefined; - + // private isWritingMode: boolean = true; // private writingModeDocs: Doc[] = []; @@ -461,10 +461,10 @@ export class CollectionFreeFormView extends CollectionSubView(); @undoBatch onGesture = (e: Event, ge: GestureUtils.GestureEvent) => { @@ -521,11 +521,11 @@ export class CollectionFreeFormView extends CollectionSubView { return { X: pt.X, Y: pt.Y, time: times[i] } }); const inkDoc = Docs.Create.InkDocument(ActiveInkColor(), CurrentUserUtils.SelectedTool, ActiveInkWidth(), ActiveInkBezierApprox(), ActiveFillColor(), ActiveArrowStart(), ActiveArrowEnd(), ActiveDash(), strokes, { title: "ink stroke", x: B.x - ActiveInkWidth() / 2, y: B.y - ActiveInkWidth() / 2, _width: B.width + ActiveInkWidth(), _height: B.height + ActiveInkWidth() }); - if (CurrentUserUtils.SelectedTool === InkTool.Write) { - this.unprocessedDocs.push(inkDoc); - CollectionFreeFormView.collectionsWithUnprocessedInk.add(this); - } - this.addDocument(inkDoc); + if (CurrentUserUtils.SelectedTool === InkTool.Write) { + this.unprocessedDocs.push(inkDoc); + CollectionFreeFormView.collectionsWithUnprocessedInk.add(this); + } + this.addDocument(inkDoc); e.stopPropagation(); break; case GestureUtils.Gestures.Box: @@ -1577,9 +1577,14 @@ export class CollectionFreeFormView extends CollectionSubView { - if (this.props.Document._isGroup) { - const inkdocs = this.childDocs.filter(s => s.type === DocumentType.INK); - InkTranscription.Instance.transcribeInk(inkdocs, math, { x: NumCast(this.layoutDoc.x) ?? 0, y: NumCast(this.layoutDoc.y) ?? 0, width: NumCast(this.layoutDoc._width) ?? 0, height: NumCast(this.layoutDoc._height) ?? 0 }, this.props.ContainingCollectionView?.addDocument); + if (this.props.Document._isGroup && this.props.Document.text) { + const text = StrCast(this.props.Document.text); + + const lines = text.split("\n"); + const width = 30 + 7 * Math.max(...lines.map(line => line.length)); + const height = 30 + 15 * lines.length; + + this.props.ContainingCollectionView?.addDocument(Docs.Create.TextDocument(text, { title: text, x: NumCast(this.layoutDoc.x) + NumCast(this.layoutDoc._width) + 20, y: NumCast(this.layoutDoc.y), _width: width, _height: height })); } } diff --git a/src/client/views/nodes/button/FontIconBox.tsx b/src/client/views/nodes/button/FontIconBox.tsx index 862c16050..fc358f106 100644 --- a/src/client/views/nodes/button/FontIconBox.tsx +++ b/src/client/views/nodes/button/FontIconBox.tsx @@ -26,6 +26,7 @@ import { EditableView } from '../../EditableView'; import { GestureOverlay } from '../../GestureOverlay'; import { Colors } from '../../global/globalEnums'; import { ActiveFillColor, ActiveInkColor, ActiveInkWidth, SetActiveFillColor, SetActiveInkColor, SetActiveInkWidth } from '../../InkingStroke'; +import { InkTranscription } from '../../InkTranscription'; import { StyleProp } from '../../StyleProvider'; import { FieldView, FieldViewProps } from '.././FieldView'; import { RichTextMenu } from '../formattedText/RichTextMenu'; @@ -702,9 +703,9 @@ export function checkInksToGroup() { if (CurrentUserUtils.SelectedTool === InkTool.Write) { CollectionFreeFormView.collectionsWithUnprocessedInk.forEach(ffView => { // TODO: nda - will probably want to go through ffView unprocessed docs and then see if any of the inksToGroup docs are in it and only use those - // find all inkDocs in ffView.unprocessedDocs that are within 200 pixels of each other - const inksToGroup = ffView.unprocessedDocs.filter(inkDoc => { - console.log(inkDoc.x, inkDoc.y); + // find all inkDocs in ffView.unprocessedDocs that are within 200 pixels of each other + const inksToGroup = ffView.unprocessedDocs.filter(inkDoc => { + console.log(inkDoc.x, inkDoc.y); }); }); } @@ -718,26 +719,26 @@ export function createInkGroup(inksToGroup?: Doc[]) { const selected = ffView.unprocessedDocs; // loop through selected an get the bound const bounds: { x: number, y: number, width?: number, height?: number }[] = [] - + selected.map(action(d => { const x = NumCast(d.x); const y = NumCast(d.y); const width = d[WidthSym](); const height = d[HeightSym](); - bounds.push({x, y, width, height}); + bounds.push({ x, y, width, height }); })) - + const aggregBounds = aggregateBounds(bounds, 0, 0); const marqViewRef = ffView._marqueeViewRef.current; - + // set the vals for bounds in marqueeView if (marqViewRef) { marqViewRef._downX = aggregBounds.x; marqViewRef._downY = aggregBounds.y; marqViewRef._lastX = aggregBounds.r; marqViewRef._lastY = aggregBounds.b; - } - + } + selected.map(action(d => { const dx = NumCast(d.x); const dy = NumCast(d.y); @@ -756,11 +757,13 @@ export function createInkGroup(inksToGroup?: Doc[]) { ffView.props.removeDocument?.(selected); // TODO: nda - this is the code to actually get a new grouped collection const newCollection = marqViewRef?.getCollection(selected, undefined, [], true); - + // nda - bug: when deleting a stroke before leaving writing mode, delete the stroke from unprocessed ink docs newCollection && ffView.props.addDocument?.(newCollection); // TODO: nda - will probably need to go through and only remove the unprocessed selected docs ffView.unprocessedDocs = []; + + InkTranscription.Instance.transcribeInk(newCollection, selected, false); }); } CollectionFreeFormView.collectionsWithUnprocessedInk.clear(); @@ -780,7 +783,7 @@ ScriptingGlobals.add(function setActiveInkTool(tool: string, checkResult?: boole // const selected = ffView.unprocessedDocs; // // loop through selected an get the bound // const bounds: { x: number, y: number, width?: number, height?: number }[] = [] - + // selected.map(action(d => { // const x = NumCast(d.x); // const y = NumCast(d.y); @@ -788,10 +791,10 @@ ScriptingGlobals.add(function setActiveInkTool(tool: string, checkResult?: boole // const height = d[HeightSym](); // bounds.push({x, y, width, height}); // })) - + // const aggregBounds = aggregateBounds(bounds, 0, 0); // const marqViewRef = ffView._marqueeViewRef.current; - + // // set the vals for bounds in marqueeView // if (marqViewRef) { // marqViewRef._downX = aggregBounds.x; @@ -799,7 +802,7 @@ ScriptingGlobals.add(function setActiveInkTool(tool: string, checkResult?: boole // marqViewRef._lastX = aggregBounds.r; // marqViewRef._lastY = aggregBounds.b; // } - + // selected.map(action(d => { // const dx = NumCast(d.x); // const dy = NumCast(d.y); @@ -818,7 +821,7 @@ ScriptingGlobals.add(function setActiveInkTool(tool: string, checkResult?: boole // ffView.props.removeDocument?.(selected); // // TODO: nda - this is the code to actually get a new grouped collection // const newCollection = marqViewRef?.getCollection(selected, undefined, [], true); - + // // nda - bug: when deleting a stroke before leaving writing mode, delete the stroke from unprocessed ink docs // newCollection && ffView.props.addDocument?.(newCollection); // ffView.unprocessedDocs = []; -- cgit v1.2.3-70-g09d2 From 73d52c3c1a0b06e0180999cb876feb6025df31b2 Mon Sep 17 00:00:00 2001 From: mehekj Date: Thu, 28 Apr 2022 16:31:57 -0400 Subject: fixed timestamps and added some scaling --- src/client/documents/Documents.ts | 1 - src/client/views/GestureOverlay.tsx | 22 +++---------- src/client/views/InkTranscription.tsx | 38 +++++++++++++--------- .../collectionFreeForm/CollectionFreeFormView.tsx | 15 ++++----- src/client/views/nodes/button/FontIconBox.tsx | 10 +++--- src/fields/InkField.ts | 3 +- src/pen-gestures/GestureUtils.ts | 3 +- 7 files changed, 41 insertions(+), 51 deletions(-) (limited to 'src/client/views/InkTranscription.tsx') diff --git a/src/client/documents/Documents.ts b/src/client/documents/Documents.ts index 884cc781d..aee9ef49d 100644 --- a/src/client/documents/Documents.ts +++ b/src/client/documents/Documents.ts @@ -1161,7 +1161,6 @@ export namespace DocUtils { created = Docs.Create.AudioDocument((field).url.href, resolved); layout = AudioBox.LayoutString; } else if (field instanceof InkField) { - console.log("Documents " + field.inkData) created = Docs.Create.InkDocument(ActiveInkColor(), CurrentUserUtils.SelectedTool, ActiveInkWidth(), ActiveInkBezierApprox(), ActiveFillColor(), ActiveArrowStart(), ActiveArrowEnd(), ActiveDash(), (field).inkData, resolved); layout = InkingStroke.LayoutString; } else if (field instanceof List && field[0] instanceof Doc) { diff --git a/src/client/views/GestureOverlay.tsx b/src/client/views/GestureOverlay.tsx index 5f0311d38..fc0a214a6 100644 --- a/src/client/views/GestureOverlay.tsx +++ b/src/client/views/GestureOverlay.tsx @@ -42,7 +42,6 @@ export class GestureOverlay extends Touchable { @observable private _menuY: number = -300; @observable private _pointerY?: number; @observable private _points: { X: number, Y: number }[] = []; - @observable private _timeStamps: number[] = []; @observable private _strokes: InkData[] = []; @observable private _palette?: JSX.Element; @observable private _clipboardDoc?: JSX.Element; @@ -61,7 +60,6 @@ export class GestureOverlay extends Touchable { private pointerIdentifier?: number; private _hands: Map = new Map(); private _holdTimer: NodeJS.Timeout | undefined; - private _strokeStartTime: number = 0; protected _multiTouchDisposer?: InteractionUtils.MultiTouchEventDisposer; @@ -299,7 +297,7 @@ export class GestureOverlay extends Touchable { pointer = fingers.reduce((a, v) => a.clientX < v.clientX || v.identifier === thumb.identifier ? a : v); } else { - console.log("not hand"); + // console.log("not hand"); } this.pointerIdentifier = pointer?.identifier; @@ -425,7 +423,6 @@ export class GestureOverlay extends Touchable { this._strokes = []; this._points = []; - this._timeStamps = []; this._possibilities = []; document.removeEventListener("touchend", this.handleHandUp); } @@ -509,8 +506,6 @@ export class GestureOverlay extends Touchable { CurrentUserUtils.SelectedTool = InkTool.Write; } this._points.push({ X: e.clientX, Y: e.clientY }); - this._timeStamps.push(0); - this._strokeStartTime = new Date().getTime(); setupMoveUpEvents(this, e, this.onPointerMove, this.onPointerUp, emptyFunction); // if (CurrentUserUtils.SelectedTool === InkTool.Highlighter) SetActiveInkColor("rgba(245, 230, 95, 0.75)"); } @@ -519,7 +514,6 @@ export class GestureOverlay extends Touchable { @action onPointerMove = (e: PointerEvent) => { this._points.push({ X: e.clientX, Y: e.clientY }); - this._timeStamps.push(new Date().getTime() - this._strokeStartTime); if (this._points.length > 1) { const B = this.svgBounds; @@ -566,7 +560,6 @@ export class GestureOverlay extends Touchable { const points = this._points.map(p => ({ X: p.X - B.left, Y: p.Y - B.top })); //push first points to so interactionUtil knows pointer is up this._points.push({ X: this._points[0].X, Y: this._points[0].Y }); - this._timeStamps.push(this._timeStamps[0]); const initialPoint = this._points[0.]; const xInGlass = initialPoint.X > (this._thumbX ?? Number.MAX_SAFE_INTEGER) && initialPoint.X < (this._thumbX ?? Number.MAX_SAFE_INTEGER) + (this.height); @@ -578,7 +571,6 @@ export class GestureOverlay extends Touchable { case ToolglassTools.InkToText: this._strokes.push(new Array(...this._points)); this._points = []; - this._timeStamps = []; CognitiveServices.Inking.Appliers.InterpretStrokes(this._strokes).then((results) => { const wordResults = results.filter((r: any) => r.category === "line"); const possibilities: string[] = []; @@ -602,7 +594,6 @@ export class GestureOverlay extends Touchable { case ToolglassTools.IgnoreGesture: this.dispatchGesture(GestureUtils.Gestures.Stroke); this._points = []; - this._timeStamps = []; break; } } @@ -611,7 +602,6 @@ export class GestureOverlay extends Touchable { this.makePolygon(this.InkShape, false); this.dispatchGesture(GestureUtils.Gestures.Stroke); this._points = []; - this._timeStamps = []; if (!CollectionFreeFormViewChrome.Instance?._keepPrimitiveMode) { this.InkShape = ""; Doc.UserDoc().activeInkTool = InkTool.None; @@ -638,7 +628,7 @@ export class GestureOverlay extends Touchable { if (!actionPerformed) { const newPoints = this._points.reduce((p, pts) => { p.push([pts.X, pts.Y]); return p; }, [] as number[][]); newPoints.pop(); - console.log("getting to bezier math"); + // console.log("getting to bezier math"); const controlPoints: { X: number, Y: number }[] = []; const bezierCurves = fitCurve(newPoints, 10); @@ -649,21 +639,19 @@ export class GestureOverlay extends Touchable { controlPoints.push({ X: curve[2][0], Y: curve[2][1] }); controlPoints.push({ X: curve[3][0], Y: curve[3][1] }); - } + } const dist = Math.sqrt((controlPoints[0].X - controlPoints.lastElement().X) * (controlPoints[0].X - controlPoints.lastElement().X) + (controlPoints[0].Y - controlPoints.lastElement().Y) * (controlPoints[0].Y - controlPoints.lastElement().Y)); if (controlPoints.length > 4 && dist < 10) controlPoints[controlPoints.length - 1] = controlPoints[0]; - this._points = controlPoints; + this._points = controlPoints; this.dispatchGesture(GestureUtils.Gestures.Stroke); // TODO: nda - check inks to group here checkInksToGroup(); } this._points = []; - this._timeStamps = []; } } else { this._points = []; - this._timeStamps = []; } CollectionFreeFormViewChrome.Instance?.primCreated(); } @@ -712,7 +700,6 @@ export class GestureOverlay extends Touchable { } } this._points = []; - this._timeStamps = []; switch (shape) { //must push an extra point in the end so InteractionUtils knows pointer is up. //must be (points[0].X,points[0]-1) @@ -836,7 +823,6 @@ export class GestureOverlay extends Touchable { gesture: gesture as any, bounds: this.getBounds(stroke ?? this._points), text: data, - times: this._timeStamps } } ) diff --git a/src/client/views/InkTranscription.tsx b/src/client/views/InkTranscription.tsx index ea5d5a1ec..a96cda743 100644 --- a/src/client/views/InkTranscription.tsx +++ b/src/client/views/InkTranscription.tsx @@ -1,12 +1,13 @@ -import { timeStamp } from 'console'; import * as iink from 'iink-js'; import { action, observable } from 'mobx'; import * as React from 'react'; -import { Doc } from '../../fields/Doc'; +import { Doc, DocListCast } from '../../fields/Doc'; import { InkData, InkField } from "../../fields/InkField"; -import { Cast, NumCast, StrCast } from '../../fields/Types'; +import { Cast, NumCast } from '../../fields/Types'; import { DocumentType } from "../documents/DocumentTypes"; import './InkTranscription.scss'; +import { Utils } from '../../Utils'; +import { timesSeries } from 'async'; export class InkTranscription extends React.Component { static Instance: InkTranscription; @@ -16,6 +17,7 @@ export class InkTranscription extends React.Component { @observable _textRegister: any; @observable _textRef: any; private currGroup?: Doc; + private containingLayout?: Doc; constructor(props: Readonly<{}>) { super(props); @@ -98,24 +100,26 @@ export class InkTranscription extends React.Component { return this._textRef = r; } - transcribeInk = (groupDoc: Doc | undefined, inkDocs: Doc[], math: boolean) => { + transcribeInk = (groupDoc: Doc | undefined, containingLayout: Doc, inkDocs: Doc[], math: boolean) => { if (!groupDoc) return; const validInks = inkDocs.filter(s => s.type === DocumentType.INK); const strokes: InkData[] = []; + const times: number[] = []; validInks.filter(i => Cast(i.data, InkField)).forEach(i => { - // TODO: interpolate missing times stamps const d = Cast(i.data, InkField, null); const left = Math.min(...d?.inkData.map(pd => pd.X) ?? [0]); const top = Math.min(...d?.inkData.map(pd => pd.Y) ?? [0]); - strokes.push(d.inkData.map(pd => ({ X: pd.X + NumCast(i.x) - left, Y: pd.Y + NumCast(i.y) - top, time: pd.time }))); + strokes.push(d.inkData.map(pd => ({ X: pd.X + NumCast(i.x) - left, Y: pd.Y + NumCast(i.y) - top }))); + times.push(NumCast(i.creationDate, 1)); }); this.currGroup = groupDoc; + this.containingLayout = containingLayout; - const pointerData = { "events": strokes.map(stroke => this.inkJSON(stroke)) }; - // console.log(JSON.stringify(pointerData)); + const pointerData = { "events": strokes.map((stroke, i) => this.inkJSON(stroke, times[i])) }; + console.log(JSON.stringify(pointerData)); const processGestures = false; if (math) { @@ -126,13 +130,17 @@ export class InkTranscription extends React.Component { } } - inkJSON = (stroke: InkData) => { + inkJSON = (stroke: InkData, time: number) => { + const scale = NumCast(this.containingLayout?._viewScale, 1); + const panX = NumCast(this.containingLayout?._panX); + const panY = NumCast(this.containingLayout?._panY); + return { "pointerType": "PEN", "pointerId": 1, - "x": stroke.map(point => point.X), - "y": stroke.map(point => point.Y), - "t": stroke.map(point => point.time), + "x": stroke.map(point => (point.X - panX) * scale), + "y": stroke.map(point => (point.Y - panY) * scale), + "t": new Array(stroke.length).fill(time), "p": new Array(stroke.length).fill(1.0) }; } @@ -142,7 +150,7 @@ export class InkTranscription extends React.Component { if (exports) { if (exports['application/x-latex']) { const latex = exports['application/x-latex']; - console.log(latex); + // console.log(latex); if (this.currGroup) { this.currGroup.text = latex; @@ -153,11 +161,11 @@ export class InkTranscription extends React.Component { } else if (exports['text/plain']) { const text = exports['text/plain']; - console.log(text); + // console.log(text); if (this.currGroup) { this.currGroup.text = text; - this.currGroup.title = text; + this.currGroup.title = text.split("\n")[0]; } ref.editor.clear(); diff --git a/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx b/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx index 023953658..3c5cdb444 100644 --- a/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx +++ b/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx @@ -517,9 +517,7 @@ export class CollectionFreeFormView extends CollectionSubView { return { X: pt.X, Y: pt.Y, time: times[i] } }); - const inkDoc = Docs.Create.InkDocument(ActiveInkColor(), CurrentUserUtils.SelectedTool, ActiveInkWidth(), ActiveInkBezierApprox(), ActiveFillColor(), ActiveArrowStart(), ActiveArrowEnd(), ActiveDash(), strokes, + const inkDoc = Docs.Create.InkDocument(ActiveInkColor(), CurrentUserUtils.SelectedTool, ActiveInkWidth(), ActiveInkBezierApprox(), ActiveFillColor(), ActiveArrowStart(), ActiveArrowEnd(), ActiveDash(), points, { title: "ink stroke", x: B.x - ActiveInkWidth() / 2, y: B.y - ActiveInkWidth() / 2, _width: B.width + ActiveInkWidth(), _height: B.height + ActiveInkWidth() }); if (CurrentUserUtils.SelectedTool === InkTool.Write) { this.unprocessedDocs.push(inkDoc); @@ -1578,13 +1576,14 @@ export class CollectionFreeFormView extends CollectionSubView { if (this.props.Document._isGroup && this.props.Document.text) { - const text = StrCast(this.props.Document.text); + if (!math) { + const text = StrCast(this.props.Document.text); - const lines = text.split("\n"); - const width = 30 + 7 * Math.max(...lines.map(line => line.length)); - const height = 30 + 15 * lines.length; + const lines = text.split("\n"); + const height = 30 + 15 * lines.length; - this.props.ContainingCollectionView?.addDocument(Docs.Create.TextDocument(text, { title: text, x: NumCast(this.layoutDoc.x) + NumCast(this.layoutDoc._width) + 20, y: NumCast(this.layoutDoc.y), _width: width, _height: height })); + this.props.ContainingCollectionView?.addDocument(Docs.Create.TextDocument(text, { title: lines[0], x: NumCast(this.layoutDoc.x) + NumCast(this.layoutDoc._width) + 20, y: NumCast(this.layoutDoc.y), _width: 200, _height: height })); + } } } diff --git a/src/client/views/nodes/button/FontIconBox.tsx b/src/client/views/nodes/button/FontIconBox.tsx index fc358f106..aca393c1a 100644 --- a/src/client/views/nodes/button/FontIconBox.tsx +++ b/src/client/views/nodes/button/FontIconBox.tsx @@ -262,7 +262,7 @@ export class FontIconBox extends DocComponent() { "Comic Sans MS", "Tahoma", "Impact", "Crimson Text"]; } } catch (e) { - console.log(e); + // console.log(e); } // Get items to place into the list @@ -699,13 +699,13 @@ ScriptingGlobals.add(function toggleItalic(checkResult?: boolean) { export function checkInksToGroup() { - console.log("getting here to inks group"); + // console.log("getting here to inks group"); if (CurrentUserUtils.SelectedTool === InkTool.Write) { CollectionFreeFormView.collectionsWithUnprocessedInk.forEach(ffView => { // TODO: nda - will probably want to go through ffView unprocessed docs and then see if any of the inksToGroup docs are in it and only use those // find all inkDocs in ffView.unprocessedDocs that are within 200 pixels of each other const inksToGroup = ffView.unprocessedDocs.filter(inkDoc => { - console.log(inkDoc.x, inkDoc.y); + // console.log(inkDoc.x, inkDoc.y); }); }); } @@ -763,7 +763,7 @@ export function createInkGroup(inksToGroup?: Doc[]) { // TODO: nda - will probably need to go through and only remove the unprocessed selected docs ffView.unprocessedDocs = []; - InkTranscription.Instance.transcribeInk(newCollection, selected, false); + InkTranscription.Instance.transcribeInk(newCollection, ffView.layoutDoc, selected, false); }); } CollectionFreeFormView.collectionsWithUnprocessedInk.clear(); @@ -845,7 +845,7 @@ ScriptingGlobals.add(function setActiveInkTool(tool: string, checkResult?: boole if (Doc.UserDoc().activeInkTool === tool && !GestureOverlay.Instance.InkShape) { Doc.UserDoc().activeInkTool = InkTool.None; } else if (tool == "write") { - console.log("write mode selected - create groupDoc here!", tool) + // console.log("write mode selected - create groupDoc here!", tool) Doc.UserDoc().activeInkTool = tool; GestureOverlay.Instance.InkShape = ""; } else { diff --git a/src/fields/InkField.ts b/src/fields/InkField.ts index 6d24c6cbc..31fa6dfac 100644 --- a/src/fields/InkField.ts +++ b/src/fields/InkField.ts @@ -20,7 +20,6 @@ export enum InkTool { export interface PointData { X: number; Y: number; - time?: number; } export type Segment = Array; @@ -85,7 +84,7 @@ export class InkField extends ObjectField { } [ToScriptString]() { - return "new InkField([" + this.inkData.map(i => `{X: ${i.X}, Y: ${i.Y}, time: ${i.time || 0}} `) + "])"; + return "new InkField([" + this.inkData.map(i => `{X: ${i.X}, Y: ${i.Y}`) + "])"; } [ToString]() { return "InkField"; diff --git a/src/pen-gestures/GestureUtils.ts b/src/pen-gestures/GestureUtils.ts index cf59cb3c6..65f2bf80c 100644 --- a/src/pen-gestures/GestureUtils.ts +++ b/src/pen-gestures/GestureUtils.ts @@ -8,8 +8,7 @@ export namespace GestureUtils { readonly gesture: Gestures, readonly points: PointData[], readonly bounds: Rect, - readonly text?: any, - readonly times?: number[] + readonly text?: any ) { } } -- cgit v1.2.3-70-g09d2 From 4f6b6306803437851959e9ad214fd61a14207a20 Mon Sep 17 00:00:00 2001 From: Naafiyan Ahmed Date: Thu, 28 Apr 2022 16:34:31 -0400 Subject: added basic code for subgrouping transcribed text --- src/client/util/InteractionUtils.tsx | 1 + src/client/views/GestureOverlay.tsx | 2 +- src/client/views/InkTranscription.tsx | 71 +++++++++++++++++++++- src/client/views/LightboxView.tsx | 3 +- src/client/views/Touchable.tsx | 2 + .../collectionFreeForm/CollectionFreeFormView.tsx | 4 +- src/client/views/nodes/DocumentLinksButton.tsx | 2 + src/client/views/nodes/button/FontIconBox.tsx | 56 ++--------------- 8 files changed, 86 insertions(+), 55 deletions(-) (limited to 'src/client/views/InkTranscription.tsx') diff --git a/src/client/util/InteractionUtils.tsx b/src/client/util/InteractionUtils.tsx index 675dff00b..e2c5fab64 100644 --- a/src/client/util/InteractionUtils.tsx +++ b/src/client/util/InteractionUtils.tsx @@ -357,6 +357,7 @@ export namespace InteractionUtils { } export function IsDragging(oldTouches: Map, newTouches: React.Touch[], leniency: number): boolean { + console.log("getting here"); for (const touch of newTouches) { if (touch) { const oldTouch = oldTouches.get(touch.identifier); diff --git a/src/client/views/GestureOverlay.tsx b/src/client/views/GestureOverlay.tsx index 5f0311d38..e689cd43c 100644 --- a/src/client/views/GestureOverlay.tsx +++ b/src/client/views/GestureOverlay.tsx @@ -212,7 +212,7 @@ export class GestureOverlay extends Touchable { const nts: any = this.getNewTouches(e); this._holdTimer && clearTimeout(this._holdTimer); this._holdTimer = undefined; - + document.dispatchEvent( new CustomEvent>("dashOnTouchMove", { diff --git a/src/client/views/InkTranscription.tsx b/src/client/views/InkTranscription.tsx index ea5d5a1ec..3b464db10 100644 --- a/src/client/views/InkTranscription.tsx +++ b/src/client/views/InkTranscription.tsx @@ -5,6 +5,7 @@ import * as React from 'react'; import { Doc } from '../../fields/Doc'; import { InkData, InkField } from "../../fields/InkField"; import { Cast, NumCast, StrCast } from '../../fields/Types'; +import { aggregateBounds } from '../../Utils'; import { DocumentType } from "../documents/DocumentTypes"; import './InkTranscription.scss'; @@ -15,12 +16,18 @@ export class InkTranscription extends React.Component { @observable _mathRef: any; @observable _textRegister: any; @observable _textRef: any; + private lastJiix: any; private currGroup?: Doc; + private wordsBoundMapping: any; + private inkDocs: Doc[]; + private ffView: any; constructor(props: Readonly<{}>) { super(props); InkTranscription.Instance = this; + this.wordsBoundMapping = new Map(); + this.inkDocs = new Array(); } componentWillUnmount() { @@ -98,8 +105,10 @@ export class InkTranscription extends React.Component { return this._textRef = r; } - transcribeInk = (groupDoc: Doc | undefined, inkDocs: Doc[], math: boolean) => { + transcribeInk = (groupDoc: Doc | undefined, inkDocs: Doc[], math: boolean, ffView?: any) => { if (!groupDoc) return; + this.inkDocs = inkDocs; + this.ffView = ffView; const validInks = inkDocs.filter(s => s.type === DocumentType.INK); @@ -137,8 +146,20 @@ export class InkTranscription extends React.Component { }; } + mmToPixel = (mm: number) => { + return ((96 * mm) / 25.4); + } + + calcBounds = (coords: any) => { + // find max and min x values and subtract + const max = Math.max(...coords); + const min = Math.min(...coords); + return max - min; + } + exportInk = (e: any, ref: any) => { const exports = e.detail.exports; + console.log(e); if (exports) { if (exports['application/x-latex']) { const latex = exports['application/x-latex']; @@ -152,12 +173,60 @@ export class InkTranscription extends React.Component { ref.editor.clear(); } else if (exports['text/plain']) { + if (exports['application/vnd.myscript.jiix']) { + this.lastJiix = JSON.parse(exports['application/vnd.myscript.jiix']); + // console.log("lastJiix:", this.lastJiix); + // TODO: nda - go through all the words and look at x,y values and figure out the selection + this.lastJiix.words.forEach((word: any) => { + // map each word to a new map + this.wordsBoundMapping[word] = new Map(); + if (word.items) { + word.items.map((item: any) => { + this.wordsBoundMapping.set(word, {xList: item.X, yList: item.Y, width: this.calcBounds(item.X), height: this.calcBounds(item.Y)}); + }) + } + }) + // make groups for each of the words + // loop through this.wordsBoundMapping + + this.wordsBoundMapping.forEach((value: any, key: any) => { + const xList = value.xList; + const yList = value.yList; + const width = value.width; + const height = value.height; + const bounds: { x: number, y: number, width?: number, height?: number }[] = [] + // loop through xList + for (let i = 0; i < xList.length; i++) { + const x = xList[i]; + const y = yList[i]; + const newBounds = { x: x, y: y }; + bounds.push(newBounds); + } + const aggregBounds = aggregateBounds(bounds, 0, 0); + + const marqViewRef = this.ffView?._marqueeViewRef.current; + + // set the vals for bounds in marqueeView + if (marqViewRef) { + marqViewRef._downX = aggregBounds.x; + marqViewRef._downY = aggregBounds.y; + marqViewRef._lastX = aggregBounds.r; + marqViewRef._lastY = aggregBounds.b; + } + + }); + + // console.log(this.wordsBoundMapping); + } const text = exports['text/plain']; console.log(text); if (this.currGroup) { + console.log("curr grouping"); this.currGroup.text = text; this.currGroup.title = text; + console.log(this.lastJiix); + console.log(this.currGroup); } ref.editor.clear(); diff --git a/src/client/views/LightboxView.tsx b/src/client/views/LightboxView.tsx index ec3bf6c18..c8f6b5f0b 100644 --- a/src/client/views/LightboxView.tsx +++ b/src/client/views/LightboxView.tsx @@ -5,7 +5,7 @@ import "normalize.css"; import * as React from 'react'; import { Doc, DocListCast, Opt } from '../../fields/Doc'; import { Cast, NumCast, StrCast } from '../../fields/Types'; -import { emptyFunction, returnEmptyDoclist, returnEmptyFilter, returnFalse, returnTrue } from '../../Utils'; +import { emptyFunction, returnEmptyDoclist, returnEmptyFilter, returnFalse, returnTrue, Utils } from '../../Utils'; import { DocUtils } from '../documents/Documents'; import { DocumentManager } from '../util/DocumentManager'; import { LinkManager } from '../util/LinkManager'; @@ -17,6 +17,7 @@ import "./LightboxView.scss"; import { DocumentView } from './nodes/DocumentView'; import { DefaultStyleProvider, wavyBorderPath } from './StyleProvider'; import { CollectionMenu } from './collections/CollectionMenu'; +import { utils } from 'mocha'; interface LightboxViewProps { PanelWidth: number; diff --git a/src/client/views/Touchable.tsx b/src/client/views/Touchable.tsx index 789756a78..c712e7ed3 100644 --- a/src/client/views/Touchable.tsx +++ b/src/client/views/Touchable.tsx @@ -127,11 +127,13 @@ export abstract class Touchable extends React.Component { } handle1PointerMove = (e: TouchEvent, me: InteractionUtils.MultiTouchEvent): any => { + console.log("getting to handle1PointersMove"); e.stopPropagation(); e.preventDefault(); } handle2PointersMove = (e: TouchEvent, me: InteractionUtils.MultiTouchEvent): any => { + console.log("getting to handle2PointersMove"); e.stopPropagation(); e.preventDefault(); } diff --git a/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx b/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx index 023953658..7c11c252a 100644 --- a/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx +++ b/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx @@ -702,9 +702,9 @@ export class CollectionFreeFormView extends CollectionSubView { + console.log("this is onPointerMove"); if (InteractionUtils.IsType(e, InteractionUtils.PENTYPE)) return; if (InteractionUtils.IsType(e, InteractionUtils.TOUCHTYPE)) { - console.log("is touch"); Doc.UserDoc().activeInkTool = InkTool.None; if (this.props.isContentActive(true)) e.stopPropagation(); } else if (!e.cancelBubble) { @@ -825,6 +825,7 @@ export class CollectionFreeFormView extends CollectionSubView) => { + console.log("getting here yeet"); if (!e.cancelBubble) { const myTouches = InteractionUtils.GetMyTargetTouches(me, this.prevPoints, true); if (myTouches[0]) { @@ -836,6 +837,7 @@ export class CollectionFreeFormView extends CollectionSubView only want to do this when collection is selected' this.pan(myTouches[0]); } } diff --git a/src/client/views/nodes/DocumentLinksButton.tsx b/src/client/views/nodes/DocumentLinksButton.tsx index 7e6ca4248..f9d2b9917 100644 --- a/src/client/views/nodes/DocumentLinksButton.tsx +++ b/src/client/views/nodes/DocumentLinksButton.tsx @@ -21,6 +21,8 @@ import { LinkDescriptionPopup } from "./LinkDescriptionPopup"; import { TaskCompletionBox } from "./TaskCompletedBox"; import React = require("react"); import { Transform } from "../../util/Transform"; +import { InkTool } from "../../../fields/InkField"; +import { CurrentUserUtils } from "../../util/CurrentUserUtils"; const higflyout = require("@hig/flyout"); export const { anchorPoints } = higflyout; diff --git a/src/client/views/nodes/button/FontIconBox.tsx b/src/client/views/nodes/button/FontIconBox.tsx index fc358f106..85130e69e 100644 --- a/src/client/views/nodes/button/FontIconBox.tsx +++ b/src/client/views/nodes/button/FontIconBox.tsx @@ -757,13 +757,17 @@ export function createInkGroup(inksToGroup?: Doc[]) { ffView.props.removeDocument?.(selected); // TODO: nda - this is the code to actually get a new grouped collection const newCollection = marqViewRef?.getCollection(selected, undefined, [], true); + if (newCollection) { + newCollection.height = newCollection[HeightSym](); + newCollection.width = newCollection[WidthSym](); + } // nda - bug: when deleting a stroke before leaving writing mode, delete the stroke from unprocessed ink docs newCollection && ffView.props.addDocument?.(newCollection); // TODO: nda - will probably need to go through and only remove the unprocessed selected docs ffView.unprocessedDocs = []; - InkTranscription.Instance.transcribeInk(newCollection, selected, false); + InkTranscription.Instance.transcribeInk(newCollection, selected, false, ffView); }); } CollectionFreeFormView.collectionsWithUnprocessedInk.clear(); @@ -778,56 +782,6 @@ export function createInkGroup(inksToGroup?: Doc[]) { ScriptingGlobals.add(function setActiveInkTool(tool: string, checkResult?: boolean) { createInkGroup(); - // if (CurrentUserUtils.SelectedTool === InkTool.Write) { - // CollectionFreeFormView.collectionsWithUnprocessedInk.forEach(ffView => { - // const selected = ffView.unprocessedDocs; - // // loop through selected an get the bound - // const bounds: { x: number, y: number, width?: number, height?: number }[] = [] - - // selected.map(action(d => { - // const x = NumCast(d.x); - // const y = NumCast(d.y); - // const width = d[WidthSym](); - // const height = d[HeightSym](); - // bounds.push({x, y, width, height}); - // })) - - // const aggregBounds = aggregateBounds(bounds, 0, 0); - // const marqViewRef = ffView._marqueeViewRef.current; - - // // set the vals for bounds in marqueeView - // if (marqViewRef) { - // marqViewRef._downX = aggregBounds.x; - // marqViewRef._downY = aggregBounds.y; - // marqViewRef._lastX = aggregBounds.r; - // marqViewRef._lastY = aggregBounds.b; - // } - - // selected.map(action(d => { - // const dx = NumCast(d.x); - // const dy = NumCast(d.y); - // delete d.x; - // delete d.y; - // delete d.activeFrame; - // delete d._timecodeToShow; // bcz: this should be automatic somehow.. along with any other properties that were logically associated with the original collection - // delete d._timecodeToHide; // bcz: this should be automatic somehow.. along with any other properties that were logically associated with the original collection - // // calculate pos based on bounds - // if (marqViewRef?.Bounds) { - // d.x = dx - marqViewRef.Bounds.left - marqViewRef.Bounds.width / 2; - // d.y = dy - marqViewRef.Bounds.top - marqViewRef.Bounds.height / 2; - // } - // return d; - // })); - // ffView.props.removeDocument?.(selected); - // // TODO: nda - this is the code to actually get a new grouped collection - // const newCollection = marqViewRef?.getCollection(selected, undefined, [], true); - - // // nda - bug: when deleting a stroke before leaving writing mode, delete the stroke from unprocessed ink docs - // newCollection && ffView.props.addDocument?.(newCollection); - // ffView.unprocessedDocs = []; - // }); - // } - // CollectionFreeFormView.collectionsWithUnprocessedInk.clear(); if (checkResult) { return ((Doc.UserDoc().activeInkTool === tool && !GestureOverlay.Instance?.InkShape) || GestureOverlay.Instance?.InkShape === tool) ? -- cgit v1.2.3-70-g09d2 From 6c9999efe775c75c217ed19c493dc175c7a556bc Mon Sep 17 00:00:00 2001 From: mehekj Date: Thu, 28 Apr 2022 17:05:19 -0400 Subject: updated timestamps to creationdate --- src/client/documents/Documents.ts | 1 + src/client/views/InkTranscription.tsx | 11 ++++++++--- 2 files changed, 9 insertions(+), 3 deletions(-) (limited to 'src/client/views/InkTranscription.tsx') diff --git a/src/client/documents/Documents.ts b/src/client/documents/Documents.ts index aee9ef49d..6936e4628 100644 --- a/src/client/documents/Documents.ts +++ b/src/client/documents/Documents.ts @@ -763,6 +763,7 @@ export namespace Docs { I.author = Doc.CurrentUserEmail; I.rotation = 0; I.data = new InkField(points); + I.creationDate = new DateField; I["acl-Public"] = Doc.UserDoc()?.defaultAclPrivate ? SharingPermissions.None : SharingPermissions.Augment; I["acl-Override"] = "None"; I.links = "@links(self)"; diff --git a/src/client/views/InkTranscription.tsx b/src/client/views/InkTranscription.tsx index a96cda743..df3a1acb3 100644 --- a/src/client/views/InkTranscription.tsx +++ b/src/client/views/InkTranscription.tsx @@ -3,7 +3,7 @@ import { action, observable } from 'mobx'; import * as React from 'react'; import { Doc, DocListCast } from '../../fields/Doc'; import { InkData, InkField } from "../../fields/InkField"; -import { Cast, NumCast } from '../../fields/Types'; +import { Cast, DateCast, NumCast } from '../../fields/Types'; import { DocumentType } from "../documents/DocumentTypes"; import './InkTranscription.scss'; import { Utils } from '../../Utils'; @@ -112,14 +112,15 @@ export class InkTranscription extends React.Component { const left = Math.min(...d?.inkData.map(pd => pd.X) ?? [0]); const top = Math.min(...d?.inkData.map(pd => pd.Y) ?? [0]); strokes.push(d.inkData.map(pd => ({ X: pd.X + NumCast(i.x) - left, Y: pd.Y + NumCast(i.y) - top }))); - times.push(NumCast(i.creationDate, 1)); + times.push(DateCast(i.creationDate).getDate().getTime()); }); this.currGroup = groupDoc; this.containingLayout = containingLayout; const pointerData = { "events": strokes.map((stroke, i) => this.inkJSON(stroke, times[i])) }; - console.log(JSON.stringify(pointerData)); + // console.log(JSON.stringify(pointerData)); + // console.log(pointerData); const processGestures = false; if (math) { @@ -163,6 +164,10 @@ export class InkTranscription extends React.Component { const text = exports['text/plain']; // console.log(text); + if (exports['application/vnd.myscript.jiix']) { + console.log(JSON.parse(exports['application/vnd.myscript.jiix'])); + } + if (this.currGroup) { this.currGroup.text = text; this.currGroup.title = text.split("\n")[0]; -- cgit v1.2.3-70-g09d2 From 71a2b8de701cc2d4058ff3d42c8183399fda3b95 Mon Sep 17 00:00:00 2001 From: Naafiyan Ahmed Date: Thu, 28 Apr 2022 21:35:48 -0400 Subject: working on mapping algo --- src/client/views/InkTranscription.tsx | 34 +++++++++++++++++++++++++++++++++- 1 file changed, 33 insertions(+), 1 deletion(-) (limited to 'src/client/views/InkTranscription.tsx') diff --git a/src/client/views/InkTranscription.tsx b/src/client/views/InkTranscription.tsx index e14a933a0..dd365e1ab 100644 --- a/src/client/views/InkTranscription.tsx +++ b/src/client/views/InkTranscription.tsx @@ -1,7 +1,7 @@ import * as iink from 'iink-js'; import { action, observable } from 'mobx'; import * as React from 'react'; -import { Doc, DocListCast } from '../../fields/Doc'; +import { DataSym, Doc, DocListCast } from '../../fields/Doc'; import { InkData, InkField } from "../../fields/InkField"; import { Cast, DateCast, NumCast } from '../../fields/Types'; import { DocumentType } from "../documents/DocumentTypes"; @@ -184,6 +184,38 @@ export class InkTranscription extends React.Component { if (exports['application/vnd.myscript.jiix']) { this.lastJiix = JSON.parse(exports['application/vnd.myscript.jiix']); // map timestamp to strokes + const timestampWord = new Map(); + this.lastJiix.words.map((word: any) => { + word.items.map((i: any) => { + const ms = Date.parse(i.timestamp); + timestampWord.set(ms, word.label); + }) + }) + + + const wordInkDocMap = new Map(); + if (this.currGroup) { + const docList = DocListCast(this.currGroup.data) + docList.forEach((inkDoc: Doc) => { + const ms = DateCast(inkDoc.creationDate).getDate().getTime(); + const word = timestampWord.get(ms); + if (!word) { + return; + } + const entry = wordInkDocMap.get(word); + if (entry) { + entry.push(inkDoc); + wordInkDocMap.set(word, entry); + } else { + const newEntry = [inkDoc]; + wordInkDocMap.set(word, newEntry); + } + + }); + console.log("getting here"); + console.log(wordInkDocMap); + + } // we can iterate through docs and for each doc index into the map by timestamp // final mapping should be word to inkDoc -- cgit v1.2.3-70-g09d2 From 45a753e0d6e812cd5653e9f0343bda5a1f231157 Mon Sep 17 00:00:00 2001 From: Naafiyan Ahmed Date: Mon, 2 May 2022 14:31:28 -0400 Subject: mapping working for ink strokes --- src/client/views/InkTranscription.tsx | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) (limited to 'src/client/views/InkTranscription.tsx') diff --git a/src/client/views/InkTranscription.tsx b/src/client/views/InkTranscription.tsx index dd365e1ab..b1f807bdf 100644 --- a/src/client/views/InkTranscription.tsx +++ b/src/client/views/InkTranscription.tsx @@ -192,12 +192,12 @@ export class InkTranscription extends React.Component { }) }) - const wordInkDocMap = new Map(); if (this.currGroup) { const docList = DocListCast(this.currGroup.data) docList.forEach((inkDoc: Doc) => { - const ms = DateCast(inkDoc.creationDate).getDate().getTime(); + // just having the times match up and be a unique value (actual timestamp doesn't matter) + const ms = DateCast(inkDoc.creationDate).getDate().getTime() + 14400000; const word = timestampWord.get(ms); if (!word) { return; @@ -212,8 +212,6 @@ export class InkTranscription extends React.Component { } }); - console.log("getting here"); - console.log(wordInkDocMap); } // we can iterate through docs and for each doc index into the map by timestamp -- cgit v1.2.3-70-g09d2 From 8147fd7d80746114a2245237e4dab2c89e2d49a2 Mon Sep 17 00:00:00 2001 From: Naafiyan Ahmed Date: Mon, 2 May 2022 16:27:49 -0400 Subject: working on getting subgroups to work --- src/client/views/InkTranscription.tsx | 87 ++++++++++++++++++++++++--- src/client/views/nodes/button/FontIconBox.tsx | 4 +- 2 files changed, 82 insertions(+), 9 deletions(-) (limited to 'src/client/views/InkTranscription.tsx') diff --git a/src/client/views/InkTranscription.tsx b/src/client/views/InkTranscription.tsx index b1f807bdf..f7260c5b4 100644 --- a/src/client/views/InkTranscription.tsx +++ b/src/client/views/InkTranscription.tsx @@ -1,13 +1,14 @@ import * as iink from 'iink-js'; import { action, observable } from 'mobx'; import * as React from 'react'; -import { DataSym, Doc, DocListCast } from '../../fields/Doc'; +import { DataSym, Doc, DocListCast, HeightSym, WidthSym } from '../../fields/Doc'; import { InkData, InkField } from "../../fields/InkField"; import { Cast, DateCast, NumCast } from '../../fields/Types'; import { DocumentType } from "../documents/DocumentTypes"; import './InkTranscription.scss'; import { aggregateBounds, Utils } from '../../Utils'; import { timesSeries } from 'async'; +import { CollectionFreeFormView } from './collections/collectionFreeForm'; export class InkTranscription extends React.Component { static Instance: InkTranscription; @@ -106,9 +107,12 @@ export class InkTranscription extends React.Component { return this._textRef = r; } - transcribeInk = (groupDoc: Doc | undefined, containingLayout: Doc, inkDocs: Doc[], math: boolean) => { + transcribeInk = (groupDoc: Doc | undefined, containingLayout: Doc, inkDocs: Doc[], math: boolean, ffView?: CollectionFreeFormView) => { if (!groupDoc) return; this.inkDocs = inkDocs; + if (ffView) { + this.ffView = ffView; + } const validInks = inkDocs.filter(s => s.type === DocumentType.INK); @@ -165,6 +169,69 @@ export class InkTranscription extends React.Component { return max - min; } + subgroupsTranscriptions = (wordInkDocMap: Map) => { + // TODO: nda - will probably want to go through ffView unprocessed docs and then see if any of the inksToGroup docs are in it and only use those + // loop through the words in wordInkDocMap + // for each word, get the inkDocs + + // iterate through the keys of wordInkDocMap + wordInkDocMap.forEach((inkDocs: Doc[], word: string) => { + const selected = wordInkDocMap.get(word); + if (!selected) { + return; + } + // loop through selected an get the bound + const bounds: { x: number, y: number, width?: number, height?: number }[] = [] + + selected.map(action(d => { + const x = NumCast(d.x); + const y = NumCast(d.y); + const width = d[WidthSym](); + const height = d[HeightSym](); + bounds.push({ x, y, width, height }); + })) + + const aggregBounds = aggregateBounds(bounds, 0, 0); + const marqViewRef = this.ffView._marqueeViewRef.current; + + // set the vals for bounds in marqueeView + if (marqViewRef) { + marqViewRef._downX = aggregBounds.x; + marqViewRef._downY = aggregBounds.y; + marqViewRef._lastX = aggregBounds.r; + marqViewRef._lastY = aggregBounds.b; + } + + selected.map(action(d => { + const dx = NumCast(d.x); + const dy = NumCast(d.y); + delete d.x; + delete d.y; + delete d.activeFrame; + delete d._timecodeToShow; // bcz: this should be automatic somehow.. along with any other properties that were logically associated with the original collection + delete d._timecodeToHide; // bcz: this should be automatic somehow.. along with any other properties that were logically associated with the original collection + // calculate pos based on bounds + if (marqViewRef?.Bounds) { + d.x = dx - marqViewRef.Bounds.left - marqViewRef.Bounds.width / 2; + d.y = dy - marqViewRef.Bounds.top - marqViewRef.Bounds.height / 2; + } + return d; + })); + this.ffView.props.removeDocument?.(selected); + // TODO: nda - this is the code to actually get a new grouped collection + const newCollection = marqViewRef?.getCollection(selected, undefined, [], true); + if (newCollection) { + newCollection.height = newCollection[HeightSym](); + newCollection.width = newCollection[WidthSym](); + newCollection.title = word; + } + + + // nda - bug: when deleting a stroke before leaving writing mode, delete the stroke from unprocessed ink docs + newCollection && this.ffView.props.addDocument?.(newCollection); + }); + } + exportInk = (e: any, ref: any) => { const exports = e.detail.exports; console.log(e); @@ -186,10 +253,12 @@ export class InkTranscription extends React.Component { // map timestamp to strokes const timestampWord = new Map(); this.lastJiix.words.map((word: any) => { - word.items.map((i: any) => { - const ms = Date.parse(i.timestamp); - timestampWord.set(ms, word.label); - }) + if (word.items) { + word.items.forEach((i: {id: string, timestamp: string, X: Array, Y: Array, F: Array}) => { + const ms = Date.parse(i.timestamp); + timestampWord.set(ms, word.label); + }) + } }) const wordInkDocMap = new Map(); @@ -210,8 +279,12 @@ export class InkTranscription extends React.Component { const newEntry = [inkDoc]; wordInkDocMap.set(word, newEntry); } - }); + // do the fun marq stuff + // generate subgroups + + console.log(wordInkDocMap); + this.subgroupsTranscriptions(wordInkDocMap); } // we can iterate through docs and for each doc index into the map by timestamp diff --git a/src/client/views/nodes/button/FontIconBox.tsx b/src/client/views/nodes/button/FontIconBox.tsx index 1c2a22226..d87b23d86 100644 --- a/src/client/views/nodes/button/FontIconBox.tsx +++ b/src/client/views/nodes/button/FontIconBox.tsx @@ -711,7 +711,7 @@ export function checkInksToGroup() { } } -export function createInkGroup(inksToGroup?: Doc[]) { +export function createInkGroup(inksToGroup?: Doc[], isSubGroup?: boolean) { // TODO nda - if document being added to is a inkGrouping then we can just add to that group if (CurrentUserUtils.SelectedTool === InkTool.Write) { CollectionFreeFormView.collectionsWithUnprocessedInk.forEach(ffView => { @@ -767,7 +767,7 @@ export function createInkGroup(inksToGroup?: Doc[]) { // TODO: nda - will probably need to go through and only remove the unprocessed selected docs ffView.unprocessedDocs = []; - InkTranscription.Instance.transcribeInk(newCollection, ffView.layoutDoc, selected, false); + InkTranscription.Instance.transcribeInk(newCollection, ffView.layoutDoc, selected, false, ffView); }); } CollectionFreeFormView.collectionsWithUnprocessedInk.clear(); -- cgit v1.2.3-70-g09d2 From 3d75e7e6bf95f3545edc959fb7eb296ec0ec8c61 Mon Sep 17 00:00:00 2001 From: Naafiyan Ahmed Date: Mon, 2 May 2022 17:09:23 -0400 Subject: working on getting subgroups to work --- src/client/views/InkTranscription.tsx | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) (limited to 'src/client/views/InkTranscription.tsx') diff --git a/src/client/views/InkTranscription.tsx b/src/client/views/InkTranscription.tsx index f7260c5b4..3d505878c 100644 --- a/src/client/views/InkTranscription.tsx +++ b/src/client/views/InkTranscription.tsx @@ -176,11 +176,12 @@ export class InkTranscription extends React.Component { // iterate through the keys of wordInkDocMap wordInkDocMap.forEach((inkDocs: Doc[], word: string) => { - const selected = wordInkDocMap.get(word); + const selected = inkDocs.slice(); if (!selected) { return; } - // loop through selected an get the bound + const marqViewRef = this.ffView._marqueeViewRef.current; + // loop through selected an get the bound const bounds: { x: number, y: number, width?: number, height?: number }[] = [] selected.map(action(d => { @@ -192,7 +193,6 @@ export class InkTranscription extends React.Component { })) const aggregBounds = aggregateBounds(bounds, 0, 0); - const marqViewRef = this.ffView._marqueeViewRef.current; // set the vals for bounds in marqueeView if (marqViewRef) { @@ -225,8 +225,6 @@ export class InkTranscription extends React.Component { newCollection.width = newCollection[WidthSym](); newCollection.title = word; } - - // nda - bug: when deleting a stroke before leaving writing mode, delete the stroke from unprocessed ink docs newCollection && this.ffView.props.addDocument?.(newCollection); }); -- cgit v1.2.3-70-g09d2 From 14b23094ea3050b96dc6aed28b5f35612c821140 Mon Sep 17 00:00:00 2001 From: Naafiyan Ahmed Date: Mon, 2 May 2022 23:16:33 -0400 Subject: got subgrouping to work naively --- src/client/views/InkTranscription.tsx | 36 +++++++++++++++++++++++++++-------- 1 file changed, 28 insertions(+), 8 deletions(-) (limited to 'src/client/views/InkTranscription.tsx') diff --git a/src/client/views/InkTranscription.tsx b/src/client/views/InkTranscription.tsx index 3d505878c..7e26b51a0 100644 --- a/src/client/views/InkTranscription.tsx +++ b/src/client/views/InkTranscription.tsx @@ -1,7 +1,7 @@ import * as iink from 'iink-js'; import { action, observable } from 'mobx'; import * as React from 'react'; -import { DataSym, Doc, DocListCast, HeightSym, WidthSym } from '../../fields/Doc'; +import { DataSym, Doc, DocListCast, HeightSym, WidthSym, LayoutSym } from '../../fields/Doc'; import { InkData, InkField } from "../../fields/InkField"; import { Cast, DateCast, NumCast } from '../../fields/Types'; import { DocumentType } from "../documents/DocumentTypes"; @@ -9,6 +9,8 @@ import './InkTranscription.scss'; import { aggregateBounds, Utils } from '../../Utils'; import { timesSeries } from 'async'; import { CollectionFreeFormView } from './collections/collectionFreeForm'; +import { DocumentManager } from "../util/DocumentManager"; + export class InkTranscription extends React.Component { static Instance: InkTranscription; @@ -169,18 +171,29 @@ export class InkTranscription extends React.Component { return max - min; } - subgroupsTranscriptions = (wordInkDocMap: Map) => { + subgroupsTranscriptions = async (wordInkDocMap: Map) => { // TODO: nda - will probably want to go through ffView unprocessed docs and then see if any of the inksToGroup docs are in it and only use those // loop through the words in wordInkDocMap // for each word, get the inkDocs // iterate through the keys of wordInkDocMap - wordInkDocMap.forEach((inkDocs: Doc[], word: string) => { + wordInkDocMap.forEach(async (inkDocs: Doc[], word: string) => { const selected = inkDocs.slice(); if (!selected) { return; } - const marqViewRef = this.ffView._marqueeViewRef.current; + console.log("selected[0].context", DocListCast(selected[0].context)); + console.log("selected[0]", DocListCast(selected[0])); + // TODO: nda - probably have to cast this to an actual Doc + const ctx = await Cast(selected[0].context, Doc); + if (!ctx) { + return; + } + const docView: CollectionFreeFormView = DocumentManager.Instance.getDocumentView(ctx)?.ComponentView as CollectionFreeFormView; + + if (!docView) return; + const marqViewRef = docView._marqueeViewRef.current; + if (!marqViewRef) return; // loop through selected an get the bound const bounds: { x: number, y: number, width?: number, height?: number }[] = [] @@ -193,8 +206,9 @@ export class InkTranscription extends React.Component { })) const aggregBounds = aggregateBounds(bounds, 0, 0); + const boundsWidth = Math.abs(aggregBounds.x - aggregBounds.r); + const boundsHeight = Math.abs(aggregBounds.y - aggregBounds.b); - // set the vals for bounds in marqueeView if (marqViewRef) { marqViewRef._downX = aggregBounds.x; marqViewRef._downY = aggregBounds.y; @@ -202,6 +216,8 @@ export class InkTranscription extends React.Component { marqViewRef._lastY = aggregBounds.b; } + // set the vals for bounds in marqueeView + selected.map(action(d => { const dx = NumCast(d.x); const dy = NumCast(d.y); @@ -211,14 +227,17 @@ export class InkTranscription extends React.Component { delete d._timecodeToShow; // bcz: this should be automatic somehow.. along with any other properties that were logically associated with the original collection delete d._timecodeToHide; // bcz: this should be automatic somehow.. along with any other properties that were logically associated with the original collection // calculate pos based on bounds + // d.x = dx - aggregBounds.x - boundsWidth / 2; + // d.y = dy - aggregBounds.y - boundsHeight / 2; if (marqViewRef?.Bounds) { d.x = dx - marqViewRef.Bounds.left - marqViewRef.Bounds.width / 2; d.y = dy - marqViewRef.Bounds.top - marqViewRef.Bounds.height / 2; } return d; })); - this.ffView.props.removeDocument?.(selected); - // TODO: nda - this is the code to actually get a new grouped collection + + docView.props.removeDocument?.(selected); + // // TODO: nda - this is the code to actually get a new grouped collection const newCollection = marqViewRef?.getCollection(selected, undefined, [], true); if (newCollection) { newCollection.height = newCollection[HeightSym](); @@ -226,7 +245,8 @@ export class InkTranscription extends React.Component { newCollection.title = word; } // nda - bug: when deleting a stroke before leaving writing mode, delete the stroke from unprocessed ink docs - newCollection && this.ffView.props.addDocument?.(newCollection); + console.log(newCollection); + newCollection && docView.props.addDocument?.(newCollection); }); } -- cgit v1.2.3-70-g09d2 From f66eefce84ce6a7c8c1fdf37f682238e65576ff6 Mon Sep 17 00:00:00 2001 From: Naafiyan Ahmed Date: Tue, 3 May 2022 16:32:13 -0400 Subject: working on subgrouping --- src/client/views/InkTranscription.tsx | 6 ------ 1 file changed, 6 deletions(-) (limited to 'src/client/views/InkTranscription.tsx') diff --git a/src/client/views/InkTranscription.tsx b/src/client/views/InkTranscription.tsx index 7e26b51a0..fa753760a 100644 --- a/src/client/views/InkTranscription.tsx +++ b/src/client/views/InkTranscription.tsx @@ -182,8 +182,6 @@ export class InkTranscription extends React.Component { if (!selected) { return; } - console.log("selected[0].context", DocListCast(selected[0].context)); - console.log("selected[0]", DocListCast(selected[0])); // TODO: nda - probably have to cast this to an actual Doc const ctx = await Cast(selected[0].context, Doc); if (!ctx) { @@ -206,8 +204,6 @@ export class InkTranscription extends React.Component { })) const aggregBounds = aggregateBounds(bounds, 0, 0); - const boundsWidth = Math.abs(aggregBounds.x - aggregBounds.r); - const boundsHeight = Math.abs(aggregBounds.y - aggregBounds.b); if (marqViewRef) { marqViewRef._downX = aggregBounds.x; @@ -227,8 +223,6 @@ export class InkTranscription extends React.Component { delete d._timecodeToShow; // bcz: this should be automatic somehow.. along with any other properties that were logically associated with the original collection delete d._timecodeToHide; // bcz: this should be automatic somehow.. along with any other properties that were logically associated with the original collection // calculate pos based on bounds - // d.x = dx - aggregBounds.x - boundsWidth / 2; - // d.y = dy - aggregBounds.y - boundsHeight / 2; if (marqViewRef?.Bounds) { d.x = dx - marqViewRef.Bounds.left - marqViewRef.Bounds.width / 2; d.y = dy - marqViewRef.Bounds.top - marqViewRef.Bounds.height / 2; -- cgit v1.2.3-70-g09d2 From 48b4af3448d87b7a0ce8aa82a903c07c8b78c2b7 Mon Sep 17 00:00:00 2001 From: Naafiyan Ahmed Date: Tue, 3 May 2022 16:53:59 -0400 Subject: removed unecessary async --- src/client/views/InkTranscription.tsx | 12 +----------- 1 file changed, 1 insertion(+), 11 deletions(-) (limited to 'src/client/views/InkTranscription.tsx') diff --git a/src/client/views/InkTranscription.tsx b/src/client/views/InkTranscription.tsx index fa753760a..9b8e822d7 100644 --- a/src/client/views/InkTranscription.tsx +++ b/src/client/views/InkTranscription.tsx @@ -171,8 +171,7 @@ export class InkTranscription extends React.Component { return max - min; } - subgroupsTranscriptions = async (wordInkDocMap: Map) => { - // TODO: nda - will probably want to go through ffView unprocessed docs and then see if any of the inksToGroup docs are in it and only use those + subgroupsTranscriptions = (wordInkDocMap: Map) => { // loop through the words in wordInkDocMap // for each word, get the inkDocs @@ -231,7 +230,6 @@ export class InkTranscription extends React.Component { })); docView.props.removeDocument?.(selected); - // // TODO: nda - this is the code to actually get a new grouped collection const newCollection = marqViewRef?.getCollection(selected, undefined, [], true); if (newCollection) { newCollection.height = newCollection[HeightSym](); @@ -292,16 +290,8 @@ export class InkTranscription extends React.Component { wordInkDocMap.set(word, newEntry); } }); - // do the fun marq stuff - // generate subgroups - - console.log(wordInkDocMap); this.subgroupsTranscriptions(wordInkDocMap); - } - // we can iterate through docs and for each doc index into the map by timestamp - - // final mapping should be word to inkDoc } const text = exports['text/plain']; // console.log(text); -- cgit v1.2.3-70-g09d2 From ac0e05eefa24aad2d90f661a33ede9c65c0fa33c Mon Sep 17 00:00:00 2001 From: mehekj Date: Tue, 3 May 2022 17:06:35 -0400 Subject: fixed make text note from ink transcription with word subgroupings --- src/client/views/InkTranscription.tsx | 10 +++++----- .../collections/collectionFreeForm/CollectionFreeFormView.tsx | 8 ++++---- 2 files changed, 9 insertions(+), 9 deletions(-) (limited to 'src/client/views/InkTranscription.tsx') diff --git a/src/client/views/InkTranscription.tsx b/src/client/views/InkTranscription.tsx index 7e26b51a0..3892f4cbb 100644 --- a/src/client/views/InkTranscription.tsx +++ b/src/client/views/InkTranscription.tsx @@ -172,7 +172,7 @@ export class InkTranscription extends React.Component { } subgroupsTranscriptions = async (wordInkDocMap: Map) => { - // TODO: nda - will probably want to go through ffView unprocessed docs and then see if any of the inksToGroup docs are in it and only use those + // TODO: nda - will probably want to go through ffView unprocessed docs and then see if any of the inksToGroup docs are in it and only use those // loop through the words in wordInkDocMap // for each word, get the inkDocs @@ -217,7 +217,7 @@ export class InkTranscription extends React.Component { } // set the vals for bounds in marqueeView - + selected.map(action(d => { const dx = NumCast(d.x); const dy = NumCast(d.y); @@ -235,7 +235,7 @@ export class InkTranscription extends React.Component { } return d; })); - + docView.props.removeDocument?.(selected); // // TODO: nda - this is the code to actually get a new grouped collection const newCollection = marqViewRef?.getCollection(selected, undefined, [], true); @@ -272,7 +272,7 @@ export class InkTranscription extends React.Component { const timestampWord = new Map(); this.lastJiix.words.map((word: any) => { if (word.items) { - word.items.forEach((i: {id: string, timestamp: string, X: Array, Y: Array, F: Array}) => { + word.items.forEach((i: { id: string, timestamp: string, X: Array, Y: Array, F: Array }) => { const ms = Date.parse(i.timestamp); timestampWord.set(ms, word.label); }) @@ -318,7 +318,7 @@ export class InkTranscription extends React.Component { if (this.currGroup) { console.log("curr grouping"); - this.currGroup.text = text; + this.currGroup.transcription = text; this.currGroup.title = text.split("\n")[0]; } diff --git a/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx b/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx index 3a5911fc9..ca8073dac 100644 --- a/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx +++ b/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx @@ -1511,9 +1511,9 @@ export class CollectionFreeFormView extends CollectionSubView s.type === DocumentType.INK).length > 0 && appearanceItems.push({ description: "Ink to text", event: () => this.transcribeStrokes(false), icon: "font" }); + this.props.Document._isGroup && this.Document.transcription && appearanceItems.push({ description: "Ink to text", event: () => this.transcribeStrokes(false), icon: "font" }); - this.props.Document._isGroup && this.childDocs.filter(s => s.type === DocumentType.INK).length > 0 && appearanceItems.push({ description: "Ink to math", event: () => this.transcribeStrokes(true), icon: "square-root-alt" }); + // this.props.Document._isGroup && this.childDocs.filter(s => s.type === DocumentType.INK).length > 0 && appearanceItems.push({ description: "Ink to math", event: () => this.transcribeStrokes(true), icon: "square-root-alt" }); !Doc.UserDoc().noviceMode ? appearanceItems.push({ description: "Arrange contents in grid", event: this.layoutDocsInGrid, icon: "table" }) : null; !appearance && ContextMenu.Instance.addItem({ description: "Appearance...", subitems: appearanceItems, icon: "eye" }); @@ -1577,9 +1577,9 @@ export class CollectionFreeFormView extends CollectionSubView { - if (this.props.Document._isGroup && this.props.Document.text) { + if (this.props.Document._isGroup && this.props.Document.transcription) { if (!math) { - const text = StrCast(this.props.Document.text); + const text = StrCast(this.props.Document.transcription); const lines = text.split("\n"); const height = 30 + 15 * lines.length; -- cgit v1.2.3-70-g09d2 From f484500cb96213f6291eb3977a614a672d778441 Mon Sep 17 00:00:00 2001 From: Naafiyan Ahmed Date: Wed, 4 May 2022 13:33:09 -0400 Subject: fix: single words no longer disappear --- src/client/views/InkTranscription.tsx | 17 +++-------------- 1 file changed, 3 insertions(+), 14 deletions(-) (limited to 'src/client/views/InkTranscription.tsx') diff --git a/src/client/views/InkTranscription.tsx b/src/client/views/InkTranscription.tsx index ca0f9792d..0e8ccd57b 100644 --- a/src/client/views/InkTranscription.tsx +++ b/src/client/views/InkTranscription.tsx @@ -1,13 +1,12 @@ import * as iink from 'iink-js'; import { action, observable } from 'mobx'; import * as React from 'react'; -import { DataSym, Doc, DocListCast, HeightSym, WidthSym, LayoutSym } from '../../fields/Doc'; +import { Doc, DocListCast, HeightSym, WidthSym } from '../../fields/Doc'; import { InkData, InkField } from "../../fields/InkField"; import { Cast, DateCast, NumCast } from '../../fields/Types'; import { DocumentType } from "../documents/DocumentTypes"; import './InkTranscription.scss'; -import { aggregateBounds, Utils } from '../../Utils'; -import { timesSeries } from 'async'; +import { aggregateBounds } from '../../Utils'; import { CollectionFreeFormView } from './collections/collectionFreeForm'; import { DocumentManager } from "../util/DocumentManager"; @@ -21,17 +20,12 @@ export class InkTranscription extends React.Component { @observable _textRef: any; private lastJiix: any; private currGroup?: Doc; - private wordsBoundMapping: any; - private inkDocs: Doc[]; - private ffView: any; private containingLayout?: Doc; constructor(props: Readonly<{}>) { super(props); InkTranscription.Instance = this; - this.wordsBoundMapping = new Map(); - this.inkDocs = new Array(); } componentWillUnmount() { @@ -111,11 +105,6 @@ export class InkTranscription extends React.Component { transcribeInk = (groupDoc: Doc | undefined, containingLayout: Doc, inkDocs: Doc[], math: boolean, ffView?: CollectionFreeFormView) => { if (!groupDoc) return; - this.inkDocs = inkDocs; - if (ffView) { - this.ffView = ffView; - } - const validInks = inkDocs.filter(s => s.type === DocumentType.INK); const strokes: InkData[] = []; @@ -290,7 +279,7 @@ export class InkTranscription extends React.Component { wordInkDocMap.set(word, newEntry); } }); - this.subgroupsTranscriptions(wordInkDocMap); + if (this.lastJiix.words.length > 1) this.subgroupsTranscriptions(wordInkDocMap); } } const text = exports['text/plain']; -- cgit v1.2.3-70-g09d2 From 33fc3811e758441e7344706004feba1599d8d833 Mon Sep 17 00:00:00 2001 From: mehekj Date: Wed, 4 May 2022 13:43:28 -0400 Subject: remove scaling (for now) --- src/client/views/InkTranscription.tsx | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'src/client/views/InkTranscription.tsx') diff --git a/src/client/views/InkTranscription.tsx b/src/client/views/InkTranscription.tsx index 0e8ccd57b..ef4510f38 100644 --- a/src/client/views/InkTranscription.tsx +++ b/src/client/views/InkTranscription.tsx @@ -142,8 +142,8 @@ export class InkTranscription extends React.Component { return { "pointerType": "PEN", "pointerId": 1, - "x": stroke.map(point => (point.X - panX) * scale), - "y": stroke.map(point => (point.Y - panY) * scale), + "x": stroke.map(point => point.X), + "y": stroke.map(point => point.Y), "t": new Array(stroke.length).fill(time), "p": new Array(stroke.length).fill(1.0) }; -- cgit v1.2.3-70-g09d2 From 4f237303dbc5398cc0f7de8e15901e1b87d57adb Mon Sep 17 00:00:00 2001 From: mehekj Date: Wed, 4 May 2022 23:10:48 -0400 Subject: use ink screen space pts for transcription --- src/client/views/InkTranscription.tsx | 25 ++++++++++--------------- src/client/views/InkingStroke.tsx | 1 + 2 files changed, 11 insertions(+), 15 deletions(-) (limited to 'src/client/views/InkTranscription.tsx') diff --git a/src/client/views/InkTranscription.tsx b/src/client/views/InkTranscription.tsx index ef4510f38..5e0667bed 100644 --- a/src/client/views/InkTranscription.tsx +++ b/src/client/views/InkTranscription.tsx @@ -9,6 +9,7 @@ import './InkTranscription.scss'; import { aggregateBounds } from '../../Utils'; import { CollectionFreeFormView } from './collections/collectionFreeForm'; import { DocumentManager } from "../util/DocumentManager"; +import { InkingStroke } from './InkingStroke'; export class InkTranscription extends React.Component { @@ -109,12 +110,14 @@ export class InkTranscription extends React.Component { const strokes: InkData[] = []; const times: number[] = []; - console.log(validInks); + // console.log(validInks); validInks.filter(i => Cast(i.data, InkField)).forEach(i => { const d = Cast(i.data, InkField, null); - const left = Math.min(...d?.inkData.map(pd => pd.X) ?? [0]); - const top = Math.min(...d?.inkData.map(pd => pd.Y) ?? [0]); - strokes.push(d.inkData.map(pd => ({ X: pd.X + NumCast(i.x) - left, Y: pd.Y + NumCast(i.y) - top }))); + // const left = Math.min(...d?.inkData.map(pd => pd.X) ?? [0]); + // const top = Math.min(...d?.inkData.map(pd => pd.Y) ?? [0]); + // strokes.push(d.inkData.map(pd => ({ X: pd.X + NumCast(i.x) - left, Y: pd.Y + NumCast(i.y) - top }))); + const inkStroke = DocumentManager.Instance.getDocumentView(i)?.ComponentView as InkingStroke; + strokes.push(d.inkData.map(pd => (inkStroke.ptToScreen({ X: pd.X, Y: pd.Y })))); times.push(DateCast(i.creationDate).getDate().getTime()); }); @@ -135,10 +138,6 @@ export class InkTranscription extends React.Component { } inkJSON = (stroke: InkData, time: number) => { - const scale = NumCast(this.containingLayout?._viewScale, 1); - const panX = NumCast(this.containingLayout?._panX); - const panY = NumCast(this.containingLayout?._panY); - return { "pointerType": "PEN", "pointerId": 1, @@ -226,14 +225,14 @@ export class InkTranscription extends React.Component { newCollection.title = word; } // nda - bug: when deleting a stroke before leaving writing mode, delete the stroke from unprocessed ink docs - console.log(newCollection); + // console.log(newCollection); newCollection && docView.props.addDocument?.(newCollection); }); } exportInk = (e: any, ref: any) => { const exports = e.detail.exports; - console.log(e); + // console.log(e); if (exports) { if (exports['application/x-latex']) { const latex = exports['application/x-latex']; @@ -285,12 +284,8 @@ export class InkTranscription extends React.Component { const text = exports['text/plain']; // console.log(text); - if (exports['application/vnd.myscript.jiix']) { - console.log(JSON.parse(exports['application/vnd.myscript.jiix'])); - } - if (this.currGroup) { - console.log("curr grouping"); + // console.log("curr grouping"); this.currGroup.transcription = text; this.currGroup.title = text.split("\n")[0]; } diff --git a/src/client/views/InkingStroke.tsx b/src/client/views/InkingStroke.tsx index 40fe6aa68..6c95f0f77 100644 --- a/src/client/views/InkingStroke.tsx +++ b/src/client/views/InkingStroke.tsx @@ -253,6 +253,7 @@ export class InkingStroke extends ViewBoxBaseComponent() { (point.X - inkLeft - inkStrokeWidth / 2) * inkScaleX + inkStrokeWidth / 2, (point.Y - inkTop - inkStrokeWidth / 2) * inkScaleY + inkStrokeWidth / 2)).map(p => ({ X: p[0], Y: p[1] })); const screenHdlPts = screenPts; + console.log(screenPts); const startMarker = StrCast(this.layoutDoc.strokeStartMarker); const endMarker = StrCast(this.layoutDoc.strokeEndMarker); -- cgit v1.2.3-70-g09d2