diff options
| -rw-r--r-- | package-lock.json | 79 | ||||
| -rw-r--r-- | package.json | 1 | ||||
| -rw-r--r-- | src/client/documents/Documents.ts | 5 | ||||
| -rw-r--r-- | src/client/views/GestureOverlay.tsx | 16 | ||||
| -rw-r--r-- | src/client/views/InkTranscription.scss | 5 | ||||
| -rw-r--r-- | src/client/views/InkTranscription.tsx | 188 | ||||
| -rw-r--r-- | src/client/views/MainView.tsx | 4 | ||||
| -rw-r--r-- | src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx | 19 | ||||
| -rw-r--r-- | src/client/views/collections/collectionFreeForm/MarqueeOptionsMenu.tsx | 11 | ||||
| -rw-r--r-- | src/client/views/collections/collectionFreeForm/MarqueeView.tsx | 16 | ||||
| -rw-r--r-- | src/client/views/nodes/EquationBox.scss | 36 | ||||
| -rw-r--r-- | src/client/views/nodes/EquationBox.tsx | 91 | ||||
| -rw-r--r-- | src/fields/InkField.ts | 3 | ||||
| -rw-r--r-- | src/pen-gestures/GestureUtils.ts | 3 | ||||
| -rw-r--r-- | src/typings/index.d.ts | 1 |
15 files changed, 444 insertions, 34 deletions
diff --git a/package-lock.json b/package-lock.json index b1431e55b..1f8b1b8f6 100644 --- a/package-lock.json +++ b/package-lock.json @@ -4579,6 +4579,16 @@ "resolved": "https://registry.npmjs.org/cli-boxes/-/cli-boxes-1.0.0.tgz", "integrity": "sha1-T6kXw+WclKAEzWH47lCdplFocUM=" }, + "clipboard": { + "version": "1.7.1", + "resolved": "https://registry.npmjs.org/clipboard/-/clipboard-1.7.1.tgz", + "integrity": "sha1-Ng1taUbpmnof7zleQrqStem1oWs=", + "requires": { + "good-listener": "^1.2.2", + "select": "^1.1.2", + "tiny-emitter": "^2.0.0" + } + }, "cliss": { "version": "0.0.2", "resolved": "https://registry.npmjs.org/cliss/-/cliss-0.0.2.tgz", @@ -5307,6 +5317,11 @@ } } }, + "crypto-js": { + "version": "3.3.0", + "resolved": "https://registry.npmjs.org/crypto-js/-/crypto-js-3.3.0.tgz", + "integrity": "sha512-DIT51nX0dCfKltpRiXV+/TVZq+Qq2NgF4644+K7Ttnla7zEzqc+kjJyiB96BHNyUTBxyjzRcZYpUdZa+QAqi6Q==" + }, "crypto-random-string": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/crypto-random-string/-/crypto-random-string-1.0.0.tgz", @@ -5885,6 +5900,11 @@ "resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz", "integrity": "sha1-3zrhmayt+31ECqrgsp4icrJOxhk=" }, + "delegate": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/delegate/-/delegate-3.2.0.tgz", + "integrity": "sha512-IofjkYBZaZivn0V8nnsMJGBr4jVLxHDheKSW88PyxS5QC4Vo9ZbZVvhzlSxY87fVq3STR6r+4cGepyHkcWOQSw==" + }, "delegates": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/delegates/-/delegates-1.0.0.tgz", @@ -7846,6 +7866,14 @@ "jquery": "*" } }, + "good-listener": { + "version": "1.2.2", + "resolved": "https://registry.npmjs.org/good-listener/-/good-listener-1.2.2.tgz", + "integrity": "sha1-1TswzfkxPf+33JoNR3CWqm0UXFA=", + "requires": { + "delegate": "^3.1.2" + } + }, "google-auth-library": { "version": "4.2.6", "resolved": "https://registry.npmjs.org/google-auth-library/-/google-auth-library-4.2.6.tgz", @@ -8462,6 +8490,29 @@ "resolved": "https://registry.npmjs.org/ignore-by-default/-/ignore-by-default-1.0.1.tgz", "integrity": "sha1-SMptcvbGo68Aqa1K5odr44ieKwk=" }, + "iink-js": { + "version": "1.5.4", + "resolved": "https://registry.npmjs.org/iink-js/-/iink-js-1.5.4.tgz", + "integrity": "sha512-WcjMmyXbSo4GY56AVVTKRyxlguh2KAGFyoH24+Zmm87ZWII12or2uLiovAyOK4IE+VVz7m0U5RMuv8i/RV/3Nw==", + "requires": { + "@babel/runtime": "^7.9.2", + "clipboard": "^1.7.1", + "crypto-js": "^3.3.0", + "d3-selection": "^1.4.1", + "json-css": "^1.5.6", + "lodash.merge": "^4.6.2", + "loglevel": "^1.6.8", + "perfect-scrollbar": "^1.5.0", + "uuid-js": "^0.7.5" + }, + "dependencies": { + "d3-selection": { + "version": "1.4.2", + "resolved": "https://registry.npmjs.org/d3-selection/-/d3-selection-1.4.2.tgz", + "integrity": "sha512-SJ0BqYihzOjDnnlfyeHT0e30k0K1+5sR3d5fNueCNeuhZTnGw4M4o8mqJchSwgKMXCNFo+e2VTChiSJ0vYtXkg==" + } + } + }, "image-data-uri": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/image-data-uri/-/image-data-uri-2.0.1.tgz", @@ -9524,6 +9575,11 @@ "resolved": "https://registry.npmjs.org/json-buffer/-/json-buffer-3.0.1.tgz", "integrity": "sha512-4bV5BfR2mqfQTJm+V5tPPdf+ZpuhiIvTuAB5g8kcrXOZpTT/QwwVRWBywX1ozr6lEuPdbHxwaJlm9G6mI2sfSQ==" }, + "json-css": { + "version": "1.5.6", + "resolved": "https://registry.npmjs.org/json-css/-/json-css-1.5.6.tgz", + "integrity": "sha1-65ZPg0ouTqobwvaY/12wB6JsfAA=" + }, "json-parse-better-errors": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/json-parse-better-errors/-/json-parse-better-errors-1.0.2.tgz", @@ -9996,8 +10052,7 @@ "loglevel": { "version": "1.6.8", "resolved": "https://registry.npmjs.org/loglevel/-/loglevel-1.6.8.tgz", - "integrity": "sha512-bsU7+gc9AJ2SqpzxwU3+1fedl8zAntbtC5XYlt3s2j1hJcn2PsXSmgN8TaLG/J1/2mod4+cE/3vNL70/c1RNCA==", - "dev": true + "integrity": "sha512-bsU7+gc9AJ2SqpzxwU3+1fedl8zAntbtC5XYlt3s2j1hJcn2PsXSmgN8TaLG/J1/2mod4+cE/3vNL70/c1RNCA==" }, "loglevelnext": { "version": "1.0.5", @@ -14793,6 +14848,11 @@ "resolved": "https://registry.npmjs.org/pend/-/pend-1.2.0.tgz", "integrity": "sha1-elfrVQpng/kRUzH89GY9XI4AelA=" }, + "perfect-scrollbar": { + "version": "1.5.5", + "resolved": "https://registry.npmjs.org/perfect-scrollbar/-/perfect-scrollbar-1.5.5.tgz", + "integrity": "sha512-dzalfutyP3e/FOpdlhVryN4AJ5XDVauVWxybSkLZmakFE2sS3y3pc4JnSprw8tGmHvkaG5Edr5T7LBTZ+WWU2g==" + }, "performance-now": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/performance-now/-/performance-now-2.1.0.tgz", @@ -16791,6 +16851,11 @@ "resolved": "https://registry.npmjs.org/section-iterator/-/section-iterator-2.0.0.tgz", "integrity": "sha1-v0RNev7rlK1Dw5rS+yYVFifMuio=" }, + "select": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/select/-/select-1.1.2.tgz", + "integrity": "sha1-DnNQrN7ICxEIUoeG7B1EGNEbOW0=" + }, "select-hose": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/select-hose/-/select-hose-2.0.0.tgz", @@ -18208,6 +18273,11 @@ "resolved": "https://registry.npmjs.org/timed-out/-/timed-out-4.0.1.tgz", "integrity": "sha1-8y6srFoXW+ol1/q1Zas+2HQe9W8=" }, + "tiny-emitter": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/tiny-emitter/-/tiny-emitter-2.1.0.tgz", + "integrity": "sha512-NB6Dk1A9xgQPMoGqC5CVXn123gWyte215ONT5Pp5a0yt4nlEoO1ZWeCwpncaekPHXO60i47ihFnZPiRPjRMq4Q==" + }, "tiny-inflate": { "version": "1.0.3", "resolved": "https://registry.npmjs.org/tiny-inflate/-/tiny-inflate-1.0.3.tgz", @@ -19185,6 +19255,11 @@ "resolved": "https://registry.npmjs.org/uuid/-/uuid-3.4.0.tgz", "integrity": "sha512-HjSDRw6gZE5JMggctHBcjVak08+KEVhSIiDzFnT9S9aegmp85S/bReBVTb4QTFaRNptJ9kuYaNhnbNEOkbKb/A==" }, + "uuid-js": { + "version": "0.7.5", + "resolved": "https://registry.npmjs.org/uuid-js/-/uuid-js-0.7.5.tgz", + "integrity": "sha1-bIhtAqU9LUDc8l2RoXC0p7JblNA=" + }, "valid-url": { "version": "1.0.9", "resolved": "https://registry.npmjs.org/valid-url/-/valid-url-1.0.9.tgz", diff --git a/package.json b/package.json index 2c3e641dd..c8455c8a3 100644 --- a/package.json +++ b/package.json @@ -196,6 +196,7 @@ "https": "^1.0.0", "https-browserify": "^1.0.0", "i": "^0.3.7", + "iink-js": "^1.5.4", "image-data-uri": "^2.0.1", "image-size": "^0.7.5", "image-size-stream": "^1.1.0", diff --git a/src/client/documents/Documents.ts b/src/client/documents/Documents.ts index 9bcd23aa0..884cc781d 100644 --- a/src/client/documents/Documents.ts +++ b/src/client/documents/Documents.ts @@ -4,7 +4,7 @@ import { DateField } from "../../fields/DateField"; import { Doc, DocListCast, DocListCastAsync, Field, HeightSym, Initializing, Opt, updateCachedAcls, WidthSym } from "../../fields/Doc"; import { Id } from "../../fields/FieldSymbols"; import { HtmlField } from "../../fields/HtmlField"; -import { InkField } from "../../fields/InkField"; +import { InkField, PointData } from "../../fields/InkField"; import { List } from "../../fields/List"; import { ProxyField } from "../../fields/Proxy"; import { RichTextField } from "../../fields/RichTextField"; @@ -738,7 +738,7 @@ export namespace Docs { return linkDoc; } - export function InkDocument(color: string, tool: string, strokeWidth: number, strokeBezier: string, fillColor: string, arrowStart: string, arrowEnd: string, dash: string, points: { X: number, Y: number }[], options: DocumentOptions = {}) { + export function InkDocument(color: string, tool: string, strokeWidth: number, strokeBezier: string, fillColor: string, arrowStart: string, arrowEnd: string, dash: string, points: PointData[], options: DocumentOptions = {}) { const I = new Doc(); I[Initializing] = true; I.type = DocumentType.INK; @@ -1161,6 +1161,7 @@ export namespace DocUtils { created = Docs.Create.AudioDocument((field).url.href, resolved); layout = AudioBox.LayoutString; } else if (field instanceof InkField) { + console.log("Documents " + field.inkData) created = Docs.Create.InkDocument(ActiveInkColor(), CurrentUserUtils.SelectedTool, ActiveInkWidth(), ActiveInkBezierApprox(), ActiveFillColor(), ActiveArrowStart(), ActiveArrowEnd(), ActiveDash(), (field).inkData, resolved); layout = InkingStroke.LayoutString; } else if (field instanceof List && field[0] instanceof Doc) { diff --git a/src/client/views/GestureOverlay.tsx b/src/client/views/GestureOverlay.tsx index 50dca0a99..16a9cfaeb 100644 --- a/src/client/views/GestureOverlay.tsx +++ b/src/client/views/GestureOverlay.tsx @@ -41,6 +41,7 @@ export class GestureOverlay extends Touchable { @observable private _menuY: number = -300; @observable private _pointerY?: number; @observable private _points: { X: number, Y: number }[] = []; + @observable private _timeStamps: number[] = []; @observable private _strokes: InkData[] = []; @observable private _palette?: JSX.Element; @observable private _clipboardDoc?: JSX.Element; @@ -59,6 +60,7 @@ export class GestureOverlay extends Touchable { private pointerIdentifier?: number; private _hands: Map<number, React.Touch[]> = new Map<number, React.Touch[]>(); private _holdTimer: NodeJS.Timeout | undefined; + private _strokeStartTime: number = 0; protected _multiTouchDisposer?: InteractionUtils.MultiTouchEventDisposer; @@ -418,6 +420,7 @@ export class GestureOverlay extends Touchable { this._strokes = []; this._points = []; + this._timeStamps = []; this._possibilities = []; document.removeEventListener("touchend", this.handleHandUp); } @@ -493,6 +496,8 @@ export class GestureOverlay extends Touchable { onPointerDown = (e: React.PointerEvent) => { if (InteractionUtils.IsType(e, InteractionUtils.PENTYPE) || [InkTool.Highlighter, InkTool.Pen].includes(CurrentUserUtils.SelectedTool)) { this._points.push({ X: e.clientX, Y: e.clientY }); + this._timeStamps.push(0); + this._strokeStartTime = new Date().getTime(); setupMoveUpEvents(this, e, this.onPointerMove, this.onPointerUp, emptyFunction); // if (CurrentUserUtils.SelectedTool === InkTool.Highlighter) SetActiveInkColor("rgba(245, 230, 95, 0.75)"); } @@ -501,6 +506,7 @@ export class GestureOverlay extends Touchable { @action onPointerMove = (e: PointerEvent) => { this._points.push({ X: e.clientX, Y: e.clientY }); + this._timeStamps.push(new Date().getTime() - this._strokeStartTime); if (this._points.length > 1) { const B = this.svgBounds; @@ -547,6 +553,7 @@ export class GestureOverlay extends Touchable { const points = this._points.map(p => ({ X: p.X - B.left, Y: p.Y - B.top })); //push first points to so interactionUtil knows pointer is up this._points.push({ X: this._points[0].X, Y: this._points[0].Y }); + this._timeStamps.push(this._timeStamps[0]); const initialPoint = this._points[0.]; const xInGlass = initialPoint.X > (this._thumbX ?? Number.MAX_SAFE_INTEGER) && initialPoint.X < (this._thumbX ?? Number.MAX_SAFE_INTEGER) + (this.height); @@ -558,6 +565,7 @@ export class GestureOverlay extends Touchable { case ToolglassTools.InkToText: this._strokes.push(new Array(...this._points)); this._points = []; + this._timeStamps = []; CognitiveServices.Inking.Appliers.InterpretStrokes(this._strokes).then((results) => { const wordResults = results.filter((r: any) => r.category === "line"); const possibilities: string[] = []; @@ -581,6 +589,7 @@ export class GestureOverlay extends Touchable { case ToolglassTools.IgnoreGesture: this.dispatchGesture(GestureUtils.Gestures.Stroke); this._points = []; + this._timeStamps = []; break; } } @@ -589,6 +598,7 @@ export class GestureOverlay extends Touchable { this.makePolygon(this.InkShape, false); this.dispatchGesture(GestureUtils.Gestures.Stroke); this._points = []; + this._timeStamps = []; if (!CollectionFreeFormViewChrome.Instance?._keepPrimitiveMode) { this.InkShape = ""; Doc.UserDoc().activeInkTool = InkTool.None; @@ -635,9 +645,11 @@ export class GestureOverlay extends Touchable { this.dispatchGesture(GestureUtils.Gestures.Stroke); } this._points = []; + this._timeStamps = []; } } else { this._points = []; + this._timeStamps = []; } CollectionFreeFormViewChrome.Instance?.primCreated(); } @@ -686,6 +698,7 @@ export class GestureOverlay extends Touchable { } } this._points = []; + this._timeStamps = []; switch (shape) { //must push an extra point in the end so InteractionUtils knows pointer is up. //must be (points[0].X,points[0]-1) @@ -808,7 +821,8 @@ export class GestureOverlay extends Touchable { points: stroke ?? this._points, gesture: gesture as any, bounds: this.getBounds(stroke ?? this._points), - text: data + text: data, + times: this._timeStamps } } ) diff --git a/src/client/views/InkTranscription.scss b/src/client/views/InkTranscription.scss new file mode 100644 index 000000000..bbb0a1afa --- /dev/null +++ b/src/client/views/InkTranscription.scss @@ -0,0 +1,5 @@ +.ink-transcription { + .error-msg { + display: none !important; + } +}
\ No newline at end of file diff --git a/src/client/views/InkTranscription.tsx b/src/client/views/InkTranscription.tsx new file mode 100644 index 000000000..950f622dd --- /dev/null +++ b/src/client/views/InkTranscription.tsx @@ -0,0 +1,188 @@ +import { timeStamp } from 'console'; +import * as iink from 'iink-js'; +import { action, observable } from 'mobx'; +import * as React from 'react'; +import { Doc } from '../../fields/Doc'; +import { InkData, InkField } from "../../fields/InkField"; +import { Cast, NumCast, StrCast } from '../../fields/Types'; +import { Docs } from "../documents/Documents"; +import './InkTranscription.scss'; + +export class InkTranscription extends React.Component { + static Instance: InkTranscription; + + @observable _mathRegister: any; + @observable _mathRef: any; + @observable _textRegister: any; + @observable _textRef: any; + private addDocument?: (doc: Doc | Doc[]) => boolean; + private bounds: { x: number, y: number, width: number, height: number } = { x: 0, y: 0, width: 0, height: 0 }; + + constructor(props: Readonly<{}>) { + super(props); + + InkTranscription.Instance = this; + } + + componentWillUnmount() { + this._mathRef.removeEventListener('exported', (e: any) => this.exportInk(e, this._mathRef)); + this._textRef.removeEventListener('exported', (e: any) => this.exportInk(e, this._textRef)); + + this._mathRef.removeEventListener('changed', (e: any) => this.callExport(this._mathRef)); + this._textRef.removeEventListener('changed', (e: any) => this.callExport(this._textRef)); + } + + @action + setMathRef = (r: any) => { + if (!this._mathRegister) { + this._mathRegister = r ? iink.register(r, { + recognitionParams: { + type: 'MATH', + protocol: 'WEBSOCKET', + server: { + host: 'cloud.myscript.com', + applicationKey: '7277ec34-0c2e-4ee1-9757-ccb657e3f89f', + hmacKey: 'f5cb18f2-1f95-4ddb-96ac-3f7c888dffc1', + websocket: { + pingEnabled: false, + autoReconnect: true + } + }, + iink: { + math: { + mimeTypes: ['application/x-latex', 'application/vnd.myscript.jiix'] + }, + export: { + jiix: { + strokes: true + } + } + } + }, + triggers: { + exportContent: 'DEMAND' + } + }) : null; + } + + r.addEventListener('exported', (e: any) => this.exportInk(e, this._mathRef)); + r.addEventListener('changed', (e: any) => this.callExport(this._mathRef)); + + return this._mathRef = r; + } + + @action + setTextRef = (r: any) => { + if (!this._textRegister) { + this._textRegister = r ? iink.register(r, { + recognitionParams: { + type: 'TEXT', + protocol: 'WEBSOCKET', + server: { + host: 'cloud.myscript.com', + applicationKey: '7277ec34-0c2e-4ee1-9757-ccb657e3f89f', + hmacKey: 'f5cb18f2-1f95-4ddb-96ac-3f7c888dffc1', + websocket: { + pingEnabled: false, + autoReconnect: true + } + }, + iink: { + text: { + mimeTypes: ['text/plain'] + }, + export: { + jiix: { + strokes: true + } + } + } + }, + triggers: { + exportContent: 'DEMAND' + } + }) : null; + } + + r.addEventListener('exported', (e: any) => this.exportInk(e, this._textRef)); + r.addEventListener('changed', (e: any) => this.callExport(this._textRef)); + + return this._textRef = r; + } + + transcribeInk = (inkdocs: Doc[], math: boolean, bounds: { x: number, y: number, width: number, height: number }, addDocument?: (doc: Doc | Doc[]) => boolean) => { + const strokes: InkData[] = []; + inkdocs.filter(i => Cast(i.data, InkField)).forEach(i => { + // TODO: interpolate missing times stamps + const d = Cast(i.data, InkField, null); + const left = Math.min(...d?.inkData.map(pd => pd.X) ?? [0]); + const top = Math.min(...d?.inkData.map(pd => pd.Y) ?? [0]); + strokes.push(d.inkData.map(pd => ({ X: pd.X + NumCast(i.x) - left, Y: pd.Y + NumCast(i.y) - top, time: pd.time }))); + }); + + this.addDocument = addDocument; + this.bounds = bounds; + + const pointerData = { "events": strokes.map(stroke => this.inkJSON(stroke)) }; + // console.log(JSON.stringify(pointerData)); + const processGestures = false; + + if (math) { + this._mathRef.editor.pointerEvents(pointerData, processGestures); + } + else { + this._textRef.editor.pointerEvents(pointerData, processGestures); + } + } + + callExport(ref: any) { + if (ref.editor.canExport) { + ref.editor.export_(); + } + } + + inkJSON = (stroke: InkData) => { + return { + "pointerType": "PEN", + "pointerId": 1, + "x": stroke.map(point => point.X), + "y": stroke.map(point => point.Y), + "t": stroke.map(point => point.time), + "p": new Array(stroke.length).fill(1.0) + }; + } + + exportInk = (e: any, ref: any) => { + const exports = e.detail.exports; + if (exports) { + if (exports['application/x-latex']) { + const latex = exports['application/x-latex']; + console.log(latex); + + this.addDocument?.(Docs.Create.EquationDocument({ title: latex, x: this.bounds.width, y: 0, _width: this.bounds.width, _height: this.bounds.height })); + } + else if (exports['text/plain']) { + const text = exports['text/plain']; + console.log(text); + this.addDocument?.(Docs.Create.TextDocument(text, { title: text, x: this.bounds.width, y: 0, _width: this.bounds.width, _height: this.bounds.height })); + } + + ref.editor.clear(); + } + } + + render() { + return ( + <div className="ink-transcription"> + <div className='math-editor' + ref={this.setMathRef} + touch-action="none"> + </div> + <div className='text-editor' + ref={this.setTextRef} + touch-action="none"> + </div> + </div> + ) + } +}
\ No newline at end of file diff --git a/src/client/views/MainView.tsx b/src/client/views/MainView.tsx index 8c0795881..84c1037dd 100644 --- a/src/client/views/MainView.tsx +++ b/src/client/views/MainView.tsx @@ -44,6 +44,7 @@ import { GestureOverlay } from './GestureOverlay'; import { DASHBOARD_SELECTOR_HEIGHT, LEFT_MENU_WIDTH } from './global/globalCssVariables.scss'; import { Colors } from './global/globalEnums'; import { KeyManager } from './GlobalKeyHandler'; +import { InkTranscription } from './InkTranscription'; import { LightboxView } from './LightboxView'; import { LinkMenu } from './linking/LinkMenu'; import "./MainView.scss"; @@ -184,7 +185,7 @@ export class MainView extends React.Component { fa.faArrowAltCircleDown, fa.faArrowAltCircleUp, fa.faArrowAltCircleLeft, fa.faArrowAltCircleRight, fa.faStopCircle, fa.faCheckCircle, fa.faGripVertical, fa.faSortUp, fa.faSortDown, fa.faTable, fa.faTh, fa.faThList, fa.faProjectDiagram, fa.faSignature, fa.faColumns, fa.faChevronCircleUp, fa.faUpload, fa.faBorderAll, fa.faBraille, fa.faChalkboard, fa.faPencilAlt, fa.faEyeSlash, fa.faSmile, fa.faIndent, fa.faOutdent, fa.faChartBar, fa.faBan, fa.faPhoneSlash, fa.faGripLines, - fa.faSave, fa.faBookmark, fa.faList, fa.faListOl, fa.faFolderPlus, fa.faLightbulb, fa.faBookOpen, fa.faMapMarkerAlt]); + fa.faSave, fa.faBookmark, fa.faList, fa.faListOl, fa.faFolderPlus, fa.faLightbulb, fa.faBookOpen, fa.faMapMarkerAlt, fa.faSquareRootAlt]); this.initAuthenticationRouters(); } @@ -636,6 +637,7 @@ export class MainView extends React.Component { <OverlayView /> <TimelineMenu /> <RichTextMenu /> + <InkTranscription /> {this.snapLines} <div className="mainView-webRef" ref={this.makeWebRef} /> <LightboxView PanelWidth={this._windowWidth} PanelHeight={this._windowHeight} maxBorder={[200, 50]} /> diff --git a/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx b/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx index e2ea81392..7d60387de 100644 --- a/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx +++ b/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx @@ -50,6 +50,7 @@ import { CollectionFreeFormRemoteCursors } from "./CollectionFreeFormRemoteCurso import "./CollectionFreeFormView.scss"; import { MarqueeView } from "./MarqueeView"; import React = require("react"); +import { InkTranscription } from "../../InkTranscription"; export const panZoomSchema = createSchema({ _panX: "number", @@ -492,7 +493,9 @@ export class CollectionFreeFormView extends CollectionSubView<Partial<collection case GestureUtils.Gestures.Stroke: const points = ge.points; const B = this.getTransform().transformBounds(ge.bounds.left, ge.bounds.top, ge.bounds.width, ge.bounds.height); - const inkDoc = Docs.Create.InkDocument(ActiveInkColor(), CurrentUserUtils.SelectedTool, ActiveInkWidth(), ActiveInkBezierApprox(), ActiveFillColor(), ActiveArrowStart(), ActiveArrowEnd(), ActiveDash(), points, + const times = ge.times; + const strokes: PointData[] = !times || times.length == 0 ? points : points.map((pt, i) => { return { X: pt.X, Y: pt.Y, time: times[i] } }); + const inkDoc = Docs.Create.InkDocument(ActiveInkColor(), CurrentUserUtils.SelectedTool, ActiveInkWidth(), ActiveInkBezierApprox(), ActiveFillColor(), ActiveArrowStart(), ActiveArrowEnd(), ActiveDash(), strokes, { title: "ink stroke", x: B.x - ActiveInkWidth() / 2, y: B.y - ActiveInkWidth() / 2, _width: B.width + ActiveInkWidth(), _height: B.height + ActiveInkWidth() }); this.addDocument(inkDoc); e.stopPropagation(); @@ -1477,6 +1480,11 @@ export class CollectionFreeFormView extends CollectionSubView<Partial<collection appearanceItems.push({ description: `${this.fitToContent ? "Make Zoomable" : "Scale to Window"}`, event: () => this.Document._fitToBox = !this.fitToContent, icon: !this.fitToContent ? "expand-arrows-alt" : "compress-arrows-alt" }); this.props.ContainingCollectionView && appearanceItems.push({ description: "Ungroup collection", event: this.promoteCollection, icon: "table" }); + + this.props.Document._isGroup && this.childDocs.filter(s => s.type === DocumentType.INK).length > 0 && appearanceItems.push({ description: "Ink to text", event: () => this.transcribeStrokes(false), icon: "font" }); + + this.props.Document._isGroup && this.childDocs.filter(s => s.type === DocumentType.INK).length > 0 && appearanceItems.push({ description: "Ink to math", event: () => this.transcribeStrokes(true), icon: "square-root-alt" }); + !Doc.UserDoc().noviceMode ? appearanceItems.push({ description: "Arrange contents in grid", event: this.layoutDocsInGrid, icon: "table" }) : null; !appearance && ContextMenu.Instance.addItem({ description: "Appearance...", subitems: appearanceItems, icon: "eye" }); @@ -1536,6 +1544,15 @@ export class CollectionFreeFormView extends CollectionSubView<Partial<collection input.click(); } + @undoBatch + @action + transcribeStrokes = (math: boolean) => { + if (this.props.Document._isGroup) { + const inkdocs = this.childDocs.filter(s => s.type === DocumentType.INK); + InkTranscription.Instance.transcribeInk(inkdocs, math, { x: NumCast(this.layoutDoc.x) ?? 0, y: NumCast(this.layoutDoc.y) ?? 0, width: NumCast(this.layoutDoc._width) ?? 0, height: NumCast(this.layoutDoc._height) ?? 0 }, this.addDocument); + } + } + @action setupDragLines = (snapToDraggedDoc: boolean = false) => { const activeDocs = this.getActiveDocuments(); diff --git a/src/client/views/collections/collectionFreeForm/MarqueeOptionsMenu.tsx b/src/client/views/collections/collectionFreeForm/MarqueeOptionsMenu.tsx index 1f59f9732..8a8b528f6 100644 --- a/src/client/views/collections/collectionFreeForm/MarqueeOptionsMenu.tsx +++ b/src/client/views/collections/collectionFreeForm/MarqueeOptionsMenu.tsx @@ -14,7 +14,6 @@ export class MarqueeOptionsMenu extends AntimodeMenu<AntimodeMenuProps> { public createCollection: (e: KeyboardEvent | React.PointerEvent | undefined, group?: boolean) => void = unimplementedFunction; public delete: (e: KeyboardEvent | React.PointerEvent | undefined) => void = unimplementedFunction; public summarize: (e: KeyboardEvent | React.PointerEvent | undefined) => void = unimplementedFunction; - public inkToText: (e: KeyboardEvent | React.PointerEvent | undefined) => void = unimplementedFunction; public showMarquee: () => void = unimplementedFunction; public hideMarquee: () => void = unimplementedFunction; public pinWithView: (e: KeyboardEvent | React.PointerEvent | undefined) => void = unimplementedFunction; @@ -64,16 +63,6 @@ export class MarqueeOptionsMenu extends AntimodeMenu<AntimodeMenuProps> { </button> </Tooltip>, ]; - if (false && !SelectionManager.Views().some(v => v.props.Document.type !== DocumentType.INK)) { - buttons.push( - <Tooltip key="inkToText" title={<div className="dash-tooltip">Change to Text</div>} placement="bottom"> - <button - className="antimodeMenu-button" - onPointerDown={this.inkToText}> - <FontAwesomeIcon icon="font" size="lg" /> - </button> - </Tooltip>); - } return this.getElement(buttons); } }
\ No newline at end of file diff --git a/src/client/views/collections/collectionFreeForm/MarqueeView.tsx b/src/client/views/collections/collectionFreeForm/MarqueeView.tsx index b10b0912f..a828cd981 100644 --- a/src/client/views/collections/collectionFreeForm/MarqueeView.tsx +++ b/src/client/views/collections/collectionFreeForm/MarqueeView.tsx @@ -1,14 +1,15 @@ import { action, computed, observable } from "mobx"; import { observer } from "mobx-react"; -import { AclAugment, AclAdmin, AclEdit, DataSym, Doc, Opt } from "../../../../fields/Doc"; +import { AclAdmin, AclAugment, AclEdit, DataSym, Doc, Opt } from "../../../../fields/Doc"; import { Id } from "../../../../fields/FieldSymbols"; import { InkData, InkField, InkTool } from "../../../../fields/InkField"; import { List } from "../../../../fields/List"; import { RichTextField } from "../../../../fields/RichTextField"; import { SchemaHeaderField } from "../../../../fields/SchemaHeaderField"; import { Cast, FieldValue, NumCast, StrCast } from "../../../../fields/Types"; +import { ImageField } from "../../../../fields/URLField"; import { GetEffectiveAcl } from "../../../../fields/util"; -import { Utils, intersectRect, returnFalse } from "../../../../Utils"; +import { intersectRect, returnFalse, Utils } from "../../../../Utils"; import { CognitiveServices } from "../../../cognitive_services/CognitiveServices"; import { Docs, DocumentOptions, DocUtils } from "../../../documents/Documents"; import { DocumentType } from "../../../documents/DocumentTypes"; @@ -21,18 +22,17 @@ import { ContextMenu } from "../../ContextMenu"; import { FormattedTextBox } from "../../nodes/formattedText/FormattedTextBox"; import { PresBox } from "../../nodes/trails/PresBox"; import { PresMovement } from "../../nodes/trails/PresEnums"; +import { VideoBox } from "../../nodes/VideoBox"; +import { pasteImageBitmap } from "../../nodes/WebBoxRenderer"; import { PreviewCursor } from "../../PreviewCursor"; +import { StyleLayers } from "../../StyleProvider"; import { CollectionDockingView } from "../CollectionDockingView"; import { SubCollectionViewProps } from "../CollectionSubView"; import { CollectionView } from "../CollectionView"; +import { TreeView } from "../TreeView"; import { MarqueeOptionsMenu } from "./MarqueeOptionsMenu"; import "./MarqueeView.scss"; import React = require("react"); -import { StyleLayers } from "../../StyleProvider"; -import { TreeView } from "../TreeView"; -import { VideoBox } from "../../nodes/VideoBox"; -import { ImageField, WebField } from "../../../../fields/URLField"; -import { pasteImageBitmap } from "../../nodes/WebBoxRenderer"; interface MarqueeViewProps { getContainerTransform: () => Transform; @@ -265,7 +265,7 @@ export class MarqueeView extends React.Component<SubCollectionViewProps & Marque MarqueeOptionsMenu.Instance.createCollection = this.collection; MarqueeOptionsMenu.Instance.delete = this.delete; MarqueeOptionsMenu.Instance.summarize = this.summary; - MarqueeOptionsMenu.Instance.inkToText = this.syntaxHighlight; + // MarqueeOptionsMenu.Instance.inkToText = this.syntaxHighlight; MarqueeOptionsMenu.Instance.showMarquee = this.showMarquee; MarqueeOptionsMenu.Instance.hideMarquee = this.hideMarquee; MarqueeOptionsMenu.Instance.jumpTo(e.clientX, e.clientY); diff --git a/src/client/views/nodes/EquationBox.scss b/src/client/views/nodes/EquationBox.scss index 6c9d53d10..229a1a485 100644 --- a/src/client/views/nodes/EquationBox.scss +++ b/src/client/views/nodes/EquationBox.scss @@ -1,3 +1,39 @@ +@import "../global/globalCssVariables.scss"; + .equationBox-cont { transform-origin: top left; + overflow: visible; + width: 100%; + height: 100%; +} + +.button { + position: absolute; + display: flex; + justify-content: center; + align-items: center; + top: 0; + right: 0; + width: 20px; + height: 20px; + border-radius: 5px; + background: $dark-gray; + color: white; + + svg { + width: 12px; + height: 12px; + } +} + +.ink-editor { + top: 20px; + min-width: 500px; + min-height: 300px; + background: $light-gray; + pointer-events: all; + + button { + float: right; + } }
\ No newline at end of file diff --git a/src/client/views/nodes/EquationBox.tsx b/src/client/views/nodes/EquationBox.tsx index c170f9867..c0c4fab09 100644 --- a/src/client/views/nodes/EquationBox.tsx +++ b/src/client/views/nodes/EquationBox.tsx @@ -1,5 +1,7 @@ +import { FontAwesomeIcon } from "@fortawesome/react-fontawesome"; import EquationEditor from 'equation-editor-react'; -import { action, reaction } from 'mobx'; +import * as iink from 'iink-js'; +import { action, observable, reaction } from 'mobx'; import { observer } from 'mobx-react'; import * as React from 'react'; import { WidthSym } from '../../../fields/Doc'; @@ -12,12 +14,16 @@ import { LightboxView } from '../LightboxView'; import './EquationBox.scss'; import { FieldView, FieldViewProps } from './FieldView'; - @observer export class EquationBox extends ViewBoxBaseComponent<FieldViewProps>() { public static LayoutString(fieldKey: string) { return FieldView.LayoutString(EquationBox, fieldKey); } public static SelectOnLoad: string = ""; + _ref: React.RefObject<EquationEditor> = React.createRef(); + @observable _inkOpen = false; + @observable _inkEditor: any; + @observable _inkRef: any; + componentDidMount() { if (EquationBox.SelectOnLoad === this.rootDoc[Id] && (!LightboxView.LightboxDoc || LightboxView.IsLightboxDocView(this.props.docViewPath()))) { this.props.select(false); @@ -39,7 +45,11 @@ export class EquationBox extends ViewBoxBaseComponent<FieldViewProps>() { } }, { fireImmediately: true }); } - plot: any; + + componentWillUnmount() { + this._inkRef.removeEventListener('exported', this.exportInk); + } + @action keyPressed = (e: KeyboardEvent) => { const _height = Number(getComputedStyle(this._ref.current!.element.current).height.replace("px", "")); @@ -65,6 +75,7 @@ export class EquationBox extends ViewBoxBaseComponent<FieldViewProps>() { } if (e.key === "Backspace" && !this.dataDoc.text) this.props.removeDocument?.(this.rootDoc); } + onChange = (str: string) => { this.dataDoc.text = str; const style = this._ref.current && getComputedStyle(this._ref.current.element.current); @@ -75,6 +86,58 @@ export class EquationBox extends ViewBoxBaseComponent<FieldViewProps>() { this.layoutDoc._height = Math.max(25, _height); } } + + @action + toggleInk = (e: React.PointerEvent) => { + e.stopPropagation(); + + this._inkOpen = !this._inkOpen; + + if (!this._inkEditor) { + this._inkEditor = this._inkRef ? iink.register(this._inkRef, { + recognitionParams: { + type: 'MATH', + protocol: 'WEBSOCKET', + server: { + host: 'cloud.myscript.com', + applicationKey: '7277ec34-0c2e-4ee1-9757-ccb657e3f89f', + hmacKey: 'f5cb18f2-1f95-4ddb-96ac-3f7c888dffc1', + }, + iink: { + math: { + mimeTypes: ['application/x-latex', 'application/vnd.myscript.jiix'] + }, + export: { + jiix: { + strokes: true + } + } + } + } + }) : null; + + this._inkRef.addEventListener('exported', this.exportInk) + } + } + + convertInk = (e: React.MouseEvent) => { + this._inkRef.editor.convert(); + } + + clearInk = (e: React.MouseEvent) => { + this._inkRef.editor.clear(); + this.onChange(""); + } + + exportInk = (e: any) => { + const exports = e.detail.exports; + if (exports && exports['application/x-latex']) { + this.onChange(exports['application/x-latex']); + console.log(JSON.parse(exports['application/vnd.myscript.jiix']).expressions[0].items[0]); + } + } + + render() { TraceMobx(); const scale = (this.props.scaling?.() || 1) * NumCast(this.layoutDoc._viewScale, 1); @@ -82,18 +145,34 @@ export class EquationBox extends ViewBoxBaseComponent<FieldViewProps>() { onPointerDown={e => !e.ctrlKey && e.stopPropagation()} style={{ transform: `scale(${scale})`, - width: `${100 / scale}%`, + width: `calc(${100 / scale}% + 25px)`, height: `${100 / scale}%`, pointerEvents: !this.props.isSelected() ? "none" : undefined, }} - onKeyDown={e => e.stopPropagation()} - > + onKeyDown={e => e.stopPropagation()}> + <EquationEditor ref={this._ref} value={this.dataDoc.text || "x"} spaceBehavesLikeTab={true} onChange={this.onChange} autoCommands="pi theta sqrt sum prod alpha beta gamma rho" autoOperatorNames="sin cos tan" /> + + <div className="button" + style={{ display: this.props.isContentActive() ? "flex" : "none" }} + onPointerDown={this.toggleInk}> + <FontAwesomeIcon icon="pencil-alt" /> + </div> + + <div className='ink-editor' + ref={action((r: any) => this._inkRef = r)} + id="editor" onPointerDown={(e) => e.stopPropagation()} + touch-action="none" + style={{ display: this._inkOpen && this.props.isContentActive() ? "block" : "none" }}> + <button onClick={this.convertInk}>convert</button> + <button onClick={this.clearInk}>clear</button> + </div> + </div>); } }
\ No newline at end of file diff --git a/src/fields/InkField.ts b/src/fields/InkField.ts index 31024e805..48271d3d3 100644 --- a/src/fields/InkField.ts +++ b/src/fields/InkField.ts @@ -19,6 +19,7 @@ export enum InkTool { export interface PointData { X: number; Y: number; + time?: number; } export type Segment = Array<Bezier>; @@ -83,7 +84,7 @@ export class InkField extends ObjectField { } [ToScriptString]() { - return "new InkField([" + this.inkData.map(i => `{X: ${i.X}, Y: ${i.Y}} `) + "])"; + return "new InkField([" + this.inkData.map(i => `{X: ${i.X}, Y: ${i.Y}, time: ${i.time || 0}} `) + "])"; } [ToString]() { return "InkField"; diff --git a/src/pen-gestures/GestureUtils.ts b/src/pen-gestures/GestureUtils.ts index 65f2bf80c..cf59cb3c6 100644 --- a/src/pen-gestures/GestureUtils.ts +++ b/src/pen-gestures/GestureUtils.ts @@ -8,7 +8,8 @@ export namespace GestureUtils { readonly gesture: Gestures, readonly points: PointData[], readonly bounds: Rect, - readonly text?: any + readonly text?: any, + readonly times?: number[] ) { } } diff --git a/src/typings/index.d.ts b/src/typings/index.d.ts index 068ac2159..bc7727426 100644 --- a/src/typings/index.d.ts +++ b/src/typings/index.d.ts @@ -8,6 +8,7 @@ declare module 'webrtc-adapter'; declare module 'bezier-curve'; declare module 'fit-curve'; declare module 'react-audio-waveform'; +declare module 'iink-js'; declare module 'reveal'; declare module 'react-reveal'; |