From 74c9d7cb47281dab59908eea5384b370f493d087 Mon Sep 17 00:00:00 2001 From: Zachary Zhang Date: Wed, 10 Jul 2024 13:29:50 -0400 Subject: work in progres --- src/client/views/DocumentDecorations.tsx | 15 +-- src/client/views/InkTranscription.scss | 5 + src/client/views/InkTranscription.tsx | 138 ++++++++++++++++---------- src/client/views/nodes/DiagramBox.tsx | 162 ++++++++++++++++--------------- 4 files changed, 181 insertions(+), 139 deletions(-) (limited to 'src') diff --git a/src/client/views/DocumentDecorations.tsx b/src/client/views/DocumentDecorations.tsx index 96ce181f8..a229b15db 100644 --- a/src/client/views/DocumentDecorations.tsx +++ b/src/client/views/DocumentDecorations.tsx @@ -152,25 +152,18 @@ export class DocumentDecorations extends ObservableReactComponent { const effectiveLayoutAcl = GetEffectiveAcl(DocumentView.Selected()[0].Document); if (effectiveLayoutAcl === AclAdmin || effectiveLayoutAcl === AclEdit || effectiveLayoutAcl === AclAugment) { - setupMoveUpEvents(this, e, moveEv => {this.onBackgroundMove(true, moveEv) - console.log("im being moved ink") - return false; - }, emptyFunction, emptyFunction); + setupMoveUpEvents(this, e, moveEv => this.onBackgroundMove(true, moveEv), emptyFunction, emptyFunction); e.stopPropagation(); } }; onTitleDown = (e: React.PointerEvent) => { - console.log("im clicked") const effectiveLayoutAcl = GetEffectiveAcl(DocumentView.SelectedDocs()[0]); if (effectiveLayoutAcl === AclAdmin || effectiveLayoutAcl === AclEdit || effectiveLayoutAcl === AclAugment) { setupMoveUpEvents( this, e, - moveEv => {this.onBackgroundMove(true, moveEv) - console.log("im being dragged") - return false; - }, + moveEv => this.onBackgroundMove(true, moveEv), emptyFunction, action(() => { const selected = DocumentView.SelectedDocs().length === 1 ? DocumentView.SelectedDocs()[0] : undefined; @@ -189,7 +182,7 @@ export class DocumentDecorations extends ObservableReactComponent { - const dragDocView = DocumentView.Selected()[0];`` + const dragDocView = DocumentView.Selected()[0]; const effectiveLayoutAcl = GetEffectiveAcl(dragDocView.Document); if (effectiveLayoutAcl !== AclAdmin && effectiveLayoutAcl !== AclEdit && effectiveLayoutAcl !== AclAugment) { return false; @@ -879,4 +872,4 @@ export class DocumentDecorations extends ObservableReactComponent ); } -} +} \ No newline at end of file diff --git a/src/client/views/InkTranscription.scss b/src/client/views/InkTranscription.scss index bbb0a1afa..18d6b8b10 100644 --- a/src/client/views/InkTranscription.scss +++ b/src/client/views/InkTranscription.scss @@ -2,4 +2,9 @@ .error-msg { display: none !important; } + .ms-editor{ + .smartguide{ + top:1000px; + } + } } \ No newline at end of file diff --git a/src/client/views/InkTranscription.tsx b/src/client/views/InkTranscription.tsx index 495bb6b83..44b63e605 100644 --- a/src/client/views/InkTranscription.tsx +++ b/src/client/views/InkTranscription.tsx @@ -1,16 +1,19 @@ import * as iink from 'iink-ts'; -import {TProtocol,TSchene,TServerConfiguration,TConfiguration,TRecognitionConfiguration} from 'iink-ts'; import { action, observable } from 'mobx'; import * as React from 'react'; import { Doc, DocListCast } from '../../fields/Doc'; import { InkData, InkField, InkTool } from '../../fields/InkField'; -import { Cast, DateCast, NumCast } from '../../fields/Types'; +import { Cast, DateCast, ImageCast, NumCast, StrCast } from '../../fields/Types'; import { aggregateBounds } from '../../Utils'; import { DocumentType } from '../documents/DocumentTypes'; import { CollectionFreeFormView } from './collections/collectionFreeForm'; import { InkingStroke } from './InkingStroke'; import './InkTranscription.scss'; +import { Docs } from '../documents/Documents'; import { DocumentView } from './nodes/DocumentView'; +import { Number } from 'mongoose'; +import { NumberArray } from 'd3'; +import { ImageField } from '../../fields/URLField'; /** * Class component that handles inking in writing mode @@ -18,12 +21,14 @@ import { DocumentView } from './nodes/DocumentView'; export class InkTranscription extends React.Component { static Instance: InkTranscription; - @observable _mathRegister: any= undefined; - @observable _mathRef: any= undefined; - @observable _textRegister: any= undefined; - @observable _textRef: any= undefined; + @observable _mathRegister: any = undefined; + @observable _mathRef: any = undefined; + @observable _textRegister: any = undefined; + @observable _textRef: any = undefined; + @observable iinkEditor: any = undefined; private lastJiix: any; private currGroup?: Doc; + private collectionFreeForm?: CollectionFreeFormView; constructor(props: Readonly<{}>) { super(props); @@ -43,57 +48,56 @@ export class InkTranscription extends React.Component { const options = { configuration: { server: { - scheme: "https", - host: "cloud.myscript.com", - applicationKey: "c0901093-5ac5-4454-8e64-0def0f13f2ca", - hmacKey: "f6465cca-1856-4492-a6a4-e2395841be2f", - protocol: "WEBSOCKET" + scheme: 'https', + host: 'cloud.myscript.com', + applicationKey: 'c0901093-5ac5-4454-8e64-0def0f13f2ca', + hmacKey: 'f6465cca-1856-4492-a6a4-e2395841be2f', + protocol: 'WEBSOCKET', }, recognition: { - type: "TEXT", - } - } + type: 'TEXT', + }, + }, }; - + editor = new iink.Editor(r, options as any); - + await editor.initialize(); - + this._textRegister = r; r?.addEventListener('exported', (e: any) => this.exportInk(e, this._textRef)); return (this._textRef = r); } - } + }; @action - setTextRef = async (r:any) => { + setTextRef = async (r: any) => { if (!this._textRegister && r) { let editor; const options = { configuration: { server: { - scheme: "https", - host: "cloud.myscript.com", - applicationKey: "c0901093-5ac5-4454-8e64-0def0f13f2ca", - hmacKey: "f6465cca-1856-4492-a6a4-e2395841be2f", - protocol: "WEBSOCKET" + scheme: 'https', + host: 'cloud.myscript.com', + applicationKey: 'c0901093-5ac5-4454-8e64-0def0f13f2ca', + hmacKey: 'f6465cca-1856-4492-a6a4-e2395841be2f', + protocol: 'WEBSOCKET', }, recognition: { - type: "TEXT", - } - } + type: 'TEXT', + }, + }, }; - + editor = new iink.Editor(r, options as any); - + await editor.initialize(); - + this.iinkEditor = editor; this._textRegister = r; r?.addEventListener('exported', (e: any) => this.exportInk(e, this._textRef)); return (this._textRef = r); } - }; /** @@ -119,14 +123,14 @@ export class InkTranscription extends React.Component { }); this.currGroup = groupDoc; - - const pointerData = { events: strokes.map((stroke, i) => this.inkJSON(stroke, times[i])) }; + const pointerData = strokes.map((stroke, i) => this.inkJSON(stroke, times[i])); + console.log(JSON.stringify(pointerData)); const processGestures = false; - if (math) { - this._mathRef.editor.pointerEvents(pointerData, processGestures); + console.log('math'); + this.iinkEditor.importPointEvents(pointerData); } else { - this._textRef.editor.pointerEvents(pointerData, processGestures); + this.iinkEditor.importPointEvents(pointerData); } }; @@ -138,13 +142,26 @@ export class InkTranscription extends React.Component { * @returns json object representation of ink data */ inkJSON = (stroke: InkData, time: number) => { + interface strokeData { + x: number; + y: number; + t: number; + p: number; + } + let strokeObjects: strokeData[] = []; + stroke.forEach(point => { + let tempObject: strokeData = { + x: point.X, + y: point.Y, + t: time, + p: 1.0, + }; + strokeObjects.push(tempObject); + }); return { pointerType: 'PEN', pointerId: 1, - x: stroke.map(point => point.X), - y: stroke.map(point => point.Y), - t: new Array(stroke.length).fill(time), - p: new Array(stroke.length).fill(1.0), + pointers: strokeObjects, }; }; @@ -163,7 +180,7 @@ export class InkTranscription extends React.Component { if (!ctx) { return; } - const docView: CollectionFreeFormView =DocumentView.getDocumentView(ctx)?.ComponentView as CollectionFreeFormView; + const docView: CollectionFreeFormView = DocumentView.getDocumentView(ctx)?.ComponentView as CollectionFreeFormView; // DocumentManager.Instance.getDocumentView(ctx)?.ComponentView as CollectionFreeFormView; if (!docView) return; @@ -179,10 +196,11 @@ export class InkTranscription extends React.Component { * @param e the event objects * @param ref the ref to the editor */ - exportInk = (e: any, ref: any) => { - const exports = e.detail.exports; + exportInk = async (e: any, ref: any) => { + const exports = e.detail['application/vnd.myscript.jiix']; + console.log(exports); if (exports) { - if (exports['application/x-latex']) { + if (exports['type'] == 'Math') { const latex = exports['application/x-latex']; if (this.currGroup) { this.currGroup.text = latex; @@ -190,7 +208,7 @@ export class InkTranscription extends React.Component { } ref.editor.clear(); - } else if (exports['text/plain']) { + } else if (exports['type'] == 'Text') { if (exports['application/vnd.myscript.jiix']) { this.lastJiix = JSON.parse(exports['application/vnd.myscript.jiix']); // map timestamp to strokes @@ -226,17 +244,33 @@ export class InkTranscription extends React.Component { if (this.lastJiix.words.length > 1) this.subgroupsTranscriptions(wordInkDocMap); } } - const text = exports['text/plain']; + const text = exports['label']; - if (this.currGroup) { + if (this.currGroup && text) { + DocumentView.getDocumentView(this.currGroup)?.ComponentView?.updateIcon?.(); + console.log(this.currGroup.icon); this.currGroup.transcription = text; - this.currGroup.title = text.split('\n')[0]; + this.currGroup.title = text; + let image = await this.getIcon(); + console.log(this.currGroup.icon, image); + if (!this.currGroup.hasTextBox) { + const newDoc = Docs.Create.TextDocument(text, { title: '', x: this.currGroup.x as number, y: (this.currGroup.y as number) + (this.currGroup.height as number) }); + this.collectionFreeForm?.addDocument(newDoc); + this.currGroup.hasTextBox = true; + } + ref.editor.clear(); } - - ref.editor.clear(); } } }; + async getIcon() { + const docView = DocumentView.getDocumentView(this.currGroup); + if (docView) { + docView.ComponentView?.updateIcon?.(); + return new Promise(res => setTimeout(() => res(ImageCast(docView.Document.icon)), 500)); + } + return undefined; + } /** * Creates the ink grouping once the user leaves the writing mode. @@ -267,6 +301,7 @@ export class InkTranscription extends React.Component { * @returns a new collection Doc or undefined if the grouping fails */ groupInkDocs(selected: Doc[], docView: CollectionFreeFormView, word?: string): Doc | undefined { + this.collectionFreeForm = docView; const bounds: { x: number; y: number; width?: number; height?: number }[] = []; // calculate the necessary bounds from the selected ink docs @@ -324,6 +359,9 @@ export class InkTranscription extends React.Component { // nda - bug: when deleting a stroke before leaving writing mode, delete the stroke from unprocessed ink docs newCollection && docView.props.addDocument?.(newCollection); + if (newCollection) { + newCollection.hasTextBox = false; + } return newCollection; } diff --git a/src/client/views/nodes/DiagramBox.tsx b/src/client/views/nodes/DiagramBox.tsx index fcc028aab..5a712b8b0 100644 --- a/src/client/views/nodes/DiagramBox.tsx +++ b/src/client/views/nodes/DiagramBox.tsx @@ -7,14 +7,14 @@ import * as React from 'react'; import { Doc, DocListCast } from '../../../fields/Doc'; import { List } from '../../../fields/List'; import { RichTextField } from '../../../fields/RichTextField'; -import { DocCast,BoolCast } from '../../../fields/Types'; +import { DocCast, BoolCast } from '../../../fields/Types'; import { GPTCallType, gptAPICall } from '../../apis/gpt/GPT'; import { DocumentType } from '../../documents/DocumentTypes'; import { Docs } from '../../documents/Documents'; import { DocumentManager } from '../../util/DocumentManager'; import { LinkManager } from '../../util/LinkManager'; import { ViewBoxAnnotatableComponent } from '../DocComponent'; -import { InkingStroke} from '../InkingStroke'; +import { InkingStroke } from '../InkingStroke'; import './DiagramBox.scss'; import { FieldView, FieldViewProps } from './FieldView'; import { PointData } from '../../../pen-gestures/GestureTypes'; @@ -94,9 +94,9 @@ export class DiagramBox extends ViewBoxAnnotatableComponent() { // () => this.lockInkStroke(), // { fireImmediately: true } // ); - this.lockInkStroke(); + this.lockInkStroke(); } - + componentDidUpdate = () => { if (typeof this.Document.drawingMermaidCode === 'string' && this.Document.menuState === 'drawing') { this.renderMermaidAsync(this.Document.drawingMermaidCode); @@ -268,13 +268,17 @@ export class DiagramBox extends ViewBoxAnnotatableComponent() { } } async lockInkStroke() { - console.log("hello") - console.log(DocListCast(this.Document.data).filter(doc => doc.title === 'rectangle').map(doc => doc.x)) + console.log('hello'); + console.log( + DocListCast(this.Document.data) + .filter(doc => doc.title === 'rectangle') + .map(doc => doc.x) + ); if (this.Document.data instanceof List) { const docArray: Doc[] = DocListCast(this.Document.data); const rectangleArray = docArray.filter(doc => doc.title === 'rectangle' || doc.title === 'circle'); - if(rectangleArray[0]){ - console.log(rectangleArray[0].x) + if (rectangleArray[0]) { + console.log(rectangleArray[0].x); } const lineArray = docArray.filter(doc => doc.title === 'line' || doc.title === 'stroke'); const timeoutPromise = () => @@ -302,88 +306,90 @@ export class DiagramBox extends ViewBoxAnnotatableComponent() { const startY = inkStrokeYArray[0] - minY + (lineArray[j]?.y as number); const endX = inkStrokeXArray[inkStrokeXArray.length - 1] - minX + (lineArray[j].x as number); const endY = inkStrokeYArray[inkStrokeYArray.length - 1] - minY + (lineArray[j].y as number); - let closestStartRect:Doc=lineArray[0]; - let closestStartDistance=9999999; - let closestEndRect:Doc=lineArray[0]; - let closestEndDistance=9999999; + let closestStartRect: Doc = lineArray[0]; + let closestStartDistance = 9999999; + let closestEndRect: Doc = lineArray[0]; + let closestEndDistance = 9999999; rectangleArray.forEach(rectangle => { - const midPoint=this.getMidPoint(rectangle) - if(this.euclideanDistance(midPoint.X,midPoint.Y,startX,startY) { + const newInkDoc = Docs.Create.AudioDocument(''); // get rid of this!! + // const newInkDoc:Doc=Docs.Create.InkDocument( + // points, + // { title: 'stroke', + // x: inkX, + // y: inkY, + // strokeWidth: Math.abs(closestEndRect.x+closestEndRect.width/2-closestStartRect.x-closestStartRect.width/2), + // _height: Math.abs(closestEndRect.y+closestEndRect.height/2-closestStartRect.y-closestStartRect.height/2), + // stroke_showLabel: BoolCast(Doc.UserDoc().activeInkHideTextLabels)}, // prettier-ignore + // 1) + + DocumentManager.Instance.AddViewRenderedCb(this.Document, docViewForYourCollection => { if (docViewForYourCollection && docViewForYourCollection.ComponentView) { - if (docViewForYourCollection.ComponentView.addDocument&&docViewForYourCollection.ComponentView.removeDocument) { - docViewForYourCollection.ComponentView?.removeDocument(inkToDelete) - docViewForYourCollection.ComponentView?.addDocument(newInkDoc); - + if (docViewForYourCollection.ComponentView.addDocument && docViewForYourCollection.ComponentView.removeDocument) { + docViewForYourCollection.ComponentView?.removeDocument(inkToDelete); + docViewForYourCollection.ComponentView?.addDocument(newInkDoc); - // const bruh2= DocListCast(this.Document.data).filter(doc => doc.title === 'line' || doc.title === 'stroke').map(doc => DocumentManager.Instance.getDocumentView(doc, this.DocumentView?.())).filter(inkView => inkView?.ComponentView instanceof InkingStroke).map(stroke => stroke?.ComponentView); // console.log(bruh2) // console.log((bruh2[0] as InkingStroke)?.inkScaledData()) } } - }); - } + }); + } } } } - getMidPoint(rectangle:Doc){ - let midPoint={X:0,Y:0} - if(typeof rectangle.x ==='number' && typeof rectangle.width==='number'&&typeof rectangle.y ==='number' && typeof rectangle.height==='number'){ - midPoint={X:rectangle.x+rectangle.width/2,Y:rectangle.y+rectangle.height/2}; + getMidPoint(rectangle: Doc) { + let midPoint = { X: 0, Y: 0 }; + if (typeof rectangle.x === 'number' && typeof rectangle.width === 'number' && typeof rectangle.y === 'number' && typeof rectangle.height === 'number') { + midPoint = { X: rectangle.x + rectangle.width / 2, Y: rectangle.y + rectangle.height / 2 }; } - return midPoint + return midPoint; } euclideanDistance(x1: number, y1: number, x2: number, y2: number): number { const deltaX = x2 - x1; @@ -486,7 +492,7 @@ export class DiagramBox extends ViewBoxAnnotatableComponent() { -