diff options
| -rw-r--r-- | package-lock.json | 14 | ||||
| -rw-r--r-- | package.json | 1 | ||||
| -rw-r--r-- | src/client/cognitive_services/CognitiveServices.ts | 11 | ||||
| -rw-r--r-- | src/client/documents/Documents.ts | 2 | ||||
| -rw-r--r-- | src/client/util/CurrentUserUtils.ts | 2 | ||||
| -rw-r--r-- | src/client/views/DocumentDecorations.tsx | 13 | ||||
| -rw-r--r-- | src/client/views/InkTranscription.tsx | 687 | ||||
| -rw-r--r-- | src/client/views/InkingStroke.tsx | 4 | ||||
| -rw-r--r-- | src/client/views/MainView.tsx | 2 | ||||
| -rw-r--r-- | src/client/views/global/globalScripts.ts | 6 | ||||
| -rw-r--r-- | src/client/views/nodes/DiagramBox.scss | 250 | ||||
| -rw-r--r-- | src/client/views/nodes/DiagramBox.tsx | 615 | ||||
| -rw-r--r-- | src/client/views/nodes/ScreenshotBox.tsx | 2 |
13 files changed, 1089 insertions, 520 deletions
diff --git a/package-lock.json b/package-lock.json index f11d8a462..6bba1abd4 100644 --- a/package-lock.json +++ b/package-lock.json @@ -126,6 +126,7 @@ "https": "^1.0.0", "https-browserify": "^1.0.0", "i": "^0.3.7", + "iink-ts": "^1.0.5", "image-data-uri": "^2.0.1", "image-size": "^1.0.2", "image-size-stream": "^1.1.0", @@ -21833,6 +21834,14 @@ "resolved": "https://registry.npmjs.org/ignore-by-default/-/ignore-by-default-1.0.1.tgz", "integrity": "sha512-Ius2VYcGNk7T90CppJqcIkS5ooHUZyIQK+ClZfMfMNFEF9VSE73Fq+906u/CWu92x4gzZMWOwfFYckPObzdEbA==" }, + "node_modules/iink-ts": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/iink-ts/-/iink-ts-1.0.5.tgz", + "integrity": "sha512-LAWWPvgcsLtouI9ExVijZbPGIWMfJtVCcuznbIDyboaPb+cHYsftcuJdjDU7TQwIpGP4hmcjNQA57XPCw4MK2A==", + "dependencies": { + "json-css": "^1.5.6" + } + }, "node_modules/image-data-uri": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/image-data-uri/-/image-data-uri-2.0.1.tgz", @@ -23419,6 +23428,11 @@ "resolved": "https://registry.npmjs.org/json-buffer/-/json-buffer-3.0.1.tgz", "integrity": "sha512-4bV5BfR2mqfQTJm+V5tPPdf+ZpuhiIvTuAB5g8kcrXOZpTT/QwwVRWBywX1ozr6lEuPdbHxwaJlm9G6mI2sfSQ==" }, + "node_modules/json-css": { + "version": "1.5.6", + "resolved": "https://registry.npmjs.org/json-css/-/json-css-1.5.6.tgz", + "integrity": "sha512-B/0T0OxZH9tSb93tXV6VOYtXqrPz/Vgz2QrCT/4NXen8HGElYkYr9V+8IrSVTMj/ftxa8cG1kcu7f3iAMlaFlQ==" + }, "node_modules/json-parse-even-better-errors": { "version": "2.3.1", "resolved": "https://registry.npmjs.org/json-parse-even-better-errors/-/json-parse-even-better-errors-2.3.1.tgz", diff --git a/package.json b/package.json index a614d108e..95fd77428 100644 --- a/package.json +++ b/package.json @@ -211,6 +211,7 @@ "https": "^1.0.0", "https-browserify": "^1.0.0", "i": "^0.3.7", + "iink-ts": "^1.0.5", "image-data-uri": "^2.0.1", "image-size": "^1.0.2", "image-size-stream": "^1.1.0", diff --git a/src/client/cognitive_services/CognitiveServices.ts b/src/client/cognitive_services/CognitiveServices.ts index 9808b6a01..9f46b8685 100644 --- a/src/client/cognitive_services/CognitiveServices.ts +++ b/src/client/cognitive_services/CognitiveServices.ts @@ -46,14 +46,15 @@ export enum Confidence { export namespace CognitiveServices { const ExecuteQuery = async <D>(service: Service, manager: APIManager<D>, data: D): Promise<any> => { const apiKey = process.env[service.toUpperCase()]; - if (!apiKey) { + if (apiKey) { + console.log(data) console.log(`No API key found for ${service}: ensure youe root directory has .env file with _CLIENT_${service.toUpperCase()}.`); return undefined; } let results: any; try { - results = await manager.requester(apiKey, manager.converter(data), service).then(json => JSON.parse(json)); + results = await manager.requester("has", manager.converter(data), service).then(json => JSON.parse(json)); } catch (e) { throw e; } @@ -137,6 +138,12 @@ export namespace CognitiveServices { points: points.map(({ X: x, Y: y }) => `${x},${y}`).join(','), language: 'en-US', })); + console.log(JSON.stringify({ + version: 1, + language: 'en-US', + unit: 'mm', + strokes, + })) return JSON.stringify({ version: 1, language: 'en-US', diff --git a/src/client/documents/Documents.ts b/src/client/documents/Documents.ts index b96fdb4bd..ff95e38bd 100644 --- a/src/client/documents/Documents.ts +++ b/src/client/documents/Documents.ts @@ -767,7 +767,7 @@ export namespace Docs { export function ComparisonDocument(text: string, options: DocumentOptions = { title: 'Comparison Box' }) { return InstanceFromProto(Prototypes.get(DocumentType.COMPARISON), text, options); } - export function DiagramDocument(options: DocumentOptions = { title: 'bruh box' }) { + export function DiagramDocument(options: DocumentOptions = { title: '' }) { return InstanceFromProto(Prototypes.get(DocumentType.DIAGRAM), undefined, options); } diff --git a/src/client/util/CurrentUserUtils.ts b/src/client/util/CurrentUserUtils.ts index e095bc659..8ece897f4 100644 --- a/src/client/util/CurrentUserUtils.ts +++ b/src/client/util/CurrentUserUtils.ts @@ -398,7 +398,7 @@ pie title Minerals in my tap water { toolTip: "Tap or drag to create a plotly node", title: "Plotly", icon: "rocket", dragFactory: doc.emptyPlotly as Doc, clickFactory: DocCast(doc.emptyMermaids)}, { toolTip: "Tap or drag to create a physics simulation",title: "Simulation", icon: "rocket",dragFactory: doc.emptySimulation as Doc, clickFactory: DocCast(doc.emptySimulation), funcs: { hidden: "IsNoviceMode()"}}, { toolTip: "Tap or drag to create a note board", title: "Notes", icon: "book", dragFactory: doc.emptyNoteboard as Doc, clickFactory: DocCast(doc.emptyNoteboard)}, - { toolTip: "Tap or drag to create an iamge", title: "Image", icon: "image", dragFactory: doc.emptyImage as Doc, clickFactory: DocCast(doc.emptyImage)}, + { toolTip: "Tap or drag to create an image", title: "Image", icon: "image", dragFactory: doc.emptyImage as Doc, clickFactory: DocCast(doc.emptyImage)}, { toolTip: "Tap or drag to create a collection", title: "Col", icon: "folder", dragFactory: doc.emptyCollection as Doc, clickFactory: DocCast(doc.emptyTab)}, { toolTip: "Tap or drag to create a webpage", title: "Web", icon: "globe-asia", dragFactory: doc.emptyWebpage as Doc, clickFactory: DocCast(doc.emptyWebpage)}, { toolTip: "Tap or drag to create a comparison box", title: "Compare", icon: "columns", dragFactory: doc.emptyComparison as Doc, clickFactory: DocCast(doc.emptyComparison)}, diff --git a/src/client/views/DocumentDecorations.tsx b/src/client/views/DocumentDecorations.tsx index 93c3e3338..96ce181f8 100644 --- a/src/client/views/DocumentDecorations.tsx +++ b/src/client/views/DocumentDecorations.tsx @@ -152,18 +152,25 @@ export class DocumentDecorations extends ObservableReactComponent<DocumentDecora onContainerDown = (e: React.PointerEvent) => { const effectiveLayoutAcl = GetEffectiveAcl(DocumentView.Selected()[0].Document); if (effectiveLayoutAcl === AclAdmin || effectiveLayoutAcl === AclEdit || effectiveLayoutAcl === AclAugment) { - setupMoveUpEvents(this, e, moveEv => this.onBackgroundMove(true, moveEv), emptyFunction, emptyFunction); + setupMoveUpEvents(this, e, moveEv => {this.onBackgroundMove(true, moveEv) + console.log("im being moved ink") + return false; + }, emptyFunction, emptyFunction); e.stopPropagation(); } }; onTitleDown = (e: React.PointerEvent) => { + console.log("im clicked") const effectiveLayoutAcl = GetEffectiveAcl(DocumentView.SelectedDocs()[0]); if (effectiveLayoutAcl === AclAdmin || effectiveLayoutAcl === AclEdit || effectiveLayoutAcl === AclAugment) { setupMoveUpEvents( this, e, - moveEv => this.onBackgroundMove(true, moveEv), + moveEv => {this.onBackgroundMove(true, moveEv) + console.log("im being dragged") + return false; + }, emptyFunction, action(() => { const selected = DocumentView.SelectedDocs().length === 1 ? DocumentView.SelectedDocs()[0] : undefined; @@ -182,7 +189,7 @@ export class DocumentDecorations extends ObservableReactComponent<DocumentDecora }; @action onBackgroundMove = (dragTitle: boolean, e: PointerEvent): boolean => { - const dragDocView = DocumentView.Selected()[0]; + const dragDocView = DocumentView.Selected()[0];`` const effectiveLayoutAcl = GetEffectiveAcl(dragDocView.Document); if (effectiveLayoutAcl !== AclAdmin && effectiveLayoutAcl !== AclEdit && effectiveLayoutAcl !== AclAugment) { return false; diff --git a/src/client/views/InkTranscription.tsx b/src/client/views/InkTranscription.tsx index 1ed8de1be..495bb6b83 100644 --- a/src/client/views/InkTranscription.tsx +++ b/src/client/views/InkTranscription.tsx @@ -1,349 +1,338 @@ -// import * as iink from 'iink-js'; -// import { action, observable } from 'mobx'; -// import * as React from 'react'; -// import { Doc, DocListCast } from '../../fields/Doc'; -// import { InkData, InkField, InkTool } from '../../fields/InkField'; -// import { Cast, DateCast, NumCast } from '../../fields/Types'; -// import { aggregateBounds } from '../../Utils'; -// import { DocumentType } from '../documents/DocumentTypes'; -// import { CollectionFreeFormView } from './collections/collectionFreeForm'; -// import { InkingStroke } from './InkingStroke'; -// import './InkTranscription.scss'; - -// /** -// * Class component that handles inking in writing mode -// */ -// export class InkTranscription extends React.Component { -// static Instance: InkTranscription; - -// @observable _mathRegister: any= undefined; -// @observable _mathRef: any= undefined; -// @observable _textRegister: any= undefined; -// @observable _textRef: any= undefined; -// private lastJiix: any; -// private currGroup?: Doc; - -// constructor(props: Readonly<{}>) { -// super(props); - -// InkTranscription.Instance = this; -// } - -// componentWillUnmount() { -// this._mathRef.removeEventListener('exported', (e: any) => this.exportInk(e, this._mathRef)); -// this._textRef.removeEventListener('exported', (e: any) => this.exportInk(e, this._textRef)); -// } - -// @action -// setMathRef = (r: any) => { -// if (!this._mathRegister) { -// this._mathRegister = r -// ? iink.register(r, { -// recognitionParams: { -// type: 'MATH', -// protocol: 'WEBSOCKET', -// server: { -// host: 'cloud.myscript.com', -// applicationKey: process.env.IINKJS_APP, -// hmacKey: process.env.IINKJS_HMAC, -// websocket: { -// pingEnabled: false, -// autoReconnect: true, -// }, -// }, -// iink: { -// math: { -// mimeTypes: ['application/x-latex', 'application/vnd.myscript.jiix'], -// }, -// export: { -// jiix: { -// strokes: true, -// }, -// }, -// }, -// }, -// }) -// : null; -// } - -// r?.addEventListener('exported', (e: any) => this.exportInk(e, this._mathRef)); - -// return (this._mathRef = r); -// }; - -// @action -// setTextRef = (r: any) => { -// if (!this._textRegister) { -// this._textRegister = r -// ? iink.register(r, { -// recognitionParams: { -// type: 'TEXT', -// protocol: 'WEBSOCKET', -// server: { -// host: 'cloud.myscript.com', -// applicationKey: '7277ec34-0c2e-4ee1-9757-ccb657e3f89f', -// hmacKey: 'f5cb18f2-1f95-4ddb-96ac-3f7c888dffc1', -// websocket: { -// pingEnabled: false, -// autoReconnect: true, -// }, -// }, -// iink: { -// text: { -// mimeTypes: ['text/plain'], -// }, -// export: { -// jiix: { -// strokes: true, -// }, -// }, -// }, -// }, -// }) -// : null; -// } - -// r?.addEventListener('exported', (e: any) => this.exportInk(e, this._textRef)); - -// return (this._textRef = r); -// }; - -// /** -// * Handles processing Dash Doc data for ink transcription. -// * -// * @param groupDoc the group which contains the ink strokes we want to transcribe -// * @param inkDocs the ink docs contained within the selected group -// * @param math boolean whether to do math transcription or not -// */ -// transcribeInk = (groupDoc: Doc | undefined, inkDocs: Doc[], math: boolean) => { -// if (!groupDoc) return; -// const validInks = inkDocs.filter(s => s.type === DocumentType.INK); - -// const strokes: InkData[] = []; -// const times: number[] = []; -// validInks -// .filter(i => Cast(i[Doc.LayoutFieldKey(i)], InkField)) -// .forEach(i => { -// const d = Cast(i[Doc.LayoutFieldKey(i)], InkField, null); -// const inkStroke = DocumentManager.Instance.getDocumentView(i)?.ComponentView as InkingStroke; -// strokes.push(d.inkData.map(pd => inkStroke.ptToScreen({ X: pd.X, Y: pd.Y }))); -// times.push(DateCast(i.author_date).getDate().getTime()); -// }); - -// this.currGroup = groupDoc; - -// const pointerData = { events: strokes.map((stroke, i) => this.inkJSON(stroke, times[i])) }; -// const processGestures = false; - -// if (math) { -// this._mathRef.editor.pointerEvents(pointerData, processGestures); -// } else { -// this._textRef.editor.pointerEvents(pointerData, processGestures); -// } -// }; - -// /** -// * Converts the Dash Ink Data to JSON. -// * -// * @param stroke The dash ink data -// * @param time the time of the stroke -// * @returns json object representation of ink data -// */ -// inkJSON = (stroke: InkData, time: number) => { -// return { -// pointerType: 'PEN', -// pointerId: 1, -// x: stroke.map(point => point.X), -// y: stroke.map(point => point.Y), -// t: new Array(stroke.length).fill(time), -// p: new Array(stroke.length).fill(1.0), -// }; -// }; - -// /** -// * Creates subgroups for each word for the whole text transcription -// * @param wordInkDocMap the mapping of words to ink strokes (Ink Docs) -// */ -// subgroupsTranscriptions = (wordInkDocMap: Map<string, Doc[]>) => { -// // iterate through the keys of wordInkDocMap -// wordInkDocMap.forEach(async (inkDocs: Doc[], word: string) => { -// const selected = inkDocs.slice(); -// if (!selected) { -// return; -// } -// const ctx = await Cast(selected[0].embedContainer, Doc); -// if (!ctx) { -// return; -// } -// const docView: CollectionFreeFormView = DocumentManager.Instance.getDocumentView(ctx)?.ComponentView as CollectionFreeFormView; - -// if (!docView) return; -// const marqViewRef = docView._marqueeViewRef.current; -// if (!marqViewRef) return; -// this.groupInkDocs(selected, docView, word); -// }); -// }; - -// /** -// * Event listener function for when the 'exported' event is heard. -// * -// * @param e the event objects -// * @param ref the ref to the editor -// */ -// exportInk = (e: any, ref: any) => { -// const exports = e.detail.exports; -// if (exports) { -// if (exports['application/x-latex']) { -// const latex = exports['application/x-latex']; -// if (this.currGroup) { -// this.currGroup.text = latex; -// this.currGroup.title = latex; -// } - -// ref.editor.clear(); -// } else if (exports['text/plain']) { -// if (exports['application/vnd.myscript.jiix']) { -// this.lastJiix = JSON.parse(exports['application/vnd.myscript.jiix']); -// // map timestamp to strokes -// const timestampWord = new Map<number, string>(); -// this.lastJiix.words.map((word: any) => { -// if (word.items) { -// word.items.forEach((i: { id: string; timestamp: string; X: Array<number>; Y: Array<number>; F: Array<number> }) => { -// const ms = Date.parse(i.timestamp); -// timestampWord.set(ms, word.label); -// }); -// } -// }); - -// const wordInkDocMap = new Map<string, Doc[]>(); -// if (this.currGroup) { -// const docList = DocListCast(this.currGroup.data); -// docList.forEach((inkDoc: Doc) => { -// // just having the times match up and be a unique value (actual timestamp doesn't matter) -// const ms = DateCast(inkDoc.author_date).getDate().getTime() + 14400000; -// const word = timestampWord.get(ms); -// if (!word) { -// return; -// } -// const entry = wordInkDocMap.get(word); -// if (entry) { -// entry.push(inkDoc); -// wordInkDocMap.set(word, entry); -// } else { -// const newEntry = [inkDoc]; -// wordInkDocMap.set(word, newEntry); -// } -// }); -// if (this.lastJiix.words.length > 1) this.subgroupsTranscriptions(wordInkDocMap); -// } -// } -// const text = exports['text/plain']; - -// if (this.currGroup) { -// this.currGroup.transcription = text; -// this.currGroup.title = text.split('\n')[0]; -// } - -// ref.editor.clear(); -// } -// } -// }; - -// /** -// * Creates the ink grouping once the user leaves the writing mode. -// */ -// createInkGroup() { -// // TODO nda - if document being added to is a inkGrouping then we can just add to that group -// if (Doc.ActiveTool === InkTool.Write) { -// CollectionFreeFormView.collectionsWithUnprocessedInk.forEach(ffView => { -// // TODO: nda - will probably want to go through ffView unprocessed docs and then see if any of the inksToGroup docs are in it and only use those -// const selected = ffView.unprocessedDocs; -// const newCollection = this.groupInkDocs( -// selected.filter(doc => doc.embedContainer), -// ffView -// ); -// ffView.unprocessedDocs = []; - -// InkTranscription.Instance.transcribeInk(newCollection, selected, false); -// }); -// } -// CollectionFreeFormView.collectionsWithUnprocessedInk.clear(); -// } - -// /** -// * Creates the groupings for a given list of ink docs on a specific doc view -// * @param selected: the list of ink docs to create a grouping of -// * @param docView: the view in which we want the grouping to be created -// * @param word: optional param if the group we are creating is a word (subgrouping individual words) -// * @returns a new collection Doc or undefined if the grouping fails -// */ -// groupInkDocs(selected: Doc[], docView: CollectionFreeFormView, word?: string): Doc | undefined { -// const bounds: { x: number; y: number; width?: number; height?: number }[] = []; - -// // calculate the necessary bounds from the selected ink docs -// selected.map( -// action(d => { -// const x = NumCast(d.x); -// const y = NumCast(d.y); -// const width = NumCast(d._width); -// const height = NumCast(d._height); -// bounds.push({ x, y, width, height }); -// }) -// ); - -// // calculate the aggregated bounds -// const aggregBounds = aggregateBounds(bounds, 0, 0); -// const marqViewRef = docView._marqueeViewRef.current; - -// // set the vals for bounds in marqueeView -// if (marqViewRef) { -// marqViewRef._downX = aggregBounds.x; -// marqViewRef._downY = aggregBounds.y; -// marqViewRef._lastX = aggregBounds.r; -// marqViewRef._lastY = aggregBounds.b; -// } - -// // map through all the selected ink strokes and create the groupings -// selected.map( -// action(d => { -// const dx = NumCast(d.x); -// const dy = NumCast(d.y); -// delete d.x; -// delete d.y; -// delete d.activeFrame; -// delete d._timecodeToShow; // bcz: this should be automatic somehow.. along with any other properties that were logically associated with the original collection -// delete d._timecodeToHide; // bcz: this should be automatic somehow.. along with any other properties that were logically associated with the original collection -// // calculate pos based on bounds -// if (marqViewRef?.Bounds) { -// d.x = dx - marqViewRef.Bounds.left - marqViewRef.Bounds.width / 2; -// d.y = dy - marqViewRef.Bounds.top - marqViewRef.Bounds.height / 2; -// } -// return d; -// }) -// ); -// docView.props.removeDocument?.(selected); -// // Gets a collection based on the selected nodes using a marquee view ref -// const newCollection = marqViewRef?.getCollection(selected, undefined, true); -// if (newCollection) { -// newCollection.width = NumCast(newCollection._width); -// newCollection.height = NumCast(newCollection._height); -// // if the grouping we are creating is an individual word -// if (word) { -// newCollection.title = word; -// } -// } - -// // nda - bug: when deleting a stroke before leaving writing mode, delete the stroke from unprocessed ink docs -// newCollection && docView.props.addDocument?.(newCollection); -// return newCollection; -// } - -// render() { -// return ( -// <div className="ink-transcription"> -// <div className="math-editor" ref={this.setMathRef} touch-action="none"></div> -// <div className="text-editor" ref={this.setTextRef} touch-action="none"></div> -// </div> -// ); -// } -// } +import * as iink from 'iink-ts'; +import {TProtocol,TSchene,TServerConfiguration,TConfiguration,TRecognitionConfiguration} from 'iink-ts'; +import { action, observable } from 'mobx'; +import * as React from 'react'; +import { Doc, DocListCast } from '../../fields/Doc'; +import { InkData, InkField, InkTool } from '../../fields/InkField'; +import { Cast, DateCast, NumCast } from '../../fields/Types'; +import { aggregateBounds } from '../../Utils'; +import { DocumentType } from '../documents/DocumentTypes'; +import { CollectionFreeFormView } from './collections/collectionFreeForm'; +import { InkingStroke } from './InkingStroke'; +import './InkTranscription.scss'; +import { DocumentView } from './nodes/DocumentView'; + +/** + * Class component that handles inking in writing mode + */ +export class InkTranscription extends React.Component { + static Instance: InkTranscription; + + @observable _mathRegister: any= undefined; + @observable _mathRef: any= undefined; + @observable _textRegister: any= undefined; + @observable _textRef: any= undefined; + private lastJiix: any; + private currGroup?: Doc; + + constructor(props: Readonly<{}>) { + super(props); + + InkTranscription.Instance = this; + } + + componentWillUnmount() { + // this._mathRef.removeEventListener('exported', (e: any) => this.exportInk(e, this._mathRef)); + // this._textRef.removeEventListener('exported', (e: any) => this.exportInk(e, this._textRef)); + } + + @action + setMathRef = async (r: any) => { + if (!this._textRegister && r) { + let editor; + const options = { + configuration: { + server: { + scheme: "https", + host: "cloud.myscript.com", + applicationKey: "c0901093-5ac5-4454-8e64-0def0f13f2ca", + hmacKey: "f6465cca-1856-4492-a6a4-e2395841be2f", + protocol: "WEBSOCKET" + }, + recognition: { + type: "TEXT", + } + } + }; + + editor = new iink.Editor(r, options as any); + + await editor.initialize(); + + this._textRegister = r; + r?.addEventListener('exported', (e: any) => this.exportInk(e, this._textRef)); + + return (this._textRef = r); + } + } + @action + setTextRef = async (r:any) => { + if (!this._textRegister && r) { + let editor; + const options = { + configuration: { + server: { + scheme: "https", + host: "cloud.myscript.com", + applicationKey: "c0901093-5ac5-4454-8e64-0def0f13f2ca", + hmacKey: "f6465cca-1856-4492-a6a4-e2395841be2f", + protocol: "WEBSOCKET" + }, + recognition: { + type: "TEXT", + } + } + }; + + editor = new iink.Editor(r, options as any); + + await editor.initialize(); + + this._textRegister = r; + r?.addEventListener('exported', (e: any) => this.exportInk(e, this._textRef)); + + return (this._textRef = r); + } + + }; + + /** + * Handles processing Dash Doc data for ink transcription. + * + * @param groupDoc the group which contains the ink strokes we want to transcribe + * @param inkDocs the ink docs contained within the selected group + * @param math boolean whether to do math transcription or not + */ + transcribeInk = (groupDoc: Doc | undefined, inkDocs: Doc[], math: boolean) => { + if (!groupDoc) return; + const validInks = inkDocs.filter(s => s.type === DocumentType.INK); + + const strokes: InkData[] = []; + const times: number[] = []; + validInks + .filter(i => Cast(i[Doc.LayoutFieldKey(i)], InkField)) + .forEach(i => { + const d = Cast(i[Doc.LayoutFieldKey(i)], InkField, null); + const inkStroke = DocumentView.getDocumentView(i)?.ComponentView as InkingStroke; + strokes.push(d.inkData.map(pd => inkStroke.ptToScreen({ X: pd.X, Y: pd.Y }))); + times.push(DateCast(i.author_date).getDate().getTime()); + }); + + this.currGroup = groupDoc; + + const pointerData = { events: strokes.map((stroke, i) => this.inkJSON(stroke, times[i])) }; + const processGestures = false; + + if (math) { + this._mathRef.editor.pointerEvents(pointerData, processGestures); + } else { + this._textRef.editor.pointerEvents(pointerData, processGestures); + } + }; + + /** + * Converts the Dash Ink Data to JSON. + * + * @param stroke The dash ink data + * @param time the time of the stroke + * @returns json object representation of ink data + */ + inkJSON = (stroke: InkData, time: number) => { + return { + pointerType: 'PEN', + pointerId: 1, + x: stroke.map(point => point.X), + y: stroke.map(point => point.Y), + t: new Array(stroke.length).fill(time), + p: new Array(stroke.length).fill(1.0), + }; + }; + + /** + * Creates subgroups for each word for the whole text transcription + * @param wordInkDocMap the mapping of words to ink strokes (Ink Docs) + */ + subgroupsTranscriptions = (wordInkDocMap: Map<string, Doc[]>) => { + // iterate through the keys of wordInkDocMap + wordInkDocMap.forEach(async (inkDocs: Doc[], word: string) => { + const selected = inkDocs.slice(); + if (!selected) { + return; + } + const ctx = await Cast(selected[0].embedContainer, Doc); + if (!ctx) { + return; + } + const docView: CollectionFreeFormView =DocumentView.getDocumentView(ctx)?.ComponentView as CollectionFreeFormView; + // DocumentManager.Instance.getDocumentView(ctx)?.ComponentView as CollectionFreeFormView; + + if (!docView) return; + const marqViewRef = docView._marqueeViewRef.current; + if (!marqViewRef) return; + this.groupInkDocs(selected, docView, word); + }); + }; + + /** + * Event listener function for when the 'exported' event is heard. + * + * @param e the event objects + * @param ref the ref to the editor + */ + exportInk = (e: any, ref: any) => { + const exports = e.detail.exports; + if (exports) { + if (exports['application/x-latex']) { + const latex = exports['application/x-latex']; + if (this.currGroup) { + this.currGroup.text = latex; + this.currGroup.title = latex; + } + + ref.editor.clear(); + } else if (exports['text/plain']) { + if (exports['application/vnd.myscript.jiix']) { + this.lastJiix = JSON.parse(exports['application/vnd.myscript.jiix']); + // map timestamp to strokes + const timestampWord = new Map<number, string>(); + this.lastJiix.words.map((word: any) => { + if (word.items) { + word.items.forEach((i: { id: string; timestamp: string; X: Array<number>; Y: Array<number>; F: Array<number> }) => { + const ms = Date.parse(i.timestamp); + timestampWord.set(ms, word.label); + }); + } + }); + + const wordInkDocMap = new Map<string, Doc[]>(); + if (this.currGroup) { + const docList = DocListCast(this.currGroup.data); + docList.forEach((inkDoc: Doc) => { + // just having the times match up and be a unique value (actual timestamp doesn't matter) + const ms = DateCast(inkDoc.author_date).getDate().getTime() + 14400000; + const word = timestampWord.get(ms); + if (!word) { + return; + } + const entry = wordInkDocMap.get(word); + if (entry) { + entry.push(inkDoc); + wordInkDocMap.set(word, entry); + } else { + const newEntry = [inkDoc]; + wordInkDocMap.set(word, newEntry); + } + }); + if (this.lastJiix.words.length > 1) this.subgroupsTranscriptions(wordInkDocMap); + } + } + const text = exports['text/plain']; + + if (this.currGroup) { + this.currGroup.transcription = text; + this.currGroup.title = text.split('\n')[0]; + } + + ref.editor.clear(); + } + } + }; + + /** + * Creates the ink grouping once the user leaves the writing mode. + */ + createInkGroup() { + // TODO nda - if document being added to is a inkGrouping then we can just add to that group + if (Doc.ActiveTool === InkTool.Write) { + CollectionFreeFormView.collectionsWithUnprocessedInk.forEach(ffView => { + // TODO: nda - will probably want to go through ffView unprocessed docs and then see if any of the inksToGroup docs are in it and only use those + const selected = ffView.unprocessedDocs; + const newCollection = this.groupInkDocs( + selected.filter(doc => doc.embedContainer), + ffView + ); + ffView.unprocessedDocs = []; + + InkTranscription.Instance.transcribeInk(newCollection, selected, false); + }); + } + CollectionFreeFormView.collectionsWithUnprocessedInk.clear(); + } + + /** + * Creates the groupings for a given list of ink docs on a specific doc view + * @param selected: the list of ink docs to create a grouping of + * @param docView: the view in which we want the grouping to be created + * @param word: optional param if the group we are creating is a word (subgrouping individual words) + * @returns a new collection Doc or undefined if the grouping fails + */ + groupInkDocs(selected: Doc[], docView: CollectionFreeFormView, word?: string): Doc | undefined { + const bounds: { x: number; y: number; width?: number; height?: number }[] = []; + + // calculate the necessary bounds from the selected ink docs + selected.map( + action(d => { + const x = NumCast(d.x); + const y = NumCast(d.y); + const width = NumCast(d._width); + const height = NumCast(d._height); + bounds.push({ x, y, width, height }); + }) + ); + + // calculate the aggregated bounds + const aggregBounds = aggregateBounds(bounds, 0, 0); + const marqViewRef = docView._marqueeViewRef.current; + + // set the vals for bounds in marqueeView + if (marqViewRef) { + marqViewRef._downX = aggregBounds.x; + marqViewRef._downY = aggregBounds.y; + marqViewRef._lastX = aggregBounds.r; + marqViewRef._lastY = aggregBounds.b; + } + + // map through all the selected ink strokes and create the groupings + selected.map( + action(d => { + const dx = NumCast(d.x); + const dy = NumCast(d.y); + delete d.x; + delete d.y; + delete d.activeFrame; + delete d._timecodeToShow; // bcz: this should be automatic somehow.. along with any other properties that were logically associated with the original collection + delete d._timecodeToHide; // bcz: this should be automatic somehow.. along with any other properties that were logically associated with the original collection + // calculate pos based on bounds + if (marqViewRef?.Bounds) { + d.x = dx - marqViewRef.Bounds.left - marqViewRef.Bounds.width / 2; + d.y = dy - marqViewRef.Bounds.top - marqViewRef.Bounds.height / 2; + } + return d; + }) + ); + docView.props.removeDocument?.(selected); + // Gets a collection based on the selected nodes using a marquee view ref + const newCollection = marqViewRef?.getCollection(selected, undefined, true); + if (newCollection) { + newCollection.width = NumCast(newCollection._width); + newCollection.height = NumCast(newCollection._height); + // if the grouping we are creating is an individual word + if (word) { + newCollection.title = word; + } + } + + // nda - bug: when deleting a stroke before leaving writing mode, delete the stroke from unprocessed ink docs + newCollection && docView.props.addDocument?.(newCollection); + return newCollection; + } + + render() { + return ( + <div className="ink-transcription"> + <div className="math-editor" ref={this.setMathRef} touch-action="none"></div> + <div className="text-editor" ref={this.setTextRef} touch-action="none"></div> + </div> + ); + } +} diff --git a/src/client/views/InkingStroke.tsx b/src/client/views/InkingStroke.tsx index 55f28f415..ce1c07f2f 100644 --- a/src/client/views/InkingStroke.tsx +++ b/src/client/views/InkingStroke.tsx @@ -104,8 +104,8 @@ export class InkingStroke extends ViewBoxAnnotatableComponent<FieldViewProps>() * analyzes the ink stroke and saves the analysis of the stroke to the 'inkAnalysis' field, * and the recognized words to the 'handwriting' */ - analyzeStrokes() { - const data: InkData = Cast(this.dataDoc[this.fieldKey], InkField)?.inkData ?? []; + analyzeStrokes=()=> { + const data: InkData = this.inkScaledData().inkData ?? []; CognitiveServices.Inking.Appliers.ConcatenateHandwriting(this.dataDoc, ['inkAnalysis', 'handwriting'], [data]); } diff --git a/src/client/views/MainView.tsx b/src/client/views/MainView.tsx index ef1bcfb64..8a3b0f27c 100644 --- a/src/client/views/MainView.tsx +++ b/src/client/views/MainView.tsx @@ -76,6 +76,7 @@ import { PresBox } from './nodes/trails'; import { AnchorMenu } from './pdf/AnchorMenu'; import { GPTPopup } from './pdf/GPTPopup/GPTPopup'; import { TopBar } from './topbar/TopBar'; +import { InkTranscription } from './InkTranscription'; const { LEFT_MENU_WIDTH, TOPBAR_HEIGHT } = require('./global/globalCssVariables.module.scss'); // prettier-ignore const _global = (window /* browser */ || global) /* node */ as any; @@ -1096,6 +1097,7 @@ export class MainView extends ObservableReactComponent<{}> { <MarqueeOptionsMenu /> <TimelineMenu /> <RichTextMenu /> + <InkTranscription/> {this.snapLines} <LightboxView key="lightbox" PanelWidth={this._windowWidth} addSplit={CollectionDockingView.AddSplit} PanelHeight={this._windowHeight} maxBorder={this.lightboxMaxBorder} /> <GPTPopup key="gptpopup" /> diff --git a/src/client/views/global/globalScripts.ts b/src/client/views/global/globalScripts.ts index 2b7de5082..bba34e302 100644 --- a/src/client/views/global/globalScripts.ts +++ b/src/client/views/global/globalScripts.ts @@ -35,6 +35,7 @@ import { ImageBox } from '../nodes/ImageBox'; import { VideoBox } from '../nodes/VideoBox'; import { WebBox } from '../nodes/WebBox'; import { RichTextMenu } from '../nodes/formattedText/RichTextMenu'; +import { InkTranscription } from '../InkTranscription'; // import { InkTranscription } from '../InkTranscription'; @@ -364,6 +365,7 @@ ScriptingGlobals.add(function toggleCharStyle(charStyle: attrname, checkResult?: export function createInkGroup(/* inksToGroup?: Doc[], isSubGroup?: boolean */) { // TODO nda - if document being added to is a inkGrouping then we can just add to that group if (Doc.ActiveTool === InkTool.Write) { + console.log('create inking group '); CollectionFreeFormView.collectionsWithUnprocessedInk.forEach(ffView => { // TODO: nda - will probably want to go through ffView unprocessed docs and then see if any of the inksToGroup docs are in it and only use those const selected = ffView.unprocessedDocs; @@ -421,14 +423,14 @@ export function createInkGroup(/* inksToGroup?: Doc[], isSubGroup?: boolean */) // TODO: nda - will probably need to go through and only remove the unprocessed selected docs ffView.unprocessedDocs = []; - // InkTranscription.Instance.transcribeInk(newCollection, selected, false); + InkTranscription.Instance.transcribeInk(newCollection, selected, false); }); } CollectionFreeFormView.collectionsWithUnprocessedInk.clear(); } function setActiveTool(tool: InkTool | Gestures, keepPrim: boolean, checkResult?: boolean) { - // InkTranscription.Instance?.createInkGroup(); + InkTranscription.Instance?.createInkGroup(); if (checkResult) { return (Doc.ActiveTool === tool && !GestureOverlay.Instance?.InkShape) || GestureOverlay.Instance?.InkShape === tool ? GestureOverlay.Instance?.KeepPrimitiveMode || ![Gestures.Circle, Gestures.Line, Gestures.Rectangle].includes(tool as Gestures) diff --git a/src/client/views/nodes/DiagramBox.scss b/src/client/views/nodes/DiagramBox.scss index d2749f1ad..4bfd4f7cb 100644 --- a/src/client/views/nodes/DiagramBox.scss +++ b/src/client/views/nodes/DiagramBox.scss @@ -1,87 +1,229 @@ -.DIYNodeBox { +.DiagramBox { + overflow:hidden; width: 100%; height: 100%; display: flex; flex-direction: column; - align-items: center; - justify-content: center; - - .DIYNodeBox-wrapper { + .buttonCollections{ + display: flex; + justify-content: center; + flex-direction: column; + height:100%; + padding:20px; + padding-right:40px; + button{ + font-size:15px; + height:100%; + width:100%; + border: none; + border-radius: 5px; + cursor: pointer; + background-color: #007bff; + color: #fff; + transition: background-color 0.3s ease; + } + button:hover { + background-color: #0056b3; + } + + } + .DiagramBox-wrapper { + overflow:hidden; width: 100%; height: 100%; display: flex; flex-direction: column; align-items: center; justify-content: center; - .DIYNodeBox { - /* existing code */ - - .DIYNodeBox-iframe { - height: 100%; - width: 100%; - border: none; - - } - } - - .search-bar { + .contentCode{ + overflow: hidden; display: flex; justify-content: center; align-items: center; - width: 100%; - padding: 10px; - - input[type="text"] { - flex: 1; - margin-right: 10px; + flex-direction:row; + padding:10px; + width:100%; + height:100%; + .topbar{ + .backButtonDrawing{ + padding: 5px 10px; + height:23px; + border-radius: 10px; + text-align: center; + padding:0; + width:50px; + font-size:10px; + position:absolute; + top:10px; + left:10px; + } + p{ + margin-left:60px + } } + .search-bar { + overflow:hidden; + position:absolute; + top:0; + .backButton{ + text-align: center; + padding:0; + width:50px; + font-size:10px; - button { - padding: 5px 10px; + } + .exampleButton{ + width:100px; + height:30px; + } + display: flex; + flex-wrap: wrap; + justify-content: space-between; + align-items: center; + width: 100%; + button { + padding: 5px 10px; + width:80px; + height:23px; + border-radius: 3px; + } + } + .exampleButtonContainer{ + display:flex; + flex-direction: column; + position: absolute; + top:37px; + right:30px; + width:50px; + z-index: 200; + button{ + width:70px; + margin:2px; + padding:0px; + height:15px; + border-radius: 3px; + } + } + textarea { + position:relative; + width:40%; + height: 100%; + height: calc(100% - 25px); + top:15px; + resize:none; + overflow: hidden; + } + .diagramBox{ + flex: 1; + display: flex; + justify-content: center; + align-items: center; + svg{ + position: relative; + top:25; + max-width: none !important; + height: calc(100% - 50px); + } } } - .content { - flex: 1; + overflow: hidden; display: flex; justify-content: center; align-items: center; + flex-direction: column; + padding:10px; width:100%; height:100%; - .diagramBox{ - flex: 1; + .topbar{ + .backButtonDrawing{ + padding: 5px 10px; + height:23px; + border-radius: 10px; + text-align: center; + padding:0; + width:50px; + font-size:10px; + position:absolute; + top:10px; + left:10px; + } + p{ + margin-left:60px + } + } + .search-bar { + overflow:hidden; + position:absolute; + top:0; + .backButton{ + text-align: center; + padding:0; + width:50px; + font-size:10px; + + } display: flex; + flex-wrap: wrap; justify-content: center; align-items: center; - width:100%; - height:100%; - svg{ + width: 100%; + textarea { flex: 1; - display: flex; - justify-content: center; - align-items: center; - width:100%; - height:100%; + height: 5px; + min-height: 20px; + resize:none; + overflow: hidden; + } + button { + padding: 5px 10px; + width:80px; + height:23px; + border-radius: 10px; + } + .rightButtons{ + display:flex; + flex-direction: column; + button { + padding: 5px 10px; + width:80px; + height:23px; + margin:2; + border-radius: 10px; + } } } - } - - .loading-circle { - position: relative; - width: 50px; - height: 50px; - border-radius: 50%; - border: 3px solid #ccc; - border-top-color: #333; - animation: spin 1s infinite linear; - } - - @keyframes spin { - 0% { - transform: rotate(0deg); + .loading-circle { + position: absolute; + display:flex; + align-items: center; + justify-content: center; + width: 50px; + height: 50px; + border-radius: 50%; + border: 3px solid #ccc; + border-top-color: #333; + animation: spin 1s infinite linear; } - 100% { - transform: rotate(360deg); + @keyframes spin { + 0% { + transform: rotate(0deg); + } + 100% { + transform: rotate(360deg); + } + } + .diagramBox{ + flex: 1; + display: flex; + justify-content: center; + align-items: center; + svg{ + position: relative; + top:25; + max-width: none !important; + height: calc(100% - 50px); + } } } } diff --git a/src/client/views/nodes/DiagramBox.tsx b/src/client/views/nodes/DiagramBox.tsx index 32969fa53..fcc028aab 100644 --- a/src/client/views/nodes/DiagramBox.tsx +++ b/src/client/views/nodes/DiagramBox.tsx @@ -1,20 +1,32 @@ +/* eslint-disable prettier/prettier */ +/* eslint-disable jsx-a11y/control-has-associated-label */ import mermaid from 'mermaid'; -import { action, makeObservable, observable, reaction } from 'mobx'; +import { action, makeObservable, observable, reaction, computed } from 'mobx'; import { observer } from 'mobx-react'; import * as React from 'react'; import { Doc, DocListCast } from '../../../fields/Doc'; import { List } from '../../../fields/List'; import { RichTextField } from '../../../fields/RichTextField'; -import { DocCast, NumCast } from '../../../fields/Types'; +import { DocCast,BoolCast } from '../../../fields/Types'; import { GPTCallType, gptAPICall } from '../../apis/gpt/GPT'; import { DocumentType } from '../../documents/DocumentTypes'; import { Docs } from '../../documents/Documents'; import { DocumentManager } from '../../util/DocumentManager'; import { LinkManager } from '../../util/LinkManager'; import { ViewBoxAnnotatableComponent } from '../DocComponent'; -import { InkingStroke } from '../InkingStroke'; +import { InkingStroke} from '../InkingStroke'; import './DiagramBox.scss'; import { FieldView, FieldViewProps } from './FieldView'; +import { PointData } from '../../../pen-gestures/GestureTypes'; +import { InkField } from '../../../fields/InkField'; + +enum menuState { + option, + mermaidCode, + drawing, + gpt, + justCreated, +} @observer export class DiagramBox extends ViewBoxAnnotatableComponent<FieldViewProps>() { @@ -27,60 +39,97 @@ export class DiagramBox extends ViewBoxAnnotatableComponent<FieldViewProps>() { super(props); makeObservable(this); } - + @observable menuState = menuState.justCreated; + @observable renderDiv: React.ReactNode; @observable inputValue = ''; + @observable createInputValue = ''; @observable loading = false; @observable errorMessage = ''; @observable mermaidCode = ''; + @observable isExampleMenuOpen = false; - @action handleInputChange = (e: React.ChangeEvent<HTMLInputElement>) => { + @action handleInputChange = (e: React.ChangeEvent<HTMLTextAreaElement>) => { this.inputValue = e.target.value; + console.log(e.target.value); }; async componentDidMount() { this._props.setContentViewBox?.(this); mermaid.initialize({ securityLevel: 'loose', startOnLoad: true, - flowchart: { useMaxWidth: true, htmlLabels: true, curve: 'cardinal' }, + darkMode: true, + flowchart: { useMaxWidth: false, htmlLabels: true, curve: 'cardinal' }, + gantt: { useMaxWidth: true, useWidth: 2000 }, }); - this.mermaidCode = 'asdasdasd'; - const docArray: Doc[] = DocListCast(this.Document.data); - let mermaidCodeDoc = docArray.filter(doc => doc.type === 'rich text'); - mermaidCodeDoc = mermaidCodeDoc.filter(doc => (doc.text as RichTextField).Text === 'mermaidCodeTitle'); - if (mermaidCodeDoc[0]) { - if (typeof mermaidCodeDoc[0].title === 'string') { - console.log(mermaidCodeDoc[0].title); - if (mermaidCodeDoc[0].title !== '') { - this.renderMermaidAsync(mermaidCodeDoc[0].title); - } - } + if (!this.Document.testValue) { + this.Document.height = 500; + this.Document.width = 500; } - // this will create a text doc far away where the user cant to save the mermaid code, where it will then be accessed when flipped to the diagram box side - // the code is stored in the title since it is much easier to change than in the text - else { - DocumentManager.Instance.AddViewRenderedCb(this.Document, docViewForYourCollection => { - if (docViewForYourCollection && docViewForYourCollection.ComponentView) { - if (docViewForYourCollection.ComponentView.addDocument && docViewForYourCollection.ComponentView.removeDocument) { - const newDoc = Docs.Create.TextDocument('mermaidCodeTitle', { title: '', x: 9999 + NumCast(this.layoutDoc._width), y: 9999 }); - docViewForYourCollection.ComponentView?.addDocument(newDoc); - } - } - }); + this.Document.testValue = 'a'; + this.mermaidCode = 'a'; + if (typeof this.Document.drawingMermaidCode === 'string' && this.Document.menuState === 'drawing') { + this.renderMermaidAsync(this.Document.drawingMermaidCode); } - console.log(this.Document.title); // this is so that ever time a new doc, text node or ink node, is created, this.createMermaidCode will run which will create a save - reaction( - () => DocListCast(this.Document.data), - () => this.convertDrawingToMermaidCode(), - { fireImmediately: true } - ); + // reaction( + // () => DocListCast(this.Document.data), + // () => this.lockInkStroke(), + // { fireImmediately: true } + // ); + // reaction( + // () => + // DocListCast(this.Document.data) + // .filter(doc => doc.type === 'rich text') + // .map(doc => (doc.text as RichTextField).Text), + // () => this.convertDrawingToMermaidCode(), + // { fireImmediately: true } + // ); + // const rectangleXValues = computed(() => + // DocListCast(this.Document.data) + // .filter(doc => doc.title === 'rectangle') + // .map(doc => doc.x) + // ); + // reaction( + // () => rectangleXValues.get(), + // () => this.lockInkStroke(), + // { fireImmediately: true } + // ); + this.lockInkStroke(); + } + + componentDidUpdate = () => { + if (typeof this.Document.drawingMermaidCode === 'string' && this.Document.menuState === 'drawing') { + this.renderMermaidAsync(this.Document.drawingMermaidCode); + } + if (typeof this.Document.gptMermaidCode === 'string' && this.Document.menuState === 'gpt') { + this.renderMermaidAsync(this.Document.gptMermaidCode); + } + }; + switchRenderDiv() { + switch (this.Document.menuState) { + case 'option': + this.renderDiv = this.renderOption(); + break; + case 'drawing': + this.renderDiv = this.renderDrawing(); + break; + case 'gpt': + this.renderDiv = this.renderGpt(); + break; + case 'mermaidCode': + this.renderDiv = this.renderMermaidCode(); + break; + default: + this.menuState = menuState.option; + this.renderDiv = this.renderOption(); + } } renderMermaid = async (str: string) => { try { const { svg, bindFunctions } = await this.mermaidDiagram(str); return { svg, bindFunctions }; } catch (error) { - console.error('Error rendering mermaid diagram:', error); + // console.error('Error rendering mermaid diagram:', error); return { svg: '', bindFunctions: undefined }; } }; @@ -92,6 +141,7 @@ export class DiagramBox extends ViewBoxAnnotatableComponent<FieldViewProps>() { const dashDiv = document.getElementById('dashDiv' + this.Document.title); if (dashDiv) { dashDiv.innerHTML = svg; + // this.changeHeightWidth(svg); if (bindFunctions) { bindFunctions(dashDiv); } @@ -100,51 +150,51 @@ export class DiagramBox extends ViewBoxAnnotatableComponent<FieldViewProps>() { console.error('Error rendering Mermaid:', error); } } + changeHeightWidth(svgString: string) { + const pattern = /width="([\d.]+)"\s*height="([\d.]+)"/; + + const match = svgString.match(pattern); + + if (match) { + const width = parseFloat(match[1]); + const height = parseFloat(match[2]); + console.log(width); + console.log(height); + this.Document.width = width; + this.Document.height = height; + } + } @action handleRenderClick = () => { - this.generateMermaidCode(); + this.mermaidCode = ''; + if (this.inputValue) { + this.generateMermaidCode(); + } }; @action async generateMermaidCode() { console.log('Generating Mermaid Code'); + const dashDiv = document.getElementById('dashDiv' + this.Document.title); + if (dashDiv) { + dashDiv.innerHTML = ''; + } this.loading = true; let prompt = ''; - // let docArray: Doc[] = DocListCast(this.Document.data); - // let mermaidCodeDoc = docArray.filter(doc => doc.type == 'rich text') - // mermaidCodeDoc=mermaidCodeDoc.filter(doc=>(doc.text as RichTextField).Text=='mermaidCodeTitle') - // if(mermaidCodeDoc[0]){ - // console.log(mermaidCodeDoc[0].title) - // if(typeof mermaidCodeDoc[0].title=='string'){ - // console.log(mermaidCodeDoc[0].title) - // if(mermaidCodeDoc[0].title!=""){ - // prompt="Edit this code "+this.inputValue+": "+mermaidCodeDoc[0].title - // console.log("you have to see me") - // } - // } - // } - // else{ prompt = 'Write this in mermaid code and only give me the mermaid code: ' + this.inputValue; - console.log('there is no text save'); // } const res = await gptAPICall(prompt, GPTCallType.MERMAID); - this.loading = false; + this.loading = true; if (res === 'Error connecting with API.') { // If GPT call failed console.error('GPT call failed'); this.errorMessage = 'GPT call failed; please try again.'; } else if (res !== null) { // If GPT call succeeded, set htmlCode;;; TODO: check if valid html - if (this.isValidCode(res)) { - this.mermaidCode = res; - console.log('GPT call succeeded:' + res); - this.errorMessage = ''; - } else { - console.error('GPT call succeeded but invalid html; please try again.'); - this.errorMessage = 'GPT call succeeded but invalid html; please try again.'; - } + this.mermaidCode = res; + console.log('GPT call succeeded:' + res); + this.errorMessage = ''; } this.renderMermaidAsync.call(this, this.removeWords(this.mermaidCode)); - this.loading = false; + this.Document.gptMermaidCode = this.removeWords(this.mermaidCode); } - isValidCode = (html: string) => true; removeWords(inputStrIn: string) { const inputStr = inputStrIn.replace('```mermaid', ''); return inputStr.replace('```', ''); @@ -164,10 +214,12 @@ export class DiagramBox extends ViewBoxAnnotatableComponent<FieldViewProps>() { }); await timeoutPromise(); const inkStrokeArray = lineArray.map(doc => DocumentManager.Instance.getDocumentView(doc, this.DocumentView?.())).filter(inkView => inkView?.ComponentView instanceof InkingStroke); - console.log(inkStrokeArray.length); - console.log(lineArray.length); if (inkStrokeArray[0] && inkStrokeArray.length === lineArray.length) { - mermaidCode = 'graph TD;'; + // if (this.isLeftRightDiagram(docArray)) { + // mermaidCode = 'graph LR;'; + // } else { + // mermaidCode = 'graph TD;'; + // } const inkingStrokeArray = inkStrokeArray.map(stroke => stroke?.ComponentView); for (let i = 0; i < rectangleArray.length; i++) { const rectangle = rectangleArray[i]; @@ -182,8 +234,6 @@ export class DiagramBox extends ViewBoxAnnotatableComponent<FieldViewProps>() { ?.inkScaledData() .inkData.map(coord => coord.Y) .map(doc => doc * inkScaleY); - console.log(inkingStrokeArray.length); - console.log(lineArray.length); // need to minX and minY to since the inkStroke.x and.y is not relative to the doc. so I have to do some calcluations const minX: number = Math.min(...inkStrokeXArray); const minY: number = Math.min(...inkStrokeYArray); @@ -197,12 +247,11 @@ export class DiagramBox extends ViewBoxAnnotatableComponent<FieldViewProps>() { if (this.isPointInBox(rectangle2, [endX, endY]) && typeof rectangle.x === 'number' && typeof rectangle2.x === 'number') { diagramExists = true; const linkedDocs: Doc[] = LinkManager.Instance.getAllRelatedLinks(lineArray[j]).map(d => DocCast(LinkManager.getOppositeAnchor(d, lineArray[j]))); - console.log(linkedDocs.length); if (linkedDocs.length !== 0) { const linkedText = (linkedDocs[0].text as RichTextField).Text; - mermaidCode += Math.abs(rectangle.x) + this.getTextInBox(rectangle, textArray) + '-->|' + linkedText + '|' + Math.abs(rectangle2.x) + this.getTextInBox(rectangle2, textArray) + ';'; + mermaidCode += Math.abs(rectangle.x) + this.getTextInBox(rectangle, textArray) + '---|' + linkedText + '|' + Math.abs(rectangle2.x) + this.getTextInBox(rectangle2, textArray) + ';'; } else { - mermaidCode += Math.abs(rectangle.x) + this.getTextInBox(rectangle, textArray) + '-->' + Math.abs(rectangle2.x) + this.getTextInBox(rectangle2, textArray) + ';'; + mermaidCode += Math.abs(rectangle.x) + this.getTextInBox(rectangle, textArray) + '---' + Math.abs(rectangle2.x) + this.getTextInBox(rectangle2, textArray) + ';'; } } } @@ -210,35 +259,160 @@ export class DiagramBox extends ViewBoxAnnotatableComponent<FieldViewProps>() { } } // this will save the text - DocumentManager.Instance.AddViewRenderedCb(this.Document, docViewForYourCollection => { - if (docViewForYourCollection && docViewForYourCollection.ComponentView) { - if (docViewForYourCollection.ComponentView.addDocument && docViewForYourCollection.ComponentView.removeDocument) { - let docs: Doc[] = DocListCast(this.Document.data); - docs = docs.filter(doc => doc.type === 'rich text'); - const mermaidCodeDoc = docs.filter(doc => (doc.text as RichTextField).Text === 'mermaidCodeTitle'); - if (mermaidCodeDoc[0]) { - if (diagramExists) { - mermaidCodeDoc[0].title = mermaidCode; - } else { - mermaidCodeDoc[0].title = ''; - } - } - } - } - }); + if (diagramExists) { + this.Document.drawingMermaidCode = mermaidCode; + } else { + this.Document.drawingMermaidCode = ''; + } } } } - testInkingStroke = () => { + async lockInkStroke() { + console.log("hello") + console.log(DocListCast(this.Document.data).filter(doc => doc.title === 'rectangle').map(doc => doc.x)) if (this.Document.data instanceof List) { const docArray: Doc[] = DocListCast(this.Document.data); + const rectangleArray = docArray.filter(doc => doc.title === 'rectangle' || doc.title === 'circle'); + if(rectangleArray[0]){ + console.log(rectangleArray[0].x) + } const lineArray = docArray.filter(doc => doc.title === 'line' || doc.title === 'stroke'); - setTimeout(() => { - const inkStrokeArray = lineArray.map(doc => DocumentManager.Instance.getDocumentView(doc, this.DocumentView?.())).filter(inkView => inkView?.ComponentView instanceof InkingStroke); - console.log(inkStrokeArray); - }); + const timeoutPromise = () => + new Promise(resolve => { + setTimeout(resolve, 0); + }); + await timeoutPromise(); + const inkStrokeArray = lineArray.map(doc => DocumentManager.Instance.getDocumentView(doc, this.DocumentView?.())).filter(inkView => inkView?.ComponentView instanceof InkingStroke); + const inkingStrokeArray = inkStrokeArray.map(stroke => stroke?.ComponentView); + for (let j = 0; j < lineArray.length; j++) { + const inkScaleX = (inkingStrokeArray[j] as InkingStroke)?.inkScaledData().inkScaleX; + const inkScaleY = (inkingStrokeArray[j] as InkingStroke)?.inkScaledData().inkScaleY; + const inkStrokeXArray = (inkingStrokeArray[j] as InkingStroke) + ?.inkScaledData() + .inkData.map(coord => coord.X) + .map(doc => doc * inkScaleX); + const inkStrokeYArray = (inkingStrokeArray[j] as InkingStroke) + ?.inkScaledData() + .inkData.map(coord => coord.Y) + .map(doc => doc * inkScaleY); + // need to minX and minY to since the inkStroke.x and.y is not relative to the doc. so I have to do some calcluations + const minX: number = Math.min(...inkStrokeXArray); + const minY: number = Math.min(...inkStrokeYArray); + const startX = inkStrokeXArray[0] - minX + (lineArray[j]?.x as number); + const startY = inkStrokeYArray[0] - minY + (lineArray[j]?.y as number); + const endX = inkStrokeXArray[inkStrokeXArray.length - 1] - minX + (lineArray[j].x as number); + const endY = inkStrokeYArray[inkStrokeYArray.length - 1] - minY + (lineArray[j].y as number); + let closestStartRect:Doc=lineArray[0]; + let closestStartDistance=9999999; + let closestEndRect:Doc=lineArray[0]; + let closestEndDistance=9999999; + rectangleArray.forEach(rectangle => { + const midPoint=this.getMidPoint(rectangle) + if(this.euclideanDistance(midPoint.X,midPoint.Y,startX,startY)<closestStartDistance&&this.euclideanDistance(midPoint.X,midPoint.Y,endX,endY)<closestEndDistance){ + if(this.euclideanDistance(midPoint.X,midPoint.Y,startX,startY)<this.euclideanDistance(midPoint.X,midPoint.Y,endX,endY)){ + closestStartDistance=this.euclideanDistance(midPoint.X,midPoint.Y,startX,startY); + closestStartRect=rectangle + } + else{ + closestEndDistance=this.euclideanDistance(midPoint.X,midPoint.Y,startX,startY); + closestEndRect=rectangle + } + } + else if(this.euclideanDistance(midPoint.X,midPoint.Y,startX,startY)<closestStartDistance){ + closestStartDistance=this.euclideanDistance(midPoint.X,midPoint.Y,startX,startY); + closestStartRect=rectangle + } + else if(this.euclideanDistance(midPoint.X,midPoint.Y,endX,endY)<closestEndDistance){ + closestEndDistance=this.euclideanDistance(midPoint.X,midPoint.Y,startX,startY); + closestEndRect=rectangle + } + + }); + const inkToDelete:Doc=lineArray[j]; + if(typeof closestStartRect.x==='number'&&typeof closestStartRect.y==='number'&&typeof closestEndRect.x==='number'&&typeof closestEndRect.y==='number'&&typeof closestStartRect.width==='number'&&typeof closestStartRect.height==='number'&&typeof closestEndRect.height==='number'&&typeof closestEndRect.width==='number'){ + const points: PointData[] = [ + { X: closestStartRect.x, Y: closestStartRect.y }, + { X: closestStartRect.x, Y: closestStartRect.y }, + { X: closestEndRect.x, Y: closestEndRect.y }, + { X: closestEndRect.x, Y: closestEndRect.y } + ]; + let inkX=0; + let inkY=0; + if(this.getMidPoint(closestEndRect).X<this.getMidPoint(closestStartRect).X){ + inkX=this.getMidPoint(closestEndRect).X + } + else{ + inkX=this.getMidPoint(closestStartRect).X + } + if(this.getMidPoint(closestEndRect).Y<this.getMidPoint(closestStartRect).Y){ + inkY=this.getMidPoint(closestEndRect).Y + } + else{ + inkY=this.getMidPoint(closestStartRect).Y + } + const newInkDoc:Doc=Docs.Create.InkDocument( + points, + { title: 'stroke', + x: inkX, + y: inkY, + _width: Math.abs(closestEndRect.x+closestEndRect.width/2-closestStartRect.x-closestStartRect.width/2), + _height: Math.abs(closestEndRect.y+closestEndRect.height/2-closestStartRect.y-closestStartRect.height/2), + stroke_showLabel: BoolCast(Doc.UserDoc().activeInkHideTextLabels)}, // prettier-ignore + 1) + + DocumentManager.Instance.AddViewRenderedCb(this.Document, (docViewForYourCollection) => { + if (docViewForYourCollection && docViewForYourCollection.ComponentView) { + if (docViewForYourCollection.ComponentView.addDocument&&docViewForYourCollection.ComponentView.removeDocument) { + docViewForYourCollection.ComponentView?.removeDocument(inkToDelete) + docViewForYourCollection.ComponentView?.addDocument(newInkDoc); + + + + // const bruh2= DocListCast(this.Document.data).filter(doc => doc.title === 'line' || doc.title === 'stroke').map(doc => DocumentManager.Instance.getDocumentView(doc, this.DocumentView?.())).filter(inkView => inkView?.ComponentView instanceof InkingStroke).map(stroke => stroke?.ComponentView); + // console.log(bruh2) + // console.log((bruh2[0] as InkingStroke)?.inkScaledData()) + } + } + }); + } + } } - }; + } + getMidPoint(rectangle:Doc){ + let midPoint={X:0,Y:0} + if(typeof rectangle.x ==='number' && typeof rectangle.width==='number'&&typeof rectangle.y ==='number' && typeof rectangle.height==='number'){ + midPoint={X:rectangle.x+rectangle.width/2,Y:rectangle.y+rectangle.height/2}; + } + return midPoint + } + euclideanDistance(x1: number, y1: number, x2: number, y2: number): number { + const deltaX = x2 - x1; + const deltaY = y2 - y1; + return Math.sqrt(deltaX * deltaX + deltaY * deltaY); + } + // isLeftRightDiagram = (docArray: Doc[]) => { + // const filteredDocs = docArray.filter(doc => doc.title === 'rectangle' || doc.title === 'circle'); + // const xDoc = filteredDocs.map(doc => doc.x) as number[]; + // const minX = Math.min(...xDoc); + // const xWidthDoc = filteredDocs.map(doc => { + // if (typeof doc.x === 'number' && typeof doc.width === 'number') { + // return doc.x + doc.width; + // } + // }) as number[]; + // const maxX = Math.max(...xWidthDoc); + // const yDoc = filteredDocs.map(doc => doc.y) as number[]; + // const minY = Math.min(...yDoc); + // const yHeightDoc = filteredDocs.map(doc => { + // if (typeof doc.x === 'number' && typeof doc.width === 'number') { + // return doc.x + doc.width; + // } + // }) as number[]; + // const maxY = Math.max(...yHeightDoc); + // if (maxX - minX > maxY - minY) { + // return true; + // } + // return false; + // }; getTextInBox = (box: Doc, richTextArray: Doc[]): string => { for (let i = 0; i < richTextArray.length; i++) { const textDoc = richTextArray[i]; @@ -261,31 +435,262 @@ export class DiagramBox extends ViewBoxAnnotatableComponent<FieldViewProps>() { } return false; }; + drawingButton = () => { + this.Document.menuState = 'drawing'; + }; + gptButton = () => { + this.Document.menuState = 'gpt'; + }; + mermaidButton = () => { + this.Document.menuState = 'mermaidCode'; + }; + optionButton = () => { + this.Document.menuState = 'option'; + }; + renderOption(): React.ReactNode { + return ( + <div className="buttonCollections"> + <button type="button" onClick={this.drawingButton}> + Drawing - Create diagram from ink drawing + </button> + <button type="button" onClick={this.gptButton}> + GPT - Generate diagram with AI prompt + </button> + <button type="button" onClick={this.mermaidButton}> + Mermaid Editor - Create diagram with mermaid code + </button> + </div> + ); + } + renderDrawing(): React.ReactNode { + return ( + <div ref={this._dragRef} className="DiagramBox-wrapper"> + <div className="content"> + <div className="topBar"> + <button className="backButtonDrawing" type="button" onClick={this.optionButton}> + Back + </button> + {!this.Document.mermaidCode && <p>Click the red pen icon to flip onto the collection side and draw a diagram with ink</p>} + </div> + <div id={'dashDiv' + this.Document.title} className="diagramBox" /> + </div> + </div> + ); + } - render() { + renderGpt(): React.ReactNode { return ( - <div ref={this._ref} className="DIYNodeBox"> - <div ref={this._dragRef} className="DIYNodeBox-wrapper"> + <div ref={this._dragRef} className="DiagramBox-wrapper"> + <div className="content"> <div className="search-bar"> - <input type="text" value={this.inputValue} onChange={this.handleInputChange} /> - <button type="button" onClick={this.handleRenderClick}> - Generate + <button className="backButton" type="button" onClick={this.optionButton}> + Back </button> + <textarea value={this.inputValue} placeholder="Enter GPT prompt" onChange={this.handleInputChange}onInput={e => this.autoResize(e.target as HTMLTextAreaElement)} /> + <div className="rightButtons"> + <button className="generateButton" type="button" onClick={this.handleRenderClick}> + Generate + </button> + <button className="convertButton" type="button" onClick={this.handleConvertButton}> + Edit + </button> + </div> </div> - <div className="content"> - {this.mermaidCode ? ( - <div id={'dashDiv' + this.Document.title} className="diagramBox" /> - ) : ( - <div>{this.loading ? <div className="loading-circle" /> : <div>{this.errorMessage ? this.errorMessage : 'Insert prompt to generate diagram'}</div>}</div> - )} + {this.mermaidCode ? ( + <div id={'dashDiv' + this.Document.title} className="diagramBox" /> + ) : ( + <div>{this.loading ? <div className="loading-circle" /> : <div>{this.errorMessage ? this.errorMessage : 'Insert prompt to generate diagram'}</div>}</div> + )} + </div> + </div> + ); + } + handleConvertButton=()=>{ + this.Document.menuState = 'mermaidCode'; + if(typeof this.Document.gptMermaidCode==='string'){ + this.createInputValue=this.removeFirstEmptyLine(this.Document.gptMermaidCode); + console.log(this.Document.gptMermaidCode) + this.renderMermaidAsync(this.Document.gptMermaidCode); + } + } + removeFirstEmptyLine(input: string): string { + const lines = input.split('\n'); + let emptyLineRemoved = false; + const resultLines = lines.filter(line => { + if (!emptyLineRemoved && line.trim() === '') { + emptyLineRemoved = true; + return false; + } + return true; + }); + return resultLines.join('\n'); + } + + renderMermaidCode(): React.ReactNode { + return ( + <div ref={this._dragRef} className="DiagramBox-wrapper"> + <div className="contentCode"> + <div className="search-bar"> + <button className="backButton" type="button" onClick={this.optionButton}> + Back + </button> + <button className="exampleButton" type="button" onClick={this.exampleButton}> + Examples + </button> </div> + {this.isExampleMenuOpen && ( + <div className="exampleButtonContainer"> + <button type="button" onClick={this.flowButton}> + Flow + </button> + <button type="button" onClick={this.pieButton}> + Pie + </button> + <button type="button" onClick={this.timelineButton}> + Timeline + </button> + <button type="button" onClick={this.classButton}> + Class + </button> + <button type="button" onClick={this.mindmapButton}> + Mindmap + </button> + </div> + )} + <textarea value={this.createInputValue} placeholder="Enter Mermaid Code" onChange={this.handleInputChangeEditor}/> + <div id={'dashDiv' + this.Document.title} className="diagramBox" /> </div> </div> ); } + exampleButton = () => { + if (this.isExampleMenuOpen) { + this.isExampleMenuOpen = false; + } else { + this.isExampleMenuOpen = true; + } + }; + flowButton = () => { + this.createInputValue = `flowchart TD + A[Christmas] -->|Get money| B(Go shopping) + B --> C{Let me think} + C -->|One| D[Laptop] + C -->|Two| E[iPhone] + C -->|Three| F[fa:fa-car Car]` + this.renderMermaidAsync(this.createInputValue); + }; + pieButton = () => { + this.createInputValue = `pie title Pets adopted by volunteers + "Dogs" : 386 + "Cats" : 85 + "Rats" : 15` + this.renderMermaidAsync(this.createInputValue); + }; + timelineButton = () => { + this.createInputValue = `gantt + title A Gantt Diagram + dateFormat YYYY-MM-DD + section Section + A task :a1, 2014-01-01, 30d + Another task :after a1 , 20d + section Another + Task in sec :2014-01-12 , 12d + another task : 24d`; + this.renderMermaidAsync(this.createInputValue); + }; + classButton = () => { + this.createInputValue = `classDiagram + Animal <|-- Duck + Animal <|-- Fish + Animal <|-- Zebra + Animal : +int age + Animal : +String gender + Animal: +isMammal() + Animal: +mate() + class Duck{ + +String beakColor + +swim() + +quack() + } + class Fish{ + -int sizeInFeet + -canEat() + } + class Zebra{ + +bool is_wild + +run() + }`; + this.renderMermaidAsync(this.createInputValue); + }; + mindmapButton = () => { + this.createInputValue = `mindmap + root((mindmap)) + Origins + Long history + ::icon(fa fa-book) + Popularisation + British popular psychology author Tony Buzan + Research + On effectivness<br/>and features + On Automatic creation + Uses + Creative techniques + Strategic planning + Argument mapping + Tools + Pen and paper + Mermaid`; + this.renderMermaidAsync(this.createInputValue); + }; + handleInputChangeEditor = (e: React.ChangeEvent<HTMLTextAreaElement>) => { + if (typeof e.target.value === 'string') { + this.createInputValue = e.target.value; + this.renderMermaidAsync(e.target.value); + } + }; + removeWhitespace(str: string): string { + return str.replace(/\s+/g, ''); + } + autoResize(element: HTMLTextAreaElement): void { + element.style.height = '5px'; + element.style.height = element.scrollHeight + 'px'; + } + timeline = `gantt + title College Timeline + dateFormat YYYY-MM-DD + section Semester 1 + Orientation :done, des1, 2023-08-01, 2023-08-03 + Classes Start :active, des2, 2023-08-04, 2023-12-15 + Midterm Exams : des3, 2023-10-15, 2023-10-20 + End of Semester : des4, 2023-12-16, 2023-12-20 + section Semester 2 + Classes Start : des5, 2024-01-10, 2024-05-15 + Spring Break : des6, 2024-03-15, 2024-03-22 + Midterm Exams : des7, 2024-03-25, 2024-03-30 + Final Exams : des8, 2024-05-10, 2024-05-15 + section Summer Break + Internship : des9, 2024-06-01, 2024-08-31 + section Semester 3 + Classes Start : des10, 2024-09-01, 2025-12-15 + Midterm Exams : des11, 2024-11-15, 2024-11-20 + End of Semester : des12, 2025-12-16, 2025-12-20 + section Semester 4 + Classes Start : des13, 2025-01-10, 2025-05-15 + Spring Break : des14, 2025-03-15, 2025-03-22 + Midterm Exams : des15, 2025-03-25, 2025-03-30 + Final Exams : des16, 2025-05-10, 2025-05-15 + Graduation : des17, 2025-05-20, 2025-05-21`; + render() { + this.switchRenderDiv(); + return ( + <div ref={this._ref} className="DiagramBox"> + {this.renderDiv} + </div> + ); + } } Docs.Prototypes.TemplateMap.set(DocumentType.DIAGRAM, { layout: { view: DiagramBox, dataField: 'dadta' }, - options: { _height: 300, _layout_fitWidth: true, _layout_nativeDimEditable: true, _layout_reflowVertical: true, waitForDoubleClickToClick: 'always', systemIcon: 'BsGlobe' }, + options: { _height: 700, _width: 700, _layout_fitWidth: false, _layout_nativeDimEditable: true, _layout_reflowVertical: true, waitForDoubleClickToClick: 'always', _layout_reflowHorizontal: true, systemIcon: 'BsGlobe' }, }); diff --git a/src/client/views/nodes/ScreenshotBox.tsx b/src/client/views/nodes/ScreenshotBox.tsx index 3be50f5e6..e6590958b 100644 --- a/src/client/views/nodes/ScreenshotBox.tsx +++ b/src/client/views/nodes/ScreenshotBox.tsx @@ -149,7 +149,7 @@ export class ScreenshotBox extends ViewBoxAnnotatableComponent<FieldViewProps>() componentDidMount() { this.dataDoc.nativeWidth = this.dataDoc.nativeHeight = 0; - this._props.setContentViewBox?.(this); // this tells the DocumentView that this ScreenshotBox is the "content" of the document. this allows the DocumentView to indirectly call getAnchor() on the AudioBox when making a link. + this._props.setContentViewBox?.(this); // this tells the DocumentView that this Box is the "content" of the document. this allows the DocumentView to indirectly call getAnchor() on the AudioBox when making a link. // this.layoutDoc.videoWall && reaction(() => ({ width: this._props.PanelWidth(), height: this._props.PanelHeight() }), // ({ width, height }) => { // if (this._camera) { |