From ed5a7755eba7563c01bc3cc40840d0ebb6cd8826 Mon Sep 17 00:00:00 2001 From: Zachary Zhang Date: Wed, 6 Mar 2024 19:46:09 -0500 Subject: add diagram document --- src/client/documents/DocumentTypes.ts | 1 + 1 file changed, 1 insertion(+) (limited to 'src/client/documents/DocumentTypes.ts') diff --git a/src/client/documents/DocumentTypes.ts b/src/client/documents/DocumentTypes.ts index 1123bcac9..570d2e9c8 100644 --- a/src/client/documents/DocumentTypes.ts +++ b/src/client/documents/DocumentTypes.ts @@ -12,6 +12,7 @@ export enum DocumentType { REC = 'recording', PDF = 'pdf', INK = 'ink', + DIAGRAM='diagram', SCREENSHOT = 'screenshot', FONTICON = 'fonticonbox', SEARCH = 'search', // search query -- cgit v1.2.3-70-g09d2 From e0641c3175bc7cf53dea924524e51f1eefa6a8b1 Mon Sep 17 00:00:00 2001 From: aidahosa1 Date: Mon, 18 Mar 2024 13:39:34 -0400 Subject: the lack of pushing is astounding actually --- src/client/documents/DocumentTypes.ts | 4 +- src/client/documents/Documents.ts | 47 ++++++ .../views/collections/CollectionCardDeckView.tsx | 164 +++++++++++++++++++++ src/client/views/collections/CollectionView.tsx | 5 + .../CollectionFreeFormLayoutEngines.tsx | 34 +++++ .../collectionFreeForm/CollectionFreeFormView.tsx | 4 +- .../collections/collectionFreeForm/MarqueeView.tsx | 20 ++- 7 files changed, 274 insertions(+), 4 deletions(-) create mode 100644 src/client/views/collections/CollectionCardDeckView.tsx (limited to 'src/client/documents/DocumentTypes.ts') diff --git a/src/client/documents/DocumentTypes.ts b/src/client/documents/DocumentTypes.ts index 1123bcac9..209b34f91 100644 --- a/src/client/documents/DocumentTypes.ts +++ b/src/client/documents/DocumentTypes.ts @@ -62,5 +62,7 @@ export enum CollectionViewType { Pile = 'pileup', StackedTimeline = 'stacked timeline', NoteTaking = 'notetaking', - Calendar = 'calendar' + Calendar = 'calendar', + Card = 'card' + } diff --git a/src/client/documents/Documents.ts b/src/client/documents/Documents.ts index 17cb6fef8..d12c61fd7 100644 --- a/src/client/documents/Documents.ts +++ b/src/client/documents/Documents.ts @@ -1219,6 +1219,15 @@ export namespace Docs { ); } + export function CardDeckDocument(documents: Array, options: DocumentOptions, id?: string) { + return InstanceFromProto( + Prototypes.get(DocumentType.COL), + new List(documents), + { backgroundColor: 'transparent', dropAction: dropActionType.move, _forceActive: true, _freeform_noZoom: true, _freeform_noAutoPan: true, ...options, _type_collection: CollectionViewType.Card }, + id + ); + } + export function LinearDocument(documents: Array, options: DocumentOptions, id?: string) { return InstanceFromProto(Prototypes.get(DocumentType.COL), new List(documents), { ...options, _type_collection: CollectionViewType.Linear }, id); } @@ -1876,6 +1885,44 @@ export namespace DocUtils { return newCollection; } } + + export function spreadCards(docList: Doc[], x: number = 0, y: number = 0, spreadAngle: number = 30, radius: number = 100, create: boolean = true) { + const totalCards = docList.length; + const halfSpreadAngle = spreadAngle * 0.5; + const angleStep = spreadAngle / (totalCards - 1); + + runInAction(() => { + docList.forEach((d, i) => { + DocUtils.iconify(d); + const angle = (-halfSpreadAngle + angleStep * i) * (Math.PI / 180); // Convert degrees to radians + d.x = x + Math.cos(angle) * radius; + d.y = y + Math.sin(angle) * radius; + d.rotation = angle; // Assuming 'd.rotation' sets the rotation, adjust accordingly + d._timecodeToShow = undefined; + }); + }); + + if (create) { + const newCollection = Docs.Create.CardDeckDocument(docList, { + title: 'card-spread', + _freeform_noZoom: true, + x: x - radius, + y: y - radius, + _width: radius * 2, + _height: radius * 2, + dragWhenActive: true, + _layout_fitWidth: false + }); + // Adjust position based on the collection's dimensions if needed + newCollection.x = NumCast(newCollection.x) + NumCast(newCollection._width) / 2 - radius; + newCollection.y = NumCast(newCollection.y) + NumCast(newCollection._height) / 2 - radius; + newCollection._width = newCollection._height = radius * 2; + return newCollection; + } + } + + + export function makeIntoPortal(doc: Doc, layoutDoc: Doc, allLinks: Doc[]) { const portalLink = allLinks.find(d => d.link_anchor_1 === doc && d.link_relationship === 'portal to:portal from'); if (!portalLink) { diff --git a/src/client/views/collections/CollectionCardDeckView.tsx b/src/client/views/collections/CollectionCardDeckView.tsx new file mode 100644 index 000000000..feb9e61cc --- /dev/null +++ b/src/client/views/collections/CollectionCardDeckView.tsx @@ -0,0 +1,164 @@ +import { action, computed, IReactionDisposer, makeObservable } from 'mobx'; +import { observer } from 'mobx-react'; +import * as React from 'react'; +import { Doc, DocListCast } from '../../../fields/Doc'; +import { ScriptField } from '../../../fields/ScriptField'; +import { NumCast, StrCast } from '../../../fields/Types'; +import { emptyFunction, returnFalse, setupMoveUpEvents } from '../../../Utils'; +import { DocUtils } from '../../documents/Documents'; +import { SelectionManager } from '../../util/SelectionManager'; +import { undoBatch, UndoManager } from '../../util/UndoManager'; +import { OpenWhere } from '../nodes/DocumentView'; +import { computePassLayout, computeStarburstLayout, computeCardDeckLayout } from './collectionFreeForm'; +import { CollectionFreeFormView } from './collectionFreeForm/CollectionFreeFormView'; +import './CollectionPileView.scss'; +import { CollectionSubView } from './CollectionSubView'; +import { dropActionType } from '../../util/DragManager'; + +@observer +export class CollectionCardView extends CollectionSubView() { + _originalChrome: any = ''; + _disposers: { [name: string]: IReactionDisposer } = {}; + + constructor(props: any) { + super(props); + makeObservable(this); + } + + componentDidMount() { + if (this.layoutEngine() !== computePassLayout.name && this.layoutEngine() !== computeCardDeckLayout.name) { + this.Document._freeform_cardEngine = computePassLayout.name; + } + this._originalChrome = this.layoutDoc._chromeHidden; + this.layoutDoc._chromeHidden = true; + } + componentWillUnmount() { + this.layoutDoc._chromeHidden = this._originalChrome; + Object.values(this._disposers).forEach(disposer => disposer?.()); + } + + layoutEngine = () => StrCast(this.Document._freeform_cardEngine); + + @undoBatch + addCardDoc = (doc: Doc | Doc[]) => { + (doc instanceof Doc ? [doc] : doc).map(d => DocUtils.iconify(d)); + return this._props.addDocument?.(doc) || false; + }; + + @undoBatch + removeCardDoc = (doc: Doc | Doc[], targetCollection: Doc | undefined, addDoc: (doc: Doc | Doc[]) => boolean) => { + (doc instanceof Doc ? [doc] : doc).forEach(d => Doc.deiconifyView(d)); + const ret = this._props.moveDocument?.(doc, targetCollection, addDoc) || false; + if (ret && !DocListCast(this.dataDoc[this.fieldKey ?? 'data']).length) this.DocumentView?.()._props.removeDocument?.(this.Document); + return ret; + }; + + @computed get toggleIcon() { + return ScriptField.MakeScript('documentView.iconify()', { documentView: 'any' }); + } + @computed get contentEvents() { + const isCard = this.layoutEngine() === computeCardDeckLayout.name; + return this._props.isContentActive() && isCard ? undefined : 'none'; + } + + // returns the contents of the cardSpread in a CollectionFreeFormView + @computed get contents() { + return ( +
+ +
+ ); + } + + // // toggles the pileup between starburst to compact + // toggleStarburst = action(() => { + // this.layoutDoc._freeform_scale = undefined; + // if (this.layoutEngine() === computeStarburstLayout.name) { + // if (NumCast(this.layoutDoc._width) !== NumCast(this.Document._starburstDiameter, 500)) { + // this.Document._starburstDiameter = NumCast(this.layoutDoc._width); + // } + // const defaultSize = 110; + // this.Document.x = NumCast(this.Document.x) + NumCast(this.layoutDoc._width) / 2 - NumCast(this.layoutDoc._freeform_pileWidth, defaultSize) / 2; + // this.Document.y = NumCast(this.Document.y) + NumCast(this.layoutDoc._height) / 2 - NumCast(this.layoutDoc._freeform_pileHeight, defaultSize) / 2; + // this.layoutDoc._width = NumCast(this.layoutDoc._freeform_pileWidth, defaultSize); + // this.layoutDoc._height = NumCast(this.layoutDoc._freeform_pileHeight, defaultSize); + // DocUtils.pileup(this.childDocs, undefined, undefined, NumCast(this.layoutDoc._width) / 2, false); + // this.layoutDoc._freeform_panX = 0; + // this.layoutDoc._freeform_panY = -10; + // this.Document._freeform_pileEngine = computePassLayout.name; + // } else { + // const defaultSize = NumCast(this.Document._starburstDiameter, 400); + // this.Document.x = NumCast(this.Document.x) + NumCast(this.layoutDoc._width) / 2 - defaultSize / 2; + // this.Document.y = NumCast(this.Document.y) + NumCast(this.layoutDoc._height) / 2 - defaultSize / 2; + // this.layoutDoc._freeform_pileWidth = NumCast(this.layoutDoc._width); + // this.layoutDoc._freeform_pileHeight = NumCast(this.layoutDoc._height); + // this.layoutDoc._freeform_panX = this.layoutDoc._freeform_panY = 0; + // this.layoutDoc._width = this.layoutDoc._height = defaultSize; + // this.layoutDoc.background; + // this.Document._freeform_pileEngine = computeStarburstLayout.name; + // } + // }); + + // for dragging documents out of the pileup view + _undoBatch: UndoManager.Batch | undefined; + pointerDown = (e: React.PointerEvent) => { + let dist = 0; + setupMoveUpEvents( + this, + e, + (e: PointerEvent, down: number[], delta: number[]) => { + if (this.layoutEngine() === 'pass' && this.childDocs.length && e.shiftKey) { + dist += Math.sqrt(delta[0] * delta[0] + delta[1] * delta[1]); + if (dist > 100) { + if (!this._undoBatch) { + this._undoBatch = UndoManager.StartBatch('layout pile'); + } + const doc = this.childDocs[0]; + doc.x = e.clientX; + doc.y = e.clientY; + this._props.addDocTab(doc, OpenWhere.inParentFromScreen) && (this._props.removeDocument?.(doc) || false); + dist = 0; + } + } + return false; + }, + () => { + this._undoBatch?.end(); + this._undoBatch = undefined; + }, + emptyFunction, + e.shiftKey && this.layoutEngine() === computePassLayout.name, + this.layoutEngine() === computePassLayout.name && e.shiftKey + ); // this sets _doubleTap + }; + + // onClick for toggling the pileup view + // @undoBatch + // onClick = (e: React.MouseEvent) => { + // if (e.button === 0) { + // SelectionManager.DeselectAll(); + // this.toggleStarburst(); + // e.stopPropagation(); + // } + // }; + + render() { + return ( +
+ {this.contents} +
+ ); + } +} diff --git a/src/client/views/collections/CollectionView.tsx b/src/client/views/collections/CollectionView.tsx index 18eb4dd1f..b7805bf3f 100644 --- a/src/client/views/collections/CollectionView.tsx +++ b/src/client/views/collections/CollectionView.tsx @@ -33,6 +33,8 @@ import { CollectionLinearView } from './collectionLinear'; import { CollectionMulticolumnView } from './collectionMulticolumn/CollectionMulticolumnView'; import { CollectionMultirowView } from './collectionMulticolumn/CollectionMultirowView'; import { CollectionSchemaView } from './collectionSchema/CollectionSchemaView'; +import { CollectionCardView } from './CollectionCardDeckView'; + export interface CollectionViewProps extends React.PropsWithChildren { isAnnotationOverlay?: boolean; // is the collection an annotation overlay (eg an overlay on an image/video/etc) isAnnotationOverlayScrollable?: boolean; // whether the annotation overlay can be vertically scrolled (just for tree views, currently) @@ -134,6 +136,9 @@ export class CollectionView extends ViewBoxAnnotatableComponent; case CollectionViewType.Time: return ; case CollectionViewType.Grid: return ; + case CollectionViewType.Card: return ; + + } }; diff --git a/src/client/views/collections/collectionFreeForm/CollectionFreeFormLayoutEngines.tsx b/src/client/views/collections/collectionFreeForm/CollectionFreeFormLayoutEngines.tsx index b8c0967c1..e972f44f1 100644 --- a/src/client/views/collections/collectionFreeForm/CollectionFreeFormLayoutEngines.tsx +++ b/src/client/views/collections/collectionFreeForm/CollectionFreeFormLayoutEngines.tsx @@ -126,6 +126,40 @@ export function computeStarburstLayout(poolData: Map, pivotDoc return normalizeResults(burstDiam, 12, docMap, poolData, viewDefsToJSX, [], 0, [divider]); } +export function computeCardDeckLayout(poolData: Map, pivotDoc: Doc, childPairs: { layout: Doc; data?: Doc }[], panelDim: number[], viewDefsToJSX: (views: ViewDefBounds[]) => ViewDefResult[], engineProps: any) { + const docMap = new Map(); + const spreadWidth = Math.min(panelDim[0], childPairs.length * 50); // Total width of the spread + const startX = -(spreadWidth / 2); // Starting X position + const fanAngle = 5; // Angle in degrees for fanning out cards + const baseZIndex = 1000; // Base Z-index to ensure cards are stacked in order + + childPairs.forEach(({ layout, data }, i) => { + const aspect = NumCast(layout._height) / NumCast(layout._width); + const docSize = Math.min(400, NumCast(layout._width)) * NumCast(pivotDoc._starburstDocScale, 1); + const posX = startX + (spreadWidth / childPairs.length) * i; + const posY = 0; // Adjust if you want to change the vertical alignment + const rotation = (i - (childPairs.length / 2)) * fanAngle; // Calculate rotation for fanning effect + + docMap.set(layout[Id], { + x: posX, + y: posY, + width: docSize, + height: docSize * aspect, + zIndex: baseZIndex + i, // Increment Z-index for each card to stack them correctly + rotation: rotation, // Optional: Add this if you want to rotate elements for a fanned effect + pair: { layout, data }, + replica: '', + color: 'white', + backgroundColor: 'white', + transition: 'all 0.3s', + }); + }); + + // This is a placeholder for the divider object and may need to be adjusted based on actual usage + const divider = { type: 'div', color: 'transparent', x: -panelDim[0] / 2, y: -panelDim[1] / 2, width: 15, height: 15, payload: undefined }; + return normalizeResults(panelDim, 12, docMap, poolData, viewDefsToJSX, [], 0, [divider]); +} + export function computePivotLayout(poolData: Map, pivotDoc: Doc, childPairs: { layout: Doc; data?: Doc }[], panelDim: number[], viewDefsToJSX: (views: ViewDefBounds[]) => ViewDefResult[], engineProps: any) { const docMap = new Map(); const fieldKey = 'data'; diff --git a/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx b/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx index 1fd453e96..f9fe306fa 100644 --- a/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx +++ b/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx @@ -49,7 +49,7 @@ import { CollectionSubView } from '../CollectionSubView'; import { TreeViewType } from '../CollectionTreeView'; import { CollectionFreeFormBackgroundGrid } from './CollectionFreeFormBackgroundGrid'; import { CollectionFreeFormInfoUI } from './CollectionFreeFormInfoUI'; -import { computePassLayout, computePivotLayout, computeStarburstLayout, computeTimelineLayout, PoolData, ViewDefBounds, ViewDefResult } from './CollectionFreeFormLayoutEngines'; +import { computePassLayout, computePivotLayout, computeStarburstLayout, computeTimelineLayout, PoolData, ViewDefBounds, ViewDefResult, computeCardDeckLayout } from './CollectionFreeFormLayoutEngines'; import { CollectionFreeFormPannableContents } from './CollectionFreeFormPannableContents'; import { CollectionFreeFormRemoteCursors } from './CollectionFreeFormRemoteCursors'; import './CollectionFreeFormView.scss'; @@ -1383,6 +1383,8 @@ export class CollectionFreeFormView extends CollectionSubView { + const selected = this.marqueeSelect(false); + SelectionManager.DeselectAll(); + selected.forEach(d => this._props.removeDocument?.(d)); + const newCollection = DocUtils.spreadCards(selected, this.Bounds.left + this.Bounds.width / 2, this.Bounds.top + this.Bounds.height / 2)!; + this._props.addDocument?.(newCollection); + this._props.selectDocuments([newCollection]); + MarqueeOptionsMenu.Instance.fadeOut(true); + this.hideMarquee(); + }); + /** * This triggers the TabDocView.PinDoc method which is the universal method * used to pin documents to the currently active presentation trail. @@ -508,6 +520,7 @@ export class MarqueeView extends ObservableReactComponent { + if (this._commandExecuted || (e as any).propagationIsStopped) { return; } @@ -518,7 +531,8 @@ export class MarqueeView extends ObservableReactComponent Date: Fri, 17 May 2024 10:39:25 -0400 Subject: manually added ChatBox from aj-starter --- package-lock.json | 359 ++++++++++++ package.json | 7 + src/client/documents/DocumentTypes.ts | 1 + src/client/documents/Documents.ts | 9 + src/client/util/CurrentUserUtils.ts | 2 + src/client/views/Main.tsx | 2 + src/client/views/nodes/ChatBox/ChatBox.scss | 228 ++++++++ src/client/views/nodes/ChatBox/ChatBox.tsx | 609 +++++++++++++++++++++ .../views/nodes/ChatBox/MessageComponent.tsx | 116 ++++ src/client/views/nodes/ChatBox/types.ts | 23 + src/client/views/nodes/DocumentContentsView.tsx | 2 +- src/fields/Types.ts | 5 +- src/server/ApiManagers/AssistantManager.ts | 131 +++++ src/server/index.ts | 10 +- 14 files changed, 1498 insertions(+), 6 deletions(-) create mode 100644 src/client/views/nodes/ChatBox/ChatBox.scss create mode 100644 src/client/views/nodes/ChatBox/ChatBox.tsx create mode 100644 src/client/views/nodes/ChatBox/MessageComponent.tsx create mode 100644 src/client/views/nodes/ChatBox/types.ts create mode 100644 src/server/ApiManagers/AssistantManager.ts (limited to 'src/client/documents/DocumentTypes.ts') diff --git a/package-lock.json b/package-lock.json index 60198132c..35ffc712c 100644 --- a/package-lock.json +++ b/package-lock.json @@ -60,6 +60,7 @@ "babel": "^6.23.0", "babel-loader": "^9.1.3", "bcrypt-nodejs": "0.0.3", + "better-react-mathjax": "^2.0.3", "bezier-curve": "^1.0.0", "bezier-js": "^6.1.4", "bingmaps-react": "^1.2.10", @@ -86,6 +87,7 @@ "csstype": "^3.1.3", "csv-parser": "^3.0.0", "csv-stringify": "^6.4.4", + "csvtojson": "^2.0.10", "D": "^1.0.0", "d3": "^7.8.5", "depcheck": "^1.4.7", @@ -107,6 +109,7 @@ "fluent-ffmpeg": "^2.1.2", "forever-agent": "^0.6.1", "fork-ts-checker-webpack-plugin": "^9.0.2", + "form-data": "^4.0.0", "formidable": "3.5.1", "function-plot": "^1.23.3", "golden-layout": "^2.6.0", @@ -133,6 +136,7 @@ "jszip": "^3.10.1", "lodash": "^4.17.21", "mapbox-gl": "^3.0.1", + "markdown-it": "^14.1.0", "mathquill": "^0.10.1-a", "md5-file": "^5.0.0", "memorystream": "^0.3.1", @@ -182,6 +186,7 @@ "react-grid-layout": "^1.4.4", "react-icons": "^5.0.1", "react-jsx-parser": "^1.29.0", + "react-latex-next": "^3.0.0", "react-loading": "^2.0.3", "react-map-gl": "^7.1.6", "react-markdown": "^9.0.1", @@ -193,8 +198,10 @@ "react-xarrows": "^2.0.2", "readline": "^1.3.0", "recharts": "^2.10.3", + "rehype-katex": "^7.0.0", "rehype-raw": "^7.0.0", "remark-gfm": "^4.0.0", + "remark-math": "^6.0.0", "request": "^2.88.2", "request-promise": "^4.2.6", "reveal.js": "^5.0.2", @@ -9347,6 +9354,11 @@ "integrity": "sha512-dRLjCWHYg4oaA77cxO64oO+7JwCwnIzkZPdrrC71jQmQtlhM556pwKo5bUzqvZndkVbeFLIIi+9TC40JNF5hNQ==", "dev": true }, + "node_modules/@types/katex": { + "version": "0.16.7", + "resolved": "https://registry.npmjs.org/@types/katex/-/katex-0.16.7.tgz", + "integrity": "sha512-HMwFiRujE5PjrgwHQ25+bsLJgowjGjm5Z8FVSf0N6PwgJrwxH0QxzHYDcKsTfV3wva0vzrpqMTJS2jXPr5BMEQ==" + }, "node_modules/@types/keygrip": { "version": "1.0.6", "resolved": "https://registry.npmjs.org/@types/keygrip/-/keygrip-1.0.6.tgz", @@ -11251,6 +11263,17 @@ "resolved": "https://registry.npmjs.org/before-after-hook/-/before-after-hook-3.0.2.tgz", "integrity": "sha512-Nik3Sc0ncrMK4UUdXQmAnRtzmNQTAAXmXIopizwZ1W1t8QmfJj+zL4OA2I7XPTPW5z5TDqv4hRo/JzouDJnX3A==" }, + "node_modules/better-react-mathjax": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/better-react-mathjax/-/better-react-mathjax-2.0.3.tgz", + "integrity": "sha512-wfifT8GFOKb1TWm2+E50I6DJpLZ5kLbch283Lu043EJtwSv0XvZDjr4YfR4d2MjAhqP6SH4VjjrKgbX8R00oCQ==", + "dependencies": { + "mathjax-full": "^3.2.2" + }, + "peerDependencies": { + "react": ">=16.8" + } + }, "node_modules/bezier-curve": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/bezier-curve/-/bezier-curve-1.0.0.tgz", @@ -15907,6 +15930,33 @@ "resolved": "https://registry.npmjs.org/csv-stringify/-/csv-stringify-6.4.6.tgz", "integrity": "sha512-h2V2XZ3uOTLilF5dPIptgUfN/o2ia/80Ie0Lly18LAnw5s8Eb7kt8rfxSUy24AztJZas9f6DPZpVlzDUtFt/ag==" }, + "node_modules/csvtojson": { + "version": "2.0.10", + "resolved": "https://registry.npmjs.org/csvtojson/-/csvtojson-2.0.10.tgz", + "integrity": "sha512-lUWFxGKyhraKCW8Qghz6Z0f2l/PqB1W3AO0HKJzGIQ5JRSlR651ekJDiGJbBT4sRNNv5ddnSGVEnsxP9XRCVpQ==", + "dependencies": { + "bluebird": "^3.5.1", + "lodash": "^4.17.3", + "strip-bom": "^2.0.0" + }, + "bin": { + "csvtojson": "bin/csvtojson" + }, + "engines": { + "node": ">=4.0.0" + } + }, + "node_modules/csvtojson/node_modules/strip-bom": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/strip-bom/-/strip-bom-2.0.0.tgz", + "integrity": "sha512-kwrX1y7czp1E69n2ajbG65mIo9dqvJ+8aBQXOGVxqwvNbsXdFM6Lq37dLAY3mknUwru8CfcCbfOLL/gMo+fi3g==", + "dependencies": { + "is-utf8": "^0.2.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, "node_modules/custom-event": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/custom-event/-/custom-event-1.0.1.tgz", @@ -18913,6 +18963,14 @@ "node": ">=8" } }, + "node_modules/esm": { + "version": "3.2.25", + "resolved": "https://registry.npmjs.org/esm/-/esm-3.2.25.tgz", + "integrity": "sha512-U1suiZ2oDVWv4zPO56S0NcR5QriEahGtdN2OR6FiOG4WJvcjBVFB0qI4+eKoWFH483PKGuLuu6V8Z4T5g63UVA==", + "engines": { + "node": ">=6" + } + }, "node_modules/espree": { "version": "9.6.1", "resolved": "https://registry.npmjs.org/espree/-/espree-9.6.1.tgz", @@ -20825,6 +20883,52 @@ "node": ">= 0.4" } }, + "node_modules/hast-util-from-dom": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/hast-util-from-dom/-/hast-util-from-dom-5.0.0.tgz", + "integrity": "sha512-d6235voAp/XR3Hh5uy7aGLbM3S4KamdW0WEgOaU1YoewnuYw4HXb5eRtv9g65m/RFGEfUY1Mw4UqCc5Y8L4Stg==", + "dependencies": { + "@types/hast": "^3.0.0", + "hastscript": "^8.0.0", + "web-namespaces": "^2.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/hast-util-from-html": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/hast-util-from-html/-/hast-util-from-html-2.0.1.tgz", + "integrity": "sha512-RXQBLMl9kjKVNkJTIO6bZyb2n+cUH8LFaSSzo82jiLT6Tfc+Pt7VQCS+/h3YwG4jaNE2TA2sdJisGWR+aJrp0g==", + "dependencies": { + "@types/hast": "^3.0.0", + "devlop": "^1.1.0", + "hast-util-from-parse5": "^8.0.0", + "parse5": "^7.0.0", + "vfile": "^6.0.0", + "vfile-message": "^4.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/hast-util-from-html-isomorphic": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/hast-util-from-html-isomorphic/-/hast-util-from-html-isomorphic-2.0.0.tgz", + "integrity": "sha512-zJfpXq44yff2hmE0XmwEOzdWin5xwH+QIhMLOScpX91e/NSGPsAzNCvLQDIEPyO2TXi+lBmU6hjLIhV8MwP2kw==", + "dependencies": { + "@types/hast": "^3.0.0", + "hast-util-from-dom": "^5.0.0", + "hast-util-from-html": "^2.0.0", + "unist-util-remove-position": "^5.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, "node_modules/hast-util-from-parse5": { "version": "8.0.1", "resolved": "https://registry.npmjs.org/hast-util-from-parse5/-/hast-util-from-parse5-8.0.1.tgz", @@ -20844,6 +20948,18 @@ "url": "https://opencollective.com/unified" } }, + "node_modules/hast-util-is-element": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/hast-util-is-element/-/hast-util-is-element-3.0.0.tgz", + "integrity": "sha512-Val9mnv2IWpLbNPqc/pUem+a7Ipj2aHacCwgNfTiK0vJKl0LF+4Ba4+v1oPHFpf3bLYmreq0/l3Gud9S5OH42g==", + "dependencies": { + "@types/hast": "^3.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, "node_modules/hast-util-parse-selector": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/hast-util-parse-selector/-/hast-util-parse-selector-4.0.0.tgz", @@ -20924,6 +21040,21 @@ "url": "https://opencollective.com/unified" } }, + "node_modules/hast-util-to-text": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/hast-util-to-text/-/hast-util-to-text-4.0.2.tgz", + "integrity": "sha512-KK6y/BN8lbaq654j7JgBydev7wuNMcID54lkRav1P0CaE1e47P72AWWPiGKXTJU271ooYzcvTAn/Zt0REnvc7A==", + "dependencies": { + "@types/hast": "^3.0.0", + "@types/unist": "^3.0.0", + "hast-util-is-element": "^3.0.0", + "unist-util-find-after": "^5.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, "node_modules/hast-util-whitespace": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/hast-util-whitespace/-/hast-util-whitespace-3.0.0.tgz", @@ -22592,6 +22723,11 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/is-utf8": { + "version": "0.2.1", + "resolved": "https://registry.npmjs.org/is-utf8/-/is-utf8-0.2.1.tgz", + "integrity": "sha512-rMYPYvCzsXywIsldgLaSoPlw5PfoB/ssr7hY4pLfcodrA5M/eArza1a9VmTiNIBNMjOGr1Ow9mTyU2o69U6U9Q==" + }, "node_modules/is-weakmap": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/is-weakmap/-/is-weakmap-2.0.2.tgz", @@ -23214,6 +23350,29 @@ "node": ">=12.0.0" } }, + "node_modules/katex": { + "version": "0.16.10", + "resolved": "https://registry.npmjs.org/katex/-/katex-0.16.10.tgz", + "integrity": "sha512-ZiqaC04tp2O5utMsl2TEZTXxa6WSC4yo0fv5ML++D3QZv/vx2Mct0mTlRx3O+uUkjfuAgOkzsCmq5MiUEsDDdA==", + "funding": [ + "https://opencollective.com/katex", + "https://github.com/sponsors/katex" + ], + "dependencies": { + "commander": "^8.3.0" + }, + "bin": { + "katex": "cli.js" + } + }, + "node_modules/katex/node_modules/commander": { + "version": "8.3.0", + "resolved": "https://registry.npmjs.org/commander/-/commander-8.3.0.tgz", + "integrity": "sha512-OkTL9umf+He2DZkUq8f8J9of7yL6RJKI24dVITBmNfZBmri9zYZQrKkuXiKhyfPSu8tUhnVBB1iKXevvnlR4Ww==", + "engines": { + "node": ">= 12" + } + }, "node_modules/kdbush": { "version": "4.0.2", "resolved": "https://registry.npmjs.org/kdbush/-/kdbush-4.0.2.tgz", @@ -23429,6 +23588,14 @@ "resolved": "https://registry.npmjs.org/lines-and-columns/-/lines-and-columns-1.2.4.tgz", "integrity": "sha512-7ylylesZQ/PV29jhEDl3Ufjo6ZX7gCqJr5F7PKrqc93v7fzSymt1BpwEU8nAUXs8qzzvqhbjhK5QZg6Mt/HkBg==" }, + "node_modules/linkify-it": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/linkify-it/-/linkify-it-5.0.0.tgz", + "integrity": "sha512-5aHCbzQRADcdP+ATqnDuhhJ/MRIqDkZX5pyjFHRRysS8vZ5AbqGEoFIb6pYHPZ+L/OC2Lc+xT8uHVVR5CAK/wQ==", + "dependencies": { + "uc.micro": "^2.0.0" + } + }, "node_modules/load-bmfont": { "version": "1.4.1", "resolved": "https://registry.npmjs.org/load-bmfont/-/load-bmfont-1.4.1.tgz", @@ -23796,6 +23963,27 @@ "vt-pbf": "^3.1.3" } }, + "node_modules/markdown-it": { + "version": "14.1.0", + "resolved": "https://registry.npmjs.org/markdown-it/-/markdown-it-14.1.0.tgz", + "integrity": "sha512-a54IwgWPaeBCAAsv13YgmALOF1elABB08FxO9i+r4VFk5Vl4pKokRPeX8u5TCgSsPi6ec1otfLjdOpVcgbpshg==", + "dependencies": { + "argparse": "^2.0.1", + "entities": "^4.4.0", + "linkify-it": "^5.0.0", + "mdurl": "^2.0.0", + "punycode.js": "^2.3.1", + "uc.micro": "^2.1.0" + }, + "bin": { + "markdown-it": "bin/markdown-it.mjs" + } + }, + "node_modules/markdown-it/node_modules/argparse": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/argparse/-/argparse-2.0.1.tgz", + "integrity": "sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q==" + }, "node_modules/markdown-table": { "version": "3.0.3", "resolved": "https://registry.npmjs.org/markdown-table/-/markdown-table-3.0.3.tgz", @@ -23819,6 +24007,17 @@ "mr-parser": "^0.2.1" } }, + "node_modules/mathjax-full": { + "version": "3.2.2", + "resolved": "https://registry.npmjs.org/mathjax-full/-/mathjax-full-3.2.2.tgz", + "integrity": "sha512-+LfG9Fik+OuI8SLwsiR02IVdjcnRCy5MufYLi0C3TdMT56L/pjB0alMVGgoWJF8pN9Rc7FESycZB9BMNWIid5w==", + "dependencies": { + "esm": "^3.2.25", + "mhchemparser": "^4.1.0", + "mj-context-menu": "^0.6.1", + "speech-rule-engine": "^4.0.6" + } + }, "node_modules/mathquill": { "version": "0.10.1-a", "resolved": "https://registry.npmjs.org/mathquill/-/mathquill-0.10.1-a.tgz", @@ -23987,6 +24186,24 @@ "url": "https://opencollective.com/unified" } }, + "node_modules/mdast-util-math": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/mdast-util-math/-/mdast-util-math-3.0.0.tgz", + "integrity": "sha512-Tl9GBNeG/AhJnQM221bJR2HPvLOSnLE/T9cJI9tlc6zwQk2nPk/4f0cHkOdEixQPC/j8UtKDdITswvLAy1OZ1w==", + "dependencies": { + "@types/hast": "^3.0.0", + "@types/mdast": "^4.0.0", + "devlop": "^1.0.0", + "longest-streak": "^3.0.0", + "mdast-util-from-markdown": "^2.0.0", + "mdast-util-to-markdown": "^2.1.0", + "unist-util-remove-position": "^5.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, "node_modules/mdast-util-mdx-expression": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/mdast-util-mdx-expression/-/mdast-util-mdx-expression-2.0.0.tgz", @@ -24109,6 +24326,11 @@ "url": "https://opencollective.com/unified" } }, + "node_modules/mdurl": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/mdurl/-/mdurl-2.0.0.tgz", + "integrity": "sha512-Lf+9+2r+Tdp5wXDXC4PcIBjTDtq4UKjCPMQhKIuzpJNW0b96kVqSwW0bT7FhRSfmAiFYgP+SCRvdrDozfh0U5w==" + }, "node_modules/media-typer": { "version": "0.3.0", "resolved": "https://registry.npmjs.org/media-typer/-/media-typer-0.3.0.tgz", @@ -24172,6 +24394,11 @@ "node": ">= 0.6" } }, + "node_modules/mhchemparser": { + "version": "4.2.1", + "resolved": "https://registry.npmjs.org/mhchemparser/-/mhchemparser-4.2.1.tgz", + "integrity": "sha512-kYmyrCirqJf3zZ9t/0wGgRZ4/ZJw//VwaRVGA75C4nhE60vtnIzhl9J9ndkX/h6hxSN7pjg/cE0VxbnNM+bnDQ==" + }, "node_modules/micromark": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/micromark/-/micromark-4.0.0.tgz", @@ -24353,6 +24580,24 @@ "url": "https://opencollective.com/unified" } }, + "node_modules/micromark-extension-math": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/micromark-extension-math/-/micromark-extension-math-3.0.0.tgz", + "integrity": "sha512-iJ2Q28vBoEovLN5o3GO12CpqorQRYDPT+p4zW50tGwTfJB+iv/VnB6Ini+gqa24K97DwptMBBIvVX6Bjk49oyQ==", + "dependencies": { + "@types/katex": "^0.16.0", + "devlop": "^1.0.0", + "katex": "^0.16.0", + "micromark-factory-space": "^2.0.0", + "micromark-util-character": "^2.0.0", + "micromark-util-symbol": "^2.0.0", + "micromark-util-types": "^2.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, "node_modules/micromark-factory-destination": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/micromark-factory-destination/-/micromark-factory-destination-2.0.0.tgz", @@ -24840,6 +25085,11 @@ "resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz", "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==" }, + "node_modules/mj-context-menu": { + "version": "0.6.1", + "resolved": "https://registry.npmjs.org/mj-context-menu/-/mj-context-menu-0.6.1.tgz", + "integrity": "sha512-7NO5s6n10TIV96d4g2uDpG7ZDpIhMh0QNfGdJw/W47JswFcosz457wqz/b5sAKvl12sxINGFCn80NZHKwxQEXA==" + }, "node_modules/mkdirp": { "version": "1.0.4", "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-1.0.4.tgz", @@ -29576,6 +29826,14 @@ "node": ">=6" } }, + "node_modules/punycode.js": { + "version": "2.3.1", + "resolved": "https://registry.npmjs.org/punycode.js/-/punycode.js-2.3.1.tgz", + "integrity": "sha512-uxFIHU0YlHYhDQtV4R9J6a52SLx28BCjT+4ieh7IGbgwVJWO+km431c4yRlREUAsAmt/uMjQUyQHNEPf0M39CA==", + "engines": { + "node": ">=6" + } + }, "node_modules/q": { "version": "1.5.1", "resolved": "https://registry.npmjs.org/q/-/q-1.5.1.tgz", @@ -30099,6 +30357,22 @@ "@types/react": "^17" } }, + "node_modules/react-latex-next": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/react-latex-next/-/react-latex-next-3.0.0.tgz", + "integrity": "sha512-x70f1b1G7TronVigsRgKHKYYVUNfZk/3bciFyYX1lYLQH2y3/TXku3+5Vap8MDbJhtopePSYBsYWS6jhzIdz+g==", + "dependencies": { + "katex": "^0.16.0" + }, + "engines": { + "node": ">=12", + "npm": ">=5" + }, + "peerDependencies": { + "react": "^15.0.0 || ^16.0.0 || ^17.0.0 || ^18.0.0", + "react-dom": "^15.0.0 || ^16.0.0 || ^17.0.0 || ^18.0.0" + } + }, "node_modules/react-loading": { "version": "2.0.3", "resolved": "https://registry.npmjs.org/react-loading/-/react-loading-2.0.3.tgz", @@ -30622,6 +30896,24 @@ "jsesc": "bin/jsesc" } }, + "node_modules/rehype-katex": { + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/rehype-katex/-/rehype-katex-7.0.0.tgz", + "integrity": "sha512-h8FPkGE00r2XKU+/acgqwWUlyzve1IiOKwsEkg4pDL3k48PiE0Pt+/uLtVHDVkN1yA4iurZN6UES8ivHVEQV6Q==", + "dependencies": { + "@types/hast": "^3.0.0", + "@types/katex": "^0.16.0", + "hast-util-from-html-isomorphic": "^2.0.0", + "hast-util-to-text": "^4.0.0", + "katex": "^0.16.0", + "unist-util-visit-parents": "^6.0.0", + "vfile": "^6.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, "node_modules/rehype-raw": { "version": "7.0.0", "resolved": "https://registry.npmjs.org/rehype-raw/-/rehype-raw-7.0.0.tgz", @@ -30661,6 +30953,21 @@ "url": "https://opencollective.com/unified" } }, + "node_modules/remark-math": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/remark-math/-/remark-math-6.0.0.tgz", + "integrity": "sha512-MMqgnP74Igy+S3WwnhQ7kqGlEerTETXMvJhrUzDikVZ2/uogJCb+WHUg97hK9/jcfc0dkD73s3LN8zU49cTEtA==", + "dependencies": { + "@types/mdast": "^4.0.0", + "mdast-util-math": "^3.0.0", + "micromark-extension-math": "^3.0.0", + "unified": "^11.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, "node_modules/remark-parse": { "version": "11.0.0", "resolved": "https://registry.npmjs.org/remark-parse/-/remark-parse-11.0.0.tgz", @@ -32129,6 +32436,27 @@ "node": ">= 6" } }, + "node_modules/speech-rule-engine": { + "version": "4.0.7", + "resolved": "https://registry.npmjs.org/speech-rule-engine/-/speech-rule-engine-4.0.7.tgz", + "integrity": "sha512-sJrL3/wHzNwJRLBdf6CjJWIlxC04iYKkyXvYSVsWVOiC2DSkHmxsqOhEeMsBA9XK+CHuNcsdkbFDnoUfAsmp9g==", + "dependencies": { + "commander": "9.2.0", + "wicked-good-xpath": "1.3.0", + "xmldom-sre": "0.1.31" + }, + "bin": { + "sre": "bin/sre" + } + }, + "node_modules/speech-rule-engine/node_modules/commander": { + "version": "9.2.0", + "resolved": "https://registry.npmjs.org/commander/-/commander-9.2.0.tgz", + "integrity": "sha512-e2i4wANQiSXgnrBlIatyHtP1odfUp0BbV5Y5nEGbxtIrStkEOAAzCUirvLBNXHLr7kwLvJl6V+4V3XV9x7Wd9w==", + "engines": { + "node": "^12.20.0 || >=14" + } + }, "node_modules/splaytree": { "version": "3.1.2", "resolved": "https://registry.npmjs.org/splaytree/-/splaytree-3.1.2.tgz", @@ -34036,6 +34364,11 @@ "resolved": "https://registry.npmjs.org/typical/-/typical-2.6.1.tgz", "integrity": "sha512-ofhi8kjIje6npGozTip9Fr8iecmYfEbS06i0JnIg+rh51KakryWF4+jX8lLKZVhy6N+ID45WYSFCxPOdTWCzNg==" }, + "node_modules/uc.micro": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/uc.micro/-/uc.micro-2.1.0.tgz", + "integrity": "sha512-ARDJmphmdvUk6Glw7y9DQ2bFkKBHwQHLi2lsaH6PPmz/Ka9sFOBsBluozhDltWmnv9u/cF6Rt87znRTPV+yp/A==" + }, "node_modules/uid-safe": { "version": "2.1.5", "resolved": "https://registry.npmjs.org/uid-safe/-/uid-safe-2.1.5.tgz", @@ -34164,6 +34497,19 @@ "node": ">=0.10.0" } }, + "node_modules/unist-util-find-after": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/unist-util-find-after/-/unist-util-find-after-5.0.0.tgz", + "integrity": "sha512-amQa0Ep2m6hE2g72AugUItjbuM8X8cGQnFoHk0pGfrFeT9GZhzN5SW8nRsiGKK7Aif4CrACPENkA6P/Lw6fHGQ==", + "dependencies": { + "@types/unist": "^3.0.0", + "unist-util-is": "^6.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, "node_modules/unist-util-is": { "version": "6.0.0", "resolved": "https://registry.npmjs.org/unist-util-is/-/unist-util-is-6.0.0.tgz", @@ -35165,6 +35511,11 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/wicked-good-xpath": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/wicked-good-xpath/-/wicked-good-xpath-1.3.0.tgz", + "integrity": "sha512-Gd9+TUn5nXdwj/hFsPVx5cuHHiF5Bwuc30jZ4+ronF1qHK5O7HD0sgmXWSEgwKquT3ClLoKPVbO6qGwVwLzvAw==" + }, "node_modules/wide-align": { "version": "1.1.5", "resolved": "https://registry.npmjs.org/wide-align/-/wide-align-1.1.5.tgz", @@ -35481,6 +35832,14 @@ "integrity": "sha512-JZnDKK8B0RCDw84FNdDAIpZK+JuJw+s7Lz8nksI7SIuU3UXJJslUthsi+uWBUYOwPFwW7W7PRLRfUKpxjtjFCw==", "dev": true }, + "node_modules/xmldom-sre": { + "version": "0.1.31", + "resolved": "https://registry.npmjs.org/xmldom-sre/-/xmldom-sre-0.1.31.tgz", + "integrity": "sha512-f9s+fUkX04BxQf+7mMWAp5zk61pciie+fFLC9hX9UVvCeJQfNHRHXpeo5MPcR0EUf57PYLdt+ZO4f3Ipk2oZUw==", + "engines": { + "node": ">=0.1" + } + }, "node_modules/xmlhttprequest-ssl": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/xmlhttprequest-ssl/-/xmlhttprequest-ssl-2.0.0.tgz", diff --git a/package.json b/package.json index 833bebf44..aa0874714 100644 --- a/package.json +++ b/package.json @@ -145,6 +145,7 @@ "babel": "^6.23.0", "babel-loader": "^9.1.3", "bcrypt-nodejs": "0.0.3", + "better-react-mathjax": "^2.0.3", "bezier-curve": "^1.0.0", "bezier-js": "^6.1.4", "bingmaps-react": "^1.2.10", @@ -171,6 +172,7 @@ "csstype": "^3.1.3", "csv-parser": "^3.0.0", "csv-stringify": "^6.4.4", + "csvtojson": "^2.0.10", "D": "^1.0.0", "d3": "^7.8.5", "depcheck": "^1.4.7", @@ -192,6 +194,7 @@ "fluent-ffmpeg": "^2.1.2", "forever-agent": "^0.6.1", "fork-ts-checker-webpack-plugin": "^9.0.2", + "form-data": "^4.0.0", "formidable": "3.5.1", "function-plot": "^1.23.3", "golden-layout": "^2.6.0", @@ -218,6 +221,7 @@ "jszip": "^3.10.1", "lodash": "^4.17.21", "mapbox-gl": "^3.0.1", + "markdown-it": "^14.1.0", "mathquill": "^0.10.1-a", "md5-file": "^5.0.0", "memorystream": "^0.3.1", @@ -267,6 +271,7 @@ "react-grid-layout": "^1.4.4", "react-icons": "^5.0.1", "react-jsx-parser": "^1.29.0", + "react-latex-next": "^3.0.0", "react-loading": "^2.0.3", "react-map-gl": "^7.1.6", "react-markdown": "^9.0.1", @@ -278,8 +283,10 @@ "react-xarrows": "^2.0.2", "readline": "^1.3.0", "recharts": "^2.10.3", + "rehype-katex": "^7.0.0", "rehype-raw": "^7.0.0", "remark-gfm": "^4.0.0", + "remark-math": "^6.0.0", "request": "^2.88.2", "request-promise": "^4.2.6", "reveal.js": "^5.0.2", diff --git a/src/client/documents/DocumentTypes.ts b/src/client/documents/DocumentTypes.ts index b4ad9c17d..0520dc375 100644 --- a/src/client/documents/DocumentTypes.ts +++ b/src/client/documents/DocumentTypes.ts @@ -20,6 +20,7 @@ export enum DocumentType { WEBCAM = 'webcam', // webcam CONFIG = 'config', // configuration document intended to specify a view layout configuration, but not be directly rendered (e.g., for saving the page# of a PDF, or view transform of a collection) SCRIPTING = 'script', // script editor + CHAT = 'chat', // chat with GPT about files EQUATION = 'equation', // equation editor FUNCPLOT = 'funcplot', // function plotter MAP = 'map', diff --git a/src/client/documents/Documents.ts b/src/client/documents/Documents.ts index 24dd5cf37..ef1c709f0 100644 --- a/src/client/documents/Documents.ts +++ b/src/client/documents/Documents.ts @@ -180,6 +180,12 @@ export class DocumentOptions { date_range?: STRt = new StrInfo('date range for calendar', false); + chat?: STRt = new StrInfo('fields related to chatBox', false); + chat_history?: STRt = new StrInfo('chat history for chatbox', false); + chat_thread_id?: STRt = new StrInfo('thread id for chatbox', false); + chat_assistant_id?: STRt = new StrInfo('assistant id for chatbox', false); + chat_vector_store_id?: STRt = new StrInfo('assistant id for chatbox', false); + wikiData?: STRt = new StrInfo('WikiData ID related to map location'); description?: STRt = new StrInfo('description of document'); _timecodeToShow?: NUMt = new NumInfo('media timecode when document should appear (e.g., when an annotation shows up as a video plays)', false); @@ -735,6 +741,9 @@ export namespace Docs { return InstanceFromProto(Prototypes.get(DocumentType.SCRIPTING), script || undefined, { ...options, layout: fieldKey ? `` /* ScriptingBox.LayoutString(fieldKey) */ : undefined }); } + export function ChatDocument(options?: DocumentOptions) { + return InstanceFromProto(Prototypes.get(DocumentType.CHAT), undefined, { ...(options || {}) }); + } // eslint-disable-next-line default-param-last export function VideoDocument(url: string, options: DocumentOptions = {}, overwriteDoc?: Doc) { return InstanceFromProto(Prototypes.get(DocumentType.VID), new VideoField(url), options, undefined, undefined, undefined, overwriteDoc); diff --git a/src/client/util/CurrentUserUtils.ts b/src/client/util/CurrentUserUtils.ts index ffc1304bf..09250d29d 100644 --- a/src/client/util/CurrentUserUtils.ts +++ b/src/client/util/CurrentUserUtils.ts @@ -368,6 +368,7 @@ pie title Minerals in my tap water {key: "Button", creator: Docs.Create.ButtonDocument, opts: { _width: 150, _height: 50, _xPadding: 10, _yPadding: 10, title_custom: true, waitForDoubleClickToClick: 'never'}, scripts: {onClick: FollowLinkScript()?.script.originalScript ?? ""}}, {key: "Script", creator: opts => Docs.Create.ScriptingDocument(null, opts), opts: { _width: 200, _height: 250, }}, {key: "DataViz", creator: opts => Docs.Create.DataVizDocument("/users/rz/Downloads/addresses.csv", opts), opts: { _width: 300, _height: 300 }}, + {key: "Chat", creator: Docs.Create.ChatDocument, opts: { _width: 300, _height: 300, }}, {key: "Header", creator: headerTemplate, opts: { _width: 300, _height: 120, _header_pointerEvents: "all", _header_height: 50, _header_fontSize: 9,_layout_autoHeightMargins: 50, _layout_autoHeight: true, treeView_HideUnrendered: true}}, {key: "ViewSlide", creator: slideView, opts: { _width: 400, _height: 300, _xMargin: 3, _yMargin: 3,}}, {key: "Trail", creator: Docs.Create.PresDocument, opts: { _width: 400, _height: 30, _type_collection: CollectionViewType.Stacking, dropAction: dropActionType.embed, treeView_HideTitle: true, _layout_fitWidth:true, layout_boxShadow: "0 0" }}, @@ -397,6 +398,7 @@ pie title Minerals in my tap water { toolTip: "Tap or drag to create a comparison box", title: "Compare", icon: "columns", dragFactory: doc.emptyComparison as Doc, clickFactory: DocCast(doc.emptyComparison)}, { toolTip: "Tap or drag to create an audio recorder", title: "Audio", icon: "microphone", dragFactory: doc.emptyAudio as Doc, clickFactory: DocCast(doc.emptyAudio), openFactoryLocation: OpenWhere.overlay}, { toolTip: "Tap or drag to create a map", title: "Map", icon: "map-marker-alt", dragFactory: doc.emptyMap as Doc, clickFactory: DocCast(doc.emptyMap)}, + { toolTip: "Tap or drag to create a chat assistant", title: "Assistant Chat", icon: "book",dragFactory: doc.emptyChat as Doc, clickFactory: DocCast(doc.emptyChat)}, { toolTip: "Tap or drag to create a screen grabber", title: "Grab", icon: "photo-video", dragFactory: doc.emptyScreengrab as Doc, clickFactory: DocCast(doc.emptyScreengrab), openFactoryLocation: OpenWhere.overlay, funcs: { hidden: "IsNoviceMode()"}}, { toolTip: "Tap or drag to create a WebCam recorder", title: "WebCam", icon: "photo-video", dragFactory: doc.emptyWebCam as Doc, clickFactory: DocCast(doc.emptyWebCam), openFactoryLocation: OpenWhere.overlay, funcs: { hidden: "IsNoviceMode()"}}, { toolTip: "Tap or drag to create a button", title: "Button", icon: "circle", dragFactory: doc.emptyButton as Doc, clickFactory: DocCast(doc.emptyButton)}, diff --git a/src/client/views/Main.tsx b/src/client/views/Main.tsx index 8968acbbb..149de59b2 100644 --- a/src/client/views/Main.tsx +++ b/src/client/views/Main.tsx @@ -28,6 +28,7 @@ import { SchemaRowBox } from './collections/collectionSchema/SchemaRowBox'; import './global/globalScripts'; import { AudioBox } from './nodes/AudioBox'; import { ComparisonBox } from './nodes/ComparisonBox'; +import { ChatBox } from './nodes/ChatBox/ChatBox'; import { DataVizBox } from './nodes/DataVizBox/DataVizBox'; import { DocumentContentsView, HTMLtag } from './nodes/DocumentContentsView'; import { EquationBox } from './nodes/EquationBox'; @@ -136,6 +137,7 @@ FieldLoader.ServerLoadStatus = { requested: 0, retrieved: 0, message: 'cache' }; MapBox, ScreenshotBox, DataVizBox, + ChatBox, HTMLtag, ComparisonBox, LoadingBox, diff --git a/src/client/views/nodes/ChatBox/ChatBox.scss b/src/client/views/nodes/ChatBox/ChatBox.scss new file mode 100644 index 000000000..f1ad3d074 --- /dev/null +++ b/src/client/views/nodes/ChatBox/ChatBox.scss @@ -0,0 +1,228 @@ +$background-color: #f8f9fa; +$text-color: #333; +$input-background: #fff; +$button-color: #007bff; +$button-hover-color: darken($button-color, 10%); +$shadow-color: rgba(0, 0, 0, 0.075); +$border-radius: 8px; + +.chatBox { + display: flex; + flex-direction: column; + width: 100%; /* Adjust the width as needed, could be in percentage */ + height: 100%; /* Adjust the height as needed, could be in percentage */ + background-color: $background-color; + font-family: 'Helvetica Neue', Arial, sans-serif; + //margin: 20px auto; + //overflow: hidden; + + .scroll-box { + flex-grow: 1; + overflow-y: scroll; + overflow-x: hidden; + height: 100%; + padding: 10px; + display: flex; + flex-direction: column-reverse; + + &::-webkit-scrollbar { + width: 8px; + } + &::-webkit-scrollbar-thumb { + background-color: darken($background-color, 10%); + border-radius: $border-radius; + } + + + .chat-content { + display: flex; + flex-direction: column; + } + + .messages { + display: flex; + flex-direction: column; + .message { + padding: 10px; + margin-bottom: 10px; + border-radius: $border-radius; + background-color: lighten($background-color, 5%); + box-shadow: 0 2px 5px $shadow-color; + //display: flex; + align-items: center; + max-width: 70%; + word-break: break-word; + .message-footer { // Assuming this is the container for the toggle button + //max-width: 70%; + + + .toggle-logs-button { + margin-top: 10px; // Padding on sides to align with the text above + width: 95%; + //display: block; // Ensures the button extends the full width of its container + text-align: center; // Centers the text inside the button + //padding: 8px 0; // Adequate padding for touch targets + background-color: $button-color; + color: #fff; + border: none; + border-radius: $border-radius; + cursor: pointer; + //transition: background-color 0.3s; + //margin-top: 10px; // Adds space above the button + box-shadow: 0 2px 4px $shadow-color; // Consistent shadow with other elements + &:hover { + background-color: $button-hover-color; + } + } + .tool-logs { + width: 100%; + background-color: $input-background; + color: $text-color; + margin-top: 5px; + //padding: 10px; + //border-radius: $border-radius; + //box-shadow: inset 0 2px 4px $shadow-color; + //transition: opacity 1s ease-in-out; + font-family: monospace; + overflow-x: auto; + max-height: 150px; // Ensuring it does not grow too large + overflow-y: auto; + } + + } + + .custom-link { + color: lightblue; + text-decoration: underline; + cursor: pointer; + } + &.user { + align-self: flex-end; + background-color: $button-color; + color: #fff; + } + + &.chatbot { + align-self: flex-start; + background-color: $input-background; + color: $text-color; + } + + span { + flex-grow: 1; + padding-right: 10px; + } + + img { + max-width: 50px; + max-height: 50px; + border-radius: 50%; + } + } + } + padding-bottom: 0; + } + + .chat-form { + display: flex; + flex-grow: 1; + //height: 50px; + bottom: 0; + width: 100%; + padding: 10px; + background-color: $input-background; + box-shadow: inset 0 -1px 2px $shadow-color; + + input[type="text"] { + flex-grow: 1; + border: 1px solid darken($input-background, 10%); + border-radius: $border-radius; + padding: 8px 12px; + margin-right: 10px; + } + + button { + padding: 8px 16px; + background-color: $button-color; + color: #fff; + border: none; + border-radius: $border-radius; + cursor: pointer; + transition: background-color 0.3s; + + &:hover { + background-color: $button-hover-color; + } + } + margin-bottom: 0; + } +} + +.initializing-overlay { + position: absolute; + top: 0; + left: 0; + width: 100%; + height: 100%; + background-color: rgba($background-color, 0.95); + display: flex; + justify-content: center; + align-items: center; + font-size: 1.5em; + color: $text-color; + z-index: 10; // Ensure it's above all other content (may be better solution) + + &::before { + content: 'Initializing...'; + font-weight: bold; + } +} + + +.modal { + position: fixed; + top: 0; + left: 0; + width: 100%; + height: 100%; + display: flex; + justify-content: center; + align-items: center; + background-color: rgba(0, 0, 0, 0.4); + + .modal-content { + background-color: $input-background; + color: $text-color; + padding: 20px; + border-radius: $border-radius; + box-shadow: 0 2px 10px $shadow-color; + display: flex; + flex-direction: column; + align-items: center; + width: auto; + min-width: 300px; + + h4 { + margin-bottom: 15px; + } + + p { + margin-bottom: 20px; + } + + button { + padding: 10px 20px; + background-color: $button-color; + color: #fff; + border: none; + border-radius: $border-radius; + cursor: pointer; + margin: 5px; + transition: background-color 0.3s; + + &:hover { + background-color: $button-hover-color; + } + } + } +} diff --git a/src/client/views/nodes/ChatBox/ChatBox.tsx b/src/client/views/nodes/ChatBox/ChatBox.tsx new file mode 100644 index 000000000..880c332ac --- /dev/null +++ b/src/client/views/nodes/ChatBox/ChatBox.tsx @@ -0,0 +1,609 @@ +import { MathJaxContext } from 'better-react-mathjax'; +import { action, makeObservable, observable, observe, reaction, runInAction } from 'mobx'; +import { observer } from 'mobx-react'; +import OpenAI, { ClientOptions } from 'openai'; +import { ImageFile, Message } from 'openai/resources/beta/threads/messages'; +import { RunStep } from 'openai/resources/beta/threads/runs/steps'; +import * as React from 'react'; +import { Doc } from '../../../../fields/Doc'; +import { Id } from '../../../../fields/FieldSymbols'; +import { CsvCast, DocCast, PDFCast, StrCast } from '../../../../fields/Types'; +import { CsvField } from '../../../../fields/URLField'; +import { Networking } from '../../../Network'; +import { DocUtils } from '../../../documents/DocUtils'; +import { DocumentType } from '../../../documents/DocumentTypes'; +import { Docs } from '../../../documents/Documents'; +import { DocumentManager } from '../../../util/DocumentManager'; +import { LinkManager } from '../../../util/LinkManager'; +import { ViewBoxAnnotatableComponent } from '../../DocComponent'; +import { FieldView, FieldViewProps } from '../FieldView'; +import './ChatBox.scss'; +import MessageComponent from './MessageComponent'; +import { ANNOTATION_LINK_TYPE, ASSISTANT_ROLE, AssistantMessage, DOWNLOAD_TYPE } from './types'; + +@observer +export class ChatBox extends ViewBoxAnnotatableComponent() { + @observable modalStatus = false; + @observable currentFile = { url: '' }; + @observable history: AssistantMessage[] = []; + @observable.deep current_message: AssistantMessage | undefined = undefined; + + @observable isLoading: boolean = false; + @observable isInitializing: boolean = true; + @observable expandedLogIndex: number | null = null; + @observable linked_docs_to_add: Doc[] = []; + + private openai: OpenAI; + private interim_history: string = ''; + private assistantID: string = ''; + private threadID: string = ''; + private _oldWheel: any; + private vectorStoreID: string = ''; + private mathJaxConfig: any; + private linkedCsvIDs: string[] = []; + + public static LayoutString(fieldKey: string) { + return FieldView.LayoutString(ChatBox, fieldKey); + } + constructor(props: FieldViewProps) { + super(props); + makeObservable(this); + this.openai = this.initializeOpenAI(); + this.history = []; + this.threadID = StrCast(this.dataDoc.thread_id); + this.assistantID = StrCast(this.dataDoc.assistant_id); + this.vectorStoreID = StrCast(this.dataDoc.vector_store_id); + this.openai = this.initializeOpenAI(); + if (this.assistantID === '' || this.threadID === '' || this.vectorStoreID === '') { + this.createAssistant(); + } else { + this.retrieveCsvUrls(); + this.isInitializing = false; + } + this.mathJaxConfig = { + loader: { load: ['input/asciimath'] }, + tex: { + inlineMath: [ + ['$', '$'], + ['\\(', '\\)'], + ], + displayMath: [ + ['$$', '$$'], + ['[', ']'], + ], + }, + }; + reaction( + () => this.history.map((msg: AssistantMessage) => ({ role: msg.role, text: msg.text, image: msg.image, tool_logs: msg.tool_logs, links: msg.links })), + serializableHistory => { + this.dataDoc.data = JSON.stringify(serializableHistory); + } + ); + } + + toggleToolLogs = (index: number) => { + this.expandedLogIndex = this.expandedLogIndex === index ? null : index; + }; + + retrieveCsvUrls() { + const linkedDocs = LinkManager.Instance.getAllRelatedLinks(this.Document) + .map(d => DocCast(LinkManager.getOppositeAnchor(d, this.Document))) + .map(d => DocCast(d?.annotationOn, d)) + .filter(d => d); + + linkedDocs.forEach(doc => { + const aiFieldId = StrCast(doc[this.Document[Id] + '_ai_field_id']); + if (CsvCast(doc.data)) { + this.linkedCsvIDs.push(StrCast(aiFieldId)); + console.log(this.linkedCsvIDs); + } + }); + } + + initializeOpenAI() { + const configuration: ClientOptions = { + apiKey: process.env.OPENAI_KEY, + dangerouslyAllowBrowser: true, + }; + return new OpenAI(configuration); + } + + onPassiveWheel = (e: WheelEvent) => { + if (this._props.isContentActive()) { + e.stopPropagation(); + } + }; + + createLink = (linkInfo: string, startIndex: number, endIndex: number, linkType: ANNOTATION_LINK_TYPE, annotationIndex: number = 0) => { + const text = this.interim_history; + const subString = this.current_message?.text.substring(startIndex, endIndex) ?? ''; + if (!text) return; + const textToDisplay = `${annotationIndex}`; + let fileInfo = linkInfo; + const fileName = subString.split('/')[subString.split('/').length - 1]; + if (linkType === ANNOTATION_LINK_TYPE.DOWNLOAD_FILE) { + fileInfo = linkInfo + '!!!' + fileName; + } + + const formattedLink = `[${textToDisplay}](${fileInfo}~~~${linkType})`; + console.log(formattedLink); + const newText = text.replace(subString, formattedLink); + runInAction(() => { + this.interim_history = newText; + console.log(newText); + this.current_message?.links?.push({ + start: startIndex, + end: endIndex, + url: linkType === ANNOTATION_LINK_TYPE.DOWNLOAD_FILE ? fileName : linkInfo, + id: linkType === ANNOTATION_LINK_TYPE.DOWNLOAD_FILE ? linkInfo : undefined, + link_type: linkType, + }); + }); + }; + + @action + createAssistant = async () => { + this.isInitializing = true; + try { + const vectorStore = await this.openai.beta.vectorStores.create({ + name: 'Vector Store for Assistant', + }); + const assistant = await this.openai.beta.assistants.create({ + name: 'Document Analyser Assistant', + instructions: ` + You will analyse documents with which you are provided. You will answer questions and provide insights based on the information in the documents. + For writing math formulas: + You have a MathJax render environment. + - Write all in-line equations within a single dollar sign, $, to render them as TeX (this means any time you want to use a dollar sign to represent a dollar sign itself, you must escape it with a backslash: "$"); + - Use a double dollar sign, $$, to render equations on a new line; + Example: $$x^2 + 3x$$ is output for "x² + 3x" to appear as TeX.`, + model: 'gpt-4-turbo', + tools: [{ type: 'file_search' }, { type: 'code_interpreter' }], + tool_resources: { + file_search: { + vector_store_ids: [vectorStore.id], + }, + code_interpreter: { + file_ids: this.linkedCsvIDs, + }, + }, + }); + const thread = await this.openai.beta.threads.create(); + + runInAction(() => { + this.dataDoc.assistant_id = assistant.id; + this.dataDoc.thread_id = thread.id; + this.dataDoc.vector_store_id = vectorStore.id; + this.assistantID = assistant.id; + this.threadID = thread.id; + this.vectorStoreID = vectorStore.id; + this.isInitializing = false; + }); + } catch (error) { + console.error('Initialization failed:', error); + this.isInitializing = false; + } + }; + + @action + runAssistant = async (inputText: string) => { + // Ensure an assistant and thread are created + if (!this.assistantID || !this.threadID || !this.vectorStoreID) { + await this.createAssistant(); + console.log('Assistant and thread created:', this.assistantID, this.threadID); + } + let currentText: string = ''; + let currentToolCallMessage: string = ''; + + // Send the user's input to the assistant + await this.openai.beta.threads.messages.create(this.threadID, { + role: 'user', + content: inputText, + }); + + // Listen to the streaming responses + const stream = this.openai.beta.threads.runs + .stream(this.threadID, { + assistant_id: this.assistantID, + }) + .on('runStepCreated', (runStep: RunStep) => { + currentText = ''; + runInAction(() => { + this.current_message = { role: ASSISTANT_ROLE.ASSISTANT, text: currentText, tool_logs: '', links: [] }; + }); + this.isLoading = true; + }) + .on('toolCallDelta', (toolCallDelta, snapshot) => { + this.isLoading = false; + if (toolCallDelta.type === 'code_interpreter') { + if (toolCallDelta.code_interpreter?.input) { + currentToolCallMessage += toolCallDelta.code_interpreter.input; + runInAction(() => { + if (this.current_message) { + this.current_message.tool_logs = currentToolCallMessage; + } + }); + } + if (toolCallDelta.code_interpreter?.outputs) { + currentToolCallMessage += '\n Code interpreter output:'; + toolCallDelta.code_interpreter.outputs.forEach(output => { + if (output.type === 'logs') { + runInAction(() => { + if (this.current_message) { + this.current_message.tool_logs += '\n|' + output.logs; + } + }); + } + }); + } + } + }) + .on('textDelta', (textDelta, snapshot) => { + this.isLoading = false; + currentText += textDelta.value; + runInAction(() => { + if (this.current_message) { + // this.current_message = {...this.current_message, text: current_text}; + this.current_message.text = currentText; + } + }); + }) + .on('messageDone', async event => { + console.log(event); + const textItem = event.content.find(item => item.type === 'text'); + if (textItem && textItem.type === 'text') { + const { text } = textItem; + console.log(text.value); + try { + runInAction(() => { + this.interim_history = text.value; + }); + } catch (e) { + console.error('Error parsing JSON response:', e); + } + + const { annotations } = text; + console.log('Annotations: ' + annotations); + let index = 0; + annotations.forEach(async annotation => { + console.log(' ' + annotation); + console.log(' ' + annotation.text); + if (annotation.type === 'file_path') { + const { file_path: filePath } = annotation; + const fileToDownload = filePath.file_id; + console.log(fileToDownload); + if (filePath) { + console.log(filePath); + console.log(fileToDownload); + this.createLink(fileToDownload, annotation.start_index, annotation.end_index, ANNOTATION_LINK_TYPE.DOWNLOAD_FILE); + } + } else { + const { file_citation: fileCitation } = annotation; + if (fileCitation) { + const citedFile = await this.openai.files.retrieve(fileCitation.file_id); + const citationUrl = citedFile.filename; + this.createLink(citationUrl, annotation.start_index, annotation.end_index, ANNOTATION_LINK_TYPE.DASH_DOC, index); + index++; + } + } + }); + runInAction(() => { + if (this.current_message) { + console.log('current message: ' + this.current_message.text); + this.current_message.text = this.interim_history; + this.history.push({ ...this.current_message }); + this.current_message = undefined; + } + }); + } + }) + .on('toolCallDone', toolCall => { + runInAction(() => { + if (this.current_message && currentToolCallMessage) { + this.current_message.tool_logs = currentToolCallMessage; + } + }); + }) + .on('imageFileDone', (content: ImageFile, snapshot: Message) => { + console.log('Image file done:', content); + }) + .on('end', () => { + console.log('Streaming done'); + }); + }; + + @action + goToLinkedDoc = async (link: string) => { + const linkedDocs = LinkManager.Instance.getAllRelatedLinks(this.Document) + .map(d => DocCast(LinkManager.getOppositeAnchor(d, this.Document))) + .map(d => DocCast(d?.annotationOn, d)) + .filter(d => d); + + const linkedDoc = linkedDocs.find(doc => { + const docUrl = CsvCast(doc.data, PDFCast(doc.data)).url.pathname.replace('/files/pdfs/', '').replace('/files/csvs/', ''); + console.log('URL: ' + docUrl + ' Citation URL: ' + link); + return link === docUrl; + }); + + if (linkedDoc) { + await DocumentManager.Instance.showDocument(DocCast(linkedDoc), { willZoomCentered: true }, () => {}); + } + }; + + @action + askGPT = async (event: React.FormEvent): Promise => { + event.preventDefault(); + + const textInput = event.currentTarget.elements.namedItem('messageInput') as HTMLInputElement; + const trimmedText = textInput.value.trim(); + + if (!this.assistantID || !this.threadID) { + try { + await this.createAssistant(); + } catch (err) { + console.error('Error:', err); + } + } + + if (trimmedText) { + try { + textInput.value = ''; + runInAction(() => { + this.history.push({ role: ASSISTANT_ROLE.USER, text: trimmedText }); + }); + await this.runAssistant(trimmedText); + this.dataDoc.data = this.history.toString(); + } catch (err) { + console.error('Error:', err); + } + } + }; + + @action + uploadLinks = async (linkedDocs: Doc[]) => { + if (this.isInitializing) { + console.log('Initialization in progress, upload aborted.'); + return; + } + const urls = linkedDocs.map(doc => CsvCast(doc.data, PDFCast(doc.data)).url.pathname); + const csvUrls = urls.filter(url => url.endsWith('.csv')); + console.log(this.assistantID, this.threadID, urls); + + const { openai_file_ids: openaiFileIds } = await Networking.PostToServer('/uploadPDFToVectorStore', { urls, threadID: this.threadID, assistantID: this.assistantID, vector_store_id: this.vectorStoreID }); + + linkedDocs.forEach((doc, i) => { + doc[this.Document[Id] + '_ai_field_id'] = openaiFileIds[i]; + console.log('AI Field ID: ' + openaiFileIds[i]); + }); + + if (csvUrls.length > 0) { + for (let i = 0; i < csvUrls.length; i++) { + this.linkedCsvIDs.push(openaiFileIds[urls.indexOf(csvUrls[i])]); + } + console.log('linked csvs:' + this.linkedCsvIDs); + await this.openai.beta.assistants.update(this.assistantID, { + tools: [{ type: 'file_search' }, { type: 'code_interpreter' }], + tool_resources: { + file_search: { + vector_store_ids: [this.vectorStoreID], + }, + code_interpreter: { + file_ids: this.linkedCsvIDs, + }, + }, + }); + } + }; + + downloadToComputer = (url: string, fileName: string) => { + fetch(url, { method: 'get', mode: 'no-cors', referrerPolicy: 'no-referrer' }) + .then(res => res.blob()) + .then(res => { + const aElement = document.createElement('a'); + aElement.setAttribute('download', fileName); + const href = URL.createObjectURL(res); + aElement.href = href; + aElement.setAttribute('target', '_blank'); + aElement.click(); + URL.revokeObjectURL(href); + }); + }; + + createDocumentInDash = async (url: string) => { + const fileSuffix = url.substring(url.lastIndexOf('.') + 1); + console.log(fileSuffix); + let doc: Doc | null = null; + switch (fileSuffix) { + case 'pdf': + doc = DocCast(await DocUtils.DocumentFromType('pdf', url, {})); + break; + case 'csv': + doc = DocCast(await DocUtils.DocumentFromType('csv', url, {})); + break; + case 'png': + case 'jpg': + case 'jpeg': + doc = DocCast(await DocUtils.DocumentFromType('image', url, {})); + break; + default: + console.error('Unsupported file type:', fileSuffix); + break; + } + if (doc) { + doc && this._props.addDocument?.(doc); + await DocumentManager.Instance.showDocument(doc, { willZoomCentered: true }, () => {}); + } + }; + + downloadFile = async (fileInfo: string, downloadType: DOWNLOAD_TYPE) => { + try { + console.log(fileInfo); + const [fileId, fileName] = fileInfo.split(/!!!/); + const { file_path: filePath } = await Networking.PostToServer('/downloadFileFromOpenAI', { file_id: fileId, file_name: fileName }); + const fileLink = CsvCast(new CsvField(filePath)).url.href; + if (downloadType === DOWNLOAD_TYPE.DASH) { + this.createDocumentInDash(fileLink); + } else { + this.downloadToComputer(fileLink, fileName); + } + } catch (error) { + console.error('Error downloading file:', error); + } + }; + + handleDownloadToDevice = () => { + this.downloadFile(this.currentFile.url, DOWNLOAD_TYPE.DEVICE); + this.modalStatus = false; // Close the modal after the action + this.currentFile = { url: '' }; // Reset the current file + }; + + handleAddToDash = () => { + // Assuming `downloadFile` is a method that handles adding to Dash + this.downloadFile(this.currentFile.url, DOWNLOAD_TYPE.DASH); + this.modalStatus = false; // Close the modal after the action + this.currentFile = { url: '' }; // Reset the current file + }; + + renderModal = () => { + if (!this.modalStatus) return null; + + return ( +
+
+

File Actions

+

Choose an action for the file:

+ + + +
+
+ ); + }; + @action + showModal = () => { + this.modalStatus = true; + }; + + @action + setCurrentFile = (file: { url: string }) => { + this.currentFile = file; + }; + + componentDidMount() { + this._props.setContentViewBox?.(this); + if (this.dataDoc.data) { + try { + const storedHistory = JSON.parse(StrCast(this.dataDoc.data)); + runInAction(() => { + this.history = storedHistory.map((msg: AssistantMessage) => ({ + role: msg.role, + text: msg.text, + quote: msg.quote, + tool_logs: msg.tool_logs, + image: msg.image, + })); + }); + } catch (e) { + console.error('Failed to parse history from dataDoc:', e); + } + } + reaction( + () => { + const linkedDocs = LinkManager.Instance.getAllRelatedLinks(this.Document) + .map(d => DocCast(LinkManager.getOppositeAnchor(d, this.Document))) + .map(d => DocCast(d?.annotationOn, d)) + .filter(d => d); + return linkedDocs; + }, + + linked => this.linked_docs_to_add.push(...linked.filter(linkedDoc => !this.linked_docs_to_add.includes(linkedDoc))) + ); + + observe( + // right now this skips during initialization which is necessary because it would be blank + // However, it will upload the same link twice when it is + this.linked_docs_to_add, + change => { + // observe pushes/splices on a user link DB 'data' field (should only happen for local changes) + switch (change.type as any) { + case 'splice': + if ((change as any).addedCount > 0) { + // maybe check here if its already in the urls datadoc array so doesn't add twice + console.log((change as any).added as Doc[]); + this.uploadLinks((change as any).added as Doc[]); + } + // (change as any).removed.forEach((link: any) => remLinkFromDoc(toRealField(link))); + break; + case 'update': // let oldValue = change.oldValue; + default: + } + }, + true + ); + } + + render() { + return ( + +
+ {this.isInitializing &&
Initializing...
} + {this.renderModal()} +
{ + this._oldWheel?.removeEventListener('wheel', this.onPassiveWheel); + this._oldWheel = r; + r?.addEventListener('wheel', this.onPassiveWheel, { passive: false }); + }}> +
+ {this.history.map((message, index) => ( + + ))} + {!this.current_message ? null : ( + + )} +
+
+
+ + +
+
+ + ); + } +} + +Docs.Prototypes.TemplateMap.set(DocumentType.CHAT, { + layout: { view: ChatBox, dataField: 'data' }, + options: { acl: '', chat: '', chat_history: '', chat_thread_id: '', chat_assistant_id: '', chat_vector_store_id: '' }, +}); diff --git a/src/client/views/nodes/ChatBox/MessageComponent.tsx b/src/client/views/nodes/ChatBox/MessageComponent.tsx new file mode 100644 index 000000000..fced0b4d5 --- /dev/null +++ b/src/client/views/nodes/ChatBox/MessageComponent.tsx @@ -0,0 +1,116 @@ +/* eslint-disable react/require-default-props */ +import React from 'react'; +import { observer } from 'mobx-react'; +import { MathJax, MathJaxContext } from 'better-react-mathjax'; +import ReactMarkdown from 'react-markdown'; +import { TbCircle0Filled, TbCircle1Filled, TbCircle2Filled, TbCircle3Filled, TbCircle4Filled, TbCircle5Filled, TbCircle6Filled, TbCircle7Filled, TbCircle8Filled, TbCircle9Filled } from 'react-icons/tb'; +import { AssistantMessage } from './types'; + +interface MessageComponentProps { + message: AssistantMessage; + toggleToolLogs: (index: number) => void; + expandedLogIndex: number | null; + index: number; + showModal: () => void; + goToLinkedDoc: (url: string) => void; + setCurrentFile: (file: { url: string }) => void; + isCurrent?: boolean; +} + +const MessageComponent: React.FC = function ({ message, toggleToolLogs, expandedLogIndex, goToLinkedDoc, index, showModal, setCurrentFile, isCurrent = false }) { + // const messageClass = `${message.role} ${isCurrent ? 'current-message' : ''}`; + + const LinkRenderer = ({ href, children }: { href: string; children: React.ReactNode }) => { + // console.log(href + " " + children) + const regex = /([a-zA-Z0-9_.!-]+)~~~(citation|file_path)/; + const matches = href.match(regex); + // console.log(href) + // console.log(matches) + const url = matches ? matches[1] : href; + const linkType = matches ? matches[2] : null; + if (linkType === 'citation') { + switch (children) { + case '0': + children = ; + break; + case '1': + children = ; + break; + case '2': + children = ; + break; + case '3': + children = ; + break; + case '4': + children = ; + break; + case '5': + children = ; + break; + case '6': + children = ; + break; + case '7': + children = ; + break; + case '8': + children = ; + break; + case '9': + children = ; + break; + default: + break; + } + } + // console.log(linkType) + const style = { + color: 'lightblue', + verticalAlign: linkType === 'citation' ? 'super' : 'baseline', + fontSize: linkType === 'citation' ? 'smaller' : 'inherit', + }; + + return ( + { + e.preventDefault(); + if (linkType === 'citation') { + goToLinkedDoc(url); + } else if (linkType === 'file_path') { + showModal(); + setCurrentFile({ url }); + } + }} + style={style}> + {children} + + ); + }; + + return ( +
+ + + {message.text ? message.text : ''} + + + {message.image && } +
+ {message.tool_logs && ( + + )} + {expandedLogIndex === index && ( +
+ 
{message.tool_logs}
+
+ )} +
+
+ ); +}; + +export default observer(MessageComponent); diff --git a/src/client/views/nodes/ChatBox/types.ts b/src/client/views/nodes/ChatBox/types.ts new file mode 100644 index 000000000..8212a7050 --- /dev/null +++ b/src/client/views/nodes/ChatBox/types.ts @@ -0,0 +1,23 @@ +export enum ASSISTANT_ROLE { + USER = 'User', + ASSISTANT = 'Assistant', +} + +export enum ANNOTATION_LINK_TYPE { + DASH_DOC = 'citation', + DOWNLOAD_FILE = 'file_path', +} + +export enum DOWNLOAD_TYPE { + DASH = 'dash', + DEVICE = 'device', +} + +export interface AssistantMessage { + role: ASSISTANT_ROLE; + text: string; + quote?: string; + image?: string; + tool_logs?: string; + links?: { start: number; end: number; url: string; id?: string; link_type: ANNOTATION_LINK_TYPE }[]; +} diff --git a/src/client/views/nodes/DocumentContentsView.tsx b/src/client/views/nodes/DocumentContentsView.tsx index 18529a429..192c7875e 100644 --- a/src/client/views/nodes/DocumentContentsView.tsx +++ b/src/client/views/nodes/DocumentContentsView.tsx @@ -79,7 +79,7 @@ export class DocumentContentsView extends ObservableReactComponent = T extends string ? 'string' : T extends number ? 'number' : T extends boolean ? 'boolean' : T extends List ? ListSpec : new (...args: any[]) => T; @@ -122,6 +122,9 @@ export function CsvCast(field: FieldResult, defaultVal: CsvField | null = null) export function WebCast(field: FieldResult, defaultVal: WebField | null = null) { return Cast(field, WebField, defaultVal); } +export function PDFCast(field: FieldResult, defaultVal: PdfField | null = null) { + return Cast(field, PdfField, defaultVal); +} export function ImageCast(field: FieldResult, defaultVal: ImageField | null = null) { return Cast(field, ImageField, defaultVal); } diff --git a/src/server/ApiManagers/AssistantManager.ts b/src/server/ApiManagers/AssistantManager.ts new file mode 100644 index 000000000..82e48167a --- /dev/null +++ b/src/server/ApiManagers/AssistantManager.ts @@ -0,0 +1,131 @@ +import * as fs from 'fs'; +import { createReadStream, writeFile } from 'fs'; +import OpenAI from 'openai'; +import * as path from 'path'; +import { promisify } from 'util'; +import * as uuid from 'uuid'; +import { filesDirectory, publicDirectory } from '..'; +import { Method } from '../RouteManager'; +import ApiManager, { Registration } from './ApiManager'; + +export enum Directory { + parsed_files = 'parsed_files', + images = 'images', + videos = 'videos', + pdfs = 'pdfs', + text = 'text', + pdf_thumbnails = 'pdf_thumbnails', + audio = 'audio', + csv = 'csv', +} + +export function serverPathToFile(directory: Directory, filename: string) { + return path.normalize(`${filesDirectory}/${directory}/${filename}`); +} + +export function pathToDirectory(directory: Directory) { + return path.normalize(`${filesDirectory}/${directory}`); +} + +export function clientPathToFile(directory: Directory, filename: string) { + return `/files/${directory}/${filename}`; +} + +const writeFileAsync = promisify(writeFile); +const readFileAsync = promisify(fs.readFile); + +export default class AssistantManager extends ApiManager { + protected initialize(register: Registration): void { + const openai = new OpenAI({ apiKey: process.env.OPENAI_KEY, dangerouslyAllowBrowser: true }); + + register({ + method: Method.POST, + subscription: '/uploadPDFToVectorStore', + secureHandler: async ({ req, res }) => { + const { urls, threadID, assistantID, vector_store_id } = req.body; + + const csvFilesIds: string[] = []; + const otherFileIds: string[] = []; + const allFileIds: string[] = []; + + const fileProcesses = urls.map(async (source: string) => { + const fullPath = path.join(publicDirectory, source); + const fileData = await openai.files.create({ file: createReadStream(fullPath), purpose: 'assistants' }); + allFileIds.push(fileData.id); + if (source.endsWith('.csv')) { + console.log(source); + csvFilesIds.push(fileData.id); + } else { + openai.beta.vectorStores.files.create(vector_store_id, { file_id: fileData.id }); + otherFileIds.push(fileData.id); + } + }); + try { + await Promise.all(fileProcesses).then(() => { + res.send({ vector_store_id: vector_store_id, openai_file_ids: allFileIds }); + }); + } catch (error) { + res.status(500).send({ error: 'Failed to process files' + error }); + } + }, + }); + + register({ + method: Method.POST, + subscription: '/downloadFileFromOpenAI', + secureHandler: async ({ req, res }) => { + const { file_id, file_name } = req.body; + //let files_directory: string; + let files_directory = '/files/openAIFiles/'; + switch (file_name.split('.').pop()) { + case 'pdf': + files_directory = '/files/pdfs/'; + break; + case 'csv': + files_directory = '/files/csv/'; + break; + case 'png': + case 'jpg': + case 'jpeg': + files_directory = '/files/images/'; + break; + default: + break; + } + + const directory = path.join(publicDirectory, files_directory); + + if (!fs.existsSync(directory)) { + fs.mkdirSync(directory); + } + const file = await openai.files.content(file_id); + const new_file_name = `${uuid.v4()}-${file_name}`; + const file_path = path.join(directory, new_file_name); + const file_array_buffer = await file.arrayBuffer(); + const bufferView = new Uint8Array(file_array_buffer); + try { + const written_file = await writeFileAsync(file_path, bufferView); + console.log(written_file); + console.log(file_path); + console.log(file_array_buffer); + console.log(bufferView); + const file_object = new File([bufferView], file_name); + //DashUploadUtils.upload(file_object, 'openAIFiles'); + res.send({ file_path: path.join(files_directory, new_file_name) }); + /* res.send( { + source: "file", + result: { + accessPaths: { + agnostic: {client: path.join('/files/openAIFiles/', `${uuid.v4()}-${file_name}`)} + }, + rawText: "", + duration: 0, + }, + } ); */ + } catch (error) { + res.status(500).send({ error: 'Failed to write file' + error }); + } + }, + }); + } +} diff --git a/src/server/index.ts b/src/server/index.ts index 1bbf8a105..3151c2975 100644 --- a/src/server/index.ts +++ b/src/server/index.ts @@ -3,7 +3,7 @@ import * as dotenv from 'dotenv'; import * as mobileDetect from 'mobile-detect'; import * as path from 'path'; import { logExecution } from './ActionUtilities'; -import { AdminPrivileges, resolvedPorts } from './SocketData'; +import AssistantManager from './ApiManagers/AssistantManager'; import DataVizManager from './ApiManagers/DataVizManager'; import DeleteManager from './ApiManagers/DeleteManager'; import DownloadManager from './ApiManagers/DownloadManager'; @@ -13,16 +13,17 @@ import SessionManager from './ApiManagers/SessionManager'; import UploadManager from './ApiManagers/UploadManager'; import UserManager from './ApiManagers/UserManager'; import UtilManager from './ApiManagers/UtilManager'; -import { GoogleCredentialsLoader, SSL } from './apis/google/CredentialsLoader'; -import { GoogleApiServerUtils } from './apis/google/GoogleApiServerUtils'; import { DashSessionAgent } from './DashSession/DashSessionAgent'; import { AppliedSessionAgent } from './DashSession/Session/agents/applied_session_agent'; import { DashStats } from './DashStats'; import { DashUploadUtils } from './DashUploadUtils'; -import { Database } from './database'; import { Logger } from './ProcessFactory'; import RouteManager, { Method, PublicHandler } from './RouteManager'; import RouteSubscriber from './RouteSubscriber'; +import { AdminPrivileges, resolvedPorts } from './SocketData'; +import { GoogleCredentialsLoader, SSL } from './apis/google/CredentialsLoader'; +import { GoogleApiServerUtils } from './apis/google/GoogleApiServerUtils'; +import { Database } from './database'; import initializeServer from './server_Initialization'; // import GooglePhotosManager from './ApiManagers/GooglePhotosManager'; @@ -72,6 +73,7 @@ function routeSetter({ addSupervisedRoute, logRegistrationOutcome }: RouteManage new UtilManager(), new GeneralGoogleManager(), /* new GooglePhotosManager(), */ new DataVizManager(), + new AssistantManager(), ]; // initialize API Managers -- cgit v1.2.3-70-g09d2