diff options
Diffstat (limited to 'src/client/views/nodes')
| -rw-r--r-- | src/client/views/nodes/ChatBox/ChatBox.scss | 228 | ||||
| -rw-r--r-- | src/client/views/nodes/ChatBox/ChatBox.tsx | 609 | ||||
| -rw-r--r-- | src/client/views/nodes/ChatBox/MessageComponent.tsx | 116 | ||||
| -rw-r--r-- | src/client/views/nodes/ChatBox/types.ts | 23 | ||||
| -rw-r--r-- | src/client/views/nodes/DiagramBox.scss | 88 | ||||
| -rw-r--r-- | src/client/views/nodes/DiagramBox.tsx | 305 | ||||
| -rw-r--r-- | src/client/views/nodes/DocumentContentsView.tsx | 2 |
7 files changed, 1370 insertions, 1 deletions
diff --git a/src/client/views/nodes/ChatBox/ChatBox.scss b/src/client/views/nodes/ChatBox/ChatBox.scss new file mode 100644 index 000000000..f1ad3d074 --- /dev/null +++ b/src/client/views/nodes/ChatBox/ChatBox.scss @@ -0,0 +1,228 @@ +$background-color: #f8f9fa; +$text-color: #333; +$input-background: #fff; +$button-color: #007bff; +$button-hover-color: darken($button-color, 10%); +$shadow-color: rgba(0, 0, 0, 0.075); +$border-radius: 8px; + +.chatBox { + display: flex; + flex-direction: column; + width: 100%; /* Adjust the width as needed, could be in percentage */ + height: 100%; /* Adjust the height as needed, could be in percentage */ + background-color: $background-color; + font-family: 'Helvetica Neue', Arial, sans-serif; + //margin: 20px auto; + //overflow: hidden; + + .scroll-box { + flex-grow: 1; + overflow-y: scroll; + overflow-x: hidden; + height: 100%; + padding: 10px; + display: flex; + flex-direction: column-reverse; + + &::-webkit-scrollbar { + width: 8px; + } + &::-webkit-scrollbar-thumb { + background-color: darken($background-color, 10%); + border-radius: $border-radius; + } + + + .chat-content { + display: flex; + flex-direction: column; + } + + .messages { + display: flex; + flex-direction: column; + .message { + padding: 10px; + margin-bottom: 10px; + border-radius: $border-radius; + background-color: lighten($background-color, 5%); + box-shadow: 0 2px 5px $shadow-color; + //display: flex; + align-items: center; + max-width: 70%; + word-break: break-word; + .message-footer { // Assuming this is the container for the toggle button + //max-width: 70%; + + + .toggle-logs-button { + margin-top: 10px; // Padding on sides to align with the text above + width: 95%; + //display: block; // Ensures the button extends the full width of its container + text-align: center; // Centers the text inside the button + //padding: 8px 0; // Adequate padding for touch targets + background-color: $button-color; + color: #fff; + border: none; + border-radius: $border-radius; + cursor: pointer; + //transition: background-color 0.3s; + //margin-top: 10px; // Adds space above the button + box-shadow: 0 2px 4px $shadow-color; // Consistent shadow with other elements + &:hover { + background-color: $button-hover-color; + } + } + .tool-logs { + width: 100%; + background-color: $input-background; + color: $text-color; + margin-top: 5px; + //padding: 10px; + //border-radius: $border-radius; + //box-shadow: inset 0 2px 4px $shadow-color; + //transition: opacity 1s ease-in-out; + font-family: monospace; + overflow-x: auto; + max-height: 150px; // Ensuring it does not grow too large + overflow-y: auto; + } + + } + + .custom-link { + color: lightblue; + text-decoration: underline; + cursor: pointer; + } + &.user { + align-self: flex-end; + background-color: $button-color; + color: #fff; + } + + &.chatbot { + align-self: flex-start; + background-color: $input-background; + color: $text-color; + } + + span { + flex-grow: 1; + padding-right: 10px; + } + + img { + max-width: 50px; + max-height: 50px; + border-radius: 50%; + } + } + } + padding-bottom: 0; + } + + .chat-form { + display: flex; + flex-grow: 1; + //height: 50px; + bottom: 0; + width: 100%; + padding: 10px; + background-color: $input-background; + box-shadow: inset 0 -1px 2px $shadow-color; + + input[type="text"] { + flex-grow: 1; + border: 1px solid darken($input-background, 10%); + border-radius: $border-radius; + padding: 8px 12px; + margin-right: 10px; + } + + button { + padding: 8px 16px; + background-color: $button-color; + color: #fff; + border: none; + border-radius: $border-radius; + cursor: pointer; + transition: background-color 0.3s; + + &:hover { + background-color: $button-hover-color; + } + } + margin-bottom: 0; + } +} + +.initializing-overlay { + position: absolute; + top: 0; + left: 0; + width: 100%; + height: 100%; + background-color: rgba($background-color, 0.95); + display: flex; + justify-content: center; + align-items: center; + font-size: 1.5em; + color: $text-color; + z-index: 10; // Ensure it's above all other content (may be better solution) + + &::before { + content: 'Initializing...'; + font-weight: bold; + } +} + + +.modal { + position: fixed; + top: 0; + left: 0; + width: 100%; + height: 100%; + display: flex; + justify-content: center; + align-items: center; + background-color: rgba(0, 0, 0, 0.4); + + .modal-content { + background-color: $input-background; + color: $text-color; + padding: 20px; + border-radius: $border-radius; + box-shadow: 0 2px 10px $shadow-color; + display: flex; + flex-direction: column; + align-items: center; + width: auto; + min-width: 300px; + + h4 { + margin-bottom: 15px; + } + + p { + margin-bottom: 20px; + } + + button { + padding: 10px 20px; + background-color: $button-color; + color: #fff; + border: none; + border-radius: $border-radius; + cursor: pointer; + margin: 5px; + transition: background-color 0.3s; + + &:hover { + background-color: $button-hover-color; + } + } + } +} diff --git a/src/client/views/nodes/ChatBox/ChatBox.tsx b/src/client/views/nodes/ChatBox/ChatBox.tsx new file mode 100644 index 000000000..880c332ac --- /dev/null +++ b/src/client/views/nodes/ChatBox/ChatBox.tsx @@ -0,0 +1,609 @@ +import { MathJaxContext } from 'better-react-mathjax'; +import { action, makeObservable, observable, observe, reaction, runInAction } from 'mobx'; +import { observer } from 'mobx-react'; +import OpenAI, { ClientOptions } from 'openai'; +import { ImageFile, Message } from 'openai/resources/beta/threads/messages'; +import { RunStep } from 'openai/resources/beta/threads/runs/steps'; +import * as React from 'react'; +import { Doc } from '../../../../fields/Doc'; +import { Id } from '../../../../fields/FieldSymbols'; +import { CsvCast, DocCast, PDFCast, StrCast } from '../../../../fields/Types'; +import { CsvField } from '../../../../fields/URLField'; +import { Networking } from '../../../Network'; +import { DocUtils } from '../../../documents/DocUtils'; +import { DocumentType } from '../../../documents/DocumentTypes'; +import { Docs } from '../../../documents/Documents'; +import { DocumentManager } from '../../../util/DocumentManager'; +import { LinkManager } from '../../../util/LinkManager'; +import { ViewBoxAnnotatableComponent } from '../../DocComponent'; +import { FieldView, FieldViewProps } from '../FieldView'; +import './ChatBox.scss'; +import MessageComponent from './MessageComponent'; +import { ANNOTATION_LINK_TYPE, ASSISTANT_ROLE, AssistantMessage, DOWNLOAD_TYPE } from './types'; + +@observer +export class ChatBox extends ViewBoxAnnotatableComponent<FieldViewProps>() { + @observable modalStatus = false; + @observable currentFile = { url: '' }; + @observable history: AssistantMessage[] = []; + @observable.deep current_message: AssistantMessage | undefined = undefined; + + @observable isLoading: boolean = false; + @observable isInitializing: boolean = true; + @observable expandedLogIndex: number | null = null; + @observable linked_docs_to_add: Doc[] = []; + + private openai: OpenAI; + private interim_history: string = ''; + private assistantID: string = ''; + private threadID: string = ''; + private _oldWheel: any; + private vectorStoreID: string = ''; + private mathJaxConfig: any; + private linkedCsvIDs: string[] = []; + + public static LayoutString(fieldKey: string) { + return FieldView.LayoutString(ChatBox, fieldKey); + } + constructor(props: FieldViewProps) { + super(props); + makeObservable(this); + this.openai = this.initializeOpenAI(); + this.history = []; + this.threadID = StrCast(this.dataDoc.thread_id); + this.assistantID = StrCast(this.dataDoc.assistant_id); + this.vectorStoreID = StrCast(this.dataDoc.vector_store_id); + this.openai = this.initializeOpenAI(); + if (this.assistantID === '' || this.threadID === '' || this.vectorStoreID === '') { + this.createAssistant(); + } else { + this.retrieveCsvUrls(); + this.isInitializing = false; + } + this.mathJaxConfig = { + loader: { load: ['input/asciimath'] }, + tex: { + inlineMath: [ + ['$', '$'], + ['\\(', '\\)'], + ], + displayMath: [ + ['$$', '$$'], + ['[', ']'], + ], + }, + }; + reaction( + () => this.history.map((msg: AssistantMessage) => ({ role: msg.role, text: msg.text, image: msg.image, tool_logs: msg.tool_logs, links: msg.links })), + serializableHistory => { + this.dataDoc.data = JSON.stringify(serializableHistory); + } + ); + } + + toggleToolLogs = (index: number) => { + this.expandedLogIndex = this.expandedLogIndex === index ? null : index; + }; + + retrieveCsvUrls() { + const linkedDocs = LinkManager.Instance.getAllRelatedLinks(this.Document) + .map(d => DocCast(LinkManager.getOppositeAnchor(d, this.Document))) + .map(d => DocCast(d?.annotationOn, d)) + .filter(d => d); + + linkedDocs.forEach(doc => { + const aiFieldId = StrCast(doc[this.Document[Id] + '_ai_field_id']); + if (CsvCast(doc.data)) { + this.linkedCsvIDs.push(StrCast(aiFieldId)); + console.log(this.linkedCsvIDs); + } + }); + } + + initializeOpenAI() { + const configuration: ClientOptions = { + apiKey: process.env.OPENAI_KEY, + dangerouslyAllowBrowser: true, + }; + return new OpenAI(configuration); + } + + onPassiveWheel = (e: WheelEvent) => { + if (this._props.isContentActive()) { + e.stopPropagation(); + } + }; + + createLink = (linkInfo: string, startIndex: number, endIndex: number, linkType: ANNOTATION_LINK_TYPE, annotationIndex: number = 0) => { + const text = this.interim_history; + const subString = this.current_message?.text.substring(startIndex, endIndex) ?? ''; + if (!text) return; + const textToDisplay = `${annotationIndex}`; + let fileInfo = linkInfo; + const fileName = subString.split('/')[subString.split('/').length - 1]; + if (linkType === ANNOTATION_LINK_TYPE.DOWNLOAD_FILE) { + fileInfo = linkInfo + '!!!' + fileName; + } + + const formattedLink = `[${textToDisplay}](${fileInfo}~~~${linkType})`; + console.log(formattedLink); + const newText = text.replace(subString, formattedLink); + runInAction(() => { + this.interim_history = newText; + console.log(newText); + this.current_message?.links?.push({ + start: startIndex, + end: endIndex, + url: linkType === ANNOTATION_LINK_TYPE.DOWNLOAD_FILE ? fileName : linkInfo, + id: linkType === ANNOTATION_LINK_TYPE.DOWNLOAD_FILE ? linkInfo : undefined, + link_type: linkType, + }); + }); + }; + + @action + createAssistant = async () => { + this.isInitializing = true; + try { + const vectorStore = await this.openai.beta.vectorStores.create({ + name: 'Vector Store for Assistant', + }); + const assistant = await this.openai.beta.assistants.create({ + name: 'Document Analyser Assistant', + instructions: ` + You will analyse documents with which you are provided. You will answer questions and provide insights based on the information in the documents. + For writing math formulas: + You have a MathJax render environment. + - Write all in-line equations within a single dollar sign, $, to render them as TeX (this means any time you want to use a dollar sign to represent a dollar sign itself, you must escape it with a backslash: "$"); + - Use a double dollar sign, $$, to render equations on a new line; + Example: $$x^2 + 3x$$ is output for "x² + 3x" to appear as TeX.`, + model: 'gpt-4-turbo', + tools: [{ type: 'file_search' }, { type: 'code_interpreter' }], + tool_resources: { + file_search: { + vector_store_ids: [vectorStore.id], + }, + code_interpreter: { + file_ids: this.linkedCsvIDs, + }, + }, + }); + const thread = await this.openai.beta.threads.create(); + + runInAction(() => { + this.dataDoc.assistant_id = assistant.id; + this.dataDoc.thread_id = thread.id; + this.dataDoc.vector_store_id = vectorStore.id; + this.assistantID = assistant.id; + this.threadID = thread.id; + this.vectorStoreID = vectorStore.id; + this.isInitializing = false; + }); + } catch (error) { + console.error('Initialization failed:', error); + this.isInitializing = false; + } + }; + + @action + runAssistant = async (inputText: string) => { + // Ensure an assistant and thread are created + if (!this.assistantID || !this.threadID || !this.vectorStoreID) { + await this.createAssistant(); + console.log('Assistant and thread created:', this.assistantID, this.threadID); + } + let currentText: string = ''; + let currentToolCallMessage: string = ''; + + // Send the user's input to the assistant + await this.openai.beta.threads.messages.create(this.threadID, { + role: 'user', + content: inputText, + }); + + // Listen to the streaming responses + const stream = this.openai.beta.threads.runs + .stream(this.threadID, { + assistant_id: this.assistantID, + }) + .on('runStepCreated', (runStep: RunStep) => { + currentText = ''; + runInAction(() => { + this.current_message = { role: ASSISTANT_ROLE.ASSISTANT, text: currentText, tool_logs: '', links: [] }; + }); + this.isLoading = true; + }) + .on('toolCallDelta', (toolCallDelta, snapshot) => { + this.isLoading = false; + if (toolCallDelta.type === 'code_interpreter') { + if (toolCallDelta.code_interpreter?.input) { + currentToolCallMessage += toolCallDelta.code_interpreter.input; + runInAction(() => { + if (this.current_message) { + this.current_message.tool_logs = currentToolCallMessage; + } + }); + } + if (toolCallDelta.code_interpreter?.outputs) { + currentToolCallMessage += '\n Code interpreter output:'; + toolCallDelta.code_interpreter.outputs.forEach(output => { + if (output.type === 'logs') { + runInAction(() => { + if (this.current_message) { + this.current_message.tool_logs += '\n|' + output.logs; + } + }); + } + }); + } + } + }) + .on('textDelta', (textDelta, snapshot) => { + this.isLoading = false; + currentText += textDelta.value; + runInAction(() => { + if (this.current_message) { + // this.current_message = {...this.current_message, text: current_text}; + this.current_message.text = currentText; + } + }); + }) + .on('messageDone', async event => { + console.log(event); + const textItem = event.content.find(item => item.type === 'text'); + if (textItem && textItem.type === 'text') { + const { text } = textItem; + console.log(text.value); + try { + runInAction(() => { + this.interim_history = text.value; + }); + } catch (e) { + console.error('Error parsing JSON response:', e); + } + + const { annotations } = text; + console.log('Annotations: ' + annotations); + let index = 0; + annotations.forEach(async annotation => { + console.log(' ' + annotation); + console.log(' ' + annotation.text); + if (annotation.type === 'file_path') { + const { file_path: filePath } = annotation; + const fileToDownload = filePath.file_id; + console.log(fileToDownload); + if (filePath) { + console.log(filePath); + console.log(fileToDownload); + this.createLink(fileToDownload, annotation.start_index, annotation.end_index, ANNOTATION_LINK_TYPE.DOWNLOAD_FILE); + } + } else { + const { file_citation: fileCitation } = annotation; + if (fileCitation) { + const citedFile = await this.openai.files.retrieve(fileCitation.file_id); + const citationUrl = citedFile.filename; + this.createLink(citationUrl, annotation.start_index, annotation.end_index, ANNOTATION_LINK_TYPE.DASH_DOC, index); + index++; + } + } + }); + runInAction(() => { + if (this.current_message) { + console.log('current message: ' + this.current_message.text); + this.current_message.text = this.interim_history; + this.history.push({ ...this.current_message }); + this.current_message = undefined; + } + }); + } + }) + .on('toolCallDone', toolCall => { + runInAction(() => { + if (this.current_message && currentToolCallMessage) { + this.current_message.tool_logs = currentToolCallMessage; + } + }); + }) + .on('imageFileDone', (content: ImageFile, snapshot: Message) => { + console.log('Image file done:', content); + }) + .on('end', () => { + console.log('Streaming done'); + }); + }; + + @action + goToLinkedDoc = async (link: string) => { + const linkedDocs = LinkManager.Instance.getAllRelatedLinks(this.Document) + .map(d => DocCast(LinkManager.getOppositeAnchor(d, this.Document))) + .map(d => DocCast(d?.annotationOn, d)) + .filter(d => d); + + const linkedDoc = linkedDocs.find(doc => { + const docUrl = CsvCast(doc.data, PDFCast(doc.data)).url.pathname.replace('/files/pdfs/', '').replace('/files/csvs/', ''); + console.log('URL: ' + docUrl + ' Citation URL: ' + link); + return link === docUrl; + }); + + if (linkedDoc) { + await DocumentManager.Instance.showDocument(DocCast(linkedDoc), { willZoomCentered: true }, () => {}); + } + }; + + @action + askGPT = async (event: React.FormEvent<HTMLFormElement>): Promise<void> => { + event.preventDefault(); + + const textInput = event.currentTarget.elements.namedItem('messageInput') as HTMLInputElement; + const trimmedText = textInput.value.trim(); + + if (!this.assistantID || !this.threadID) { + try { + await this.createAssistant(); + } catch (err) { + console.error('Error:', err); + } + } + + if (trimmedText) { + try { + textInput.value = ''; + runInAction(() => { + this.history.push({ role: ASSISTANT_ROLE.USER, text: trimmedText }); + }); + await this.runAssistant(trimmedText); + this.dataDoc.data = this.history.toString(); + } catch (err) { + console.error('Error:', err); + } + } + }; + + @action + uploadLinks = async (linkedDocs: Doc[]) => { + if (this.isInitializing) { + console.log('Initialization in progress, upload aborted.'); + return; + } + const urls = linkedDocs.map(doc => CsvCast(doc.data, PDFCast(doc.data)).url.pathname); + const csvUrls = urls.filter(url => url.endsWith('.csv')); + console.log(this.assistantID, this.threadID, urls); + + const { openai_file_ids: openaiFileIds } = await Networking.PostToServer('/uploadPDFToVectorStore', { urls, threadID: this.threadID, assistantID: this.assistantID, vector_store_id: this.vectorStoreID }); + + linkedDocs.forEach((doc, i) => { + doc[this.Document[Id] + '_ai_field_id'] = openaiFileIds[i]; + console.log('AI Field ID: ' + openaiFileIds[i]); + }); + + if (csvUrls.length > 0) { + for (let i = 0; i < csvUrls.length; i++) { + this.linkedCsvIDs.push(openaiFileIds[urls.indexOf(csvUrls[i])]); + } + console.log('linked csvs:' + this.linkedCsvIDs); + await this.openai.beta.assistants.update(this.assistantID, { + tools: [{ type: 'file_search' }, { type: 'code_interpreter' }], + tool_resources: { + file_search: { + vector_store_ids: [this.vectorStoreID], + }, + code_interpreter: { + file_ids: this.linkedCsvIDs, + }, + }, + }); + } + }; + + downloadToComputer = (url: string, fileName: string) => { + fetch(url, { method: 'get', mode: 'no-cors', referrerPolicy: 'no-referrer' }) + .then(res => res.blob()) + .then(res => { + const aElement = document.createElement('a'); + aElement.setAttribute('download', fileName); + const href = URL.createObjectURL(res); + aElement.href = href; + aElement.setAttribute('target', '_blank'); + aElement.click(); + URL.revokeObjectURL(href); + }); + }; + + createDocumentInDash = async (url: string) => { + const fileSuffix = url.substring(url.lastIndexOf('.') + 1); + console.log(fileSuffix); + let doc: Doc | null = null; + switch (fileSuffix) { + case 'pdf': + doc = DocCast(await DocUtils.DocumentFromType('pdf', url, {})); + break; + case 'csv': + doc = DocCast(await DocUtils.DocumentFromType('csv', url, {})); + break; + case 'png': + case 'jpg': + case 'jpeg': + doc = DocCast(await DocUtils.DocumentFromType('image', url, {})); + break; + default: + console.error('Unsupported file type:', fileSuffix); + break; + } + if (doc) { + doc && this._props.addDocument?.(doc); + await DocumentManager.Instance.showDocument(doc, { willZoomCentered: true }, () => {}); + } + }; + + downloadFile = async (fileInfo: string, downloadType: DOWNLOAD_TYPE) => { + try { + console.log(fileInfo); + const [fileId, fileName] = fileInfo.split(/!!!/); + const { file_path: filePath } = await Networking.PostToServer('/downloadFileFromOpenAI', { file_id: fileId, file_name: fileName }); + const fileLink = CsvCast(new CsvField(filePath)).url.href; + if (downloadType === DOWNLOAD_TYPE.DASH) { + this.createDocumentInDash(fileLink); + } else { + this.downloadToComputer(fileLink, fileName); + } + } catch (error) { + console.error('Error downloading file:', error); + } + }; + + handleDownloadToDevice = () => { + this.downloadFile(this.currentFile.url, DOWNLOAD_TYPE.DEVICE); + this.modalStatus = false; // Close the modal after the action + this.currentFile = { url: '' }; // Reset the current file + }; + + handleAddToDash = () => { + // Assuming `downloadFile` is a method that handles adding to Dash + this.downloadFile(this.currentFile.url, DOWNLOAD_TYPE.DASH); + this.modalStatus = false; // Close the modal after the action + this.currentFile = { url: '' }; // Reset the current file + }; + + renderModal = () => { + if (!this.modalStatus) return null; + + return ( + <div className="modal"> + <div className="modal-content"> + <h4>File Actions</h4> + <p>Choose an action for the file:</p> + <button type="button" onClick={this.handleDownloadToDevice}> + Download to Device + </button> + <button type="button" onClick={this.handleAddToDash}> + Add to Dash + </button> + <button + type="button" + onClick={() => { + this.modalStatus = false; + }}> + Cancel + </button> + </div> + </div> + ); + }; + @action + showModal = () => { + this.modalStatus = true; + }; + + @action + setCurrentFile = (file: { url: string }) => { + this.currentFile = file; + }; + + componentDidMount() { + this._props.setContentViewBox?.(this); + if (this.dataDoc.data) { + try { + const storedHistory = JSON.parse(StrCast(this.dataDoc.data)); + runInAction(() => { + this.history = storedHistory.map((msg: AssistantMessage) => ({ + role: msg.role, + text: msg.text, + quote: msg.quote, + tool_logs: msg.tool_logs, + image: msg.image, + })); + }); + } catch (e) { + console.error('Failed to parse history from dataDoc:', e); + } + } + reaction( + () => { + const linkedDocs = LinkManager.Instance.getAllRelatedLinks(this.Document) + .map(d => DocCast(LinkManager.getOppositeAnchor(d, this.Document))) + .map(d => DocCast(d?.annotationOn, d)) + .filter(d => d); + return linkedDocs; + }, + + linked => this.linked_docs_to_add.push(...linked.filter(linkedDoc => !this.linked_docs_to_add.includes(linkedDoc))) + ); + + observe( + // right now this skips during initialization which is necessary because it would be blank + // However, it will upload the same link twice when it is + this.linked_docs_to_add, + change => { + // observe pushes/splices on a user link DB 'data' field (should only happen for local changes) + switch (change.type as any) { + case 'splice': + if ((change as any).addedCount > 0) { + // maybe check here if its already in the urls datadoc array so doesn't add twice + console.log((change as any).added as Doc[]); + this.uploadLinks((change as any).added as Doc[]); + } + // (change as any).removed.forEach((link: any) => remLinkFromDoc(toRealField(link))); + break; + case 'update': // let oldValue = change.oldValue; + default: + } + }, + true + ); + } + + render() { + return ( + <MathJaxContext config={this.mathJaxConfig}> + <div className="chatBox"> + {this.isInitializing && <div className="initializing-overlay">Initializing...</div>} + {this.renderModal()} + <div + className="scroll-box chat-content" + ref={r => { + this._oldWheel?.removeEventListener('wheel', this.onPassiveWheel); + this._oldWheel = r; + r?.addEventListener('wheel', this.onPassiveWheel, { passive: false }); + }}> + <div className="messages"> + {this.history.map((message, index) => ( + <MessageComponent + key={index} + message={message} + toggleToolLogs={this.toggleToolLogs} + expandedLogIndex={this.expandedLogIndex} + index={index} + showModal={this.showModal} + goToLinkedDoc={this.goToLinkedDoc} + setCurrentFile={this.setCurrentFile} + /> + ))} + {!this.current_message ? null : ( + <MessageComponent + key={this.history.length} + message={this.current_message} + toggleToolLogs={this.toggleToolLogs} + expandedLogIndex={this.expandedLogIndex} + index={this.history.length} + showModal={this.showModal} + goToLinkedDoc={this.goToLinkedDoc} + setCurrentFile={this.setCurrentFile} + isCurrent + /> + )} + </div> + </div> + <form onSubmit={this.askGPT} className="chat-form"> + <input type="text" name="messageInput" autoComplete="off" placeholder="Type a message..." /> + <button type="submit">Send</button> + </form> + </div> + </MathJaxContext> + ); + } +} + +Docs.Prototypes.TemplateMap.set(DocumentType.CHAT, { + layout: { view: ChatBox, dataField: 'data' }, + options: { acl: '', chat: '', chat_history: '', chat_thread_id: '', chat_assistant_id: '', chat_vector_store_id: '' }, +}); diff --git a/src/client/views/nodes/ChatBox/MessageComponent.tsx b/src/client/views/nodes/ChatBox/MessageComponent.tsx new file mode 100644 index 000000000..fced0b4d5 --- /dev/null +++ b/src/client/views/nodes/ChatBox/MessageComponent.tsx @@ -0,0 +1,116 @@ +/* eslint-disable react/require-default-props */ +import React from 'react'; +import { observer } from 'mobx-react'; +import { MathJax, MathJaxContext } from 'better-react-mathjax'; +import ReactMarkdown from 'react-markdown'; +import { TbCircle0Filled, TbCircle1Filled, TbCircle2Filled, TbCircle3Filled, TbCircle4Filled, TbCircle5Filled, TbCircle6Filled, TbCircle7Filled, TbCircle8Filled, TbCircle9Filled } from 'react-icons/tb'; +import { AssistantMessage } from './types'; + +interface MessageComponentProps { + message: AssistantMessage; + toggleToolLogs: (index: number) => void; + expandedLogIndex: number | null; + index: number; + showModal: () => void; + goToLinkedDoc: (url: string) => void; + setCurrentFile: (file: { url: string }) => void; + isCurrent?: boolean; +} + +const MessageComponent: React.FC<MessageComponentProps> = function ({ message, toggleToolLogs, expandedLogIndex, goToLinkedDoc, index, showModal, setCurrentFile, isCurrent = false }) { + // const messageClass = `${message.role} ${isCurrent ? 'current-message' : ''}`; + + const LinkRenderer = ({ href, children }: { href: string; children: React.ReactNode }) => { + // console.log(href + " " + children) + const regex = /([a-zA-Z0-9_.!-]+)~~~(citation|file_path)/; + const matches = href.match(regex); + // console.log(href) + // console.log(matches) + const url = matches ? matches[1] : href; + const linkType = matches ? matches[2] : null; + if (linkType === 'citation') { + switch (children) { + case '0': + children = <TbCircle0Filled />; + break; + case '1': + children = <TbCircle1Filled />; + break; + case '2': + children = <TbCircle2Filled />; + break; + case '3': + children = <TbCircle3Filled />; + break; + case '4': + children = <TbCircle4Filled />; + break; + case '5': + children = <TbCircle5Filled />; + break; + case '6': + children = <TbCircle6Filled />; + break; + case '7': + children = <TbCircle7Filled />; + break; + case '8': + children = <TbCircle8Filled />; + break; + case '9': + children = <TbCircle9Filled />; + break; + default: + break; + } + } + // console.log(linkType) + const style = { + color: 'lightblue', + verticalAlign: linkType === 'citation' ? 'super' : 'baseline', + fontSize: linkType === 'citation' ? 'smaller' : 'inherit', + }; + + return ( + <a + href="#" + onClick={e => { + e.preventDefault(); + if (linkType === 'citation') { + goToLinkedDoc(url); + } else if (linkType === 'file_path') { + showModal(); + setCurrentFile({ url }); + } + }} + style={style}> + {children} + </a> + ); + }; + + return ( + <div className={`message ${message.role}`}> + <MathJaxContext> + <MathJax dynamic hideUntilTypeset="every"> + <ReactMarkdown components={{ a: LinkRenderer }}>{message.text ? message.text : ''}</ReactMarkdown> + </MathJax> + </MathJaxContext> + {message.image && <img src={message.image} alt="" />} + <div className="message-footer"> + {message.tool_logs && ( + <button className="toggle-logs-button" onClick={() => toggleToolLogs(index)}> + {expandedLogIndex === index ? 'Hide Code Interpreter Logs' : 'Show Code Interpreter Logs'} + </button> + )} + {expandedLogIndex === index && ( + <div className="tool-logs"> + <pre>{message.tool_logs}</pre> + </div> + )} + </div> + </div> + ); +}; + +export default observer(MessageComponent); diff --git a/src/client/views/nodes/ChatBox/types.ts b/src/client/views/nodes/ChatBox/types.ts new file mode 100644 index 000000000..8212a7050 --- /dev/null +++ b/src/client/views/nodes/ChatBox/types.ts @@ -0,0 +1,23 @@ +export enum ASSISTANT_ROLE { + USER = 'User', + ASSISTANT = 'Assistant', +} + +export enum ANNOTATION_LINK_TYPE { + DASH_DOC = 'citation', + DOWNLOAD_FILE = 'file_path', +} + +export enum DOWNLOAD_TYPE { + DASH = 'dash', + DEVICE = 'device', +} + +export interface AssistantMessage { + role: ASSISTANT_ROLE; + text: string; + quote?: string; + image?: string; + tool_logs?: string; + links?: { start: number; end: number; url: string; id?: string; link_type: ANNOTATION_LINK_TYPE }[]; +} diff --git a/src/client/views/nodes/DiagramBox.scss b/src/client/views/nodes/DiagramBox.scss new file mode 100644 index 000000000..d2749f1ad --- /dev/null +++ b/src/client/views/nodes/DiagramBox.scss @@ -0,0 +1,88 @@ +.DIYNodeBox { + width: 100%; + height: 100%; + display: flex; + flex-direction: column; + align-items: center; + justify-content: center; + + .DIYNodeBox-wrapper { + width: 100%; + height: 100%; + display: flex; + flex-direction: column; + align-items: center; + justify-content: center; + .DIYNodeBox { + /* existing code */ + + .DIYNodeBox-iframe { + height: 100%; + width: 100%; + border: none; + + } + } + + .search-bar { + display: flex; + justify-content: center; + align-items: center; + width: 100%; + padding: 10px; + + input[type="text"] { + flex: 1; + margin-right: 10px; + } + + button { + padding: 5px 10px; + } + } + + .content { + flex: 1; + display: flex; + justify-content: center; + align-items: center; + width:100%; + height:100%; + .diagramBox{ + flex: 1; + display: flex; + justify-content: center; + align-items: center; + width:100%; + height:100%; + svg{ + flex: 1; + display: flex; + justify-content: center; + align-items: center; + width:100%; + height:100%; + } + } + } + + .loading-circle { + position: relative; + width: 50px; + height: 50px; + border-radius: 50%; + border: 3px solid #ccc; + border-top-color: #333; + animation: spin 1s infinite linear; + } + + @keyframes spin { + 0% { + transform: rotate(0deg); + } + 100% { + transform: rotate(360deg); + } + } + } +}
\ No newline at end of file diff --git a/src/client/views/nodes/DiagramBox.tsx b/src/client/views/nodes/DiagramBox.tsx new file mode 100644 index 000000000..fa7e5868a --- /dev/null +++ b/src/client/views/nodes/DiagramBox.tsx @@ -0,0 +1,305 @@ +import { makeObservable, observable, action, reaction } from 'mobx'; +import { observer } from 'mobx-react'; +import * as React from 'react'; +import { ViewBoxAnnotatableComponent, ViewBoxInterface } from '../DocComponent'; +import { StyleProp } from '../StyleProvider'; +import './DiagramBox.scss'; +import { FieldView, FieldViewProps } from './FieldView'; +import { PinProps, PresBox } from './trails'; +import mermaid from 'mermaid'; +import { Doc, DocListCast } from '../../../fields/Doc'; +import { List } from '../../../fields/List'; +import { RichTextField } from '../../../fields/RichTextField'; +import { ContextMenu } from '../ContextMenu'; +import { gptAPICall, GPTCallType } from '../../apis/gpt/GPT'; +import { ChatCompletionMessageParam } from 'openai/resources/chat/completions'; +import OpenAI, { ClientOptions } from 'openai'; +import { line } from 'd3'; +import { InkingStroke } from '../InkingStroke'; +import { DocumentManager } from '../../util/DocumentManager'; +import { C } from '@fullcalendar/core/internal-common'; +import { Docs } from '../../documents/Documents'; +import { NumCast } from '../../../fields/Types'; +import { LinkManager } from '../../util/LinkManager'; +import { CsvCast, DocCast, StrCast } from '../../../fields/Types'; +import { DocumentType } from '../../documents/DocumentTypes'; + +@observer +export class DiagramBox extends ViewBoxAnnotatableComponent<FieldViewProps>() implements ViewBoxInterface { + public static LayoutString(fieldKey: string) { + return FieldView.LayoutString(DiagramBox, fieldKey); + } + private _ref: React.RefObject<HTMLDivElement> = React.createRef(); + private _dragRef = React.createRef<HTMLDivElement>(); + constructor(props: FieldViewProps) { + super(props); + makeObservable(this); + } + + @observable inputValue = ''; + @observable loading = false; + @observable errorMessage = ''; + @observable mermaidCode = ''; + + @action handleInputChange = (e: React.ChangeEvent<HTMLInputElement>) => { + this.inputValue = e.target.value; + }; + async componentDidMount() { + this._props.setContentViewBox?.(this); + mermaid.initialize({ + securityLevel: 'loose', + startOnLoad: true, + flowchart: { useMaxWidth: true, htmlLabels: true, curve: 'cardinal' }, + }); + this.mermaidCode = 'asdasdasd'; + let docArray: Doc[] = DocListCast(this.Document.data); + let mermaidCodeDoc = docArray.filter(doc => doc.type == 'rich text'); + mermaidCodeDoc = mermaidCodeDoc.filter(doc => (doc.text as RichTextField).Text == 'mermaidCodeTitle'); + if (mermaidCodeDoc[0]) { + if (typeof mermaidCodeDoc[0].title == 'string') { + console.log(mermaidCodeDoc[0].title); + if (mermaidCodeDoc[0].title != '') { + this.renderMermaidAsync(mermaidCodeDoc[0].title); + } + } + } + //this will create a text doc far away where the user cant to save the mermaid code, where it will then be accessed when flipped to the diagram box side + //the code is stored in the title since it is much easier to change than in the text + else { + DocumentManager.Instance.AddViewRenderedCb(this.Document, docViewForYourCollection => { + if (docViewForYourCollection && docViewForYourCollection.ComponentView) { + if (docViewForYourCollection.ComponentView.addDocument && docViewForYourCollection.ComponentView.removeDocument) { + let newDoc = Docs.Create.TextDocument('mermaidCodeTitle', { title: '', x: 9999 + NumCast(this.layoutDoc._width), y: 9999 }); + docViewForYourCollection.ComponentView?.addDocument(newDoc); + } + } + }); + } + console.log(this.Document.title); + //this is so that ever time a new doc, text node or ink node, is created, this.createMermaidCode will run which will create a save + reaction( + () => DocListCast(this.Document.data), + docs => { + console.log('reaction happened'); + this.convertDrawingToMermaidCode(); + }, + { fireImmediately: true } + ); + } + renderMermaid = async (str: string) => { + try { + const { svg, bindFunctions } = await this.mermaidDiagram(str); + return { svg, bindFunctions }; + } catch (error) { + console.error('Error rendering mermaid diagram:', error); + return { svg: '', bindFunctions: undefined }; + } + }; + mermaidDiagram = async (str: string) => { + return await mermaid.render('graph' + Date.now(), str); + }; + + async renderMermaidAsync(mermaidCode: string) { + try { + const { svg, bindFunctions } = await this.renderMermaid(mermaidCode); + const dashDiv = document.getElementById('dashDiv' + this.Document.title); + if (dashDiv) { + dashDiv.innerHTML = svg; + if (bindFunctions) { + bindFunctions(dashDiv); + } + } + } catch (error) { + console.error('Error rendering Mermaid:', error); + } + } + @action handleRenderClick = () => { + this.generateMermaidCode(); + }; + @action async generateMermaidCode() { + console.log('Generating Mermaid Code'); + this.loading = true; + let prompt = ''; + // let docArray: Doc[] = DocListCast(this.Document.data); + // let mermaidCodeDoc = docArray.filter(doc => doc.type == 'rich text') + // mermaidCodeDoc=mermaidCodeDoc.filter(doc=>(doc.text as RichTextField).Text=='mermaidCodeTitle') + // if(mermaidCodeDoc[0]){ + // console.log(mermaidCodeDoc[0].title) + // if(typeof mermaidCodeDoc[0].title=='string'){ + // console.log(mermaidCodeDoc[0].title) + // if(mermaidCodeDoc[0].title!=""){ + // prompt="Edit this code "+this.inputValue+": "+mermaidCodeDoc[0].title + // console.log("you have to see me") + // } + // } + // } + // else{ + prompt = 'Write this in mermaid code and only give me the mermaid code: ' + this.inputValue; + console.log('there is no text save'); + //} + let res = await gptAPICall(prompt, GPTCallType.MERMAID); + this.loading = false; + if (res == 'Error connecting with API.') { + // If GPT call failed + console.error('GPT call failed'); + this.errorMessage = 'GPT call failed; please try again.'; + } else if (res != null) { + // If GPT call succeeded, set htmlCode;;; TODO: check if valid html + if (this.isValidCode(res)) { + this.mermaidCode = res; + console.log('GPT call succeeded:' + res); + this.errorMessage = ''; + } else { + console.error('GPT call succeeded but invalid html; please try again.'); + this.errorMessage = 'GPT call succeeded but invalid html; please try again.'; + } + } + this.renderMermaidAsync.call(this, this.removeWords(this.mermaidCode)); + this.loading = false; + } + isValidCode = (html: string) => { + return true; + }; + removeWords(inputStr: string) { + inputStr = inputStr.replace('```mermaid', ''); + return inputStr.replace('```', ''); + } + //method to convert the drawings on collection node side the mermaid code + async convertDrawingToMermaidCode() { + let mermaidCode = ''; + let diagramExists = false; + if (this.Document.data instanceof List) { + let docArray: Doc[] = DocListCast(this.Document.data); + let rectangleArray = docArray.filter(doc => doc.title == 'rectangle' || doc.title == 'circle'); + let lineArray = docArray.filter(doc => doc.title == 'line' || doc.title == 'stroke'); + let textArray = docArray.filter(doc => doc.type == 'rich text'); + const timeoutPromise = () => + new Promise(resolve => { + setTimeout(resolve, 0); + }); + await timeoutPromise(); + let inkStrokeArray = lineArray.map(doc => DocumentManager.Instance.getDocumentView(doc, this.DocumentView?.())).filter(inkView => inkView?.ComponentView instanceof InkingStroke); + console.log(inkStrokeArray.length); + console.log(lineArray.length); + if (inkStrokeArray[0] && inkStrokeArray.length == lineArray.length) { + mermaidCode = 'graph TD;'; + let inkingStrokeArray = inkStrokeArray.map(stroke => stroke?.ComponentView); + for (let i = 0; i < rectangleArray.length; i++) { + const rectangle = rectangleArray[i]; + for (let j = 0; j < lineArray.length; j++) { + let inkScaleX = (inkingStrokeArray[j] as InkingStroke)?.inkScaledData().inkScaleX; + let inkScaleY = (inkingStrokeArray[j] as InkingStroke)?.inkScaledData().inkScaleY; + let inkStrokeXArray = (inkingStrokeArray[j] as InkingStroke) + ?.inkScaledData() + .inkData.map(coord => coord.X) + .map(doc => doc * inkScaleX); + let inkStrokeYArray = (inkingStrokeArray[j] as InkingStroke) + ?.inkScaledData() + .inkData.map(coord => coord.Y) + .map(doc => doc * inkScaleY); + console.log(inkingStrokeArray.length); + console.log(lineArray.length); + //need to minX and minY to since the inkStroke.x and.y is not relative to the doc. so I have to do some calcluations + let minX: number = Math.min(...inkStrokeXArray); + let minY: number = Math.min(...inkStrokeYArray); + let startX = inkStrokeXArray[0] - minX + (lineArray[j]?.x as number); + let startY = inkStrokeYArray[0] - minY + (lineArray[j]?.y as number); + let endX = inkStrokeXArray[inkStrokeXArray.length - 1] - minX + (lineArray[j].x as number); + let endY = inkStrokeYArray[inkStrokeYArray.length - 1] - minY + (lineArray[j].y as number); + if (this.isPointInBox(rectangle, [startX, startY])) { + for (let k = 0; k < rectangleArray.length; k++) { + const rectangle2 = rectangleArray[k]; + if (this.isPointInBox(rectangle2, [endX, endY]) && typeof rectangle.x === 'number' && typeof rectangle2.x === 'number') { + diagramExists = true; + const linkedDocs: Doc[] = LinkManager.Instance.getAllRelatedLinks(lineArray[j]).map(d => DocCast(LinkManager.getOppositeAnchor(d, lineArray[j]))); + console.log(linkedDocs.length); + if (linkedDocs.length != 0) { + let linkedText = (linkedDocs[0].text as RichTextField).Text; + mermaidCode += Math.abs(rectangle.x) + this.getTextInBox(rectangle, textArray) + '-->|' + linkedText + '|' + Math.abs(rectangle2.x) + this.getTextInBox(rectangle2, textArray) + ';'; + } else { + mermaidCode += Math.abs(rectangle.x) + this.getTextInBox(rectangle, textArray) + '-->' + Math.abs(rectangle2.x) + this.getTextInBox(rectangle2, textArray) + ';'; + } + } + } + } + } + } + //this will save the text + DocumentManager.Instance.AddViewRenderedCb(this.Document, docViewForYourCollection => { + if (docViewForYourCollection && docViewForYourCollection.ComponentView) { + if (docViewForYourCollection.ComponentView.addDocument && docViewForYourCollection.ComponentView.removeDocument) { + let docArray: Doc[] = DocListCast(this.Document.data); + docArray = docArray.filter(doc => doc.type == 'rich text'); + let mermaidCodeDoc = docArray.filter(doc => (doc.text as RichTextField).Text == 'mermaidCodeTitle'); + if (mermaidCodeDoc[0]) { + if (diagramExists) { + mermaidCodeDoc[0].title = mermaidCode; + } else { + mermaidCodeDoc[0].title = ''; + } + } + } + } + }); + } + } + } + testInkingStroke = () => { + if (this.Document.data instanceof List) { + let docArray: Doc[] = DocListCast(this.Document.data); + let lineArray = docArray.filter(doc => doc.title == 'line' || doc.title == 'stroke'); + setTimeout(() => { + let inkStrokeArray = lineArray.map(doc => DocumentManager.Instance.getDocumentView(doc, this.DocumentView?.())).filter(inkView => inkView?.ComponentView instanceof InkingStroke); + console.log(inkStrokeArray); + }); + } + }; + getTextInBox = (box: Doc, richTextArray: Doc[]): string => { + for (let i = 0; i < richTextArray.length; i++) { + let textDoc = richTextArray[i]; + if (typeof textDoc.x === 'number' && typeof textDoc.y === 'number' && typeof box.x === 'number' && typeof box.height === 'number' && typeof box.width === 'number' && typeof box.y === 'number') { + if (textDoc.x > box.x && textDoc.x < box.x + box.width && textDoc.y > box.y && textDoc.y < box.y + box.height) { + if (box.title == 'rectangle') { + return '(' + (textDoc.text as RichTextField)?.Text + ')'; + } + if (box.title == 'circle') { + return '((' + (textDoc.text as RichTextField)?.Text + '))'; + } + } + } + } + return '( )'; + }; + isPointInBox = (box: Doc, line: number[]): boolean => { + if (typeof line[0] === 'number' && typeof box.x === 'number' && typeof box.width === 'number' && typeof box.height === 'number' && typeof box.y === 'number' && typeof line[1] === 'number') { + return line[0] < box.x + box.width && line[0] > box.x && line[1] > box.y && line[1] < box.y + box.height; + } else { + return false; + } + }; + + render() { + return ( + <div ref={this._ref} className="DIYNodeBox"> + <div ref={this._dragRef} className="DIYNodeBox-wrapper"> + <div className="search-bar"> + <input type="text" value={this.inputValue} onChange={this.handleInputChange} /> + <button onClick={this.handleRenderClick}>Generate</button> + </div> + <div className="content"> + {this.mermaidCode ? ( + <div id={'dashDiv' + this.Document.title} className="diagramBox"></div> + ) : ( + <div>{this.loading ? <div className="loading-circle"></div> : <div>{this.errorMessage ? this.errorMessage : 'Insert prompt to generate diagram'}</div>}</div> + )} + </div> + </div> + </div> + ); + } +} + +Docs.Prototypes.TemplateMap.set(DocumentType.DIAGRAM, { + layout: { view: DiagramBox, dataField: 'dadta' }, + options: { _height: 300, _layout_fitWidth: true, _layout_nativeDimEditable: true, _layout_reflowVertical: true, waitForDoubleClickToClick: 'always', systemIcon: 'BsGlobe' }, +}); diff --git a/src/client/views/nodes/DocumentContentsView.tsx b/src/client/views/nodes/DocumentContentsView.tsx index 18529a429..192c7875e 100644 --- a/src/client/views/nodes/DocumentContentsView.tsx +++ b/src/client/views/nodes/DocumentContentsView.tsx @@ -79,7 +79,7 @@ export class DocumentContentsView extends ObservableReactComponent<DocumentConte * Set of all available rendering componets for Docs (e.g., ImageBox, CollectionFreeFormView, etc) */ private static Components: { [key: string]: any }; - public static Init(defaultLayoutString: string, components:{ [key: string]: any}) { + public static Init(defaultLayoutString: string, components: { [key: string]: any }) { DocumentContentsView.DefaultLayoutString = defaultLayoutString; DocumentContentsView.Components = components; } |
