From 7ff649c6582b7e00dcf2d7fe607699f850310ba0 Mon Sep 17 00:00:00 2001 From: "A.J. Shulman" Date: Thu, 30 May 2024 13:58:40 -0400 Subject: fixing chatbox features still messages appear weird when using code interpreter and now doesn't detect file links --- .../views/nodes/ChatBox/MessageComponent.tsx | 37 ++-------------------- 1 file changed, 2 insertions(+), 35 deletions(-) (limited to 'src/client/views/nodes/ChatBox/MessageComponent.tsx') diff --git a/src/client/views/nodes/ChatBox/MessageComponent.tsx b/src/client/views/nodes/ChatBox/MessageComponent.tsx index fced0b4d5..ef6ce83b5 100644 --- a/src/client/views/nodes/ChatBox/MessageComponent.tsx +++ b/src/client/views/nodes/ChatBox/MessageComponent.tsx @@ -3,8 +3,8 @@ import React from 'react'; import { observer } from 'mobx-react'; import { MathJax, MathJaxContext } from 'better-react-mathjax'; import ReactMarkdown from 'react-markdown'; -import { TbCircle0Filled, TbCircle1Filled, TbCircle2Filled, TbCircle3Filled, TbCircle4Filled, TbCircle5Filled, TbCircle6Filled, TbCircle7Filled, TbCircle8Filled, TbCircle9Filled } from 'react-icons/tb'; import { AssistantMessage } from './types'; +import { TbInfoCircleFilled } from 'react-icons/tb'; interface MessageComponentProps { message: AssistantMessage; @@ -29,40 +29,7 @@ const MessageComponent: React.FC = function ({ message, t const url = matches ? matches[1] : href; const linkType = matches ? matches[2] : null; if (linkType === 'citation') { - switch (children) { - case '0': - children = ; - break; - case '1': - children = ; - break; - case '2': - children = ; - break; - case '3': - children = ; - break; - case '4': - children = ; - break; - case '5': - children = ; - break; - case '6': - children = ; - break; - case '7': - children = ; - break; - case '8': - children = ; - break; - case '9': - children = ; - break; - default: - break; - } + children = ; } // console.log(linkType) const style = { -- cgit v1.2.3-70-g09d2 From e297c75cdcc8bb5b1b138d1272f1f6f27b222f4c Mon Sep 17 00:00:00 2001 From: "A.J. Shulman" Date: Thu, 27 Jun 2024 16:36:40 -0400 Subject: fixing to work with python API also added follow up questions --- package-lock.json | 2 +- package.json | 2 +- src/client/views/nodes/ChatBox/ChatBox.scss | 77 +++-- src/client/views/nodes/ChatBox/ChatBox.tsx | 351 ++++++--------------- .../views/nodes/ChatBox/MessageComponent.tsx | 53 +++- src/server/ApiManagers/AssistantManager.ts | 33 ++ 6 files changed, 218 insertions(+), 300 deletions(-) (limited to 'src/client/views/nodes/ChatBox/MessageComponent.tsx') diff --git a/package-lock.json b/package-lock.json index b143dc5c1..9d58ead46 100644 --- a/package-lock.json +++ b/package-lock.json @@ -56,7 +56,7 @@ "adm-zip": "^0.5.10", "archiver": "^7.0.1", "async": "^3.2.5", - "axios": "^1.6.2", + "axios": "^1.7.2", "babel": "^6.23.0", "babel-loader": "^9.1.3", "bcrypt-nodejs": "0.0.3", diff --git a/package.json b/package.json index 52f627b63..49c0e4d91 100644 --- a/package.json +++ b/package.json @@ -141,7 +141,7 @@ "adm-zip": "^0.5.10", "archiver": "^7.0.1", "async": "^3.2.5", - "axios": "^1.6.2", + "axios": "^1.7.2", "babel": "^6.23.0", "babel-loader": "^9.1.3", "bcrypt-nodejs": "0.0.3", diff --git a/src/client/views/nodes/ChatBox/ChatBox.scss b/src/client/views/nodes/ChatBox/ChatBox.scss index a08b98f50..f1e3d3d67 100644 --- a/src/client/views/nodes/ChatBox/ChatBox.scss +++ b/src/client/views/nodes/ChatBox/ChatBox.scss @@ -9,12 +9,10 @@ $border-radius: 8px; .chatBox { display: flex; flex-direction: column; - width: 100%; /* Adjust the width as needed, could be in percentage */ - height: 100%; /* Adjust the height as needed, could be in percentage */ + width: 100%; + height: 100%; background-color: $background-color; font-family: 'Helvetica Neue', Arial, sans-serif; - //margin: 20px auto; - //overflow: hidden; .scroll-box { flex-grow: 1; @@ -24,6 +22,7 @@ $border-radius: 8px; padding: 10px; display: flex; flex-direction: column-reverse; + padding-bottom: 0; &::-webkit-scrollbar { width: 8px; @@ -47,28 +46,24 @@ $border-radius: 8px; border-radius: $border-radius; background-color: lighten($background-color, 5%); box-shadow: 0 2px 5px $shadow-color; - //display: flex; - align-items: center; - max-width: 70%; + align-items: flex-start; + max-width: 90%; + width: 100%; word-break: break-word; + .message-footer { - // Assuming this is the container for the toggle button - //max-width: 70%; + width: 100%; .toggle-logs-button { - margin-top: 10px; // Padding on sides to align with the text above + margin-top: 10px; width: 95%; - //display: block; // Ensures the button extends the full width of its container - text-align: center; // Centers the text inside the button - //padding: 8px 0; // Adequate padding for touch targets + text-align: center; background-color: $button-color; color: #fff; border: none; border-radius: $border-radius; cursor: pointer; - //transition: background-color 0.3s; - //margin-top: 10px; // Adds space above the button - box-shadow: 0 2px 4px $shadow-color; // Consistent shadow with other elements + box-shadow: 0 2px 4px $shadow-color; &:hover { background-color: $button-hover-color; } @@ -78,13 +73,9 @@ $border-radius: 8px; background-color: $input-background; color: $text-color; margin-top: 5px; - //padding: 10px; - //border-radius: $border-radius; - //box-shadow: inset 0 2px 4px $shadow-color; - //transition: opacity 1s ease-in-out; font-family: monospace; overflow-x: auto; - max-height: 150px; // Ensuring it does not grow too large + max-height: 150px; overflow-y: auto; } } @@ -118,18 +109,17 @@ $border-radius: 8px; } } } - padding-bottom: 0; } .chat-form { display: flex; - flex-grow: 1; - //height: 50px; + flex-grow: 0; bottom: 0; width: 100%; padding: 10px; background-color: $input-background; box-shadow: inset 0 -1px 2px $shadow-color; + margin-bottom: 0; input[type='text'] { flex-grow: 1; @@ -147,12 +137,12 @@ $border-radius: 8px; border-radius: $border-radius; cursor: pointer; transition: background-color 0.3s; + min-width: 80px; &:hover { background-color: $button-hover-color; } } - margin-bottom: 0; } } @@ -168,7 +158,7 @@ $border-radius: 8px; align-items: center; font-size: 1.5em; color: $text-color; - z-index: 10; // Ensure it's above all other content (may be better solution) + z-index: 10; &::before { content: 'Initializing...'; @@ -214,7 +204,6 @@ $border-radius: 8px; border: none; border-radius: $border-radius; cursor: pointer; - //margin: 5px; transition: background-color 0.3s; &:hover { @@ -223,3 +212,37 @@ $border-radius: 8px; } } } + +.follow-up-questions { + margin-top: 10px; + width: 100%; + + h4 { + margin-bottom: 5px; + font-size: 14px; + } + + .follow-up-button { + background-color: #f0f0f0; + border: 1px solid #ddd; + border-radius: 8px; + padding: 8px 10px; + margin: 4px 0; + cursor: pointer; + transition: background-color 0.3s; + display: block; + width: 100%; + text-align: left; + white-space: normal; + word-wrap: break-word; + font-size: 12px; + color: $text-color; + min-height: 40px; + height: auto; // Allow the button to expand as needed + line-height: 1.3; + + &:hover { + background-color: #e0e0e0; + } + } +} diff --git a/src/client/views/nodes/ChatBox/ChatBox.tsx b/src/client/views/nodes/ChatBox/ChatBox.tsx index 390f13ce7..b986c7393 100644 --- a/src/client/views/nodes/ChatBox/ChatBox.tsx +++ b/src/client/views/nodes/ChatBox/ChatBox.tsx @@ -21,6 +21,7 @@ import './ChatBox.scss'; import MessageComponent from './MessageComponent'; import { ANNOTATION_LINK_TYPE, ASSISTANT_ROLE, AssistantMessage, DOWNLOAD_TYPE } from './types'; import { Annotation } from 'mobx/dist/internal'; +import { FormEvent } from 'react'; @observer export class ChatBox extends ViewBoxAnnotatableComponent() { @@ -33,7 +34,7 @@ export class ChatBox extends ViewBoxAnnotatableComponent() { @observable isInitializing: boolean = true; @observable expandedLogIndex: number | null = null; @observable linked_docs_to_add: Doc[] = []; - + @observable inputValue: string = ''; private openai: OpenAI; private assistantID: string = ''; private threadID: string = ''; @@ -160,240 +161,66 @@ export class ChatBox extends ViewBoxAnnotatableComponent() { } }; - @action - runAssistant = async (inputText: string) => { - // Ensure an assistant and thread are created - if (!this.assistantID || !this.threadID || !this.vectorStoreID) { - await this.createAssistant(); - console.log('Assistant and thread created:', this.assistantID, this.threadID); - } - - let currentText: string = ''; - let currentToolCallMessage: string = ''; - - // Send the user's input to the assistant - await this.openai.beta.threads.messages.create(this.threadID, { - role: 'user', - content: inputText, - }); - - // Listen to the streaming responses - const stream = this.openai.beta.threads.runs - .stream(this.threadID, { - assistant_id: this.assistantID, - }) - .on('runStepCreated', (runStep: RunStep) => { - runInAction(() => { - this.isLoading = true; - //intentionally don't merge run steps' messages and keep them as seperate messages on the interface - this.current_message = { role: ASSISTANT_ROLE.ASSISTANT, text: '', tool_logs: '' }; - }); - }) - .on('toolCallDelta', (toolCallDelta, snapshot) => { - if (toolCallDelta.type === 'code_interpreter') { - if (toolCallDelta.code_interpreter?.input) { - currentToolCallMessage += toolCallDelta.code_interpreter.input; - - if (this.current_message) { - this.current_message.tool_logs = currentToolCallMessage; - } - } - - if (toolCallDelta.code_interpreter?.outputs) { - currentToolCallMessage += '\n Code interpreter output:'; - toolCallDelta.code_interpreter.outputs.forEach(output => { - if (output.type === 'logs') { - runInAction(() => { - if (this.current_message) { - this.current_message.tool_logs += '\n|' + output.logs; - } - }); - } - }); - } - } - }) - .on('textDelta', (textDelta, snapshot) => { - currentText += textDelta.value; - runInAction(() => { - if (this.current_message) { - this.current_message.text = currentText; - } - }); - }) - .on('messageDone', async message => { - console.log(this.current_message); - const textItem = message.content.find(item => item.type === 'text'); - if (textItem && textItem.type === 'text') { - const { text } = textItem; - console.log(text.value); - try { - if (this.current_message) { - this.current_message.text = text.value; - } - } catch (e) { - console.error('Error parsing JSON response:', e); - } - console.log(this.current_message); - - const { annotations } = text; - console.log('Annotations: ' + annotations); - - await this.createLinks( - annotations.filter( - (annotation => { - const seenAnnotationTexts = new Set(); - return annotation => { - if (seenAnnotationTexts.has(annotation.text)) { - return false; - } else { - seenAnnotationTexts.add(annotation.text); - return true; - } - }; - })() - ) - ); - } - runInAction(() => { - if (this.current_message) { - console.log(this.current_message); - this.history.push({ ...this.current_message }); - this.current_message = undefined; - } - }); - }) - .on('toolCallDone', async toolCall => { - runInAction(() => { - if (this.current_message?.tool_logs) { - this.history.push({ ...this.current_message }); - this.current_message = undefined; - } - }); - }) - .on('imageFileDone', (content: ImageFile, snapshot: Message) => { - console.log('Image file done:', content); - }); - }; - - createLinks = async (annotations: OpenAI.Beta.Threads.Messages.Annotation[]) => { - console.log(this.current_message); - let text = this.current_message?.text; - console.log(text); - await Promise.all( - annotations.map(async annotation => { - const subString = annotation.text; - const textToDisplay = `DASHLINK`; - let fileInfo = ''; - let formattedLink = ''; - const fileName = subString.split('/')[subString.split('/').length - 1]; - - if (annotation.type === 'file_path') { - const { file_path: filePath } = annotation; - if (filePath) { - fileInfo = filePath.file_id + '!!!' + fileName; - formattedLink = `[${textToDisplay}](${fileInfo}~~~${ANNOTATION_LINK_TYPE.DOWNLOAD_FILE})`; - } - } else { - const { file_citation: fileCitation } = annotation; - if (fileCitation) { - const citedFile = await this.openai.files.retrieve(fileCitation.file_id); - formattedLink = `[${textToDisplay}](${citedFile.filename}~~~${ANNOTATION_LINK_TYPE.DASH_DOC})`; - } - } - - console.log(formattedLink); - text = text?.split(subString).join(formattedLink); - console.log(text); - }) - ); - runInAction(() => { - if (this.current_message) this.current_message.text = text || ''; - }); - }; - - @action - goToLinkedDoc = async (link: string) => { - const linkedDocs = LinkManager.Instance.getAllRelatedLinks(this.Document) - .map(d => DocCast(LinkManager.getOppositeAnchor(d, this.Document))) - .map(d => DocCast(d?.annotationOn, d)) - .filter(d => d); - - const linkedDoc = linkedDocs.find(doc => { - const docUrl = CsvCast(doc.data, PDFCast(doc.data)).url.pathname.replace('/files/pdfs/', '').replace('/files/csvs/', ''); - console.log('URL: ' + docUrl + ' Citation URL: ' + link); - return link === docUrl; - }); - - if (linkedDoc) { - await DocumentManager.Instance.showDocument(DocCast(linkedDoc), { willZoomCentered: true }, () => {}); - } - }; - @action askGPT = async (event: React.FormEvent): Promise => { event.preventDefault(); + this.inputValue = ''; const textInput = event.currentTarget.elements.namedItem('messageInput') as HTMLInputElement; const trimmedText = textInput.value.trim(); - if (!this.assistantID || !this.threadID) { - try { - await this.createAssistant(); - } catch (err) { - console.error('Error:', err); - } - } - if (trimmedText) { try { textInput.value = ''; runInAction(() => { this.history.push({ role: ASSISTANT_ROLE.USER, text: trimmedText }); }); - await this.runAssistant(trimmedText); - this.dataDoc.data = this.history.toString(); + const { response } = await Networking.PostToServer('/askAgent', { input: trimmedText }); + runInAction(() => { + this.history.push({ role: ASSISTANT_ROLE.ASSISTANT, text: response }); + }); + this.dataDoc.data = JSON.stringify(this.history); } catch (err) { console.error('Error:', err); } } }; - @action - uploadLinks = async (linkedDocs: Doc[]) => { - if (this.isInitializing) { - console.log('Initialization in progress, upload aborted.'); - return; - } - const urls = linkedDocs.map(doc => CsvCast(doc.data, PDFCast(doc.data)).url.pathname); - const csvUrls = urls.filter(url => url.endsWith('.csv')); - console.log(this.assistantID, this.threadID, urls); - - const { openai_file_ids: openaiFileIds } = await Networking.PostToServer('/uploadPDFToVectorStore', { urls, threadID: this.threadID, assistantID: this.assistantID, vector_store_id: this.vectorStoreID }); - - linkedDocs.forEach((doc, i) => { - doc[this.Document[Id] + '_ai_field_id'] = openaiFileIds[i]; - console.log('AI Field ID: ' + openaiFileIds[i]); - }); - - if (csvUrls.length > 0) { - for (let i = 0; i < csvUrls.length; i++) { - this.linkedCsvIDs.push(openaiFileIds[urls.indexOf(csvUrls[i])]); - } - console.log('linked csvs:' + this.linkedCsvIDs); - await this.openai.beta.assistants.update(this.assistantID, { - tools: [{ type: 'file_search' }, { type: 'code_interpreter' }], - tool_resources: { - file_search: { - vector_store_ids: [this.vectorStoreID], - }, - code_interpreter: { - file_ids: this.linkedCsvIDs, - }, - }, - }); - } - }; + // @action + // uploadLinks = async (linkedDocs: Doc[]) => { + // if (this.isInitializing) { + // console.log('Initialization in progress, upload aborted.'); + // return; + // } + // const urls: string[] = linkedDocs.map(doc => CsvCast(doc.data, PDFCast(doc.data)).url.pathname); + // const csvUrls: string[] = urls.filter(url => url.endsWith('.csv')); + // console.log(this.assistantID, this.threadID, urls); + + // await Networking.PostToServer('/uploadPDFs', { file_path: urls[0] }); + + // // linkedDocs.forEach((doc, i) => { + // // doc[this.Document[Id] + '_ai_field_id'] = openaiFileIds[i]; + // // console.log('AI Field ID: ' + openaiFileIds[i]); + // // }); + + // // if (csvUrls.length > 0) { + // // for (let i = 0; i < csvUrls.length; i++) { + // // this.linkedCsvIDs.push(openaiFileIds[urls.indexOf(csvUrls[i])]); + // // } + // // console.log('linked csvs:' + this.linkedCsvIDs); + // // await this.openai.beta.assistants.update(this.assistantID, { + // // tools: [{ type: 'file_search' }, { type: 'code_interpreter' }], + // // tool_resources: { + // // file_search: { + // // vector_store_ids: [this.vectorStoreID], + // // }, + // // code_interpreter: { + // // file_ids: this.linkedCsvIDs, + // // }, + // // }, + // // }); + // // } + // }; downloadToComputer = (url: string, fileName: string) => { fetch(url, { method: 'get', mode: 'no-cors', referrerPolicy: 'no-referrer' }) @@ -555,53 +382,59 @@ export class ChatBox extends ViewBoxAnnotatableComponent() { ); } + @action + handleFollowUpClick = (question: string) => { + console.log('Follow-up question clicked:', question); + this.inputValue = question; + }; render() { return ( - -
- {this.isInitializing &&
Initializing...
} - {this.renderModal()} -
{ - this._oldWheel?.removeEventListener('wheel', this.onPassiveWheel); - this._oldWheel = r; - r?.addEventListener('wheel', this.onPassiveWheel, { passive: false }); - }}> -
- {this.history.map((message, index) => ( - - ))} - {!this.current_message ? null : ( - - )} -
+ /** **/ +
+ {this.isInitializing &&
Initializing...
} + {this.renderModal()} +
{ + this._oldWheel?.removeEventListener('wheel', this.onPassiveWheel); + this._oldWheel = r; + r?.addEventListener('wheel', this.onPassiveWheel, { passive: false }); + }}> +
+ {this.history.map((message, index) => ( + {}} + setCurrentFile={this.setCurrentFile} + onFollowUpClick={this.handleFollowUpClick} + /> + ))} + {!this.current_message ? null : ( + {}} + setCurrentFile={this.setCurrentFile} + onFollowUpClick={this.handleFollowUpClick} + /> + )}
-
- - -
- +
+ (this.inputValue = e.target.value)} /> + +
+
+ /** **/ ); } } diff --git a/src/client/views/nodes/ChatBox/MessageComponent.tsx b/src/client/views/nodes/ChatBox/MessageComponent.tsx index ef6ce83b5..15c0811fb 100644 --- a/src/client/views/nodes/ChatBox/MessageComponent.tsx +++ b/src/client/views/nodes/ChatBox/MessageComponent.tsx @@ -14,24 +14,29 @@ interface MessageComponentProps { showModal: () => void; goToLinkedDoc: (url: string) => void; setCurrentFile: (file: { url: string }) => void; + onFollowUpClick: (question: string) => void; // New prop isCurrent?: boolean; } -const MessageComponent: React.FC = function ({ message, toggleToolLogs, expandedLogIndex, goToLinkedDoc, index, showModal, setCurrentFile, isCurrent = false }) { - // const messageClass = `${message.role} ${isCurrent ? 'current-message' : ''}`; - +const MessageComponent: React.FC = function ({ + message, + toggleToolLogs, + expandedLogIndex, + goToLinkedDoc, + index, + showModal, + setCurrentFile, + onFollowUpClick, // New prop + isCurrent = false, +}) { const LinkRenderer = ({ href, children }: { href: string; children: React.ReactNode }) => { - // console.log(href + " " + children) const regex = /([a-zA-Z0-9_.!-]+)~~~(citation|file_path)/; const matches = href.match(regex); - // console.log(href) - // console.log(matches) const url = matches ? matches[1] : href; const linkType = matches ? matches[2] : null; if (linkType === 'citation') { children = ; } - // console.log(linkType) const style = { color: 'lightblue', verticalAlign: linkType === 'citation' ? 'super' : 'baseline', @@ -56,14 +61,38 @@ const MessageComponent: React.FC = function ({ message, t ); }; + const parseMessage = (text: string) => { + const answerMatch = text.match(/([\s\S]*?)<\/answer>/); + const followUpMatch = text.match(/([\s\S]*?)<\/follow_up_question>/); + + const answer = answerMatch ? answerMatch[1] : text; + const followUpQuestions = followUpMatch + ? followUpMatch[1] + .split('\n') + .filter(q => q.trim()) + .map(q => q.replace(/^\d+\.\s*/, '').trim()) + : []; + + return { answer, followUpQuestions }; + }; + + const { answer, followUpQuestions } = parseMessage(message.text); + console.log('Parsed answer:', answer); + console.log('Parsed follow-up questions:', followUpQuestions); return (
- - - {message.text ? message.text : ''} - - + {answer} {message.image && } + {followUpQuestions.length > 0 && ( +
+

Follow-up Questions:

+ {followUpQuestions.map((question, idx) => ( + + ))} +
+ )}
{message.tool_logs && (
- /** **/ ); } } diff --git a/src/client/views/nodes/ChatBox/MessageComponent.tsx b/src/client/views/nodes/ChatBox/MessageComponent.tsx index 15c0811fb..1baf6d7d5 100644 --- a/src/client/views/nodes/ChatBox/MessageComponent.tsx +++ b/src/client/views/nodes/ChatBox/MessageComponent.tsx @@ -1,110 +1,73 @@ -/* eslint-disable react/require-default-props */ import React from 'react'; import { observer } from 'mobx-react'; -import { MathJax, MathJaxContext } from 'better-react-mathjax'; -import ReactMarkdown from 'react-markdown'; -import { AssistantMessage } from './types'; +import { AssistantMessage, CHUNK_TYPE, Citation } from './types'; import { TbInfoCircleFilled } from 'react-icons/tb'; interface MessageComponentProps { message: AssistantMessage; - toggleToolLogs: (index: number) => void; - expandedLogIndex: number | null; index: number; - showModal: () => void; - goToLinkedDoc: (url: string) => void; - setCurrentFile: (file: { url: string }) => void; - onFollowUpClick: (question: string) => void; // New prop - isCurrent?: boolean; + onFollowUpClick: (question: string) => void; + onCitationClick: (citation: Citation) => void; + updateMessageCitations: (index: number, citations: Citation[]) => void; } -const MessageComponent: React.FC = function ({ - message, - toggleToolLogs, - expandedLogIndex, - goToLinkedDoc, - index, - showModal, - setCurrentFile, - onFollowUpClick, // New prop - isCurrent = false, -}) { - const LinkRenderer = ({ href, children }: { href: string; children: React.ReactNode }) => { - const regex = /([a-zA-Z0-9_.!-]+)~~~(citation|file_path)/; - const matches = href.match(regex); - const url = matches ? matches[1] : href; - const linkType = matches ? matches[2] : null; - if (linkType === 'citation') { - children = ; - } - const style = { - color: 'lightblue', - verticalAlign: linkType === 'citation' ? 'super' : 'baseline', - fontSize: linkType === 'citation' ? 'smaller' : 'inherit', - }; +const MessageComponent: React.FC = function ({ message, index, onFollowUpClick, onCitationClick, updateMessageCitations }) { + const LinkRenderer = ({ children }: { children: React.ReactNode }) => { + const text = children as string; + const citationRegex = /([^<]*)<\/citation>/g; + const parts = []; + let lastIndex = 0; + let match; + const citations: Citation[] = []; - return ( - { - e.preventDefault(); - if (linkType === 'citation') { - goToLinkedDoc(url); - } else if (linkType === 'file_path') { - showModal(); - setCurrentFile({ url }); - } - }} - style={style}> - {children} - - ); - }; + while ((match = citationRegex.exec(text)) !== null) { + const [fullMatch, chunkId, type, content] = match; + const citation: Citation = { chunk_id: chunkId, type: type as CHUNK_TYPE, text: content }; + citations.push(citation); + + parts.push(text.slice(lastIndex, match.index)); + parts.push( + { + e.preventDefault(); + onCitationClick(citation); + }} + style={{ + color: 'lightblue', + verticalAlign: 'super', + fontSize: 'smaller', + }}> + + + ); + lastIndex = match.index + fullMatch.length; + } - const parseMessage = (text: string) => { - const answerMatch = text.match(/([\s\S]*?)<\/answer>/); - const followUpMatch = text.match(/([\s\S]*?)<\/follow_up_question>/); + parts.push(text.slice(lastIndex)); - const answer = answerMatch ? answerMatch[1] : text; - const followUpQuestions = followUpMatch - ? followUpMatch[1] - .split('\n') - .filter(q => q.trim()) - .map(q => q.replace(/^\d+\.\s*/, '').trim()) - : []; + // Update the message's citations in the ChatBox's history + updateMessageCitations(index, citations); - return { answer, followUpQuestions }; + return <>{parts}; }; - const { answer, followUpQuestions } = parseMessage(message.text); - console.log('Parsed answer:', answer); - console.log('Parsed follow-up questions:', followUpQuestions); return (
- {answer} - {message.image && } - {followUpQuestions.length > 0 && ( +
+ {message.text} +
+ {message.follow_up_questions && message.follow_up_questions.length > 0 && (

Follow-up Questions:

- {followUpQuestions.map((question, idx) => ( + {message.follow_up_questions.map((question, idx) => ( ))}
)} -
- {message.tool_logs && ( - - )} - {expandedLogIndex === index && ( -
- 
{message.tool_logs}
-
- )} -
); }; diff --git a/src/client/views/nodes/ChatBox/tools/BaseTool.ts b/src/client/views/nodes/ChatBox/tools/BaseTool.ts index 3511d9528..903161bd5 100644 --- a/src/client/views/nodes/ChatBox/tools/BaseTool.ts +++ b/src/client/views/nodes/ChatBox/tools/BaseTool.ts @@ -1,6 +1,6 @@ import { Tool } from '../types'; -export abstract class BaseTool implements Tool { +export abstract class BaseTool = Record> implements Tool { constructor( public name: string, public description: string, @@ -9,7 +9,7 @@ export abstract class BaseTool implements Tool { public briefSummary: string ) {} - abstract execute(args: Record): Promise; + abstract execute(args: T): Promise; getActionRule(): Record { return { diff --git a/src/client/views/nodes/ChatBox/tools/CalculateTool.ts b/src/client/views/nodes/ChatBox/tools/CalculateTool.ts index b881d90fa..818332c44 100644 --- a/src/client/views/nodes/ChatBox/tools/CalculateTool.ts +++ b/src/client/views/nodes/ChatBox/tools/CalculateTool.ts @@ -1,6 +1,6 @@ import { BaseTool } from './BaseTool'; -export class CalculateTool extends BaseTool { +export class CalculateTool extends BaseTool<{ expression: string }> { constructor() { super( 'calculate', diff --git a/src/client/views/nodes/ChatBox/tools/RAGTool.ts b/src/client/views/nodes/ChatBox/tools/RAGTool.ts index 84d5430e7..185efa0ba 100644 --- a/src/client/views/nodes/ChatBox/tools/RAGTool.ts +++ b/src/client/views/nodes/ChatBox/tools/RAGTool.ts @@ -1,8 +1,9 @@ import { BaseTool } from './BaseTool'; import { Vectorstore } from '../vectorstore/VectorstoreUpload'; import { Chunk } from '../types'; +import * as fs from 'fs'; -export class RAGTool extends BaseTool { +export class RAGTool extends BaseTool<{ hypothetical_document_chunk: string }> { constructor( private vectorstore: Vectorstore, summaries: string @@ -59,16 +60,26 @@ export class RAGTool extends BaseTool { for (const chunk of relevantChunks) { content.push({ type: 'text', - text: ``, + text: ``, }); - if (chunk.metadata.type === 'image') { - // Implement image loading and base64 encoding here - // For now, we'll just add a placeholder - content.push({ - type: 'image_url', - image_url: { url: chunk.metadata.file_path }, - }); + if (chunk.metadata.type === 'image' || chunk.metadata.type === 'table') { + try { + const imageBuffer = fs.readFileSync(chunk.metadata.file_path); + const base64Image = imageBuffer.toString('base64'); + if (base64Image) { + content.push({ + type: 'image_url', + image_url: { + url: `data:image/jpeg;base64,${base64Image}`, + }, + }); + } else { + console.log(`Failed to encode image for chunk ${chunk.id}`); + } + } catch (error) { + console.error(`Error reading image file for chunk ${chunk.id}:`, error); + } } content.push({ type: 'text', text: `${chunk.metadata.text}\n\n` }); diff --git a/src/client/views/nodes/ChatBox/tools/WikipediaTool.ts b/src/client/views/nodes/ChatBox/tools/WikipediaTool.ts index 0aef58f61..8ef2830d4 100644 --- a/src/client/views/nodes/ChatBox/tools/WikipediaTool.ts +++ b/src/client/views/nodes/ChatBox/tools/WikipediaTool.ts @@ -1,7 +1,7 @@ import { BaseTool } from './BaseTool'; import axios from 'axios'; -export class WikipediaTool extends BaseTool { +export class WikipediaTool extends BaseTool<{ title: string }> { constructor() { super( 'wikipedia', diff --git a/src/client/views/nodes/ChatBox/types.ts b/src/client/views/nodes/ChatBox/types.ts index c60973be3..0270b6256 100644 --- a/src/client/views/nodes/ChatBox/types.ts +++ b/src/client/views/nodes/ChatBox/types.ts @@ -6,6 +6,7 @@ export enum ASSISTANT_ROLE { export enum CHUNK_TYPE { TEXT = 'text', IMAGE = 'image', + TABLE = 'table', } export interface AssistantMessage { @@ -18,9 +19,7 @@ export interface AssistantMessage { export interface Citation { text: string; type: CHUNK_TYPE; - span: [number, number]; chunk_id: string; - direct_text?: string; } export interface Chunk { @@ -46,13 +45,13 @@ export interface AI_Document { type: string; } -export interface Tool { +export interface Tool = Record> { name: string; description: string; parameters: Record; useRules: string; briefSummary: string; - execute: (args: Record) => Promise; + execute: (args: T) => Promise; getActionRule: () => Record; } diff --git a/src/client/views/nodes/ChatBox/vectorstore/VectorstoreUpload.ts b/src/client/views/nodes/ChatBox/vectorstore/VectorstoreUpload.ts index d16e117b6..1f483ad61 100644 --- a/src/client/views/nodes/ChatBox/vectorstore/VectorstoreUpload.ts +++ b/src/client/views/nodes/ChatBox/vectorstore/VectorstoreUpload.ts @@ -2,6 +2,7 @@ import { Pinecone, Index, IndexList, PineconeRecord, RecordMetadata, QueryRespon import { CohereClient } from 'cohere-ai'; import { EmbedResponse } from 'cohere-ai/api'; import dotenv from 'dotenv'; + import { Chunk, AI_Document } from '../types'; dotenv.config(); -- cgit v1.2.3-70-g09d2 From d2c968cb3705b314396c0503b089f8a233a26502 Mon Sep 17 00:00:00 2001 From: "A.J. Shulman" Date: Wed, 10 Jul 2024 17:54:59 -0400 Subject: Working now somewhat --- src/client/views/nodes/ChatBox/Agent.ts | 83 ++++++++++++++-------- src/client/views/nodes/ChatBox/ChatBot.ts | 16 ----- src/client/views/nodes/ChatBox/ChatBox.tsx | 27 +++++-- .../views/nodes/ChatBox/MessageComponent.tsx | 8 +-- src/client/views/nodes/ChatBox/prompts.ts | 5 +- 5 files changed, 77 insertions(+), 62 deletions(-) delete mode 100644 src/client/views/nodes/ChatBox/ChatBot.ts (limited to 'src/client/views/nodes/ChatBox/MessageComponent.tsx') diff --git a/src/client/views/nodes/ChatBox/Agent.ts b/src/client/views/nodes/ChatBox/Agent.ts index 4c2838540..355acb19f 100644 --- a/src/client/views/nodes/ChatBox/Agent.ts +++ b/src/client/views/nodes/ChatBox/Agent.ts @@ -18,7 +18,7 @@ export class Agent { private summaries: string; constructor(private vectorstore: Vectorstore) { - this.client = new OpenAI({ apiKey: process.env.OPENAI_API_KEY }); + this.client = new OpenAI({ apiKey: process.env.OPENAI_KEY, dangerouslyAllowBrowser: true }); this.summaries = this.vectorstore ? this.vectorstore.getSummaries() : 'No documents available.'; this.tools = { wikipedia: new WikipediaTool(), @@ -40,7 +40,7 @@ export class Agent { return history; } - async askAgent(question: string, maxTurns: number = 5): Promise { + async askAgent(question: string, maxTurns: number = 8): Promise { console.log(`Starting query: ${question}`); this.messages.push({ role: 'user', content: question }); const chatHistory = this.formatChatHistory(); @@ -51,6 +51,10 @@ export class Agent { this.interMessages.push({ role: 'assistant', content: `${question}` }); + const parser = new XMLParser(); + const builder = new XMLBuilder(); + let currentAction: string | undefined; + for (let i = 0; i < maxTurns; i++) { console.log(`Turn ${i + 1}/${maxTurns}`); @@ -58,42 +62,58 @@ export class Agent { console.log(`Bot response: ${result}`); this.interMessages.push({ role: 'assistant', content: result }); + let parsedResult; try { - const parser = new XMLParser(); - const parsedResult = parser.parse(result); - const step = parsedResult[`step${i + 1}`]; - - if (step.thought) console.log(`Thought: ${step.thought}`); - if (step.action) { - console.log(`Action: ${step.action}`); - const action = step.action; - const actionRules = new XMLBuilder().build({ - action_rules: this.tools[action].getActionRule(), - }); - this.interMessages.push({ role: 'user', content: actionRules }); - } - if (step.action_input) { - const actionInput = new XMLBuilder().build({ action_input: step.action_input }); + parsedResult = parser.parse(result); + } catch (error) { + console.log('Error: Invalid XML response from bot'); + return 'Invalid response format.'; + } + + const step = parsedResult[Object.keys(parsedResult)[0]]; + + for (const key in step) { + if (key === 'thought') { + console.log(`Thought: ${step[key]}`); + } else if (key === 'action') { + currentAction = step[key] as string; + console.log(`Action: ${currentAction}`); + if (this.tools[currentAction]) { + const nextPrompt = [ + { + type: 'text', + text: builder.build({ action_rules: this.tools[currentAction].getActionRule() }), + }, + ]; + this.interMessages.push({ role: 'assistant', content: nextPrompt }); + break; + } else { + console.log('Error: No valid action'); + } + } else if (key === 'action_input') { + const actionInput = builder.build({ action_input: step[key] }); console.log(`Action input: ${actionInput}`); - try { - const observation = await this.processAction(step.action, step.action_input); - const nextPrompt = [{ type: 'text', text: '' }, ...observation, { type: 'text', text: '' }]; - this.interMessages.push({ role: 'user', content: nextPrompt }); - } catch (e) { - console.error(`Error processing action: ${e}`); - return `${e}`; + if (currentAction) { + try { + const observation = await this.processAction(currentAction, step[key]); + const nextPrompt = [{ type: 'text', text: '' }, ...observation, { type: 'text', text: '' }]; + this.interMessages.push({ role: 'assistant', content: nextPrompt }); + break; + } catch (error) { + console.log(`Error processing action: ${error}`); + return `${error}`; + } + } else { + console.log('Error: Action input without a valid action'); + return 'Action input without a valid action'; } - } - if (step.answer) { + } else if (key === 'answer') { console.log('Answer found. Ending query.'); - const answerContent = new XMLBuilder().build({ answer: step.answer }); + const answerContent = builder.build({ answer: step[key] }); this.messages.push({ role: 'assistant', content: answerContent }); this.interMessages = []; return answerContent; } - } catch (e) { - console.error('Error: Invalid XML response from bot'); - return 'Invalid response format.'; } } @@ -102,8 +122,9 @@ export class Agent { } private async execute(): Promise { + console.log('Messages: ' + this.interMessages); const completion = await this.client.chat.completions.create({ - model: 'gpt-4', + model: 'gpt-4o', messages: this.interMessages as ChatCompletionMessageParam[], temperature: 0, }); diff --git a/src/client/views/nodes/ChatBox/ChatBot.ts b/src/client/views/nodes/ChatBox/ChatBot.ts deleted file mode 100644 index 8b5e0982c..000000000 --- a/src/client/views/nodes/ChatBox/ChatBot.ts +++ /dev/null @@ -1,16 +0,0 @@ -import { Agent } from './Agent'; -import { Vectorstore } from './vectorstore/VectorstoreUpload'; -import dotenv from 'dotenv'; -dotenv.config(); - -export class ChatBot { - private agent: Agent; - - constructor(vectorstore: Vectorstore) { - this.agent = new Agent(vectorstore); - } - - async ask(question: string): Promise { - return await this.agent.askAgent(question); - } -} diff --git a/src/client/views/nodes/ChatBox/ChatBox.tsx b/src/client/views/nodes/ChatBox/ChatBox.tsx index 3ecb2d340..2ce1ebdd2 100644 --- a/src/client/views/nodes/ChatBox/ChatBox.tsx +++ b/src/client/views/nodes/ChatBox/ChatBox.tsx @@ -16,7 +16,7 @@ import { ASSISTANT_ROLE, AssistantMessage, AI_Document, convertToAIDocument, Cit import { Vectorstore } from './vectorstore/VectorstoreUpload'; import { CollectionFreeFormDocumentView } from '../CollectionFreeFormDocumentView'; import { CollectionFreeFormView } from '../../collections/collectionFreeForm'; -import { ChatBot } from './ChatBot'; +import { Agent } from './Agent'; import dotenv from 'dotenv'; dotenv.config(); @@ -34,7 +34,7 @@ export class ChatBox extends ViewBoxAnnotatableComponent() { private documents: AI_Document[] = []; private _oldWheel: any; private vectorstore: Vectorstore; - private chatbot: ChatBot; // Add the ChatBot instance + private agent: Agent; // Add the ChatBot instance public static LayoutString(fieldKey: string) { return FieldView.LayoutString(ChatBox, fieldKey); @@ -48,7 +48,7 @@ export class ChatBox extends ViewBoxAnnotatableComponent() { this.openai = this.initializeOpenAI(); this.getOtherDocs(); this.vectorstore = new Vectorstore(); - this.chatbot = new ChatBot(this.vectorstore); // Initialize the ChatBot + this.agent = new Agent(this.vectorstore); // Initialize the Agent reaction( () => this.history.map((msg: AssistantMessage) => ({ role: msg.role, text: msg.text, follow_up_questions: msg.follow_up_questions, citations: msg.citations })), @@ -58,6 +58,7 @@ export class ChatBox extends ViewBoxAnnotatableComponent() { ); } + @action getOtherDocs = async () => { const visible_docs = (CollectionFreeFormDocumentView.from(this._props.DocumentView?.())?._props.parent as CollectionFreeFormView)?.childDocs .filter(doc => doc != this.Document) @@ -76,6 +77,7 @@ export class ChatBox extends ViewBoxAnnotatableComponent() { doc['ai_document'] = document_json; } }); + this.isInitializing = false; }; @action @@ -120,7 +122,7 @@ export class ChatBox extends ViewBoxAnnotatableComponent() { this.history.push({ role: ASSISTANT_ROLE.USER, text: trimmedText }); }); this.isLoading = true; - const response = await this.chatbot.ask(trimmedText); // Use the chatbot to get the response + const response = await this.agent.askAgent(trimmedText); // Use the chatbot to get the response runInAction(() => { this.history.push(this.parseAssistantResponse(response)); }); @@ -142,8 +144,21 @@ export class ChatBox extends ViewBoxAnnotatableComponent() { const answerElement = xmlDoc.querySelector('answer'); const followUpQuestionsElement = xmlDoc.querySelector('follow_up_questions'); - const text = answerElement ? answerElement.innerHTML || '' : ''; // Use innerHTML to preserve citation tags - const followUpQuestions = followUpQuestionsElement ? Array.from(followUpQuestionsElement.querySelectorAll('question')).map(q => q.textContent || '') : []; + let text = ''; + let followUpQuestions: string[] = []; + + if (answerElement) { + // Remove the follow_up_questions element from the answer + const followUpElement = answerElement.querySelector('follow_up_questions'); + if (followUpElement) { + followUpElement.remove(); + } + text = answerElement.innerHTML.trim(); + } + + if (followUpQuestionsElement) { + followUpQuestions = Array.from(followUpQuestionsElement.querySelectorAll('question')).map(q => q.textContent || ''); + } return { role: ASSISTANT_ROLE.ASSISTANT, diff --git a/src/client/views/nodes/ChatBox/MessageComponent.tsx b/src/client/views/nodes/ChatBox/MessageComponent.tsx index 1baf6d7d5..91671a24a 100644 --- a/src/client/views/nodes/ChatBox/MessageComponent.tsx +++ b/src/client/views/nodes/ChatBox/MessageComponent.tsx @@ -12,8 +12,7 @@ interface MessageComponentProps { } const MessageComponent: React.FC = function ({ message, index, onFollowUpClick, onCitationClick, updateMessageCitations }) { - const LinkRenderer = ({ children }: { children: React.ReactNode }) => { - const text = children as string; + const renderContent = (text: string) => { const citationRegex = /([^<]*)<\/citation>/g; const parts = []; let lastIndex = 0; @@ -47,7 +46,6 @@ const MessageComponent: React.FC = function ({ message, i parts.push(text.slice(lastIndex)); - // Update the message's citations in the ChatBox's history updateMessageCitations(index, citations); return <>{parts}; @@ -55,9 +53,7 @@ const MessageComponent: React.FC = function ({ message, i return (
-
- {message.text} -
+
{renderContent(message.text)}
{message.follow_up_questions && message.follow_up_questions.length > 0 && (

Follow-up Questions:

diff --git a/src/client/views/nodes/ChatBox/prompts.ts b/src/client/views/nodes/ChatBox/prompts.ts index 8835265e4..ffea13788 100644 --- a/src/client/views/nodes/ChatBox/prompts.ts +++ b/src/client/views/nodes/ChatBox/prompts.ts @@ -6,12 +6,12 @@ export function getReactPrompt(tools: Tool[], chatHistory: string): string { const toolDescriptions = tools.map(tool => `${tool.name}:\n${tool.briefSummary}`).join('\n*****\n'); return ` - You run in a loop of Thought, Action, PAUSE, Action Input, Pause, Observation. + You run in a loop of Thought, Action, (PAUSE), Action Input, (PAUSE), Observation. (this Thought/Action/PAUSE/Action Input/PAUSE/Observation can repeat N times) Contain each stage of the loop within an XML element that specifies the stage type (e.g. content of the thought). At the end of the loop, you output an Answer with the answer content contained within an XML element with an tag. At the end of the answer should be an array of 3 potential follow-up questions for the user to ask you next, contained within a key. Use to describe your thoughts about the question you have been asked. - Use to specify run one of the actions available to you - then return a element. + Use to specify run one of the actions available to you. Then, you will be provided with action rules within an element that specifies how you should structure the input to the action and what the output of that action will look like - then return another element. Then, provide within an element each parameter, with parameter names as element tags themselves with their values inside, following the structure defined in the action rules. Observation, in an element will be the result of running those actions. @@ -28,7 +28,6 @@ export function getReactPrompt(tools: Tool[], chatHistory: string): string { I should look up France on Wikipedia wikipedia - You will be called again with this: -- cgit v1.2.3-70-g09d2 From f1cdfc1d02488c4a513fbf67f729f702526a345d Mon Sep 17 00:00:00 2001 From: "A.J. Shulman" Date: Thu, 11 Jul 2024 12:06:04 -0400 Subject: not working well (files don't get filename immediately_ --- src/client/documents/DocumentTypes.ts | 1 + src/client/documents/Documents.ts | 5 +++ src/client/views/nodes/ChatBox/ChatBox.tsx | 36 +++++++---------- .../views/nodes/ChatBox/MessageComponent.tsx | 33 +++++++++++++++- .../views/nodes/ChatBox/tools/CollectionTool.ts | 0 src/client/views/nodes/ChatBox/tools/RAGTool.ts | 9 +++++ .../nodes/ChatBox/vectorstore/VectorstoreUpload.ts | 45 +++++++++++++++++----- 7 files changed, 95 insertions(+), 34 deletions(-) create mode 100644 src/client/views/nodes/ChatBox/tools/CollectionTool.ts (limited to 'src/client/views/nodes/ChatBox/MessageComponent.tsx') diff --git a/src/client/documents/DocumentTypes.ts b/src/client/documents/DocumentTypes.ts index 8f95068db..cb1625381 100644 --- a/src/client/documents/DocumentTypes.ts +++ b/src/client/documents/DocumentTypes.ts @@ -28,6 +28,7 @@ export enum DocumentType { DATAVIZ = 'dataviz', LOADING = 'loading', SIMULATION = 'simulation', // physics simulation + MESSAGE = 'message', // chat message // special purpose wrappers that either take no data or are compositions of lower level types LINK = 'link', diff --git a/src/client/documents/Documents.ts b/src/client/documents/Documents.ts index a67e6b4f6..ea5eca804 100644 --- a/src/client/documents/Documents.ts +++ b/src/client/documents/Documents.ts @@ -791,6 +791,11 @@ export namespace Docs { export function RTFDocument(field: RichTextField, options: DocumentOptions = {}, fieldKey: string = 'text') { return InstanceFromProto(Prototypes.get(DocumentType.RTF), field, options, undefined, fieldKey); } + + export function MessageDocument(field: string, options: DocumentOptions = {}, fieldKey: string = 'data') { + return InstanceFromProto(Prototypes.get(DocumentType.MESSAGE), field, options, undefined, fieldKey); + } + export function TextDocument(text: string, options: DocumentOptions = {}, fieldKey: string = 'text') { const rtf = { doc: { diff --git a/src/client/views/nodes/ChatBox/ChatBox.tsx b/src/client/views/nodes/ChatBox/ChatBox.tsx index 64ab2888b..9b2a92564 100644 --- a/src/client/views/nodes/ChatBox/ChatBox.tsx +++ b/src/client/views/nodes/ChatBox/ChatBox.tsx @@ -11,7 +11,7 @@ import { LinkManager } from '../../../util/LinkManager'; import { ViewBoxAnnotatableComponent } from '../../DocComponent'; import { FieldView, FieldViewProps } from '../FieldView'; import './ChatBox.scss'; -import MessageComponent from './MessageComponent'; +import MessageComponentBox from './MessageComponent'; import { ASSISTANT_ROLE, AssistantMessage, AI_Document, convertToAIDocument, Citation } from './types'; import { Vectorstore } from './vectorstore/VectorstoreUpload'; import { CollectionFreeFormDocumentView } from '../CollectionFreeFormDocumentView'; @@ -19,6 +19,7 @@ import { CollectionFreeFormView } from '../../collections/collectionFreeForm'; import { Agent } from './Agent'; import dotenv from 'dotenv'; import { DocData } from '../../../../fields/DocSymbols'; +import { DocumentView } from '../DocumentView'; dotenv.config(); @observer @@ -33,7 +34,7 @@ export class ChatBox extends ViewBoxAnnotatableComponent() { @observable inputValue: string = ''; @observable private _visibleDocs: Doc[] = []; private openai: OpenAI; - private vectorstore_id: string; + // private vectorstore_id: string; private documents: AI_Document[] = []; private _oldWheel: any; private vectorstore: Vectorstore; @@ -61,22 +62,8 @@ export class ChatBox extends ViewBoxAnnotatableComponent() { } @action - addAIDocs = async (visible_docs: Doc[]) => { - console.log('All Docs:', visible_docs); - visible_docs?.forEach(async doc => { - if (doc[DocData].ai_document) { - this.documents.push(convertToAIDocument(JSON.parse(StrCast(doc[DocData].ai_document)))); - } else { - const local_file_path: string = CsvCast(doc.data, PDFCast(doc.data))?.url?.pathname; - if (local_file_path) { - const { document_json } = await Networking.PostToServer('/createDocument', { file_path: local_file_path }); - const ai_document: AI_Document = convertToAIDocument(document_json); - this.documents.push(ai_document); - await this.vectorstore.addDocument(ai_document); - doc[DocData].ai_document = JSON.stringify(document_json); - } - } - }); + addDocsToVectorstore = async (visible_docs: Doc[]) => { + await this.vectorstore.addAIDocs(visible_docs); this.isInitializing = false; }; @@ -107,6 +94,10 @@ export class ChatBox extends ViewBoxAnnotatableComponent() { } }; + // getAssistantResponse() { + // return Docs.Create.MessageDocument(text, {}); + // } + @action askGPT = async (event: React.FormEvent): Promise => { event.preventDefault(); @@ -241,7 +232,7 @@ export class ChatBox extends ViewBoxAnnotatableComponent() { reaction( () => this.visibleDocs, visibleDocs => { - this._visibleDocs = visibleDocs; + this._visibleDocs.push(...visibleDocs.filter(visibleDoc => !this._visibleDocs.includes(visibleDoc))); } ); observe( @@ -255,7 +246,7 @@ export class ChatBox extends ViewBoxAnnotatableComponent() { if ((change as any).addedCount > 0) { // maybe check here if its already in the urls datadoc array so doesn't add twice console.log((change as any).added as Doc[]); - this.addAIDocs((change as any).added as Doc[]); + this.addDocsToVectorstore((change as any).added as Doc[]); } // (change as any).removed.forEach((link: any) => remLinkFromDoc(toRealField(link))); break; @@ -295,10 +286,11 @@ export class ChatBox extends ViewBoxAnnotatableComponent() { }}>
{this.history.map((message, index) => ( - + // + ))} {this.current_message && ( - void; } -const MessageComponent: React.FC = function ({ message, index, onFollowUpClick, onCitationClick, updateMessageCitations }) { +const MessageComponentBox: React.FC = function ({ message, index, onFollowUpClick, onCitationClick, updateMessageCitations }) { + // public static LayoutString(fieldKey: string) { + // return FieldView.LayoutString(MessageComponentBox, fieldKey); + // } + + // the presentation view that renders this slide + + // @computed + // get chatBoxView() { + // return this.DocumentView?.().containerViewPath?.().lastElement()?.ComponentView as ChatBox; + // } + const renderContent = (text: string) => { const citationRegex = /([^<]*)<\/citation>/g; const parts = []; @@ -68,4 +81,20 @@ const MessageComponent: React.FC = function ({ message, i ); }; -export default observer(MessageComponent); +// Docs.Prototypes.TemplateMap.set(DocumentType.MESSAGE, { +// layout: { view: MessageComponentBox, dataField: 'data' }, +// options: { +// acl: '', +// _height: 35, +// _xMargin: 10, +// _yMargin: 10, +// _layout_nativeDimEditable: true, +// _layout_reflowVertical: true, +// _layout_reflowHorizontal: true, +// defaultDoubleClick: 'ignore', +// systemIcon: 'BsFileEarmarkTextFill', +// layout_borderRounding: '10px', +// }, +// }); + +export default observer(MessageComponentBox); diff --git a/src/client/views/nodes/ChatBox/tools/CollectionTool.ts b/src/client/views/nodes/ChatBox/tools/CollectionTool.ts new file mode 100644 index 000000000..e69de29bb diff --git a/src/client/views/nodes/ChatBox/tools/RAGTool.ts b/src/client/views/nodes/ChatBox/tools/RAGTool.ts index 185efa0ba..36e4bc3ce 100644 --- a/src/client/views/nodes/ChatBox/tools/RAGTool.ts +++ b/src/client/views/nodes/ChatBox/tools/RAGTool.ts @@ -39,6 +39,7 @@ export class RAGTool extends BaseTool<{ hypothetical_document_chunk: string }> { !!!IMPORTANT Before you close the tag with , within the answer tags provide a set of 3 follow-up questions inside a tag and individually within tags. These should relate to the document, the current query, and the chat_history and should aim to help the user better understand whatever they are looking for. Also, ensure that the answer tags are wrapped with the correct step tags as well.`, + `Performs a RAG (Retrieval-Augmented Generation) search on user documents and returns a set of document chunks (either images or text) that can be used to provide a grounded response based on user documents @@ -49,6 +50,14 @@ export class RAGTool extends BaseTool<{ hypothetical_document_chunk: string }> { ); } + changeSummaries(summaries: string) { + this.briefSummary = `Performs a RAG (Retrieval-Augmented Generation) search on user documents and returns a set of document chunks (either images or text) that can be used to provide a grounded response based on user documents. + +!!!IMPORTANT Use the RAG tool ANYTIME the question may potentially (even if you are not sure) relate to one of the user's documents. +Here are the summaries of the user's documents: +${summaries}`; + } + async execute(args: { hypothetical_document_chunk: string }): Promise { const relevantChunks = await this.vectorstore.retrieve(args.hypothetical_document_chunk); return this.getFormattedChunks(relevantChunks); diff --git a/src/client/views/nodes/ChatBox/vectorstore/VectorstoreUpload.ts b/src/client/views/nodes/ChatBox/vectorstore/VectorstoreUpload.ts index 1f483ad61..5e8e6b23a 100644 --- a/src/client/views/nodes/ChatBox/vectorstore/VectorstoreUpload.ts +++ b/src/client/views/nodes/ChatBox/vectorstore/VectorstoreUpload.ts @@ -3,7 +3,11 @@ import { CohereClient } from 'cohere-ai'; import { EmbedResponse } from 'cohere-ai/api'; import dotenv from 'dotenv'; -import { Chunk, AI_Document } from '../types'; +import { Chunk, AI_Document, convertToAIDocument } from '../types'; +import { Doc } from '../../../../../fields/Doc'; +import { DocData } from '../../../../../fields/DocSymbols'; +import { CsvCast, PDFCast, StrCast } from '../../../../../fields/Types'; +import { Networking } from '../../../../Network'; dotenv.config(); @@ -12,7 +16,7 @@ export class Vectorstore { private index!: Index; private cohere: CohereClient; private indexName: string = 'pdf-chatbot'; - private documents: AI_Document[] = []; + documents: AI_Document[] = []; constructor() { const pineconeApiKey = process.env.PINECONE_API_KEY; @@ -49,10 +53,35 @@ export class Vectorstore { this.index = this.pinecone.Index(this.indexName); } - async addDocument(document: AI_Document) { - this.documents.push(document); - await this.indexDocument(document); - console.log(`Document added: ${document.file_name}`); + async addAIDocs(visible_docs: Doc[]) { + console.log('All Docs:', visible_docs); + visible_docs?.forEach(async doc => { + await this.addAIDoc(doc); + }); + } + + async addAIDoc(doc: Doc) { + if (doc[DocData]?.ai_document) { + this.documents.push(convertToAIDocument(JSON.parse(StrCast(doc[DocData].ai_document)))); + console.log(`Document already added: ${doc[DocData].file_name}`); + } else { + console.log(doc); + const local_file_path: string = CsvCast(doc.data)?.url?.pathname ?? PDFCast(doc.data)?.url?.pathname; + console.log('Local File Path:', local_file_path); + if (local_file_path) { + const { document_json } = await Networking.PostToServer('/createDocument', { file_path: local_file_path }); + console.log('Document JSON:', document_json); + const ai_document: AI_Document = convertToAIDocument(document_json); + this.documents.push(ai_document); + await this.indexDocument(ai_document); + console.log(`Document added: ${ai_document.file_name}`); + doc[DocData].ai_document = JSON.stringify(document_json); + } + } + } + + getSummaries(): string { + return this.documents.map((doc, index) => `${index + 1}) ${doc.summary}`).join('\n') + '\n'; } private async indexDocument(document: AI_Document) { @@ -111,8 +140,4 @@ export class Vectorstore { return []; } } - - getSummaries(): string { - return this.documents.map((doc, index) => `${index + 1}) ${doc.summary}`).join('\n') + '\n'; - } } -- cgit v1.2.3-70-g09d2 From ef79b7d617035c52fea159225ba9a39b8222e8f4 Mon Sep 17 00:00:00 2001 From: "A.J. Shulman" Date: Mon, 15 Jul 2024 12:31:35 -0400 Subject: citation parsing working much better --- .vscode/launch.json | 15 ++-- src/client/views/nodes/ChatBox/Agent.ts | 8 +- src/client/views/nodes/ChatBox/AnswerParser.ts | 60 ++++++++++++++ src/client/views/nodes/ChatBox/ChatBox.tsx | 52 ++++-------- .../views/nodes/ChatBox/MessageComponent.tsx | 95 ++++++++-------------- src/client/views/nodes/ChatBox/types.ts | 18 +++- .../nodes/ChatBox/vectorstore/VectorstoreUpload.ts | 5 +- 7 files changed, 138 insertions(+), 115 deletions(-) create mode 100644 src/client/views/nodes/ChatBox/AnswerParser.ts (limited to 'src/client/views/nodes/ChatBox/MessageComponent.tsx') diff --git a/.vscode/launch.json b/.vscode/launch.json index e4c31361c..e9a07cf93 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -59,7 +59,7 @@ "webRoot": "${workspaceFolder}" }, { - "type": "node", + "type": "pwa-node", "request": "attach", "name": "Typescript Server", "protocol": "inspector", @@ -68,7 +68,7 @@ "remoteRoot": "${workspaceFolder}" }, { - "type": "node", + "type": "pwa-node", "request": "launch", "name": "Current TS File", "runtimeExecutable": "npx", @@ -76,7 +76,7 @@ "port": 9229 }, { - "type": "node", + "type": "pwa-node", "request": "launch", "name": "Mocha Tests", "program": "${workspaceFolder}/node_modules/mocha/bin/_mocha", @@ -86,7 +86,7 @@ "protocol": "inspector" }, { - "type": "node", + "type": "pwa-node", "request": "launch", "name": "Mocha Current File", "program": "${workspaceFolder}/node_modules/mocha/bin/_mocha", @@ -96,9 +96,6 @@ "protocol": "inspector" } ], - - "resolveSourceMapLocations": [ - "${workspaceFolder}/**", - "!**/node_modules/**" - ] + + "resolveSourceMapLocations": ["${workspaceFolder}/**", "!**/node_modules/**"] } diff --git a/src/client/views/nodes/ChatBox/Agent.ts b/src/client/views/nodes/ChatBox/Agent.ts index 6757b2ce8..fd3c6e5e8 100644 --- a/src/client/views/nodes/ChatBox/Agent.ts +++ b/src/client/views/nodes/ChatBox/Agent.ts @@ -95,6 +95,8 @@ export class Agent { break; } else { console.log('Error: No valid action'); + this.interMessages.push({ role: 'user', content: 'No valid action, try again.' }); + break; } } else if (key === 'action_input') { const actionInput = builder.build({ action_input: step[key] }); @@ -116,11 +118,7 @@ export class Agent { } } else if (key === 'answer') { console.log('Answer found. Ending query.'); - const answerContent = builder.build({ answer: step[key] }); - this.messages.push({ role: 'assistant', content: answerContent }); - this.interMessages = []; - console.log(this.messages); - return answerContent; + return result; } } } diff --git a/src/client/views/nodes/ChatBox/AnswerParser.ts b/src/client/views/nodes/ChatBox/AnswerParser.ts new file mode 100644 index 000000000..f77d2261d --- /dev/null +++ b/src/client/views/nodes/ChatBox/AnswerParser.ts @@ -0,0 +1,60 @@ +import { ASSISTANT_ROLE, AssistantMessage, Citation, getChunkType } from './types'; + +export class AnswerParser { + static parse(xml: string): AssistantMessage { + const answerRegex = /([\s\S]*?)<\/answer>/; + const citationRegex = /(.*?)<\/citation>/g; + const followUpQuestionsRegex = /([\s\S]*?)<\/follow_up_questions>/; + const questionRegex = /(.*?)<\/question>/g; + + const answerMatch = answerRegex.exec(xml); + const followUpQuestionsMatch = followUpQuestionsRegex.exec(xml); + + if (!answerMatch) { + throw new Error('Invalid XML: Missing tag.'); + } + + const rawTextContent = answerMatch[1].trim(); + const textContentWithCitations = rawTextContent.replace(citationRegex, ''); + const textContent = textContentWithCitations.replace(followUpQuestionsRegex, '').trim(); + + let citations: Citation[] = []; + let match: RegExpExecArray | null; + + let plainTextOffset = 0; + let citationOffset = 0; + + while ((match = citationRegex.exec(rawTextContent)) !== null) { + const [fullMatch, chunk_id, type, direct_text] = match; + const citationStartIndex = match.index; + const citationPlainStart = citationStartIndex - citationOffset; + + citations.push({ + direct_text: direct_text.trim(), + type: getChunkType(type), + chunk_id: chunk_id, + location: citationPlainStart, + }); + + citationOffset += fullMatch.length; + } + + let followUpQuestions: string[] = []; + if (followUpQuestionsMatch) { + const questionsText = followUpQuestionsMatch[1]; + let questionMatch: RegExpExecArray | null; + + while ((questionMatch = questionRegex.exec(questionsText)) !== null) { + followUpQuestions.push(questionMatch[1].trim()); + } + } + const assistantResponse: AssistantMessage = { + role: ASSISTANT_ROLE.ASSISTANT, + text_content: textContent, + follow_up_questions: followUpQuestions, + citations: citations, + }; + + return assistantResponse; + } +} diff --git a/src/client/views/nodes/ChatBox/ChatBox.tsx b/src/client/views/nodes/ChatBox/ChatBox.tsx index 9b2a92564..bae6bbaa6 100644 --- a/src/client/views/nodes/ChatBox/ChatBox.tsx +++ b/src/client/views/nodes/ChatBox/ChatBox.tsx @@ -12,7 +12,7 @@ import { ViewBoxAnnotatableComponent } from '../../DocComponent'; import { FieldView, FieldViewProps } from '../FieldView'; import './ChatBox.scss'; import MessageComponentBox from './MessageComponent'; -import { ASSISTANT_ROLE, AssistantMessage, AI_Document, convertToAIDocument, Citation } from './types'; +import { ASSISTANT_ROLE, AssistantMessage, AI_Document, convertToAIDocument, Citation, CHUNK_TYPE } from './types'; import { Vectorstore } from './vectorstore/VectorstoreUpload'; import { CollectionFreeFormDocumentView } from '../CollectionFreeFormDocumentView'; import { CollectionFreeFormView } from '../../collections/collectionFreeForm'; @@ -20,6 +20,7 @@ import { Agent } from './Agent'; import dotenv from 'dotenv'; import { DocData } from '../../../../fields/DocSymbols'; import { DocumentView } from '../DocumentView'; +import { AnswerParser } from './AnswerParser'; dotenv.config(); @observer @@ -47,14 +48,13 @@ export class ChatBox extends ViewBoxAnnotatableComponent() { constructor(props: FieldViewProps) { super(props); makeObservable(this); - this.openai = this.initializeOpenAI(); - this.history = [{ role: ASSISTANT_ROLE.ASSISTANT, text: 'Welcome to the Document Analyser Assistant! Link a document or ask questions to get started.' }]; + this.history = [{ role: ASSISTANT_ROLE.ASSISTANT, text_content: 'Welcome to the Document Analyser Assistant! Link a document or ask questions to get started.' }]; this.openai = this.initializeOpenAI(); this.vectorstore = new Vectorstore(); this.agent = new Agent(this.vectorstore); // Initialize the Agent reaction( - () => this.history.map((msg: AssistantMessage) => ({ role: msg.role, text: msg.text, follow_up_questions: msg.follow_up_questions, citations: msg.citations })), + () => this.history.map((msg: AssistantMessage) => ({ role: msg.role, text_content: msg.text_content, follow_up_questions: msg.follow_up_questions, citations: msg.citations })), serializableHistory => { this.dataDoc.data = JSON.stringify(serializableHistory); } @@ -110,19 +110,19 @@ export class ChatBox extends ViewBoxAnnotatableComponent() { try { textInput.value = ''; runInAction(() => { - this.history.push({ role: ASSISTANT_ROLE.USER, text: trimmedText }); + this.history.push({ role: ASSISTANT_ROLE.USER, text_content: trimmedText }); this.isLoading = true; }); const response = await this.agent.askAgent(trimmedText); // Use the chatbot to get the response runInAction(() => { - this.history.push(this.parseAssistantResponse(response)); + this.history.push(AnswerParser.parse(response)); }); this.dataDoc.data = JSON.stringify(this.history); } catch (err) { console.error('Error:', err); runInAction(() => { - this.history.push({ role: ASSISTANT_ROLE.ASSISTANT, text: 'Sorry, I encountered an error while processing your request.' }); + this.history.push({ role: ASSISTANT_ROLE.ASSISTANT, text_content: 'Sorry, I encountered an error while processing your request.' }); }); } finally { runInAction(() => { @@ -132,35 +132,6 @@ export class ChatBox extends ViewBoxAnnotatableComponent() { } }; - parseAssistantResponse(response: string): AssistantMessage { - const parser = new DOMParser(); - const xmlDoc = parser.parseFromString(response, 'text/xml'); - const answerElement = xmlDoc.querySelector('answer'); - const followUpQuestionsElement = xmlDoc.querySelector('follow_up_questions'); - - let text = ''; - let followUpQuestions: string[] = []; - - if (answerElement) { - // Remove the follow_up_questions element from the answer - const followUpElement = answerElement.querySelector('follow_up_questions'); - if (followUpElement) { - followUpElement.remove(); - } - text = answerElement.innerHTML.trim(); - } - - if (followUpQuestionsElement) { - followUpQuestions = Array.from(followUpQuestionsElement.querySelectorAll('question')).map(q => q.textContent || ''); - } - - return { - role: ASSISTANT_ROLE.ASSISTANT, - text, - follow_up_questions: followUpQuestions, - }; - } - @action updateMessageCitations = (index: number, citations: Citation[]) => { if (this.history[index]) { @@ -219,7 +190,7 @@ export class ChatBox extends ViewBoxAnnotatableComponent() { this.history.push( ...storedHistory.map((msg: AssistantMessage) => ({ role: msg.role, - text: msg.text, + text_content: msg.text_content, follow_up_questions: msg.follow_up_questions, citations: msg.citations, })) @@ -246,6 +217,7 @@ export class ChatBox extends ViewBoxAnnotatableComponent() { if ((change as any).addedCount > 0) { // maybe check here if its already in the urls datadoc array so doesn't add twice console.log((change as any).added as Doc[]); + console.log('here!'); this.addDocsToVectorstore((change as any).added as Doc[]); } // (change as any).removed.forEach((link: any) => remLinkFromDoc(toRealField(link))); @@ -265,7 +237,11 @@ export class ChatBox extends ViewBoxAnnotatableComponent() { @computed get visibleDocs() { - return (CollectionFreeFormDocumentView.from(this._props.DocumentView?.())?._props.parent as CollectionFreeFormView)?.childDocs.filter(doc => doc != this.Document) ?? []; + //return (CollectionFreeFormDocumentView.from(this._props.DocumentView?.())?._props.parent as CollectionFreeFormView)?.childDocs.filter(doc => doc != this.Document) ?? []; + return LinkManager.Instance.getAllRelatedLinks(this.Document) + .map(d => DocCast(LinkManager.getOppositeAnchor(d, this.Document))) + .map(d => DocCast(d?.annotationOn, d)) + .filter(d => d); } @action diff --git a/src/client/views/nodes/ChatBox/MessageComponent.tsx b/src/client/views/nodes/ChatBox/MessageComponent.tsx index 38faf7e00..76faff10b 100644 --- a/src/client/views/nodes/ChatBox/MessageComponent.tsx +++ b/src/client/views/nodes/ChatBox/MessageComponent.tsx @@ -1,9 +1,6 @@ import React from 'react'; import { observer } from 'mobx-react'; -import { AssistantMessage, CHUNK_TYPE, Citation } from './types'; -import { TbInfoCircleFilled } from 'react-icons/tb'; -import { Docs } from '../../../documents/Documents'; -import { DocumentType } from '../../../documents/DocumentTypes'; +import { AssistantMessage, Citation } from './types'; interface MessageComponentProps { message: AssistantMessage; @@ -14,59 +11,53 @@ interface MessageComponentProps { } const MessageComponentBox: React.FC = function ({ message, index, onFollowUpClick, onCitationClick, updateMessageCitations }) { - // public static LayoutString(fieldKey: string) { - // return FieldView.LayoutString(MessageComponentBox, fieldKey); - // } - - // the presentation view that renders this slide - - // @computed - // get chatBoxView() { - // return this.DocumentView?.().containerViewPath?.().lastElement()?.ComponentView as ChatBox; - // } + const renderContent = (content: string) => { + if (!message.citations || message.citations.length === 0) { + return content; + } - const renderContent = (text: string) => { - const citationRegex = /([^<]*)<\/citation>/g; const parts = []; let lastIndex = 0; - let match; - const citations: Citation[] = []; - - while ((match = citationRegex.exec(text)) !== null) { - const [fullMatch, chunkId, type, content] = match; - const citation: Citation = { chunk_id: chunkId, type: type as CHUNK_TYPE, text: content }; - citations.push(citation); - parts.push(text.slice(lastIndex, match.index)); - parts.push( - { - e.preventDefault(); - onCitationClick(citation); - }} + message.citations.forEach((citation, idx) => { + const location = citation.location; + const textBefore = content.slice(lastIndex, location); + const citationButton = ( + ); - lastIndex = match.index + fullMatch.length; - } - - parts.push(text.slice(lastIndex)); + parts.push(textBefore, citationButton); + lastIndex = location; + }); - updateMessageCitations(index, citations); + parts.push(content.slice(lastIndex)); - return <>{parts}; + return parts; }; return (
-
{renderContent(message.text)}
+
{renderContent(message.text_content)}
{message.follow_up_questions && message.follow_up_questions.length > 0 && (

Follow-up Questions:

@@ -81,20 +72,4 @@ const MessageComponentBox: React.FC = function ({ message ); }; -// Docs.Prototypes.TemplateMap.set(DocumentType.MESSAGE, { -// layout: { view: MessageComponentBox, dataField: 'data' }, -// options: { -// acl: '', -// _height: 35, -// _xMargin: 10, -// _yMargin: 10, -// _layout_nativeDimEditable: true, -// _layout_reflowVertical: true, -// _layout_reflowHorizontal: true, -// defaultDoubleClick: 'ignore', -// systemIcon: 'BsFileEarmarkTextFill', -// layout_borderRounding: '10px', -// }, -// }); - export default observer(MessageComponentBox); diff --git a/src/client/views/nodes/ChatBox/types.ts b/src/client/views/nodes/ChatBox/types.ts index 0270b6256..a0d295e92 100644 --- a/src/client/views/nodes/ChatBox/types.ts +++ b/src/client/views/nodes/ChatBox/types.ts @@ -9,17 +9,31 @@ export enum CHUNK_TYPE { TABLE = 'table', } +export function getChunkType(type: string): CHUNK_TYPE { + switch (type.toLowerCase()) { + case 'text': + return CHUNK_TYPE.TEXT; + case 'image': + return CHUNK_TYPE.IMAGE; + case 'table': + return CHUNK_TYPE.TABLE; + default: + return CHUNK_TYPE.TEXT; + } +} + export interface AssistantMessage { role: ASSISTANT_ROLE; - text: string; + text_content: string; follow_up_questions?: string[]; citations?: Citation[]; } export interface Citation { - text: string; + direct_text?: string; type: CHUNK_TYPE; chunk_id: string; + location: number; } export interface Chunk { diff --git a/src/client/views/nodes/ChatBox/vectorstore/VectorstoreUpload.ts b/src/client/views/nodes/ChatBox/vectorstore/VectorstoreUpload.ts index 5e8e6b23a..d3b1cb4e7 100644 --- a/src/client/views/nodes/ChatBox/vectorstore/VectorstoreUpload.ts +++ b/src/client/views/nodes/ChatBox/vectorstore/VectorstoreUpload.ts @@ -58,14 +58,17 @@ export class Vectorstore { visible_docs?.forEach(async doc => { await this.addAIDoc(doc); }); + return; } async addAIDoc(doc: Doc) { if (doc[DocData]?.ai_document) { this.documents.push(convertToAIDocument(JSON.parse(StrCast(doc[DocData].ai_document)))); - console.log(`Document already added: ${doc[DocData].file_name}`); + console.log(`Document already added: ${doc.file_name}`); } else { console.log(doc); + console.log(PDFCast(doc.data)?.url?.pathname); + console.log(CsvCast(doc.data)?.url?.pathname); const local_file_path: string = CsvCast(doc.data)?.url?.pathname ?? PDFCast(doc.data)?.url?.pathname; console.log('Local File Path:', local_file_path); if (local_file_path) { -- cgit v1.2.3-70-g09d2 From 97fdb44133c6aed043f84fd345d5ac57125e5405 Mon Sep 17 00:00:00 2001 From: "A.J. Shulman" Date: Mon, 15 Jul 2024 13:47:39 -0400 Subject: attempt at adding links --- package-lock.json | 12 ++ package.json | 1 + src/client/views/nodes/ChatBox/ChatBox.scss | 128 ++++++++++----------- src/client/views/nodes/ChatBox/ChatBox.tsx | 80 ++++++++----- .../views/nodes/ChatBox/MessageComponent.tsx | 2 +- .../nodes/ChatBox/vectorstore/VectorstoreUpload.ts | 22 +++- 6 files changed, 143 insertions(+), 102 deletions(-) (limited to 'src/client/views/nodes/ChatBox/MessageComponent.tsx') diff --git a/package-lock.json b/package-lock.json index 713174741..79aac1ece 100644 --- a/package-lock.json +++ b/package-lock.json @@ -141,6 +141,7 @@ "lodash": "^4.17.21", "mapbox-gl": "^3.0.1", "markdown-it": "^14.1.0", + "markdown-to-jsx": "^7.4.7", "mathquill": "^0.10.1-a", "md5-file": "^5.0.0", "memorystream": "^0.3.1", @@ -25639,6 +25640,17 @@ "url": "https://github.com/sponsors/wooorm" } }, + "node_modules/markdown-to-jsx": { + "version": "7.4.7", + "resolved": "https://registry.npmjs.org/markdown-to-jsx/-/markdown-to-jsx-7.4.7.tgz", + "integrity": "sha512-0+ls1IQZdU6cwM1yu0ZjjiVWYtkbExSyUIFU2ZeDIFuZM1W42Mh4OlJ4nb4apX4H8smxDHRdFaoIVJGwfv5hkg==", + "engines": { + "node": ">= 10" + }, + "peerDependencies": { + "react": ">= 0.14.0" + } + }, "node_modules/material-colors": { "version": "1.2.6", "resolved": "https://registry.npmjs.org/material-colors/-/material-colors-1.2.6.tgz", diff --git a/package.json b/package.json index e1895ad81..bdc049694 100644 --- a/package.json +++ b/package.json @@ -226,6 +226,7 @@ "lodash": "^4.17.21", "mapbox-gl": "^3.0.1", "markdown-it": "^14.1.0", + "markdown-to-jsx": "^7.4.7", "mathquill": "^0.10.1-a", "md5-file": "^5.0.0", "memorystream": "^0.3.1", diff --git a/src/client/views/nodes/ChatBox/ChatBox.scss b/src/client/views/nodes/ChatBox/ChatBox.scss index f1e3d3d67..75171fe56 100644 --- a/src/client/views/nodes/ChatBox/ChatBox.scss +++ b/src/client/views/nodes/ChatBox/ChatBox.scss @@ -5,6 +5,10 @@ $button-color: #007bff; $button-hover-color: darken($button-color, 10%); $shadow-color: rgba(0, 0, 0, 0.075); $border-radius: 8px; +$citation-color: #ff6347; +$citation-hover-color: darken($citation-color, 10%); +$follow-up-bg-color: #e9ecef; +$follow-up-hover-bg-color: #dee2e6; .chatBox { display: flex; @@ -40,51 +44,41 @@ $border-radius: 8px; .messages { display: flex; flex-direction: column; + .message { - padding: 10px; + padding: 10px 15px; margin-bottom: 10px; border-radius: $border-radius; background-color: lighten($background-color, 5%); box-shadow: 0 2px 5px $shadow-color; align-items: flex-start; max-width: 90%; - width: 100%; + width: fit-content; word-break: break-word; + position: relative; - .message-footer { - width: 100%; - - .toggle-logs-button { - margin-top: 10px; - width: 95%; - text-align: center; - background-color: $button-color; - color: #fff; - border: none; - border-radius: $border-radius; - cursor: pointer; - box-shadow: 0 2px 4px $shadow-color; - &:hover { - background-color: $button-hover-color; - } - } - .tool-logs { - width: 100%; - background-color: $input-background; - color: $text-color; - margin-top: 5px; - font-family: monospace; - overflow-x: auto; - max-height: 150px; - overflow-y: auto; + .citation-button { + background-color: $citation-color; + color: #fff; + border: none; + border-radius: 50%; + cursor: pointer; + width: 20px; + height: 20px; + display: inline-flex; + align-items: center; + justify-content: center; + font-size: 12px; + font-weight: bold; + margin: 0 2px; + padding: 0; + transition: background-color 0.3s; + + &:hover { + background-color: $citation-hover-color; } } - .custom-link { - color: lightblue; - text-decoration: underline; - cursor: pointer; - } &.user { align-self: flex-end; background-color: $button-color; @@ -108,6 +102,40 @@ $border-radius: 8px; border-radius: 50%; } } + + .follow-up-questions { + margin-top: 10px; + width: 100%; + + h4 { + margin-bottom: 5px; + font-size: 14px; + } + + .follow-up-button { + background-color: $follow-up-bg-color; + border: 1px solid #ddd; + border-radius: 8px; + padding: 8px 10px; + margin: 4px 0; + cursor: pointer; + transition: background-color 0.3s; + display: block; + width: 100%; + text-align: left; + white-space: normal; + word-wrap: break-word; + font-size: 12px; + color: $text-color; + min-height: 40px; + height: auto; + line-height: 1.3; + + &:hover { + background-color: $follow-up-hover-bg-color; + } + } + } } } @@ -212,37 +240,3 @@ $border-radius: 8px; } } } - -.follow-up-questions { - margin-top: 10px; - width: 100%; - - h4 { - margin-bottom: 5px; - font-size: 14px; - } - - .follow-up-button { - background-color: #f0f0f0; - border: 1px solid #ddd; - border-radius: 8px; - padding: 8px 10px; - margin: 4px 0; - cursor: pointer; - transition: background-color 0.3s; - display: block; - width: 100%; - text-align: left; - white-space: normal; - word-wrap: break-word; - font-size: 12px; - color: $text-color; - min-height: 40px; - height: auto; // Allow the button to expand as needed - line-height: 1.3; - - &:hover { - background-color: #e0e0e0; - } - } -} diff --git a/src/client/views/nodes/ChatBox/ChatBox.tsx b/src/client/views/nodes/ChatBox/ChatBox.tsx index bae6bbaa6..4d7381a57 100644 --- a/src/client/views/nodes/ChatBox/ChatBox.tsx +++ b/src/client/views/nodes/ChatBox/ChatBox.tsx @@ -12,15 +12,20 @@ import { ViewBoxAnnotatableComponent } from '../../DocComponent'; import { FieldView, FieldViewProps } from '../FieldView'; import './ChatBox.scss'; import MessageComponentBox from './MessageComponent'; -import { ASSISTANT_ROLE, AssistantMessage, AI_Document, convertToAIDocument, Citation, CHUNK_TYPE } from './types'; +import { ASSISTANT_ROLE, AssistantMessage, AI_Document, convertToAIDocument, Citation, CHUNK_TYPE, Chunk, getChunkType } from './types'; import { Vectorstore } from './vectorstore/VectorstoreUpload'; import { CollectionFreeFormDocumentView } from '../CollectionFreeFormDocumentView'; import { CollectionFreeFormView } from '../../collections/collectionFreeForm'; import { Agent } from './Agent'; import dotenv from 'dotenv'; -import { DocData } from '../../../../fields/DocSymbols'; +import { DocData, DocViews } from '../../../../fields/DocSymbols'; import { DocumentView } from '../DocumentView'; import { AnswerParser } from './AnswerParser'; +import { DocumentManager } from '../../../util/DocumentManager'; +import { UUID } from 'bson'; +import { v4 as uuidv4 } from 'uuid'; +import { aS } from '@fullcalendar/core/internal-common'; + dotenv.config(); @observer @@ -31,11 +36,10 @@ export class ChatBox extends ViewBoxAnnotatableComponent() { @observable isLoading: boolean = false; @observable isInitializing: boolean = true; @observable expandedScratchpadIndex: number | null = null; - @observable linked_docs_to_add: Doc[] = []; @observable inputValue: string = ''; - @observable private _visibleDocs: Doc[] = []; + @observable private currently_linked: Doc[] = []; private openai: OpenAI; - // private vectorstore_id: string; + private vectorstore_id: string; private documents: AI_Document[] = []; private _oldWheel: any; private vectorstore: Vectorstore; @@ -50,9 +54,14 @@ export class ChatBox extends ViewBoxAnnotatableComponent() { makeObservable(this); this.history = [{ role: ASSISTANT_ROLE.ASSISTANT, text_content: 'Welcome to the Document Analyser Assistant! Link a document or ask questions to get started.' }]; this.openai = this.initializeOpenAI(); - this.vectorstore = new Vectorstore(); + if (StrCast(this.dataDoc.vectorstore_id) == '') { + this.vectorstore_id = uuidv4(); + this.dataDoc.vectorstore_id = this.vectorstore_id; + } else { + this.vectorstore_id = StrCast(this.dataDoc.vectorstore_id); + } + this.vectorstore = new Vectorstore(this.vectorstore_id); this.agent = new Agent(this.vectorstore); // Initialize the Agent - reaction( () => this.history.map((msg: AssistantMessage) => ({ role: msg.role, text_content: msg.text_content, follow_up_questions: msg.follow_up_questions, citations: msg.citations })), serializableHistory => { @@ -61,19 +70,17 @@ export class ChatBox extends ViewBoxAnnotatableComponent() { ); } - @action - addDocsToVectorstore = async (visible_docs: Doc[]) => { - await this.vectorstore.addAIDocs(visible_docs); - this.isInitializing = false; + addDocsToVectorstore = async (linkedDocs: Doc[]) => { + await this.vectorstore.addAIDocs(linkedDocs); }; - @action - uploadNewDocument = async (newDoc: Doc) => { - const local_file_path: string = CsvCast(newDoc.data, PDFCast(newDoc.data)).url.pathname; - const { document_json } = await Networking.PostToServer('/createDocument', { file_path: local_file_path }); - this.documents.push(...document_json.map(convertToAIDocument)); - newDoc['ai_document'] = document_json; - }; + // @action + // uploadNewDocument = async (newDoc: Doc) => { + // const local_file_path: string = CsvCast(newDoc.data, PDFCast(newDoc.data)).url.pathname; + // const { document_json } = await Networking.PostToServer('/createDocument', { file_path: local_file_path }); + // this.documents.push(...document_json.map(convertToAIDocument)); + // //newDoc['ai_document'] = document_json; + // }; @action toggleToolLogs = (index: number) => { @@ -142,6 +149,24 @@ export class ChatBox extends ViewBoxAnnotatableComponent() { @action handleCitationClick = (citation: Citation) => { console.log('Citation clicked:', citation); + const currentLinkedDocs: Doc[] = this.linkedDocs; + const chunk_id = citation.chunk_id; + for (let doc of currentLinkedDocs) { + const doc_chunks: Chunk[] = JSON.parse(StrCast(doc.ai_document)).chunks; + const chunk_file_name = doc_chunks.find(chunk => chunk.id === chunk_id)?.metadata.file_path; + const doc_url = CsvCast(doc.data, PDFCast(doc.data)).url.pathname; + console.log('URL: ' + doc_url + ' Citation URL: ' + chunk_file_name); + //const ai_field_id = doc[this.Document[Id] + '_ai_field_id']; + if (chunk_file_name == doc_url) { + DocumentManager.Instance.showDocument(doc, {}, () => { + console.log(doc.data); + //look at context path for each docview and choose the doc view that has as + //its parent the same collection view the chatbox is in + const first_view = Array.from(doc[DocViews])[0]; + first_view.ComponentView?.search?.(citation.direct_text); + }); + } + } // You can implement additional functionality here, such as showing a modal with the full citation content }; @@ -183,6 +208,7 @@ export class ChatBox extends ViewBoxAnnotatableComponent() { componentDidMount() { this._props.setContentViewBox?.(this); + this.currently_linked = this.linkedDocs; if (this.dataDoc.data) { try { const storedHistory = JSON.parse(StrCast(this.dataDoc.data)); @@ -201,15 +227,15 @@ export class ChatBox extends ViewBoxAnnotatableComponent() { } } reaction( - () => this.visibleDocs, - visibleDocs => { - this._visibleDocs.push(...visibleDocs.filter(visibleDoc => !this._visibleDocs.includes(visibleDoc))); + () => this.linkedDocs, + linkedDocs => { + this.currently_linked.push(...linkedDocs.filter(linkedDoc => !this.currently_linked.includes(linkedDoc))); } ); observe( // right now this skips during initialization which is necessary because it would be blank // However, it will upload the same link twice when it is - this._visibleDocs, + this.currently_linked, change => { // observe pushes/splices on a user link DB 'data' field (should only happen for local changes) switch (change.type as any) { @@ -228,15 +254,13 @@ export class ChatBox extends ViewBoxAnnotatableComponent() { }, true ); - runInAction(() => { - if (!this._visibleDocs.length) { - this.isInitializing = false; - } - }); + if (this.isInitializing) { + this.isInitializing = false; + } } @computed - get visibleDocs() { + get linkedDocs() { //return (CollectionFreeFormDocumentView.from(this._props.DocumentView?.())?._props.parent as CollectionFreeFormView)?.childDocs.filter(doc => doc != this.Document) ?? []; return LinkManager.Instance.getAllRelatedLinks(this.Document) .map(d => DocCast(LinkManager.getOppositeAnchor(d, this.Document))) diff --git a/src/client/views/nodes/ChatBox/MessageComponent.tsx b/src/client/views/nodes/ChatBox/MessageComponent.tsx index 76faff10b..e18224405 100644 --- a/src/client/views/nodes/ChatBox/MessageComponent.tsx +++ b/src/client/views/nodes/ChatBox/MessageComponent.tsx @@ -35,7 +35,7 @@ const MessageComponentBox: React.FC = function ({ message height: '20px', borderRadius: '50%', border: 'none', - background: '#007bff', + background: '#ff6347', color: 'white', fontSize: '12px', fontWeight: 'bold', diff --git a/src/client/views/nodes/ChatBox/vectorstore/VectorstoreUpload.ts b/src/client/views/nodes/ChatBox/vectorstore/VectorstoreUpload.ts index d3b1cb4e7..3a889bff2 100644 --- a/src/client/views/nodes/ChatBox/vectorstore/VectorstoreUpload.ts +++ b/src/client/views/nodes/ChatBox/vectorstore/VectorstoreUpload.ts @@ -16,9 +16,10 @@ export class Vectorstore { private index!: Index; private cohere: CohereClient; private indexName: string = 'pdf-chatbot'; + private id: string; documents: AI_Document[] = []; - constructor() { + constructor(id: string) { const pineconeApiKey = process.env.PINECONE_API_KEY; if (!pineconeApiKey) { throw new Error('PINECONE_API_KEY is not defined.'); @@ -30,6 +31,7 @@ export class Vectorstore { this.cohere = new CohereClient({ token: process.env.COHERE_API_KEY, }); + this.id = id; this.initializeIndex(); } @@ -62,23 +64,28 @@ export class Vectorstore { } async addAIDoc(doc: Doc) { - if (doc[DocData]?.ai_document) { - this.documents.push(convertToAIDocument(JSON.parse(StrCast(doc[DocData].ai_document)))); + if (doc.ai_document) { + if (doc.ai_document === 'IN PROGRESS') { + console.log('Already in progress.'); + return; + } + this.documents.push(convertToAIDocument(JSON.parse(StrCast(doc.ai_document)))); console.log(`Document already added: ${doc.file_name}`); } else { + doc.ai_document = 'IN PROGRESS'; console.log(doc); console.log(PDFCast(doc.data)?.url?.pathname); console.log(CsvCast(doc.data)?.url?.pathname); const local_file_path: string = CsvCast(doc.data)?.url?.pathname ?? PDFCast(doc.data)?.url?.pathname; console.log('Local File Path:', local_file_path); - if (local_file_path) { + if (local_file_path !== undefined || local_file_path !== null || local_file_path !== '') { const { document_json } = await Networking.PostToServer('/createDocument', { file_path: local_file_path }); console.log('Document JSON:', document_json); const ai_document: AI_Document = convertToAIDocument(document_json); this.documents.push(ai_document); await this.indexDocument(ai_document); console.log(`Document added: ${ai_document.file_name}`); - doc[DocData].ai_document = JSON.stringify(document_json); + doc.ai_document = JSON.stringify(document_json); } } } @@ -94,7 +101,7 @@ export class Vectorstore { ({ id: chunk.id, values: chunk.values, - metadata: chunk.metadata as RecordMetadata, + metadata: { ...chunk.metadata, vectorestore_id: this.id } as RecordMetadata, }) as PineconeRecord ); await this.index.upsert(pineconeRecords); @@ -125,6 +132,9 @@ export class Vectorstore { const queryResponse: QueryResponse = await this.index.query({ vector: queryEmbedding, + filter: { + vectorstore_id: this.id, + }, topK, includeValues: true, includeMetadata: true, -- cgit v1.2.3-70-g09d2 From 0340c24eccce3d90c03934dec14d574128fb32ef Mon Sep 17 00:00:00 2001 From: "A.J. Shulman" Date: Wed, 17 Jul 2024 12:06:40 -0400 Subject: added image citation highlights --- src/client/views/nodes/ChatBox/AnswerParser.ts | 4 +- src/client/views/nodes/ChatBox/ChatBox.scss | 4 +- src/client/views/nodes/ChatBox/ChatBox.tsx | 128 +++++++++++---------- .../views/nodes/ChatBox/MessageComponent.tsx | 2 +- src/client/views/nodes/ChatBox/types.ts | 69 +++++------ .../nodes/ChatBox/vectorstore/VectorstoreUpload.ts | 16 ++- 6 files changed, 115 insertions(+), 108 deletions(-) (limited to 'src/client/views/nodes/ChatBox/MessageComponent.tsx') diff --git a/src/client/views/nodes/ChatBox/AnswerParser.ts b/src/client/views/nodes/ChatBox/AnswerParser.ts index f77d2261d..1162d46b0 100644 --- a/src/client/views/nodes/ChatBox/AnswerParser.ts +++ b/src/client/views/nodes/ChatBox/AnswerParser.ts @@ -1,4 +1,5 @@ import { ASSISTANT_ROLE, AssistantMessage, Citation, getChunkType } from './types'; +import { v4 as uuid } from 'uuid'; export class AnswerParser { static parse(xml: string): AssistantMessage { @@ -33,7 +34,8 @@ export class AnswerParser { direct_text: direct_text.trim(), type: getChunkType(type), chunk_id: chunk_id, - location: citationPlainStart, + text_location: citationPlainStart, + citation_id: uuid(), }); citationOffset += fullMatch.length; diff --git a/src/client/views/nodes/ChatBox/ChatBox.scss b/src/client/views/nodes/ChatBox/ChatBox.scss index 75171fe56..e39938c4f 100644 --- a/src/client/views/nodes/ChatBox/ChatBox.scss +++ b/src/client/views/nodes/ChatBox/ChatBox.scss @@ -174,7 +174,7 @@ $follow-up-hover-bg-color: #dee2e6; } } -.initializing-overlay { +.uploading-overlay { position: absolute; top: 0; left: 0; @@ -189,7 +189,7 @@ $follow-up-hover-bg-color: #dee2e6; z-index: 10; &::before { - content: 'Initializing...'; + content: 'Uploading Docs...'; font-weight: bold; } } diff --git a/src/client/views/nodes/ChatBox/ChatBox.tsx b/src/client/views/nodes/ChatBox/ChatBox.tsx index c7ae9a354..8b4a7bd0a 100644 --- a/src/client/views/nodes/ChatBox/ChatBox.tsx +++ b/src/client/views/nodes/ChatBox/ChatBox.tsx @@ -2,7 +2,7 @@ import { action, computed, makeObservable, observable, observe, reaction, runInA import { observer } from 'mobx-react'; import OpenAI, { ClientOptions } from 'openai'; import * as React from 'react'; -import { Doc } from '../../../../fields/Doc'; +import { Doc, DocListCast } from '../../../../fields/Doc'; import { CsvCast, DocCast, PDFCast, StrCast } from '../../../../fields/Types'; import { Networking } from '../../../Network'; import { DocumentType } from '../../../documents/DocumentTypes'; @@ -12,7 +12,7 @@ import { ViewBoxAnnotatableComponent } from '../../DocComponent'; import { FieldView, FieldViewProps } from '../FieldView'; import './ChatBox.scss'; import MessageComponentBox from './MessageComponent'; -import { ASSISTANT_ROLE, AssistantMessage, AI_Document, convertToAIDocument, Citation, CHUNK_TYPE, Chunk, getChunkType } from './types'; +import { ASSISTANT_ROLE, AssistantMessage, AI_Document, Citation, CHUNK_TYPE, Chunk, getChunkType } from './types'; import { Vectorstore } from './vectorstore/VectorstoreUpload'; import { CollectionFreeFormDocumentView } from '../CollectionFreeFormDocumentView'; import { CollectionFreeFormView } from '../../collections/collectionFreeForm'; @@ -26,6 +26,7 @@ import { UUID } from 'bson'; import { v4 as uuidv4 } from 'uuid'; import { aS } from '@fullcalendar/core/internal-common'; import { computeRect } from '@fullcalendar/core/internal'; +import { DocUtils } from '../../../documents/DocUtils'; dotenv.config(); @@ -35,7 +36,7 @@ export class ChatBox extends ViewBoxAnnotatableComponent() { @observable.deep current_message: AssistantMessage | undefined = undefined; @observable isLoading: boolean = false; - @observable isInitializing: boolean = false; + @observable isUploadingDocs: boolean = false; @observable expandedScratchpadIndex: number | null = null; @observable inputValue: string = ''; @observable private linked_docs_to_add: ObservableSet = observable.set(); @@ -75,9 +76,7 @@ export class ChatBox extends ViewBoxAnnotatableComponent() { @action addDocToVectorstore = async (newLinkedDoc: Doc) => { - this.isInitializing = true; await this.vectorstore.addAIDoc(newLinkedDoc); - this.isInitializing = false; }; // @action @@ -158,75 +157,76 @@ export class ChatBox extends ViewBoxAnnotatableComponent() { const currentLinkedDocs: Doc[] = this.linkedDocs; const chunk_id = citation.chunk_id; for (let doc of currentLinkedDocs) { - //console.log(JSON.parse(StrCast(doc.chunk_simpl))); - console.log(JSON.stringify(StrCast(doc.chunk_simpl))); - const doc_chunk_simpl = JSON.parse(StrCast(doc.chunk_simpl) as string); - console.log(doc_chunk_simpl); - const text_chunks = doc_chunk_simpl.text_chunks as [{ chunk_id: string; start_page: number; end_page: number }] | []; - const image_chunks = doc_chunk_simpl.image_chunks as [{ chunk_id: string; location: string; page: number; page_width: number; page_height: number }] | []; - - const found_text_chunk = text_chunks.find(chunk => chunk.chunk_id === chunk_id); - if (found_text_chunk) { - const doc_url = CsvCast(doc.data, PDFCast(doc.data)).url.pathname; - console.log('URL: ' + doc_url); - - //const ai_field_id = doc[this.Document[Id] + '_ai_field_id']; - DocumentManager.Instance.showDocument(doc, { willZoomCentered: true }, () => { - console.log(doc.data); - //look at context path for each docview and choose the doc view that has as - //its parent the same collection view the chatbox is in - const first_view = Array.from(doc[DocViews])[0]; - first_view.ComponentView?.search?.(citation.direct_text); - }); - } - - const found_image_chunk = image_chunks.find(chunk => chunk.chunk_id === chunk_id); - if (found_image_chunk) { - const location_string: string = found_image_chunk.location; + if (doc.chunk_simpl) { + //console.log(JSON.parse(StrCast(doc.chunk_simpl))); + const doc_chunk_simpl = JSON.parse(StrCast(doc.chunk_simpl)); + console.log(doc_chunk_simpl); + const text_chunks = doc_chunk_simpl.text_chunks as [{ chunk_id: string; start_page: number; end_page: number }] | []; + const image_chunks = doc_chunk_simpl.image_chunks as [{ chunk_id: string; location: string; page: number }] | []; + + const found_text_chunk = text_chunks.find(chunk => chunk.chunk_id === chunk_id); + if (found_text_chunk) { + const doc_url = CsvCast(doc.data, PDFCast(doc.data)).url.pathname; + console.log('URL: ' + doc_url); + + //const ai_field_id = doc[this.Document[Id] + '_ai_field_id']; + DocumentManager.Instance.showDocument(doc, { willZoomCentered: true }, () => { + console.log(doc.data); + //look at context path for each docview and choose the doc view that has as + //its parent the same collection view the chatbox is in + const first_view = Array.from(doc[DocViews])[0]; + first_view.ComponentView?.search?.(citation.direct_text); + }); + } - // Extract variables from location_string - const values = location_string.replace(/[\[\]]/g, '').split(','); + const found_image_chunk = image_chunks.find(chunk => chunk.chunk_id === chunk_id); + if (found_image_chunk) { + const location_string: string = found_image_chunk.location; - // Ensure we have exactly 4 values - if (values.length !== 4) { - console.error('Location string must contain exactly 4 numbers'); - return; // or handle this error as appropriate - } + // Extract variables from location_string + const values = location_string.replace(/[\[\]]/g, '').split(','); - const x1 = parseInt(values[0]) * (parseInt(StrCast(doc.width)) / found_image_chunk.page_width); - const y1 = parseInt(values[1]) * (parseInt(StrCast(doc.height)) / found_image_chunk.page_height); - const x2 = parseInt(values[2]) * (parseInt(StrCast(doc.width)) / found_image_chunk.page_width); - const y2 = parseInt(values[3]) * (parseInt(StrCast(doc.height)) / found_image_chunk.page_height); + // Ensure we have exactly 4 values + if (values.length !== 4) { + console.error('Location string must contain exactly 4 numbers'); + return; // or handle this error as appropriate + } - // Parse values to numbers - // const [x1, y1, x2, y2] = values.map(Number); + const x1 = parseFloat(values[0]) * Doc.NativeWidth(doc); + const y1 = parseFloat(values[1]) * Doc.NativeHeight(doc); + const x2 = parseFloat(values[2]) * Doc.NativeWidth(doc); + const y2 = parseFloat(values[3]) * Doc.NativeHeight(doc); - // Check if any parsing resulted in NaN - if ([x1, y1, x2, y2].some(isNaN)) { - console.error('All values in location string must be valid numbers'); - return; // or handle this error as appropriate - } + const annotationKey = Doc.LayoutFieldKey(doc) + '_annotations'; - const highlight_doc = Docs.Create.FreeformDocument([], { - x: x1, - y: y1, - _width: x2 - x1, - _height: y2 - y1, - backgroundColor: 'rgba(255, 255, 0, 0.5)', - }); + const existingDoc = DocListCast(doc[DocData][annotationKey]).find(d => d.citation_id === citation.citation_id); + const highlight_doc = existingDoc ?? this.createImageCitationHighlight(x1, y1, x2, y2, citation, annotationKey, doc); - Doc.AddDocToList(doc[DocData], Doc.LayoutFieldKey(doc) + '_annotations', highlight_doc); - highlight_doc.annotationOn = doc; - Doc.SetContainer(highlight_doc, doc); - DocumentManager.Instance.showDocument(highlight_doc, { willZoomCentered: true }, () => {}); + DocumentManager.Instance.showDocument(highlight_doc, { willZoomCentered: true }, () => {}); + } } } // You can implement additional functionality here, such as showing a modal with the full citation content }; + createImageCitationHighlight = (x1: number, y1: number, x2: number, y2: number, citation: Citation, annotationKey: string, pdfDoc: Doc): Doc => { + const highlight_doc = Docs.Create.FreeformDocument([], { + x: x1, + y: y1, + _width: x2 - x1, + _height: y2 - y1, + backgroundColor: 'rgba(255, 255, 0, 0.5)', + }); + highlight_doc[DocData].citation_id = citation.citation_id; + Doc.AddDocToList(pdfDoc[DocData], annotationKey, highlight_doc); + highlight_doc.annotationOn = pdfDoc; + Doc.SetContainer(highlight_doc, pdfDoc); + return highlight_doc; + }; + // @action // uploadLinks = async (linkedDocs: Doc[]) => { - // if (this.isInitializing) { + // if (this.isUploadingDocs) { // console.log('Initialization in progress, upload aborted.'); // return; // } @@ -293,7 +293,13 @@ export class ChatBox extends ViewBoxAnnotatableComponent() { observe(this.linked_docs_to_add, change => { if (change.type === 'add') { + runInAction(() => { + this.isUploadingDocs = true; + }); this.addDocToVectorstore(change.newValue); + runInAction(() => { + this.isUploadingDocs = false; + }); } else if (change.type === 'delete') { console.log('Deleted docs: ', change.oldValue); } @@ -358,7 +364,7 @@ export class ChatBox extends ViewBoxAnnotatableComponent() { render() { return (
- {this.isInitializing &&
Initializing...
} + {this.isUploadingDocs &&
}
{ diff --git a/src/client/views/nodes/ChatBox/MessageComponent.tsx b/src/client/views/nodes/ChatBox/MessageComponent.tsx index e18224405..9f3dee990 100644 --- a/src/client/views/nodes/ChatBox/MessageComponent.tsx +++ b/src/client/views/nodes/ChatBox/MessageComponent.tsx @@ -20,7 +20,7 @@ const MessageComponentBox: React.FC = function ({ message let lastIndex = 0; message.citations.forEach((citation, idx) => { - const location = citation.location; + const location = citation.text_location; const textBefore = content.slice(lastIndex, location); const citationButton = ( ); - parts.push(textBefore, citationButton); + parts.push(citationButton); lastIndex = location; }); - parts.push(content.slice(lastIndex)); + parts.push({content.slice(lastIndex)}); return parts; }; diff --git a/src/client/views/nodes/ChatBox/prompts.ts b/src/client/views/nodes/ChatBox/prompts.ts index c011d65b7..d5eb99cb2 100644 --- a/src/client/views/nodes/ChatBox/prompts.ts +++ b/src/client/views/nodes/ChatBox/prompts.ts @@ -19,6 +19,8 @@ export function getReactPrompt(tools: Tool[], chatHistory: string): string { Your available actions are: ***** ${toolDescriptions} + ***** + no_tool: Use this when no external tool or action is required to answer the question. ********** Example: You will be called with: @@ -74,13 +76,59 @@ export function getReactPrompt(tools: Tool[], chatHistory: string): string { + ********** + Example: +You will be called with: +What is 2 + 2? + +You will then output: + + This is a simple arithmetic question that doesn't require any external tool. + no_tool + + +THEN PAUSE AND DO NOT OUTPUT ANYTHING. + +You will be called again with this: + + { + "no_tool": { + "name": "no_tool", + "description": "Use when no external tool or action is required", + "parameters": [] + } + } + + +You will then output: + + + + +THEN PAUSE AND DO NOT OUTPUT ANYTHING. + +You will then be called again with this: +No tool used. Proceed with answering the question. + +You then output: + + + 2 + 2 equals 4. + + What is 3 + 3? + Can you explain the concept of addition? + What is 2 * 2? + + + ********** Here is the history of your conversation with the user (all loop steps are ommitted, so it is just the user query and final answer): ${chatHistory} Use context from the past conversation if necessary. ********** If the response is inadequate, repeat the loop, either trying a different tool or changing the parameters for the action input. - + ********** + !!!IMPORTANT Only use tools when they are absolutely necessary to answer the question. If you have enough information or knowledge to answer the question without using a tool, use the "no_tool" action instead. !!!IMPORTANT When you have an Answer, Write your entire response inside an element (which itself should be inside the step element for the current step). After you finish the answer, provide an array of 3 follow-up questions inside a array. These should relate to the query and the response and should aim to help the user better understand whatever they are looking for. ********** !!!IMPORTANT Every response, provide in full parsable and valid XML with the root element being the step number (e.g. ), iterated every time you output something new. diff --git a/src/client/views/nodes/ChatBox/tools/CollectionTool.ts b/src/client/views/nodes/ChatBox/tools/CollectionTool.ts deleted file mode 100644 index e69de29bb..000000000 diff --git a/src/client/views/nodes/ChatBox/tools/CreateCollectionTool.ts b/src/client/views/nodes/ChatBox/tools/CreateCollectionTool.ts new file mode 100644 index 000000000..26ac0d7cc --- /dev/null +++ b/src/client/views/nodes/ChatBox/tools/CreateCollectionTool.ts @@ -0,0 +1,35 @@ +import { DocCast } from '../../../../../fields/Types'; +import { DocServer } from '../../../../DocServer'; +import { Docs } from '../../../../documents/Documents'; +import { DocumentView } from '../../DocumentView'; +import { OpenWhere } from '../../OpenWhere'; +import { BaseTool } from './BaseTool'; + +export class GetDocsContentTool extends BaseTool<{ title: string; document_ids: string[] }> { + private _docView: DocumentView; + constructor(docView: DocumentView) { + super( + 'retrieveDocs', + 'Retrieves the contents of all Documents that the user is interacting with in Dash ', + { + title: { + type: 'string', + description: 'the title of the collection that you will be making', + required: 'true', + }, + }, + 'Provide a mathematical expression to calculate that would work with JavaScript eval().', + 'Runs a calculation and returns the number - uses JavaScript so be sure to use floating point syntax if necessary' + ); + this._docView = docView; + } + + async execute(args: { title: string; document_ids: string[] }): Promise { + // Note: Using eval() can be dangerous. Consider using a safer alternative. + const docs = args.document_ids.map(doc_id => DocCast(DocServer.GetCachedRefField(doc_id))); + const collection = Docs.Create.FreeformDocument(docs, { title: args.title }); + this._docView._props.addDocTab(collection, OpenWhere.addRight); //in future, create popup prompting user where to add + return [{ type: 'text', text: 'Collection created in Dash called ' + args.title }]; + } +} +//export function create_collection(docView: DocumentView, document_ids: string[], title: string): string {} diff --git a/src/client/views/nodes/ChatBox/tools/GetDocsTool.ts b/src/client/views/nodes/ChatBox/tools/GetDocsTool.ts new file mode 100644 index 000000000..f970ca8ee --- /dev/null +++ b/src/client/views/nodes/ChatBox/tools/GetDocsTool.ts @@ -0,0 +1,29 @@ +import { DocCast } from '../../../../../fields/Types'; +import { DocServer } from '../../../../DocServer'; +import { Docs } from '../../../../documents/Documents'; +import { DocumentView } from '../../DocumentView'; +import { OpenWhere } from '../../OpenWhere'; +import { BaseTool } from './BaseTool'; + +export class GetDocsTool extends BaseTool<{ title: string; document_ids: string[] }> { + private _docView: DocumentView; + constructor(docView: DocumentView) { + super( + 'retrieveDocs', + 'Retrieves the contents of all Documents that the user is interacting with in Dash', + {}, + 'No need to provide anything. Just run the tool and it will retrieve the contents of all Documents that the user is interacting with in Dash.', + 'Returns the the documents in Dash in JSON form. This will include the title of the document, the location in the FreeFormDocument, and the content of the document, any applicable data fields, the layout of the document, etc.' + ); + this._docView = docView; + } + + async execute(args: { title: string; document_ids: string[] }): Promise { + // Note: Using eval() can be dangerous. Consider using a safer alternative. + const docs = args.document_ids.map(doc_id => DocCast(DocServer.GetCachedRefField(doc_id))); + const collection = Docs.Create.FreeformDocument(docs, { title: args.title }); + this._docView._props.addDocTab(collection, OpenWhere.addRight); //in future, create popup prompting user where to add + return [{ type: 'text', text: 'Collection created in Dash called ' + args.title }]; + } +} +//export function create_collection(docView: DocumentView, document_ids: string[], title: string): string {} diff --git a/src/client/views/nodes/ChatBox/tools/NoTool.ts b/src/client/views/nodes/ChatBox/tools/NoTool.ts new file mode 100644 index 000000000..1f0830a77 --- /dev/null +++ b/src/client/views/nodes/ChatBox/tools/NoTool.ts @@ -0,0 +1,18 @@ +// tools/NoTool.ts +import { BaseTool } from './BaseTool'; + +export class NoTool extends BaseTool<{}> { + constructor() { + super( + 'no_tool', + 'Use this when no external tool or action is required to answer the question.', + {}, + 'When using the "no_tool" action, simply provide an empty element. The observation will always be "No tool used. Proceed with answering the question."', + 'Use when no external tool or action is required to answer the question.' + ); + } + + async execute(args: {}): Promise { + return [{ type: 'text', text: 'No tool used. Proceed with answering the question.' }]; + } +} -- cgit v1.2.3-70-g09d2 From 834ca4e21fead079e681b963e2d533d93a53cb91 Mon Sep 17 00:00:00 2001 From: "A.J. Shulman" Date: Mon, 22 Jul 2024 10:49:02 -0400 Subject: fixing summaries by having them part of the ReAct prompt --- src/client/views/nodes/ChatBox/Agent.ts | 8 +- src/client/views/nodes/ChatBox/ChatBox.tsx | 80 +----- .../views/nodes/ChatBox/MessageComponent.tsx | 11 +- src/client/views/nodes/ChatBox/prompts.ts | 295 +++++++++++---------- src/client/views/nodes/ChatBox/tools/RAGTool.ts | 11 +- 5 files changed, 172 insertions(+), 233 deletions(-) (limited to 'src/client/views/nodes/ChatBox/MessageComponent.tsx') diff --git a/src/client/views/nodes/ChatBox/Agent.ts b/src/client/views/nodes/ChatBox/Agent.ts index a3b1d083c..d494928f9 100644 --- a/src/client/views/nodes/ChatBox/Agent.ts +++ b/src/client/views/nodes/ChatBox/Agent.ts @@ -19,25 +19,27 @@ export class Agent { private interMessages: AgentMessage[] = []; private vectorstore: Vectorstore; private _history: () => string; + private _summaries: () => string; constructor(_vectorstore: Vectorstore, summaries: () => string, history: () => string) { this.client = new OpenAI({ apiKey: process.env.OPENAI_KEY, dangerouslyAllowBrowser: true }); this.vectorstore = _vectorstore; this._history = history; + this._summaries = summaries; this.tools = { wikipedia: new WikipediaTool(), calculate: new CalculateTool(), - rag: new RAGTool(this.vectorstore, summaries), + rag: new RAGTool(this.vectorstore), no_tool: new NoTool(), }; } - async askAgent(question: string, maxTurns: number = 8): Promise { + async askAgent(question: string, maxTurns: number = 10): Promise { console.log(`Starting query: ${question}`); this.messages.push({ role: 'user', content: question }); const chatHistory = this._history(); console.log(`Chat history: ${chatHistory}`); - const systemPrompt = getReactPrompt(Object.values(this.tools), chatHistory); + const systemPrompt = getReactPrompt(Object.values(this.tools), this._summaries, chatHistory); console.log(`System prompt: ${systemPrompt}`); this.interMessages = [{ role: 'system', content: systemPrompt }]; diff --git a/src/client/views/nodes/ChatBox/ChatBox.tsx b/src/client/views/nodes/ChatBox/ChatBox.tsx index e3a164b3e..49c9b3292 100644 --- a/src/client/views/nodes/ChatBox/ChatBox.tsx +++ b/src/client/views/nodes/ChatBox/ChatBox.tsx @@ -4,7 +4,6 @@ import OpenAI, { ClientOptions } from 'openai'; import * as React from 'react'; import { Doc, DocListCast } from '../../../../fields/Doc'; import { CsvCast, DocCast, PDFCast, StrCast } from '../../../../fields/Types'; -import { Networking } from '../../../Network'; import { DocumentType } from '../../../documents/DocumentTypes'; import { Docs } from '../../../documents/Documents'; import { LinkManager } from '../../../util/LinkManager'; @@ -14,19 +13,12 @@ import './ChatBox.scss'; import MessageComponentBox from './MessageComponent'; import { ASSISTANT_ROLE, AssistantMessage, AI_Document, Citation, CHUNK_TYPE, Chunk, getChunkType } from './types'; import { Vectorstore } from './vectorstore/VectorstoreUpload'; -import { CollectionFreeFormDocumentView } from '../CollectionFreeFormDocumentView'; -import { CollectionFreeFormView } from '../../collections/collectionFreeForm'; import { Agent } from './Agent'; import dotenv from 'dotenv'; import { DocData, DocViews } from '../../../../fields/DocSymbols'; -import { DocumentView } from '../DocumentView'; import { AnswerParser } from './AnswerParser'; import { DocumentManager } from '../../../util/DocumentManager'; -import { UUID } from 'bson'; import { v4 as uuidv4 } from 'uuid'; -import { aS } from '@fullcalendar/core/internal-common'; -import { computeRect } from '@fullcalendar/core/internal'; -import { DocUtils } from '../../../documents/DocUtils'; dotenv.config(); @@ -78,14 +70,6 @@ export class ChatBox extends ViewBoxAnnotatableComponent() { await this.vectorstore.addAIDoc(newLinkedDoc); }; - // @action - // uploadNewDocument = async (newDoc: Doc) => { - // const local_file_path: string = CsvCast(newDoc.data, PDFCast(newDoc.data)).url.pathname; - // const { document_json } = await Networking.PostToServer('/createDocument', { file_path: local_file_path }); - // this.documents.push(...document_json.map(convertToAIDocument)); - // //newDoc['ai_document'] = document_json; - // }; - @action toggleToolLogs = (index: number) => { this.expandedScratchpadIndex = this.expandedScratchpadIndex === index ? null : index; @@ -105,10 +89,6 @@ export class ChatBox extends ViewBoxAnnotatableComponent() { } }; - // getAssistantResponse() { - // return Docs.Create.MessageDocument(text, {}); - // } - @action askGPT = async (event: React.FormEvent): Promise => { event.preventDefault(); @@ -223,42 +203,6 @@ export class ChatBox extends ViewBoxAnnotatableComponent() { return highlight_doc; }; - // @action - // uploadLinks = async (linkedDocs: Doc[]) => { - // if (this.isUploadingDocs) { - // console.log('Initialization in progress, upload aborted.'); - // return; - // } - // const urls: string[] = linkedDocs.map(doc => CsvCast(doc.data, PDFCast(doc.data)).url.pathname); - // const csvUrls: string[] = urls.filter(url => url.endsWith('.csv')); - // console.log(this.assistantID, this.threadID, urls); - - // await Networking.PostToServer('/uploadPDFs', { file_path: urls[0] }); - - // // linkedDocs.forEach((doc, i) => { - // // doc[this.Document[Id] + '_ai_field_id'] = openaiFileIds[i]; - // // console.log('AI Field ID: ' + openaiFileIds[i]); - // // }); - - // // if (csvUrls.length > 0) { - // // for (let i = 0; i < csvUrls.length; i++) { - // // this.linkedCsvIDs.push(openaiFileIds[urls.indexOf(csvUrls[i])]); - // // } - // // console.log('linked csvs:' + this.linkedCsvIDs); - // // await this.openai.beta.assistants.update(this.assistantID, { - // // tools: [{ type: 'file_search' }, { type: 'code_interpreter' }], - // // tool_resources: { - // // file_search: { - // // vector_store_ids: [this.vectorStoreID], - // // }, - // // code_interpreter: { - // // file_ids: this.linkedCsvIDs, - // // }, - // // }, - // // }); - // // } - // }; - componentDidMount() { this._props.setContentViewBox?.(this); if (this.dataDoc.data) { @@ -307,20 +251,8 @@ export class ChatBox extends ViewBoxAnnotatableComponent() { }); } - // case 'splice': - // if ((change as any).addedCount > 0) { - // // maybe check here if its already in the urls datadoc array so doesn't add twice - // console.log((change as any).added as Doc[]); - // this.addDocsToVectorstore((change as any).added as Doc[]); - // } - // // (change as any).removed.forEach((link: any) => remLinkFromDoc(toRealField(link))); - // break; - // case 'update': // let oldValue = change.oldValue; - // default: - @computed get linkedDocs() { - //return (CollectionFreeFormDocumentView.from(this._props.DocumentView?.())?._props.parent as CollectionFreeFormView)?.childDocs.filter(doc => doc != this.Document) ?? []; return LinkManager.Instance.getAllRelatedLinks(this.Document) .map(d => DocCast(LinkManager.getOppositeAnchor(d, this.Document))) .map(d => DocCast(d?.annotationOn, d)) @@ -334,6 +266,7 @@ export class ChatBox extends ViewBoxAnnotatableComponent() { .map(d => DocCast(LinkManager.getOppositeAnchor(d, this.Document))) .map(d => DocCast(d?.annotationOn, d)) .filter(d => d) + .filter(d => d.summary) .map((doc, index) => `${index + 1}) ${doc.summary}`) .join('\n') + '\n' ); @@ -375,19 +308,8 @@ export class ChatBox extends ViewBoxAnnotatableComponent() { }}>
{this.history.map((message, index) => ( - // ))} - {this.current_message && ( - - )}
diff --git a/src/client/views/nodes/ChatBox/MessageComponent.tsx b/src/client/views/nodes/ChatBox/MessageComponent.tsx index fb4a56bc3..56fde8bb2 100644 --- a/src/client/views/nodes/ChatBox/MessageComponent.tsx +++ b/src/client/views/nodes/ChatBox/MessageComponent.tsx @@ -1,7 +1,7 @@ import React from 'react'; import { observer } from 'mobx-react'; -import ReactMarkdown from 'react-markdown'; import { AssistantMessage, Citation } from './types'; +import Markdown from 'react-markdown'; interface MessageComponentProps { message: AssistantMessage; @@ -14,7 +14,7 @@ interface MessageComponentProps { const MessageComponentBox: React.FC = function ({ message, index, onFollowUpClick, onCitationClick, updateMessageCitations }) { const renderContent = (content: string) => { if (!message.citations || message.citations.length === 0) { - return {content}; + return content; } const parts = []; @@ -23,10 +23,9 @@ const MessageComponentBox: React.FC = function ({ message message.citations.forEach((citation, idx) => { const location = citation.text_location; const textBefore = content.slice(lastIndex, location); - parts.push({textBefore}); const citationButton = ( ); - parts.push(citationButton); + parts.push(textBefore, citationButton); lastIndex = location; }); - parts.push({content.slice(lastIndex)}); + parts.push(content.slice(lastIndex)); return parts; }; diff --git a/src/client/views/nodes/ChatBox/prompts.ts b/src/client/views/nodes/ChatBox/prompts.ts index 5c353337f..4109ffc17 100644 --- a/src/client/views/nodes/ChatBox/prompts.ts +++ b/src/client/views/nodes/ChatBox/prompts.ts @@ -2,147 +2,170 @@ import { Tool } from './types'; -export function getReactPrompt(tools: Tool[], chatHistory: string): string { +export function getReactPrompt(tools: Tool[], summaries: () => string, chatHistory: string): string { const toolDescriptions = tools.map(tool => `${tool.name}:\n${tool.briefSummary}`).join('\n*****\n'); return ` - -You are an advanced AI assistant with access to various tools. Your task is to answer user queries accurately and efficiently. Follow these instructions meticulously: - -1. Operation Loop: - You operate in a loop of Thought, Action, Observation, and Answer. Each iteration is numbered (step1, step2, etc.). - -2. Response Structure: - a. Enclose each step in numbered XML tags: , , etc. - b. Within each step, use the following tags as needed: - - Your reasoning process - - The tool you choose to use - - Parameters for the chosen tool - - Your final response (only in the last step) - -3. Detailed Process: - a. Analyze the user's query carefully. - b. Determine if a tool is necessary or if you can answer directly. - c. If a tool is needed: - - Select the most appropriate tool. - - Use to specify the tool. - - Wait for action rules to be provided. - - Provide tool parameters in . - - Wait for the observation from the tool. - d. If no tool is needed, use the 'no_tool' action. - e. Based on observations or your knowledge, formulate your answer. - f. Provide the final answer in the tag, including follow-up questions. - -4. Available Tools: -${toolDescriptions} - no_tool: Use when no external tool is required to answer the question. - -5. Critical Rules: - - Use tools ONLY when absolutely necessary for accurate answers. - - Ensure ALL XML is valid, properly nested, and complete. - - ALWAYS pause after and tags for system processing. - - If the initial answer is inadequate, iterate through additional steps to refine it. - - Utilize context from past conversations when relevant (provided in Chat History). - - ALWAYS include your final response within a single tag. - -6. Answer Format: - Your final tag must contain: - - The complete answer to the user's query. - - An array of EXACTLY 3 follow-up questions within tags. - -7. Example Interaction: -What is the population of Tokyo, and how does it compare to New York City? - - - To answer this question accurately, I need to look up the current population figures for both Tokyo and New York City. I'll use the Wikipedia tool for this information. - wikipedia - - -[SYSTEM PAUSE] - - - { - "wikipedia": { - "name": "wikipedia", - "description": "Search Wikipedia and return a summary", - "parameters": [ - { - "title": { - "type": "string", - "description": "The title of the Wikipedia article to search", - "required": "true" - } + + You are an advanced AI assistant with access to various tools. Your task is to answer user queries accurately and efficiently. Follow these instructions meticulously: + + 1. Operation Loop: + You operate in a loop of Thought, Action, (STOP), *Action Rules*, Action Input, (STOP), *Observation*, and Answer. Each iteration is numbered (step1, step2, etc.). + + 2. Response Structure: + a. Enclose each step in numbered XML tags: , , etc. + b. Within each step, use the following tags as needed: + - Your reasoning process + - The tool you choose to use + - Parameters for the chosen tool + - Your final response (only in the last step) + c. Stop after and tags for system processing. You will receive a user response after each time you stop in the form of either action rules or an observation. + + 3. Detailed Process: + a. Analyze the user's query carefully. + b. Determine if a tool is necessary or if you can answer directly. + c. If a tool is needed: + c.1) Select the most appropriate tool. + c.2) Use to specify the tool. + c.3) End your response in order for action rules to be provided. + c.4) Based on the action rules, provide tool parameters in . + c.5) End your response again and wait for the observation from the tool. + d. If no tool is needed, use the 'no_tool' action but still follow the same structure. + e. Use the retrieval (RAG) tool ANYTIME the question may potentially (even if you are not sure) relate to one of the user's documents. Here are the summaries of the user's documents: + ${summaries()} + f. Based on observations or your knowledge, formulate your answer. + g. Provide the final answer in the tag, including follow-up questions. + + 4. Available Tools: + ${toolDescriptions} + no_tool: Use when no external tool is required to answer the question. If a user document may relate to the query, do not use, and instead, use the RAG tool (even if unsure). + + 5. !!!Critical Rules!!!: + - Use tools ONLY when absolutely necessary for accurate answers (except when query may relate to user documents, then use RAG tool ALWAYS to start). + - Only provide one step at a time and only write assistant steps. Do not skip steps. Do not provide multiple steps at once. Decide the step that you will provide based on previous steps taken. + - Ensure ALL XML is valid, properly nested, and complete. + - ALWAYS stop after and tags. + - If the initial answer is inadequate, iterate through additional steps to refine it. + - Utilize context from past conversations when relevant (provided in Chat History). + - ALWAYS include your final response within a single tag. + + 6. Answer Format: + Your final tag must contain: + - The complete answer to the user's query. + - An array of EXACTLY 3 follow-up questions within tags. + + 7. Example Interaction (YOU ONLY OUTPUT THE ASSISTANT STEPS): + SYSTEM: + + ***SYSTEM MESSAGE ELIDED*** + + + USER: + + What is the population of Tokyo, and how does it compare to New York City? + + + ASSISTANT: + + To answer this question accurately, I need to look up the current population figures for both Tokyo and New York City. I'll use the Wikipedia tool for this information. + wikipedia + + + USER: + + + { + "wikipedia": { + "name": "wikipedia", + "description": "Search Wikipedia and return a summary", + "parameters": [ + { + "title": { + "type": "string", + "description": "The title of the Wikipedia article to search", + "required": "true" + } + } + ] } - ] - } - } - - - - - Tokyo - - - -[SYSTEM PAUSE] - -Tokyo is the capital and most populous prefecture of Japan. The Tokyo metropolitan area, which includes Tokyo and several surrounding prefectures, is the world's most populous metropolitan area with an estimated 37.468 million residents as of 2018. - - - Now that I have information about Tokyo, I need to get information about New York City for comparison. - wikipedia - - -[SYSTEM PAUSE] - - - { - "wikipedia": { - "name": "wikipedia", - "description": "Search Wikipedia and return a summary", - "parameters": [ - { - "title": { - "type": "string", - "description": "The title of the Wikipedia article to search", - "required": "true" - } + } + + + + ASSISTANT: + + + Tokyo + + + + USER: + + Tokyo is the capital and most populous prefecture of Japan. The Tokyo metropolitan area, which includes Tokyo and several surrounding prefectures, is the world's most populous metropolitan area with an estimated 37.468 million residents as of 2018. + + + ASSISTANT: + + Now that I have information about Tokyo, I need to get information about New York City for comparison. + wikipedia + + + USER: + + + { + "wikipedia": { + "name": "wikipedia", + "description": "Search Wikipedia and return a summary", + "parameters": [ + { + "title": { + "type": "string", + "description": "The title of the Wikipedia article to search", + "required": "true" + } + } + ] } - ] - } - } - - - - - New York City - - - -[SYSTEM PAUSE] - -New York City is the most populous city in the United States. With an estimated 2020 population of 8,804,190 distributed over 300.46 square miles (778.2 km2), New York City is also the most densely populated major city in the United States. - - - Now that I have the population information for both Tokyo and New York City, I can provide a comprehensive answer comparing the two. - - The population of Tokyo metropolitan area is approximately 37.468 million (as of 2018), while New York City has a population of about 8.8 million (as of 2020). Tokyo's population is significantly larger, more than four times that of New York City. It's important to note that the Tokyo figure refers to the broader metropolitan area, while the New York City figure is for the city proper. Even accounting for this difference, Tokyo remains substantially more populous than New York City. - - - What factors contribute to Tokyo's significantly larger population compared to New York City? - How do the population densities of Tokyo and New York City compare? - What challenges do these megacities face due to their large populations? - - - - -8. Chat History: -${chatHistory} - -Remember to use this history for context when appropriate. - -Now, process the user's query and provide your response following the format and rules outlined above. Ensure your final answer is comprehensive and entirely contained within a single tag. -`; + } + + + + ASSISTANT: + + + New York City + + + + USER: + + New York City is the most populous city in the United States. With an estimated 2020 population of 8,804,190 distributed over 300.46 square miles (778.2 km2), New York City is also the most densely populated major city in the United States. + + + ASSISTANT: + + Now that I have the population information for both Tokyo and New York City, I can provide a comprehensive answer comparing the two. + + The population of Tokyo metropolitan area is approximately 37.468 million (as of 2018), while New York City has a population of about 8.8 million (as of 2020). Tokyo's population is significantly larger, more than four times that of New York City. It's important to note that the Tokyo figure refers to the broader metropolitan area, while the New York City figure is for the city proper. Even accounting for this difference, Tokyo remains substantially more populous than New York City. + + + What factors contribute to Tokyo's significantly larger population compared to New York City? + How do the population densities of Tokyo and New York City compare? + What challenges do these megacities face due to their large populations? + + + + + 8. Chat History: + ${chatHistory} + + Remember to use this history for context when appropriate. + + Now, process the user's query and provide your response following the format and rules outlined above. Ensure your final answer is comprehensive and entirely contained within a single tag. + + !!!IMPORTANT Very importantly, even if you use no tool or have an answer, follow the structure and output ONE step at a time. You will be provided with user steps and will output the appropriate single assistant step. FOLLOW THE STRUCTURE; Do not skip to the answer right away or skip steps. +`; } export function getSummarizedChunksPrompt(chunks: string): string { diff --git a/src/client/views/nodes/ChatBox/tools/RAGTool.ts b/src/client/views/nodes/ChatBox/tools/RAGTool.ts index 0a4529974..5bc31dbab 100644 --- a/src/client/views/nodes/ChatBox/tools/RAGTool.ts +++ b/src/client/views/nodes/ChatBox/tools/RAGTool.ts @@ -5,10 +5,7 @@ import * as fs from 'fs'; import { Networking } from '../../../../Network'; export class RAGTool extends BaseTool<{ hypothetical_document_chunk: string }> { - constructor( - private vectorstore: Vectorstore, - summaries: () => string - ) { + constructor(private vectorstore: Vectorstore) { super( 'rag', 'Perform a RAG search on user documents', @@ -43,11 +40,7 @@ export class RAGTool extends BaseTool<{ hypothetical_document_chunk: string }> { `Performs a RAG (Retrieval-Augmented Generation) search on user documents and returns a set of document chunks (either images or text) that can be used to provide a grounded response based on - user documents - - !!!IMPORTANT Use the RAG tool ANYTIME the question may potentially (even if you are not sure) relate to one of the user's documents. - Here are the summaries of the user's documents: - ${summaries()}` + user documents` ); } -- cgit v1.2.3-70-g09d2 From e7b7f7000534200d75f3519ffb13d5b22dbc5481 Mon Sep 17 00:00:00 2001 From: "A.J. Shulman" Date: Mon, 22 Jul 2024 12:19:27 -0400 Subject: attempting new RAG prompt and formatting of citations vs grounded text --- src/client/views/nodes/ChatBox/Agent.ts | 20 ++-- src/client/views/nodes/ChatBox/AnswerParser.ts | 84 ++++++++++++---- src/client/views/nodes/ChatBox/ChatBox.tsx | 19 ++-- .../views/nodes/ChatBox/MessageComponent.tsx | 84 ++++++++-------- src/client/views/nodes/ChatBox/tools/RAGTool.ts | 106 ++++++++++++++++++--- src/client/views/nodes/ChatBox/types.ts | 15 ++- 6 files changed, 237 insertions(+), 91 deletions(-) (limited to 'src/client/views/nodes/ChatBox/MessageComponent.tsx') diff --git a/src/client/views/nodes/ChatBox/Agent.ts b/src/client/views/nodes/ChatBox/Agent.ts index d494928f9..ca1b5c60c 100644 --- a/src/client/views/nodes/ChatBox/Agent.ts +++ b/src/client/views/nodes/ChatBox/Agent.ts @@ -43,14 +43,14 @@ export class Agent { console.log(`System prompt: ${systemPrompt}`); this.interMessages = [{ role: 'system', content: systemPrompt }]; - this.interMessages.push({ role: 'user', content: `${question}` }); + this.interMessages.push({ role: 'user', content: `${question}` }); const parser = new XMLParser(); const builder = new XMLBuilder(); let currentAction: string | undefined; - for (let i = 0; i < maxTurns; i++) { - console.log(`Turn ${i + 1}/${maxTurns}`); + for (let i = 1; i < maxTurns; i++) { + console.log(`Turn ${i}/${maxTurns}`); const result = await this.execute(); console.log(`Bot response: ${result}`); @@ -73,17 +73,20 @@ export class Agent { currentAction = step[key] as string; console.log(`Action: ${currentAction}`); if (this.tools[currentAction]) { + i++; const nextPrompt = [ { type: 'text', - text: builder.build({ action_rules: this.tools[currentAction].getActionRule() }), + text: `` + builder.build({ action_rules: this.tools[currentAction].getActionRule() }) + `<\step>`, }, ]; this.interMessages.push({ role: 'user', content: nextPrompt }); + break; } else { console.log('Error: No valid action'); - this.interMessages.push({ role: 'user', content: 'No valid action, try again.' }); + i++; + this.interMessages.push({ role: 'user', content: `No valid action, try again.` }); break; } } else if (key === 'action_input') { @@ -92,7 +95,12 @@ export class Agent { if (currentAction) { try { const observation = await this.processAction(currentAction, step[key]); - const nextPrompt = [{ type: 'text', text: '' }, ...observation, { type: 'text', text: '' }]; + // const stepElement = parsedResult.documentElement; + // const rootTagName = stepElement.tagName; + // const match = rootTagName.match(/step(\d+)/); + // const currentStep = match ? parseInt(match[1]) + 1 : 1; + i++; + const nextPrompt = [{ type: 'text', text: `` }, ...observation, { type: 'text', text: '' }]; console.log(observation); this.interMessages.push({ role: 'user', content: nextPrompt }); break; diff --git a/src/client/views/nodes/ChatBox/AnswerParser.ts b/src/client/views/nodes/ChatBox/AnswerParser.ts index 1162d46b0..4b6c817fd 100644 --- a/src/client/views/nodes/ChatBox/AnswerParser.ts +++ b/src/client/views/nodes/ChatBox/AnswerParser.ts @@ -1,12 +1,13 @@ -import { ASSISTANT_ROLE, AssistantMessage, Citation, getChunkType } from './types'; +import { ASSISTANT_ROLE, AssistantMessage, Citation, CHUNK_TYPE, TEXT_TYPE, getChunkType } from './types'; import { v4 as uuid } from 'uuid'; export class AnswerParser { static parse(xml: string): AssistantMessage { const answerRegex = /([\s\S]*?)<\/answer>/; - const citationRegex = /(.*?)<\/citation>/g; + const citationRegex = /([\s\S]*?)<\/citation>/g; const followUpQuestionsRegex = /([\s\S]*?)<\/follow_up_questions>/; const questionRegex = /(.*?)<\/question>/g; + const groundedTextRegex = /([\s\S]*?)<\/grounded_text>/g; const answerMatch = answerRegex.exec(xml); const followUpQuestionsMatch = followUpQuestionsRegex.exec(xml); @@ -16,45 +17,86 @@ export class AnswerParser { } const rawTextContent = answerMatch[1].trim(); - const textContentWithCitations = rawTextContent.replace(citationRegex, ''); - const textContent = textContentWithCitations.replace(followUpQuestionsRegex, '').trim(); - + let textContent: AssistantMessage['content'] = []; let citations: Citation[] = []; - let match: RegExpExecArray | null; - - let plainTextOffset = 0; - let citationOffset = 0; - - while ((match = citationRegex.exec(rawTextContent)) !== null) { - const [fullMatch, chunk_id, type, direct_text] = match; - const citationStartIndex = match.index; - const citationPlainStart = citationStartIndex - citationOffset; + let contentIndex = 0; + // Parse citations + let citationMatch; + while ((citationMatch = citationRegex.exec(rawTextContent)) !== null) { + const [_, index, chunk_id, type, direct_text] = citationMatch; citations.push({ direct_text: direct_text.trim(), type: getChunkType(type), - chunk_id: chunk_id, - text_location: citationPlainStart, + chunk_id, citation_id: uuid(), }); + } - citationOffset += fullMatch.length; + // Parse text content (normal and grounded) + let lastIndex = 0; + let matches = []; + + // Find all grounded text matches + let groundedTextMatch; + while ((groundedTextMatch = groundedTextRegex.exec(rawTextContent)) !== null) { + matches.push({ + type: 'grounded', + index: groundedTextMatch.index, + length: groundedTextMatch[0].length, + citationIndexes: groundedTextMatch[1], + text: groundedTextMatch[2], + }); + } + + // Sort matches by their index in the original text + matches.sort((a, b) => a.index - b.index); + + // Process normal and grounded text in order + for (let i = 0; i <= matches.length; i++) { + const currentMatch = matches[i]; + const nextMatchIndex = currentMatch ? currentMatch.index : rawTextContent.length; + + // Add normal text before the current grounded text (or end of content) + if (nextMatchIndex > lastIndex) { + const normalText = rawTextContent.slice(lastIndex, nextMatchIndex).trim(); + if (normalText) { + textContent.push({ + index: contentIndex++, + type: TEXT_TYPE.NORMAL, + text: normalText, + citation_ids: null, + }); + } + } + + // Add grounded text if there's a match + if (currentMatch) { + const citationIds = currentMatch.citationIndexes.split(',').map(index => citations[parseInt(index) - 1].citation_id); + textContent.push({ + index: contentIndex++, + type: TEXT_TYPE.GROUNDED, + text: currentMatch.text.trim(), + citation_ids: citationIds, + }); + lastIndex = currentMatch.index + currentMatch.length; + } } let followUpQuestions: string[] = []; if (followUpQuestionsMatch) { const questionsText = followUpQuestionsMatch[1]; - let questionMatch: RegExpExecArray | null; - + let questionMatch; while ((questionMatch = questionRegex.exec(questionsText)) !== null) { followUpQuestions.push(questionMatch[1].trim()); } } + const assistantResponse: AssistantMessage = { role: ASSISTANT_ROLE.ASSISTANT, - text_content: textContent, + content: textContent, follow_up_questions: followUpQuestions, - citations: citations, + citations, }; return assistantResponse; diff --git a/src/client/views/nodes/ChatBox/ChatBox.tsx b/src/client/views/nodes/ChatBox/ChatBox.tsx index 49c9b3292..9e604073d 100644 --- a/src/client/views/nodes/ChatBox/ChatBox.tsx +++ b/src/client/views/nodes/ChatBox/ChatBox.tsx @@ -11,7 +11,7 @@ import { ViewBoxAnnotatableComponent } from '../../DocComponent'; import { FieldView, FieldViewProps } from '../FieldView'; import './ChatBox.scss'; import MessageComponentBox from './MessageComponent'; -import { ASSISTANT_ROLE, AssistantMessage, AI_Document, Citation, CHUNK_TYPE, Chunk, getChunkType } from './types'; +import { ASSISTANT_ROLE, AssistantMessage, AI_Document, Citation, CHUNK_TYPE, Chunk, getChunkType, TEXT_TYPE } from './types'; import { Vectorstore } from './vectorstore/VectorstoreUpload'; import { Agent } from './Agent'; import dotenv from 'dotenv'; @@ -58,7 +58,7 @@ export class ChatBox extends ViewBoxAnnotatableComponent() { this.agent = new Agent(this.vectorstore, this.retrieveSummaries, this.retrieveFormattedHistory); reaction( - () => this.history.map((msg: AssistantMessage) => ({ role: msg.role, text_content: msg.text_content, follow_up_questions: msg.follow_up_questions, citations: msg.citations })), + () => this.history.map((msg: AssistantMessage) => ({ role: msg.role, content: msg.content, follow_up_questions: msg.follow_up_questions, citations: msg.citations })), serializableHistory => { this.dataDoc.data = JSON.stringify(serializableHistory); } @@ -101,7 +101,7 @@ export class ChatBox extends ViewBoxAnnotatableComponent() { try { textInput.value = ''; runInAction(() => { - this.history.push({ role: ASSISTANT_ROLE.USER, text_content: trimmedText }); + this.history.push({ role: ASSISTANT_ROLE.USER, content: [{ index: 0, type: TEXT_TYPE.NORMAL, text: trimmedText, citation_ids: null }] }); this.isLoading = true; }); @@ -113,7 +113,7 @@ export class ChatBox extends ViewBoxAnnotatableComponent() { } catch (err) { console.error('Error:', err); runInAction(() => { - this.history.push({ role: ASSISTANT_ROLE.ASSISTANT, text_content: 'Sorry, I encountered an error while processing your request.' }); + this.history.push({ role: ASSISTANT_ROLE.USER, content: [{ index: 0, type: TEXT_TYPE.NORMAL, text: 'Sorry, I encountered an error while processing your request.', citation_ids: null }] }); }); } finally { runInAction(() => { @@ -212,7 +212,7 @@ export class ChatBox extends ViewBoxAnnotatableComponent() { this.history.push( ...storedHistory.map((msg: AssistantMessage) => ({ role: msg.role, - text_content: msg.text_content, + content: msg.content, follow_up_questions: msg.follow_up_questions, citations: msg.citations, })) @@ -222,7 +222,12 @@ export class ChatBox extends ViewBoxAnnotatableComponent() { console.error('Failed to parse history from dataDoc:', e); } } else { - this.history = [{ role: ASSISTANT_ROLE.ASSISTANT, text_content: 'Welcome to the Document Analyser Assistant! Link a document or ask questions to get started.' }]; + runInAction(() => { + this.history.push({ + role: ASSISTANT_ROLE.USER, + content: [{ index: 0, type: TEXT_TYPE.NORMAL, text: 'Welcome to the Document Analyser Assistant! Link a document or ask questions to get started.', citation_ids: null }], + }); + }); } reaction( () => { @@ -276,7 +281,7 @@ export class ChatBox extends ViewBoxAnnotatableComponent() { get formattedHistory(): string { let history = '\n'; for (const message of this.history) { - history += `<${message.role}>${message.text_content}\n`; + history += `<${message.role}>${message.content.map(content => content.text).join(' ')}\n`; } history += ''; return history; diff --git a/src/client/views/nodes/ChatBox/MessageComponent.tsx b/src/client/views/nodes/ChatBox/MessageComponent.tsx index 56fde8bb2..fd7c445c5 100644 --- a/src/client/views/nodes/ChatBox/MessageComponent.tsx +++ b/src/client/views/nodes/ChatBox/MessageComponent.tsx @@ -1,6 +1,6 @@ import React from 'react'; import { observer } from 'mobx-react'; -import { AssistantMessage, Citation } from './types'; +import { AssistantMessage, Citation, MessageContent, TEXT_TYPE } from './types'; import Markdown from 'react-markdown'; interface MessageComponentProps { @@ -12,53 +12,51 @@ interface MessageComponentProps { } const MessageComponentBox: React.FC = function ({ message, index, onFollowUpClick, onCitationClick, updateMessageCitations }) { - const renderContent = (content: string) => { - if (!message.citations || message.citations.length === 0) { - return content; - } - - const parts = []; - let lastIndex = 0; - - message.citations.forEach((citation, idx) => { - const location = citation.text_location; - const textBefore = content.slice(lastIndex, location); - const citationButton = ( - + const renderContent = (item: MessageContent) => { + const i = item.index; + if (item.type === TEXT_TYPE.GROUNDED) { + const citation_ids = item.citation_ids || []; + return ( + + {item.text} + {citation_ids.map((id, idx) => { + const citation = message.citations?.find(c => c.citation_id === id); + if (!citation) return null; + return ( + + ); + })} + ); - parts.push(textBefore, citationButton); - lastIndex = location; - }); - - parts.push(content.slice(lastIndex)); - - return parts; + } else { + return {item.text}; + } }; return (
-
{renderContent(message.text_content)}
+
{message.content && message.content.map(messageFragment => {renderContent(messageFragment)})}
{message.follow_up_questions && message.follow_up_questions.length > 0 && (

Follow-up Questions:

diff --git a/src/client/views/nodes/ChatBox/tools/RAGTool.ts b/src/client/views/nodes/ChatBox/tools/RAGTool.ts index 5bc31dbab..4b29d6bce 100644 --- a/src/client/views/nodes/ChatBox/tools/RAGTool.ts +++ b/src/client/views/nodes/ChatBox/tools/RAGTool.ts @@ -17,26 +17,108 @@ export class RAGTool extends BaseTool<{ hypothetical_document_chunk: string }> { required: 'true', }, }, - `Your task is to first provide a response to the user's prompt based on the information given in the chunks and considering the chat history. Follow these steps: + ` + Your task is to provide a comprehensive response to the user's prompt based on the given chunks and chat history. Follow these structural guidelines meticulously: - 1. Carefully read and analyze the provided chunks, which may include text, images, or tables. Each chunk has an associated chunk_id. + 1. Overall Structure: + + [Main content with nested grounded_text tags] + + [Individual citation tags] + + + [Three question tags] + + - 2. Review the prompt and chat history to understand the context of the user's question or request. + 2. Grounded Text Tag Structure: + - Basic format: + + [Your generated text based on chunk information] + - 3. Formulate a response that addresses the prompt using information from the relevant chunks. Your response should be informative and directly answer the user's question or request. + - Nested format: + + [General information] + + [More specific information] + + - 4. Use citations to support your response. Citations should contain direct textual references to the granular, specific part of the original chunk that applies to the situation—with no text ommitted. Citations should be in the following format: - - For text: relevant direct text from the chunk that the citation in referencing specifically - - For images or tables: + - Multiple citation indices: + + [Information synthesized from multiple chunks] + - Place citations after the sentences they apply to. You can use multiple citations in a row. + 3. Citation Tag Structure: + + [For text: relevant subset of original chunk] + [For image/table: leave empty] + - 5. If there's insufficient information in the provided chunks to answer the prompt sufficiently, ALWAYS respond with RAG not applicable + 4. Detailed Grounded Text Guidelines: + a. Wrap all information derived from chunks in grounded_text tags. + b. Nest grounded_text tags when presenting hierarchical or increasingly specific information or when a larger section of generated text is best grounded by one subset of a chunk and smaller sections of that generated text are best grounded by other subsets of either the same or different chunk(s). + c. Use a single grounded_text tag for closely related information that references the same citation (subset of text from a chunk). + d. Combine multiple citation indices for synthesized information from multiple citations. + e. Ensure every grounded_text tag has at least one corresponding citation. + f. Grounded text can be as short as a few words or as long as several sentences. + d. Avoid overlapping grounded_text tags; instead, use nesting or sequential tags. - Write your entire response, including follow-up questions, inside tags. Remember to use the citation format for both text and image references, and maintain a conversational tone throughout your response. + 5. Detailed Citation Guidelines: + a. Create a unique citation for each distinct piece of information from the chunks that is used to support grounded_text. + b. Ensure each citation has a unique index number. + c. Specify the correct type: "text", "image", or "table". + d. For text chunks, include only the relevant subset of the original text that the grounded_text is based on. + e. For image/table chunks, leave the citation content empty. + f. One citation can be used for multiple grounded_text tags if they are based on the same information. + g. One text chunk can have multiple citations if different parts of the text have different important information. + h. !!!DO NOT OVERCITE - only include citations for information that is directly relevant to the grounded_text. - !!!IMPORTANT Before you close the tag with , within the answer tags provide a set of 3 follow-up questions inside a tag and individually within tags. These should relate to the document, the current query, and the chat_history and should aim to help the user better understand whatever they are looking for. - Also, ensure that the answer tags are wrapped with the correct step tags as well.`, + 6. Structural Integrity Checks: + a. Ensure all opening tags have corresponding closing tags. + b. Verify that all grounded_text tags have valid citation_index attributes. + c. Check that all cited indices in grounded_text tags have corresponding citations. + d. Confirm proper nesting - tags opened last should be closed first. + + Example of grounded_text usage: + + + + Artificial Intelligence (AI) is revolutionizing various sectors, with healthcare experiencing significant transformations in areas such as diagnosis and treatment planning. + + In the field of medical diagnosis, AI has shown remarkable capabilities, particularly in radiology. For instance, AI systems have drastically improved mammogram analysis, achieving 99% accuracy at a rate 30 times faster than human radiologists. + + This advancement not only enhances the efficiency of healthcare systems but also significantly reduces the occurrence of false positives, leading to fewer unnecessary biopsies and reduced patient stress. + + + + + + Beyond diagnosis, AI is playing a crucial role in drug discovery and development. By analyzing vast amounts of genetic and molecular data, AI algorithms can identify potential drug candidates much faster than traditional methods. + + This could potentially reduce the time and cost of bringing new medications to market, especially for rare diseases that have historically received less attention due to limited market potential. + + + + [... rest of the content ...] + + + Artificial Intelligence is revolutionizing various industries, with healthcare being one of the most profoundly affected sectors. + AI has shown particular promise in the field of radiology, enhancing the accuracy and speed of image analysis. + According to recent studies, AI systems have achieved 99% accuracy in mammogram analysis, performing the task 30 times faster than human radiologists. + The improvement in mammogram accuracy has led to a significant reduction in false positives, decreasing the need for unnecessary biopsies and reducing patient anxiety. + AI is accelerating the drug discovery process by analyzing complex molecular and genetic data to identify potential drug candidates. + The use of AI in drug discovery could significantly reduce the time and cost associated with bringing new medications to market, particularly for rare diseases. + + + + How might AI-driven personalized medicine impact the cost and accessibility of healthcare in the future? + What measures can be taken to ensure that AI systems in healthcare are free from biases and equally effective for diverse populations? + How could the role of healthcare professionals evolve as AI becomes more integrated into medical practices? + + + `, `Performs a RAG (Retrieval-Augmented Generation) search on user documents and returns a set of document chunks (either images or text) that can be used to provide a grounded response based on diff --git a/src/client/views/nodes/ChatBox/types.ts b/src/client/views/nodes/ChatBox/types.ts index d702d5c41..10c80c05a 100644 --- a/src/client/views/nodes/ChatBox/types.ts +++ b/src/client/views/nodes/ChatBox/types.ts @@ -6,6 +6,11 @@ export enum ASSISTANT_ROLE { ASSISTANT = 'assistant', } +export enum TEXT_TYPE { + NORMAL = 'normal', + GROUNDED = 'grounded', +} + export enum CHUNK_TYPE { TEXT = 'text', IMAGE = 'image', @@ -27,16 +32,22 @@ export function getChunkType(type: string): CHUNK_TYPE { export interface AssistantMessage { role: ASSISTANT_ROLE; - text_content: string; + content: MessageContent[]; follow_up_questions?: string[]; citations?: Citation[]; } +export interface MessageContent { + index: number; + type: TEXT_TYPE; + text: string; + citation_ids: string[] | null; +} + export interface Citation { direct_text?: string; type: CHUNK_TYPE; chunk_id: string; - text_location: number; citation_id: string; } -- cgit v1.2.3-70-g09d2 From 56f6637e9d9052c4bc7724d2713573ac67141631 Mon Sep 17 00:00:00 2001 From: "A.J. Shulman" Date: Mon, 22 Jul 2024 13:20:36 -0400 Subject: works decently well --- src/client/views/nodes/ChatBox/Agent.ts | 4 +- src/client/views/nodes/ChatBox/AnswerParser.ts | 70 ++++++++-------------- .../views/nodes/ChatBox/MessageComponent.tsx | 2 +- src/client/views/nodes/ChatBox/prompts.ts | 4 +- src/client/views/nodes/ChatBox/tools/RAGTool.ts | 52 ++++++---------- 5 files changed, 47 insertions(+), 85 deletions(-) (limited to 'src/client/views/nodes/ChatBox/MessageComponent.tsx') diff --git a/src/client/views/nodes/ChatBox/Agent.ts b/src/client/views/nodes/ChatBox/Agent.ts index ca1b5c60c..b9d137270 100644 --- a/src/client/views/nodes/ChatBox/Agent.ts +++ b/src/client/views/nodes/ChatBox/Agent.ts @@ -22,7 +22,7 @@ export class Agent { private _summaries: () => string; constructor(_vectorstore: Vectorstore, summaries: () => string, history: () => string) { - this.client = new OpenAI({ apiKey: process.env.OPENAI_KEY, dangerouslyAllowBrowser: true }); + this.client = new OpenAI({ apiKey: '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', dangerouslyAllowBrowser: true }); this.vectorstore = _vectorstore; this._history = history; this._summaries = summaries; @@ -34,7 +34,7 @@ export class Agent { }; } - async askAgent(question: string, maxTurns: number = 10): Promise { + async askAgent(question: string, maxTurns: number = 20): Promise { console.log(`Starting query: ${question}`); this.messages.push({ role: 'user', content: question }); const chatHistory = this._history(); diff --git a/src/client/views/nodes/ChatBox/AnswerParser.ts b/src/client/views/nodes/ChatBox/AnswerParser.ts index 4b6c817fd..dd7ec3499 100644 --- a/src/client/views/nodes/ChatBox/AnswerParser.ts +++ b/src/client/views/nodes/ChatBox/AnswerParser.ts @@ -17,71 +17,49 @@ export class AnswerParser { } const rawTextContent = answerMatch[1].trim(); - let textContent: AssistantMessage['content'] = []; + let content: AssistantMessage['content'] = []; let citations: Citation[] = []; let contentIndex = 0; // Parse citations let citationMatch; + const citationMap = new Map(); while ((citationMatch = citationRegex.exec(rawTextContent)) !== null) { const [_, index, chunk_id, type, direct_text] = citationMatch; + const citation_id = uuid(); + citationMap.set(index, citation_id); citations.push({ direct_text: direct_text.trim(), type: getChunkType(type), chunk_id, - citation_id: uuid(), + citation_id, }); } - // Parse text content (normal and grounded) - let lastIndex = 0; - let matches = []; + // Parse grounded text content + const parseGroundedText = (text: string): AssistantMessage['content'] => { + const result: AssistantMessage['content'] = []; + let lastIndex = 0; + let match; - // Find all grounded text matches - let groundedTextMatch; - while ((groundedTextMatch = groundedTextRegex.exec(rawTextContent)) !== null) { - matches.push({ - type: 'grounded', - index: groundedTextMatch.index, - length: groundedTextMatch[0].length, - citationIndexes: groundedTextMatch[1], - text: groundedTextMatch[2], - }); - } - - // Sort matches by their index in the original text - matches.sort((a, b) => a.index - b.index); - - // Process normal and grounded text in order - for (let i = 0; i <= matches.length; i++) { - const currentMatch = matches[i]; - const nextMatchIndex = currentMatch ? currentMatch.index : rawTextContent.length; - - // Add normal text before the current grounded text (or end of content) - if (nextMatchIndex > lastIndex) { - const normalText = rawTextContent.slice(lastIndex, nextMatchIndex).trim(); - if (normalText) { - textContent.push({ - index: contentIndex++, - type: TEXT_TYPE.NORMAL, - text: normalText, - citation_ids: null, - }); - } - } + while ((match = groundedTextRegex.exec(text)) !== null) { + const [fullMatch, citationIndex, groundedText] = match; + const citation_ids = citationIndex.split(',').map(index => citationMap.get(index) || ''); - // Add grounded text if there's a match - if (currentMatch) { - const citationIds = currentMatch.citationIndexes.split(',').map(index => citations[parseInt(index) - 1].citation_id); - textContent.push({ + result.push({ index: contentIndex++, type: TEXT_TYPE.GROUNDED, - text: currentMatch.text.trim(), - citation_ids: citationIds, + text: groundedText.trim(), + citation_ids, }); - lastIndex = currentMatch.index + currentMatch.length; + + lastIndex = match.index + fullMatch.length; } - } + + return result; + }; + + content = parseGroundedText(rawTextContent); let followUpQuestions: string[] = []; if (followUpQuestionsMatch) { @@ -94,7 +72,7 @@ export class AnswerParser { const assistantResponse: AssistantMessage = { role: ASSISTANT_ROLE.ASSISTANT, - content: textContent, + content, follow_up_questions: followUpQuestions, citations, }; diff --git a/src/client/views/nodes/ChatBox/MessageComponent.tsx b/src/client/views/nodes/ChatBox/MessageComponent.tsx index fd7c445c5..d24a55d23 100644 --- a/src/client/views/nodes/ChatBox/MessageComponent.tsx +++ b/src/client/views/nodes/ChatBox/MessageComponent.tsx @@ -24,7 +24,7 @@ const MessageComponentBox: React.FC = function ({ message if (!citation) return null; return (
diff --git a/src/client/views/nodes/ChatBox/MessageComponent.tsx b/src/client/views/nodes/ChatBox/MessageComponent.tsx index 07bfd4e3d..70b0527a2 100644 --- a/src/client/views/nodes/ChatBox/MessageComponent.tsx +++ b/src/client/views/nodes/ChatBox/MessageComponent.tsx @@ -62,6 +62,14 @@ const MessageComponentBox: React.FC = function ({ message return (
+
+ {message.thoughts && + message.thoughts.map((thought, index) => ( + + Thought: {thought} + + ))} +
{message.content && message.content.map(messageFragment => {renderContent(messageFragment)})}
{message.follow_up_questions && message.follow_up_questions.length > 0 && (
diff --git a/src/client/views/nodes/ChatBox/types.ts b/src/client/views/nodes/ChatBox/types.ts index 1c7aaa4b7..391f124e0 100644 --- a/src/client/views/nodes/ChatBox/types.ts +++ b/src/client/views/nodes/ChatBox/types.ts @@ -47,6 +47,7 @@ export interface AssistantMessage { role: ASSISTANT_ROLE; content: MessageContent[]; follow_up_questions?: string[]; + thoughts?: string[]; citations?: Citation[]; } -- cgit v1.2.3-70-g09d2 From 2c38022a7f21d4b498277b18ad31baf24ac3a143 Mon Sep 17 00:00:00 2001 From: "A.J. Shulman" Date: Sun, 18 Aug 2024 10:12:35 -0400 Subject: Attempting streaming content --- src/client/views/nodes/ChatBox/Agent.ts | 198 +++++++++++++++++---- src/client/views/nodes/ChatBox/ChatBox.tsx | 43 ++--- .../views/nodes/ChatBox/MessageComponent.tsx | 32 +++- src/client/views/nodes/ChatBox/StreamParser.ts | 125 +++++++++++++ src/client/views/nodes/ChatBox/types.ts | 1 + 5 files changed, 326 insertions(+), 73 deletions(-) create mode 100644 src/client/views/nodes/ChatBox/StreamParser.ts (limited to 'src/client/views/nodes/ChatBox/MessageComponent.tsx') diff --git a/src/client/views/nodes/ChatBox/Agent.ts b/src/client/views/nodes/ChatBox/Agent.ts index 8bad29d9a..2c7c40e0c 100644 --- a/src/client/views/nodes/ChatBox/Agent.ts +++ b/src/client/views/nodes/ChatBox/Agent.ts @@ -1,5 +1,5 @@ import OpenAI from 'openai'; -import { Tool, AgentMessage } from './types'; +import { Tool, AgentMessage, AssistantMessage, TEXT_TYPE, CHUNK_TYPE, ASSISTANT_ROLE } from './types'; import { getReactPrompt } from './prompts'; import { XMLParser, XMLBuilder } from 'fast-xml-parser'; import { Vectorstore } from './vectorstore/Vectorstore'; @@ -12,6 +12,9 @@ import { WebsiteInfoScraperTool } from './tools/WebsiteInfoScraperTool'; import { SearchTool } from './tools/SearchTool'; import { NoTool } from './tools/NoTool'; import { on } from 'events'; +import { StreamParser } from './StreamParser'; +import { v4 as uuidv4 } from 'uuid'; +import { AnswerParser } from './AnswerParser'; dotenv.config(); @@ -41,7 +44,7 @@ export class Agent { }; } - async askAgent(question: string, maxTurns: number = 20, onUpdate: (update: string) => void): Promise { + async askAgent(question: string, maxTurns: number = 20, onUpdate: (update: AssistantMessage) => void): Promise { console.log(`Starting query: ${question}`); this.messages.push({ role: 'user', content: question }); const chatHistory = this._history(); @@ -53,12 +56,18 @@ export class Agent { const builder = new XMLBuilder({ ignoreAttributes: false, attributeNamePrefix: '@_' }); let currentAction: string | undefined; - let thoughtNumber = 0; + let assistantMessage: AssistantMessage = { + role: ASSISTANT_ROLE.ASSISTANT, + content: [], + thoughts: [], + actions: [], + citations: [], + }; for (let i = 2; i < maxTurns; i += 2) { console.log(`Turn ${i}/${maxTurns}`); - const result = await this.execute(onUpdate, thoughtNumber); + const result = await this.execute(assistantMessage, onUpdate); this.interMessages.push({ role: 'assistant', content: result }); let parsedResult; @@ -66,23 +75,25 @@ export class Agent { parsedResult = parser.parse(result); } catch (error) { console.log('Error: Invalid XML response from bot'); - return 'Invalid response format.'; + return assistantMessage; } const stage = parsedResult.stage; if (!stage) { console.log('Error: No stage found in response'); - return 'Invalid response format: No stage found.'; + return assistantMessage; } for (const key in stage) { - if (key === 'thought') { - console.log(`Thought: ${stage[key]}`); - thoughtNumber++; - } else if (key === 'action') { + if (!assistantMessage.actions) { + assistantMessage.actions = []; + } + if (key === 'action') { currentAction = stage[key] as string; console.log(`Action: ${currentAction}`); + assistantMessage.actions.push({ index: assistantMessage.actions.length, action: currentAction, action_input: '' }); + onUpdate({ ...assistantMessage }); if (this.tools[currentAction]) { const nextPrompt = `` + builder.build({ action_rules: this.tools[currentAction].getActionRule() }) + ``; this.interMessages.push({ role: 'user', content: nextPrompt }); @@ -93,8 +104,12 @@ export class Agent { break; } } else if (key === 'action_input') { - const actionInput = builder.build({ action_input: stage[key] }); + const actionInput = stage[key]; console.log(`Action input: ${actionInput}`); + if (currentAction && assistantMessage.actions.length > 0) { + assistantMessage.actions[assistantMessage.actions.length - 1].action_input = actionInput; + onUpdate({ ...assistantMessage }); + } if (currentAction) { try { const observation = await this.processAction(currentAction, stage[key]); @@ -104,24 +119,28 @@ export class Agent { break; } catch (error) { console.log(`Error processing action: ${error}`); - return `${error}`; + return assistantMessage; } } else { console.log('Error: Action input without a valid action'); - return 'Action input without a valid action'; + return assistantMessage; } } else if (key === 'answer') { console.log('Answer found. Ending query.'); - onUpdate(`ANSWER:${stage[key]}`); - return result; + const parsedAnswer = AnswerParser.parse(`${stage[key]}`); + assistantMessage.content = parsedAnswer.content; + assistantMessage.follow_up_questions = parsedAnswer.follow_up_questions; + assistantMessage.citations = parsedAnswer.citations; + onUpdate({ ...assistantMessage }); + return assistantMessage; } } } console.log('Reached maximum turns. Ending query.'); - return 'Reached maximum turns without finding an answer'; + return assistantMessage; } - private async execute(onUpdate: (update: string) => void, thoughtNumber: number): Promise { + private async execute(assistantMessage: AssistantMessage, onUpdate: (update: AssistantMessage) => void): Promise { const stream = await this.client.chat.completions.create({ model: 'gpt-4o', messages: this.interMessages as ChatCompletionMessageParam[], @@ -130,32 +149,147 @@ export class Agent { }); let fullResponse = ''; + let currentTag = ''; let currentContent = ''; + let isInsideTag = false; + let isInsideActionInput = false; + let actionInputContent = ''; + + if (!assistantMessage.actions) { + assistantMessage.actions = []; + } for await (const chunk of stream) { const content = chunk.choices[0]?.delta?.content || ''; fullResponse += content; - currentContent += content; + for (const char of content) { + if (char === '<') { + isInsideTag = true; + if (currentTag && currentContent) { + if (currentTag === 'action_input') { + assistantMessage.actions[assistantMessage.actions.length - 1].action_input = actionInputContent; + actionInputContent = ''; + } else { + this.processStreamedContent(currentTag, currentContent, assistantMessage); + } + onUpdate({ ...assistantMessage }); + } + currentTag = ''; + currentContent = ''; + } else if (char === '>') { + isInsideTag = false; + if (currentTag === 'action_input') { + isInsideActionInput = true; + } else if (currentTag === '/action_input') { + isInsideActionInput = false; + assistantMessage.actions[assistantMessage.actions.length - 1].action_input = actionInputContent; + actionInputContent = ''; + onUpdate({ ...assistantMessage }); + } + if (currentTag.startsWith('/')) { + currentTag = ''; + } + } else if (isInsideTag) { + currentTag += char; + } else if (isInsideActionInput) { + actionInputContent += char; + } else { + currentContent += char; + if (currentTag === 'thought' || currentTag === 'action') { + this.processStreamedContent(currentTag, currentContent, assistantMessage); + onUpdate({ ...assistantMessage }); + } + } + } + } - console.log(currentContent); + return fullResponse; + } - if (currentContent.includes('')) { - onUpdate(`THOUGHT${thoughtNumber}:${currentContent}`); - } - if (currentContent.includes('')) { - currentContent = ''; - } - if (currentContent.includes('')) { - onUpdate(`ANSWER_START:${currentContent}`); - } - if (currentContent.includes('')) { - onUpdate(`ANSWER_END:${currentContent}`); - currentContent = ''; + private processStreamedContent(tag: string, content: string, assistantMessage: AssistantMessage) { + if (!assistantMessage.thoughts) { + assistantMessage.thoughts = []; + } + if (!assistantMessage.actions) { + assistantMessage.actions = []; + } + switch (tag) { + case 'thought': + if (assistantMessage.thoughts.length > 0) { + assistantMessage.thoughts[assistantMessage.thoughts.length - 1] = content; + } else { + assistantMessage.thoughts.push(content); + } + break; + case 'action': + if (assistantMessage.actions.length > 0) { + assistantMessage.actions[assistantMessage.actions.length - 1].action = content; + } else { + assistantMessage.actions.push({ index: assistantMessage.actions.length, action: content, action_input: '' }); + } + break; + case 'action_input': + if (assistantMessage.actions.length > 0) { + assistantMessage.actions[assistantMessage.actions.length - 1].action_input = content; + } + break; + } + } + + private processAnswer(content: string, assistantMessage: AssistantMessage) { + const groundedTextRegex = /([\s\S]*?)<\/grounded_text>/g; + let lastIndex = 0; + let match; + + while ((match = groundedTextRegex.exec(content)) !== null) { + const [fullMatch, citationIndex, groundedText] = match; + + // Add normal text before the grounded text + if (match.index > lastIndex) { + const normalText = content.slice(lastIndex, match.index).trim(); + if (normalText) { + assistantMessage.content.push({ + index: assistantMessage.content.length, + type: TEXT_TYPE.NORMAL, + text: normalText, + citation_ids: null, + }); + } } + + // Add grounded text + const citation_id = uuidv4(); + assistantMessage.content.push({ + index: assistantMessage.content.length, + type: TEXT_TYPE.GROUNDED, + text: groundedText.trim(), + citation_ids: [citation_id], + }); + + // Add citation + assistantMessage.citations?.push({ + citation_id, + chunk_id: '', + type: CHUNK_TYPE.TEXT, + direct_text: '', + }); + + lastIndex = match.index + fullMatch.length; } - return fullResponse; + // Add any remaining normal text after the last grounded text + if (lastIndex < content.length) { + const remainingText = content.slice(lastIndex).trim(); + if (remainingText) { + assistantMessage.content.push({ + index: assistantMessage.content.length, + type: TEXT_TYPE.NORMAL, + text: remainingText, + citation_ids: null, + }); + } + } } private async processAction(action: string, actionInput: any): Promise { diff --git a/src/client/views/nodes/ChatBox/ChatBox.tsx b/src/client/views/nodes/ChatBox/ChatBox.tsx index d38c71810..099c0298e 100644 --- a/src/client/views/nodes/ChatBox/ChatBox.tsx +++ b/src/client/views/nodes/ChatBox/ChatBox.tsx @@ -149,45 +149,24 @@ export class ChatBox extends ViewBoxAnnotatableComponent() { if (trimmedText) { try { - //Make everything go through the answer parser - //Pass in all the updates to the AnswrParser and it will create the assistant messasge that will be the current message including adding in the thoughts and also waiting for the asnwer and also showing tool progress textInput.value = ''; this.history.push({ role: ASSISTANT_ROLE.USER, content: [{ index: 0, type: TEXT_TYPE.NORMAL, text: trimmedText, citation_ids: null }] }); this.isLoading = true; - this.current_message = { role: ASSISTANT_ROLE.ASSISTANT, content: [], thoughts: [] }; - - let currentThought = ''; - - this.current_message?.thoughts?.push(currentThought); - - const onUpdate = (update: string) => { - const thoughtNumber = Number(update.match(/^THOUGHT(\d+):/)?.[1] ?? 0); - const regex = /\s*([\s\S]*?)(?:<\/thought>|$)/; - const match = update.match(regex); - const currentThought = match ? match[1].trim() : ''; - //const numericPrefix = Number(update.match(/^\d+/)?.[0]); - if (update.startsWith('THOUGHT')) { - console.log('Thought:', currentThought, thoughtNumber); - if (this.current_message?.thoughts) { - if (this.current_message.thoughts.length <= thoughtNumber) { - this.current_message.thoughts.push(currentThought); - } else { - this.current_message.thoughts[thoughtNumber] = currentThought; - } - } - console.log('Thoughts:', this.current_message?.thoughts); - } + this.current_message = { role: ASSISTANT_ROLE.ASSISTANT, content: [], thoughts: [], actions: [], citations: [] }; + + const onUpdate = (update: AssistantMessage) => { + runInAction(() => { + this.current_message = { ...update }; + }); }; - const response = await this.agent.askAgent(trimmedText, 20, onUpdate); - const parsedAnswer = AnswerParser.parse(response); - parsedAnswer.thoughts = this.current_message?.thoughts; + const finalMessage = await this.agent.askAgent(trimmedText, 20, onUpdate); - if (this.current_message) { - this.history.push(parsedAnswer); + runInAction(() => { + this.history.push({ ...finalMessage }); this.current_message = undefined; - } - this.dataDoc.data = JSON.stringify(this.history); + this.dataDoc.data = JSON.stringify(this.history); + }); } catch (err) { console.error('Error:', err); this.history.push({ role: ASSISTANT_ROLE.ASSISTANT, content: [{ index: 0, type: TEXT_TYPE.NORMAL, text: 'Sorry, I encountered an error while processing your request.', citation_ids: null }] }); diff --git a/src/client/views/nodes/ChatBox/MessageComponent.tsx b/src/client/views/nodes/ChatBox/MessageComponent.tsx index 70b0527a2..e82dcd5f7 100644 --- a/src/client/views/nodes/ChatBox/MessageComponent.tsx +++ b/src/client/views/nodes/ChatBox/MessageComponent.tsx @@ -55,21 +55,35 @@ const MessageComponentBox: React.FC = function ({ message {item.text} ); + } else if ('query' in item) { + // Handle the case where the item has a query property + return ( + + {JSON.stringify(item.query)} + + ); } else { - return {item.text}; + // Fallback for any other unexpected cases + return {JSON.stringify(item)}; } }; return (
-
- {message.thoughts && - message.thoughts.map((thought, index) => ( - - Thought: {thought} - - ))} -
+ {message.thoughts && + message.thoughts.map((thought, idx) => ( +
+ Thought: {thought} +
+ ))} + {message.actions && + message.actions.map((action, idx) => ( +
+ Action: {action.action} +
+ Input: {action.action_input} +
+ ))}
{message.content && message.content.map(messageFragment => {renderContent(messageFragment)})}
{message.follow_up_questions && message.follow_up_questions.length > 0 && (
diff --git a/src/client/views/nodes/ChatBox/StreamParser.ts b/src/client/views/nodes/ChatBox/StreamParser.ts new file mode 100644 index 000000000..9b087663a --- /dev/null +++ b/src/client/views/nodes/ChatBox/StreamParser.ts @@ -0,0 +1,125 @@ +import { AssistantMessage, ASSISTANT_ROLE, TEXT_TYPE, Citation, CHUNK_TYPE } from './types'; +import { v4 as uuidv4 } from 'uuid'; + +export class StreamParser { + private currentMessage: AssistantMessage; + private currentTag: string | null = null; + private buffer: string = ''; + private citationIndex: number = 1; + + constructor() { + this.currentMessage = { + role: ASSISTANT_ROLE.ASSISTANT, + content: [], + thoughts: [], + actions: [], + citations: [], + }; + } + + parse(chunk: string): AssistantMessage { + this.buffer += chunk; + + while (this.buffer.length > 0) { + if (this.currentTag === null) { + const openTagMatch = this.buffer.match(/<(\w+)>/); + if (openTagMatch) { + this.currentTag = openTagMatch[1]; + this.buffer = this.buffer.slice(openTagMatch.index! + openTagMatch[0].length); + } else { + break; + } + } else { + const closeTagIndex = this.buffer.indexOf(``); + if (closeTagIndex !== -1) { + const content = this.buffer.slice(0, closeTagIndex); + this.processTag(this.currentTag, content); + this.buffer = this.buffer.slice(closeTagIndex + this.currentTag.length + 3); + this.currentTag = null; + } else { + break; + } + } + } + + return this.currentMessage; + } + + private processTag(tag: string, content: string) { + switch (tag) { + case 'thought': + this.currentMessage.thoughts!.push(content); + break; + case 'action': + this.currentMessage.actions!.push({ index: this.currentMessage.actions!.length, action: content, action_input: '' }); + break; + case 'action_input': + if (this.currentMessage.actions!.length > 0) { + this.currentMessage.actions![this.currentMessage.actions!.length - 1].action_input = content; + } + break; + case 'answer': + this.processAnswer(content); + break; + } + } + + private processAnswer(content: string) { + const groundedTextRegex = /([\s\S]*?)<\/grounded_text>/g; + let lastIndex = 0; + let match; + + while ((match = groundedTextRegex.exec(content)) !== null) { + const [fullMatch, citationIndex, groundedText] = match; + + // Add normal text before the grounded text + if (match.index > lastIndex) { + const normalText = content.slice(lastIndex, match.index).trim(); + if (normalText) { + this.currentMessage.content.push({ + index: this.currentMessage.content.length, + type: TEXT_TYPE.NORMAL, + text: normalText, + citation_ids: null, + }); + } + } + + // Add grounded text + const citation_id = uuidv4(); + this.currentMessage.content.push({ + index: this.currentMessage.content.length, + type: TEXT_TYPE.GROUNDED, + text: groundedText.trim(), + citation_ids: [citation_id], + }); + + // Add citation + this.currentMessage.citations!.push({ + citation_id, + chunk_id: '', + type: CHUNK_TYPE.TEXT, + direct_text: '', + }); + + lastIndex = match.index + fullMatch.length; + } + + // Add any remaining normal text after the last grounded text + if (lastIndex < content.length) { + const remainingText = content.slice(lastIndex).trim(); + if (remainingText) { + this.currentMessage.content.push({ + index: this.currentMessage.content.length, + type: TEXT_TYPE.NORMAL, + text: remainingText, + citation_ids: null, + }); + } + } + } + + getResult(): AssistantMessage { + return this.currentMessage; + } +} diff --git a/src/client/views/nodes/ChatBox/types.ts b/src/client/views/nodes/ChatBox/types.ts index 391f124e0..efeec7b93 100644 --- a/src/client/views/nodes/ChatBox/types.ts +++ b/src/client/views/nodes/ChatBox/types.ts @@ -48,6 +48,7 @@ export interface AssistantMessage { content: MessageContent[]; follow_up_questions?: string[]; thoughts?: string[]; + actions?: { index: number; action: string; action_input: string }[]; citations?: Citation[]; } -- cgit v1.2.3-70-g09d2 From 4b6ce2ffcb82c1a7467ef7ed8b67b97094a8f6b6 Mon Sep 17 00:00:00 2001 From: "A.J. Shulman" Date: Mon, 19 Aug 2024 12:53:00 -0400 Subject: Streaming wiht thoughts and actions working much better but still get error for web search --- src/client/views/nodes/ChatBox/Agent.ts | 112 +++++++-------------- src/client/views/nodes/ChatBox/AnswerParser.ts | 17 ++-- src/client/views/nodes/ChatBox/ChatBox.tsx | 25 +++-- .../views/nodes/ChatBox/MessageComponent.tsx | 32 +++--- src/client/views/nodes/ChatBox/prompts.ts | 50 ++++++--- src/client/views/nodes/ChatBox/types.ts | 17 +++- 6 files changed, 126 insertions(+), 127 deletions(-) (limited to 'src/client/views/nodes/ChatBox/MessageComponent.tsx') diff --git a/src/client/views/nodes/ChatBox/Agent.ts b/src/client/views/nodes/ChatBox/Agent.ts index 413ecbd41..ae08271ee 100644 --- a/src/client/views/nodes/ChatBox/Agent.ts +++ b/src/client/views/nodes/ChatBox/Agent.ts @@ -1,5 +1,5 @@ import OpenAI from 'openai'; -import { Tool, AgentMessage, AssistantMessage, TEXT_TYPE, CHUNK_TYPE, ASSISTANT_ROLE } from './types'; +import { Tool, AgentMessage, AssistantMessage, TEXT_TYPE, CHUNK_TYPE, ASSISTANT_ROLE, ProcessingInfo, PROCESSING_TYPE } from './types'; import { getReactPrompt } from './prompts'; import { XMLParser, XMLBuilder } from 'fast-xml-parser'; import { Vectorstore } from './vectorstore/Vectorstore'; @@ -29,6 +29,8 @@ export class Agent { private _csvData: () => { filename: string; id: string; text: string }[]; private actionNumber: number = 0; private thoughtNumber: number = 0; + private processingNumber: number = 0; + private processingInfo: ProcessingInfo[] = []; constructor(_vectorstore: Vectorstore, summaries: () => string, history: () => string, csvData: () => { filename: string; id: string; text: string }[], addLinkedUrlDoc: (url: string, id: string) => void) { this.client = new OpenAI({ apiKey: process.env.OPENAI_KEY, dangerouslyAllowBrowser: true }); @@ -46,7 +48,7 @@ export class Agent { }; } - async askAgent(question: string, maxTurns: number = 30, onUpdate: (update: AssistantMessage) => void): Promise { + async askAgent(question: string, onUpdate: (update: ProcessingInfo[]) => void, maxTurns: number = 30): Promise { console.log(`Starting query: ${question}`); this.messages.push({ role: 'user', content: question }); const chatHistory = this._history(); @@ -57,48 +59,34 @@ export class Agent { const builder = new XMLBuilder({ ignoreAttributes: false, attributeNamePrefix: '@_' }); let currentAction: string | undefined; - let assistantMessage: AssistantMessage = { - role: ASSISTANT_ROLE.ASSISTANT, - content: [], - thoughts: [], - actions: [], - citations: [], - }; + this.processingInfo = []; for (let i = 2; i < maxTurns; i += 2) { console.log(`Turn ${i}/${maxTurns}`); - const result = await this.execute(assistantMessage, onUpdate); + const result = await this.execute(onUpdate); this.interMessages.push({ role: 'assistant', content: result }); let parsedResult; try { parsedResult = parser.parse(result); } catch (error) { - console.log('Error: Invalid XML response from bot'); - assistantMessage.content.push({ index: assistantMessage.content.length, type: TEXT_TYPE.ERROR, text: 'Invalid response from bot', citation_ids: null }); - return assistantMessage; + throw new Error(`Error parsing response: ${error}`); } const stage = parsedResult.stage; if (!stage) { - console.log('Error: No stage found in response'); - assistantMessage.content.push({ index: assistantMessage.content.length, type: TEXT_TYPE.ERROR, text: 'Invalid response from bot', citation_ids: null }); - return assistantMessage; + throw new Error(`Error: No stage found in response`); } for (const key in stage) { - if (!assistantMessage.actions) { - assistantMessage.actions = []; - } if (key === 'thought') { console.log(`Thought: ${stage[key]}`); - this.thoughtNumber++; + this.processingNumber++; } else if (key === 'action') { currentAction = stage[key] as string; console.log(`Action: ${currentAction}`); - onUpdate({ ...assistantMessage }); if (this.tools[currentAction]) { const nextPrompt = [ { @@ -118,35 +106,29 @@ export class Agent { console.log(`Action input: ${actionInput}`); if (currentAction) { try { - const observation = await this.processAction(currentAction, stage[key]); + const observation = await this.processAction(currentAction, stage[key].inputs); const nextPrompt = [{ type: 'text', text: ` ` }, ...observation, { type: 'text', text: '' }]; console.log(observation); this.interMessages.push({ role: 'user', content: nextPrompt }); - this.actionNumber++; //might not work with no tool + this.processingNumber++; break; } catch (error) { - console.log(`Error processing action: ${error}`); - assistantMessage.content.push({ index: assistantMessage.content.length, type: TEXT_TYPE.ERROR, text: 'Invalid response from bot', citation_ids: null }); - return assistantMessage; + throw new Error(`Error processing action: ${error}`); } } else { - console.log('Error: Action input without a valid action'); - assistantMessage.content.push({ index: assistantMessage.content.length, type: TEXT_TYPE.ERROR, text: 'Invalid response from bot', citation_ids: null }); - return assistantMessage; + throw new Error('Error: Action input without a valid action'); } } else if (key === 'answer') { console.log('Answer found. Ending query.'); - const parsedAnswer = AnswerParser.parse(result, assistantMessage); - onUpdate({ ...parsedAnswer }); + const parsedAnswer = AnswerParser.parse(result, this.processingInfo); return parsedAnswer; } } } - console.log('Reached maximum turns. Ending query.'); - return assistantMessage; + throw new Error('Reached maximum turns. Ending query.'); } - private async execute(assistantMessage: AssistantMessage, onUpdate: (update: AssistantMessage) => void): Promise { + private async execute(onUpdate: (update: ProcessingInfo[]) => void): Promise { const stream = await this.client.chat.completions.create({ model: 'gpt-4o', messages: this.interMessages as ChatCompletionMessageParam[], @@ -158,12 +140,6 @@ export class Agent { let currentTag: string = ''; let currentContent: string = ''; let isInsideTag: boolean = false; - let isInsideActionInput: boolean = false; - let actionInputContent: string = ''; - - if (!assistantMessage.actions) { - assistantMessage.actions = []; - } for await (const chunk of stream) { const content = chunk.choices[0]?.delta?.content || ''; @@ -172,40 +148,20 @@ export class Agent { for (const char of content) { if (char === '<') { isInsideTag = true; - if (currentTag && currentContent) { - if (currentTag === 'action_input') { - assistantMessage.actions[assistantMessage.actions.length - 1].action_input = actionInputContent; - actionInputContent = ''; - } else { - this.processStreamedContent(currentTag, currentContent, assistantMessage); - } - onUpdate({ ...assistantMessage }); - } currentTag = ''; currentContent = ''; } else if (char === '>') { isInsideTag = false; - if (currentTag === 'action_input') { - isInsideActionInput = true; - } else if (currentTag === '/action_input') { - isInsideActionInput = false; - console.log('Action input:', actionInputContent); - assistantMessage.actions[assistantMessage.actions.length - 1].action_input = actionInputContent; - actionInputContent = ''; - onUpdate({ ...assistantMessage }); - } if (currentTag.startsWith('/')) { currentTag = ''; } } else if (isInsideTag) { currentTag += char; - } else if (isInsideActionInput) { - actionInputContent += char; } else { currentContent += char; - if (currentTag === 'thought' || currentTag === 'action') { - this.processStreamedContent(currentTag, currentContent, assistantMessage); - onUpdate({ ...assistantMessage }); + if (currentTag === 'thought' || currentTag === 'action_input_description') { + this.processStreamedContent(currentTag, currentContent); + onUpdate(this.processingInfo); } } } @@ -214,24 +170,24 @@ export class Agent { return fullResponse; } - private processStreamedContent(tag: string, content: string, assistantMessage: AssistantMessage) { - if (!assistantMessage.thoughts) { - assistantMessage.thoughts = []; - } - if (!assistantMessage.actions) { - assistantMessage.actions = []; - } + private processStreamedContent(tag: string, streamed_content: string) { + const current_info = this.processingInfo.find(info => info.index === this.processingNumber); switch (tag) { case 'thought': - assistantMessage.thoughts[this.thoughtNumber] = content; - break; - case 'action': - assistantMessage.actions[this.actionNumber].action = content; - - break; - case 'action_input': - assistantMessage.actions[this.actionNumber].action_input = content; + if (current_info) { + current_info.content = streamed_content; + } else { + console.log(`Adding thought: ${streamed_content}`); + this.processingInfo.push({ index: this.processingNumber, type: PROCESSING_TYPE.THOUGHT, content: streamed_content.trim() }); + } break; + case 'action_input_description': + if (current_info) { + current_info.content = streamed_content; + } else { + console.log(`Adding thought: ${streamed_content}`); + this.processingInfo.push({ index: this.processingNumber, type: PROCESSING_TYPE.ACTION, content: streamed_content.trim() }); + } } } diff --git a/src/client/views/nodes/ChatBox/AnswerParser.ts b/src/client/views/nodes/ChatBox/AnswerParser.ts index 68637b7c7..1d46a366d 100644 --- a/src/client/views/nodes/ChatBox/AnswerParser.ts +++ b/src/client/views/nodes/ChatBox/AnswerParser.ts @@ -1,8 +1,8 @@ -import { ASSISTANT_ROLE, AssistantMessage, Citation, CHUNK_TYPE, TEXT_TYPE, getChunkType } from './types'; +import { ASSISTANT_ROLE, AssistantMessage, Citation, CHUNK_TYPE, TEXT_TYPE, getChunkType, ProcessingInfo } from './types'; import { v4 as uuid } from 'uuid'; export class AnswerParser { - static parse(xml: string, currentMessage: AssistantMessage): AssistantMessage { + static parse(xml: string, processingInfo: ProcessingInfo[]): AssistantMessage { const answerRegex = /([\s\S]*?)<\/answer>/; const citationsRegex = /([\s\S]*?)<\/citations>/; const citationRegex = /([\s\S]*?)<\/citation>/g; @@ -102,10 +102,15 @@ export class AnswerParser { followUpQuestions.push(questionMatch[1].trim()); } } - currentMessage.content = currentMessage.content.concat(content); - currentMessage.citations = citations; - currentMessage.follow_up_questions = followUpQuestions; - return currentMessage; + const assistantResponse: AssistantMessage = { + role: ASSISTANT_ROLE.ASSISTANT, + content, + follow_up_questions: followUpQuestions, + citations, + processing_info: processingInfo, + }; + + return assistantResponse; } } diff --git a/src/client/views/nodes/ChatBox/ChatBox.tsx b/src/client/views/nodes/ChatBox/ChatBox.tsx index 099c0298e..36416a330 100644 --- a/src/client/views/nodes/ChatBox/ChatBox.tsx +++ b/src/client/views/nodes/ChatBox/ChatBox.tsx @@ -11,7 +11,7 @@ import { ViewBoxAnnotatableComponent } from '../../DocComponent'; import { FieldView, FieldViewProps } from '../FieldView'; import './ChatBox.scss'; import MessageComponentBox from './MessageComponent'; -import { ASSISTANT_ROLE, AssistantMessage, AI_Document, Citation, CHUNK_TYPE, RAGChunk, getChunkType, TEXT_TYPE, SimplifiedChunk } from './types'; +import { ASSISTANT_ROLE, AssistantMessage, AI_Document, Citation, CHUNK_TYPE, RAGChunk, getChunkType, TEXT_TYPE, SimplifiedChunk, ProcessingInfo } from './types'; import { Vectorstore } from './vectorstore/Vectorstore'; import { Agent } from './Agent'; import dotenv from 'dotenv'; @@ -150,26 +150,30 @@ export class ChatBox extends ViewBoxAnnotatableComponent() { if (trimmedText) { try { textInput.value = ''; - this.history.push({ role: ASSISTANT_ROLE.USER, content: [{ index: 0, type: TEXT_TYPE.NORMAL, text: trimmedText, citation_ids: null }] }); + this.history.push({ role: ASSISTANT_ROLE.USER, content: [{ index: 0, type: TEXT_TYPE.NORMAL, text: trimmedText, citation_ids: null }], processing_info: [] }); this.isLoading = true; - this.current_message = { role: ASSISTANT_ROLE.ASSISTANT, content: [], thoughts: [], actions: [], citations: [] }; + this.current_message = { role: ASSISTANT_ROLE.ASSISTANT, content: [], citations: [], processing_info: [] }; - const onUpdate = (update: AssistantMessage) => { + const onUpdate = (update: ProcessingInfo[]) => { runInAction(() => { - this.current_message = { ...update }; + if (this.current_message) { + this.current_message = { ...this.current_message, processing_info: update }; + } }); }; - const finalMessage = await this.agent.askAgent(trimmedText, 20, onUpdate); + const finalMessage = await this.agent.askAgent(trimmedText, onUpdate); runInAction(() => { - this.history.push({ ...finalMessage }); - this.current_message = undefined; - this.dataDoc.data = JSON.stringify(this.history); + if (this.current_message) { + this.history.push({ ...finalMessage }); + this.current_message = undefined; + this.dataDoc.data = JSON.stringify(this.history); + } }); } catch (err) { console.error('Error:', err); - this.history.push({ role: ASSISTANT_ROLE.ASSISTANT, content: [{ index: 0, type: TEXT_TYPE.NORMAL, text: 'Sorry, I encountered an error while processing your request.', citation_ids: null }] }); + this.history.push({ role: ASSISTANT_ROLE.ASSISTANT, content: [{ index: 0, type: TEXT_TYPE.ERROR, text: 'Sorry, I encountered an error while processing your request.', citation_ids: null }], processing_info: [] }); } finally { this.isLoading = false; } @@ -295,6 +299,7 @@ export class ChatBox extends ViewBoxAnnotatableComponent() { this.history.push({ role: ASSISTANT_ROLE.ASSISTANT, content: [{ index: 0, type: TEXT_TYPE.NORMAL, text: 'Welcome to the Document Analyser Assistant! Link a document or ask questions to get started.', citation_ids: null }], + processing_info: [], }); }); } diff --git a/src/client/views/nodes/ChatBox/MessageComponent.tsx b/src/client/views/nodes/ChatBox/MessageComponent.tsx index e82dcd5f7..0b8fa6b96 100644 --- a/src/client/views/nodes/ChatBox/MessageComponent.tsx +++ b/src/client/views/nodes/ChatBox/MessageComponent.tsx @@ -1,6 +1,6 @@ import React from 'react'; import { observer } from 'mobx-react'; -import { AssistantMessage, Citation, MessageContent, TEXT_TYPE } from './types'; +import { AssistantMessage, Citation, MessageContent, PROCESSING_TYPE, ProcessingInfo, TEXT_TYPE } from './types'; import Markdown from 'react-markdown'; interface MessageComponentProps { @@ -68,22 +68,24 @@ const MessageComponentBox: React.FC = function ({ message } }; + console.log(message.processing_info); + return (
- {message.thoughts && - message.thoughts.map((thought, idx) => ( -
- Thought: {thought} -
- ))} - {message.actions && - message.actions.map((action, idx) => ( -
- Action: {action.action} -
- Input: {action.action_input} -
- ))} + {message.processing_info && + (message.processing_info as ProcessingInfo[]).map(item => + item.type === PROCESSING_TYPE.THOUGHT ? ( +
+ Thought: {item.content} +
+ ) : item.type === PROCESSING_TYPE.ACTION ? ( +
+ Action: {item.content} +
+ ) : ( +
+ ) + )}
{message.content && message.content.map(messageFragment => {renderContent(messageFragment)})}
{message.follow_up_questions && message.follow_up_questions.length > 0 && (
diff --git a/src/client/views/nodes/ChatBox/prompts.ts b/src/client/views/nodes/ChatBox/prompts.ts index 5f4f79a7f..4a67ac1d1 100644 --- a/src/client/views/nodes/ChatBox/prompts.ts +++ b/src/client/views/nodes/ChatBox/prompts.ts @@ -16,7 +16,7 @@ export function getReactPrompt(tools: Tool[], summaries: () => string, chatHisto return ` - You are an advanced AI assistant equipped with various tools to answer user queries accurately and efficiently. Your task is to provide a comprehensive response based on the user’s prompt using available tools, chat history, and provided information. Follow these guidelines meticulously to ensure the accuracy and structure of your response. + You are an advanced AI assistant equipped with various tools to answer user queries accurately and efficiently. Your task is to provide a comprehensive response based on the user's prompt using available tools, chat history, and provided information. Follow these guidelines meticulously to ensure the accuracy and structure of your response. @@ -42,7 +42,7 @@ export function getReactPrompt(tools: Tool[], summaries: () => string, chatHisto <citation> - Provide citations for each grounded text, referencing the tool or chunk used. - <question> - Include exactly three follow-up questions from the user’s perspective within these tags. + <question> - Include exactly three follow-up questions from the user's perspective within these tags. @@ -73,20 +73,20 @@ export function getReactPrompt(tools: Tool[], summaries: () => string, chatHisto Specify the correct type: "text", "image", "table", "csv", or "url". For text-based information, include only the relevant subset of the original information that the <grounded_text> is based on. For image, table, csv, or url citation types, leave the citation content empty. - ALL CITATIONS MUST use the chunk_id field to reference the source, whether it’s from RAG, search + website scraping, data analysis, or any other tool. + ALL CITATIONS MUST use the chunk_id field to reference the source, whether it's from RAG, search + website scraping, data analysis, or any other tool. One citation can be used for multiple <grounded_text> tags if they are based on the same tool or chunk information. !!!DO NOT OVERCITE - only include citations for information that is directly relevant to the <grounded_text>. - Analyze the user’s query carefully. + Analyze the user's query carefully. Determine whether a tool is required to answer the query accurately. If a tool is necessary: Select the most appropriate tool. Use the <action> tag to specify the tool. End your response after the <action> tag and wait for action rules to be provided. - Based on the action rules, provide the necessary tool parameters within the <action_input> tag. + Based on the action rules, provide the necessary tool parameters within the <action_input> tag, including a brief description of what you're doing with the action. End your response again and wait for the observation from the tool. If no tool is needed, use the 'no_tool' action but still follow the same response structure. @@ -97,8 +97,8 @@ export function getReactPrompt(tools: Tool[], summaries: () => string, chatHisto Your final <answer> tag must contain: - The complete answer to the user’s query, with grounded information wrapped in <grounded_text> tags and general information wrapped in <normal_text> tags. - Exactly three follow-up questions written from the user’s perspective, enclosed within <follow_up_questions> tags. + The complete answer to the user's query, with grounded information wrapped in <grounded_text> tags and general information wrapped in <normal_text> tags. + Exactly three follow-up questions written from the user's perspective, enclosed within <follow_up_questions> tags. @@ -138,9 +138,12 @@ export function getReactPrompt(tools: Tool[], summaries: () => string, chatHisto - - The user is asking for key moments and statements from official sources regarding the 2010 Vancouver Winter Olympics. Search the provided documents for any press releases or official statements that highlight significant events, achievements, or noteworthy aspects of the games. - + Searching user documents for official statements and key moments of the 2010 Vancouver Winter Olympics. + + + The user is asking for key moments and statements from official sources regarding the 2010 Vancouver Winter Olympics. Search the provided documents for any press releases or official statements that highlight significant events, achievements, or noteworthy aspects of the games. + + @@ -177,7 +180,10 @@ export function getReactPrompt(tools: Tool[], summaries: () => string, chatHisto - 2010_Vancouver_Olympics_Medal_Count.csv + Analyzing the medal count data for the 2010 Vancouver Winter Olympics to compare country performances. + + 2010_Vancouver_Olympics_Medal_Count.csv + @@ -248,7 +254,10 @@ export function getReactPrompt(tools: Tool[], summaries: () => string, chatHisto - Best hiking trails Yosemite National Park + Searching the web for information about the best hiking trails in Yosemite National Park. + + Best hiking trails Yosemite National Park + @@ -283,7 +292,10 @@ export function getReactPrompt(tools: Tool[], summaries: () => string, chatHisto - https://www.nps.gov/yose/planyourvisit/hiking.htm + Scraping information about hiking trails from the official Yosemite National Park website. + + https://www.nps.gov/yose/planyourvisit/hiking.htm + @@ -309,7 +321,10 @@ export function getReactPrompt(tools: Tool[], summaries: () => string, chatHisto - https://www.alltrails.com/parks/us/california/yosemite-national-park + Scraping user reviews and ratings for Yosemite hiking trails from AllTrails. + + https://www.alltrails.com/parks/us/california/yosemite-national-park + @@ -335,7 +350,10 @@ export function getReactPrompt(tools: Tool[], summaries: () => string, chatHisto - https://www.yosemitehikes.com/ + Gathering detailed trail descriptions and hiking information from YosemiteHikes.com. + + https://www.yosemitehikes.com/ + @@ -410,7 +428,7 @@ export function getReactPrompt(tools: Tool[], summaries: () => string, chatHisto - Now, process the user’s query and provide your response following the format and rules outlined above. Ensure your final answer is comprehensive, correctly cited, and entirely contained within the structured tags. Do not get stuck in infinite loops and keep responses concise, grounded, and most importantly, HELPFUL AND USEFUL! + Now, process the user's query and provide your response following the format and rules outlined above. Ensure your final answer is comprehensive, correctly cited, and entirely contained within the structured tags. Do not get stuck in infinite loops and keep responses concise, grounded, and most importantly, HELPFUL AND USEFUL! `; diff --git a/src/client/views/nodes/ChatBox/types.ts b/src/client/views/nodes/ChatBox/types.ts index b4e66bdbe..09f14f019 100644 --- a/src/client/views/nodes/ChatBox/types.ts +++ b/src/client/views/nodes/ChatBox/types.ts @@ -1,6 +1,8 @@ import { breadcrumbsClasses } from '@mui/material'; import { Doc } from '../../../../fields/Doc'; import { StrCast } from '../../../../fields/Types'; +import e from 'cors'; +import { index } from 'd3'; export enum ASSISTANT_ROLE { USER = 'user', @@ -21,6 +23,12 @@ export enum CHUNK_TYPE { CSV = 'CSV', } +export enum PROCESSING_TYPE { + THOUGHT = 'thought', + ACTION = 'action', + //eventually migrate error to here +} + export function getChunkType(type: string): CHUNK_TYPE { switch (type.toLowerCase()) { case 'text': @@ -44,13 +52,18 @@ export function getChunkType(type: string): CHUNK_TYPE { } } +export interface ProcessingInfo { + index: number; + type: PROCESSING_TYPE; + content: string; +} + export interface AssistantMessage { role: ASSISTANT_ROLE; content: MessageContent[]; follow_up_questions?: string[]; - thoughts?: string[]; - actions?: { index: number; action: string; action_input: string }[]; citations?: Citation[]; + processing_info: ProcessingInfo[]; } export interface MessageContent { -- cgit v1.2.3-70-g09d2 From 4c0c7794c85cfdbcd61a7ee5cb9a29494fd0444b Mon Sep 17 00:00:00 2001 From: "A.J. Shulman" Date: Tue, 20 Aug 2024 15:17:25 -0400 Subject: better styling, now thoughts and actions are hidden, scroll works better MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit next steps: - [ ] Ensure it doesn’t create more web documents when one already exists - [ ] Citations should not be rendered on the next line but on the same line as the text - [ ] If invalid XML, run get 3.5 to verify and fix XML based one examples - [ ] Making sure if you ask for other information, it doesn’t go to the same website. Providing website history in use rules for the search tool and website scraper tool or in the prompt directly --- package-lock.json | 633 ++++++++++++++++++++- package.json | 1 + src/client/util/CurrentUserUtils.ts | 2 +- src/client/views/nodes/ChatBox/ChatBox.scss | 402 ++++++------- src/client/views/nodes/ChatBox/ChatBox.tsx | 101 +++- .../views/nodes/ChatBox/MessageComponent.scss | 10 - .../views/nodes/ChatBox/MessageComponent.tsx | 97 ++-- src/server/ApiManagers/AssistantManager.ts | 64 +-- 8 files changed, 978 insertions(+), 332 deletions(-) delete mode 100644 src/client/views/nodes/ChatBox/MessageComponent.scss (limited to 'src/client/views/nodes/ChatBox/MessageComponent.tsx') diff --git a/package-lock.json b/package-lock.json index 601addf4e..68c9e9746 100644 --- a/package-lock.json +++ b/package-lock.json @@ -180,6 +180,7 @@ "prosemirror-transform": "^1.8.0", "prosemirror-view": "^1.32.5", "pug": "^3.0.2", + "puppeteer": "^23.1.0", "query-string": "^7.1.3", "querystring-es3": "^0.2.1", "raw-loader": "^4.0.2", @@ -5204,6 +5205,140 @@ "url": "https://opencollective.com/popperjs" } }, + "node_modules/@puppeteer/browsers": { + "version": "2.3.1", + "resolved": "https://registry.npmjs.org/@puppeteer/browsers/-/browsers-2.3.1.tgz", + "integrity": "sha512-uK7o3hHkK+naEobMSJ+2ySYyXtQkBxIH8Gn4MK9ciePjNV+Pf+PgY/W7iPzn2MTjl3stcYB5AlcTmPYw7AXDwA==", + "dependencies": { + "debug": "^4.3.6", + "extract-zip": "^2.0.1", + "progress": "^2.0.3", + "proxy-agent": "^6.4.0", + "semver": "^7.6.3", + "tar-fs": "^3.0.6", + "unbzip2-stream": "^1.4.3", + "yargs": "^17.7.2" + }, + "bin": { + "browsers": "lib/cjs/main-cli.js" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/@puppeteer/browsers/node_modules/ansi-styles": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz", + "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==", + "dependencies": { + "color-convert": "^2.0.1" + }, + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/chalk/ansi-styles?sponsor=1" + } + }, + "node_modules/@puppeteer/browsers/node_modules/cliui": { + "version": "8.0.1", + "resolved": "https://registry.npmjs.org/cliui/-/cliui-8.0.1.tgz", + "integrity": "sha512-BSeNnyus75C4//NQ9gQt1/csTXyo/8Sb+afLAkzAptFuMsod9HFokGNudZpi/oQV73hnVK+sR+5PVRMd+Dr7YQ==", + "dependencies": { + "string-width": "^4.2.0", + "strip-ansi": "^6.0.1", + "wrap-ansi": "^7.0.0" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/@puppeteer/browsers/node_modules/color-convert": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz", + "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==", + "dependencies": { + "color-name": "~1.1.4" + }, + "engines": { + "node": ">=7.0.0" + } + }, + "node_modules/@puppeteer/browsers/node_modules/color-name": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz", + "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==" + }, + "node_modules/@puppeteer/browsers/node_modules/emoji-regex": { + "version": "8.0.0", + "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz", + "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==" + }, + "node_modules/@puppeteer/browsers/node_modules/semver": { + "version": "7.6.3", + "resolved": "https://registry.npmjs.org/semver/-/semver-7.6.3.tgz", + "integrity": "sha512-oVekP1cKtI+CTDvHWYFUcMtsK/00wmAEfyqKfNdARm8u1wNVhSgaX7A8d4UuIlUI5e84iEwOhs7ZPYRmzU9U6A==", + "bin": { + "semver": "bin/semver.js" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/@puppeteer/browsers/node_modules/string-width": { + "version": "4.2.3", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", + "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", + "dependencies": { + "emoji-regex": "^8.0.0", + "is-fullwidth-code-point": "^3.0.0", + "strip-ansi": "^6.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/@puppeteer/browsers/node_modules/wrap-ansi": { + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-7.0.0.tgz", + "integrity": "sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==", + "dependencies": { + "ansi-styles": "^4.0.0", + "string-width": "^4.1.0", + "strip-ansi": "^6.0.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/chalk/wrap-ansi?sponsor=1" + } + }, + "node_modules/@puppeteer/browsers/node_modules/yargs": { + "version": "17.7.2", + "resolved": "https://registry.npmjs.org/yargs/-/yargs-17.7.2.tgz", + "integrity": "sha512-7dSzzRQ++CKnNI/krKnYRV7JKKPUXMEh61soaHKg9mrWEhzFWhFnxPxGl+69cD1Ou63C13NUPCnmIcrvqCuM6w==", + "dependencies": { + "cliui": "^8.0.1", + "escalade": "^3.1.1", + "get-caller-file": "^2.0.5", + "require-directory": "^2.1.1", + "string-width": "^4.2.3", + "y18n": "^5.0.5", + "yargs-parser": "^21.1.1" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/@puppeteer/browsers/node_modules/yargs-parser": { + "version": "21.1.1", + "resolved": "https://registry.npmjs.org/yargs-parser/-/yargs-parser-21.1.1.tgz", + "integrity": "sha512-tVpsJW7DdjecAiFpbIB1e3qxIQsE6NoPc5/eTdrbbIC4h0LVsWhnoa3g+m2HclBIujHzsxZ4VJVA+GUuc2/LBw==", + "engines": { + "node": ">=12" + } + }, "node_modules/@react-aria/actiongroup": { "version": "3.7.5", "resolved": "https://registry.npmjs.org/@react-aria/actiongroup/-/actiongroup-3.7.5.tgz", @@ -8786,6 +8921,11 @@ "resolved": "https://registry.npmjs.org/@tokenizer/token/-/token-0.3.0.tgz", "integrity": "sha512-OvjF+z51L3ov0OyAU0duzsYuvO01PH7x4t6DJx+guahgTnBHkhJdG7soQeTSFLWN3efnHyibZ4Z8l2EuWwJN3A==" }, + "node_modules/@tootallnate/quickjs-emscripten": { + "version": "0.23.0", + "resolved": "https://registry.npmjs.org/@tootallnate/quickjs-emscripten/-/quickjs-emscripten-0.23.0.tgz", + "integrity": "sha512-C5Mc6rdnsaJDjO3UpGW/CQTHtCKaYlScZTly4JIu97Jxo/odCiH0ITnDXSJPTOrEKk/ycSZ0AOgTmkDtkOsvIA==" + }, "node_modules/@tsconfig/node10": { "version": "1.0.11", "resolved": "https://registry.npmjs.org/@tsconfig/node10/-/node10-1.0.11.tgz", @@ -11530,6 +11670,15 @@ "resolved": "https://registry.npmjs.org/@types/yargs-parser/-/yargs-parser-21.0.3.tgz", "integrity": "sha512-I4q9QU9MQv4oEOz4tAHJtNz1cwuLxn2F3xcc2iV5WdqLPpUnj30aUuxt1mAxYTG+oe8CZMV/+6rU4S4gRDzqtQ==" }, + "node_modules/@types/yauzl": { + "version": "2.10.3", + "resolved": "https://registry.npmjs.org/@types/yauzl/-/yauzl-2.10.3.tgz", + "integrity": "sha512-oJoftv0LSuaDZE3Le4DbKX+KS9G36NzOeSap90UIK0yMA/NhKJhqlSGtNDORNRaIbQfzjXDrQa0ytJ6mNRGz/Q==", + "optional": true, + "dependencies": { + "@types/node": "*" + } + }, "node_modules/@types/youtube": { "version": "0.0.50", "resolved": "https://registry.npmjs.org/@types/youtube/-/youtube-0.0.50.tgz", @@ -12594,6 +12743,17 @@ "node": ">=0.10.0" } }, + "node_modules/ast-types": { + "version": "0.13.4", + "resolved": "https://registry.npmjs.org/ast-types/-/ast-types-0.13.4.tgz", + "integrity": "sha512-x1FCFnFifvYDDzTaLII71vG5uvDwgtmDTEVWAxrgeiR8VjMONcCXJx7E+USjDtHlwFmt9MysbqgF9b9Vjr6w+w==", + "dependencies": { + "tslib": "^2.0.1" + }, + "engines": { + "node": ">=4" + } + }, "node_modules/ast-types-flow": { "version": "0.0.8", "resolved": "https://registry.npmjs.org/ast-types-flow/-/ast-types-flow-0.0.8.tgz", @@ -12854,6 +13014,41 @@ "integrity": "sha512-qMKFd2qG/36aA4GwvKq8MxnPgCQAmBWmSyLWsJcbn8v03wvIPQ/hG1Ms8bPzndZxMDoHpxez5VOS+gC9Yi24/Q==", "optional": true }, + "node_modules/bare-fs": { + "version": "2.3.1", + "resolved": "https://registry.npmjs.org/bare-fs/-/bare-fs-2.3.1.tgz", + "integrity": "sha512-W/Hfxc/6VehXlsgFtbB5B4xFcsCl+pAh30cYhoFyXErf6oGrwjh8SwiPAdHgpmWonKuYpZgGywN0SXt7dgsADA==", + "optional": true, + "dependencies": { + "bare-events": "^2.0.0", + "bare-path": "^2.0.0", + "bare-stream": "^2.0.0" + } + }, + "node_modules/bare-os": { + "version": "2.4.0", + "resolved": "https://registry.npmjs.org/bare-os/-/bare-os-2.4.0.tgz", + "integrity": "sha512-v8DTT08AS/G0F9xrhyLtepoo9EJBJ85FRSMbu1pQUlAf6A8T0tEEQGMVObWeqpjhSPXsE0VGlluFBJu2fdoTNg==", + "optional": true + }, + "node_modules/bare-path": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/bare-path/-/bare-path-2.1.3.tgz", + "integrity": "sha512-lh/eITfU8hrj9Ru5quUp0Io1kJWIk1bTjzo7JH1P5dWmQ2EL4hFUlfI8FonAhSlgIfhn63p84CDY/x+PisgcXA==", + "optional": true, + "dependencies": { + "bare-os": "^2.1.0" + } + }, + "node_modules/bare-stream": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/bare-stream/-/bare-stream-2.1.3.tgz", + "integrity": "sha512-tiDAH9H/kP+tvNO5sczyn9ZAA7utrSMobyDchsnyyXBuUe2FSQWbxhtuHB8jwpHYYevVo2UJpcmvvjrbHboUUQ==", + "optional": true, + "dependencies": { + "streamx": "^2.18.0" + } + }, "node_modules/Base64": { "version": "0.2.1", "resolved": "https://registry.npmjs.org/Base64/-/Base64-0.2.1.tgz", @@ -12894,6 +13089,14 @@ "node": ">=6.0.0" } }, + "node_modules/basic-ftp": { + "version": "5.0.5", + "resolved": "https://registry.npmjs.org/basic-ftp/-/basic-ftp-5.0.5.tgz", + "integrity": "sha512-4Bcg1P8xhUuqcii/S0Z9wiHIrQVPMermM1any+MX5GeGD7faD3/msQUDGLol9wOcz4/jbg/WJnGqoJF6LiBdtg==", + "engines": { + "node": ">=10.0.0" + } + }, "node_modules/batch": { "version": "0.6.1", "resolved": "https://registry.npmjs.org/batch/-/batch-0.6.1.tgz", @@ -16765,6 +16968,19 @@ "node": ">=6.0" } }, + "node_modules/chromium-bidi": { + "version": "0.6.4", + "resolved": "https://registry.npmjs.org/chromium-bidi/-/chromium-bidi-0.6.4.tgz", + "integrity": "sha512-8zoq6ogmhQQkAKZVKO2ObFTl4uOkqoX1PlKQX3hZQ5E9cbUotcAb7h4pTNVAGGv8Z36PF3CtdOriEp/Rz82JqQ==", + "dependencies": { + "mitt": "3.0.1", + "urlpattern-polyfill": "10.0.0", + "zod": "3.23.8" + }, + "peerDependencies": { + "devtools-protocol": "*" + } + }, "node_modules/ci-info": { "version": "3.9.0", "resolved": "https://registry.npmjs.org/ci-info/-/ci-info-3.9.0.tgz", @@ -18345,9 +18561,9 @@ "integrity": "sha512-okzr3f11N6WuqYtZSvm+F776mB41wRZMhKP+hc34YdW+KmtYYK9iqvHSwo2k9FEH3fhGXvOPV6yz2IcSrfRUDg==" }, "node_modules/debug": { - "version": "4.3.5", - "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.5.tgz", - "integrity": "sha512-pt0bNEmneDIvdL1Xsd9oDQ/wrQRkXDT4AUWlNZNPKvW5x/jyO9VFXkJUP07vQ2upmw5PlaITaPKc31jK13V+jg==", + "version": "4.3.6", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.6.tgz", + "integrity": "sha512-O/09Bd4Z1fBrU4VzkhFqVgpPzaGbw6Sm9FEkBT1A/YBXQFGuuSxa1dN2nxgxS34JmKXqYx8CZAwEVoJFImUXIg==", "dependencies": { "ms": "2.1.2" }, @@ -18582,6 +18798,19 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/degenerator": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/degenerator/-/degenerator-5.0.1.tgz", + "integrity": "sha512-TllpMR/t0M5sqCXfj85i4XaAzxmS5tVA16dqvdkMwGmzI+dXLXnw3J+3Vdv7VKw+ThlTMboK6i9rnZ6Nntj5CQ==", + "dependencies": { + "ast-types": "^0.13.4", + "escodegen": "^2.1.0", + "esprima": "^4.0.1" + }, + "engines": { + "node": ">= 14" + } + }, "node_modules/delaunator": { "version": "5.0.1", "resolved": "https://registry.npmjs.org/delaunator/-/delaunator-5.0.1.tgz", @@ -18738,6 +18967,11 @@ "url": "https://github.com/sponsors/wooorm" } }, + "node_modules/devtools-protocol": { + "version": "0.0.1312386", + "resolved": "https://registry.npmjs.org/devtools-protocol/-/devtools-protocol-0.0.1312386.tgz", + "integrity": "sha512-DPnhUXvmvKT2dFA/j7B+riVLUt9Q6RKJlcppojL5CoRywJJKLDYnRlw0gTFKfgDPHP5E04UoB71SxoJlVZy8FA==" + }, "node_modules/dezalgo": { "version": "1.0.4", "resolved": "https://registry.npmjs.org/dezalgo/-/dezalgo-1.0.4.tgz", @@ -19011,6 +19245,14 @@ "node": ">=0.10.0" } }, + "node_modules/end-of-stream": { + "version": "1.4.4", + "resolved": "https://registry.npmjs.org/end-of-stream/-/end-of-stream-1.4.4.tgz", + "integrity": "sha512-+uw1inIHVPQoaVuHzRyXd21icM+cnt4CzD5rW+NC1wjOUSTOs+Te7FOv7AhN7vS9x/oIyhLP5PR1H+phQAHu5Q==", + "dependencies": { + "once": "^1.4.0" + } + }, "node_modules/engine.io": { "version": "6.5.5", "resolved": "https://registry.npmjs.org/engine.io/-/engine.io-6.5.5.tgz", @@ -19114,6 +19356,14 @@ "url": "https://github.com/fb55/entities?sponsor=1" } }, + "node_modules/env-paths": { + "version": "2.2.1", + "resolved": "https://registry.npmjs.org/env-paths/-/env-paths-2.2.1.tgz", + "integrity": "sha512-+h1lkLKhZMTYjog1VEpJNG7NZJWcuc2DDk/qsqSTRRCOXiLjeQ1d1/udrUGhqMxUgAlwKNZ0cf2uqan5GLuS2A==", + "engines": { + "node": ">=6" + } + }, "node_modules/envinfo": { "version": "7.13.0", "resolved": "https://registry.npmjs.org/envinfo/-/envinfo-7.13.0.tgz", @@ -19343,6 +19593,35 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/escodegen": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/escodegen/-/escodegen-2.1.0.tgz", + "integrity": "sha512-2NlIDTwUWJN0mRPQOdtQBzbUHvdGY2P1VXSyU83Q3xKxM7WHX2Ql8dKq782Q9TgQUNOLEzEYu9bzLNj1q88I5w==", + "dependencies": { + "esprima": "^4.0.1", + "estraverse": "^5.2.0", + "esutils": "^2.0.2" + }, + "bin": { + "escodegen": "bin/escodegen.js", + "esgenerate": "bin/esgenerate.js" + }, + "engines": { + "node": ">=6.0" + }, + "optionalDependencies": { + "source-map": "~0.6.1" + } + }, + "node_modules/escodegen/node_modules/source-map": { + "version": "0.6.1", + "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz", + "integrity": "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==", + "optional": true, + "engines": { + "node": ">=0.10.0" + } + }, "node_modules/eslint": { "version": "8.57.0", "resolved": "https://registry.npmjs.org/eslint/-/eslint-8.57.0.tgz", @@ -21286,6 +21565,39 @@ "resolved": "https://registry.npmjs.org/extract-colors/-/extract-colors-4.0.6.tgz", "integrity": "sha512-U+pYyQKXCSHOmtZPIEJBGLJjLDiqS+oOub2ILA3a7UGt9+IvZvwAN3hOPFjUgT+gX/apSBwP5vBgnKMlV0fy8Q==" }, + "node_modules/extract-zip": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/extract-zip/-/extract-zip-2.0.1.tgz", + "integrity": "sha512-GDhU9ntwuKyGXdZBUgTIe+vXnWj0fppUEtMDL0+idd5Sta8TGpHssn/eusA9mrPr9qNDym6SxAYZjNvCn/9RBg==", + "dependencies": { + "debug": "^4.1.1", + "get-stream": "^5.1.0", + "yauzl": "^2.10.0" + }, + "bin": { + "extract-zip": "cli.js" + }, + "engines": { + "node": ">= 10.17.0" + }, + "optionalDependencies": { + "@types/yauzl": "^2.9.1" + } + }, + "node_modules/extract-zip/node_modules/get-stream": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/get-stream/-/get-stream-5.2.0.tgz", + "integrity": "sha512-nBF+F1rAZVCu/p7rjzgA+Yb4lfYXrpl7a6VmJrU8wF9I1CKvP/QwPNZHnOlwbTkY6dvtFIzFMSyQXbLoTQPRpA==", + "dependencies": { + "pump": "^3.0.0" + }, + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/extsprintf": { "version": "1.3.0", "resolved": "https://registry.npmjs.org/extsprintf/-/extsprintf-1.3.0.tgz", @@ -21410,6 +21722,14 @@ "node": ">=0.8.0" } }, + "node_modules/fd-slicer": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/fd-slicer/-/fd-slicer-1.1.0.tgz", + "integrity": "sha512-cE1qsB/VwyQozZ+q1dGxR8LBYNZeofhEdUNGSMbQD3Gw2lAzX9Zb3uIU6Ebc/Fmyjo9AWWfnn0AUCHqtevs/8g==", + "dependencies": { + "pend": "~1.2.0" + } + }, "node_modules/fetch-blob": { "version": "2.1.2", "resolved": "https://registry.npmjs.org/fetch-blob/-/fetch-blob-2.1.2.tgz", @@ -22480,6 +22800,41 @@ "url": "https://github.com/privatenumber/get-tsconfig?sponsor=1" } }, + "node_modules/get-uri": { + "version": "6.0.3", + "resolved": "https://registry.npmjs.org/get-uri/-/get-uri-6.0.3.tgz", + "integrity": "sha512-BzUrJBS9EcUb4cFol8r4W3v1cPsSyajLSthNkz5BxbpDcHN5tIrM10E2eNvfnvBn3DaT3DUgx0OpsBKkaOpanw==", + "dependencies": { + "basic-ftp": "^5.0.2", + "data-uri-to-buffer": "^6.0.2", + "debug": "^4.3.4", + "fs-extra": "^11.2.0" + }, + "engines": { + "node": ">= 14" + } + }, + "node_modules/get-uri/node_modules/data-uri-to-buffer": { + "version": "6.0.2", + "resolved": "https://registry.npmjs.org/data-uri-to-buffer/-/data-uri-to-buffer-6.0.2.tgz", + "integrity": "sha512-7hvf7/GW8e86rW0ptuwS3OcBGDjIi6SZva7hCyWC0yYry2cOPmLIjXAUHI6DK2HsnwJd9ifmt57i8eV2n4YNpw==", + "engines": { + "node": ">= 14" + } + }, + "node_modules/get-uri/node_modules/fs-extra": { + "version": "11.2.0", + "resolved": "https://registry.npmjs.org/fs-extra/-/fs-extra-11.2.0.tgz", + "integrity": "sha512-PmDi3uwK5nFuXh7XDTlVnS17xJS7vW36is2+w3xcv8SVxiB4NyATf4ctkVY5bkSjX0Y4nbvZCq1/EjtEyr9ktw==", + "dependencies": { + "graceful-fs": "^4.2.0", + "jsonfile": "^6.0.1", + "universalify": "^2.0.0" + }, + "engines": { + "node": ">=14.14" + } + }, "node_modules/get-value": { "version": "2.0.6", "resolved": "https://registry.npmjs.org/get-value/-/get-value-2.0.6.tgz", @@ -24224,6 +24579,28 @@ "loose-envify": "^1.0.0" } }, + "node_modules/ip-address": { + "version": "9.0.5", + "resolved": "https://registry.npmjs.org/ip-address/-/ip-address-9.0.5.tgz", + "integrity": "sha512-zHtQzGojZXTwZTHQqra+ETKd4Sn3vgi7uBmlPoXVWZqYvuKmtI0l/VZTjqGmJY9x88GGOaZ9+G9ES8hC4T4X8g==", + "dependencies": { + "jsbn": "1.1.0", + "sprintf-js": "^1.1.3" + }, + "engines": { + "node": ">= 12" + } + }, + "node_modules/ip-address/node_modules/jsbn": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/jsbn/-/jsbn-1.1.0.tgz", + "integrity": "sha512-4bYVV3aAMtDTTu4+xsDYa6sy9GyJ69/amsu9sYF2zqjiEoZA5xJi3BrfX3uY+/IekIu7MwdObdbDWpoZdBv3/A==" + }, + "node_modules/ip-address/node_modules/sprintf-js": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/sprintf-js/-/sprintf-js-1.1.3.tgz", + "integrity": "sha512-Oo+0REFV59/rz3gfJNKQiBlwfHaSESl1pcGyABQsnnIfWOFt6JNj5gCog2U6MLZ//IGYD+nA8nI+mTShREReaA==" + }, "node_modules/ipaddr.js": { "version": "1.9.1", "resolved": "https://registry.npmjs.org/ipaddr.js/-/ipaddr.js-1.9.1.tgz", @@ -32566,6 +32943,11 @@ "resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz", "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==" }, + "node_modules/mitt": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/mitt/-/mitt-3.0.1.tgz", + "integrity": "sha512-vKivATfr97l2/QBCYAkXYDbrIWPM2IIKEl7YPhjCvKlG3kE2gm+uBo6nEXK3M5/Ffh/FLpKExzOQ3JJoJGFKBw==" + }, "node_modules/mj-context-menu": { "version": "0.6.1", "resolved": "https://registry.npmjs.org/mj-context-menu/-/mj-context-menu-0.6.1.tgz", @@ -33083,6 +33465,14 @@ "resolved": "https://registry.npmjs.org/neo-async/-/neo-async-2.6.2.tgz", "integrity": "sha512-Yd3UES5mWCSqR+qNT93S3UoYUkqAZ9lLg8a7g9rimsWmYGK8cVToA4/sF3RrshdyV3sAGMXVUmpMYOw+dLpOuw==" }, + "node_modules/netmask": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/netmask/-/netmask-2.0.2.tgz", + "integrity": "sha512-dBpDMdxv9Irdq66304OLfEmQ9tbNRFnFTuZiLo+bD+r332bBmMJ8GBLXklIXXgxd3+v9+KUnZaUR5PJMa75Gsg==", + "engines": { + "node": ">= 0.4.0" + } + }, "node_modules/nextafter": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/nextafter/-/nextafter-1.0.0.tgz", @@ -36182,6 +36572,36 @@ "node": ">=6" } }, + "node_modules/pac-proxy-agent": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/pac-proxy-agent/-/pac-proxy-agent-7.0.2.tgz", + "integrity": "sha512-BFi3vZnO9X5Qt6NRz7ZOaPja3ic0PhlsmCRYLOpN11+mWBCR6XJDqW5RF3j8jm4WGGQZtBA+bTfxYzeKW73eHg==", + "dependencies": { + "@tootallnate/quickjs-emscripten": "^0.23.0", + "agent-base": "^7.0.2", + "debug": "^4.3.4", + "get-uri": "^6.0.1", + "http-proxy-agent": "^7.0.0", + "https-proxy-agent": "^7.0.5", + "pac-resolver": "^7.0.1", + "socks-proxy-agent": "^8.0.4" + }, + "engines": { + "node": ">= 14" + } + }, + "node_modules/pac-resolver": { + "version": "7.0.1", + "resolved": "https://registry.npmjs.org/pac-resolver/-/pac-resolver-7.0.1.tgz", + "integrity": "sha512-5NPgf87AT2STgwa2ntRMr45jTKrYBGkVU36yT0ig/n/GMAa3oPqhZfIQ2kMEimReg0+t9kZViDVZ83qfVUlckg==", + "dependencies": { + "degenerator": "^5.0.0", + "netmask": "^2.0.2" + }, + "engines": { + "node": ">= 14" + } + }, "node_modules/package-json-from-dist": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/package-json-from-dist/-/package-json-from-dist-1.0.0.tgz", @@ -36621,6 +37041,11 @@ "url": "https://github.com/sponsors/Borewit" } }, + "node_modules/pend": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/pend/-/pend-1.2.0.tgz", + "integrity": "sha512-F3asv42UuXchdzt+xXqfW1OGlVBe+mxa2mqI0pg5yAHZPvFmY3Y6drSf/GQ1A86WgWEN9Kzh/WrgKa6iGcHXLg==" + }, "node_modules/performance-now": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/performance-now/-/performance-now-2.1.0.tgz", @@ -37021,7 +37446,6 @@ "version": "2.0.3", "resolved": "https://registry.npmjs.org/progress/-/progress-2.0.3.tgz", "integrity": "sha512-7PiHtLll5LdnKIMw100I+8xJXR5gW2QwWYkT6iJva0bXitZKa/XMrSbdmg3r2Xnaidz9Qumd0VPaMrZlF9V9sA==", - "dev": true, "engines": { "node": ">=0.4.0" } @@ -37175,6 +37599,32 @@ "node": ">= 0.10" } }, + "node_modules/proxy-agent": { + "version": "6.4.0", + "resolved": "https://registry.npmjs.org/proxy-agent/-/proxy-agent-6.4.0.tgz", + "integrity": "sha512-u0piLU+nCOHMgGjRbimiXmA9kM/L9EHh3zL81xCdp7m+Y2pHIsnmbdDoEDoAz5geaonNR6q6+yOPQs6n4T6sBQ==", + "dependencies": { + "agent-base": "^7.0.2", + "debug": "^4.3.4", + "http-proxy-agent": "^7.0.1", + "https-proxy-agent": "^7.0.3", + "lru-cache": "^7.14.1", + "pac-proxy-agent": "^7.0.1", + "proxy-from-env": "^1.1.0", + "socks-proxy-agent": "^8.0.2" + }, + "engines": { + "node": ">= 14" + } + }, + "node_modules/proxy-agent/node_modules/lru-cache": { + "version": "7.18.3", + "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-7.18.3.tgz", + "integrity": "sha512-jumlc0BIUrS3qJGgIkWZsyfAM7NCWiBcCDhnd+3NNM5KbBmLTgHVfWBcg6W+rLUsIpzpERPsvwUP7CckAQSOoA==", + "engines": { + "node": ">=12" + } + }, "node_modules/proxy-from-env": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/proxy-from-env/-/proxy-from-env-1.1.0.tgz", @@ -37302,6 +37752,15 @@ "resolved": "https://registry.npmjs.org/pug-walk/-/pug-walk-2.0.0.tgz", "integrity": "sha512-yYELe9Q5q9IQhuvqsZNwA5hfPkMJ8u92bQLIMcsMxf/VADjNtEYptU+inlufAFYcWdHlwNfZOEnOOQrZrcyJCQ==" }, + "node_modules/pump": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/pump/-/pump-3.0.0.tgz", + "integrity": "sha512-LwZy+p3SFs1Pytd/jYct4wpv49HiYCqd9Rlc5ZVdk0V+8Yzv6jR5Blk3TRmPL1ft69TxP0IMZGJ+WPFU2BFhww==", + "dependencies": { + "end-of-stream": "^1.1.0", + "once": "^1.3.1" + } + }, "node_modules/punycode": { "version": "2.3.1", "resolved": "https://registry.npmjs.org/punycode/-/punycode-2.3.1.tgz", @@ -37318,6 +37777,83 @@ "node": ">=6" } }, + "node_modules/puppeteer": { + "version": "23.1.0", + "resolved": "https://registry.npmjs.org/puppeteer/-/puppeteer-23.1.0.tgz", + "integrity": "sha512-m+CyicDlGN1AVUeOsCa6/+KQydJzxfsPowL7fQy+VGNeaWafB0m8G5aGfXdfZztKMxzCsdz7KNNzbJPeG9wwFw==", + "hasInstallScript": true, + "dependencies": { + "@puppeteer/browsers": "2.3.1", + "chromium-bidi": "0.6.4", + "cosmiconfig": "^9.0.0", + "devtools-protocol": "0.0.1312386", + "puppeteer-core": "23.1.0", + "typed-query-selector": "^2.12.0" + }, + "bin": { + "puppeteer": "lib/cjs/puppeteer/node/cli.js" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/puppeteer-core": { + "version": "23.1.0", + "resolved": "https://registry.npmjs.org/puppeteer-core/-/puppeteer-core-23.1.0.tgz", + "integrity": "sha512-SvAsu+xnLN2FMXE/59bp3s3WXp8ewqUGzVV4AQtml/2xmsciZnU/bXcCW+eETHPWQ6Agg2vTI7QzWXPpEARK2g==", + "dependencies": { + "@puppeteer/browsers": "2.3.1", + "chromium-bidi": "0.6.4", + "debug": "^4.3.6", + "devtools-protocol": "0.0.1312386", + "typed-query-selector": "^2.12.0", + "ws": "^8.18.0" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/puppeteer/node_modules/argparse": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/argparse/-/argparse-2.0.1.tgz", + "integrity": "sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q==" + }, + "node_modules/puppeteer/node_modules/cosmiconfig": { + "version": "9.0.0", + "resolved": "https://registry.npmjs.org/cosmiconfig/-/cosmiconfig-9.0.0.tgz", + "integrity": "sha512-itvL5h8RETACmOTFc4UfIyB2RfEHi71Ax6E/PivVxq9NseKbOWpeyHEOIbmAw1rs8Ak0VursQNww7lf7YtUwzg==", + "dependencies": { + "env-paths": "^2.2.1", + "import-fresh": "^3.3.0", + "js-yaml": "^4.1.0", + "parse-json": "^5.2.0" + }, + "engines": { + "node": ">=14" + }, + "funding": { + "url": "https://github.com/sponsors/d-fischer" + }, + "peerDependencies": { + "typescript": ">=4.9.5" + }, + "peerDependenciesMeta": { + "typescript": { + "optional": true + } + } + }, + "node_modules/puppeteer/node_modules/js-yaml": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.1.0.tgz", + "integrity": "sha512-wpxZs9NoxZaJESJGIZTyDEaYpl0FKSA+FB9aJiyemKhMwkxQg63h4T1KJgUGHpTqPDNRcmmYLugrRjJlBtWvRA==", + "dependencies": { + "argparse": "^2.0.1" + }, + "bin": { + "js-yaml": "bin/js-yaml.js" + } + }, "node_modules/q": { "version": "1.5.1", "resolved": "https://registry.npmjs.org/q/-/q-1.5.1.tgz", @@ -40274,6 +40810,15 @@ "node": ">=4" } }, + "node_modules/smart-buffer": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/smart-buffer/-/smart-buffer-4.2.0.tgz", + "integrity": "sha512-94hK0Hh8rPqQl2xXc3HsaBoOXKV20MToPkcXvwbISWLEs+64sBq5kFgn2kJDHb1Pry9yrP0dxrCI9RRci7RXKg==", + "engines": { + "node": ">= 6.0.0", + "npm": ">= 3.0.0" + } + }, "node_modules/socket.io": { "version": "4.7.5", "resolved": "https://registry.npmjs.org/socket.io/-/socket.io-4.7.5.tgz", @@ -40366,6 +40911,32 @@ "uuid": "dist/bin/uuid" } }, + "node_modules/socks": { + "version": "2.8.3", + "resolved": "https://registry.npmjs.org/socks/-/socks-2.8.3.tgz", + "integrity": "sha512-l5x7VUUWbjVFbafGLxPWkYsHIhEvmF85tbIeFZWc8ZPtoMyybuEhL7Jye/ooC4/d48FgOjSJXgsF/AJPYCW8Zw==", + "dependencies": { + "ip-address": "^9.0.5", + "smart-buffer": "^4.2.0" + }, + "engines": { + "node": ">= 10.0.0", + "npm": ">= 3.0.0" + } + }, + "node_modules/socks-proxy-agent": { + "version": "8.0.4", + "resolved": "https://registry.npmjs.org/socks-proxy-agent/-/socks-proxy-agent-8.0.4.tgz", + "integrity": "sha512-GNAq/eg8Udq2x0eNiFkr9gRg5bA7PXEWagQdeRX4cPSG+X/8V38v637gim9bjFptMk1QWsCTr0ttrJEiXbNnRw==", + "dependencies": { + "agent-base": "^7.1.1", + "debug": "^4.3.4", + "socks": "^2.8.3" + }, + "engines": { + "node": ">= 14" + } + }, "node_modules/sort-asc": { "version": "0.2.0", "resolved": "https://registry.npmjs.org/sort-asc/-/sort-asc-0.2.0.tgz", @@ -41241,6 +41812,19 @@ "node": ">=10" } }, + "node_modules/tar-fs": { + "version": "3.0.6", + "resolved": "https://registry.npmjs.org/tar-fs/-/tar-fs-3.0.6.tgz", + "integrity": "sha512-iokBDQQkUyeXhgPYaZxmczGPhnhXZ0CmrqI+MOb/WFGS9DW5wnfrLgtjUJBvz50vQ3qfRwJ62QVoCFu8mPVu5w==", + "dependencies": { + "pump": "^3.0.0", + "tar-stream": "^3.1.5" + }, + "optionalDependencies": { + "bare-fs": "^2.1.1", + "bare-path": "^2.1.0" + } + }, "node_modules/tar-stream": { "version": "3.1.7", "resolved": "https://registry.npmjs.org/tar-stream/-/tar-stream-3.1.7.tgz", @@ -41403,8 +41987,7 @@ "node_modules/through": { "version": "2.3.8", "resolved": "https://registry.npmjs.org/through/-/through-2.3.8.tgz", - "integrity": "sha512-w89qg7PI8wAdvX60bMDP+bFoD5Dvhm9oLheFp5O4a2QF0cSBGsBX4qZmadPMvVqlLJBBci+WqGGOAPvcDeNSVg==", - "dev": true + "integrity": "sha512-w89qg7PI8wAdvX60bMDP+bFoD5Dvhm9oLheFp5O4a2QF0cSBGsBX4qZmadPMvVqlLJBBci+WqGGOAPvcDeNSVg==" }, "node_modules/thunky": { "version": "1.1.0", @@ -42350,6 +42933,11 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/typed-query-selector": { + "version": "2.12.0", + "resolved": "https://registry.npmjs.org/typed-query-selector/-/typed-query-selector-2.12.0.tgz", + "integrity": "sha512-SbklCd1F0EiZOyPiW192rrHZzZ5sBijB6xM+cpmrwDqObvdtunOHHIk9fCGsoK5JVIYXoyEp4iEdE3upFH3PAg==" + }, "node_modules/typescript": { "version": "5.5.3", "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.5.3.tgz", @@ -42458,6 +43046,15 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/unbzip2-stream": { + "version": "1.4.3", + "resolved": "https://registry.npmjs.org/unbzip2-stream/-/unbzip2-stream-1.4.3.tgz", + "integrity": "sha512-mlExGW4w71ebDJviH16lQLtZS32VKqsSfk80GCfUlwT/4/hNRFsoscrF/c++9xinkMzECL1uL9DDwXqFWkruPg==", + "dependencies": { + "buffer": "^5.2.1", + "through": "^2.3.8" + } + }, "node_modules/undefsafe": { "version": "2.0.5", "resolved": "https://registry.npmjs.org/undefsafe/-/undefsafe-2.0.5.tgz", @@ -42860,6 +43457,11 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/urlpattern-polyfill": { + "version": "10.0.0", + "resolved": "https://registry.npmjs.org/urlpattern-polyfill/-/urlpattern-polyfill-10.0.0.tgz", + "integrity": "sha512-H/A06tKD7sS1O1X2SshBVeA5FLycRpjqiBeqGKmBwBDBy28EnRjORxTNe269KSSr5un5qyWi1iL61wLxpd+ZOg==" + }, "node_modules/use-composed-ref": { "version": "1.3.0", "resolved": "https://registry.npmjs.org/use-composed-ref/-/use-composed-ref-1.3.0.tgz", @@ -43968,7 +44570,6 @@ "version": "8.18.0", "resolved": "https://registry.npmjs.org/ws/-/ws-8.18.0.tgz", "integrity": "sha512-8VbfWfHLbbwu3+N6OKsOMpBdT4kXPDDB9cJk2bJ6mh9ucxdlnNvH1e+roYkKmN9Nxw2yjz7VzeO9oOz2zJ04Pw==", - "dev": true, "engines": { "node": ">=10.0.0" }, @@ -44161,6 +44762,23 @@ "node": ">=8" } }, + "node_modules/yauzl": { + "version": "2.10.0", + "resolved": "https://registry.npmjs.org/yauzl/-/yauzl-2.10.0.tgz", + "integrity": "sha512-p4a9I6X6nu6IhoGmBqAcbJy1mlC4j27vEPZX9F4L4/vZT3Lyq1VkFHw/V/PUcB9Buo+DG3iHkT0x3Qya58zc3g==", + "dependencies": { + "buffer-crc32": "~0.2.3", + "fd-slicer": "~1.1.0" + } + }, + "node_modules/yauzl/node_modules/buffer-crc32": { + "version": "0.2.13", + "resolved": "https://registry.npmjs.org/buffer-crc32/-/buffer-crc32-0.2.13.tgz", + "integrity": "sha512-VO9Ht/+p3SN7SKWqcrgEzjGbRSJYTx+Q1pTQC0wrWqHx0vpJraQ6GtHx8tvcg1rlK1byhU5gccxgOgj7B0TDkQ==", + "engines": { + "node": "*" + } + }, "node_modules/yn": { "version": "3.1.1", "resolved": "https://registry.npmjs.org/yn/-/yn-3.1.1.tgz", @@ -44198,7 +44816,6 @@ "version": "3.23.8", "resolved": "https://registry.npmjs.org/zod/-/zod-3.23.8.tgz", "integrity": "sha512-XBx9AXhXktjUqnepgTiE5flcKIYWi/rme0Eaj+5Y0lftuGBq+jyRu/md4WnuxqgP1ubdpNCsYEYPxrzVHD8d6g==", - "peer": true, "funding": { "url": "https://github.com/sponsors/colinhacks" } diff --git a/package.json b/package.json index 544917e78..57a8ffc05 100644 --- a/package.json +++ b/package.json @@ -265,6 +265,7 @@ "prosemirror-transform": "^1.8.0", "prosemirror-view": "^1.32.5", "pug": "^3.0.2", + "puppeteer": "^23.1.0", "query-string": "^7.1.3", "querystring-es3": "^0.2.1", "raw-loader": "^4.0.2", diff --git a/src/client/util/CurrentUserUtils.ts b/src/client/util/CurrentUserUtils.ts index e095bc659..280830442 100644 --- a/src/client/util/CurrentUserUtils.ts +++ b/src/client/util/CurrentUserUtils.ts @@ -371,7 +371,7 @@ pie title Minerals in my tap water {key: "Button", creator: Docs.Create.ButtonDocument, opts: { _width: 150, _height: 50, _xPadding: 10, _yPadding: 10, title_custom: true, waitForDoubleClickToClick: 'never'}, scripts: {onClick: FollowLinkScript()?.script.originalScript ?? ""}}, {key: "Script", creator: opts => Docs.Create.ScriptingDocument(null, opts), opts: { _width: 200, _height: 250, }}, {key: "DataViz", creator: opts => Docs.Create.DataVizDocument("/users/rz/Downloads/addresses.csv", opts), opts: { _width: 300, _height: 300 }}, - {key: "Chat", creator: Docs.Create.ChatDocument, opts: { _width: 300, _height: 300, }}, + {key: "Chat", creator: Docs.Create.ChatDocument, opts: { _width: 500, _height: 500, }}, {key: "Header", creator: headerTemplate, opts: { _width: 300, _height: 120, _header_pointerEvents: "all", _header_height: 50, _header_fontSize: 9,_layout_autoHeightMargins: 50, _layout_autoHeight: true, treeView_HideUnrendered: true}}, {key: "ViewSlide", creator: slideView, opts: { _width: 400, _height: 300, _xMargin: 3, _yMargin: 3,}}, {key: "Trail", creator: Docs.Create.PresDocument, opts: { _width: 400, _height: 30, _type_collection: CollectionViewType.Stacking, dropAction: dropActionType.embed, treeView_HideTitle: true, _layout_fitWidth:true, layout_boxShadow: "0 0" }}, diff --git a/src/client/views/nodes/ChatBox/ChatBox.scss b/src/client/views/nodes/ChatBox/ChatBox.scss index 91bb3aba7..76fa05ce8 100644 --- a/src/client/views/nodes/ChatBox/ChatBox.scss +++ b/src/client/views/nodes/ChatBox/ChatBox.scss @@ -1,246 +1,246 @@ -$background-color: #f8f9fa; +@import url('https://fonts.googleapis.com/css2?family=Atkinson+Hyperlegible:ital,wght@0,400;0,700;1,400;1,700&display=swap'); + +$primary-color: #4a90e2; +$secondary-color: #f5f8fa; $text-color: #333; -$input-background: #fff; -$button-color: #007bff; -$button-hover-color: darken($button-color, 10%); -$shadow-color: rgba(0, 0, 0, 0.075); -$border-radius: 8px; -$citation-color: #ff6347; -$citation-hover-color: darken($citation-color, 10%); -$follow-up-bg-color: #e9ecef; -$follow-up-hover-bg-color: #dee2e6; - -.chatBox { +$light-text-color: #777; +$border-color: #e1e8ed; +$shadow-color: rgba(0, 0, 0, 0.1); +$transition: all 0.3s ease; + +.chat-box { display: flex; flex-direction: column; - width: 100%; height: 100%; - background-color: $background-color; - font-family: 'Helvetica Neue', Arial, sans-serif; + background-color: #fff; + font-family: + 'Atkinson Hyperlegible', + -apple-system, + BlinkMacSystemFont, + 'Segoe UI', + Roboto, + Helvetica, + Arial, + sans-serif; + border-radius: 12px; + overflow: hidden; + box-shadow: 0 4px 12px $shadow-color; + + .chat-header { + background-color: $primary-color; + color: white; + padding: 15px; + text-align: center; + box-shadow: 0 2px 4px $shadow-color; + height: fit-content; + h2 { + margin: 0; + font-size: 1.3em; + font-weight: 500; + } + } - .scroll-box { + .chat-messages { flex-grow: 1; - overflow-y: scroll; - overflow-x: hidden; - height: 100%; - padding: 10px; + overflow-y: auto; + padding: 20px; display: flex; - flex-direction: column-reverse; - padding-bottom: 0; + flex-direction: column; &::-webkit-scrollbar { - width: 8px; + width: 6px; } + &::-webkit-scrollbar-thumb { - background-color: darken($background-color, 10%); - border-radius: $border-radius; + background-color: $border-color; + border-radius: 3px; } + } - .chat-content { - display: flex; - flex-direction: column; + .chat-input { + display: flex; + padding: 20px; + border-top: 1px solid $border-color; + background-color: #fff; + + input { + flex-grow: 1; + padding: 12px 15px; + border: 1px solid $border-color; + border-radius: 24px; + font-size: 15px; + transition: $transition; + + &:focus { + outline: none; + border-color: $primary-color; + box-shadow: 0 0 0 2px rgba($primary-color, 0.2); + } } - .messages { + .submit-button { + background-color: $primary-color; + color: white; + border: none; + border-radius: 50%; + width: 48px; + height: 48px; + margin-left: 10px; + cursor: pointer; + transition: $transition; display: flex; - flex-direction: column; - - .message { - padding: 10px 15px; - margin-bottom: 10px; - border-radius: $border-radius; - background-color: lighten($background-color, 5%); - box-shadow: 0 2px 5px $shadow-color; - align-items: flex-start; - max-width: 90%; - width: fit-content; - word-break: break-word; - position: relative; - - .citation-button { - background-color: $citation-color; - color: #fff; - border: none; - border-radius: 50%; - cursor: pointer; - width: 20px; - height: 20px; - display: inline-flex; - align-items: center; - justify-content: center; - font-size: 12px; - font-weight: bold; - margin: 0 2px; - padding: 0; - transition: background-color 0.3s; - - &:hover { - background-color: $citation-hover-color; - } - } - - &.user { - align-self: flex-end; - background-color: $button-color; - color: #fff; - } - - &.chatbot { - align-self: flex-start; - background-color: $input-background; - color: $text-color; - } - - span { - flex-grow: 1; - padding-right: 10px; - } - - img { - max-width: 50px; - max-height: 50px; - border-radius: 50%; - } + align-items: center; + justify-content: center; + + &:hover { + background-color: darken($primary-color, 10%); + } + + &:disabled { + background-color: $light-text-color; + cursor: not-allowed; } - .follow-up-questions { - margin-top: 10px; - width: 100%; - - h4 { - margin-bottom: 5px; - font-size: 14px; - } - - .follow-up-button { - background-color: $follow-up-bg-color; - border: 1px solid #ddd; - border-radius: 8px; - padding: 8px 10px; - margin: 4px 0; - cursor: pointer; - transition: background-color 0.3s; - display: block; - width: 100%; - text-align: left; - white-space: normal; - word-wrap: break-word; - font-size: 12px; - color: $text-color; - min-height: 40px; - height: auto; - line-height: 1.3; - - &:hover { - background-color: $follow-up-hover-bg-color; - } - } + .spinner { + height: 24px; + width: 24px; + border: 3px solid rgba(255, 255, 255, 0.3); + border-top: 3px solid #fff; + border-radius: 50%; + animation: spin 1s linear infinite; } } } +} - .chat-form { - display: flex; - flex-grow: 0; - bottom: 0; - width: 100%; - padding: 10px; - background-color: $input-background; - box-shadow: inset 0 -1px 2px $shadow-color; - margin-bottom: 0; +.message { + max-width: 80%; + margin-bottom: 20px; + padding: 16px 20px; + border-radius: 18px; + font-size: 15px; + line-height: 1.5; + box-shadow: 0 2px 4px $shadow-color; + + &.user { + align-self: flex-end; + background-color: $primary-color; + color: white; + border-bottom-right-radius: 4px; + } - input[type='text'] { - flex-grow: 1; - border: 1px solid darken($input-background, 10%); - border-radius: $border-radius; - padding: 8px 12px; - margin-right: 10px; + &.chatbot { + align-self: flex-start; + background-color: $secondary-color; + color: $text-color; + border-bottom-left-radius: 4px; + } + + .toggle-info { + background-color: transparent; + color: $primary-color; + border: 1px solid $primary-color; + width: 100%; + height: fit-content; + border-radius: 8px; + padding: 10px 16px; + font-size: 14px; + cursor: pointer; + transition: $transition; + margin-top: 10px; + + &:hover { + background-color: rgba($primary-color, 0.1); } + } +} - button { - padding: 8px 16px; - background-color: $button-color; - color: #fff; - border: none; - border-radius: $border-radius; - cursor: pointer; - transition: background-color 0.3s; - min-width: 80px; +.follow-up-questions { + margin-top: 15px; - &:hover { - background-color: $button-hover-color; - } + h4 { + font-size: 15px; + font-weight: 600; + margin-bottom: 10px; + } + + .questions-list { + display: flex; + flex-direction: column; + gap: 10px; + } + + .follow-up-button { + background-color: #fff; + color: $primary-color; + border: 1px solid $primary-color; + border-radius: 8px; + padding: 10px 16px; + font-size: 14px; + cursor: pointer; + transition: $transition; + text-align: left; + white-space: normal; + word-wrap: break-word; + width: 100%; + height: fit-content; + + &:hover { + background-color: $primary-color; + color: #fff; } } } -.uploading-overlay { - position: absolute; - top: 0; - left: 0; - width: 100%; - height: 100%; - background-color: rgba($background-color, 0.95); - display: flex; - justify-content: center; +.citation-button { + display: inline-flex; align-items: center; - font-size: 1.5em; + justify-content: center; + width: 20px; + height: 20px; + border-radius: 50%; + background-color: rgba(0, 0, 0, 0.1); color: $text-color; - z-index: 10; - - &::before { - content: 'Uploading Docs...'; - font-weight: bold; + font-size: 12px; + font-weight: bold; + margin-left: 5px; + cursor: pointer; + transition: $transition; + vertical-align: middle; + + &:hover { + background-color: rgba(0, 0, 0, 0.2); } } -.modal { - position: fixed; +.uploading-overlay { + position: absolute; top: 0; left: 0; - width: 100%; - height: 100%; + right: 0; + bottom: 0; + background-color: rgba(255, 255, 255, 0.8); display: flex; justify-content: center; align-items: center; - background-color: rgba(0, 0, 0, 0.4); - - .modal-content { - background-color: $input-background; - color: $text-color; - padding: 20px; - border-radius: $border-radius; - box-shadow: 0 2px 10px $shadow-color; - display: flex; - flex-direction: column; - align-items: center; - width: auto; - min-width: 300px; - - h4 { - margin-bottom: 15px; - } - - p { - margin-bottom: 20px; - } + z-index: 1000; +} - button { - padding: 10px 20px; - background-color: $button-color; - color: #fff; - border: none; - border-radius: $border-radius; - cursor: pointer; - transition: background-color 0.3s; +@keyframes spin { + 0% { + transform: rotate(0deg); + } + 100% { + transform: rotate(360deg); + } +} - &:hover { - background-color: $button-hover-color; - } - } +@media (max-width: 768px) { + .chat-box { + border-radius: 0; } - .thought-text { - color: #6c757d; - font-style: italic; + + .message { + max-width: 90%; } } diff --git a/src/client/views/nodes/ChatBox/ChatBox.tsx b/src/client/views/nodes/ChatBox/ChatBox.tsx index 36416a330..4a98f8dc1 100644 --- a/src/client/views/nodes/ChatBox/ChatBox.tsx +++ b/src/client/views/nodes/ChatBox/ChatBox.tsx @@ -21,6 +21,8 @@ import { DocumentManager } from '../../../util/DocumentManager'; import { v4 as uuidv4 } from 'uuid'; import { chunk } from 'lodash'; import { DocUtils } from '../../../documents/DocUtils'; +import { createRef } from 'react'; +import { ClientUtils } from '../../../../ClientUtils'; dotenv.config(); @@ -37,10 +39,10 @@ export class ChatBox extends ViewBoxAnnotatableComponent() { @observable private linked_csv_files: { filename: string; id: string; text: string }[] = []; private openai: OpenAI; private vectorstore_id: string; - private documents: AI_Document[] = []; - private _oldWheel: any; private vectorstore: Vectorstore; private agent: Agent; // Add the ChatBot instance + private _oldWheel: HTMLDivElement | null = null; + private messagesRef: React.RefObject; public static LayoutString(fieldKey: string) { return FieldView.LayoutString(ChatBox, fieldKey); @@ -59,6 +61,7 @@ export class ChatBox extends ViewBoxAnnotatableComponent() { } this.vectorstore = new Vectorstore(this.vectorstore_id, this.retrieveDocIds); this.agent = new Agent(this.vectorstore, this.retrieveSummaries, this.retrieveFormattedHistory, this.retrieveCSVData, this.addLinkedUrlDoc); + this.messagesRef = React.createRef(); reaction( () => this.history.map((msg: AssistantMessage) => ({ role: msg.role, content: msg.content, follow_up_questions: msg.follow_up_questions, citations: msg.citations })), @@ -133,6 +136,23 @@ export class ChatBox extends ViewBoxAnnotatableComponent() { return new OpenAI(configuration); } + addScrollListener = () => { + if (this.messagesRef.current) { + this.messagesRef.current.addEventListener('wheel', this.onPassiveWheel, { passive: false }); + } + }; + + removeScrollListener = () => { + if (this.messagesRef.current) { + this.messagesRef.current.removeEventListener('wheel', this.onPassiveWheel); + } + }; + + scrollToBottom = () => { + if (this.messagesRef.current) { + } + }; + onPassiveWheel = (e: WheelEvent) => { if (this._props.isContentActive()) { e.stopPropagation(); @@ -160,6 +180,7 @@ export class ChatBox extends ViewBoxAnnotatableComponent() { this.current_message = { ...this.current_message, processing_info: update }; } }); + this.scrollToBottom(); }; const finalMessage = await this.agent.askAgent(trimmedText, onUpdate); @@ -176,8 +197,10 @@ export class ChatBox extends ViewBoxAnnotatableComponent() { this.history.push({ role: ASSISTANT_ROLE.ASSISTANT, content: [{ index: 0, type: TEXT_TYPE.ERROR, text: 'Sorry, I encountered an error while processing your request.', citation_ids: null }], processing_info: [] }); } finally { this.isLoading = false; + this.scrollToBottom(); } } + this.scrollToBottom(); }; @action @@ -202,6 +225,11 @@ export class ChatBox extends ViewBoxAnnotatableComponent() { doc.chunk_simpl = JSON.stringify({ chunks: [chunkToAdd] }); }; + @computed + get userName() { + return ClientUtils.CurrentUserEmail; + } + @action handleCitationClick = (citation: Citation) => { console.log('Citation clicked:', citation); @@ -276,6 +304,10 @@ export class ChatBox extends ViewBoxAnnotatableComponent() { return highlight_doc; }; + componentDidUpdate() { + this.scrollToBottom(); + } + componentDidMount() { this._props.setContentViewBox?.(this); if (this.dataDoc.data) { @@ -332,6 +364,11 @@ export class ChatBox extends ViewBoxAnnotatableComponent() { console.log('Deleted docs: ', change.oldValue); } }); + this.addScrollListener(); + } + + componentWillUnmount() { + this.removeScrollListener(); } @computed @@ -411,35 +448,41 @@ export class ChatBox extends ViewBoxAnnotatableComponent() { }; render() { return ( -
- {this.isUploadingDocs &&
} -
{ - this._oldWheel?.removeEventListener('wheel', this.onPassiveWheel); - this._oldWheel = r; - r?.addEventListener('wheel', this.onPassiveWheel, { passive: false }); - }}> -
- {this.history.map((message, index) => ( - - ))} - {!this.current_message ? null : ( - - )} +
+ {this.isUploadingDocs && ( +
+
+ )} +
+

{this.userName()}'s AI Assistant

- - (this.inputValue = e.target.value)} /> -
diff --git a/src/client/views/nodes/ChatBox/MessageComponent.scss b/src/client/views/nodes/ChatBox/MessageComponent.scss deleted file mode 100644 index 6fcc0e5e7..000000000 --- a/src/client/views/nodes/ChatBox/MessageComponent.scss +++ /dev/null @@ -1,10 +0,0 @@ -MessageComponent-citation { - color: lightblue; - vertical-align: super; - font-size: smaller; -} -MessageComponent-file_path { - color: lightblue; - vertical-align: baseline; - font-size: inherit; -} diff --git a/src/client/views/nodes/ChatBox/MessageComponent.tsx b/src/client/views/nodes/ChatBox/MessageComponent.tsx index 0b8fa6b96..00e9795e3 100644 --- a/src/client/views/nodes/ChatBox/MessageComponent.tsx +++ b/src/client/views/nodes/ChatBox/MessageComponent.tsx @@ -1,7 +1,7 @@ -import React from 'react'; +import React, { useState } from 'react'; import { observer } from 'mobx-react'; import { AssistantMessage, Citation, MessageContent, PROCESSING_TYPE, ProcessingInfo, TEXT_TYPE } from './types'; -import Markdown from 'react-markdown'; +import ReactMarkdown from 'react-markdown'; interface MessageComponentProps { message: AssistantMessage; @@ -12,37 +12,20 @@ interface MessageComponentProps { } const MessageComponentBox: React.FC = function ({ message, index, onFollowUpClick, onCitationClick, updateMessageCitations }) { + const [dropdownOpen, setDropdownOpen] = useState(false); + const renderContent = (item: MessageContent) => { const i = item.index; if (item.type === TEXT_TYPE.GROUNDED) { const citation_ids = item.citation_ids || []; return ( - {item.text} + {item.text} {citation_ids.map((id, idx) => { const citation = message.citations?.find(c => c.citation_id === id); if (!citation) return null; return ( - ); @@ -52,49 +35,65 @@ const MessageComponentBox: React.FC = function ({ message } else if (item.type === TEXT_TYPE.NORMAL) { return ( - {item.text} + {item.text} ); } else if ('query' in item) { - // Handle the case where the item has a query property return ( - {JSON.stringify(item.query)} + {JSON.stringify(item.query)} ); } else { - // Fallback for any other unexpected cases - return {JSON.stringify(item)}; + return ( + + {JSON.stringify(item)} + + ); } }; - console.log(message.processing_info); + const hasProcessingInfo = message.processing_info && message.processing_info.length > 0; + + const renderProcessingInfo = (info: ProcessingInfo) => { + if (info.type === PROCESSING_TYPE.THOUGHT) { + return ( +
+ Thought: {info.content} +
+ ); + } else if (info.type === PROCESSING_TYPE.ACTION) { + return ( +
+ Action: {info.content} +
+ ); + } else { + return null; + } + }; return (
- {message.processing_info && - (message.processing_info as ProcessingInfo[]).map(item => - item.type === PROCESSING_TYPE.THOUGHT ? ( -
- Thought: {item.content} -
- ) : item.type === PROCESSING_TYPE.ACTION ? ( -
- Action: {item.content} -
- ) : ( -
- ) - )} -
{message.content && message.content.map(messageFragment => {renderContent(messageFragment)})}
+
{message.content && message.content.map(messageFragment => {renderContent(messageFragment)})}
+ {hasProcessingInfo && ( +
+ + {dropdownOpen &&
{message.processing_info.map(renderProcessingInfo)}
} +
+ )} {message.follow_up_questions && message.follow_up_questions.length > 0 && (

Follow-up Questions:

- {message.follow_up_questions.map((question, idx) => ( - - ))} +
+ {message.follow_up_questions.map((question, idx) => ( + + ))} +
)}
diff --git a/src/server/ApiManagers/AssistantManager.ts b/src/server/ApiManagers/AssistantManager.ts index cd26ca79b..9b85dbbe8 100644 --- a/src/server/ApiManagers/AssistantManager.ts +++ b/src/server/ApiManagers/AssistantManager.ts @@ -14,6 +14,8 @@ import { PartitionResponse } from 'unstructured-client/sdk/models/operations'; import { ChunkingStrategy, Strategy } from 'unstructured-client/sdk/models/shared'; import * as cheerio from 'cheerio'; import { ScrapflyClient, ScrapeConfig } from 'scrapfly-sdk'; +import { google } from 'googleapis'; +import puppeteer from 'puppeteer'; export enum Directory { parsed_files = 'parsed_files', @@ -55,6 +57,7 @@ export default class AssistantManager extends ApiManager { }, }); const scrapflyClient = new ScrapflyClient({ key: process.env._CLIENT_SCRAPFLY_API_KEY! }); + const customsearch = google.customsearch('v1'); register({ method: Method.POST, @@ -89,20 +92,18 @@ export default class AssistantManager extends ApiManager { secureHandler: async ({ req, res }) => { const { query } = req.body; try { - const response = await axios.get('http://api.serpstack.com/search', { - params: { - access_key: process.env._CLIENT_SERPSTACK_API_KEY, - query: query, - }, + const response = await customsearch.cse.list({ + q: query, + cx: process.env._CLIENT_GOOGLE_SEARCH_ENGINE_ID, + key: process.env._CLIENT_GOOGLE_API_KEY, + safe: 'active', }); - console.log(response.data); - const results = response.data.organic_results.map((result: any) => ({ - url: result.url, - snippet: result.snippet, - })); - - console.log(results); + const results = + response.data.items?.map((item: any) => ({ + url: item.link, + snippet: item.snippet, + })) || []; res.send({ results }); } catch (error: any) { @@ -144,6 +145,7 @@ export default class AssistantManager extends ApiManager { const scrapedImagesDirectory = pathToDirectory(Directory.scrape_images); const filePath = serverPathToFile(Directory.scrape_images, url_filename); + // Check if the image already exists if (fs.existsSync(filePath)) { const imageBuffer = await readFileAsync(filePath); const base64Image = imageBuffer.toString('base64'); @@ -151,33 +153,27 @@ export default class AssistantManager extends ApiManager { return res.send({ website_image_base64: base64Image }); } + // Create the directory if it doesn't exist if (!fs.existsSync(scrapedImagesDirectory)) { fs.mkdirSync(scrapedImagesDirectory); } - const result = await scrapflyClient.scrape( - new ScrapeConfig({ - url: url, - render_js: true, - screenshots: { everything: 'fullpage' }, - }) - ); - - const screenshotPromises = Object.entries(result.result.screenshots).map(async ([name, screenshot]) => { - const response = await axios.get(screenshot.url, { - params: { - key: process.env._CLIENT_SCRAPFLY_API_KEY!, - options: 'print_media_format', - proxy_pool: 'public_residential_pool', - }, - responseType: 'arraybuffer', - }); - await fs.promises.writeFile(filePath, response.data); - return response.data.toString('base64'); + // Launch Puppeteer to take a screenshot of the webpage + const browser = await puppeteer.launch({ + args: ['--no-sandbox', '--disable-setuid-sandbox'], }); - - const base64Screenshots = await Promise.all(screenshotPromises); - res.send({ website_image_base64: base64Screenshots[0] }); + const page = await browser.newPage(); + await page.setUserAgent('Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36'); + await page.goto(url, { waitUntil: 'networkidle2' }); + const screenshotBuffer = await page.screenshot({ fullPage: true }); + await browser.close(); + + // Save the screenshot to the file system + await writeFileAsync(filePath, screenshotBuffer); + + // Return the base64-encoded image + const base64Image = Buffer.from(screenshotBuffer).toString('base64'); + res.send({ website_image_base64: base64Image }); } catch (error: any) { console.error('Error scraping website:', error); res.status(500).send({ error: 'Failed to scrape website', details: error.message }); -- cgit v1.2.3-70-g09d2 From e5464e4c04ef6f8a2bbf868b43bbcdba54239406 Mon Sep 17 00:00:00 2001 From: "A.J. Shulman" Date: Wed, 21 Aug 2024 14:13:56 -0400 Subject: fixed citations so they show different numbers --- package-lock.json | 21 ++++++++++ package.json | 2 + .../views/nodes/ChatBox/MessageComponent.tsx | 3 +- .../nodes/ChatBox/tools/WebsiteInfoScraperTool.ts | 17 ++------ src/server/ApiManagers/AssistantManager.ts | 47 ++++++++++------------ 5 files changed, 51 insertions(+), 39 deletions(-) (limited to 'src/client/views/nodes/ChatBox/MessageComponent.tsx') diff --git a/package-lock.json b/package-lock.json index 68c9e9746..a7e31663f 100644 --- a/package-lock.json +++ b/package-lock.json @@ -26,6 +26,7 @@ "@fullcalendar/daygrid": "^6.1.10", "@fullcalendar/multimonth": "^6.1.10", "@internationalized/date": "^3.5.0", + "@mozilla/readability": "^0.5.0", "@mui/icons-material": "^5.14.19", "@mui/material": "^5.14.19", "@octokit/core": "^6.0.1", @@ -271,6 +272,7 @@ "@types/howler": "^2.2.11", "@types/html-to-text": "^9.0.4", "@types/jquery": "^3.5.29", + "@types/jsdom": "^21.1.7", "@types/libxmljs": "^0.18.12", "@types/lodash": "^4.14.202", "@types/mocha": "^10.0.6", @@ -4755,6 +4757,14 @@ "sparse-bitfield": "^3.0.3" } }, + "node_modules/@mozilla/readability": { + "version": "0.5.0", + "resolved": "https://registry.npmjs.org/@mozilla/readability/-/readability-0.5.0.tgz", + "integrity": "sha512-Z+CZ3QaosfFaTqvhQsIktyGrjFjSC0Fa4EMph4mqKnWhmyoGICsV/8QK+8HpXut6zV7zwfWwqDmEjtk1Qf6EgQ==", + "engines": { + "node": ">=14.0.0" + } + }, "node_modules/@mui/base": { "version": "5.0.0-beta.40", "resolved": "https://registry.npmjs.org/@mui/base/-/base-5.0.0-beta.40.tgz", @@ -11150,6 +11160,17 @@ "@types/sizzle": "*" } }, + "node_modules/@types/jsdom": { + "version": "21.1.7", + "resolved": "https://registry.npmjs.org/@types/jsdom/-/jsdom-21.1.7.tgz", + "integrity": "sha512-yOriVnggzrnQ3a9OKOCxaVuSug3w3/SbOj5i7VwXWZEyUNl3bLF9V3MfxGbZKuwqJOQyRfqXyROBB1CoZLFWzA==", + "dev": true, + "dependencies": { + "@types/node": "*", + "@types/tough-cookie": "*", + "parse5": "^7.0.0" + } + }, "node_modules/@types/json-schema": { "version": "7.0.15", "resolved": "https://registry.npmjs.org/@types/json-schema/-/json-schema-7.0.15.tgz", diff --git a/package.json b/package.json index 57a8ffc05..2b2a9ab89 100644 --- a/package.json +++ b/package.json @@ -43,6 +43,7 @@ "@types/howler": "^2.2.11", "@types/html-to-text": "^9.0.4", "@types/jquery": "^3.5.29", + "@types/jsdom": "^21.1.7", "@types/libxmljs": "^0.18.12", "@types/lodash": "^4.14.202", "@types/mocha": "^10.0.6", @@ -111,6 +112,7 @@ "@fullcalendar/daygrid": "^6.1.10", "@fullcalendar/multimonth": "^6.1.10", "@internationalized/date": "^3.5.0", + "@mozilla/readability": "^0.5.0", "@mui/icons-material": "^5.14.19", "@mui/material": "^5.14.19", "@octokit/core": "^6.0.1", diff --git a/src/client/views/nodes/ChatBox/MessageComponent.tsx b/src/client/views/nodes/ChatBox/MessageComponent.tsx index 00e9795e3..3edfb272c 100644 --- a/src/client/views/nodes/ChatBox/MessageComponent.tsx +++ b/src/client/views/nodes/ChatBox/MessageComponent.tsx @@ -16,6 +16,7 @@ const MessageComponentBox: React.FC = function ({ message const renderContent = (item: MessageContent) => { const i = item.index; + console.log('item', item, 'index', i); if (item.type === TEXT_TYPE.GROUNDED) { const citation_ids = item.citation_ids || []; return ( @@ -26,7 +27,7 @@ const MessageComponentBox: React.FC = function ({ message if (!citation) return null; return ( ); })} diff --git a/src/client/views/nodes/ChatBox/tools/WebsiteInfoScraperTool.ts b/src/client/views/nodes/ChatBox/tools/WebsiteInfoScraperTool.ts index 739329bea..4588b5aec 100644 --- a/src/client/views/nodes/ChatBox/tools/WebsiteInfoScraperTool.ts +++ b/src/client/views/nodes/ChatBox/tools/WebsiteInfoScraperTool.ts @@ -17,8 +17,8 @@ export class WebsiteInfoScraperTool extends BaseTool<{ url: string | string[] }> max_inputs: '3', }, }, - 'Provide up to 3 URLs of websites that you have identified as the most relevant from the previous search. This tool will provide screenshots of those specific websites. It will also create a document from the scraped content for future reference. When providing a response to the user, ideally reference as many of the websites as possible in order to provide a well grounded result.', - 'Returns the full content of the webpages as images for analysis.' + 'Provide up to 3 URLs of websites that you have identified as the most relevant from the previous search. This tool will provide the text content of those specific websites. When providing a final response to the user based on information from these chunks, ideally cite as many of the url chunks as possible (ground your infromation from multiple sources, if possible) in order to provide a well grounded result.', + 'Returns the text content of the webpages for analysis.' ); this._addLinkedUrlDoc = addLinkedUrlDoc; } @@ -29,19 +29,10 @@ export class WebsiteInfoScraperTool extends BaseTool<{ url: string | string[] }> for (const url of urls) { try { - const { website_image_base64 } = await Networking.PostToServer('/scrapeWebsite', { url }); + const { website_plain_text } = await Networking.PostToServer('/scrapeWebsite', { url }); const id = uuidv4(); this._addLinkedUrlDoc(url, id); - results.push( - { type: 'text', text: ` ` }, - { - type: 'image_url', - image_url: { - url: `data:image/jpeg;base64,${website_image_base64}`, - }, - }, - { type: 'text', text: `\n` } - ); + results.push({ type: 'text', text: `\n${website_plain_text}\n\n` }); } catch (error) { results.push({ type: 'text', text: `An error occurred while scraping the website: ${url}` }); } diff --git a/src/server/ApiManagers/AssistantManager.ts b/src/server/ApiManagers/AssistantManager.ts index afaeaf961..a59a2d22d 100644 --- a/src/server/ApiManagers/AssistantManager.ts +++ b/src/server/ApiManagers/AssistantManager.ts @@ -15,7 +15,9 @@ import { ChunkingStrategy, Strategy } from 'unstructured-client/sdk/models/share import * as cheerio from 'cheerio'; import { ScrapflyClient, ScrapeConfig } from 'scrapfly-sdk'; import { google } from 'googleapis'; -import puppeteer from 'puppeteer'; +import * as puppeteer from 'puppeteer'; +import { JSDOM } from 'jsdom'; +import { Readability } from '@mozilla/readability'; export enum Directory { parsed_files = 'parsed_files', @@ -142,39 +144,34 @@ export default class AssistantManager extends ApiManager { secureHandler: async ({ req, res }) => { const { url } = req.body; try { - const url_filename = url.replace(/\./g, '-').replace(/\//g, '_') + '.jpg'; - const scrapedImagesDirectory = pathToDirectory(Directory.scrape_images); - const filePath = serverPathToFile(Directory.scrape_images, url_filename); - - // Check if the image already exists - if (fs.existsSync(filePath)) { - const imageBuffer = await readFileAsync(filePath); - const base64Image = imageBuffer.toString('base64'); - console.log('Image already exists'); - return res.send({ website_image_base64: base64Image }); - } - - // Create the directory if it doesn't exist - if (!fs.existsSync(scrapedImagesDirectory)) { - fs.mkdirSync(scrapedImagesDirectory); - } - - // Launch Puppeteer to take a screenshot of the webpage + // Launch Puppeteer to navigate to the webpage const browser = await puppeteer.launch({ args: ['--no-sandbox', '--disable-setuid-sandbox'], }); const page = await browser.newPage(); await page.setUserAgent('Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36'); await page.goto(url, { waitUntil: 'networkidle2' }); - const screenshotBuffer = await page.screenshot({ fullPage: true }); + + // Get the HTML content of the page + const htmlContent = await page.content(); await browser.close(); - // Save the screenshot to the file system - await writeFileAsync(filePath, screenshotBuffer); + // Use JSDOM to parse the HTML content + const dom = new JSDOM(htmlContent, { url }); - // Return the base64-encoded image - const base64Image = Buffer.from(screenshotBuffer).toString('base64'); - res.send({ website_image_base64: base64Image }); + // Use Readability to extract the readable content + const reader = new Readability(dom.window.document); + const article = reader.parse(); + + if (article) { + // Extract the plain text from the article content + const plainText = article.textContent; + + // Return the plain text content + res.send({ website_plain_text: plainText }); + } else { + res.status(500).send({ error: 'Failed to extract readable content' }); + } } catch (error: any) { console.error('Error scraping website:', error); res.status(500).send({ error: 'Failed to scrape website', details: error.message }); -- cgit v1.2.3-70-g09d2 From 484eb670b291afa07f2f7b976fafe02bdc9ac71d Mon Sep 17 00:00:00 2001 From: "A.J. Shulman" Date: Wed, 21 Aug 2024 16:06:31 -0400 Subject: added answer streaming parsing so it provides realtime parsing and then follow-up questions and citations are added when its finished --- src/client/views/nodes/ChatBox/Agent.ts | 54 ++++----- src/client/views/nodes/ChatBox/AnswerParser.ts | 2 +- src/client/views/nodes/ChatBox/ChatBox.tsx | 16 ++- .../views/nodes/ChatBox/MessageComponent.tsx | 3 +- src/client/views/nodes/ChatBox/StreamParser.ts | 125 --------------------- .../views/nodes/ChatBox/StreamedAnswerParser.ts | 73 ++++++++++++ 6 files changed, 111 insertions(+), 162 deletions(-) delete mode 100644 src/client/views/nodes/ChatBox/StreamParser.ts create mode 100644 src/client/views/nodes/ChatBox/StreamedAnswerParser.ts (limited to 'src/client/views/nodes/ChatBox/MessageComponent.tsx') diff --git a/src/client/views/nodes/ChatBox/Agent.ts b/src/client/views/nodes/ChatBox/Agent.ts index 43138bf94..4ccb179f0 100644 --- a/src/client/views/nodes/ChatBox/Agent.ts +++ b/src/client/views/nodes/ChatBox/Agent.ts @@ -12,9 +12,9 @@ import { WebsiteInfoScraperTool } from './tools/WebsiteInfoScraperTool'; import { SearchTool } from './tools/SearchTool'; import { NoTool } from './tools/NoTool'; import { on } from 'events'; -import { StreamParser } from './StreamParser'; import { v4 as uuidv4 } from 'uuid'; import { AnswerParser } from './AnswerParser'; +import { StreamedAnswerParser } from './StreamedAnswerParser'; dotenv.config(); @@ -31,6 +31,7 @@ export class Agent { private thoughtNumber: number = 0; private processingNumber: number = 0; private processingInfo: ProcessingInfo[] = []; + private streamedAnswerParser: StreamedAnswerParser = new StreamedAnswerParser(); constructor(_vectorstore: Vectorstore, summaries: () => string, history: () => string, csvData: () => { filename: string; id: string; text: string }[], addLinkedUrlDoc: (url: string, id: string) => void) { this.client = new OpenAI({ apiKey: process.env.OPENAI_KEY, dangerouslyAllowBrowser: true }); @@ -48,7 +49,7 @@ export class Agent { }; } - async askAgent(question: string, onUpdate: (update: ProcessingInfo[]) => void, maxTurns: number = 30): Promise { + async askAgent(question: string, onProcessingUpdate: (processingUpdate: ProcessingInfo[]) => void, onAnswerUpdate: (answerUpdate: string) => void, maxTurns: number = 30): Promise { console.log(`Starting query: ${question}`); this.messages.push({ role: 'user', content: question }); const chatHistory = this._history(); @@ -74,7 +75,7 @@ export class Agent { console.log(this.interMessages); console.log(`Turn ${i}/${maxTurns}`); - const result = await this.execute(onUpdate); + const result = await this.execute(onProcessingUpdate, onAnswerUpdate); this.interMessages.push({ role: 'assistant', content: result }); let parsedResult; @@ -133,6 +134,7 @@ export class Agent { } } else if (key === 'answer') { console.log('Answer found. Ending query.'); + this.streamedAnswerParser.reset(); const parsedAnswer = AnswerParser.parse(result, this.processingInfo); return parsedAnswer; } @@ -141,7 +143,7 @@ export class Agent { throw new Error('Reached maximum turns. Ending query.'); } - private async execute(onUpdate: (update: ProcessingInfo[]) => void): Promise { + private async execute(onProcessingUpdate: (processingUpdate: ProcessingInfo[]) => void, onAnswerUpdate: (answerUpdate: string) => void): Promise { const stream = await this.client.chat.completions.create({ model: 'gpt-4o', messages: this.interMessages as ChatCompletionMessageParam[], @@ -155,11 +157,18 @@ export class Agent { let isInsideTag: boolean = false; for await (const chunk of stream) { - const content = chunk.choices[0]?.delta?.content || ''; + let content = chunk.choices[0]?.delta?.content || ''; fullResponse += content; for (const char of content) { - if (char === '<') { + if (currentTag === 'answer') { + currentContent += char; + console.log(char); + const streamedAnswer = this.streamedAnswerParser.parse(char); + console.log(streamedAnswer); + onAnswerUpdate(streamedAnswer); + continue; + } else if (char === '<') { isInsideTag = true; currentTag = ''; currentContent = ''; @@ -170,11 +179,15 @@ export class Agent { } } else if (isInsideTag) { currentTag += char; - } else { + } else if (currentTag === 'thought' || currentTag === 'action_input_description') { currentContent += char; - if (currentTag === 'thought' || currentTag === 'action_input_description') { - this.processStreamedContent(currentTag, currentContent); - onUpdate(this.processingInfo); + const current_info = this.processingInfo.find(info => info.index === this.processingNumber); + if (current_info) { + current_info.content = currentContent.trim(); + onProcessingUpdate(this.processingInfo); + } else { + this.processingInfo.push({ index: this.processingNumber, type: currentTag === 'thought' ? PROCESSING_TYPE.THOUGHT : PROCESSING_TYPE.ACTION, content: currentContent.trim() }); + onProcessingUpdate(this.processingInfo); } } } @@ -183,27 +196,6 @@ export class Agent { return fullResponse; } - private processStreamedContent(tag: string, streamed_content: string) { - const current_info = this.processingInfo.find(info => info.index === this.processingNumber); - switch (tag) { - case 'thought': - if (current_info) { - current_info.content = streamed_content; - } else { - console.log(`Adding thought: ${streamed_content}`); - this.processingInfo.push({ index: this.processingNumber, type: PROCESSING_TYPE.THOUGHT, content: streamed_content.trim() }); - } - break; - case 'action_input_description': - if (current_info) { - current_info.content = streamed_content; - } else { - console.log(`Adding thought: ${streamed_content}`); - this.processingInfo.push({ index: this.processingNumber, type: PROCESSING_TYPE.ACTION, content: streamed_content.trim() }); - } - } - } - private async processAction(action: string, actionInput: any): Promise { if (!(action in this.tools)) { throw new Error(`Unknown action: ${action}`); diff --git a/src/client/views/nodes/ChatBox/AnswerParser.ts b/src/client/views/nodes/ChatBox/AnswerParser.ts index 1d46a366d..b18083a27 100644 --- a/src/client/views/nodes/ChatBox/AnswerParser.ts +++ b/src/client/views/nodes/ChatBox/AnswerParser.ts @@ -56,7 +56,7 @@ export class AnswerParser { while ((match = groundedTextRegex.exec(rawTextContent)) !== null) { const [fullMatch, citationIndex, groundedText] = match; - // Add normal text before the grounded text + // Add normal text that is before the grounded text if (match.index > lastIndex) { const normalText = rawTextContent.slice(lastIndex, match.index).trim(); if (normalText) { diff --git a/src/client/views/nodes/ChatBox/ChatBox.tsx b/src/client/views/nodes/ChatBox/ChatBox.tsx index 1366eb772..45f5c0a65 100644 --- a/src/client/views/nodes/ChatBox/ChatBox.tsx +++ b/src/client/views/nodes/ChatBox/ChatBox.tsx @@ -11,7 +11,7 @@ import { ViewBoxAnnotatableComponent } from '../../DocComponent'; import { FieldView, FieldViewProps } from '../FieldView'; import './ChatBox.scss'; import MessageComponentBox from './MessageComponent'; -import { ASSISTANT_ROLE, AssistantMessage, AI_Document, Citation, CHUNK_TYPE, RAGChunk, getChunkType, TEXT_TYPE, SimplifiedChunk, ProcessingInfo } from './types'; +import { ASSISTANT_ROLE, AssistantMessage, AI_Document, Citation, CHUNK_TYPE, RAGChunk, getChunkType, TEXT_TYPE, SimplifiedChunk, ProcessingInfo, MessageContent } from './types'; import { Vectorstore } from './vectorstore/Vectorstore'; import { Agent } from './Agent'; import dotenv from 'dotenv'; @@ -175,16 +175,24 @@ export class ChatBox extends ViewBoxAnnotatableComponent() { this.isLoading = true; this.current_message = { role: ASSISTANT_ROLE.ASSISTANT, content: [], citations: [], processing_info: [] }; - const onUpdate = (update: ProcessingInfo[]) => { + const onProcessingUpdate = (processingUpdate: ProcessingInfo[]) => { runInAction(() => { if (this.current_message) { - this.current_message = { ...this.current_message, processing_info: update }; + this.current_message = { ...this.current_message, processing_info: processingUpdate }; } }); this.scrollToBottom(); }; - const finalMessage = await this.agent.askAgent(trimmedText, onUpdate); + const onAnswerUpdate = (answerUpdate: string) => { + runInAction(() => { + if (this.current_message) { + this.current_message = { ...this.current_message, content: [{ text: answerUpdate, type: TEXT_TYPE.NORMAL, index: 0, citation_ids: [] }] }; + } + }); + }; + + const finalMessage = await this.agent.askAgent(trimmedText, onProcessingUpdate, onAnswerUpdate); runInAction(() => { if (this.current_message) { diff --git a/src/client/views/nodes/ChatBox/MessageComponent.tsx b/src/client/views/nodes/ChatBox/MessageComponent.tsx index 3edfb272c..d0e78c751 100644 --- a/src/client/views/nodes/ChatBox/MessageComponent.tsx +++ b/src/client/views/nodes/ChatBox/MessageComponent.tsx @@ -76,15 +76,16 @@ const MessageComponentBox: React.FC = function ({ message return (
-
{message.content && message.content.map(messageFragment => {renderContent(messageFragment)})}
{hasProcessingInfo && (
{dropdownOpen &&
{message.processing_info.map(renderProcessingInfo)}
} +
)} +
{message.content && message.content.map(messageFragment => {renderContent(messageFragment)})}
{message.follow_up_questions && message.follow_up_questions.length > 0 && (

Follow-up Questions:

diff --git a/src/client/views/nodes/ChatBox/StreamParser.ts b/src/client/views/nodes/ChatBox/StreamParser.ts deleted file mode 100644 index 9b087663a..000000000 --- a/src/client/views/nodes/ChatBox/StreamParser.ts +++ /dev/null @@ -1,125 +0,0 @@ -import { AssistantMessage, ASSISTANT_ROLE, TEXT_TYPE, Citation, CHUNK_TYPE } from './types'; -import { v4 as uuidv4 } from 'uuid'; - -export class StreamParser { - private currentMessage: AssistantMessage; - private currentTag: string | null = null; - private buffer: string = ''; - private citationIndex: number = 1; - - constructor() { - this.currentMessage = { - role: ASSISTANT_ROLE.ASSISTANT, - content: [], - thoughts: [], - actions: [], - citations: [], - }; - } - - parse(chunk: string): AssistantMessage { - this.buffer += chunk; - - while (this.buffer.length > 0) { - if (this.currentTag === null) { - const openTagMatch = this.buffer.match(/<(\w+)>/); - if (openTagMatch) { - this.currentTag = openTagMatch[1]; - this.buffer = this.buffer.slice(openTagMatch.index! + openTagMatch[0].length); - } else { - break; - } - } else { - const closeTagIndex = this.buffer.indexOf(``); - if (closeTagIndex !== -1) { - const content = this.buffer.slice(0, closeTagIndex); - this.processTag(this.currentTag, content); - this.buffer = this.buffer.slice(closeTagIndex + this.currentTag.length + 3); - this.currentTag = null; - } else { - break; - } - } - } - - return this.currentMessage; - } - - private processTag(tag: string, content: string) { - switch (tag) { - case 'thought': - this.currentMessage.thoughts!.push(content); - break; - case 'action': - this.currentMessage.actions!.push({ index: this.currentMessage.actions!.length, action: content, action_input: '' }); - break; - case 'action_input': - if (this.currentMessage.actions!.length > 0) { - this.currentMessage.actions![this.currentMessage.actions!.length - 1].action_input = content; - } - break; - case 'answer': - this.processAnswer(content); - break; - } - } - - private processAnswer(content: string) { - const groundedTextRegex = /([\s\S]*?)<\/grounded_text>/g; - let lastIndex = 0; - let match; - - while ((match = groundedTextRegex.exec(content)) !== null) { - const [fullMatch, citationIndex, groundedText] = match; - - // Add normal text before the grounded text - if (match.index > lastIndex) { - const normalText = content.slice(lastIndex, match.index).trim(); - if (normalText) { - this.currentMessage.content.push({ - index: this.currentMessage.content.length, - type: TEXT_TYPE.NORMAL, - text: normalText, - citation_ids: null, - }); - } - } - - // Add grounded text - const citation_id = uuidv4(); - this.currentMessage.content.push({ - index: this.currentMessage.content.length, - type: TEXT_TYPE.GROUNDED, - text: groundedText.trim(), - citation_ids: [citation_id], - }); - - // Add citation - this.currentMessage.citations!.push({ - citation_id, - chunk_id: '', - type: CHUNK_TYPE.TEXT, - direct_text: '', - }); - - lastIndex = match.index + fullMatch.length; - } - - // Add any remaining normal text after the last grounded text - if (lastIndex < content.length) { - const remainingText = content.slice(lastIndex).trim(); - if (remainingText) { - this.currentMessage.content.push({ - index: this.currentMessage.content.length, - type: TEXT_TYPE.NORMAL, - text: remainingText, - citation_ids: null, - }); - } - } - } - - getResult(): AssistantMessage { - return this.currentMessage; - } -} diff --git a/src/client/views/nodes/ChatBox/StreamedAnswerParser.ts b/src/client/views/nodes/ChatBox/StreamedAnswerParser.ts new file mode 100644 index 000000000..3585cab4a --- /dev/null +++ b/src/client/views/nodes/ChatBox/StreamedAnswerParser.ts @@ -0,0 +1,73 @@ +import { threadId } from 'worker_threads'; + +enum ParserState { + Outside, + InGroundedText, + InNormalText, +} + +export class StreamedAnswerParser { + private state: ParserState = ParserState.Outside; + private buffer: string = ''; + private result: string = ''; + private isStartOfLine: boolean = true; + + public parse(char: string): string { + switch (this.state) { + case ParserState.Outside: + if (char === '<') { + this.buffer = '<'; + } else if (char === '>') { + if (this.buffer.startsWith('') { + this.state = ParserState.Outside; + this.buffer = ''; + } else if (this.buffer.startsWith('') { + this.state = ParserState.Outside; + this.buffer = ''; + } else if (this.buffer.startsWith('<')) { + this.buffer += char; + } else { + this.processChar(char); + } + break; + } + + return this.result.trim(); + } + + private processChar(char: string): void { + if (this.isStartOfLine && char === ' ') { + // Skip leading spaces + return; + } + if (char === '\n') { + this.result += char; + this.isStartOfLine = true; + } else { + this.result += char; + this.isStartOfLine = false; + } + } + + public reset(): void { + this.state = ParserState.Outside; + this.buffer = ''; + this.result = ''; + this.isStartOfLine = true; + } +} -- cgit v1.2.3-70-g09d2 From 3d290c126623404b56958d16becc244d04548658 Mon Sep 17 00:00:00 2001 From: "A.J. Shulman" Date: Thu, 22 Aug 2024 11:17:28 -0400 Subject: fixed table citations and now formatted chunks show table info as well --- src/client/views/nodes/ChatBox/ChatBox.tsx | 13 ++++++++----- src/client/views/nodes/ChatBox/MessageComponent.tsx | 2 +- src/client/views/nodes/ChatBox/prompts.ts | 2 ++ src/client/views/nodes/ChatBox/tools/RAGTool.ts | 5 +++-- src/server/ApiManagers/AssistantManager.ts | 10 ++++------ 5 files changed, 18 insertions(+), 14 deletions(-) (limited to 'src/client/views/nodes/ChatBox/MessageComponent.tsx') diff --git a/src/client/views/nodes/ChatBox/ChatBox.tsx b/src/client/views/nodes/ChatBox/ChatBox.tsx index 45f5c0a65..d79233fae 100644 --- a/src/client/views/nodes/ChatBox/ChatBox.tsx +++ b/src/client/views/nodes/ChatBox/ChatBox.tsx @@ -251,9 +251,12 @@ export class ChatBox extends ViewBoxAnnotatableComponent() { const docChunkSimpl = JSON.parse(StrCast(doc.chunk_simpl)) as { chunks: SimplifiedChunk[] }; console.log(docChunkSimpl); const foundChunk = docChunkSimpl.chunks.find(chunk => chunk.chunkId === chunkId); + console.log(foundChunk); if (foundChunk) { - switch (getChunkType(foundChunk.chunkType)) { + console.log(getChunkType(foundChunk.chunkType)); + switch (foundChunk.chunkType) { case CHUNK_TYPE.IMAGE: + case CHUNK_TYPE.TABLE: const values = foundChunk.location?.replace(/[\[\]]/g, '').split(','); if (values?.length !== 4) { @@ -262,9 +265,9 @@ export class ChatBox extends ViewBoxAnnotatableComponent() { } const x1 = parseFloat(values[0]) * Doc.NativeWidth(doc); - const y1 = parseFloat(values[1]) * Doc.NativeHeight(doc); + const y1 = parseFloat(values[1]) * Doc.NativeHeight(doc) + foundChunk.startPage * Doc.NativeHeight(doc); const x2 = parseFloat(values[2]) * Doc.NativeWidth(doc); - const y2 = parseFloat(values[3]) * Doc.NativeHeight(doc); + const y2 = parseFloat(values[3]) * Doc.NativeHeight(doc) + foundChunk.startPage * Doc.NativeHeight(doc); const annotationKey = Doc.LayoutFieldKey(doc) + '_annotations'; @@ -290,7 +293,7 @@ export class ChatBox extends ViewBoxAnnotatableComponent() { }); break; default: - console.log('Chunk type not supported'); + console.log('Chunk type not supported', foundChunk.chunkType); break; } } @@ -339,7 +342,7 @@ export class ChatBox extends ViewBoxAnnotatableComponent() { runInAction(() => { this.history.push({ role: ASSISTANT_ROLE.ASSISTANT, - content: [{ index: 0, type: TEXT_TYPE.NORMAL, text: 'Welcome to the Document Analyser Assistant! Link a document or ask questions to get started.', citation_ids: null }], + content: [{ index: 0, type: TEXT_TYPE.NORMAL, text: `Hey, ${this.userName()} Welcome to the Your Friendly Assistant! Link a document or ask questions about anything to get started.`, citation_ids: null }], processing_info: [], }); }); diff --git a/src/client/views/nodes/ChatBox/MessageComponent.tsx b/src/client/views/nodes/ChatBox/MessageComponent.tsx index d0e78c751..812e52ee0 100644 --- a/src/client/views/nodes/ChatBox/MessageComponent.tsx +++ b/src/client/views/nodes/ChatBox/MessageComponent.tsx @@ -16,7 +16,7 @@ const MessageComponentBox: React.FC = function ({ message const renderContent = (item: MessageContent) => { const i = item.index; - console.log('item', item, 'index', i); + //console.log('item', item, 'index', i); if (item.type === TEXT_TYPE.GROUNDED) { const citation_ids = item.citation_ids || []; return ( diff --git a/src/client/views/nodes/ChatBox/prompts.ts b/src/client/views/nodes/ChatBox/prompts.ts index 845b4d255..76d958daf 100644 --- a/src/client/views/nodes/ChatBox/prompts.ts +++ b/src/client/views/nodes/ChatBox/prompts.ts @@ -27,6 +27,8 @@ export function getReactPrompt(tools: Tool[], summaries: () => string, chatHisto If a tool is needed, ALWAYS select the most appropriate tool based on the user's query. If the query could relate to user documents or require external information (e.g., RAG, search + website scraping, data analysis), USE the appropriate tool to gather that information. If there are no user docs or the user docs have not yielded helpful information, use the search tool to find websites followed by the website scraper tool to get useful infromation from one of those websites. You can use the website scraper (or even the search tool), multiple times to find information from multiple websites either from the same search or different searches. + Always follow the response structure provided in the instructions. + If you are stuck in a loop because a tool isn't helping you even though you think it should, use the 'no_tool' action to proceed with the response and ask the user for more information or clarification or let them know you cannot answer their question and why. diff --git a/src/client/views/nodes/ChatBox/tools/RAGTool.ts b/src/client/views/nodes/ChatBox/tools/RAGTool.ts index 26fa2adc5..fcd0ea43f 100644 --- a/src/client/views/nodes/ChatBox/tools/RAGTool.ts +++ b/src/client/views/nodes/ChatBox/tools/RAGTool.ts @@ -54,9 +54,10 @@ export class RAGTool extends BaseTool<{ hypothetical_document_chunk: string }> { 5. Detailed Citation Guidelines: a. Create a unique citation for each distinct piece of information from the chunks that is used to support grounded_text. + b. ALL TEXT CITATIONS must have direct text in its element content (e.g. DIRECT TEXT HERE) that is a relevant subset of the original text chunk that is being cited specifically. b. Ensure each citation has a unique index number. c. Specify the correct type: "text", "image", or "table". - d. For text chunks, include only the relevant subset of the original text that the grounded_text is based on. + d. For text chunks, the content of the citation should ALWAYS have the relevant subset of the original text that the grounded_text is based on. e. For image/table chunks, leave the citation content empty. f. One citation can be used for multiple grounded_text tags if they are based on the same chunk information. g. !!!DO NOT OVERCITE - only include citations for information that is directly relevant to the grounded_text. @@ -120,7 +121,7 @@ export class RAGTool extends BaseTool<{ hypothetical_document_chunk: string }> { async getFormattedChunks(relevantChunks: RAGChunk[]): Promise<{ type: string; text?: string; image_url?: { url: string } }[]> { try { const { formattedChunks } = await Networking.PostToServer('/formatChunks', { relevantChunks }); - + console.log('Formatted Chunks:', formattedChunks); if (!formattedChunks) { throw new Error('Failed to format chunks'); } diff --git a/src/server/ApiManagers/AssistantManager.ts b/src/server/ApiManagers/AssistantManager.ts index a59a2d22d..9bc5bf128 100644 --- a/src/server/ApiManagers/AssistantManager.ts +++ b/src/server/ApiManagers/AssistantManager.ts @@ -189,7 +189,7 @@ export default class AssistantManager extends ApiManager { try { // Read file data and convert to base64 - const file_data = fs.readFileSync(public_path, { encoding: 'base64' }); + const file_data: string = fs.readFileSync(public_path, { encoding: 'base64' }); const response = await axios.post( 'http://localhost:8080/createDocument', @@ -265,12 +265,12 @@ export default class AssistantManager extends ApiManager { for (const chunk of relevantChunks) { content.push({ type: 'text', - text: ``, + text: ``, }); if (chunk.metadata.type === 'image' || chunk.metadata.type === 'table') { try { - const filePath = serverPathToFile(Directory.parsed_files, chunk.metadata.file_path); + const filePath = serverPathToFile(Directory.chunk_images, chunk.metadata.file_path); const imageBuffer = await readFileAsync(filePath); const base64Image = imageBuffer.toString('base64'); if (base64Image) { @@ -286,10 +286,8 @@ export default class AssistantManager extends ApiManager { } catch (error) { console.error(`Error reading image file for chunk ${chunk.id}:`, error); } - content.push({ type: 'text', text: `\n\n` }); - } else { - content.push({ type: 'text', text: `${chunk.metadata.text}\n\n` }); } + content.push({ type: 'text', text: `${chunk.metadata.text}\n\n` }); } content.push({ type: 'text', text: '' }); -- cgit v1.2.3-70-g09d2 From 4791cd23af08da70895204a3a7fbaf889d9af2d5 Mon Sep 17 00:00:00 2001 From: "A.J. Shulman" Date: Sat, 7 Sep 2024 12:43:05 -0400 Subject: completely restructured, added comments, and significantly reduced the length of the prompt (~72% shorter and cheaper) --- src/client/views/Main.tsx | 2 +- src/client/views/nodes/ChatBox/Agent.ts | 254 ------- src/client/views/nodes/ChatBox/ChatBox.scss | 279 -------- src/client/views/nodes/ChatBox/ChatBox.tsx | 756 --------------------- .../views/nodes/ChatBox/MessageComponent.tsx | 105 --- src/client/views/nodes/ChatBox/ProgressBar.scss | 69 -- src/client/views/nodes/ChatBox/ProgressBar.tsx | 13 - src/client/views/nodes/ChatBox/prompts.ts | 465 ------------- .../nodes/ChatBox/response_parsers/AnswerParser.ts | 125 ---- .../response_parsers/StreamedAnswerParser.ts | 73 -- src/client/views/nodes/ChatBox/tools.ts | 26 - src/client/views/nodes/ChatBox/tools/BaseTool.ts | 24 - .../views/nodes/ChatBox/tools/CalculateTool.ts | 26 - .../views/nodes/ChatBox/tools/CreateCSVTool.ts | 51 -- .../nodes/ChatBox/tools/CreateCollectionTool.ts | 36 - .../views/nodes/ChatBox/tools/DataAnalysisTool.ts | 59 -- .../views/nodes/ChatBox/tools/GetDocsTool.ts | 29 - src/client/views/nodes/ChatBox/tools/NoTool.ts | 18 - src/client/views/nodes/ChatBox/tools/RAGTool.ts | 138 ---- src/client/views/nodes/ChatBox/tools/SearchTool.ts | 54 -- .../nodes/ChatBox/tools/WebsiteInfoScraperTool.ts | 43 -- .../views/nodes/ChatBox/tools/WikipediaTool.ts | 37 - src/client/views/nodes/ChatBox/types.ts | 169 ----- .../views/nodes/ChatBox/vectorstore/Vectorstore.ts | 258 ------- .../views/nodes/chatbot/agentsystem/Agent.ts | 278 ++++++++ .../views/nodes/chatbot/agentsystem/prompts.ts | 181 +++++ .../nodes/chatbot/chatboxcomponents/ChatBox.scss | 279 ++++++++ .../nodes/chatbot/chatboxcomponents/ChatBox.tsx | 745 ++++++++++++++++++++ .../chatbot/chatboxcomponents/MessageComponent.tsx | 146 ++++ .../chatbot/chatboxcomponents/ProgressBar.scss | 69 ++ .../chatbot/chatboxcomponents/ProgressBar.tsx | 22 + .../nodes/chatbot/response_parsers/AnswerParser.ts | 125 ++++ .../response_parsers/StreamedAnswerParser.ts | 73 ++ src/client/views/nodes/chatbot/tools/BaseTool.ts | 24 + .../views/nodes/chatbot/tools/CalculateTool.ts | 26 + .../views/nodes/chatbot/tools/CreateCSVTool.ts | 51 ++ .../nodes/chatbot/tools/CreateCollectionTool.ts | 36 + .../views/nodes/chatbot/tools/DataAnalysisTool.ts | 59 ++ .../views/nodes/chatbot/tools/GetDocsTool.ts | 29 + src/client/views/nodes/chatbot/tools/NoTool.ts | 18 + src/client/views/nodes/chatbot/tools/RAGTool.ts | 138 ++++ src/client/views/nodes/chatbot/tools/SearchTool.ts | 54 ++ .../nodes/chatbot/tools/WebsiteInfoScraperTool.ts | 83 +++ .../views/nodes/chatbot/tools/WikipediaTool.ts | 37 + src/client/views/nodes/chatbot/types/types.ts | 129 ++++ .../views/nodes/chatbot/vectorstore/Vectorstore.ts | 258 +++++++ src/server/ApiManagers/AssistantManager.ts | 2 +- 47 files changed, 2862 insertions(+), 3109 deletions(-) delete mode 100644 src/client/views/nodes/ChatBox/Agent.ts delete mode 100644 src/client/views/nodes/ChatBox/ChatBox.scss delete mode 100644 src/client/views/nodes/ChatBox/ChatBox.tsx delete mode 100644 src/client/views/nodes/ChatBox/MessageComponent.tsx delete mode 100644 src/client/views/nodes/ChatBox/ProgressBar.scss delete mode 100644 src/client/views/nodes/ChatBox/ProgressBar.tsx delete mode 100644 src/client/views/nodes/ChatBox/prompts.ts delete mode 100644 src/client/views/nodes/ChatBox/response_parsers/AnswerParser.ts delete mode 100644 src/client/views/nodes/ChatBox/response_parsers/StreamedAnswerParser.ts delete mode 100644 src/client/views/nodes/ChatBox/tools.ts delete mode 100644 src/client/views/nodes/ChatBox/tools/BaseTool.ts delete mode 100644 src/client/views/nodes/ChatBox/tools/CalculateTool.ts delete mode 100644 src/client/views/nodes/ChatBox/tools/CreateCSVTool.ts delete mode 100644 src/client/views/nodes/ChatBox/tools/CreateCollectionTool.ts delete mode 100644 src/client/views/nodes/ChatBox/tools/DataAnalysisTool.ts delete mode 100644 src/client/views/nodes/ChatBox/tools/GetDocsTool.ts delete mode 100644 src/client/views/nodes/ChatBox/tools/NoTool.ts delete mode 100644 src/client/views/nodes/ChatBox/tools/RAGTool.ts delete mode 100644 src/client/views/nodes/ChatBox/tools/SearchTool.ts delete mode 100644 src/client/views/nodes/ChatBox/tools/WebsiteInfoScraperTool.ts delete mode 100644 src/client/views/nodes/ChatBox/tools/WikipediaTool.ts delete mode 100644 src/client/views/nodes/ChatBox/types.ts delete mode 100644 src/client/views/nodes/ChatBox/vectorstore/Vectorstore.ts create mode 100644 src/client/views/nodes/chatbot/agentsystem/Agent.ts create mode 100644 src/client/views/nodes/chatbot/agentsystem/prompts.ts create mode 100644 src/client/views/nodes/chatbot/chatboxcomponents/ChatBox.scss create mode 100644 src/client/views/nodes/chatbot/chatboxcomponents/ChatBox.tsx create mode 100644 src/client/views/nodes/chatbot/chatboxcomponents/MessageComponent.tsx create mode 100644 src/client/views/nodes/chatbot/chatboxcomponents/ProgressBar.scss create mode 100644 src/client/views/nodes/chatbot/chatboxcomponents/ProgressBar.tsx create mode 100644 src/client/views/nodes/chatbot/response_parsers/AnswerParser.ts create mode 100644 src/client/views/nodes/chatbot/response_parsers/StreamedAnswerParser.ts create mode 100644 src/client/views/nodes/chatbot/tools/BaseTool.ts create mode 100644 src/client/views/nodes/chatbot/tools/CalculateTool.ts create mode 100644 src/client/views/nodes/chatbot/tools/CreateCSVTool.ts create mode 100644 src/client/views/nodes/chatbot/tools/CreateCollectionTool.ts create mode 100644 src/client/views/nodes/chatbot/tools/DataAnalysisTool.ts create mode 100644 src/client/views/nodes/chatbot/tools/GetDocsTool.ts create mode 100644 src/client/views/nodes/chatbot/tools/NoTool.ts create mode 100644 src/client/views/nodes/chatbot/tools/RAGTool.ts create mode 100644 src/client/views/nodes/chatbot/tools/SearchTool.ts create mode 100644 src/client/views/nodes/chatbot/tools/WebsiteInfoScraperTool.ts create mode 100644 src/client/views/nodes/chatbot/tools/WikipediaTool.ts create mode 100644 src/client/views/nodes/chatbot/types/types.ts create mode 100644 src/client/views/nodes/chatbot/vectorstore/Vectorstore.ts (limited to 'src/client/views/nodes/ChatBox/MessageComponent.tsx') diff --git a/src/client/views/Main.tsx b/src/client/views/Main.tsx index f7cd0e925..e1f7dd233 100644 --- a/src/client/views/Main.tsx +++ b/src/client/views/Main.tsx @@ -27,7 +27,7 @@ import { CollectionSchemaView } from './collections/collectionSchema/CollectionS import { SchemaRowBox } from './collections/collectionSchema/SchemaRowBox'; import './global/globalScripts'; import { AudioBox } from './nodes/AudioBox'; -import { ChatBox } from './nodes/ChatBox/ChatBox'; +import { ChatBox } from './nodes/chatbot/chatboxcomponents/ChatBox'; import { ComparisonBox } from './nodes/ComparisonBox'; import { DataVizBox } from './nodes/DataVizBox/DataVizBox'; import { DiagramBox } from './nodes/DiagramBox'; diff --git a/src/client/views/nodes/ChatBox/Agent.ts b/src/client/views/nodes/ChatBox/Agent.ts deleted file mode 100644 index 9eb069c78..000000000 --- a/src/client/views/nodes/ChatBox/Agent.ts +++ /dev/null @@ -1,254 +0,0 @@ -import OpenAI from 'openai'; -import { Tool, AgentMessage, AssistantMessage, TEXT_TYPE, CHUNK_TYPE, ASSISTANT_ROLE, ProcessingInfo, PROCESSING_TYPE } from './types'; -import { getReactPrompt } from './prompts'; -import { XMLParser, XMLBuilder } from 'fast-xml-parser'; -import { Vectorstore } from './vectorstore/Vectorstore'; -import { ChatCompletionMessageParam } from 'openai/resources'; -import dotenv from 'dotenv'; -import { CalculateTool } from './tools/CalculateTool'; -import { RAGTool } from './tools/RAGTool'; -import { DataAnalysisTool } from './tools/DataAnalysisTool'; -import { WebsiteInfoScraperTool } from './tools/WebsiteInfoScraperTool'; -import { SearchTool } from './tools/SearchTool'; -import { NoTool } from './tools/NoTool'; -import { on } from 'events'; -import { v4 as uuidv4 } from 'uuid'; -import { AnswerParser } from './response_parsers/AnswerParser'; -import { StreamedAnswerParser } from './response_parsers/StreamedAnswerParser'; -import { CreateCSVTool } from './tools/CreateCSVTool'; - -dotenv.config(); - -export class Agent { - private client: OpenAI; - private tools: Record>; - private messages: AgentMessage[] = []; - private interMessages: AgentMessage[] = []; - private vectorstore: Vectorstore; - private _history: () => string; - private _summaries: () => string; - private _csvData: () => { filename: string; id: string; text: string }[]; - private actionNumber: number = 0; - private thoughtNumber: number = 0; - private processingNumber: number = 0; - private processingInfo: ProcessingInfo[] = []; - private streamedAnswerParser: StreamedAnswerParser = new StreamedAnswerParser(); - - constructor( - _vectorstore: Vectorstore, - summaries: () => string, - history: () => string, - csvData: () => { filename: string; id: string; text: string }[], - addLinkedUrlDoc: (url: string, id: string) => void, - createCSVInDash: (url: string, title: string, id: string, data: string) => void - ) { - this.client = new OpenAI({ apiKey: process.env.OPENAI_KEY, dangerouslyAllowBrowser: true }); - this.vectorstore = _vectorstore; - this._history = history; - this._summaries = summaries; - this._csvData = csvData; - this.tools = { - calculate: new CalculateTool(), - rag: new RAGTool(this.vectorstore), - dataAnalysis: new DataAnalysisTool(csvData), - websiteInfoScraper: new WebsiteInfoScraperTool(addLinkedUrlDoc), - searchTool: new SearchTool(addLinkedUrlDoc), - createCSV: new CreateCSVTool(createCSVInDash), - no_tool: new NoTool(), - }; - } - - async askAgent(question: string, onProcessingUpdate: (processingUpdate: ProcessingInfo[]) => void, onAnswerUpdate: (answerUpdate: string) => void, maxTurns: number = 30): Promise { - console.log(`Starting query: ${question}`); - this.messages.push({ role: 'user', content: question }); - const chatHistory = this._history(); - const systemPrompt = getReactPrompt(Object.values(this.tools), this._summaries, chatHistory); - this.interMessages = [{ role: 'system', content: systemPrompt }]; - this.interMessages.push({ role: 'user', content: `${question}` }); - const parser = new XMLParser({ - ignoreAttributes: false, - attributeNamePrefix: '@_', - textNodeName: '_text', - isArray: (name, jpath, isLeafNode, isAttribute) => { - // Convert tags with the same name to arrays - return ['query', 'url'].indexOf(name) !== -1; - }, - }); - const builder = new XMLBuilder({ ignoreAttributes: false, attributeNamePrefix: '@_' }); - - let currentAction: string | undefined; - - this.processingInfo = []; - - for (let i = 2; i < maxTurns; i += 2) { - console.log(this.interMessages); - console.log(`Turn ${i}/${maxTurns}`); - - const result = await this.execute(onProcessingUpdate, onAnswerUpdate); - this.interMessages.push({ role: 'assistant', content: result }); - - let parsedResult; - try { - parsedResult = parser.parse(result); - } catch (error) { - throw new Error(`Error parsing response: ${error}`); - } - - const stage = parsedResult.stage; - - if (!stage) { - throw new Error(`Error: No stage found in response`); - } - - for (const key in stage) { - if (key === 'thought') { - console.log(`Thought: ${stage[key]}`); - this.processingNumber++; - } else if (key === 'action') { - currentAction = stage[key] as string; - console.log(`Action: ${currentAction}`); - if (this.tools[currentAction]) { - const nextPrompt = [ - { - type: 'text', - text: `` + builder.build({ action_rules: this.tools[currentAction].getActionRule() }) + ``, - }, - ]; - this.interMessages.push({ role: 'user', content: nextPrompt }); - break; - } else { - console.log('Error: No valid action'); - this.interMessages.push({ role: 'user', content: `No valid action, try again.` }); - break; - } - } else if (key === 'action_input') { - const actionInput = stage[key]; - console.log(`Action input:`, actionInput.inputs); - if (currentAction) { - try { - // Parse the inputs - //const parsedInputs = this.parseActionInputs(actionInput.inputs); - //console.log(`Parsed inputs:`, parsedInputs); - const observation = await this.processAction(currentAction, actionInput.inputs); - const nextPrompt = [{ type: 'text', text: ` ` }, ...observation, { type: 'text', text: '' }]; - console.log(observation); - this.interMessages.push({ role: 'user', content: nextPrompt }); - this.processingNumber++; - break; - } catch (error) { - throw new Error(`Error processing action: ${error}`); - } - } else { - throw new Error('Error: Action input without a valid action'); - } - } else if (key === 'answer') { - console.log('Answer found. Ending query.'); - this.streamedAnswerParser.reset(); - const parsedAnswer = AnswerParser.parse(result, this.processingInfo); - return parsedAnswer; - } - } - } - throw new Error('Reached maximum turns. Ending query.'); - } - - private async execute(onProcessingUpdate: (processingUpdate: ProcessingInfo[]) => void, onAnswerUpdate: (answerUpdate: string) => void): Promise { - const stream = await this.client.chat.completions.create({ - model: 'gpt-4o', - messages: this.interMessages as ChatCompletionMessageParam[], - temperature: 0, - stream: true, - }); - - let fullResponse: string = ''; - let currentTag: string = ''; - let currentContent: string = ''; - let isInsideTag: boolean = false; - - for await (const chunk of stream) { - let content = chunk.choices[0]?.delta?.content || ''; - fullResponse += content; - - for (const char of content) { - if (currentTag === 'answer') { - currentContent += char; - //console.log(char); - const streamedAnswer = this.streamedAnswerParser.parse(char); - //console.log(streamedAnswer); - onAnswerUpdate(streamedAnswer); - continue; - } else if (char === '<') { - isInsideTag = true; - currentTag = ''; - currentContent = ''; - } else if (char === '>') { - isInsideTag = false; - if (currentTag.startsWith('/')) { - currentTag = ''; - } - } else if (isInsideTag) { - currentTag += char; - } else if (currentTag === 'thought' || currentTag === 'action_input_description') { - currentContent += char; - const current_info = this.processingInfo.find(info => info.index === this.processingNumber); - if (current_info) { - current_info.content = currentContent.trim(); - onProcessingUpdate(this.processingInfo); - } else { - this.processingInfo.push({ index: this.processingNumber, type: currentTag === 'thought' ? PROCESSING_TYPE.THOUGHT : PROCESSING_TYPE.ACTION, content: currentContent.trim() }); - onProcessingUpdate(this.processingInfo); - } - } - } - } - - return fullResponse; - } - - private async processAction(action: string, actionInput: any): Promise { - if (!(action in this.tools)) { - throw new Error(`Unknown action: ${action}`); - } - - const tool = this.tools[action]; - const args: Record = {}; - - // for (const paramName in tool.parameters) { - // if (actionInput[paramName] !== undefined) { - // if (Array.isArray(actionInput[paramName])) { - // // If the input is already an array, use it as is - // args[paramName] = actionInput[paramName]; - // } else if (typeof actionInput[paramName] === 'object' && actionInput[paramName] !== null) { - // // If the input is an object, check if it has multiple of the same tag - // const values = Object.values(actionInput[paramName]); - // if (values.length > 1) { - // // If there are multiple values, convert to an array - // args[paramName] = values; - // } else { - // // If there's only one value, use it directly - // args[paramName] = values[0]; - // } - // } else { - // // For single values, use them as is - // args[paramName] = actionInput[paramName]; - // } - // } else if (tool.parameters[paramName].required === 'true') { - // throw new Error(`Missing required parameter '${paramName}' for action '${action}'`); - // } - // } - - return await tool.execute(actionInput); - } - - private parseActionInputs(inputs: any): Record { - const parsedInputs: Record = {}; - for (const key in inputs) { - if (Array.isArray(inputs[key])) { - parsedInputs[key] = inputs[key].map((item: any) => item._text); - } else { - parsedInputs[key] = inputs[key]._text; - } - } - return parsedInputs; - } -} diff --git a/src/client/views/nodes/ChatBox/ChatBox.scss b/src/client/views/nodes/ChatBox/ChatBox.scss deleted file mode 100644 index 42f6a0d61..000000000 --- a/src/client/views/nodes/ChatBox/ChatBox.scss +++ /dev/null @@ -1,279 +0,0 @@ -@import url('https://fonts.googleapis.com/css2?family=Atkinson+Hyperlegible:ital,wght@0,400;0,700;1,400;1,700&display=swap'); - -$primary-color: #4a90e2; -$secondary-color: #f5f8fa; -$text-color: #333; -$light-text-color: #777; -$border-color: #e1e8ed; -$shadow-color: rgba(0, 0, 0, 0.1); -$transition: all 0.3s ease; -.chat-box { - display: flex; - flex-direction: column; - height: 100%; - background-color: #fff; - font-family: - 'Atkinson Hyperlegible', - -apple-system, - BlinkMacSystemFont, - 'Segoe UI', - Roboto, - Helvetica, - Arial, - sans-serif; - border-radius: 12px; - overflow: hidden; - box-shadow: 0 4px 12px $shadow-color; - position: relative; - - .chat-header { - background-color: $primary-color; - color: white; - padding: 15px; - text-align: center; - box-shadow: 0 2px 4px $shadow-color; - height: fit-content; - - h2 { - margin: 0; - font-size: 1.3em; - font-weight: 500; - } - } - - .chat-messages { - flex-grow: 1; - overflow-y: auto; - padding: 20px; - display: flex; - flex-direction: column; - gap: 10px; // Added to give space between elements - - &::-webkit-scrollbar { - width: 6px; - } - - &::-webkit-scrollbar-thumb { - background-color: $border-color; - border-radius: 3px; - } - } - - .chat-input { - display: flex; - padding: 20px; - border-top: 1px solid $border-color; - background-color: #fff; - - input { - flex-grow: 1; - padding: 12px 15px; - border: 1px solid $border-color; - border-radius: 24px; - font-size: 15px; - transition: $transition; - - &:focus { - outline: none; - border-color: $primary-color; - box-shadow: 0 0 0 2px rgba($primary-color, 0.2); - } - } - - .submit-button { - background-color: $primary-color; - color: white; - border: none; - border-radius: 50%; - width: 48px; - height: 48px; - margin-left: 10px; - cursor: pointer; - transition: $transition; - display: flex; - align-items: center; - justify-content: center; - - &:hover { - background-color: darken($primary-color, 10%); - } - - &:disabled { - background-color: $light-text-color; - cursor: not-allowed; - } - - .spinner { - height: 24px; - width: 24px; - border: 3px solid rgba(255, 255, 255, 0.3); - border-top: 3px solid #fff; - border-radius: 50%; - animation: spin 1s linear infinite; - display: flex; - align-items: center; - justify-content: center; - } - } - } - .citation-popup { - position: fixed; - bottom: 50px; - left: 50%; - transform: translateX(-50%); - background-color: rgba(0, 0, 0, 0.8); - color: white; - padding: 10px 20px; - border-radius: 10px; - box-shadow: 0 4px 12px rgba(0, 0, 0, 0.2); - z-index: 1000; - animation: fadeIn 0.3s ease-in-out; - - p { - margin: 0; - font-size: 14px; - } - - @keyframes fadeIn { - from { - opacity: 0; - } - to { - opacity: 1; - } - } - } -} - -.message { - max-width: 80%; - margin-bottom: 20px; - padding: 16px 20px; - border-radius: 18px; - font-size: 15px; - line-height: 1.5; - box-shadow: 0 2px 4px $shadow-color; - word-wrap: break-word; // To handle long words - - &.user { - align-self: flex-end; - background-color: $primary-color; - color: white; - border-bottom-right-radius: 4px; - } - - &.chatbot { - align-self: flex-start; - background-color: $secondary-color; - color: $text-color; - border-bottom-left-radius: 4px; - } - - .toggle-info { - background-color: transparent; - color: $primary-color; - border: 1px solid $primary-color; - width: 100%; - height: fit-content; - border-radius: 8px; - padding: 10px 16px; - font-size: 14px; - cursor: pointer; - transition: $transition; - margin-top: 10px; - - &:hover { - background-color: rgba($primary-color, 0.1); - } - } -} - -.follow-up-questions { - margin-top: 15px; - - h4 { - font-size: 15px; - font-weight: 600; - margin-bottom: 10px; - } - - .questions-list { - display: flex; - flex-direction: column; - gap: 10px; - } - - .follow-up-button { - background-color: #fff; - color: $primary-color; - border: 1px solid $primary-color; - border-radius: 8px; - padding: 10px 16px; - font-size: 14px; - cursor: pointer; - transition: $transition; - text-align: left; - white-space: normal; - word-wrap: break-word; - width: 100%; - height: fit-content; - - &:hover { - background-color: $primary-color; - color: #fff; - } - } -} - -.citation-button { - display: inline-flex; - align-items: center; - justify-content: center; - width: 20px; - height: 20px; - border-radius: 50%; - background-color: rgba(0, 0, 0, 0.1); - color: $text-color; - font-size: 12px; - font-weight: bold; - margin-left: 5px; - cursor: pointer; - transition: $transition; - vertical-align: middle; - - &:hover { - background-color: rgba(0, 0, 0, 0.2); - } -} - -.uploading-overlay { - position: absolute; - top: 0; - left: 0; - right: 0; - bottom: 0; - background-color: rgba(255, 255, 255, 0.8); - display: flex; - justify-content: center; - align-items: center; - z-index: 1000; -} - -@keyframes spin { - 0% { - transform: rotate(0deg); - } - 100% { - transform: rotate(360deg); - } -} - -@media (max-width: 768px) { - .chat-box { - border-radius: 0; - } - - .message { - max-width: 90%; - } -} diff --git a/src/client/views/nodes/ChatBox/ChatBox.tsx b/src/client/views/nodes/ChatBox/ChatBox.tsx deleted file mode 100644 index 98a2e6002..000000000 --- a/src/client/views/nodes/ChatBox/ChatBox.tsx +++ /dev/null @@ -1,756 +0,0 @@ -import { action, computed, makeObservable, observable, observe, reaction, runInAction, ObservableSet } from 'mobx'; -import { observer } from 'mobx-react'; -import OpenAI, { ClientOptions } from 'openai'; -import * as React from 'react'; -import { Doc, DocListCast } from '../../../../fields/Doc'; -import { CsvCast, DocCast, PDFCast, RTFCast, StrCast } from '../../../../fields/Types'; -import { DocumentType } from '../../../documents/DocumentTypes'; -import { Docs } from '../../../documents/Documents'; -import { LinkManager } from '../../../util/LinkManager'; -import { ViewBoxAnnotatableComponent } from '../../DocComponent'; -import { FieldView, FieldViewProps } from '../FieldView'; -import './ChatBox.scss'; -import MessageComponentBox from './MessageComponent'; -import { ASSISTANT_ROLE, AssistantMessage, Citation, CHUNK_TYPE, TEXT_TYPE, SimplifiedChunk, ProcessingInfo } from './types'; -import { Vectorstore } from './vectorstore/Vectorstore'; -import { Agent } from './Agent'; -import dotenv from 'dotenv'; -import { DocData, DocViews } from '../../../../fields/DocSymbols'; -import { DocumentManager } from '../../../util/DocumentManager'; -import { v4 as uuidv4 } from 'uuid'; -import { DocUtils } from '../../../documents/DocUtils'; -import { ClientUtils } from '../../../../ClientUtils'; -import { ProgressBar } from './ProgressBar'; -import { DocumentView } from '../DocumentView'; -import { Networking } from '../../../Network'; - -dotenv.config(); - -/** - * ChatBox is the main class responsible for managing the interaction between the user and the assistant, - * handling documents, and integrating with OpenAI for tasks such as document analysis, chat functionality, - * and vector store interactions. - */ -@observer -export class ChatBox extends ViewBoxAnnotatableComponent() { - // MobX observable properties to track UI state and data - @observable history: AssistantMessage[] = []; - @observable.deep current_message: AssistantMessage | undefined = undefined; - @observable isLoading: boolean = false; - @observable uploadProgress: number = 0; - @observable currentStep: string = ''; - @observable expandedScratchpadIndex: number | null = null; - @observable inputValue: string = ''; - @observable private linked_docs_to_add: ObservableSet = observable.set(); - @observable private linked_csv_files: { filename: string; id: string; text: string }[] = []; - @observable private isUploadingDocs: boolean = false; - @observable private citationPopup: { text: string; visible: boolean } = { text: '', visible: false }; - - // Private properties for managing OpenAI API, vector store, agent, and UI elements - private openai: OpenAI; - private vectorstore_id: string; - private vectorstore: Vectorstore; - private agent: Agent; - private _oldWheel: HTMLDivElement | null = null; - private messagesRef: React.RefObject; - - /** - * Static method that returns the layout string for the field. - * @param fieldKey Key to get the layout string. - */ - public static LayoutString(fieldKey: string) { - return FieldView.LayoutString(ChatBox, fieldKey); - } - - /** - * Constructor initializes the component, sets up OpenAI, vector store, and agent instances, - * and observes changes in the chat history to save the state in dataDoc. - * @param props The properties passed to the component. - */ - constructor(props: FieldViewProps) { - super(props); - makeObservable(this); // Enable MobX observables - - // Initialize OpenAI, vectorstore, and agent - this.openai = this.initializeOpenAI(); - if (StrCast(this.dataDoc.vectorstore_id) == '') { - console.log('new_id'); - this.vectorstore_id = uuidv4(); - this.dataDoc.vectorstore_id = this.vectorstore_id; - } else { - this.vectorstore_id = StrCast(this.dataDoc.vectorstore_id); - } - this.vectorstore = new Vectorstore(this.vectorstore_id, this.retrieveDocIds); - this.agent = new Agent(this.vectorstore, this.retrieveSummaries, this.retrieveFormattedHistory, this.retrieveCSVData, this.addLinkedUrlDoc, this.createCSVInDash); - this.messagesRef = React.createRef(); - - // Reaction to update dataDoc when chat history changes - reaction( - () => - this.history.map((msg: AssistantMessage) => ({ - role: msg.role, - content: msg.content, - follow_up_questions: msg.follow_up_questions, - citations: msg.citations, - })), - serializableHistory => { - this.dataDoc.data = JSON.stringify(serializableHistory); - } - ); - } - - /** - * Adds a document to the vectorstore for AI-based analysis. - * Handles the upload progress and errors during the process. - * @param newLinkedDoc The new document to add. - */ - @action - addDocToVectorstore = async (newLinkedDoc: Doc) => { - this.uploadProgress = 0; - this.currentStep = 'Initializing...'; - this.isUploadingDocs = true; - - try { - // Add the document to the vectorstore - await this.vectorstore.addAIDoc(newLinkedDoc, this.updateProgress); - } catch (error) { - console.error('Error uploading document:', error); - this.currentStep = 'Error during upload'; - } finally { - this.isUploadingDocs = false; - this.uploadProgress = 0; - this.currentStep = ''; - } - }; - - /** - * Updates the upload progress and the current step in the UI. - * @param progress The percentage of the progress. - * @param step The current step name. - */ - @action - updateProgress = (progress: number, step: string) => { - console.log('Progress:', progress, step); - this.uploadProgress = progress; - this.currentStep = step; - }; - - /** - * Adds a CSV file for analysis by sending it to OpenAI and generating a summary. - * @param newLinkedDoc The linked document representing the CSV file. - * @param id Optional ID for the document. - */ - @action - addCSVForAnalysis = async (newLinkedDoc: Doc, id?: string) => { - console.log('adding csv file for analysis'); - if (!newLinkedDoc.chunk_simpl) { - // Convert document text to CSV data - const csvData: string = StrCast(newLinkedDoc.text); - console.log('CSV Data:', csvData); - - // Generate a summary using OpenAI API - const completion = await this.openai.chat.completions.create({ - messages: [ - { - role: 'system', - content: - 'You are an AI assistant tasked with summarizing the content of a CSV file. You will be provided with the data from the CSV file and your goal is to generate a concise summary that captures the main themes, trends, and key points represented in the data.', - }, - { - role: 'user', - content: `Please provide a comprehensive summary of the CSV file based on the provided data. Ensure the summary highlights the most important information, patterns, and insights. Your response should be in paragraph form and be concise. - CSV Data: - ${csvData} - ********** - Summary:`, - }, - ], - model: 'gpt-3.5-turbo', - }); - - const csvId = id ?? uuidv4(); - - // Add CSV details to linked files - this.linked_csv_files.push({ - filename: CsvCast(newLinkedDoc.data).url.pathname, - id: csvId, - text: csvData, - }); - - // Add a chunk for the CSV and assign the summary - const chunkToAdd = { - chunkId: csvId, - chunkType: CHUNK_TYPE.CSV, - }; - newLinkedDoc.chunk_simpl = JSON.stringify({ chunks: [chunkToAdd] }); - newLinkedDoc.summary = completion.choices[0].message.content!; - } - }; - - /** - * Toggles the tool logs, expanding or collapsing the scratchpad at the given index. - * @param index Index of the tool log to toggle. - */ - @action - toggleToolLogs = (index: number) => { - this.expandedScratchpadIndex = this.expandedScratchpadIndex === index ? null : index; - }; - - /** - * Initializes the OpenAI API client using the API key from environment variables. - * @returns OpenAI client instance. - */ - initializeOpenAI() { - console.log(process.env.OPENAI_KEY); - const configuration: ClientOptions = { - apiKey: process.env.OPENAI_KEY, - dangerouslyAllowBrowser: true, - }; - return new OpenAI(configuration); - } - - /** - * Adds a scroll event listener to detect user scrolling and handle passive wheel events. - */ - addScrollListener = () => { - if (this.messagesRef.current) { - this.messagesRef.current.addEventListener('wheel', this.onPassiveWheel, { passive: false }); - } - }; - - /** - * Removes the scroll event listener from the chat messages container. - */ - removeScrollListener = () => { - if (this.messagesRef.current) { - this.messagesRef.current.removeEventListener('wheel', this.onPassiveWheel); - } - }; - - /** - * Scrolls the chat messages container to the bottom, ensuring the latest message is visible. - */ - scrollToBottom = () => { - if (this.messagesRef.current) { - this.messagesRef.current.scrollTop = this.messagesRef.current.scrollHeight; - } - }; - - /** - * Event handler for detecting wheel scrolling and stopping the event propagation. - * @param e The wheel event. - */ - onPassiveWheel = (e: WheelEvent) => { - if (this._props.isContentActive()) { - e.stopPropagation(); - } - }; - - /** - * Sends the user's input to OpenAI, displays the loading indicator, and updates the chat history. - * @param event The form submission event. - */ - @action - askGPT = async (event: React.FormEvent): Promise => { - event.preventDefault(); - this.inputValue = ''; - - // Extract the user's message - const textInput = event.currentTarget.elements.namedItem('messageInput') as HTMLInputElement; - const trimmedText = textInput.value.trim(); - - if (trimmedText) { - try { - textInput.value = ''; - // Add the user's message to the history - this.history.push({ - role: ASSISTANT_ROLE.USER, - content: [{ index: 0, type: TEXT_TYPE.NORMAL, text: trimmedText, citation_ids: null }], - processing_info: [], - }); - this.isLoading = true; - this.current_message = { - role: ASSISTANT_ROLE.ASSISTANT, - content: [], - citations: [], - processing_info: [], - }; - - // Define callbacks for real-time processing updates - const onProcessingUpdate = (processingUpdate: ProcessingInfo[]) => { - runInAction(() => { - if (this.current_message) { - this.current_message = { - ...this.current_message, - processing_info: processingUpdate, - }; - } - }); - this.scrollToBottom(); - }; - - const onAnswerUpdate = (answerUpdate: string) => { - runInAction(() => { - if (this.current_message) { - this.current_message = { - ...this.current_message, - content: [{ text: answerUpdate, type: TEXT_TYPE.NORMAL, index: 0, citation_ids: [] }], - }; - } - }); - }; - - // Send the user's question to the assistant and get the final message - const finalMessage = await this.agent.askAgent(trimmedText, onProcessingUpdate, onAnswerUpdate); - - // Update the history with the final assistant message - runInAction(() => { - if (this.current_message) { - this.history.push({ ...finalMessage }); - this.current_message = undefined; - this.dataDoc.data = JSON.stringify(this.history); - } - }); - } catch (err) { - console.error('Error:', err); - // Handle error in processing - this.history.push({ - role: ASSISTANT_ROLE.ASSISTANT, - content: [{ index: 0, type: TEXT_TYPE.ERROR, text: 'Sorry, I encountered an error while processing your request.', citation_ids: null }], - processing_info: [], - }); - } finally { - this.isLoading = false; - this.scrollToBottom(); - } - } - this.scrollToBottom(); - }; - - /** - * Updates the citations for a given message in the chat history. - * @param index The index of the message in the history. - * @param citations The list of citations to add to the message. - */ - @action - updateMessageCitations = (index: number, citations: Citation[]) => { - if (this.history[index]) { - this.history[index].citations = citations; - } - }; - - /** - * Adds a linked document from a URL for future reference and analysis. - * @param url The URL of the document to add. - * @param id The unique identifier for the document. - */ - @action - addLinkedUrlDoc = async (url: string, id: string) => { - const doc = Docs.Create.WebDocument(url, { data_useCors: true }); - console.log('Adding URL:', url); - - const linkDoc = Docs.Create.LinkDocument(this.Document, doc); - LinkManager.Instance.addLink(linkDoc); - let canDisplay; - - try { - // Fetch the URL content through the proxy - const { data } = await Networking.PostToServer('/proxyFetch', { url }); - - // Simulating header behavior since we can't fetch headers via proxy - const xFrameOptions = data.headers?.['x-frame-options']; - - if (xFrameOptions && xFrameOptions.toUpperCase() === 'SAMEORIGIN') { - console.log('URL cannot be displayed in an iframe:', url); - canDisplay = false; - } else { - console.log('URL can be displayed in an iframe:', url); - canDisplay = true; - } - } catch (error) { - console.error('Error fetching the URL from the server:', error); - } - - const chunkToAdd = { - chunkId: id, - chunkType: CHUNK_TYPE.URL, - url: url, - canDisplay: canDisplay, - }; - - doc.chunk_simpl = JSON.stringify({ chunks: [chunkToAdd] }); - }; - - /** - * Getter to retrieve the current user's name from the client utils. - */ - @computed - get userName() { - return ClientUtils.CurrentUserEmail; - } - - /** - * Creates a CSV document in the dashboard and adds it for analysis. - * @param url The URL of the CSV. - * @param title The title of the CSV document. - * @param id The unique ID for the document. - * @param data The CSV data content. - */ - @action - createCSVInDash = async (url: string, title: string, id: string, data: string) => { - console.log('Creating CSV in Dash:', url, title); - const doc = DocCast(await DocUtils.DocumentFromType('csv', url, { title: title, text: RTFCast(data) })); - - const linkDoc = Docs.Create.LinkDocument(this.Document, doc); - LinkManager.Instance.addLink(linkDoc); - - doc && this._props.addDocument?.(doc); - await DocumentManager.Instance.showDocument(doc, { willZoomCentered: true }, () => {}); - - this.addCSVForAnalysis(doc, id); - }; - - /** - * Event handler to manage citations click in the message components. - * @param citation The citation object clicked by the user. - */ - @action - handleCitationClick = (citation: Citation) => { - console.log('Citation clicked:', citation); - const currentLinkedDocs: Doc[] = this.linkedDocs; - - const chunkId = citation.chunk_id; - - // Loop through the linked documents to find the matching chunk and handle its display - for (let doc of currentLinkedDocs) { - if (doc.chunk_simpl) { - const docChunkSimpl = JSON.parse(StrCast(doc.chunk_simpl)) as { chunks: SimplifiedChunk[] }; - const foundChunk = docChunkSimpl.chunks.find(chunk => chunk.chunkId === chunkId); - if (foundChunk) { - // Handle different types of chunks (image, text, table, etc.) - switch (foundChunk.chunkType) { - case CHUNK_TYPE.IMAGE: - case CHUNK_TYPE.TABLE: - const values = foundChunk.location?.replace(/[\[\]]/g, '').split(','); - - if (values?.length !== 4) { - console.error('Location string must contain exactly 4 numbers'); - return; - } - - const x1 = parseFloat(values[0]) * Doc.NativeWidth(doc); - const y1 = parseFloat(values[1]) * Doc.NativeHeight(doc) + foundChunk.startPage * Doc.NativeHeight(doc); - const x2 = parseFloat(values[2]) * Doc.NativeWidth(doc); - const y2 = parseFloat(values[3]) * Doc.NativeHeight(doc) + foundChunk.startPage * Doc.NativeHeight(doc); - - const annotationKey = Doc.LayoutFieldKey(doc) + '_annotations'; - - const existingDoc = DocListCast(doc[DocData][annotationKey]).find(d => d.citation_id === citation.citation_id); - const highlightDoc = existingDoc ?? this.createImageCitationHighlight(x1, y1, x2, y2, citation, annotationKey, doc); - - DocumentManager.Instance.showDocument(highlightDoc, { willZoomCentered: true }, () => {}); - break; - case CHUNK_TYPE.TEXT: - this.citationPopup = { text: citation.direct_text ?? 'No text available', visible: true }; - setTimeout(() => (this.citationPopup.visible = false), 3000); // Hide after 3 seconds - - DocumentManager.Instance.showDocument(doc, { willZoomCentered: true }, () => { - const firstView = Array.from(doc[DocViews])[0] as DocumentView; - firstView.ComponentView?.search?.(citation.direct_text ?? ''); - }); - break; - case CHUNK_TYPE.URL: - if (!foundChunk.canDisplay) { - window.open(StrCast(doc.displayUrl), '_blank'); - } else if (foundChunk.canDisplay) { - DocumentManager.Instance.showDocument(doc, { willZoomCentered: true }, () => {}); - } - break; - case CHUNK_TYPE.CSV: - DocumentManager.Instance.showDocument(doc, { willZoomCentered: true }, () => {}); - break; - default: - console.log('Chunk type not supported', foundChunk.chunkType); - break; - } - } - } - } - }; - - /** - * Creates an annotation highlight on a PDF document for image citations. - * @param x1 X-coordinate of the top-left corner of the highlight. - * @param y1 Y-coordinate of the top-left corner of the highlight. - * @param x2 X-coordinate of the bottom-right corner of the highlight. - * @param y2 Y-coordinate of the bottom-right corner of the highlight. - * @param citation The citation object to associate with the highlight. - * @param annotationKey The key used to store the annotation. - * @param pdfDoc The document where the highlight is created. - * @returns The highlighted document. - */ - createImageCitationHighlight = (x1: number, y1: number, x2: number, y2: number, citation: Citation, annotationKey: string, pdfDoc: Doc): Doc => { - const highlight_doc = Docs.Create.FreeformDocument([], { - x: x1, - y: y1, - _width: x2 - x1, - _height: y2 - y1, - backgroundColor: 'rgba(255, 255, 0, 0.5)', - }); - highlight_doc[DocData].citation_id = citation.citation_id; - Doc.AddDocToList(pdfDoc[DocData], annotationKey, highlight_doc); - highlight_doc.annotationOn = pdfDoc; - Doc.SetContainer(highlight_doc, pdfDoc); - return highlight_doc; - }; - - /** - * Lifecycle method that triggers when the component updates. - * Ensures the chat is scrolled to the bottom when new messages are added. - */ - componentDidUpdate() { - this.scrollToBottom(); - } - - /** - * Lifecycle method that triggers when the component mounts. - * Initializes scroll listeners, sets up document reactions, and loads chat history from dataDoc if available. - */ - componentDidMount() { - this._props.setContentViewBox?.(this); - if (this.dataDoc.data) { - try { - const storedHistory = JSON.parse(StrCast(this.dataDoc.data)); - runInAction(() => { - this.history.push( - ...storedHistory.map((msg: AssistantMessage) => ({ - role: msg.role, - content: msg.content, - follow_up_questions: msg.follow_up_questions, - citations: msg.citations, - })) - ); - }); - } catch (e) { - console.error('Failed to parse history from dataDoc:', e); - } - } else { - // Default welcome message - runInAction(() => { - this.history.push({ - role: ASSISTANT_ROLE.ASSISTANT, - content: [ - { - index: 0, - type: TEXT_TYPE.NORMAL, - text: `Hey, ${this.userName()}! Welcome to Your Friendly Assistant. Link a document or ask questions to get started.`, - citation_ids: null, - }, - ], - processing_info: [], - }); - }); - } - - // Set up reactions for linked documents - reaction( - () => { - const linkedDocs = LinkManager.Instance.getAllRelatedLinks(this.Document) - .map(d => DocCast(LinkManager.getOppositeAnchor(d, this.Document))) - .map(d => DocCast(d?.annotationOn, d)) - .filter(d => d); - return linkedDocs; - }, - linked => linked.forEach(doc => this.linked_docs_to_add.add(doc)) - ); - - // Observe changes to linked documents and handle document addition - observe(this.linked_docs_to_add, change => { - if (change.type === 'add') { - if (PDFCast(change.newValue.data)) { - this.addDocToVectorstore(change.newValue); - } else if (CsvCast(change.newValue.data)) { - this.addCSVForAnalysis(change.newValue); - } - } else if (change.type === 'delete') { - console.log('Deleted docs: ', change.oldValue); - } - }); - this.addScrollListener(); - } - - /** - * Lifecycle method that triggers when the component unmounts. - * Removes scroll listeners to avoid memory leaks. - */ - componentWillUnmount() { - this.removeScrollListener(); - } - - /** - * Getter that retrieves all linked documents for the current document. - */ - @computed - get linkedDocs() { - return LinkManager.Instance.getAllRelatedLinks(this.Document) - .map(d => DocCast(LinkManager.getOppositeAnchor(d, this.Document))) - .map(d => DocCast(d?.annotationOn, d)) - .filter(d => d); - } - - /** - * Getter that retrieves document IDs of linked documents that have AI-related content. - */ - @computed - get docIds() { - return LinkManager.Instance.getAllRelatedLinks(this.Document) - .map(d => DocCast(LinkManager.getOppositeAnchor(d, this.Document))) - .map(d => DocCast(d?.annotationOn, d)) - .filter(d => d) - .filter(d => d.ai_doc_id) - .map(d => StrCast(d.ai_doc_id)); - } - - /** - * Getter that retrieves summaries of all linked documents. - */ - @computed - get summaries(): string { - return ( - LinkManager.Instance.getAllRelatedLinks(this.Document) - .map(d => DocCast(LinkManager.getOppositeAnchor(d, this.Document))) - .map(d => DocCast(d?.annotationOn, d)) - .filter(d => d) - .filter(d => d.summary) - .map((doc, index) => { - if (PDFCast(doc.data)) { - return `${doc.summary}`; - } else if (CsvCast(doc.data)) { - return `${doc.summary}`; - } else { - return `${index + 1}) ${doc.summary}`; - } - }) - .join('\n') + '\n' - ); - } - - /** - * Getter that retrieves all linked CSV files for analysis. - */ - @computed - get linkedCSVs(): { filename: string; id: string; text: string }[] { - return this.linked_csv_files; - } - - /** - * Getter that formats the entire chat history as a string for the agent's system message. - */ - @computed - get formattedHistory(): string { - let history = '\n'; - for (const message of this.history) { - history += `<${message.role}>${message.content.map(content => content.text).join(' ')}`; - if (message.loop_summary) { - history += `${message.loop_summary}`; - } - history += `\n`; - } - history += ''; - return history; - } - - // Other helper methods for retrieving document data and processing - - retrieveSummaries = () => { - return this.summaries; - }; - - retrieveCSVData = () => { - return this.linkedCSVs; - }; - - retrieveFormattedHistory = () => { - return this.formattedHistory; - }; - - retrieveDocIds = () => { - return this.docIds; - }; - - /** - * Handles follow-up questions when the user clicks on them. - * Automatically sets the input value to the clicked follow-up question. - * @param question The follow-up question clicked by the user. - */ - @action - handleFollowUpClick = (question: string) => { - console.log('Follow-up question clicked:', question); - this.inputValue = question; - }; - - /** - * Renders the chat interface, including the message list, input field, and other UI elements. - */ - render() { - return ( -
- {this.isUploadingDocs && ( -
-
- -
{this.currentStep}
-
-
- )} -
-

{this.userName()}'s AI Assistant

-
-
- {this.history.map((message, index) => ( - - ))} - {this.current_message && ( - - )} -
-
- (this.inputValue = e.target.value)} /> - -
- {/* Popup for citation */} - {this.citationPopup.visible && ( -
-

- Text from your document: {this.citationPopup.text} -

-
- )} -
- ); - } -} - -/** - * Register the ChatBox component as the template for CHAT document types. - */ -Docs.Prototypes.TemplateMap.set(DocumentType.CHAT, { - layout: { view: ChatBox, dataField: 'data' }, - options: { acl: '', chat: '', chat_history: '', chat_thread_id: '', chat_assistant_id: '', chat_vector_store_id: '' }, -}); diff --git a/src/client/views/nodes/ChatBox/MessageComponent.tsx b/src/client/views/nodes/ChatBox/MessageComponent.tsx deleted file mode 100644 index 812e52ee0..000000000 --- a/src/client/views/nodes/ChatBox/MessageComponent.tsx +++ /dev/null @@ -1,105 +0,0 @@ -import React, { useState } from 'react'; -import { observer } from 'mobx-react'; -import { AssistantMessage, Citation, MessageContent, PROCESSING_TYPE, ProcessingInfo, TEXT_TYPE } from './types'; -import ReactMarkdown from 'react-markdown'; - -interface MessageComponentProps { - message: AssistantMessage; - index: number; - onFollowUpClick: (question: string) => void; - onCitationClick: (citation: Citation) => void; - updateMessageCitations: (index: number, citations: Citation[]) => void; -} - -const MessageComponentBox: React.FC = function ({ message, index, onFollowUpClick, onCitationClick, updateMessageCitations }) { - const [dropdownOpen, setDropdownOpen] = useState(false); - - const renderContent = (item: MessageContent) => { - const i = item.index; - //console.log('item', item, 'index', i); - if (item.type === TEXT_TYPE.GROUNDED) { - const citation_ids = item.citation_ids || []; - return ( - - {item.text} - {citation_ids.map((id, idx) => { - const citation = message.citations?.find(c => c.citation_id === id); - if (!citation) return null; - return ( - - ); - })} - - ); - } else if (item.type === TEXT_TYPE.NORMAL) { - return ( - - {item.text} - - ); - } else if ('query' in item) { - return ( - - {JSON.stringify(item.query)} - - ); - } else { - return ( - - {JSON.stringify(item)} - - ); - } - }; - - const hasProcessingInfo = message.processing_info && message.processing_info.length > 0; - - const renderProcessingInfo = (info: ProcessingInfo) => { - if (info.type === PROCESSING_TYPE.THOUGHT) { - return ( -
- Thought: {info.content} -
- ); - } else if (info.type === PROCESSING_TYPE.ACTION) { - return ( -
- Action: {info.content} -
- ); - } else { - return null; - } - }; - - return ( -
- {hasProcessingInfo && ( -
- - {dropdownOpen &&
{message.processing_info.map(renderProcessingInfo)}
} -
-
- )} -
{message.content && message.content.map(messageFragment => {renderContent(messageFragment)})}
- {message.follow_up_questions && message.follow_up_questions.length > 0 && ( -
-

Follow-up Questions:

-
- {message.follow_up_questions.map((question, idx) => ( - - ))} -
-
- )} -
- ); -}; - -export default observer(MessageComponentBox); diff --git a/src/client/views/nodes/ChatBox/ProgressBar.scss b/src/client/views/nodes/ChatBox/ProgressBar.scss deleted file mode 100644 index ff5be4a38..000000000 --- a/src/client/views/nodes/ChatBox/ProgressBar.scss +++ /dev/null @@ -1,69 +0,0 @@ -.spinner-container { - display: flex; - flex-direction: column; - justify-content: center; - align-items: center; - height: 100%; -} - -.spinner { - width: 60px; - height: 60px; - position: relative; - margin-bottom: 20px; // Space between spinner and text -} - -.double-bounce1, -.double-bounce2 { - width: 100%; - height: 100%; - border-radius: 50%; - background-color: #4a90e2; - opacity: 0.6; - position: absolute; - top: 0; - left: 0; - animation: bounce 2s infinite ease-in-out; -} - -.double-bounce2 { - animation-delay: -1s; -} - -@keyframes bounce { - 0%, - 100% { - transform: scale(0); - } - 50% { - transform: scale(1); - } -} - -.uploading-overlay { - position: absolute; - top: 0; - left: 0; - right: 0; - bottom: 0; - background-color: rgba(255, 255, 255, 0.8); - display: flex; - align-items: center; - justify-content: center; - z-index: 1000; -} - -.progress-container { - display: flex; - flex-direction: column; - align-items: center; - text-align: center; -} - -.step-name { - font-size: 18px; - color: #333; - text-align: center; - width: 100%; - margin-top: -10px; // Adjust to move the text closer to the spinner -} diff --git a/src/client/views/nodes/ChatBox/ProgressBar.tsx b/src/client/views/nodes/ChatBox/ProgressBar.tsx deleted file mode 100644 index 0aa07213f..000000000 --- a/src/client/views/nodes/ChatBox/ProgressBar.tsx +++ /dev/null @@ -1,13 +0,0 @@ -import React from 'react'; -import './ProgressBar.scss'; - -export const ProgressBar: React.FC = () => { - return ( -
-
-
-
-
-
- ); -}; diff --git a/src/client/views/nodes/ChatBox/prompts.ts b/src/client/views/nodes/ChatBox/prompts.ts deleted file mode 100644 index 0a356189b..000000000 --- a/src/client/views/nodes/ChatBox/prompts.ts +++ /dev/null @@ -1,465 +0,0 @@ -// prompts.ts - -import { Tool } from './types'; - -export function getReactPrompt(tools: Tool[], summaries: () => string, chatHistory: string): string { - const toolDescriptions: string = tools - .map( - tool => ` - - ${tool.name} - ${tool.briefSummary} - - ` - ) - .join('\n'); - - return ` - - You are an advanced AI assistant equipped with various tools to answer user queries accurately and efficiently. Your task is to provide a comprehensive response based on the user's prompt using available tools, chat history, and provided information. Follow these guidelines meticulously to ensure the accuracy and structure of your response. - - - - **MOST IMPORTANT**: Always output responses within stage number tags, using the stage number and the system role as the root tag (e.g., , , etc.). This is crucial and should never be overlooked. - **STOP after every stage and wait for the system to provide the next input (e.g., action rules or observations). - Only output **ONE stage at a time** in your responses. Do not skip stages or provide multiple stages at once. Thus, you should only output even stage number root tags. - Always structure your responses using valid, well-formed XML with properly nested tags. - If a tool is needed, ALWAYS select the most appropriate tool based on the user's query. - If the query could relate to user documents or require external information (e.g., RAG, search + website scraping, data analysis), USE the appropriate tool to gather that information. - If there are no user docs or the user docs have not yielded helpful information, use the search tool to find websites followed by the website scraper tool to get useful infromation from one of those websites. You can use the website scraper (or even the search tool), multiple times to find information from multiple websites either from the same search or different searches. - Ensure at the end of every final answer, you provide exactly three follow-up questions from the user's perspective—from the perspective that they are asking the question. - Always follow the response structure provided in the instructions. - If a tool doesn't work—or yield helpful results—after two tries, EITHER use another tool or proceed with the response and ask the user for more information or clarification or let them know you cannot answer their question and why. DO NOT CONTINUE WITH THE SAME TOOL 3 TIMES. - Use multiple tools in conjunction with each other to provide a comprehensive answer to the user's query, if needed (i.e. for the prompt "create a CSV showing historical bird migration trends", you could use the search tool and the webscraper tool to get the info, and then use the create CSV tool to create the CSV) - - - - - When providing your final response, use the following structure: - - - <grounded_text> - Wrap text that is derived from tool-based or chunk-based information within these tags, ensuring proper citation. - <normal_text> - Wrap text that is not derived from tool-based or chunk-based information within these tags. - - <citation> - Provide citations for each grounded text, referencing the tool or chunk used. - - - <question> - Include exactly three follow-up questions from the user's perspective within these tags. - - - <loop_summary> - Provide a summary of the actions and tools used by the assistant throughout the interaction within these tags. - - - - - - Wrap all information derived from tools (e.g., RAG, search + website scraping, data analysis)—which will be provided in chunks—in <grounded_text> tags. - DO NOT PUT ANYTHING THAT IS NOT DIRECTLY DERIVED FROM TOOLS OR CHUNKS IN <grounded_text> TAGS. - Use a single <grounded_text> tag for sequential and closely related information that references the same citation. - If other citations are used sequentially, create new <grounded_text> tags. - Ensure each <grounded_text> tag has corresponding citations (up to three, and one is fine). Separate multiple citation indices with commas. - Grounded text can be as short as a few words or as long as several sentences. - Avoid overlapping or nesting <grounded_text> tags; use sequential tags instead. - Grounded text tags should always have a citation_index attribute that references a citation index number that the text is grounded in. - Content within the <grounded_text> tags should be in Markdown format. - - - - Wrap all text that is not derived from tools or chunks in <normal_text> tags (any text outputted in the answer that is not in a <grounded_text> tag should be within a normal text tag). - Ensure that these tags are used for your reasoning, background knowledge, or general information that does not require a citation. - Do not use <normal_text> tags for information that needs grounding or citation. - Anything that is in any user docs should be grounded text and cited, not normal text, even if it is background or general information. - Content within the <normal_text> tags should be in Markdown format. - - - - Create a unique citation for each distinct piece of information from tools or chunks that is used to support <grounded_text>. - Ensure each citation has a unique index number. - Specify the correct type: "text", "image", "table", "csv", or "url". - For text-based information, include only the relevant subset of the original information that the <grounded_text> is based on. - For image, table, csv, or url citation types, leave the citation content empty. - ALL CITATIONS MUST use the chunk_id field to reference the source, whether it's from RAG, search + website scraping, data analysis, or any other tool. - One citation can be used for multiple <grounded_text> tags if they are based on the same tool or chunk information. - !!!DO NOT OVERCITE - only include citations for information that is directly relevant to the <grounded_text>. - - - - Analyze the user's query carefully. - Determine whether a tool is required to answer the query accurately. - If a tool is necessary: - - Select the most appropriate tool. - Use the <action> tag to specify the tool. - End your response after the <action> tag and wait for action rules to be provided. - Based on the action rules, provide the necessary tool parameters within the <inputs>. The <inputs> tag should be within the <action_input> tag, and should follow an <action_description> tag that contains a brief description of what you're doing with the action. - For each input, you may provide as many different iterations of the same tag (i.e. to provide multiple inputs to the tool) as is specified in the input's max_input's field. - End your response again and wait for the observation from the tool. - - If no tool is needed, use the 'no_tool' action but still follow the same response structure. - If the query might relate to user documents or requires external information, **ALWAYS** use the appropriate tool to retrieve the information (either rag or dataAnalysis). - Once all observations are collected, or if no tool was needed, provide your comprehensive answer within the <answer> tag, using the <grounded_text> and <normal_text> tags as required. - - - - Your final <answer> tag must contain: - - The complete answer to the user's query, with grounded information wrapped in <grounded_text> tags and general information wrapped in <normal_text> tags. - Exactly three follow-up questions written from the user's perspective, enclosed within <follow_up_questions> tags. - A concise <loop_summary> that describes the actions and tools used throughout the interaction. - - - - - ${toolDescriptions} - If no external tool is required to answer the question, use the 'no_tool' action. However, if the query might relate to user documents or require external information, do not use 'no_tool'—instead, use the appropriate tool (RAG, search + website scraping, data analysis), even if unsure. - - - - ENSURE THAT YOU ONLY USE TOOLS THAT ANALYZE OR OTHERWISE USE USER DOCS IF THE QUERY APPLIES TO ONE OF THESE USER SUMMARIES (AT LEAST SOMEWHAT). IF THERE ARE NO SUMMARIES, THERE ARE NO USER DOCUMENTS. - - ${summaries()} - - - - - These examples are not, by any means, exhaustive in terms of how tools can be used in conjunction with one another. They simply are to provide you with examples of how to structure your outputs and use some of the tools in some contexts. - YOU ONLY OUTPUT THE ASSISTANT STAGES: - - - ***SYSTEM MESSAGE ELIDED*** - - - Could you provide a detailed overview of the 2010 Vancouver Winter Olympics's impact, including the overall summary of the games, key moments from official statements, and how the medal count compared across countries? - - - - - Since the user has Olympics related docs, I will use the RAG tool to find relevant information from the user's documents, specifically focusing on key moments and statements from an official press release. - - rag - - - - ***Action rules elided*** - - - - - Searching user documents for official statements and key moments of the 2010 Vancouver Winter Olympics. - - - The user is asking for key moments and statements from official sources regarding the 2010 Vancouver Winter Olympics. Search the provided documents for any press releases or official statements that highlight significant events, achievements, or noteworthy aspects of the games. - - - - - - - - PRESS RELEASE: 2010 VANCOUVER WINTER OLYMPICS - - Vancouver, BC - - - The 2010 Winter Olympics, officially known as the XXI Olympic Winter Games, took place in Vancouver, British Columbia, Canada, from February 12 to 28, 2010. It featured 86 events in 15 disciplines across 7 sports, with 2,566 athletes from 82 National Olympic Committees participating. This edition of the Winter Olympics was notable for being the first hosted by Canada since the 1988 Winter Olympics in Calgary. - - Overall, the 2010 Winter Olympics have been hailed as a monumental success, showcasing the best of Canadian culture and athletic prowess. The Opening Ceremony, held on February 12, 2010, was a spectacular celebration of the nation's history and diversity... - - Key Moments: - - The Canadian men's and women's ice hockey teams both claimed gold medals, an achievement that brought immense pride to the host nation. - - Tragedy struck early in the games with the death of Georgian luger Nodar Kumaritashvili during a training run. This incident led to increased scrutiny of safety measures across all events... - - ***More chunk content elided for brevity*** - - The games concluded on February 28, 2010, with a memorable Closing Ceremony that highlighted the achievements of the athletes and the spirit of unity that defined the 2010 Vancouver Winter Olympics. - - - - - - With information on key moments from the press release, I will now analyze the medal count data and economic impact data to provide a comprehensive overview of the 2010 Vancouver Winter Olympics's impact. - - dataAnalysis - - - - ***Action rules elided*** - - - - - Analyzing the medal count data and economic impact data for the 2010 Vancouver Winter Olympics to compare country performances and assess the games' overall impact. - - 2010_Vancouver_Olympics_Medal_Count.csv - 2010_Vancouver_Olympics_Economic_Impact.csv - - - - - - - Country,Gold,Silver,Bronze,Total - USA,9,15,13,37 - Germany,10,13,7,30 - Canada,14,7,5,26 - Norway,9,8,6,23 - Russia,3,5,7,15 - South Korea,6,6,2,14 - China,5,2,4,11 - Sweden,5,2,4,11 - France,2,3,6,11 - Austria,4,6,6,16 - - - Category,Value - Total Cost (CAD),7.7 billion - Revenue Generated (CAD),2.5 billion - Jobs Created,45000 - Tourism Increase (%),35 - Infrastructure Investment (CAD),1.9 billion - Post-Olympic Annual Tourism Revenue Increase (CAD),220 million - Venues Repurposed,8 - Carbon Footprint Offset (%),118 - - - - - - Now that I have gathered comprehensive information from the official press release, medal count data, and economic impact data, I will provide a detailed answer to the user's query, incorporating all aspects of the 2010 Vancouver Winter Olympics's impact. - - - - The 2010 Winter Olympics, officially known as the XXI Olympic Winter Games, took place in Vancouver, British Columbia, Canada, from February 12 to 28, 2010. It featured 86 events in 15 disciplines across 7 sports, with 2,566 athletes from 82 National Olympic Committees participating. - - - - The Opening Ceremony was one of the most spectacular in Olympic history, celebrating Canadian culture and heritage. A significant highlight for the host nation was the victory of the Canadian men's and women's ice hockey teams, both securing gold medals, which was a source of immense national pride. - - - - In terms of medal performance, the United States led the overall tally with 37 medals (9 gold, 15 silver, 13 bronze), followed by Germany with 30 medals (10 gold, 13 silver, 7 bronze). Canada, while placing third in total medals with 26, achieved a record number of 14 gold medals, the highest of any country in the 2010 games. Other notable performances came from Norway (23 medals), Russia (15 medals), and South Korea (14 medals). - - - - The economic impact of the 2010 Vancouver Winter Olympics was significant. The total cost of hosting the games was approximately 7.7 billion CAD, while generating revenue of 2.5 billion CAD. The games created about 45,000 jobs and led to a 35% increase in tourism during the event. The long-term benefits include an annual increase of 220 million CAD in tourism revenue and 1.9 billion CAD invested in infrastructure. - - - - From an environmental perspective, the Vancouver Olympics made efforts to be sustainable, with 118% of the carbon footprint offset and 8 venues repurposed after the games, demonstrating a commitment to leaving a positive legacy. - - - - However, the games were also marked by tragedy with the unfortunate death of Georgian luger Nodar Kumaritashvili during a training run, which deeply affected the Olympic community and led to increased scrutiny of safety measures across all events. - - - - Despite this tragedy, the 2010 Vancouver Winter Olympics were generally hailed as a monumental success, showcasing the best of Canadian culture and athletic prowess. The games concluded on February 28, 2010, with a memorable Closing Ceremony that highlighted the achievements of the athletes and the spirit of unity that defined these Olympics. - - - - - - - - - - - - - - How did the 2010 Vancouver Winter Olympics compare to previous Winter Olympics in terms of economic impact and sustainability efforts? - What specific infrastructure improvements were made in Vancouver as a result of hosting the 2010 Winter Olympics? - How did the performance of Canadian athletes in the 2010 Vancouver Winter Olympics compare to their performance in previous Winter Olympics? - - - The assistant used the RAG tool to gather information from the user's documents, focusing on key moments and official statements. It then used the dataAnalysis tool to analyze the medal count and economic impact data, providing a comprehensive overview of the 2010 Vancouver Winter Olympics's impact. - - - - - - - ***SYSTEM MESSAGE ELIDED*** - - - What are the best hiking trails in Yosemite National Park? And which are the safest? - - - - To provide comprehensive information about the best hiking trails in Yosemite National Park, I'll use the search tool to find relevant websites with hiking recommendations, safety information, and general park information. - - searchTool - - - ***Action rules elided*** - - - - Searching the web for information about the best and safest hiking trails in Yosemite National Park. - - Best hiking trails Yosemite National Park - Safest hiking trails Yosemite National Park - Information about hiking in Yosemite National Park - - - - - - - https://www.nps.gov/yose/planyourvisit/hiking.htm - Official National Park Service website with hiking information for Yosemite - - - https://www.alltrails.com/parks/us/california/yosemite-national-park - AllTrails page featuring user-rated hiking trails in Yosemite - - - https://www.yosemitehikes.com/ - Comprehensive guide to Yosemite hikes with detailed trail information - - - - - - I've found several relevant websites about hiking in Yosemite National Park. I'll start by scraping content from the official National Park Service website for the most reliable information, then move on to AllTrails for user ratings and reviews, and finally to YosemiteHikes.com for more detailed trail descriptions. - - websiteInfoScraper - - - ***Action rules elided*** - - - - Scraping information from three websites about hiking trails in Yosemite National Park. - - https://www.nps.gov/yose/planyourvisit/hiking.htm - https://www.alltrails.com/parks/us/california/yosemite-national-park - https://www.yosemitehikes.com/ - - - - - - - ***NPS general planning website image elided*** - - - ***AllTrails website image elided*** - - - ***YosemiteHikes.com website image elided*** - - - - - - The information gathered from the three websites provides a good overview of the best hiking trails and safety considerations in Yosemite National Park. However, I noticed that there's limited information about seasonal trail conditions and closures, which could be crucial for planning a safe hike. To address this, I'll scrape additional information from the Yosemite National Park's current conditions page. - - websiteInfoScraper - - - ***Action rules elided*** - - - - Scraping information about current trail conditions and closures in Yosemite National Park. - - https://www.nps.gov/yose/planyourvisit/conditions.htm - - - - - - - ***NPS conditions website image elided*** - - - - - - Now that I have additional information about current trail conditions and seasonal closures, I can provide a more comprehensive and up-to-date answer to the user's query about the best and safest hiking trails in Yosemite National Park. - - - - Yosemite National Park offers a wide range of hiking trails suitable for various skill levels. Some of the best hiking trails in Yosemite, based on the information gathered, include: - - 1. Mist Trail to Vernal Fall: A moderate 3-mile round-trip hike in Yosemite Valley, offering close-up views of waterfalls... ***REST ELIDED FOR BREVITY*** - - - ***CONTENT ELIDED FOR BREVITY*** - - - - ***CONTENT ELIDED FOR BREVITY*** - - - Regarding safety, some of the safest trails for hikers of all skill levels include: - - - 1. Lower Yosemite Fall Trail: An easy 1-mile loop in Yosemite Valley... ***REST ELIDED FOR BREVITY*** - - - - ***CONTENT ELIDED FOR BREVITY*** - - - To ensure a safe hiking experience in Yosemite, regardless of the trail you choose, follow these safety tips: - - - 1. Stay on marked trails at all times... - ***REST ELIDED FOR BREVITY*** - - - Remember that even the "safest" trails can pose risks if proper precautions are not taken. Always hike within your ability level and be prepared for the specific conditions of the trail you choose. It's essential to check for daily updates on trail conditions before starting your hike, as they can change rapidly, especially during spring and fall. - - - - https://www.nps.gov/yose/planyourvisit/hiking.htm - - - https://www.alltrails.com/parks/us/california/yosemite-national-park - - - https://www.yosemitehikes.com/ - - - https://www.nps.gov/yose/planyourvisit/conditions.htm - - - - What is the best time of year to hike in Yosemite National Park? - Are there any special permits required for hiking certain trails in Yosemite? - What wildlife might I encounter while hiking in Yosemite, and how should I prepare for potential encounters? - - - The assistant used the search tool to find relevant websites, then scraped information from those sites using the websiteInfoScraper tool. Additional information about current trail conditions was gathered from the NPS website to provide a comprehensive and safe hiking guide. - - - - - - - - ${chatHistory} - Use this history for context when appropriate. - - - - Now, process the user's query and provide your response following the format and rules outlined above. Ensure your final answer is comprehensive, correctly cited, and entirely contained within the structured tags. Do not get stuck in infinite loops and keep responses concise, grounded, and most importantly, HELPFUL AND USEFUL! - - -`; -} - -export function getSummarizedChunksPrompt(chunks: string): string { - return `Please provide a comprehensive summary of what you think the document from which these chunks originated. - Ensure the summary captures the main ideas and key points from all provided chunks. Be concise and brief and only provide the summary in paragraph form. - - Text chunks: - \`\`\` - ${chunks} - \`\`\``; -} - -export function getSummarizedSystemPrompt(): string { - return 'You are an AI assistant tasked with summarizing a document. You are provided with important chunks from the document and provide a summary, as best you can, of what the document will contain overall. Be concise and brief with your response.'; -} diff --git a/src/client/views/nodes/ChatBox/response_parsers/AnswerParser.ts b/src/client/views/nodes/ChatBox/response_parsers/AnswerParser.ts deleted file mode 100644 index 79b53b0a3..000000000 --- a/src/client/views/nodes/ChatBox/response_parsers/AnswerParser.ts +++ /dev/null @@ -1,125 +0,0 @@ -import { ASSISTANT_ROLE, AssistantMessage, Citation, CHUNK_TYPE, TEXT_TYPE, getChunkType, ProcessingInfo } from '../types'; -import { v4 as uuid } from 'uuid'; - -export class AnswerParser { - static parse(xml: string, processingInfo: ProcessingInfo[]): AssistantMessage { - const answerRegex = /([\s\S]*?)<\/answer>/; - const citationsRegex = /([\s\S]*?)<\/citations>/; - const citationRegex = /([\s\S]*?)<\/citation>/g; - const followUpQuestionsRegex = /([\s\S]*?)<\/follow_up_questions>/; - const questionRegex = /(.*?)<\/question>/g; - const groundedTextRegex = /([\s\S]*?)<\/grounded_text>/g; - const normalTextRegex = /([\s\S]*?)<\/normal_text>/g; - const loopSummaryRegex = /([\s\S]*?)<\/loop_summary>/; - - const answerMatch = answerRegex.exec(xml); - const citationsMatch = citationsRegex.exec(xml); - const followUpQuestionsMatch = followUpQuestionsRegex.exec(xml); - const loopSummaryMatch = loopSummaryRegex.exec(xml); - - if (!answerMatch) { - throw new Error('Invalid XML: Missing tag.'); - } - - let rawTextContent = answerMatch[1].trim(); - let content: AssistantMessage['content'] = []; - let citations: Citation[] = []; - let contentIndex = 0; - - // Remove citations and follow-up questions from rawTextContent - if (citationsMatch) { - rawTextContent = rawTextContent.replace(citationsMatch[0], '').trim(); - } - if (followUpQuestionsMatch) { - rawTextContent = rawTextContent.replace(followUpQuestionsMatch[0], '').trim(); - } - if (loopSummaryMatch) { - rawTextContent = rawTextContent.replace(loopSummaryMatch[0], '').trim(); - } - - // Parse citations - let citationMatch; - const citationMap = new Map(); - if (citationsMatch) { - const citationsContent = citationsMatch[1]; - while ((citationMatch = citationRegex.exec(citationsContent)) !== null) { - const [_, index, chunk_id, type, direct_text] = citationMatch; - const citation_id = uuid(); - citationMap.set(index, citation_id); - citations.push({ - direct_text: direct_text.trim(), - type: getChunkType(type), - chunk_id, - citation_id, - }); - } - } - - rawTextContent = rawTextContent.replace(normalTextRegex, '$1'); - - // Parse text content (normal and grounded) - let lastIndex = 0; - let match; - - while ((match = groundedTextRegex.exec(rawTextContent)) !== null) { - const [fullMatch, citationIndex, groundedText] = match; - - // Add normal text that is before the grounded text - if (match.index > lastIndex) { - const normalText = rawTextContent.slice(lastIndex, match.index).trim(); - if (normalText) { - content.push({ - index: contentIndex++, - type: TEXT_TYPE.NORMAL, - text: normalText, - citation_ids: null, - }); - } - } - - // Add grounded text - const citation_ids = citationIndex.split(',').map(index => citationMap.get(index) || ''); - content.push({ - index: contentIndex++, - type: TEXT_TYPE.GROUNDED, - text: groundedText.trim(), - citation_ids, - }); - - lastIndex = match.index + fullMatch.length; - } - - // Add any remaining normal text after the last grounded text - if (lastIndex < rawTextContent.length) { - const remainingText = rawTextContent.slice(lastIndex).trim(); - if (remainingText) { - content.push({ - index: contentIndex++, - type: TEXT_TYPE.NORMAL, - text: remainingText, - citation_ids: null, - }); - } - } - - let followUpQuestions: string[] = []; - if (followUpQuestionsMatch) { - const questionsText = followUpQuestionsMatch[1]; - let questionMatch; - while ((questionMatch = questionRegex.exec(questionsText)) !== null) { - followUpQuestions.push(questionMatch[1].trim()); - } - } - - const assistantResponse: AssistantMessage = { - role: ASSISTANT_ROLE.ASSISTANT, - content, - follow_up_questions: followUpQuestions, - citations, - processing_info: processingInfo, - loop_summary: loopSummaryMatch ? loopSummaryMatch[1].trim() : undefined, - }; - - return assistantResponse; - } -} diff --git a/src/client/views/nodes/ChatBox/response_parsers/StreamedAnswerParser.ts b/src/client/views/nodes/ChatBox/response_parsers/StreamedAnswerParser.ts deleted file mode 100644 index 3585cab4a..000000000 --- a/src/client/views/nodes/ChatBox/response_parsers/StreamedAnswerParser.ts +++ /dev/null @@ -1,73 +0,0 @@ -import { threadId } from 'worker_threads'; - -enum ParserState { - Outside, - InGroundedText, - InNormalText, -} - -export class StreamedAnswerParser { - private state: ParserState = ParserState.Outside; - private buffer: string = ''; - private result: string = ''; - private isStartOfLine: boolean = true; - - public parse(char: string): string { - switch (this.state) { - case ParserState.Outside: - if (char === '<') { - this.buffer = '<'; - } else if (char === '>') { - if (this.buffer.startsWith('') { - this.state = ParserState.Outside; - this.buffer = ''; - } else if (this.buffer.startsWith('') { - this.state = ParserState.Outside; - this.buffer = ''; - } else if (this.buffer.startsWith('<')) { - this.buffer += char; - } else { - this.processChar(char); - } - break; - } - - return this.result.trim(); - } - - private processChar(char: string): void { - if (this.isStartOfLine && char === ' ') { - // Skip leading spaces - return; - } - if (char === '\n') { - this.result += char; - this.isStartOfLine = true; - } else { - this.result += char; - this.isStartOfLine = false; - } - } - - public reset(): void { - this.state = ParserState.Outside; - this.buffer = ''; - this.result = ''; - this.isStartOfLine = true; - } -} diff --git a/src/client/views/nodes/ChatBox/tools.ts b/src/client/views/nodes/ChatBox/tools.ts deleted file mode 100644 index 4035280a8..000000000 --- a/src/client/views/nodes/ChatBox/tools.ts +++ /dev/null @@ -1,26 +0,0 @@ -import { DocCast } from '../../../../fields/Types'; -import { DocServer } from '../../../DocServer'; -import { Docs } from '../../../documents/Documents'; -import { DocUtils } from '../../../documents/DocUtils'; -import { TabDocView } from '../../collections/TabDocView'; -import { DocumentView } from '../DocumentView'; -import { OpenWhere } from '../OpenWhere'; - -export function retrieval(json: any): string { - return ''; -} - -export function create_collection(docView: DocumentView, document_ids: string[], title: string): string { - const docs = document_ids.map(doc_id => DocCast(DocServer.GetCachedRefField(doc_id))); - const collection = Docs.Create.FreeformDocument(docs, { title }); - docView._props.addDocTab(collection, OpenWhere.addRight); //in future, create popup prompting user where to add - return 'Collection created in Dash called ' + title; -} - -export function create_link(docView: DocumentView, document_ids: string[]): string { - //Make document_ids a size 2 array - const docs = document_ids.map(doc_id => DocCast(DocServer.GetCachedRefField(doc_id))); - const linkDoc = DocUtils.MakeLink(docs[0], docs[1], {})!; - DocumentView.linkCommonAncestor(linkDoc)?.ComponentView?.addDocument?.(linkDoc); - return 'Link created between ' + docs[0].title + ' and ' + docs[1].title; -} diff --git a/src/client/views/nodes/ChatBox/tools/BaseTool.ts b/src/client/views/nodes/ChatBox/tools/BaseTool.ts deleted file mode 100644 index 2e2267653..000000000 --- a/src/client/views/nodes/ChatBox/tools/BaseTool.ts +++ /dev/null @@ -1,24 +0,0 @@ -import { Tool } from '../types'; - -export abstract class BaseTool = Record> implements Tool { - constructor( - public name: string, - public description: string, - public parameters: Record, - public citationRules: string, - public briefSummary: string - ) {} - - abstract execute(args: T): Promise; - - getActionRule(): Record { - return { - [this.name]: { - name: this.name, - citationRules: this.citationRules, - description: this.description, - parameters: this.parameters, - }, - }; - } -} diff --git a/src/client/views/nodes/ChatBox/tools/CalculateTool.ts b/src/client/views/nodes/ChatBox/tools/CalculateTool.ts deleted file mode 100644 index 74b7ca27b..000000000 --- a/src/client/views/nodes/ChatBox/tools/CalculateTool.ts +++ /dev/null @@ -1,26 +0,0 @@ -import { BaseTool } from './BaseTool'; - -export class CalculateTool extends BaseTool<{ expression: string }> { - constructor() { - super( - 'calculate', - 'Perform a calculation', - { - expression: { - type: 'string', - description: 'The mathematical expression to evaluate', - required: 'true', - max_inputs: '1', - }, - }, - 'Provide a mathematical expression to calculate that would work with JavaScript eval().', - 'Runs a calculation and returns the number - uses JavaScript so be sure to use floating point syntax if necessary' - ); - } - - async execute(args: { expression: string }): Promise { - // Note: Using eval() can be dangerous. Consider using a safer alternative. - const result = eval(args.expression); - return [{ type: 'text', text: result.toString() }]; - } -} diff --git a/src/client/views/nodes/ChatBox/tools/CreateCSVTool.ts b/src/client/views/nodes/ChatBox/tools/CreateCSVTool.ts deleted file mode 100644 index 55015846b..000000000 --- a/src/client/views/nodes/ChatBox/tools/CreateCSVTool.ts +++ /dev/null @@ -1,51 +0,0 @@ -import { BaseTool } from './BaseTool'; -import { Networking } from '../../../../Network'; - -export class CreateCSVTool extends BaseTool<{ csvData: string; filename: string }> { - private _handleCSVResult: (url: string, filename: string, id: string, data: string) => void; - - constructor(handleCSVResult: (url: string, title: string, id: string, data: string) => void) { - super( - 'createCSV', - 'Creates a CSV file from raw CSV data and saves it to the server', - { - type: 'object', - properties: { - csvData: { - type: 'string', - description: 'A string of comma-separated values representing the CSV data.', - }, - filename: { - type: 'string', - description: 'The base name of the CSV file to be created. Should end in ".csv".', - }, - }, - required: ['csvData', 'filename'], - }, - 'Provide a CSV string and a filename to create a CSV file.', - 'Creates a CSV file from the provided CSV string and saves it to the server with a unique identifier, returning the file URL and UUID.' - ); - this._handleCSVResult = handleCSVResult; - } - - async execute(args: { csvData: string; filename: string }): Promise { - try { - console.log('Creating CSV file:', args.filename, ' with data:', args.csvData); - // Post the raw CSV data to the createCSV endpoint on the server - const { fileUrl, id } = await Networking.PostToServer('/createCSV', { filename: args.filename, data: args.csvData }); - - // Handle the result by invoking the callback - this._handleCSVResult(fileUrl, args.filename, id, args.csvData); - - return [ - { - type: 'text', - text: `File successfully created: ${fileUrl}. \nNow a CSV file with this data and the name ${args.filename} is available as a user doc.`, - }, - ]; - } catch (error) { - console.error('Error creating CSV file:', error); - throw new Error('Failed to create CSV file.'); - } - } -} diff --git a/src/client/views/nodes/ChatBox/tools/CreateCollectionTool.ts b/src/client/views/nodes/ChatBox/tools/CreateCollectionTool.ts deleted file mode 100644 index 573428179..000000000 --- a/src/client/views/nodes/ChatBox/tools/CreateCollectionTool.ts +++ /dev/null @@ -1,36 +0,0 @@ -import { DocCast } from '../../../../../fields/Types'; -import { DocServer } from '../../../../DocServer'; -import { Docs } from '../../../../documents/Documents'; -import { DocumentView } from '../../DocumentView'; -import { OpenWhere } from '../../OpenWhere'; -import { BaseTool } from './BaseTool'; - -export class GetDocsContentTool extends BaseTool<{ title: string; document_ids: string[] }> { - private _docView: DocumentView; - constructor(docView: DocumentView) { - super( - 'retrieveDocs', - 'Retrieves the contents of all Documents that the user is interacting with in Dash ', - { - title: { - type: 'string', - description: 'the title of the collection that you will be making', - required: 'true', - max_inputs: '1', - }, - }, - 'Provide a mathematical expression to calculate that would work with JavaScript eval().', - 'Runs a calculation and returns the number - uses JavaScript so be sure to use floating point syntax if necessary' - ); - this._docView = docView; - } - - async execute(args: { title: string; document_ids: string[] }): Promise { - // Note: Using eval() can be dangerous. Consider using a safer alternative. - const docs = args.document_ids.map(doc_id => DocCast(DocServer.GetCachedRefField(doc_id))); - const collection = Docs.Create.FreeformDocument(docs, { title: args.title }); - this._docView._props.addDocTab(collection, OpenWhere.addRight); //in future, create popup prompting user where to add - return [{ type: 'text', text: 'Collection created in Dash called ' + args.title }]; - } -} -//export function create_collection(docView: DocumentView, document_ids: string[], title: string): string {} diff --git a/src/client/views/nodes/ChatBox/tools/DataAnalysisTool.ts b/src/client/views/nodes/ChatBox/tools/DataAnalysisTool.ts deleted file mode 100644 index a12ee46e5..000000000 --- a/src/client/views/nodes/ChatBox/tools/DataAnalysisTool.ts +++ /dev/null @@ -1,59 +0,0 @@ -import { BaseTool } from './BaseTool'; - -export class DataAnalysisTool extends BaseTool<{ csv_file_name: string | string[] }> { - private csv_files_function: () => { filename: string; id: string; text: string }[]; - - constructor(csv_files: () => { filename: string; id: string; text: string }[]) { - super( - 'dataAnalysis', - 'Analyzes, and provides insights, from one or more CSV files', - { - csv_file_name: { - type: 'string', - description: 'Name(s) of the CSV file(s) to analyze', - required: 'true', - max_inputs: '3', - }, - }, - 'Provide the name(s) of up to 3 CSV files to analyze based on the user query and whichever available CSV files may be relevant.', - 'Provides the full CSV file text for your analysis based on the user query and the available CSV file(s). ' - ); - this.csv_files_function = csv_files; - } - - getFileContent(filename: string): string | undefined { - const files = this.csv_files_function(); - const file = files.find(f => f.filename === filename); - return file?.text; - } - - getFileID(filename: string): string | undefined { - const files = this.csv_files_function(); - const file = files.find(f => f.filename === filename); - return file?.id; - } - - async execute(args: { csv_file_name: string | string[] }): Promise { - const filenames = Array.isArray(args.csv_file_name) ? args.csv_file_name : [args.csv_file_name]; - const results = []; - - for (const filename of filenames) { - const fileContent = this.getFileContent(filename); - const fileID = this.getFileID(filename); - - if (fileContent && fileID) { - results.push({ - type: 'text', - text: `${fileContent}`, - }); - } else { - results.push({ - type: 'text', - text: `File not found: ${filename}`, - }); - } - } - - return results; - } -} diff --git a/src/client/views/nodes/ChatBox/tools/GetDocsTool.ts b/src/client/views/nodes/ChatBox/tools/GetDocsTool.ts deleted file mode 100644 index f970ca8ee..000000000 --- a/src/client/views/nodes/ChatBox/tools/GetDocsTool.ts +++ /dev/null @@ -1,29 +0,0 @@ -import { DocCast } from '../../../../../fields/Types'; -import { DocServer } from '../../../../DocServer'; -import { Docs } from '../../../../documents/Documents'; -import { DocumentView } from '../../DocumentView'; -import { OpenWhere } from '../../OpenWhere'; -import { BaseTool } from './BaseTool'; - -export class GetDocsTool extends BaseTool<{ title: string; document_ids: string[] }> { - private _docView: DocumentView; - constructor(docView: DocumentView) { - super( - 'retrieveDocs', - 'Retrieves the contents of all Documents that the user is interacting with in Dash', - {}, - 'No need to provide anything. Just run the tool and it will retrieve the contents of all Documents that the user is interacting with in Dash.', - 'Returns the the documents in Dash in JSON form. This will include the title of the document, the location in the FreeFormDocument, and the content of the document, any applicable data fields, the layout of the document, etc.' - ); - this._docView = docView; - } - - async execute(args: { title: string; document_ids: string[] }): Promise { - // Note: Using eval() can be dangerous. Consider using a safer alternative. - const docs = args.document_ids.map(doc_id => DocCast(DocServer.GetCachedRefField(doc_id))); - const collection = Docs.Create.FreeformDocument(docs, { title: args.title }); - this._docView._props.addDocTab(collection, OpenWhere.addRight); //in future, create popup prompting user where to add - return [{ type: 'text', text: 'Collection created in Dash called ' + args.title }]; - } -} -//export function create_collection(docView: DocumentView, document_ids: string[], title: string): string {} diff --git a/src/client/views/nodes/ChatBox/tools/NoTool.ts b/src/client/views/nodes/ChatBox/tools/NoTool.ts deleted file mode 100644 index 1f0830a77..000000000 --- a/src/client/views/nodes/ChatBox/tools/NoTool.ts +++ /dev/null @@ -1,18 +0,0 @@ -// tools/NoTool.ts -import { BaseTool } from './BaseTool'; - -export class NoTool extends BaseTool<{}> { - constructor() { - super( - 'no_tool', - 'Use this when no external tool or action is required to answer the question.', - {}, - 'When using the "no_tool" action, simply provide an empty element. The observation will always be "No tool used. Proceed with answering the question."', - 'Use when no external tool or action is required to answer the question.' - ); - } - - async execute(args: {}): Promise { - return [{ type: 'text', text: 'No tool used. Proceed with answering the question.' }]; - } -} diff --git a/src/client/views/nodes/ChatBox/tools/RAGTool.ts b/src/client/views/nodes/ChatBox/tools/RAGTool.ts deleted file mode 100644 index 544b9daba..000000000 --- a/src/client/views/nodes/ChatBox/tools/RAGTool.ts +++ /dev/null @@ -1,138 +0,0 @@ -import { BaseTool } from './BaseTool'; -import { Vectorstore } from '../vectorstore/Vectorstore'; -import { RAGChunk } from '../types'; -import * as fs from 'fs'; -import { Networking } from '../../../../Network'; -import { file } from 'jszip'; -import { ChatCompletion, ChatCompletionContentPart, ChatCompletionMessageParam } from 'openai/resources'; - -export class RAGTool extends BaseTool { - constructor(private vectorstore: Vectorstore) { - super( - 'rag', - 'Perform a RAG search on user documents', - { - hypothetical_document_chunk: { - type: 'string', - description: - "Detailed version of the prompt that is effectively a hypothetical document chunk that would be ideal to embed and compare to the vectors of real document chunks to fetch the most relevant document chunks to answer the user's query", - required: 'true', - }, - }, - ` - Your task is to provide a comprehensive response to the user's prompt based on the given chunks and chat history. Follow these structural guidelines meticulously: - - 1. Overall Structure: - - [Main content with grounded_text tags interspersed with normal plain text (information that is not derived from chunks' information)] - - [Individual citation tags] - - - [Three question tags] - - - - 2. Grounded Text Tag Structure: - - Basic format: - - [Your generated text based on information from a subset of a chunk (a citation's direct text)] - - - 3. Citation Tag Structure: - - [For text: relevant subset of original chunk] - [For image/table: leave empty] - - - 4. Detailed Grounded Text Guidelines: - a. Wrap all information derived from chunks in grounded_text tags. - b. DO NOT PUT ANYTHING THAT IS NOT DIRECTLY DERIVED FROM INFORMATION FROM CHUNKS (EITHER IMAGE, TABLE, OR TEXT) IN GROUNDED_TEXT TAGS. - c. Use a single grounded_text tag for suquential and closely related information that references the same citation. If other citations' information are used sequentially, create new grounded_text tags. - d. Ensure every grounded_text tag has up to a few corresponding citations (should not be more than 3 and only 1 is fine). Multiple citation indices should be separated by commas. - e. Grounded text can be as short as a few words or as long as several sentences. - f. Avoid overlapping or nesting grounded_text tags; instead, use sequential tags. - - 5. Detailed Citation Guidelines: - a. Create a unique citation for each distinct piece of information from the chunks that is used to support grounded_text. - b. ALL TEXT CITATIONS must have direct text in its element content (e.g. DIRECT TEXT HERE) that is a relevant SUBSET of the original text chunk that is being cited specifically. - c. DO NOT paraphrase or summarize the text; use the original text as much as possible. - d. DO NOT USE THE FULL TEXT CHUNK as the citation content; only use the relevant subset of the text that the grounded_text is base. AS SHORT AS POSSIBLE WHILE PROVIDING INFORMATION (ONE TO TWO SENTENCES USUALLY)! - e. Ensure each citation has a unique index number. - f. Specify the correct type: "text", "image", or "table". - g. For text chunks, the content of the citation should ALWAYS have the relevant subset of the original text that the grounded_text is based on. - h. For image/table chunks, leave the citation content empty. - i. One citation can be used for multiple grounded_text tags if they are based on the same chunk information. - j. !!!DO NOT OVERCITE - only include citations for information that is directly relevant to the grounded_text. - - 6. Structural Integrity Checks: - a. Ensure all opening tags have corresponding closing tags. - b. Verify that all grounded_text tags have valid citation_index attributes (they should be equal to the associated citation(s) index field—not their chunk_id field). - c. Check that all cited indices in grounded_text tags have corresponding citations. - - Example of grounded_text usage: - - - - Artificial Intelligence (AI) is revolutionizing various sectors, with healthcare experiencing significant transformations in areas such as diagnosis and treatment planning. - - - In the field of medical diagnosis, AI has shown remarkable capabilities, particularly in radiology. For instance, AI systems have drastically improved mammogram analysis, achieving 99% accuracy at a rate 30 times faster than human radiologists. - - - This advancement not only enhances the efficiency of healthcare systems but also significantly reduces the occurrence of false positives, leading to fewer unnecessary biopsies and reduced patient stress. - - - - Beyond diagnosis, AI is playing a crucial role in drug discovery and development. By analyzing vast amounts of genetic and molecular data, AI algorithms can identify potential drug candidates much faster than traditional methods. - - - This could potentially reduce the time and cost of bringing new medications to market, especially for rare diseases that have historically received less attention due to limited market potential. - - - [... rest of the content ...] - - - Artificial Intelligence is revolutionizing various industries, with healthcare being one of the most profoundly affected sectors. - AI has shown particular promise in the field of radiology, enhancing the accuracy and speed of image analysis. - According to recent studies, AI systems have achieved 99% accuracy in mammogram analysis, performing the task 30 times faster than human radiologists. - The improvement in mammogram accuracy has led to a significant reduction in false positives, decreasing the need for unnecessary biopsies and reducing patient anxiety. - AI is accelerating the drug discovery process by analyzing complex molecular and genetic data to identify potential drug candidates. - The use of AI in drug discovery could significantly reduce the time and cost associated with bringing new medications to market, particularly for rare diseases. - - - - How might AI-driven personalized medicine impact the cost and accessibility of healthcare in the future? - What measures can be taken to ensure that AI systems in healthcare are free from biases and equally effective for diverse populations? - How could the role of healthcare professionals evolve as AI becomes more integrated into medical practices? - - - `, - - `Performs a RAG (Retrieval-Augmented Generation) search on user documents and returns a - set of document chunks (either images or text) that can be used to provide a grounded response based on - user documents` - ); - } - - async execute(args: { hypothetical_document_chunk: string }): Promise { - const relevantChunks = await this.vectorstore.retrieve(args.hypothetical_document_chunk); - const formatted_chunks = await this.getFormattedChunks(relevantChunks); - return formatted_chunks; - } - - async getFormattedChunks(relevantChunks: RAGChunk[]): Promise { - try { - const { formattedChunks } = await Networking.PostToServer('/formatChunks', { relevantChunks }); - - if (!formattedChunks) { - throw new Error('Failed to format chunks'); - } - - return formattedChunks; - } catch (error) { - console.error('Error formatting chunks:', error); - throw error; - } - } -} diff --git a/src/client/views/nodes/ChatBox/tools/SearchTool.ts b/src/client/views/nodes/ChatBox/tools/SearchTool.ts deleted file mode 100644 index b926cbadc..000000000 --- a/src/client/views/nodes/ChatBox/tools/SearchTool.ts +++ /dev/null @@ -1,54 +0,0 @@ -import { max } from 'lodash'; -import { Networking } from '../../../../Network'; -import { BaseTool } from './BaseTool'; -import { v4 as uuidv4 } from 'uuid'; - -export class SearchTool extends BaseTool<{ query: string | string[] }> { - private _addLinkedUrlDoc: (url: string, id: string) => void; - private _max_results: number; - constructor(addLinkedUrlDoc: (url: string, id: string) => void, max_results: number = 5) { - super( - 'searchTool', - 'Search the web to find a wide range of websites related to a query or multiple queries', - { - query: { - type: 'string', - description: 'The search query or queries to use for finding websites', - required: 'true', - max_inputs: '3', - }, - }, - 'Provide up to 3 search queries to find a broad range of websites. This tool is intended to help you identify relevant websites, but not to be used for providing the final answer. Use this information to determine which specific website to investigate further.', - 'Returns a list of websites and their overviews based on the search queries, helping to identify which websites might contain relevant information.' - ); - this._addLinkedUrlDoc = addLinkedUrlDoc; - this._max_results = max_results; - } - - async execute(args: { query: string | string[] }): Promise { - const queries = Array.isArray(args.query) ? args.query : [args.query]; - const allResults = []; - - for (const query of queries) { - try { - const { results } = await Networking.PostToServer('/getWebSearchResults', { query, max_results: this._max_results }); - const data: { type: string; text: string }[] = results.map((result: { url: string; snippet: string }) => { - const id = uuidv4(); - return { - type: 'text', - text: ` - ${result.url} - ${result.snippet} - `, - }; - }); - allResults.push(...data); - } catch (error) { - console.log(error); - allResults.push({ type: 'text', text: `An error occurred while performing the web search for query: ${query}` }); - } - } - - return allResults; - } -} diff --git a/src/client/views/nodes/ChatBox/tools/WebsiteInfoScraperTool.ts b/src/client/views/nodes/ChatBox/tools/WebsiteInfoScraperTool.ts deleted file mode 100644 index 4588b5aec..000000000 --- a/src/client/views/nodes/ChatBox/tools/WebsiteInfoScraperTool.ts +++ /dev/null @@ -1,43 +0,0 @@ -import { Networking } from '../../../../Network'; -import { BaseTool } from './BaseTool'; -import { v4 as uuidv4 } from 'uuid'; - -export class WebsiteInfoScraperTool extends BaseTool<{ url: string | string[] }> { - private _addLinkedUrlDoc: (url: string, id: string) => void; - - constructor(addLinkedUrlDoc: (url: string, id: string) => void) { - super( - 'websiteInfoScraper', - 'Scrape detailed information from specific websites identified as relevant to the user query', - { - url: { - type: 'string', - description: 'The URL(s) of the website(s) to scrape', - required: 'true', - max_inputs: '3', - }, - }, - 'Provide up to 3 URLs of websites that you have identified as the most relevant from the previous search. This tool will provide the text content of those specific websites. When providing a final response to the user based on information from these chunks, ideally cite as many of the url chunks as possible (ground your infromation from multiple sources, if possible) in order to provide a well grounded result.', - 'Returns the text content of the webpages for analysis.' - ); - this._addLinkedUrlDoc = addLinkedUrlDoc; - } - - async execute(args: { url: string | string[] }): Promise { - const urls = Array.isArray(args.url) ? args.url : [args.url]; - const results = []; - - for (const url of urls) { - try { - const { website_plain_text } = await Networking.PostToServer('/scrapeWebsite', { url }); - const id = uuidv4(); - this._addLinkedUrlDoc(url, id); - results.push({ type: 'text', text: `\n${website_plain_text}\n\n` }); - } catch (error) { - results.push({ type: 'text', text: `An error occurred while scraping the website: ${url}` }); - } - } - - return results; - } -} diff --git a/src/client/views/nodes/ChatBox/tools/WikipediaTool.ts b/src/client/views/nodes/ChatBox/tools/WikipediaTool.ts deleted file mode 100644 index 143d91d80..000000000 --- a/src/client/views/nodes/ChatBox/tools/WikipediaTool.ts +++ /dev/null @@ -1,37 +0,0 @@ -import { title } from 'process'; -import { Networking } from '../../../../Network'; -import { BaseTool } from './BaseTool'; -import axios from 'axios'; -import { v4 as uuidv4 } from 'uuid'; - -export class WikipediaTool extends BaseTool<{ title: string }> { - private _addLinkedUrlDoc: (url: string, id: string) => void; - constructor(addLinkedUrlDoc: (url: string, id: string) => void) { - super( - 'wikipedia', - 'Search Wikipedia and return a summary', - { - title: { - type: 'string', - description: 'The title of the Wikipedia article to search', - required: true, - }, - }, - 'Provide simply the title you want to search on Wikipedia and nothing more. If re-using this tool, try a different title for different information.', - 'Returns a summary from searching an article title on Wikipedia' - ); - this._addLinkedUrlDoc = addLinkedUrlDoc; - } - - async execute(args: { title: string }): Promise { - try { - const { text } = await Networking.PostToServer('/getWikipediaSummary', { title: args.title }); - const id = uuidv4(); - const url = `https://en.wikipedia.org/wiki/${args.title.replace(/ /g, '_')}`; - this._addLinkedUrlDoc(url, id); - return [{ type: 'text', text: ` ${text} ` }]; - } catch (error) { - return [{ type: 'text', text: 'An error occurred while fetching the article.' }]; - } - } -} diff --git a/src/client/views/nodes/ChatBox/types.ts b/src/client/views/nodes/ChatBox/types.ts deleted file mode 100644 index a12c52592..000000000 --- a/src/client/views/nodes/ChatBox/types.ts +++ /dev/null @@ -1,169 +0,0 @@ -import { breadcrumbsClasses } from '@mui/material'; -import { Doc } from '../../../../fields/Doc'; -import { StrCast } from '../../../../fields/Types'; -import e from 'cors'; -import { index } from 'd3'; - -export enum ASSISTANT_ROLE { - USER = 'user', - ASSISTANT = 'assistant', -} - -export enum TEXT_TYPE { - NORMAL = 'normal', - GROUNDED = 'grounded', - ERROR = 'error', -} - -export enum CHUNK_TYPE { - TEXT = 'text', - IMAGE = 'image', - TABLE = 'table', - URL = 'url', - CSV = 'CSV', -} - -export enum PROCESSING_TYPE { - THOUGHT = 'thought', - ACTION = 'action', - //eventually migrate error to here -} - -export function getChunkType(type: string): CHUNK_TYPE { - switch (type.toLowerCase()) { - case 'text': - return CHUNK_TYPE.TEXT; - break; - case 'image': - return CHUNK_TYPE.IMAGE; - break; - case 'table': - return CHUNK_TYPE.TABLE; - break; - case 'CSV': - return CHUNK_TYPE.CSV; - break; - case 'url': - return CHUNK_TYPE.URL; - break; - default: - return CHUNK_TYPE.TEXT; - break; - } -} - -export interface ProcessingInfo { - index: number; - type: PROCESSING_TYPE; - content: string; -} - -export interface AssistantMessage { - role: ASSISTANT_ROLE; - content: MessageContent[]; - follow_up_questions?: string[]; - citations?: Citation[]; - processing_info: ProcessingInfo[]; - loop_summary?: string; -} - -export interface MessageContent { - index: number; - type: TEXT_TYPE; - text: string; - citation_ids: string[] | null; -} - -export interface Citation { - direct_text?: string; - type: CHUNK_TYPE; - chunk_id: string; - citation_id: string; - url?: string; -} - -export interface RAGChunk { - id: string; - values: number[]; - metadata: { - text: string; - type: CHUNK_TYPE; - original_document: string; - file_path: string; - doc_id: string; - location: string; - start_page: number; - end_page: number; - base64_data?: string | undefined; - page_width?: number | undefined; - page_height?: number | undefined; - }; -} - -export interface SimplifiedChunk { - chunkId: string; - startPage: number; - endPage: number; - location?: string; - chunkType: CHUNK_TYPE; - url?: string; - canDisplay?: boolean; -} - -export interface AI_Document { - purpose: string; - file_name: string; - num_pages: number; - summary: string; - chunks: RAGChunk[]; - type: string; -} - -export interface Tool = Record> { - name: string; - description: string; - parameters: Record; - citationRules: string; - briefSummary: string; - execute: (args: T) => Promise; - getActionRule: () => Record; -} - -export interface AgentMessage { - role: 'system' | 'user' | 'assistant'; - content: string | { type: string; text?: string; image_url?: { url: string } }[]; -} - -// export function convertToAIDocument(json: any): AI_Document { -// if (!json) { -// throw new Error('Invalid JSON object'); -// } - -// const chunks: Chunk[] = json.chunks.map((chunk: any) => ({ -// id: chunk.id, -// values: chunk.values, -// metadata: { -// text: chunk.metadata.text, -// type: chunk.metadata.type as CHUNK_TYPE, // Ensure type casting -// original_document: chunk.metadata.original_document, -// file_path: chunk.metadata.file_path, -// location: chunk.metadata.location, -// start_page: chunk.metadata.start_page, -// end_page: chunk.metadata.end_page, -// base64_data: chunk.metadata.base64_data, -// width: chunk.metadata.width, -// height: chunk.metadata.height, -// }, -// })); - -// const aiDocument: AI_Document = { -// purpose: json.purpose, -// file_name: json.file_name, -// num_pages: json.num_pages, -// summary: json.summary, -// chunks: chunks, -// type: json.type, -// }; - -// return aiDocument; -// } diff --git a/src/client/views/nodes/ChatBox/vectorstore/Vectorstore.ts b/src/client/views/nodes/ChatBox/vectorstore/Vectorstore.ts deleted file mode 100644 index cc3b1ccd5..000000000 --- a/src/client/views/nodes/ChatBox/vectorstore/Vectorstore.ts +++ /dev/null @@ -1,258 +0,0 @@ -import { Pinecone, Index, IndexList, PineconeRecord, RecordMetadata, QueryResponse } from '@pinecone-database/pinecone'; -import { CohereClient } from 'cohere-ai'; -import { EmbedResponse } from 'cohere-ai/api'; -import dotenv from 'dotenv'; -import { RAGChunk, AI_Document, CHUNK_TYPE } from '../types'; -import { Doc } from '../../../../../fields/Doc'; -import { CsvCast, PDFCast, StrCast } from '../../../../../fields/Types'; -import { Networking } from '../../../../Network'; - -dotenv.config(); - -/** - * The Vectorstore class integrates with Pinecone for vector-based document indexing and retrieval, - * and Cohere for text embedding. It handles AI document management, uploads, and query-based retrieval. - */ -export class Vectorstore { - private pinecone: Pinecone; // Pinecone client for managing the vector index. - private index!: Index; // The specific Pinecone index used for document chunks. - private cohere: CohereClient; // Cohere client for generating embeddings. - private indexName: string = 'pdf-chatbot'; // Default name for the index. - private _id: string; // Unique ID for the Vectorstore instance. - private _doc_ids: string[] = []; // List of document IDs handled by this instance. - - documents: AI_Document[] = []; // Store the documents indexed in the vectorstore. - - /** - * Constructor initializes the Pinecone and Cohere clients, sets up the document ID list, - * and initializes the Pinecone index. - * @param id The unique identifier for the vectorstore instance. - * @param doc_ids A function that returns a list of document IDs. - */ - constructor(id: string, doc_ids: () => string[]) { - const pineconeApiKey = process.env.PINECONE_API_KEY; - if (!pineconeApiKey) { - throw new Error('PINECONE_API_KEY is not defined.'); - } - - // Initialize Pinecone and Cohere clients with API keys from the environment. - this.pinecone = new Pinecone({ apiKey: pineconeApiKey }); - this.cohere = new CohereClient({ token: process.env.COHERE_API_KEY }); - this._id = id; - this._doc_ids = doc_ids(); - this.initializeIndex(); - } - - /** - * Initializes the Pinecone index by checking if it exists, and creating it if not. - * The index is set to use the cosine metric for vector similarity. - */ - private async initializeIndex() { - const indexList: IndexList = await this.pinecone.listIndexes(); - - // Check if the index already exists, otherwise create it. - if (!indexList.indexes?.some(index => index.name === this.indexName)) { - await this.pinecone.createIndex({ - name: this.indexName, - dimension: 1024, - metric: 'cosine', - spec: { - serverless: { - cloud: 'aws', - region: 'us-east-1', - }, - }, - }); - } - - // Set the index for future use. - this.index = this.pinecone.Index(this.indexName); - } - - /** - * Adds an AI document to the vectorstore. This method handles document chunking, uploading to the - * vectorstore, and updating the progress for long-running tasks like file uploads. - * @param doc The document to be added to the vectorstore. - * @param progressCallback Callback to update the progress of the upload. - */ - async addAIDoc(doc: Doc, progressCallback: (progress: number, step: string) => void) { - console.log('Adding AI Document:', doc); - const ai_document_status: string = StrCast(doc.ai_document_status); - - // Skip if the document is already in progress or completed. - if (ai_document_status !== undefined && ai_document_status.trim() !== '' && ai_document_status !== '{}') { - if (ai_document_status === 'IN PROGRESS') { - console.log('Already in progress.'); - return; - } - if (!this._doc_ids.includes(StrCast(doc.ai_doc_id))) { - this._doc_ids.push(StrCast(doc.ai_doc_id)); - } - } else { - // Start processing the document. - doc.ai_document_status = 'PROGRESS'; - console.log(doc); - - // Get the local file path (CSV or PDF). - const local_file_path: string = CsvCast(doc.data)?.url?.pathname ?? PDFCast(doc.data)?.url?.pathname; - console.log('Local File Path:', local_file_path); - - if (local_file_path) { - console.log('Creating AI Document...'); - // Start the document creation process by sending the file to the server. - const { jobId } = await Networking.PostToServer('/createDocument', { file_path: local_file_path }); - - // Poll the server for progress updates. - let inProgress: boolean = true; - let result: any = null; - while (inProgress) { - // Polling interval for status updates. - await new Promise(resolve => setTimeout(resolve, 2000)); - - // Check if the job is completed. - const resultResponse = await Networking.FetchFromServer(`/getResult/${jobId}`); - const resultResponseJson = JSON.parse(resultResponse); - if (resultResponseJson.status === 'completed') { - console.log('Result here:', resultResponseJson); - result = resultResponseJson; - break; - } - - // Fetch progress information and update the progress callback. - const progressResponse = await Networking.FetchFromServer(`/getProgress/${jobId}`); - const progressResponseJson = JSON.parse(progressResponse); - if (progressResponseJson) { - const progress = progressResponseJson.progress; - const step = progressResponseJson.step; - progressCallback(progress, step); - } - } - - // Once completed, process the document and add it to the vectorstore. - console.log('Document JSON:', result); - this.documents.push(result); - await this.indexDocument(result); - console.log(`Document added: ${result.file_name}`); - - // Update document metadata such as summary, purpose, and vectorstore ID. - doc.summary = result.summary; - doc.ai_doc_id = result.doc_id; - this._doc_ids.push(result.doc_id); - doc.ai_purpose = result.purpose; - - if (!doc.vectorstore_id) { - doc.vectorstore_id = JSON.stringify([this._id]); - } else { - doc.vectorstore_id = JSON.stringify(JSON.parse(StrCast(doc.vectorstore_id)).concat([this._id])); - } - - if (!doc.chunk_simpl) { - doc.chunk_simpl = JSON.stringify({ chunks: [] }); - } - - // Process each chunk of the document and update the document's chunk_simpl field. - result.chunks.forEach((chunk: RAGChunk) => { - const chunkToAdd = { - chunkId: chunk.id, - startPage: chunk.metadata.start_page, - endPage: chunk.metadata.end_page, - location: chunk.metadata.location, - chunkType: chunk.metadata.type as CHUNK_TYPE, - text: chunk.metadata.text, - }; - const new_chunk_simpl = JSON.parse(StrCast(doc.chunk_simpl)); - new_chunk_simpl.chunks = new_chunk_simpl.chunks.concat(chunkToAdd); - doc.chunk_simpl = JSON.stringify(new_chunk_simpl); - }); - - // Mark the document status as completed. - doc.ai_document_status = 'COMPLETED'; - } - } - } - - /** - * Indexes the processed document by uploading the document's vector chunks to the Pinecone index. - * @param document The processed document containing its chunks and metadata. - */ - private async indexDocument(document: any) { - console.log('Uploading vectors to content namespace...'); - - // Prepare Pinecone records for each chunk in the document. - const pineconeRecords: PineconeRecord[] = (document.chunks as RAGChunk[]).map(chunk => ({ - id: chunk.id, - values: chunk.values, - metadata: { ...chunk.metadata } as RecordMetadata, - })); - - // Upload the records to Pinecone. - await this.index.upsert(pineconeRecords); - } - - /** - * Retrieves the top K document chunks relevant to the user's query. - * This involves embedding the query using Cohere, then querying Pinecone for matching vectors. - * @param query The search query string. - * @param topK The number of top results to return (default is 10). - * @returns A list of document chunks that match the query. - */ - async retrieve(query: string, topK: number = 10): Promise { - console.log(`Retrieving chunks for query: ${query}`); - try { - // Generate an embedding for the query using Cohere. - const queryEmbeddingResponse: EmbedResponse = await this.cohere.embed({ - texts: [query], - model: 'embed-english-v3.0', - inputType: 'search_query', - }); - - let queryEmbedding: number[]; - - // Extract the embedding from the response. - if (Array.isArray(queryEmbeddingResponse.embeddings)) { - queryEmbedding = queryEmbeddingResponse.embeddings[0]; - } else if (queryEmbeddingResponse.embeddings && 'embeddings' in queryEmbeddingResponse.embeddings) { - queryEmbedding = (queryEmbeddingResponse.embeddings as { embeddings: number[][] }).embeddings[0]; - } else { - throw new Error('Invalid embedding response format'); - } - - if (!Array.isArray(queryEmbedding)) { - throw new Error('Query embedding is not an array'); - } - - // Query the Pinecone index using the embedding and filter by document IDs. - const queryResponse: QueryResponse = await this.index.query({ - vector: queryEmbedding, - filter: { - doc_id: { $in: this._doc_ids }, - }, - topK, - includeValues: true, - includeMetadata: true, - }); - - // Map the results into RAGChunks and return them. - return queryResponse.matches.map( - match => - ({ - id: match.id, - values: match.values as number[], - metadata: match.metadata as { - text: string; - type: string; - original_document: string; - file_path: string; - doc_id: string; - location: string; - start_page: number; - end_page: number; - }, - }) as RAGChunk - ); - } catch (error) { - console.error(`Error retrieving chunks: ${error}`); - return []; - } - } -} diff --git a/src/client/views/nodes/chatbot/agentsystem/Agent.ts b/src/client/views/nodes/chatbot/agentsystem/Agent.ts new file mode 100644 index 000000000..180d05cf3 --- /dev/null +++ b/src/client/views/nodes/chatbot/agentsystem/Agent.ts @@ -0,0 +1,278 @@ +import OpenAI from 'openai'; +import { Tool, AgentMessage, AssistantMessage, TEXT_TYPE, CHUNK_TYPE, ASSISTANT_ROLE, ProcessingInfo, PROCESSING_TYPE } from '../types/types'; +import { getReactPrompt } from './prompts'; +import { XMLParser, XMLBuilder } from 'fast-xml-parser'; +import { Vectorstore } from '../vectorstore/Vectorstore'; +import { ChatCompletionMessageParam } from 'openai/resources'; +import dotenv from 'dotenv'; +import { CalculateTool } from '../tools/CalculateTool'; +import { RAGTool } from '../tools/RAGTool'; +import { DataAnalysisTool } from '../tools/DataAnalysisTool'; +import { WebsiteInfoScraperTool } from '../tools/WebsiteInfoScraperTool'; +import { SearchTool } from '../tools/SearchTool'; +import { NoTool } from '../tools/NoTool'; +import { v4 as uuidv4 } from 'uuid'; +import { AnswerParser } from '../response_parsers/AnswerParser'; +import { StreamedAnswerParser } from '../response_parsers/StreamedAnswerParser'; +import { CreateCSVTool } from '../tools/CreateCSVTool'; + +dotenv.config(); + +/** + * The Agent class handles the interaction between the assistant and the tools available, + * processes user queries, and manages the communication flow between the tools and OpenAI. + */ +export class Agent { + // Private properties + private client: OpenAI; + private tools: Record>; + private messages: AgentMessage[] = []; + private interMessages: AgentMessage[] = []; + private vectorstore: Vectorstore; + private _history: () => string; + private _summaries: () => string; + private _csvData: () => { filename: string; id: string; text: string }[]; + private actionNumber: number = 0; + private thoughtNumber: number = 0; + private processingNumber: number = 0; + private processingInfo: ProcessingInfo[] = []; + private streamedAnswerParser: StreamedAnswerParser = new StreamedAnswerParser(); + + /** + * The constructor initializes the agent with the vector store and toolset, and sets up the OpenAI client. + * @param _vectorstore Vector store instance for document storage and retrieval. + * @param summaries A function to retrieve document summaries. + * @param history A function to retrieve chat history. + * @param csvData A function to retrieve CSV data linked to the assistant. + * @param addLinkedUrlDoc A function to add a linked document from a URL. + * @param createCSVInDash A function to create a CSV document in the dashboard. + */ + constructor( + _vectorstore: Vectorstore, + summaries: () => string, + history: () => string, + csvData: () => { filename: string; id: string; text: string }[], + addLinkedUrlDoc: (url: string, id: string) => void, + createCSVInDash: (url: string, title: string, id: string, data: string) => void + ) { + // Initialize OpenAI client with API key from environment + this.client = new OpenAI({ apiKey: process.env.OPENAI_KEY, dangerouslyAllowBrowser: true }); + this.vectorstore = _vectorstore; + this._history = history; + this._summaries = summaries; + this._csvData = csvData; + + // Define available tools for the assistant + this.tools = { + calculate: new CalculateTool(), + rag: new RAGTool(this.vectorstore), + dataAnalysis: new DataAnalysisTool(csvData), + websiteInfoScraper: new WebsiteInfoScraperTool(addLinkedUrlDoc), + searchTool: new SearchTool(addLinkedUrlDoc), + createCSV: new CreateCSVTool(createCSVInDash), + no_tool: new NoTool(), + }; + } + + /** + * This method handles the conversation flow with the assistant, processes user queries, + * and manages the assistant's decision-making process, including tool actions. + * @param question The user's question. + * @param onProcessingUpdate Callback function for processing updates. + * @param onAnswerUpdate Callback function for answer updates. + * @param maxTurns The maximum number of turns to allow in the conversation. + * @returns The final response from the assistant. + */ + async askAgent(question: string, onProcessingUpdate: (processingUpdate: ProcessingInfo[]) => void, onAnswerUpdate: (answerUpdate: string) => void, maxTurns: number = 30): Promise { + console.log(`Starting query: ${question}`); + + // Push user's question to message history + this.messages.push({ role: 'user', content: question }); + + // Retrieve chat history and generate system prompt + const chatHistory = this._history(); + const systemPrompt = getReactPrompt(Object.values(this.tools), this._summaries, chatHistory); + + // Initialize intermediate messages + this.interMessages = [{ role: 'system', content: systemPrompt }]; + this.interMessages.push({ role: 'user', content: `${question}` }); + + // Setup XML parser and builder + const parser = new XMLParser({ + ignoreAttributes: false, + attributeNamePrefix: '@_', + textNodeName: '_text', + isArray: (name, jpath, isLeafNode, isAttribute) => ['query', 'url'].indexOf(name) !== -1, + }); + const builder = new XMLBuilder({ ignoreAttributes: false, attributeNamePrefix: '@_' }); + + let currentAction: string | undefined; + this.processingInfo = []; + + // Conversation loop (up to maxTurns) + for (let i = 2; i < maxTurns; i += 2) { + console.log(this.interMessages); + console.log(`Turn ${i}/${maxTurns}`); + + // Execute a step in the conversation and get the result + const result = await this.execute(onProcessingUpdate, onAnswerUpdate); + this.interMessages.push({ role: 'assistant', content: result }); + + let parsedResult; + try { + // Parse XML result from the assistant + parsedResult = parser.parse(result); + } catch (error) { + throw new Error(`Error parsing response: ${error}`); + } + + // Extract the stage from the parsed result + const stage = parsedResult.stage; + if (!stage) { + throw new Error(`Error: No stage found in response`); + } + + // Handle different stage elements (thoughts, actions, inputs, answers) + for (const key in stage) { + if (key === 'thought') { + // Handle assistant's thoughts + console.log(`Thought: ${stage[key]}`); + this.processingNumber++; + } else if (key === 'action') { + // Handle action stage + currentAction = stage[key] as string; + console.log(`Action: ${currentAction}`); + + if (this.tools[currentAction]) { + // Prepare the next action based on the current tool + const nextPrompt = [ + { + type: 'text', + text: `` + builder.build({ action_rules: this.tools[currentAction].getActionRule() }) + ``, + }, + ]; + this.interMessages.push({ role: 'user', content: nextPrompt }); + break; + } else { + // Handle error in case of an invalid action + console.log('Error: No valid action'); + this.interMessages.push({ role: 'user', content: `No valid action, try again.` }); + break; + } + } else if (key === 'action_input') { + // Handle action input stage + const actionInput = stage[key]; + console.log(`Action input:`, actionInput.inputs); + + if (currentAction) { + try { + // Process the action with its input + const observation = await this.processAction(currentAction, actionInput.inputs); + const nextPrompt = [{ type: 'text', text: ` ` }, ...observation, { type: 'text', text: '' }]; + console.log(observation); + this.interMessages.push({ role: 'user', content: nextPrompt }); + this.processingNumber++; + break; + } catch (error) { + throw new Error(`Error processing action: ${error}`); + } + } else { + throw new Error('Error: Action input without a valid action'); + } + } else if (key === 'answer') { + // If an answer is found, end the query + console.log('Answer found. Ending query.'); + this.streamedAnswerParser.reset(); + const parsedAnswer = AnswerParser.parse(result, this.processingInfo); + return parsedAnswer; + } + } + } + + throw new Error('Reached maximum turns. Ending query.'); + } + + /** + * Executes a step in the conversation, processing the assistant's response and parsing it in real-time. + * @param onProcessingUpdate Callback for processing updates. + * @param onAnswerUpdate Callback for answer updates. + * @returns The full response from the assistant. + */ + private async execute(onProcessingUpdate: (processingUpdate: ProcessingInfo[]) => void, onAnswerUpdate: (answerUpdate: string) => void): Promise { + // Stream OpenAI response for real-time updates + const stream = await this.client.chat.completions.create({ + model: 'gpt-4o', + messages: this.interMessages as ChatCompletionMessageParam[], + temperature: 0, + stream: true, + }); + + let fullResponse: string = ''; + let currentTag: string = ''; + let currentContent: string = ''; + let isInsideTag: boolean = false; + + // Process each chunk of the streamed response + for await (const chunk of stream) { + let content = chunk.choices[0]?.delta?.content || ''; + fullResponse += content; + + // Parse the streamed content character by character + for (const char of content) { + if (currentTag === 'answer') { + // Handle answer parsing for real-time updates + currentContent += char; + const streamedAnswer = this.streamedAnswerParser.parse(char); + onAnswerUpdate(streamedAnswer); + continue; + } else if (char === '<') { + // Start of a new tag + isInsideTag = true; + currentTag = ''; + currentContent = ''; + } else if (char === '>') { + // End of the tag + isInsideTag = false; + if (currentTag.startsWith('/')) { + currentTag = ''; + } + } else if (isInsideTag) { + // Append characters to the tag name + currentTag += char; + } else if (currentTag === 'thought' || currentTag === 'action_input_description') { + // Handle processing information for thought or action input description + currentContent += char; + const current_info = this.processingInfo.find(info => info.index === this.processingNumber); + if (current_info) { + current_info.content = currentContent.trim(); + onProcessingUpdate(this.processingInfo); + } else { + this.processingInfo.push({ + index: this.processingNumber, + type: currentTag === 'thought' ? PROCESSING_TYPE.THOUGHT : PROCESSING_TYPE.ACTION, + content: currentContent.trim(), + }); + onProcessingUpdate(this.processingInfo); + } + } + } + } + + return fullResponse; + } + + /** + * Processes a specific action by invoking the appropriate tool with the provided inputs. + * @param action The action to perform. + * @param actionInput The inputs for the action. + * @returns The result of the action. + */ + private async processAction(action: string, actionInput: any): Promise { + if (!(action in this.tools)) { + throw new Error(`Unknown action: ${action}`); + } + + const tool = this.tools[action]; + return await tool.execute(actionInput); + } +} diff --git a/src/client/views/nodes/chatbot/agentsystem/prompts.ts b/src/client/views/nodes/chatbot/agentsystem/prompts.ts new file mode 100644 index 000000000..9daabc35f --- /dev/null +++ b/src/client/views/nodes/chatbot/agentsystem/prompts.ts @@ -0,0 +1,181 @@ +// prompts.ts + +import { Tool } from '../types/types'; + +export function getReactPrompt(tools: Tool[], summaries: () => string, chatHistory: string): string { + const toolDescriptions = tools + .map( + tool => ` + + ${tool.name} + ${tool.briefSummary} + ` + ) + .join('\n'); + + return ` + + You are an advanced AI assistant equipped with tools to answer user queries efficiently. You operate in a loop that is RIGIDLY structured and requires the use of specific tags and formats for your responses. Your goal is to provide accurate and well-structured answers to user queries. Below are the guidelines and information you can use to structure your approach to accomplishing this task. + + + + **STRUCTURE**: Always use the correct stage tags (e.g., ) for every response. Use only even-numbered stages for your responses. + **STOP after every stage and wait for input. Do not combine multiple stages in one response.** + If a tool is needed, select the most appropriate tool based on the query. + **If one tool does not yield satisfactory results or fails twice, try another tool that might work better for the query.** + Ensure that **ALL answers follow the answer structure**: grounded text wrapped in tags with corresponding citations, normal text in tags, and three follow-up questions at the end. + + + + + - All information derived from tools or user documents must be wrapped in these tags with proper citation. + - Use this tag for text not derived from tools or user documents. + + - Provide proper citations for each , referencing the tool or document chunk used. + + - Provide exactly three user-perspective follow-up questions. + - Summarize the actions and tools used in the conversation. + + + + + **Wrap ALL tool-based information** in tags and provide citations. + Use separate tags for distinct information or when switching to a different tool or document. + Ensure that **EVERY** tag includes a citation index referencing the source of the information. + Over-citing is discouraged—only cite the information that is directly relevant to the user's query. + + + + Wrap general information or reasoning **not derived from tools or documents** in tags. + Never put information derived from user documents or tools in tags—use for those. + + + + Carefully analyze the user query and determine if a tool is necessary to provide an accurate answer. + If a tool is needed, choose the most appropriate one and **stop after the action** to wait for system input. + If no tool is needed, use the 'no_tool' action but follow the structure. + When all observations are complete, format the final answer using and tags with appropriate citations. + Include exactly three follow-up questions from the user's perspective. + Provide a loop summary at the end of the conversation. + + + + ${toolDescriptions} + If no external tool is required, use 'no_tool', but if there might be relevant external information, use the appropriate tool. + + + + ${summaries()} + + + + ${chatHistory} + + + + + + Can you provide key moments from the 2022 World Cup and its impact on tourism in Qatar? + + + + + I will use the RAG tool to retrieve key moments from the user's World Cup documents. Afterward, I will use the website scraper tool to gather tourism impact data on Qatar. + + rag + + + + ***Action rules omitted*** + + + + + Searching user documents for key moments from the 2022 World Cup. + + Key moments from the 2022 World Cup. Goals, assists, big wins, big losses. + + + + + + + + The 2022 FIFA World Cup saw Argentina win, with Lionel Messi's performance being a key highlight. It was widely celebrated as a historical moment in sports. + + + + + + + With key moments from the World Cup retrieved, I will now use the website scraper tool to gather data on Qatar’s tourism impact during the World Cup. + + websiteInfoScraper + + + + ***Action rules omitted*** + + + + + Scraping websites for information about Qatar's tourism impact during the 2022 World Cup. + + Tourism impact of the 2022 World Cup in Qatar + + + + + + + + https://www.qatartourism.com/world-cup-impact + During the 2022 World Cup, Qatar saw a 40% increase in tourism, with over 1.5 million visitors attending. + + + + + + + Now that I have gathered both key moments from the World Cup and tourism impact data from Qatar, I will summarize the information in my final response. + + + **The 2022 World Cup** saw Argentina crowned champions, with **Lionel Messi** leading his team to victory, marking a historic moment in sports. + **Qatar** experienced a **40% increase in tourism** during the World Cup, welcoming over **1.5 million visitors**, significantly boosting its economy. + Moments like **Messi’s triumph** often become ingrained in the legacy of World Cups, immortalizing these tournaments in both sports and cultural memory. The **long-term implications** of the World Cup on Qatar's **economy, tourism**, and **global image** remain important areas of interest as the country continues to build on the momentum generated by hosting this prestigious event. + + Key moments from the 2022 World Cup. + + + + What long-term effects has the World Cup had on Qatar's economy and infrastructure? + Can you compare Qatar's tourism numbers with previous World Cup hosts? + How has Qatar’s image on the global stage evolved post-World Cup? + + + The assistant first used the RAG tool to extract key moments from the user documents about the 2022 World Cup. Then, the assistant utilized the website scraping tool to gather data on Qatar's tourism impact. Both tools provided valuable information, and no additional tools were needed. + + + + + + + + Process the user's query according to these rules. Ensure your final answer is comprehensive, well-structured, and includes citations where appropriate. + +`; +} + +export function getSummarizedChunksPrompt(chunks: string): string { + return `Please provide a comprehensive summary of what you think the document from which these chunks originated. + Ensure the summary captures the main ideas and key points from all provided chunks. Be concise and brief and only provide the summary in paragraph form. + + Text chunks: + \`\`\` + ${chunks} + \`\`\``; +} + +export function getSummarizedSystemPrompt(): string { + return 'You are an AI assistant tasked with summarizing a document. You are provided with important chunks from the document and provide a summary, as best you can, of what the document will contain overall. Be concise and brief with your response.'; +} diff --git a/src/client/views/nodes/chatbot/chatboxcomponents/ChatBox.scss b/src/client/views/nodes/chatbot/chatboxcomponents/ChatBox.scss new file mode 100644 index 000000000..42f6a0d61 --- /dev/null +++ b/src/client/views/nodes/chatbot/chatboxcomponents/ChatBox.scss @@ -0,0 +1,279 @@ +@import url('https://fonts.googleapis.com/css2?family=Atkinson+Hyperlegible:ital,wght@0,400;0,700;1,400;1,700&display=swap'); + +$primary-color: #4a90e2; +$secondary-color: #f5f8fa; +$text-color: #333; +$light-text-color: #777; +$border-color: #e1e8ed; +$shadow-color: rgba(0, 0, 0, 0.1); +$transition: all 0.3s ease; +.chat-box { + display: flex; + flex-direction: column; + height: 100%; + background-color: #fff; + font-family: + 'Atkinson Hyperlegible', + -apple-system, + BlinkMacSystemFont, + 'Segoe UI', + Roboto, + Helvetica, + Arial, + sans-serif; + border-radius: 12px; + overflow: hidden; + box-shadow: 0 4px 12px $shadow-color; + position: relative; + + .chat-header { + background-color: $primary-color; + color: white; + padding: 15px; + text-align: center; + box-shadow: 0 2px 4px $shadow-color; + height: fit-content; + + h2 { + margin: 0; + font-size: 1.3em; + font-weight: 500; + } + } + + .chat-messages { + flex-grow: 1; + overflow-y: auto; + padding: 20px; + display: flex; + flex-direction: column; + gap: 10px; // Added to give space between elements + + &::-webkit-scrollbar { + width: 6px; + } + + &::-webkit-scrollbar-thumb { + background-color: $border-color; + border-radius: 3px; + } + } + + .chat-input { + display: flex; + padding: 20px; + border-top: 1px solid $border-color; + background-color: #fff; + + input { + flex-grow: 1; + padding: 12px 15px; + border: 1px solid $border-color; + border-radius: 24px; + font-size: 15px; + transition: $transition; + + &:focus { + outline: none; + border-color: $primary-color; + box-shadow: 0 0 0 2px rgba($primary-color, 0.2); + } + } + + .submit-button { + background-color: $primary-color; + color: white; + border: none; + border-radius: 50%; + width: 48px; + height: 48px; + margin-left: 10px; + cursor: pointer; + transition: $transition; + display: flex; + align-items: center; + justify-content: center; + + &:hover { + background-color: darken($primary-color, 10%); + } + + &:disabled { + background-color: $light-text-color; + cursor: not-allowed; + } + + .spinner { + height: 24px; + width: 24px; + border: 3px solid rgba(255, 255, 255, 0.3); + border-top: 3px solid #fff; + border-radius: 50%; + animation: spin 1s linear infinite; + display: flex; + align-items: center; + justify-content: center; + } + } + } + .citation-popup { + position: fixed; + bottom: 50px; + left: 50%; + transform: translateX(-50%); + background-color: rgba(0, 0, 0, 0.8); + color: white; + padding: 10px 20px; + border-radius: 10px; + box-shadow: 0 4px 12px rgba(0, 0, 0, 0.2); + z-index: 1000; + animation: fadeIn 0.3s ease-in-out; + + p { + margin: 0; + font-size: 14px; + } + + @keyframes fadeIn { + from { + opacity: 0; + } + to { + opacity: 1; + } + } + } +} + +.message { + max-width: 80%; + margin-bottom: 20px; + padding: 16px 20px; + border-radius: 18px; + font-size: 15px; + line-height: 1.5; + box-shadow: 0 2px 4px $shadow-color; + word-wrap: break-word; // To handle long words + + &.user { + align-self: flex-end; + background-color: $primary-color; + color: white; + border-bottom-right-radius: 4px; + } + + &.chatbot { + align-self: flex-start; + background-color: $secondary-color; + color: $text-color; + border-bottom-left-radius: 4px; + } + + .toggle-info { + background-color: transparent; + color: $primary-color; + border: 1px solid $primary-color; + width: 100%; + height: fit-content; + border-radius: 8px; + padding: 10px 16px; + font-size: 14px; + cursor: pointer; + transition: $transition; + margin-top: 10px; + + &:hover { + background-color: rgba($primary-color, 0.1); + } + } +} + +.follow-up-questions { + margin-top: 15px; + + h4 { + font-size: 15px; + font-weight: 600; + margin-bottom: 10px; + } + + .questions-list { + display: flex; + flex-direction: column; + gap: 10px; + } + + .follow-up-button { + background-color: #fff; + color: $primary-color; + border: 1px solid $primary-color; + border-radius: 8px; + padding: 10px 16px; + font-size: 14px; + cursor: pointer; + transition: $transition; + text-align: left; + white-space: normal; + word-wrap: break-word; + width: 100%; + height: fit-content; + + &:hover { + background-color: $primary-color; + color: #fff; + } + } +} + +.citation-button { + display: inline-flex; + align-items: center; + justify-content: center; + width: 20px; + height: 20px; + border-radius: 50%; + background-color: rgba(0, 0, 0, 0.1); + color: $text-color; + font-size: 12px; + font-weight: bold; + margin-left: 5px; + cursor: pointer; + transition: $transition; + vertical-align: middle; + + &:hover { + background-color: rgba(0, 0, 0, 0.2); + } +} + +.uploading-overlay { + position: absolute; + top: 0; + left: 0; + right: 0; + bottom: 0; + background-color: rgba(255, 255, 255, 0.8); + display: flex; + justify-content: center; + align-items: center; + z-index: 1000; +} + +@keyframes spin { + 0% { + transform: rotate(0deg); + } + 100% { + transform: rotate(360deg); + } +} + +@media (max-width: 768px) { + .chat-box { + border-radius: 0; + } + + .message { + max-width: 90%; + } +} diff --git a/src/client/views/nodes/chatbot/chatboxcomponents/ChatBox.tsx b/src/client/views/nodes/chatbot/chatboxcomponents/ChatBox.tsx new file mode 100644 index 000000000..6dc691798 --- /dev/null +++ b/src/client/views/nodes/chatbot/chatboxcomponents/ChatBox.tsx @@ -0,0 +1,745 @@ +import { action, computed, makeObservable, observable, observe, reaction, runInAction, ObservableSet } from 'mobx'; +import { observer } from 'mobx-react'; +import OpenAI, { ClientOptions } from 'openai'; +import * as React from 'react'; +import { Doc, DocListCast } from '../../../../../fields/Doc'; +import { CsvCast, DocCast, PDFCast, RTFCast, StrCast } from '../../../../../fields/Types'; +import { DocumentType } from '../../../../documents/DocumentTypes'; +import { Docs } from '../../../../documents/Documents'; +import { LinkManager } from '../../../../util/LinkManager'; +import { ViewBoxAnnotatableComponent } from '../../../DocComponent'; +import { FieldView, FieldViewProps } from '../../FieldView'; +import './ChatBox.scss'; +import MessageComponentBox from './MessageComponent'; +import { ASSISTANT_ROLE, AssistantMessage, Citation, CHUNK_TYPE, TEXT_TYPE, SimplifiedChunk, ProcessingInfo } from '../types/types'; +import { Vectorstore } from '../vectorstore/Vectorstore'; +import { Agent } from '../agentsystem/Agent'; +import dotenv from 'dotenv'; +import { DocData, DocViews } from '../../../../../fields/DocSymbols'; +import { DocumentManager } from '../../../../util/DocumentManager'; +import { v4 as uuidv4 } from 'uuid'; +import { DocUtils } from '../../../../documents/DocUtils'; +import { ClientUtils } from '../../../../../ClientUtils'; +import { ProgressBar } from './ProgressBar'; +import { DocumentView } from '../../DocumentView'; +import { Networking } from '../../../../Network'; + +dotenv.config(); + +/** + * ChatBox is the main class responsible for managing the interaction between the user and the assistant, + * handling documents, and integrating with OpenAI for tasks such as document analysis, chat functionality, + * and vector store interactions. + */ +@observer +export class ChatBox extends ViewBoxAnnotatableComponent() { + // MobX observable properties to track UI state and data + @observable history: AssistantMessage[] = []; + @observable.deep current_message: AssistantMessage | undefined = undefined; + @observable isLoading: boolean = false; + @observable uploadProgress: number = 0; + @observable currentStep: string = ''; + @observable expandedScratchpadIndex: number | null = null; + @observable inputValue: string = ''; + @observable private linked_docs_to_add: ObservableSet = observable.set(); + @observable private linked_csv_files: { filename: string; id: string; text: string }[] = []; + @observable private isUploadingDocs: boolean = false; + @observable private citationPopup: { text: string; visible: boolean } = { text: '', visible: false }; + + // Private properties for managing OpenAI API, vector store, agent, and UI elements + private openai: OpenAI; + private vectorstore_id: string; + private vectorstore: Vectorstore; + private agent: Agent; + private _oldWheel: HTMLDivElement | null = null; + private messagesRef: React.RefObject; + + /** + * Static method that returns the layout string for the field. + * @param fieldKey Key to get the layout string. + */ + public static LayoutString(fieldKey: string) { + return FieldView.LayoutString(ChatBox, fieldKey); + } + + /** + * Constructor initializes the component, sets up OpenAI, vector store, and agent instances, + * and observes changes in the chat history to save the state in dataDoc. + * @param props The properties passed to the component. + */ + constructor(props: FieldViewProps) { + super(props); + makeObservable(this); // Enable MobX observables + + // Initialize OpenAI, vectorstore, and agent + this.openai = this.initializeOpenAI(); + if (StrCast(this.dataDoc.vectorstore_id) == '') { + this.vectorstore_id = uuidv4(); + this.dataDoc.vectorstore_id = this.vectorstore_id; + } else { + this.vectorstore_id = StrCast(this.dataDoc.vectorstore_id); + } + this.vectorstore = new Vectorstore(this.vectorstore_id, this.retrieveDocIds); + this.agent = new Agent(this.vectorstore, this.retrieveSummaries, this.retrieveFormattedHistory, this.retrieveCSVData, this.addLinkedUrlDoc, this.createCSVInDash); + this.messagesRef = React.createRef(); + + // Reaction to update dataDoc when chat history changes + reaction( + () => + this.history.map((msg: AssistantMessage) => ({ + role: msg.role, + content: msg.content, + follow_up_questions: msg.follow_up_questions, + citations: msg.citations, + })), + serializableHistory => { + this.dataDoc.data = JSON.stringify(serializableHistory); + } + ); + } + + /** + * Adds a document to the vectorstore for AI-based analysis. + * Handles the upload progress and errors during the process. + * @param newLinkedDoc The new document to add. + */ + @action + addDocToVectorstore = async (newLinkedDoc: Doc) => { + this.uploadProgress = 0; + this.currentStep = 'Initializing...'; + this.isUploadingDocs = true; + + try { + // Add the document to the vectorstore + await this.vectorstore.addAIDoc(newLinkedDoc, this.updateProgress); + } catch (error) { + console.error('Error uploading document:', error); + this.currentStep = 'Error during upload'; + } finally { + this.isUploadingDocs = false; + this.uploadProgress = 0; + this.currentStep = ''; + } + }; + + /** + * Updates the upload progress and the current step in the UI. + * @param progress The percentage of the progress. + * @param step The current step name. + */ + @action + updateProgress = (progress: number, step: string) => { + this.uploadProgress = progress; + this.currentStep = step; + }; + + /** + * Adds a CSV file for analysis by sending it to OpenAI and generating a summary. + * @param newLinkedDoc The linked document representing the CSV file. + * @param id Optional ID for the document. + */ + @action + addCSVForAnalysis = async (newLinkedDoc: Doc, id?: string) => { + if (!newLinkedDoc.chunk_simpl) { + // Convert document text to CSV data + const csvData: string = StrCast(newLinkedDoc.text); + + // Generate a summary using OpenAI API + const completion = await this.openai.chat.completions.create({ + messages: [ + { + role: 'system', + content: + 'You are an AI assistant tasked with summarizing the content of a CSV file. You will be provided with the data from the CSV file and your goal is to generate a concise summary that captures the main themes, trends, and key points represented in the data.', + }, + { + role: 'user', + content: `Please provide a comprehensive summary of the CSV file based on the provided data. Ensure the summary highlights the most important information, patterns, and insights. Your response should be in paragraph form and be concise. + CSV Data: + ${csvData} + ********** + Summary:`, + }, + ], + model: 'gpt-3.5-turbo', + }); + + const csvId = id ?? uuidv4(); + + // Add CSV details to linked files + this.linked_csv_files.push({ + filename: CsvCast(newLinkedDoc.data).url.pathname, + id: csvId, + text: csvData, + }); + + // Add a chunk for the CSV and assign the summary + const chunkToAdd = { + chunkId: csvId, + chunkType: CHUNK_TYPE.CSV, + }; + newLinkedDoc.chunk_simpl = JSON.stringify({ chunks: [chunkToAdd] }); + newLinkedDoc.summary = completion.choices[0].message.content!; + } + }; + + /** + * Toggles the tool logs, expanding or collapsing the scratchpad at the given index. + * @param index Index of the tool log to toggle. + */ + @action + toggleToolLogs = (index: number) => { + this.expandedScratchpadIndex = this.expandedScratchpadIndex === index ? null : index; + }; + + /** + * Initializes the OpenAI API client using the API key from environment variables. + * @returns OpenAI client instance. + */ + initializeOpenAI() { + const configuration: ClientOptions = { + apiKey: process.env.OPENAI_KEY, + dangerouslyAllowBrowser: true, + }; + return new OpenAI(configuration); + } + + /** + * Adds a scroll event listener to detect user scrolling and handle passive wheel events. + */ + addScrollListener = () => { + if (this.messagesRef.current) { + this.messagesRef.current.addEventListener('wheel', this.onPassiveWheel, { passive: false }); + } + }; + + /** + * Removes the scroll event listener from the chat messages container. + */ + removeScrollListener = () => { + if (this.messagesRef.current) { + this.messagesRef.current.removeEventListener('wheel', this.onPassiveWheel); + } + }; + + /** + * Scrolls the chat messages container to the bottom, ensuring the latest message is visible. + */ + scrollToBottom = () => { + if (this.messagesRef.current) { + this.messagesRef.current.scrollTop = this.messagesRef.current.scrollHeight; + } + }; + + /** + * Event handler for detecting wheel scrolling and stopping the event propagation. + * @param e The wheel event. + */ + onPassiveWheel = (e: WheelEvent) => { + if (this._props.isContentActive()) { + e.stopPropagation(); + } + }; + + /** + * Sends the user's input to OpenAI, displays the loading indicator, and updates the chat history. + * @param event The form submission event. + */ + @action + askGPT = async (event: React.FormEvent): Promise => { + event.preventDefault(); + this.inputValue = ''; + + // Extract the user's message + const textInput = event.currentTarget.elements.namedItem('messageInput') as HTMLInputElement; + const trimmedText = textInput.value.trim(); + + if (trimmedText) { + try { + textInput.value = ''; + // Add the user's message to the history + this.history.push({ + role: ASSISTANT_ROLE.USER, + content: [{ index: 0, type: TEXT_TYPE.NORMAL, text: trimmedText, citation_ids: null }], + processing_info: [], + }); + this.isLoading = true; + this.current_message = { + role: ASSISTANT_ROLE.ASSISTANT, + content: [], + citations: [], + processing_info: [], + }; + + // Define callbacks for real-time processing updates + const onProcessingUpdate = (processingUpdate: ProcessingInfo[]) => { + runInAction(() => { + if (this.current_message) { + this.current_message = { + ...this.current_message, + processing_info: processingUpdate, + }; + } + }); + this.scrollToBottom(); + }; + + const onAnswerUpdate = (answerUpdate: string) => { + runInAction(() => { + if (this.current_message) { + this.current_message = { + ...this.current_message, + content: [{ text: answerUpdate, type: TEXT_TYPE.NORMAL, index: 0, citation_ids: [] }], + }; + } + }); + }; + + // Send the user's question to the assistant and get the final message + const finalMessage = await this.agent.askAgent(trimmedText, onProcessingUpdate, onAnswerUpdate); + + // Update the history with the final assistant message + runInAction(() => { + if (this.current_message) { + this.history.push({ ...finalMessage }); + this.current_message = undefined; + this.dataDoc.data = JSON.stringify(this.history); + } + }); + } catch (err) { + console.error('Error:', err); + // Handle error in processing + this.history.push({ + role: ASSISTANT_ROLE.ASSISTANT, + content: [{ index: 0, type: TEXT_TYPE.ERROR, text: 'Sorry, I encountered an error while processing your request.', citation_ids: null }], + processing_info: [], + }); + } finally { + this.isLoading = false; + this.scrollToBottom(); + } + } + this.scrollToBottom(); + }; + + /** + * Updates the citations for a given message in the chat history. + * @param index The index of the message in the history. + * @param citations The list of citations to add to the message. + */ + @action + updateMessageCitations = (index: number, citations: Citation[]) => { + if (this.history[index]) { + this.history[index].citations = citations; + } + }; + + /** + * Adds a linked document from a URL for future reference and analysis. + * @param url The URL of the document to add. + * @param id The unique identifier for the document. + */ + @action + addLinkedUrlDoc = async (url: string, id: string) => { + const doc = Docs.Create.WebDocument(url, { data_useCors: true }); + + const linkDoc = Docs.Create.LinkDocument(this.Document, doc); + LinkManager.Instance.addLink(linkDoc); + let canDisplay; + + try { + // Fetch the URL content through the proxy + const { data } = await Networking.PostToServer('/proxyFetch', { url }); + + // Simulating header behavior since we can't fetch headers via proxy + const xFrameOptions = data.headers?.['x-frame-options']; + + if (xFrameOptions && xFrameOptions.toUpperCase() === 'SAMEORIGIN') { + canDisplay = false; + } else { + canDisplay = true; + } + } catch (error) { + console.error('Error fetching the URL from the server:', error); + } + + const chunkToAdd = { + chunkId: id, + chunkType: CHUNK_TYPE.URL, + url: url, + canDisplay: canDisplay, + }; + + doc.chunk_simpl = JSON.stringify({ chunks: [chunkToAdd] }); + }; + + /** + * Getter to retrieve the current user's name from the client utils. + */ + @computed + get userName() { + return ClientUtils.CurrentUserEmail; + } + + /** + * Creates a CSV document in the dashboard and adds it for analysis. + * @param url The URL of the CSV. + * @param title The title of the CSV document. + * @param id The unique ID for the document. + * @param data The CSV data content. + */ + @action + createCSVInDash = async (url: string, title: string, id: string, data: string) => { + const doc = DocCast(await DocUtils.DocumentFromType('csv', url, { title: title, text: RTFCast(data) })); + + const linkDoc = Docs.Create.LinkDocument(this.Document, doc); + LinkManager.Instance.addLink(linkDoc); + + doc && this._props.addDocument?.(doc); + await DocumentManager.Instance.showDocument(doc, { willZoomCentered: true }, () => {}); + + this.addCSVForAnalysis(doc, id); + }; + + /** + * Event handler to manage citations click in the message components. + * @param citation The citation object clicked by the user. + */ + @action + handleCitationClick = (citation: Citation) => { + const currentLinkedDocs: Doc[] = this.linkedDocs; + + const chunkId = citation.chunk_id; + + // Loop through the linked documents to find the matching chunk and handle its display + for (let doc of currentLinkedDocs) { + if (doc.chunk_simpl) { + const docChunkSimpl = JSON.parse(StrCast(doc.chunk_simpl)) as { chunks: SimplifiedChunk[] }; + const foundChunk = docChunkSimpl.chunks.find(chunk => chunk.chunkId === chunkId); + if (foundChunk) { + // Handle different types of chunks (image, text, table, etc.) + switch (foundChunk.chunkType) { + case CHUNK_TYPE.IMAGE: + case CHUNK_TYPE.TABLE: + const values = foundChunk.location?.replace(/[\[\]]/g, '').split(','); + + if (values?.length !== 4) { + console.error('Location string must contain exactly 4 numbers'); + return; + } + + const x1 = parseFloat(values[0]) * Doc.NativeWidth(doc); + const y1 = parseFloat(values[1]) * Doc.NativeHeight(doc) + foundChunk.startPage * Doc.NativeHeight(doc); + const x2 = parseFloat(values[2]) * Doc.NativeWidth(doc); + const y2 = parseFloat(values[3]) * Doc.NativeHeight(doc) + foundChunk.startPage * Doc.NativeHeight(doc); + + const annotationKey = Doc.LayoutFieldKey(doc) + '_annotations'; + + const existingDoc = DocListCast(doc[DocData][annotationKey]).find(d => d.citation_id === citation.citation_id); + const highlightDoc = existingDoc ?? this.createImageCitationHighlight(x1, y1, x2, y2, citation, annotationKey, doc); + + DocumentManager.Instance.showDocument(highlightDoc, { willZoomCentered: true }, () => {}); + break; + case CHUNK_TYPE.TEXT: + this.citationPopup = { text: citation.direct_text ?? 'No text available', visible: true }; + setTimeout(() => (this.citationPopup.visible = false), 3000); // Hide after 3 seconds + + DocumentManager.Instance.showDocument(doc, { willZoomCentered: true }, () => { + const firstView = Array.from(doc[DocViews])[0] as DocumentView; + firstView.ComponentView?.search?.(citation.direct_text ?? ''); + }); + break; + case CHUNK_TYPE.URL: + if (!foundChunk.canDisplay) { + window.open(StrCast(doc.displayUrl), '_blank'); + } else if (foundChunk.canDisplay) { + DocumentManager.Instance.showDocument(doc, { willZoomCentered: true }, () => {}); + } + break; + case CHUNK_TYPE.CSV: + DocumentManager.Instance.showDocument(doc, { willZoomCentered: true }, () => {}); + break; + default: + console.error('Chunk type not recognized:', foundChunk.chunkType); + break; + } + } + } + } + }; + + /** + * Creates an annotation highlight on a PDF document for image citations. + * @param x1 X-coordinate of the top-left corner of the highlight. + * @param y1 Y-coordinate of the top-left corner of the highlight. + * @param x2 X-coordinate of the bottom-right corner of the highlight. + * @param y2 Y-coordinate of the bottom-right corner of the highlight. + * @param citation The citation object to associate with the highlight. + * @param annotationKey The key used to store the annotation. + * @param pdfDoc The document where the highlight is created. + * @returns The highlighted document. + */ + createImageCitationHighlight = (x1: number, y1: number, x2: number, y2: number, citation: Citation, annotationKey: string, pdfDoc: Doc): Doc => { + const highlight_doc = Docs.Create.FreeformDocument([], { + x: x1, + y: y1, + _width: x2 - x1, + _height: y2 - y1, + backgroundColor: 'rgba(255, 255, 0, 0.5)', + }); + highlight_doc[DocData].citation_id = citation.citation_id; + Doc.AddDocToList(pdfDoc[DocData], annotationKey, highlight_doc); + highlight_doc.annotationOn = pdfDoc; + Doc.SetContainer(highlight_doc, pdfDoc); + return highlight_doc; + }; + + /** + * Lifecycle method that triggers when the component updates. + * Ensures the chat is scrolled to the bottom when new messages are added. + */ + componentDidUpdate() { + this.scrollToBottom(); + } + + /** + * Lifecycle method that triggers when the component mounts. + * Initializes scroll listeners, sets up document reactions, and loads chat history from dataDoc if available. + */ + componentDidMount() { + this._props.setContentViewBox?.(this); + if (this.dataDoc.data) { + try { + const storedHistory = JSON.parse(StrCast(this.dataDoc.data)); + runInAction(() => { + this.history.push( + ...storedHistory.map((msg: AssistantMessage) => ({ + role: msg.role, + content: msg.content, + follow_up_questions: msg.follow_up_questions, + citations: msg.citations, + })) + ); + }); + } catch (e) { + console.error('Failed to parse history from dataDoc:', e); + } + } else { + // Default welcome message + runInAction(() => { + this.history.push({ + role: ASSISTANT_ROLE.ASSISTANT, + content: [ + { + index: 0, + type: TEXT_TYPE.NORMAL, + text: `Hey, ${this.userName()}! Welcome to Your Friendly Assistant. Link a document or ask questions to get started.`, + citation_ids: null, + }, + ], + processing_info: [], + }); + }); + } + + // Set up reactions for linked documents + reaction( + () => { + const linkedDocs = LinkManager.Instance.getAllRelatedLinks(this.Document) + .map(d => DocCast(LinkManager.getOppositeAnchor(d, this.Document))) + .map(d => DocCast(d?.annotationOn, d)) + .filter(d => d); + return linkedDocs; + }, + linked => linked.forEach(doc => this.linked_docs_to_add.add(doc)) + ); + + // Observe changes to linked documents and handle document addition + observe(this.linked_docs_to_add, change => { + if (change.type === 'add') { + if (PDFCast(change.newValue.data)) { + this.addDocToVectorstore(change.newValue); + } else if (CsvCast(change.newValue.data)) { + this.addCSVForAnalysis(change.newValue); + } + } else if (change.type === 'delete') { + // Handle document removal + } + }); + this.addScrollListener(); + } + + /** + * Lifecycle method that triggers when the component unmounts. + * Removes scroll listeners to avoid memory leaks. + */ + componentWillUnmount() { + this.removeScrollListener(); + } + + /** + * Getter that retrieves all linked documents for the current document. + */ + @computed + get linkedDocs() { + return LinkManager.Instance.getAllRelatedLinks(this.Document) + .map(d => DocCast(LinkManager.getOppositeAnchor(d, this.Document))) + .map(d => DocCast(d?.annotationOn, d)) + .filter(d => d); + } + + /** + * Getter that retrieves document IDs of linked documents that have AI-related content. + */ + @computed + get docIds() { + return LinkManager.Instance.getAllRelatedLinks(this.Document) + .map(d => DocCast(LinkManager.getOppositeAnchor(d, this.Document))) + .map(d => DocCast(d?.annotationOn, d)) + .filter(d => d) + .filter(d => d.ai_doc_id) + .map(d => StrCast(d.ai_doc_id)); + } + + /** + * Getter that retrieves summaries of all linked documents. + */ + @computed + get summaries(): string { + return ( + LinkManager.Instance.getAllRelatedLinks(this.Document) + .map(d => DocCast(LinkManager.getOppositeAnchor(d, this.Document))) + .map(d => DocCast(d?.annotationOn, d)) + .filter(d => d) + .filter(d => d.summary) + .map((doc, index) => { + if (PDFCast(doc.data)) { + return `${doc.summary}`; + } else if (CsvCast(doc.data)) { + return `${doc.summary}`; + } else { + return `${index + 1}) ${doc.summary}`; + } + }) + .join('\n') + '\n' + ); + } + + /** + * Getter that retrieves all linked CSV files for analysis. + */ + @computed + get linkedCSVs(): { filename: string; id: string; text: string }[] { + return this.linked_csv_files; + } + + /** + * Getter that formats the entire chat history as a string for the agent's system message. + */ + @computed + get formattedHistory(): string { + let history = '\n'; + for (const message of this.history) { + history += `<${message.role}>${message.content.map(content => content.text).join(' ')}`; + if (message.loop_summary) { + history += `${message.loop_summary}`; + } + history += `\n`; + } + history += ''; + return history; + } + + // Other helper methods for retrieving document data and processing + + retrieveSummaries = () => { + return this.summaries; + }; + + retrieveCSVData = () => { + return this.linkedCSVs; + }; + + retrieveFormattedHistory = () => { + return this.formattedHistory; + }; + + retrieveDocIds = () => { + return this.docIds; + }; + + /** + * Handles follow-up questions when the user clicks on them. + * Automatically sets the input value to the clicked follow-up question. + * @param question The follow-up question clicked by the user. + */ + @action + handleFollowUpClick = (question: string) => { + this.inputValue = question; + }; + + /** + * Renders the chat interface, including the message list, input field, and other UI elements. + */ + render() { + return ( +
+ {this.isUploadingDocs && ( +
+
+ +
{this.currentStep}
+
+
+ )} +
+

{this.userName()}'s AI Assistant

+
+
+ {this.history.map((message, index) => ( + + ))} + {this.current_message && ( + + )} +
+
+ (this.inputValue = e.target.value)} /> + +
+ {/* Popup for citation */} + {this.citationPopup.visible && ( +
+

+ Text from your document: {this.citationPopup.text} +

+
+ )} +
+ ); + } +} + +/** + * Register the ChatBox component as the template for CHAT document types. + */ +Docs.Prototypes.TemplateMap.set(DocumentType.CHAT, { + layout: { view: ChatBox, dataField: 'data' }, + options: { acl: '', chat: '', chat_history: '', chat_thread_id: '', chat_assistant_id: '', chat_vector_store_id: '' }, +}); diff --git a/src/client/views/nodes/chatbot/chatboxcomponents/MessageComponent.tsx b/src/client/views/nodes/chatbot/chatboxcomponents/MessageComponent.tsx new file mode 100644 index 000000000..801becb64 --- /dev/null +++ b/src/client/views/nodes/chatbot/chatboxcomponents/MessageComponent.tsx @@ -0,0 +1,146 @@ +import React, { useState } from 'react'; +import { observer } from 'mobx-react'; +import { AssistantMessage, Citation, MessageContent, PROCESSING_TYPE, ProcessingInfo, TEXT_TYPE } from '../types/types'; +import ReactMarkdown from 'react-markdown'; + +/** + * Props for the MessageComponentBox. + * @interface MessageComponentProps + * @property {AssistantMessage} message - The message data to display. + * @property {number} index - The index of the message. + * @property {Function} onFollowUpClick - Callback to handle follow-up question clicks. + * @property {Function} onCitationClick - Callback to handle citation clicks. + * @property {Function} updateMessageCitations - Function to update message citations. + */ +interface MessageComponentProps { + message: AssistantMessage; + index: number; + onFollowUpClick: (question: string) => void; + onCitationClick: (citation: Citation) => void; + updateMessageCitations: (index: number, citations: Citation[]) => void; +} + +/** + * MessageComponentBox displays the content of an AssistantMessage including text, citations, + * processing information, and follow-up questions. + * @param {MessageComponentProps} props - The props for the component. + */ +const MessageComponentBox: React.FC = ({ message, index, onFollowUpClick, onCitationClick, updateMessageCitations }) => { + // State for managing whether the dropdown is open or closed for processing info + const [dropdownOpen, setDropdownOpen] = useState(false); + + /** + * Renders the content of the message based on the type (e.g., grounded text, normal text). + * @param {MessageContent} item - The content item to render. + * @returns {JSX.Element} JSX element rendering the content. + */ + const renderContent = (item: MessageContent) => { + const i = item.index; + + // Handle grounded text with citations + if (item.type === TEXT_TYPE.GROUNDED) { + const citation_ids = item.citation_ids || []; + return ( + + {item.text} + {citation_ids.map((id, idx) => { + const citation = message.citations?.find(c => c.citation_id === id); + if (!citation) return null; + return ( + + ); + })} + + ); + } + + // Handle normal text + else if (item.type === TEXT_TYPE.NORMAL) { + return ( + + {item.text} + + ); + } + + // Handle query type content + else if ('query' in item) { + return ( + + {JSON.stringify(item.query)} + + ); + } + + // Fallback for any other content type + else { + return ( + + {JSON.stringify(item)} + + ); + } + }; + + // Check if the message contains processing information (thoughts/actions) + const hasProcessingInfo = message.processing_info && message.processing_info.length > 0; + + /** + * Renders processing information such as thoughts or actions during message handling. + * @param {ProcessingInfo} info - The processing information to render. + * @returns {JSX.Element | null} JSX element rendering the processing info or null. + */ + const renderProcessingInfo = (info: ProcessingInfo) => { + if (info.type === PROCESSING_TYPE.THOUGHT) { + return ( +
+ Thought: {info.content} +
+ ); + } else if (info.type === PROCESSING_TYPE.ACTION) { + return ( +
+ Action: {info.content} +
+ ); + } + return null; + }; + + return ( +
+ {/* Processing Information Dropdown */} + {hasProcessingInfo && ( +
+ + {dropdownOpen &&
{message.processing_info.map(renderProcessingInfo)}
} +
+
+ )} + + {/* Message Content */} +
{message.content && message.content.map(messageFragment => {renderContent(messageFragment)})}
+ + {/* Follow-up Questions Section */} + {message.follow_up_questions && message.follow_up_questions.length > 0 && ( +
+

Follow-up Questions:

+
+ {message.follow_up_questions.map((question, idx) => ( + + ))} +
+
+ )} +
+ ); +}; + +// Export the observer-wrapped component to allow MobX to react to state changes +export default observer(MessageComponentBox); diff --git a/src/client/views/nodes/chatbot/chatboxcomponents/ProgressBar.scss b/src/client/views/nodes/chatbot/chatboxcomponents/ProgressBar.scss new file mode 100644 index 000000000..ff5be4a38 --- /dev/null +++ b/src/client/views/nodes/chatbot/chatboxcomponents/ProgressBar.scss @@ -0,0 +1,69 @@ +.spinner-container { + display: flex; + flex-direction: column; + justify-content: center; + align-items: center; + height: 100%; +} + +.spinner { + width: 60px; + height: 60px; + position: relative; + margin-bottom: 20px; // Space between spinner and text +} + +.double-bounce1, +.double-bounce2 { + width: 100%; + height: 100%; + border-radius: 50%; + background-color: #4a90e2; + opacity: 0.6; + position: absolute; + top: 0; + left: 0; + animation: bounce 2s infinite ease-in-out; +} + +.double-bounce2 { + animation-delay: -1s; +} + +@keyframes bounce { + 0%, + 100% { + transform: scale(0); + } + 50% { + transform: scale(1); + } +} + +.uploading-overlay { + position: absolute; + top: 0; + left: 0; + right: 0; + bottom: 0; + background-color: rgba(255, 255, 255, 0.8); + display: flex; + align-items: center; + justify-content: center; + z-index: 1000; +} + +.progress-container { + display: flex; + flex-direction: column; + align-items: center; + text-align: center; +} + +.step-name { + font-size: 18px; + color: #333; + text-align: center; + width: 100%; + margin-top: -10px; // Adjust to move the text closer to the spinner +} diff --git a/src/client/views/nodes/chatbot/chatboxcomponents/ProgressBar.tsx b/src/client/views/nodes/chatbot/chatboxcomponents/ProgressBar.tsx new file mode 100644 index 000000000..b9fd08742 --- /dev/null +++ b/src/client/views/nodes/chatbot/chatboxcomponents/ProgressBar.tsx @@ -0,0 +1,22 @@ +import React from 'react'; +import './ProgressBar.scss'; + +/** + * ProgressBar is a functional React component that displays a loading spinner + * to indicate progress or ongoing processing. It uses two bouncing elements + * to create a smooth animation that represents an active state. + * + * The animation consists of two divs (`double-bounce1` and `double-bounce2`), + * each of which will bounce in and out of view, creating a pulsating effect. + */ +export const ProgressBar: React.FC = () => { + return ( +
+ {/* Spinner div containing two bouncing elements */} +
+
{/* First bouncing element */} +
{/* Second bouncing element */} +
+
+ ); +}; diff --git a/src/client/views/nodes/chatbot/response_parsers/AnswerParser.ts b/src/client/views/nodes/chatbot/response_parsers/AnswerParser.ts new file mode 100644 index 000000000..3b4fdb6f5 --- /dev/null +++ b/src/client/views/nodes/chatbot/response_parsers/AnswerParser.ts @@ -0,0 +1,125 @@ +import { ASSISTANT_ROLE, AssistantMessage, Citation, CHUNK_TYPE, TEXT_TYPE, getChunkType, ProcessingInfo } from '../types/types'; +import { v4 as uuid } from 'uuid'; + +export class AnswerParser { + static parse(xml: string, processingInfo: ProcessingInfo[]): AssistantMessage { + const answerRegex = /([\s\S]*?)<\/answer>/; + const citationsRegex = /([\s\S]*?)<\/citations>/; + const citationRegex = /([\s\S]*?)<\/citation>/g; + const followUpQuestionsRegex = /([\s\S]*?)<\/follow_up_questions>/; + const questionRegex = /(.*?)<\/question>/g; + const groundedTextRegex = /([\s\S]*?)<\/grounded_text>/g; + const normalTextRegex = /([\s\S]*?)<\/normal_text>/g; + const loopSummaryRegex = /([\s\S]*?)<\/loop_summary>/; + + const answerMatch = answerRegex.exec(xml); + const citationsMatch = citationsRegex.exec(xml); + const followUpQuestionsMatch = followUpQuestionsRegex.exec(xml); + const loopSummaryMatch = loopSummaryRegex.exec(xml); + + if (!answerMatch) { + throw new Error('Invalid XML: Missing tag.'); + } + + let rawTextContent = answerMatch[1].trim(); + let content: AssistantMessage['content'] = []; + let citations: Citation[] = []; + let contentIndex = 0; + + // Remove citations and follow-up questions from rawTextContent + if (citationsMatch) { + rawTextContent = rawTextContent.replace(citationsMatch[0], '').trim(); + } + if (followUpQuestionsMatch) { + rawTextContent = rawTextContent.replace(followUpQuestionsMatch[0], '').trim(); + } + if (loopSummaryMatch) { + rawTextContent = rawTextContent.replace(loopSummaryMatch[0], '').trim(); + } + + // Parse citations + let citationMatch; + const citationMap = new Map(); + if (citationsMatch) { + const citationsContent = citationsMatch[1]; + while ((citationMatch = citationRegex.exec(citationsContent)) !== null) { + const [_, index, chunk_id, type, direct_text] = citationMatch; + const citation_id = uuid(); + citationMap.set(index, citation_id); + citations.push({ + direct_text: direct_text.trim(), + type: getChunkType(type), + chunk_id, + citation_id, + }); + } + } + + rawTextContent = rawTextContent.replace(normalTextRegex, '$1'); + + // Parse text content (normal and grounded) + let lastIndex = 0; + let match; + + while ((match = groundedTextRegex.exec(rawTextContent)) !== null) { + const [fullMatch, citationIndex, groundedText] = match; + + // Add normal text that is before the grounded text + if (match.index > lastIndex) { + const normalText = rawTextContent.slice(lastIndex, match.index).trim(); + if (normalText) { + content.push({ + index: contentIndex++, + type: TEXT_TYPE.NORMAL, + text: normalText, + citation_ids: null, + }); + } + } + + // Add grounded text + const citation_ids = citationIndex.split(',').map(index => citationMap.get(index) || ''); + content.push({ + index: contentIndex++, + type: TEXT_TYPE.GROUNDED, + text: groundedText.trim(), + citation_ids, + }); + + lastIndex = match.index + fullMatch.length; + } + + // Add any remaining normal text after the last grounded text + if (lastIndex < rawTextContent.length) { + const remainingText = rawTextContent.slice(lastIndex).trim(); + if (remainingText) { + content.push({ + index: contentIndex++, + type: TEXT_TYPE.NORMAL, + text: remainingText, + citation_ids: null, + }); + } + } + + let followUpQuestions: string[] = []; + if (followUpQuestionsMatch) { + const questionsText = followUpQuestionsMatch[1]; + let questionMatch; + while ((questionMatch = questionRegex.exec(questionsText)) !== null) { + followUpQuestions.push(questionMatch[1].trim()); + } + } + + const assistantResponse: AssistantMessage = { + role: ASSISTANT_ROLE.ASSISTANT, + content, + follow_up_questions: followUpQuestions, + citations, + processing_info: processingInfo, + loop_summary: loopSummaryMatch ? loopSummaryMatch[1].trim() : undefined, + }; + + return assistantResponse; + } +} diff --git a/src/client/views/nodes/chatbot/response_parsers/StreamedAnswerParser.ts b/src/client/views/nodes/chatbot/response_parsers/StreamedAnswerParser.ts new file mode 100644 index 000000000..3585cab4a --- /dev/null +++ b/src/client/views/nodes/chatbot/response_parsers/StreamedAnswerParser.ts @@ -0,0 +1,73 @@ +import { threadId } from 'worker_threads'; + +enum ParserState { + Outside, + InGroundedText, + InNormalText, +} + +export class StreamedAnswerParser { + private state: ParserState = ParserState.Outside; + private buffer: string = ''; + private result: string = ''; + private isStartOfLine: boolean = true; + + public parse(char: string): string { + switch (this.state) { + case ParserState.Outside: + if (char === '<') { + this.buffer = '<'; + } else if (char === '>') { + if (this.buffer.startsWith('') { + this.state = ParserState.Outside; + this.buffer = ''; + } else if (this.buffer.startsWith('') { + this.state = ParserState.Outside; + this.buffer = ''; + } else if (this.buffer.startsWith('<')) { + this.buffer += char; + } else { + this.processChar(char); + } + break; + } + + return this.result.trim(); + } + + private processChar(char: string): void { + if (this.isStartOfLine && char === ' ') { + // Skip leading spaces + return; + } + if (char === '\n') { + this.result += char; + this.isStartOfLine = true; + } else { + this.result += char; + this.isStartOfLine = false; + } + } + + public reset(): void { + this.state = ParserState.Outside; + this.buffer = ''; + this.result = ''; + this.isStartOfLine = true; + } +} diff --git a/src/client/views/nodes/chatbot/tools/BaseTool.ts b/src/client/views/nodes/chatbot/tools/BaseTool.ts new file mode 100644 index 000000000..b57f1c8e4 --- /dev/null +++ b/src/client/views/nodes/chatbot/tools/BaseTool.ts @@ -0,0 +1,24 @@ +import { Tool } from '../types/types'; + +export abstract class BaseTool = Record> implements Tool { + constructor( + public name: string, + public description: string, + public parameters: Record, + public citationRules: string, + public briefSummary: string + ) {} + + abstract execute(args: T): Promise; + + getActionRule(): Record { + return { + [this.name]: { + name: this.name, + citationRules: this.citationRules, + description: this.description, + parameters: this.parameters, + }, + }; + } +} diff --git a/src/client/views/nodes/chatbot/tools/CalculateTool.ts b/src/client/views/nodes/chatbot/tools/CalculateTool.ts new file mode 100644 index 000000000..74b7ca27b --- /dev/null +++ b/src/client/views/nodes/chatbot/tools/CalculateTool.ts @@ -0,0 +1,26 @@ +import { BaseTool } from './BaseTool'; + +export class CalculateTool extends BaseTool<{ expression: string }> { + constructor() { + super( + 'calculate', + 'Perform a calculation', + { + expression: { + type: 'string', + description: 'The mathematical expression to evaluate', + required: 'true', + max_inputs: '1', + }, + }, + 'Provide a mathematical expression to calculate that would work with JavaScript eval().', + 'Runs a calculation and returns the number - uses JavaScript so be sure to use floating point syntax if necessary' + ); + } + + async execute(args: { expression: string }): Promise { + // Note: Using eval() can be dangerous. Consider using a safer alternative. + const result = eval(args.expression); + return [{ type: 'text', text: result.toString() }]; + } +} diff --git a/src/client/views/nodes/chatbot/tools/CreateCSVTool.ts b/src/client/views/nodes/chatbot/tools/CreateCSVTool.ts new file mode 100644 index 000000000..55015846b --- /dev/null +++ b/src/client/views/nodes/chatbot/tools/CreateCSVTool.ts @@ -0,0 +1,51 @@ +import { BaseTool } from './BaseTool'; +import { Networking } from '../../../../Network'; + +export class CreateCSVTool extends BaseTool<{ csvData: string; filename: string }> { + private _handleCSVResult: (url: string, filename: string, id: string, data: string) => void; + + constructor(handleCSVResult: (url: string, title: string, id: string, data: string) => void) { + super( + 'createCSV', + 'Creates a CSV file from raw CSV data and saves it to the server', + { + type: 'object', + properties: { + csvData: { + type: 'string', + description: 'A string of comma-separated values representing the CSV data.', + }, + filename: { + type: 'string', + description: 'The base name of the CSV file to be created. Should end in ".csv".', + }, + }, + required: ['csvData', 'filename'], + }, + 'Provide a CSV string and a filename to create a CSV file.', + 'Creates a CSV file from the provided CSV string and saves it to the server with a unique identifier, returning the file URL and UUID.' + ); + this._handleCSVResult = handleCSVResult; + } + + async execute(args: { csvData: string; filename: string }): Promise { + try { + console.log('Creating CSV file:', args.filename, ' with data:', args.csvData); + // Post the raw CSV data to the createCSV endpoint on the server + const { fileUrl, id } = await Networking.PostToServer('/createCSV', { filename: args.filename, data: args.csvData }); + + // Handle the result by invoking the callback + this._handleCSVResult(fileUrl, args.filename, id, args.csvData); + + return [ + { + type: 'text', + text: `File successfully created: ${fileUrl}. \nNow a CSV file with this data and the name ${args.filename} is available as a user doc.`, + }, + ]; + } catch (error) { + console.error('Error creating CSV file:', error); + throw new Error('Failed to create CSV file.'); + } + } +} diff --git a/src/client/views/nodes/chatbot/tools/CreateCollectionTool.ts b/src/client/views/nodes/chatbot/tools/CreateCollectionTool.ts new file mode 100644 index 000000000..573428179 --- /dev/null +++ b/src/client/views/nodes/chatbot/tools/CreateCollectionTool.ts @@ -0,0 +1,36 @@ +import { DocCast } from '../../../../../fields/Types'; +import { DocServer } from '../../../../DocServer'; +import { Docs } from '../../../../documents/Documents'; +import { DocumentView } from '../../DocumentView'; +import { OpenWhere } from '../../OpenWhere'; +import { BaseTool } from './BaseTool'; + +export class GetDocsContentTool extends BaseTool<{ title: string; document_ids: string[] }> { + private _docView: DocumentView; + constructor(docView: DocumentView) { + super( + 'retrieveDocs', + 'Retrieves the contents of all Documents that the user is interacting with in Dash ', + { + title: { + type: 'string', + description: 'the title of the collection that you will be making', + required: 'true', + max_inputs: '1', + }, + }, + 'Provide a mathematical expression to calculate that would work with JavaScript eval().', + 'Runs a calculation and returns the number - uses JavaScript so be sure to use floating point syntax if necessary' + ); + this._docView = docView; + } + + async execute(args: { title: string; document_ids: string[] }): Promise { + // Note: Using eval() can be dangerous. Consider using a safer alternative. + const docs = args.document_ids.map(doc_id => DocCast(DocServer.GetCachedRefField(doc_id))); + const collection = Docs.Create.FreeformDocument(docs, { title: args.title }); + this._docView._props.addDocTab(collection, OpenWhere.addRight); //in future, create popup prompting user where to add + return [{ type: 'text', text: 'Collection created in Dash called ' + args.title }]; + } +} +//export function create_collection(docView: DocumentView, document_ids: string[], title: string): string {} diff --git a/src/client/views/nodes/chatbot/tools/DataAnalysisTool.ts b/src/client/views/nodes/chatbot/tools/DataAnalysisTool.ts new file mode 100644 index 000000000..a12ee46e5 --- /dev/null +++ b/src/client/views/nodes/chatbot/tools/DataAnalysisTool.ts @@ -0,0 +1,59 @@ +import { BaseTool } from './BaseTool'; + +export class DataAnalysisTool extends BaseTool<{ csv_file_name: string | string[] }> { + private csv_files_function: () => { filename: string; id: string; text: string }[]; + + constructor(csv_files: () => { filename: string; id: string; text: string }[]) { + super( + 'dataAnalysis', + 'Analyzes, and provides insights, from one or more CSV files', + { + csv_file_name: { + type: 'string', + description: 'Name(s) of the CSV file(s) to analyze', + required: 'true', + max_inputs: '3', + }, + }, + 'Provide the name(s) of up to 3 CSV files to analyze based on the user query and whichever available CSV files may be relevant.', + 'Provides the full CSV file text for your analysis based on the user query and the available CSV file(s). ' + ); + this.csv_files_function = csv_files; + } + + getFileContent(filename: string): string | undefined { + const files = this.csv_files_function(); + const file = files.find(f => f.filename === filename); + return file?.text; + } + + getFileID(filename: string): string | undefined { + const files = this.csv_files_function(); + const file = files.find(f => f.filename === filename); + return file?.id; + } + + async execute(args: { csv_file_name: string | string[] }): Promise { + const filenames = Array.isArray(args.csv_file_name) ? args.csv_file_name : [args.csv_file_name]; + const results = []; + + for (const filename of filenames) { + const fileContent = this.getFileContent(filename); + const fileID = this.getFileID(filename); + + if (fileContent && fileID) { + results.push({ + type: 'text', + text: `${fileContent}`, + }); + } else { + results.push({ + type: 'text', + text: `File not found: ${filename}`, + }); + } + } + + return results; + } +} diff --git a/src/client/views/nodes/chatbot/tools/GetDocsTool.ts b/src/client/views/nodes/chatbot/tools/GetDocsTool.ts new file mode 100644 index 000000000..f970ca8ee --- /dev/null +++ b/src/client/views/nodes/chatbot/tools/GetDocsTool.ts @@ -0,0 +1,29 @@ +import { DocCast } from '../../../../../fields/Types'; +import { DocServer } from '../../../../DocServer'; +import { Docs } from '../../../../documents/Documents'; +import { DocumentView } from '../../DocumentView'; +import { OpenWhere } from '../../OpenWhere'; +import { BaseTool } from './BaseTool'; + +export class GetDocsTool extends BaseTool<{ title: string; document_ids: string[] }> { + private _docView: DocumentView; + constructor(docView: DocumentView) { + super( + 'retrieveDocs', + 'Retrieves the contents of all Documents that the user is interacting with in Dash', + {}, + 'No need to provide anything. Just run the tool and it will retrieve the contents of all Documents that the user is interacting with in Dash.', + 'Returns the the documents in Dash in JSON form. This will include the title of the document, the location in the FreeFormDocument, and the content of the document, any applicable data fields, the layout of the document, etc.' + ); + this._docView = docView; + } + + async execute(args: { title: string; document_ids: string[] }): Promise { + // Note: Using eval() can be dangerous. Consider using a safer alternative. + const docs = args.document_ids.map(doc_id => DocCast(DocServer.GetCachedRefField(doc_id))); + const collection = Docs.Create.FreeformDocument(docs, { title: args.title }); + this._docView._props.addDocTab(collection, OpenWhere.addRight); //in future, create popup prompting user where to add + return [{ type: 'text', text: 'Collection created in Dash called ' + args.title }]; + } +} +//export function create_collection(docView: DocumentView, document_ids: string[], title: string): string {} diff --git a/src/client/views/nodes/chatbot/tools/NoTool.ts b/src/client/views/nodes/chatbot/tools/NoTool.ts new file mode 100644 index 000000000..1f0830a77 --- /dev/null +++ b/src/client/views/nodes/chatbot/tools/NoTool.ts @@ -0,0 +1,18 @@ +// tools/NoTool.ts +import { BaseTool } from './BaseTool'; + +export class NoTool extends BaseTool<{}> { + constructor() { + super( + 'no_tool', + 'Use this when no external tool or action is required to answer the question.', + {}, + 'When using the "no_tool" action, simply provide an empty element. The observation will always be "No tool used. Proceed with answering the question."', + 'Use when no external tool or action is required to answer the question.' + ); + } + + async execute(args: {}): Promise { + return [{ type: 'text', text: 'No tool used. Proceed with answering the question.' }]; + } +} diff --git a/src/client/views/nodes/chatbot/tools/RAGTool.ts b/src/client/views/nodes/chatbot/tools/RAGTool.ts new file mode 100644 index 000000000..c24306dcd --- /dev/null +++ b/src/client/views/nodes/chatbot/tools/RAGTool.ts @@ -0,0 +1,138 @@ +import { BaseTool } from './BaseTool'; +import { Vectorstore } from '../vectorstore/Vectorstore'; +import { RAGChunk } from '../types/types'; +import * as fs from 'fs'; +import { Networking } from '../../../../Network'; +import { file } from 'jszip'; +import { ChatCompletion, ChatCompletionContentPart, ChatCompletionMessageParam } from 'openai/resources'; + +export class RAGTool extends BaseTool { + constructor(private vectorstore: Vectorstore) { + super( + 'rag', + 'Perform a RAG search on user documents', + { + hypothetical_document_chunk: { + type: 'string', + description: + "Detailed version of the prompt that is effectively a hypothetical document chunk that would be ideal to embed and compare to the vectors of real document chunks to fetch the most relevant document chunks to answer the user's query", + required: 'true', + }, + }, + ` + Your task is to provide a comprehensive response to the user's prompt based on the given chunks and chat history. Follow these structural guidelines meticulously: + + 1. Overall Structure: + + [Main content with grounded_text tags interspersed with normal plain text (information that is not derived from chunks' information)] + + [Individual citation tags] + + + [Three question tags] + + + + 2. Grounded Text Tag Structure: + - Basic format: + + [Your generated text based on information from a subset of a chunk (a citation's direct text)] + + + 3. Citation Tag Structure: + + [For text: relevant subset of original chunk] + [For image/table: leave empty] + + + 4. Detailed Grounded Text Guidelines: + a. Wrap all information derived from chunks in grounded_text tags. + b. DO NOT PUT ANYTHING THAT IS NOT DIRECTLY DERIVED FROM INFORMATION FROM CHUNKS (EITHER IMAGE, TABLE, OR TEXT) IN GROUNDED_TEXT TAGS. + c. Use a single grounded_text tag for suquential and closely related information that references the same citation. If other citations' information are used sequentially, create new grounded_text tags. + d. Ensure every grounded_text tag has up to a few corresponding citations (should not be more than 3 and only 1 is fine). Multiple citation indices should be separated by commas. + e. Grounded text can be as short as a few words or as long as several sentences. + f. Avoid overlapping or nesting grounded_text tags; instead, use sequential tags. + + 5. Detailed Citation Guidelines: + a. Create a unique citation for each distinct piece of information from the chunks that is used to support grounded_text. + b. ALL TEXT CITATIONS must have direct text in its element content (e.g. DIRECT TEXT HERE) that is a relevant SUBSET of the original text chunk that is being cited specifically. + c. DO NOT paraphrase or summarize the text; use the original text as much as possible. + d. DO NOT USE THE FULL TEXT CHUNK as the citation content; only use the relevant subset of the text that the grounded_text is base. AS SHORT AS POSSIBLE WHILE PROVIDING INFORMATION (ONE TO TWO SENTENCES USUALLY)! + e. Ensure each citation has a unique index number. + f. Specify the correct type: "text", "image", or "table". + g. For text chunks, the content of the citation should ALWAYS have the relevant subset of the original text that the grounded_text is based on. + h. For image/table chunks, leave the citation content empty. + i. One citation can be used for multiple grounded_text tags if they are based on the same chunk information. + j. !!!DO NOT OVERCITE - only include citations for information that is directly relevant to the grounded_text. + + 6. Structural Integrity Checks: + a. Ensure all opening tags have corresponding closing tags. + b. Verify that all grounded_text tags have valid citation_index attributes (they should be equal to the associated citation(s) index field—not their chunk_id field). + c. Check that all cited indices in grounded_text tags have corresponding citations. + + Example of grounded_text usage: + + + + Artificial Intelligence (AI) is revolutionizing various sectors, with healthcare experiencing significant transformations in areas such as diagnosis and treatment planning. + + + In the field of medical diagnosis, AI has shown remarkable capabilities, particularly in radiology. For instance, AI systems have drastically improved mammogram analysis, achieving 99% accuracy at a rate 30 times faster than human radiologists. + + + This advancement not only enhances the efficiency of healthcare systems but also significantly reduces the occurrence of false positives, leading to fewer unnecessary biopsies and reduced patient stress. + + + + Beyond diagnosis, AI is playing a crucial role in drug discovery and development. By analyzing vast amounts of genetic and molecular data, AI algorithms can identify potential drug candidates much faster than traditional methods. + + + This could potentially reduce the time and cost of bringing new medications to market, especially for rare diseases that have historically received less attention due to limited market potential. + + + [... rest of the content ...] + + + Artificial Intelligence is revolutionizing various industries, with healthcare being one of the most profoundly affected sectors. + AI has shown particular promise in the field of radiology, enhancing the accuracy and speed of image analysis. + According to recent studies, AI systems have achieved 99% accuracy in mammogram analysis, performing the task 30 times faster than human radiologists. + The improvement in mammogram accuracy has led to a significant reduction in false positives, decreasing the need for unnecessary biopsies and reducing patient anxiety. + AI is accelerating the drug discovery process by analyzing complex molecular and genetic data to identify potential drug candidates. + The use of AI in drug discovery could significantly reduce the time and cost associated with bringing new medications to market, particularly for rare diseases. + + + + How might AI-driven personalized medicine impact the cost and accessibility of healthcare in the future? + What measures can be taken to ensure that AI systems in healthcare are free from biases and equally effective for diverse populations? + How could the role of healthcare professionals evolve as AI becomes more integrated into medical practices? + + + `, + + `Performs a RAG (Retrieval-Augmented Generation) search on user documents and returns a + set of document chunks (either images or text) that can be used to provide a grounded response based on + user documents` + ); + } + + async execute(args: { hypothetical_document_chunk: string }): Promise { + const relevantChunks = await this.vectorstore.retrieve(args.hypothetical_document_chunk); + const formatted_chunks = await this.getFormattedChunks(relevantChunks); + return formatted_chunks; + } + + async getFormattedChunks(relevantChunks: RAGChunk[]): Promise { + try { + const { formattedChunks } = await Networking.PostToServer('/formatChunks', { relevantChunks }); + + if (!formattedChunks) { + throw new Error('Failed to format chunks'); + } + + return formattedChunks; + } catch (error) { + console.error('Error formatting chunks:', error); + throw error; + } + } +} diff --git a/src/client/views/nodes/chatbot/tools/SearchTool.ts b/src/client/views/nodes/chatbot/tools/SearchTool.ts new file mode 100644 index 000000000..b926cbadc --- /dev/null +++ b/src/client/views/nodes/chatbot/tools/SearchTool.ts @@ -0,0 +1,54 @@ +import { max } from 'lodash'; +import { Networking } from '../../../../Network'; +import { BaseTool } from './BaseTool'; +import { v4 as uuidv4 } from 'uuid'; + +export class SearchTool extends BaseTool<{ query: string | string[] }> { + private _addLinkedUrlDoc: (url: string, id: string) => void; + private _max_results: number; + constructor(addLinkedUrlDoc: (url: string, id: string) => void, max_results: number = 5) { + super( + 'searchTool', + 'Search the web to find a wide range of websites related to a query or multiple queries', + { + query: { + type: 'string', + description: 'The search query or queries to use for finding websites', + required: 'true', + max_inputs: '3', + }, + }, + 'Provide up to 3 search queries to find a broad range of websites. This tool is intended to help you identify relevant websites, but not to be used for providing the final answer. Use this information to determine which specific website to investigate further.', + 'Returns a list of websites and their overviews based on the search queries, helping to identify which websites might contain relevant information.' + ); + this._addLinkedUrlDoc = addLinkedUrlDoc; + this._max_results = max_results; + } + + async execute(args: { query: string | string[] }): Promise { + const queries = Array.isArray(args.query) ? args.query : [args.query]; + const allResults = []; + + for (const query of queries) { + try { + const { results } = await Networking.PostToServer('/getWebSearchResults', { query, max_results: this._max_results }); + const data: { type: string; text: string }[] = results.map((result: { url: string; snippet: string }) => { + const id = uuidv4(); + return { + type: 'text', + text: ` + ${result.url} + ${result.snippet} + `, + }; + }); + allResults.push(...data); + } catch (error) { + console.log(error); + allResults.push({ type: 'text', text: `An error occurred while performing the web search for query: ${query}` }); + } + } + + return allResults; + } +} diff --git a/src/client/views/nodes/chatbot/tools/WebsiteInfoScraperTool.ts b/src/client/views/nodes/chatbot/tools/WebsiteInfoScraperTool.ts new file mode 100644 index 000000000..2118218f6 --- /dev/null +++ b/src/client/views/nodes/chatbot/tools/WebsiteInfoScraperTool.ts @@ -0,0 +1,83 @@ +import { Networking } from '../../../../Network'; +import { BaseTool } from './BaseTool'; +import { v4 as uuidv4 } from 'uuid'; + +export class WebsiteInfoScraperTool extends BaseTool<{ url: string | string[] }> { + private _addLinkedUrlDoc: (url: string, id: string) => void; + + constructor(addLinkedUrlDoc: (url: string, id: string) => void) { + super( + 'websiteInfoScraper', + 'Scrape detailed information from specific websites relevant to the user query', + { + url: { + type: 'string', + description: 'The URL(s) of the website(s) to scrape', + required: true, + max_inputs: 3, + }, + }, + ` + Your task is to provide a comprehensive response to the user's prompt using the content scraped from relevant websites. Ensure you follow these guidelines for structuring your response: + + 1. Grounded Text Tag Structure: + - Wrap all text derived from the scraped website(s) in tags. + - **Do not include non-sourced information** in tags. + - Use a single tag for content derived from a single website. If citing multiple websites, create new tags for each. + - Ensure each tag has a citation index corresponding to the scraped URL. + + 2. Citation Tag Structure: + - Create a tag for each distinct piece of information used from the website(s). + - Each tag must reference a URL chunk using the chunk_id attribute. + - For URL-based citations, leave the citation content empty, but reference the chunk_id and type as 'url'. + + 3. Structural Integrity Checks: + - Ensure all opening and closing tags are matched properly. + - Verify that all citation_index attributes in tags correspond to valid citations. + - Do not over-cite—cite only the most relevant parts of the websites. + + Example Usage: + + + + Based on data from the World Bank, economic growth has stabilized in recent years, following a surge in investments. + + + According to information retrieved from the International Monetary Fund, the inflation rate has been gradually decreasing since 2020. + + + + + + + + + What are the long-term economic impacts of increased investments on GDP? + How might inflation trends affect future monetary policy? + Are there additional factors that could influence economic growth beyond investments and inflation? + + + `, + 'Returns the text content of the webpages for further analysis and grounding.' + ); + this._addLinkedUrlDoc = addLinkedUrlDoc; + } + + async execute(args: { url: string | string[] }): Promise { + const urls = Array.isArray(args.url) ? args.url : [args.url]; + const results = []; + + for (const url of urls) { + try { + const { website_plain_text } = await Networking.PostToServer('/scrapeWebsite', { url }); + const id = uuidv4(); + this._addLinkedUrlDoc(url, id); + results.push({ type: 'text', text: `\n${website_plain_text}\n\n` }); + } catch (error) { + results.push({ type: 'text', text: `An error occurred while scraping the website: ${url}` }); + } + } + + return results; + } +} diff --git a/src/client/views/nodes/chatbot/tools/WikipediaTool.ts b/src/client/views/nodes/chatbot/tools/WikipediaTool.ts new file mode 100644 index 000000000..143d91d80 --- /dev/null +++ b/src/client/views/nodes/chatbot/tools/WikipediaTool.ts @@ -0,0 +1,37 @@ +import { title } from 'process'; +import { Networking } from '../../../../Network'; +import { BaseTool } from './BaseTool'; +import axios from 'axios'; +import { v4 as uuidv4 } from 'uuid'; + +export class WikipediaTool extends BaseTool<{ title: string }> { + private _addLinkedUrlDoc: (url: string, id: string) => void; + constructor(addLinkedUrlDoc: (url: string, id: string) => void) { + super( + 'wikipedia', + 'Search Wikipedia and return a summary', + { + title: { + type: 'string', + description: 'The title of the Wikipedia article to search', + required: true, + }, + }, + 'Provide simply the title you want to search on Wikipedia and nothing more. If re-using this tool, try a different title for different information.', + 'Returns a summary from searching an article title on Wikipedia' + ); + this._addLinkedUrlDoc = addLinkedUrlDoc; + } + + async execute(args: { title: string }): Promise { + try { + const { text } = await Networking.PostToServer('/getWikipediaSummary', { title: args.title }); + const id = uuidv4(); + const url = `https://en.wikipedia.org/wiki/${args.title.replace(/ /g, '_')}`; + this._addLinkedUrlDoc(url, id); + return [{ type: 'text', text: ` ${text} ` }]; + } catch (error) { + return [{ type: 'text', text: 'An error occurred while fetching the article.' }]; + } + } +} diff --git a/src/client/views/nodes/chatbot/types/types.ts b/src/client/views/nodes/chatbot/types/types.ts new file mode 100644 index 000000000..f5d14ad6a --- /dev/null +++ b/src/client/views/nodes/chatbot/types/types.ts @@ -0,0 +1,129 @@ +export enum ASSISTANT_ROLE { + USER = 'user', + ASSISTANT = 'assistant', +} + +export enum TEXT_TYPE { + NORMAL = 'normal', + GROUNDED = 'grounded', + ERROR = 'error', +} + +export enum CHUNK_TYPE { + TEXT = 'text', + IMAGE = 'image', + TABLE = 'table', + URL = 'url', + CSV = 'CSV', +} + +export enum PROCESSING_TYPE { + THOUGHT = 'thought', + ACTION = 'action', + //eventually migrate error to here +} + +export function getChunkType(type: string): CHUNK_TYPE { + switch (type.toLowerCase()) { + case 'text': + return CHUNK_TYPE.TEXT; + break; + case 'image': + return CHUNK_TYPE.IMAGE; + break; + case 'table': + return CHUNK_TYPE.TABLE; + break; + case 'CSV': + return CHUNK_TYPE.CSV; + break; + case 'url': + return CHUNK_TYPE.URL; + break; + default: + return CHUNK_TYPE.TEXT; + break; + } +} + +export interface ProcessingInfo { + index: number; + type: PROCESSING_TYPE; + content: string; +} + +export interface AssistantMessage { + role: ASSISTANT_ROLE; + content: MessageContent[]; + follow_up_questions?: string[]; + citations?: Citation[]; + processing_info: ProcessingInfo[]; + loop_summary?: string; +} + +export interface MessageContent { + index: number; + type: TEXT_TYPE; + text: string; + citation_ids: string[] | null; +} + +export interface Citation { + direct_text?: string; + type: CHUNK_TYPE; + chunk_id: string; + citation_id: string; + url?: string; +} + +export interface RAGChunk { + id: string; + values: number[]; + metadata: { + text: string; + type: CHUNK_TYPE; + original_document: string; + file_path: string; + doc_id: string; + location: string; + start_page: number; + end_page: number; + base64_data?: string | undefined; + page_width?: number | undefined; + page_height?: number | undefined; + }; +} + +export interface SimplifiedChunk { + chunkId: string; + startPage: number; + endPage: number; + location?: string; + chunkType: CHUNK_TYPE; + url?: string; + canDisplay?: boolean; +} + +export interface AI_Document { + purpose: string; + file_name: string; + num_pages: number; + summary: string; + chunks: RAGChunk[]; + type: string; +} + +export interface Tool = Record> { + name: string; + description: string; + parameters: Record; + citationRules: string; + briefSummary: string; + execute: (args: T) => Promise; + getActionRule: () => Record; +} + +export interface AgentMessage { + role: 'system' | 'user' | 'assistant'; + content: string | { type: string; text?: string; image_url?: { url: string } }[]; +} diff --git a/src/client/views/nodes/chatbot/vectorstore/Vectorstore.ts b/src/client/views/nodes/chatbot/vectorstore/Vectorstore.ts new file mode 100644 index 000000000..07a2b73bc --- /dev/null +++ b/src/client/views/nodes/chatbot/vectorstore/Vectorstore.ts @@ -0,0 +1,258 @@ +import { Pinecone, Index, IndexList, PineconeRecord, RecordMetadata, QueryResponse } from '@pinecone-database/pinecone'; +import { CohereClient } from 'cohere-ai'; +import { EmbedResponse } from 'cohere-ai/api'; +import dotenv from 'dotenv'; +import { RAGChunk, AI_Document, CHUNK_TYPE } from '../types/types'; +import { Doc } from '../../../../../fields/Doc'; +import { CsvCast, PDFCast, StrCast } from '../../../../../fields/Types'; +import { Networking } from '../../../../Network'; + +dotenv.config(); + +/** + * The Vectorstore class integrates with Pinecone for vector-based document indexing and retrieval, + * and Cohere for text embedding. It handles AI document management, uploads, and query-based retrieval. + */ +export class Vectorstore { + private pinecone: Pinecone; // Pinecone client for managing the vector index. + private index!: Index; // The specific Pinecone index used for document chunks. + private cohere: CohereClient; // Cohere client for generating embeddings. + private indexName: string = 'pdf-chatbot'; // Default name for the index. + private _id: string; // Unique ID for the Vectorstore instance. + private _doc_ids: string[] = []; // List of document IDs handled by this instance. + + documents: AI_Document[] = []; // Store the documents indexed in the vectorstore. + + /** + * Constructor initializes the Pinecone and Cohere clients, sets up the document ID list, + * and initializes the Pinecone index. + * @param id The unique identifier for the vectorstore instance. + * @param doc_ids A function that returns a list of document IDs. + */ + constructor(id: string, doc_ids: () => string[]) { + const pineconeApiKey = process.env.PINECONE_API_KEY; + if (!pineconeApiKey) { + throw new Error('PINECONE_API_KEY is not defined.'); + } + + // Initialize Pinecone and Cohere clients with API keys from the environment. + this.pinecone = new Pinecone({ apiKey: pineconeApiKey }); + this.cohere = new CohereClient({ token: process.env.COHERE_API_KEY }); + this._id = id; + this._doc_ids = doc_ids(); + this.initializeIndex(); + } + + /** + * Initializes the Pinecone index by checking if it exists, and creating it if not. + * The index is set to use the cosine metric for vector similarity. + */ + private async initializeIndex() { + const indexList: IndexList = await this.pinecone.listIndexes(); + + // Check if the index already exists, otherwise create it. + if (!indexList.indexes?.some(index => index.name === this.indexName)) { + await this.pinecone.createIndex({ + name: this.indexName, + dimension: 1024, + metric: 'cosine', + spec: { + serverless: { + cloud: 'aws', + region: 'us-east-1', + }, + }, + }); + } + + // Set the index for future use. + this.index = this.pinecone.Index(this.indexName); + } + + /** + * Adds an AI document to the vectorstore. This method handles document chunking, uploading to the + * vectorstore, and updating the progress for long-running tasks like file uploads. + * @param doc The document to be added to the vectorstore. + * @param progressCallback Callback to update the progress of the upload. + */ + async addAIDoc(doc: Doc, progressCallback: (progress: number, step: string) => void) { + console.log('Adding AI Document:', doc); + const ai_document_status: string = StrCast(doc.ai_document_status); + + // Skip if the document is already in progress or completed. + if (ai_document_status !== undefined && ai_document_status.trim() !== '' && ai_document_status !== '{}') { + if (ai_document_status === 'IN PROGRESS') { + console.log('Already in progress.'); + return; + } + if (!this._doc_ids.includes(StrCast(doc.ai_doc_id))) { + this._doc_ids.push(StrCast(doc.ai_doc_id)); + } + } else { + // Start processing the document. + doc.ai_document_status = 'PROGRESS'; + console.log(doc); + + // Get the local file path (CSV or PDF). + const local_file_path: string = CsvCast(doc.data)?.url?.pathname ?? PDFCast(doc.data)?.url?.pathname; + console.log('Local File Path:', local_file_path); + + if (local_file_path) { + console.log('Creating AI Document...'); + // Start the document creation process by sending the file to the server. + const { jobId } = await Networking.PostToServer('/createDocument', { file_path: local_file_path }); + + // Poll the server for progress updates. + let inProgress: boolean = true; + let result: any = null; + while (inProgress) { + // Polling interval for status updates. + await new Promise(resolve => setTimeout(resolve, 2000)); + + // Check if the job is completed. + const resultResponse = await Networking.FetchFromServer(`/getResult/${jobId}`); + const resultResponseJson = JSON.parse(resultResponse); + if (resultResponseJson.status === 'completed') { + console.log('Result here:', resultResponseJson); + result = resultResponseJson; + break; + } + + // Fetch progress information and update the progress callback. + const progressResponse = await Networking.FetchFromServer(`/getProgress/${jobId}`); + const progressResponseJson = JSON.parse(progressResponse); + if (progressResponseJson) { + const progress = progressResponseJson.progress; + const step = progressResponseJson.step; + progressCallback(progress, step); + } + } + + // Once completed, process the document and add it to the vectorstore. + console.log('Document JSON:', result); + this.documents.push(result); + await this.indexDocument(result); + console.log(`Document added: ${result.file_name}`); + + // Update document metadata such as summary, purpose, and vectorstore ID. + doc.summary = result.summary; + doc.ai_doc_id = result.doc_id; + this._doc_ids.push(result.doc_id); + doc.ai_purpose = result.purpose; + + if (!doc.vectorstore_id) { + doc.vectorstore_id = JSON.stringify([this._id]); + } else { + doc.vectorstore_id = JSON.stringify(JSON.parse(StrCast(doc.vectorstore_id)).concat([this._id])); + } + + if (!doc.chunk_simpl) { + doc.chunk_simpl = JSON.stringify({ chunks: [] }); + } + + // Process each chunk of the document and update the document's chunk_simpl field. + result.chunks.forEach((chunk: RAGChunk) => { + const chunkToAdd = { + chunkId: chunk.id, + startPage: chunk.metadata.start_page, + endPage: chunk.metadata.end_page, + location: chunk.metadata.location, + chunkType: chunk.metadata.type as CHUNK_TYPE, + text: chunk.metadata.text, + }; + const new_chunk_simpl = JSON.parse(StrCast(doc.chunk_simpl)); + new_chunk_simpl.chunks = new_chunk_simpl.chunks.concat(chunkToAdd); + doc.chunk_simpl = JSON.stringify(new_chunk_simpl); + }); + + // Mark the document status as completed. + doc.ai_document_status = 'COMPLETED'; + } + } + } + + /** + * Indexes the processed document by uploading the document's vector chunks to the Pinecone index. + * @param document The processed document containing its chunks and metadata. + */ + private async indexDocument(document: any) { + console.log('Uploading vectors to content namespace...'); + + // Prepare Pinecone records for each chunk in the document. + const pineconeRecords: PineconeRecord[] = (document.chunks as RAGChunk[]).map(chunk => ({ + id: chunk.id, + values: chunk.values, + metadata: { ...chunk.metadata } as RecordMetadata, + })); + + // Upload the records to Pinecone. + await this.index.upsert(pineconeRecords); + } + + /** + * Retrieves the top K document chunks relevant to the user's query. + * This involves embedding the query using Cohere, then querying Pinecone for matching vectors. + * @param query The search query string. + * @param topK The number of top results to return (default is 10). + * @returns A list of document chunks that match the query. + */ + async retrieve(query: string, topK: number = 10): Promise { + console.log(`Retrieving chunks for query: ${query}`); + try { + // Generate an embedding for the query using Cohere. + const queryEmbeddingResponse: EmbedResponse = await this.cohere.embed({ + texts: [query], + model: 'embed-english-v3.0', + inputType: 'search_query', + }); + + let queryEmbedding: number[]; + + // Extract the embedding from the response. + if (Array.isArray(queryEmbeddingResponse.embeddings)) { + queryEmbedding = queryEmbeddingResponse.embeddings[0]; + } else if (queryEmbeddingResponse.embeddings && 'embeddings' in queryEmbeddingResponse.embeddings) { + queryEmbedding = (queryEmbeddingResponse.embeddings as { embeddings: number[][] }).embeddings[0]; + } else { + throw new Error('Invalid embedding response format'); + } + + if (!Array.isArray(queryEmbedding)) { + throw new Error('Query embedding is not an array'); + } + + // Query the Pinecone index using the embedding and filter by document IDs. + const queryResponse: QueryResponse = await this.index.query({ + vector: queryEmbedding, + filter: { + doc_id: { $in: this._doc_ids }, + }, + topK, + includeValues: true, + includeMetadata: true, + }); + + // Map the results into RAGChunks and return them. + return queryResponse.matches.map( + match => + ({ + id: match.id, + values: match.values as number[], + metadata: match.metadata as { + text: string; + type: string; + original_document: string; + file_path: string; + doc_id: string; + location: string; + start_page: number; + end_page: number; + }, + }) as RAGChunk + ); + } catch (error) { + console.error(`Error retrieving chunks: ${error}`); + return []; + } + } +} diff --git a/src/server/ApiManagers/AssistantManager.ts b/src/server/ApiManagers/AssistantManager.ts index 58d35ce64..88a81c5de 100644 --- a/src/server/ApiManagers/AssistantManager.ts +++ b/src/server/ApiManagers/AssistantManager.ts @@ -8,7 +8,7 @@ import { filesDirectory, publicDirectory } from '../SocketData'; import { Method } from '../RouteManager'; import ApiManager, { Registration } from './ApiManager'; import axios from 'axios'; -import { RAGChunk } from '../../client/views/nodes/ChatBox/types'; +import { RAGChunk } from '../../client/views/nodes/chatbot/types/types'; import { UnstructuredClient } from 'unstructured-client'; import { PartitionResponse } from 'unstructured-client/sdk/models/operations'; import { ChunkingStrategy, Strategy } from 'unstructured-client/sdk/models/shared'; -- cgit v1.2.3-70-g09d2