/** * @file ChatBox.tsx * @description This file defines the ChatBox component, which manages user interactions with * an AI assistant. It handles document uploads, chat history, message input, and integration * with the OpenAI API. The ChatBox is MobX-observable and tracks the progress of tasks such as * document analysis and AI-driven summaries. It also maintains real-time chat functionality * with support for follow-up questions and citation management. */ import dotenv from 'dotenv'; import { ObservableSet, action, computed, makeObservable, observable, observe, reaction, runInAction } from 'mobx'; import { observer } from 'mobx-react'; import OpenAI, { ClientOptions } from 'openai'; import * as React from 'react'; import { v4 as uuidv4 } from 'uuid'; import { ClientUtils } from '../../../../../ClientUtils'; import { Doc, DocListCast } from '../../../../../fields/Doc'; import { DocData, DocViews } from '../../../../../fields/DocSymbols'; import { CsvCast, DocCast, PDFCast, RTFCast, StrCast } from '../../../../../fields/Types'; import { Networking } from '../../../../Network'; import { DocUtils } from '../../../../documents/DocUtils'; import { DocumentType } from '../../../../documents/DocumentTypes'; import { Docs, DocumentOptions } from '../../../../documents/Documents'; import { DocumentManager } from '../../../../util/DocumentManager'; import { LinkManager } from '../../../../util/LinkManager'; import { ViewBoxAnnotatableComponent } from '../../../DocComponent'; import { DocumentView } from '../../DocumentView'; import { FieldView, FieldViewProps } from '../../FieldView'; import { PDFBox } from '../../PDFBox'; import { Agent } from '../agentsystem/Agent'; import { ASSISTANT_ROLE, AssistantMessage, CHUNK_TYPE, Citation, ProcessingInfo, SimplifiedChunk, TEXT_TYPE } from '../types/types'; import { Vectorstore } from '../vectorstore/Vectorstore'; import './ChatBox.scss'; import MessageComponentBox from './MessageComponent'; import { ProgressBar } from './ProgressBar'; import { RichTextField } from '../../../../../fields/RichTextField'; dotenv.config(); /** * ChatBox is the main class responsible for managing the interaction between the user and the assistant, * handling documents, and integrating with OpenAI for tasks such as document analysis, chat functionality, * and vector store interactions. */ @observer export class ChatBox extends ViewBoxAnnotatableComponent() { // MobX observable properties to track UI state and data @observable history: AssistantMessage[] = []; @observable.deep current_message: AssistantMessage | undefined = undefined; @observable isLoading: boolean = false; @observable uploadProgress: number = 0; @observable currentStep: string = ''; @observable expandedScratchpadIndex: number | null = null; @observable inputValue: string = ''; @observable private linked_docs_to_add: ObservableSet = observable.set(); @observable private linked_csv_files: { filename: string; id: string; text: string }[] = []; @observable private isUploadingDocs: boolean = false; @observable private citationPopup: { text: string; visible: boolean } = { text: '', visible: false }; // Private properties for managing OpenAI API, vector store, agent, and UI elements private openai: OpenAI; private vectorstore_id: string; private vectorstore: Vectorstore; private agent: Agent; private messagesRef: React.RefObject; /** * Static method that returns the layout string for the field. * @param fieldKey Key to get the layout string. */ public static LayoutString(fieldKey: string) { return FieldView.LayoutString(ChatBox, fieldKey); } /** * Constructor initializes the component, sets up OpenAI, vector store, and agent instances, * and observes changes in the chat history to save the state in dataDoc. * @param props The properties passed to the component. */ constructor(props: FieldViewProps) { super(props); makeObservable(this); // Enable MobX observables // Initialize OpenAI, vectorstore, and agent this.openai = this.initializeOpenAI(); if (StrCast(this.dataDoc.vectorstore_id) == '') { this.vectorstore_id = uuidv4(); this.dataDoc.vectorstore_id = this.vectorstore_id; } else { this.vectorstore_id = StrCast(this.dataDoc.vectorstore_id); } this.vectorstore = new Vectorstore(this.vectorstore_id, this.retrieveDocIds); this.agent = new Agent(this.vectorstore, this.retrieveSummaries, this.retrieveFormattedHistory, this.retrieveCSVData, this.addLinkedUrlDoc, this.createDocInDash, this.createCSVInDash); this.messagesRef = React.createRef(); // Reaction to update dataDoc when chat history changes reaction( () => this.history.map((msg: AssistantMessage) => ({ role: msg.role, content: msg.content, follow_up_questions: msg.follow_up_questions, citations: msg.citations, })), serializableHistory => { this.dataDoc.data = JSON.stringify(serializableHistory); } ); } /** * Adds a document to the vectorstore for AI-based analysis. * Handles the upload progress and errors during the process. * @param newLinkedDoc The new document to add. */ @action addDocToVectorstore = async (newLinkedDoc: Doc) => { this.uploadProgress = 0; this.currentStep = 'Initializing...'; this.isUploadingDocs = true; try { // Add the document to the vectorstore await this.vectorstore.addAIDoc(newLinkedDoc, this.updateProgress); } catch (error) { console.error('Error uploading document:', error); this.currentStep = 'Error during upload'; } finally { this.isUploadingDocs = false; this.uploadProgress = 0; this.currentStep = ''; } }; /** * Updates the upload progress and the current step in the UI. * @param progress The percentage of the progress. * @param step The current step name. */ @action updateProgress = (progress: number, step: string) => { this.uploadProgress = progress; this.currentStep = step; }; /** * Adds a CSV file for analysis by sending it to OpenAI and generating a summary. * @param newLinkedDoc The linked document representing the CSV file. * @param id Optional ID for the document. */ @action addCSVForAnalysis = async (newLinkedDoc: Doc, id?: string) => { if (!newLinkedDoc.chunk_simpl) { // Convert document text to CSV data const csvData: string = StrCast(newLinkedDoc.text); // Generate a summary using OpenAI API const completion = await this.openai.chat.completions.create({ messages: [ { role: 'system', content: 'You are an AI assistant tasked with summarizing the content of a CSV file. You will be provided with the data from the CSV file and your goal is to generate a concise summary that captures the main themes, trends, and key points represented in the data.', }, { role: 'user', content: `Please provide a comprehensive summary of the CSV file based on the provided data. Ensure the summary highlights the most important information, patterns, and insights. Your response should be in paragraph form and be concise. CSV Data: ${csvData} ********** Summary:`, }, ], model: 'gpt-3.5-turbo', }); const csvId = id ?? uuidv4(); // Add CSV details to linked files this.linked_csv_files.push({ filename: CsvCast(newLinkedDoc.data).url.pathname, id: csvId, text: csvData, }); // Add a chunk for the CSV and assign the summary const chunkToAdd = { chunkId: csvId, chunkType: CHUNK_TYPE.CSV, }; newLinkedDoc.chunk_simpl = JSON.stringify({ chunks: [chunkToAdd] }); newLinkedDoc.summary = completion.choices[0].message.content!; } }; /** * Toggles the tool logs, expanding or collapsing the scratchpad at the given index. * @param index Index of the tool log to toggle. */ @action toggleToolLogs = (index: number) => { this.expandedScratchpadIndex = this.expandedScratchpadIndex === index ? null : index; }; /** * Initializes the OpenAI API client using the API key from environment variables. * @returns OpenAI client instance. */ initializeOpenAI() { const configuration: ClientOptions = { apiKey: process.env.OPENAI_KEY, dangerouslyAllowBrowser: true, }; return new OpenAI(configuration); } /** * Adds a scroll event listener to detect user scrolling and handle passive wheel events. */ addScrollListener = () => { if (this.messagesRef.current) { this.messagesRef.current.addEventListener('wheel', this.onPassiveWheel, { passive: false }); } }; /** * Removes the scroll event listener from the chat messages container. */ removeScrollListener = () => { if (this.messagesRef.current) { this.messagesRef.current.removeEventListener('wheel', this.onPassiveWheel); } }; /** * Scrolls the chat messages container to the bottom, ensuring the latest message is visible. */ scrollToBottom = () => { // if (this.messagesRef.current) { // this.messagesRef.current.scrollTop = this.messagesRef.current.scrollHeight; // } }; /** * Event handler for detecting wheel scrolling and stopping the event propagation. * @param e The wheel event. */ onPassiveWheel = (e: WheelEvent) => { if (this._props.isContentActive()) { e.stopPropagation(); } }; /** * Sends the user's input to OpenAI, displays the loading indicator, and updates the chat history. * @param event The form submission event. */ @action askGPT = async (event: React.FormEvent): Promise => { event.preventDefault(); this.inputValue = ''; // Extract the user's message const textInput = (event.currentTarget as HTMLFormElement).elements.namedItem('messageInput') as HTMLInputElement; const trimmedText = textInput.value.trim(); if (trimmedText) { try { textInput.value = ''; // Add the user's message to the history this.history.push({ role: ASSISTANT_ROLE.USER, content: [{ index: 0, type: TEXT_TYPE.NORMAL, text: trimmedText, citation_ids: null }], processing_info: [], }); this.isLoading = true; this.current_message = { role: ASSISTANT_ROLE.ASSISTANT, content: [], citations: [], processing_info: [], }; // Define callbacks for real-time processing updates const onProcessingUpdate = (processingUpdate: ProcessingInfo[]) => { runInAction(() => { if (this.current_message) { this.current_message = { ...this.current_message, processing_info: processingUpdate, }; } }); this.scrollToBottom(); }; const onAnswerUpdate = (answerUpdate: string) => { runInAction(() => { if (this.current_message) { this.current_message = { ...this.current_message, content: [{ text: answerUpdate, type: TEXT_TYPE.NORMAL, index: 0, citation_ids: [] }], }; } }); }; // Send the user's question to the assistant and get the final message const finalMessage = await this.agent.askAgent(trimmedText, onProcessingUpdate, onAnswerUpdate); // Update the history with the final assistant message runInAction(() => { if (this.current_message) { this.history.push({ ...finalMessage }); this.current_message = undefined; this.dataDoc.data = JSON.stringify(this.history); } }); } catch (err) { console.error('Error:', err); // Handle error in processing this.history.push({ role: ASSISTANT_ROLE.ASSISTANT, content: [{ index: 0, type: TEXT_TYPE.ERROR, text: 'Sorry, I encountered an error while processing your request.', citation_ids: null }], processing_info: [], }); } finally { this.isLoading = false; this.scrollToBottom(); } } this.scrollToBottom(); }; /** * Updates the citations for a given message in the chat history. * @param index The index of the message in the history. * @param citations The list of citations to add to the message. */ @action updateMessageCitations = (index: number, citations: Citation[]) => { if (this.history[index]) { this.history[index].citations = citations; } }; /** * Adds a linked document from a URL for future reference and analysis. * @param url The URL of the document to add. * @param id The unique identifier for the document. */ @action addLinkedUrlDoc = async (url: string, id: string) => { const doc = Docs.Create.WebDocument(url, { data_useCors: true }); const linkDoc = Docs.Create.LinkDocument(this.Document, doc); LinkManager.Instance.addLink(linkDoc); const chunkToAdd = { chunkId: id, chunkType: CHUNK_TYPE.URL, url: url, }; doc.chunk_simpl = JSON.stringify({ chunks: [chunkToAdd] }); }; /** * Getter to retrieve the current user's name from the client utils. */ @computed get userName() { return ClientUtils.CurrentUserEmail; } /** * Creates a CSV document in the dashboard and adds it for analysis. * @param url The URL of the CSV. * @param title The title of the CSV document. * @param id The unique ID for the document. * @param data The CSV data content. */ @action createCSVInDash = async (url: string, title: string, id: string, data: string) => { const doc = DocCast(await DocUtils.DocumentFromType('csv', url, { title: title, text: RTFCast(data) })); const linkDoc = Docs.Create.LinkDocument(this.Document, doc); LinkManager.Instance.addLink(linkDoc); doc && this._props.addDocument?.(doc); await DocumentManager.Instance.showDocument(doc, { willZoomCentered: true }, () => {}); this.addCSVForAnalysis(doc, id); }; /** * Creates a text document in the dashboard and adds it for analysis. * @param title The title of the doc. * @param text_content The text of the document. * @param options Other optional document options (e.g. color) * @param id The unique ID for the document. */ @action createDocInDash = async (doc_type: string, data: string | undefined, options: DocumentOptions, id: string) => { let doc; switch (doc_type.toLowerCase()) { case 'text': doc = Docs.Create.TextDocument(data || '', options); break; case 'image': doc = Docs.Create.ImageDocument(data || '', options); break; case 'pdf': doc = Docs.Create.PdfDocument(data || '', options); break; case 'video': doc = Docs.Create.VideoDocument(data || '', options); break; case 'audio': doc = Docs.Create.AudioDocument(data || '', options); break; case 'web': doc = Docs.Create.WebDocument(data || '', options); break; case 'equation': doc = Docs.Create.EquationDocument(data || '', options); break; case 'functionplot': case 'function_plot': doc = Docs.Create.FunctionPlotDocument([], options); break; case 'dataviz': case 'data_viz': doc = Docs.Create.DataVizDocument(data || '', options); break; case 'chat': doc = Docs.Create.ChatDocument(options); break; // Add more cases for other document types default: console.error('Unknown or unsupported document type:', doc_type); return; } const linkDoc = Docs.Create.LinkDocument(this.Document, doc); LinkManager.Instance.addLink(linkDoc); doc && this._props.addDocument?.(doc); await DocumentManager.Instance.showDocument(doc, { willZoomCentered: true }, () => {}); }; /** * Event handler to manage citations click in the message components. * @param citation The citation object clicked by the user. */ @action handleCitationClick = (citation: Citation) => { const currentLinkedDocs: Doc[] = this.linkedDocs; const chunkId = citation.chunk_id; // Loop through the linked documents to find the matching chunk and handle its display for (const doc of currentLinkedDocs) { if (doc.chunk_simpl) { const docChunkSimpl = JSON.parse(StrCast(doc.chunk_simpl)) as { chunks: SimplifiedChunk[] }; const foundChunk = docChunkSimpl.chunks.find(chunk => chunk.chunkId === chunkId); if (foundChunk) { // Handle different types of chunks (image, text, table, etc.) switch (foundChunk.chunkType) { case CHUNK_TYPE.IMAGE: case CHUNK_TYPE.TABLE: { const values = foundChunk.location?.replace(/[[\]]/g, '').split(','); if (values?.length !== 4) { console.error('Location string must contain exactly 4 numbers'); return; } const x1 = parseFloat(values[0]) * Doc.NativeWidth(doc); const y1 = parseFloat(values[1]) * Doc.NativeHeight(doc) + foundChunk.startPage * Doc.NativeHeight(doc); const x2 = parseFloat(values[2]) * Doc.NativeWidth(doc); const y2 = parseFloat(values[3]) * Doc.NativeHeight(doc) + foundChunk.startPage * Doc.NativeHeight(doc); const annotationKey = Doc.LayoutFieldKey(doc) + '_annotations'; const existingDoc = DocListCast(doc[DocData][annotationKey]).find(d => d.citation_id === citation.citation_id); const highlightDoc = existingDoc ?? this.createImageCitationHighlight(x1, y1, x2, y2, citation, annotationKey, doc); DocumentManager.Instance.showDocument(highlightDoc, { willZoomCentered: true }, () => {}); } break; case CHUNK_TYPE.TEXT: this.citationPopup = { text: citation.direct_text ?? 'No text available', visible: true }; setTimeout(() => (this.citationPopup.visible = false), 3000); // Hide after 3 seconds DocumentManager.Instance.showDocument(doc, { willZoomCentered: true }, () => { const firstView = Array.from(doc[DocViews])[0] as DocumentView; (firstView.ComponentView as PDFBox)?.gotoPage?.(foundChunk.startPage); (firstView.ComponentView as PDFBox)?.search?.(citation.direct_text ?? ''); }); break; case CHUNK_TYPE.URL: DocumentManager.Instance.showDocument(doc, { willZoomCentered: true }, () => {}); break; case CHUNK_TYPE.CSV: DocumentManager.Instance.showDocument(doc, { willZoomCentered: true }, () => {}); break; default: console.error('Chunk type not recognized:', foundChunk.chunkType); break; } } } } }; /** * Creates an annotation highlight on a PDF document for image citations. * @param x1 X-coordinate of the top-left corner of the highlight. * @param y1 Y-coordinate of the top-left corner of the highlight. * @param x2 X-coordinate of the bottom-right corner of the highlight. * @param y2 Y-coordinate of the bottom-right corner of the highlight. * @param citation The citation object to associate with the highlight. * @param annotationKey The key used to store the annotation. * @param pdfDoc The document where the highlight is created. * @returns The highlighted document. */ createImageCitationHighlight = (x1: number, y1: number, x2: number, y2: number, citation: Citation, annotationKey: string, pdfDoc: Doc): Doc => { const highlight_doc = Docs.Create.FreeformDocument([], { x: x1, y: y1, _width: x2 - x1, _height: y2 - y1, backgroundColor: 'rgba(255, 255, 0, 0.5)', }); highlight_doc[DocData].citation_id = citation.citation_id; Doc.AddDocToList(pdfDoc[DocData], annotationKey, highlight_doc); highlight_doc.annotationOn = pdfDoc; Doc.SetContainer(highlight_doc, pdfDoc); return highlight_doc; }; /** * Lifecycle method that triggers when the component updates. * Ensures the chat is scrolled to the bottom when new messages are added. */ componentDidUpdate() { this.scrollToBottom(); } /** * Lifecycle method that triggers when the component mounts. * Initializes scroll listeners, sets up document reactions, and loads chat history from dataDoc if available. */ componentDidMount() { this._props.setContentViewBox?.(this); if (this.dataDoc.data) { try { const storedHistory = JSON.parse(StrCast(this.dataDoc.data)); runInAction(() => { this.history.push( ...storedHistory.map((msg: AssistantMessage) => ({ role: msg.role, content: msg.content, follow_up_questions: msg.follow_up_questions, citations: msg.citations, })) ); }); } catch (e) { console.error('Failed to parse history from dataDoc:', e); } } else { // Default welcome message runInAction(() => { this.history.push({ role: ASSISTANT_ROLE.ASSISTANT, content: [ { index: 0, type: TEXT_TYPE.NORMAL, text: `Hey, ${this.userName()}! Welcome to Your Friendly Assistant. Link a document or ask questions to get started.`, citation_ids: null, }, ], processing_info: [], }); }); } // Set up reactions for linked documents reaction( () => { const linkedDocs = LinkManager.Instance.getAllRelatedLinks(this.Document) .map(d => DocCast(LinkManager.getOppositeAnchor(d, this.Document))) .map(d => DocCast(d?.annotationOn, d)) .filter(d => d); return linkedDocs; }, linked => linked.forEach(doc => this.linked_docs_to_add.add(doc)) ); // Observe changes to linked documents and handle document addition observe(this.linked_docs_to_add, change => { if (change.type === 'add') { if (PDFCast(change.newValue.data)) { this.addDocToVectorstore(change.newValue); } else if (CsvCast(change.newValue.data)) { this.addCSVForAnalysis(change.newValue); } } else if (change.type === 'delete') { // Handle document removal } }); this.addScrollListener(); } /** * Lifecycle method that triggers when the component unmounts. * Removes scroll listeners to avoid memory leaks. */ componentWillUnmount() { this.removeScrollListener(); } /** * Getter that retrieves all linked documents for the current document. */ @computed get linkedDocs() { return LinkManager.Instance.getAllRelatedLinks(this.Document) .map(d => DocCast(LinkManager.getOppositeAnchor(d, this.Document))) .map(d => DocCast(d?.annotationOn, d)) .filter(d => d); } /** * Getter that retrieves document IDs of linked documents that have AI-related content. */ @computed get docIds() { return LinkManager.Instance.getAllRelatedLinks(this.Document) .map(d => DocCast(LinkManager.getOppositeAnchor(d, this.Document))) .map(d => DocCast(d?.annotationOn, d)) .filter(d => d) .filter(d => d.ai_doc_id) .map(d => StrCast(d.ai_doc_id)); } /** * Getter that retrieves summaries of all linked documents. */ @computed get summaries(): string { return ( LinkManager.Instance.getAllRelatedLinks(this.Document) .map(d => DocCast(LinkManager.getOppositeAnchor(d, this.Document))) .map(d => DocCast(d?.annotationOn, d)) .filter(d => d) .filter(d => d.summary) .map((doc, index) => { if (PDFCast(doc.data)) { return `${doc.summary}`; } else if (CsvCast(doc.data)) { return `${doc.summary}`; } else { return `${index + 1}) ${doc.summary}`; } }) .join('\n') + '\n' ); } /** * Getter that retrieves all linked CSV files for analysis. */ @computed get linkedCSVs(): { filename: string; id: string; text: string }[] { return this.linked_csv_files; } /** * Getter that formats the entire chat history as a string for the agent's system message. */ @computed get formattedHistory(): string { let history = '\n'; for (const message of this.history) { history += `<${message.role}>${message.content.map(content => content.text).join(' ')}`; if (message.loop_summary) { history += `${message.loop_summary}`; } history += `\n`; } history += ''; return history; } // Other helper methods for retrieving document data and processing retrieveSummaries = () => { return this.summaries; }; retrieveCSVData = () => { return this.linkedCSVs; }; retrieveFormattedHistory = () => { return this.formattedHistory; }; retrieveDocIds = () => { return this.docIds; }; /** * Handles follow-up questions when the user clicks on them. * Automatically sets the input value to the clicked follow-up question. * @param question The follow-up question clicked by the user. */ @action handleFollowUpClick = (question: string) => { this.inputValue = question; }; /** * Renders the chat interface, including the message list, input field, and other UI elements. */ render() { return (
{this.isUploadingDocs && (
{this.currentStep}
)}

{this.userName()}'s AI Assistant

{this.history.map((message, index) => ( ))} {this.current_message && ( )}
(this.inputValue = e.target.value)} />
{/* Popup for citation */} {this.citationPopup.visible && (

Text from your document: {this.citationPopup.text}

)}
); } } /** * Register the ChatBox component as the template for CHAT document types. */ Docs.Prototypes.TemplateMap.set(DocumentType.CHAT, { layout: { view: ChatBox, dataField: 'data' }, options: { acl: '', chat: '', chat_history: '', chat_thread_id: '', chat_assistant_id: '', chat_vector_store_id: '' }, });