diff options
Diffstat (limited to 'src/client/views/nodes/ChatBox/ChatBox.tsx')
| -rw-r--r-- | src/client/views/nodes/ChatBox/ChatBox.tsx | 886 |
1 files changed, 412 insertions, 474 deletions
diff --git a/src/client/views/nodes/ChatBox/ChatBox.tsx b/src/client/views/nodes/ChatBox/ChatBox.tsx index 880c332ac..383be0bb7 100644 --- a/src/client/views/nodes/ChatBox/ChatBox.tsx +++ b/src/client/views/nodes/ChatBox/ChatBox.tsx @@ -1,106 +1,157 @@ -import { MathJaxContext } from 'better-react-mathjax'; -import { action, makeObservable, observable, observe, reaction, runInAction } from 'mobx'; +import { action, computed, makeObservable, observable, observe, reaction, runInAction, ObservableSet } from 'mobx'; import { observer } from 'mobx-react'; import OpenAI, { ClientOptions } from 'openai'; -import { ImageFile, Message } from 'openai/resources/beta/threads/messages'; -import { RunStep } from 'openai/resources/beta/threads/runs/steps'; import * as React from 'react'; -import { Doc } from '../../../../fields/Doc'; -import { Id } from '../../../../fields/FieldSymbols'; -import { CsvCast, DocCast, PDFCast, StrCast } from '../../../../fields/Types'; -import { CsvField } from '../../../../fields/URLField'; -import { Networking } from '../../../Network'; -import { DocUtils } from '../../../documents/DocUtils'; +import { Doc, DocListCast } from '../../../../fields/Doc'; +import { CsvCast, DocCast, PDFCast, RTFCast, StrCast } from '../../../../fields/Types'; import { DocumentType } from '../../../documents/DocumentTypes'; import { Docs } from '../../../documents/Documents'; -import { DocumentManager } from '../../../util/DocumentManager'; import { LinkManager } from '../../../util/LinkManager'; import { ViewBoxAnnotatableComponent } from '../../DocComponent'; import { FieldView, FieldViewProps } from '../FieldView'; import './ChatBox.scss'; -import MessageComponent from './MessageComponent'; -import { ANNOTATION_LINK_TYPE, ASSISTANT_ROLE, AssistantMessage, DOWNLOAD_TYPE } from './types'; +import MessageComponentBox from './MessageComponent'; +import { ASSISTANT_ROLE, AssistantMessage, AI_Document, Citation, CHUNK_TYPE, RAGChunk, getChunkType, TEXT_TYPE, SimplifiedChunk, ProcessingInfo, MessageContent } from './types'; +import { Vectorstore } from './vectorstore/Vectorstore'; +import { Agent } from './Agent'; +import dotenv from 'dotenv'; +import { DocData, DocViews } from '../../../../fields/DocSymbols'; +import { AnswerParser } from './AnswerParser'; +import { DocumentManager } from '../../../util/DocumentManager'; +import { v4 as uuidv4 } from 'uuid'; +import { chunk } from 'lodash'; +import { DocUtils } from '../../../documents/DocUtils'; +import { createRef } from 'react'; +import { ClientUtils } from '../../../../ClientUtils'; +import { ProgressBar } from './ProgressBar'; + +dotenv.config(); @observer export class ChatBox extends ViewBoxAnnotatableComponent<FieldViewProps>() { - @observable modalStatus = false; - @observable currentFile = { url: '' }; @observable history: AssistantMessage[] = []; @observable.deep current_message: AssistantMessage | undefined = undefined; @observable isLoading: boolean = false; - @observable isInitializing: boolean = true; - @observable expandedLogIndex: number | null = null; - @observable linked_docs_to_add: Doc[] = []; - + @observable uploadProgress: number = 0; // Track progress percentage + @observable currentStep: string = ''; // Track current step name + @observable expandedScratchpadIndex: number | null = null; + @observable inputValue: string = ''; + @observable private linked_docs_to_add: ObservableSet<Doc> = observable.set(); + @observable private linked_csv_files: { filename: string; id: string; text: string }[] = []; + @observable private isUploadingDocs: boolean = false; private openai: OpenAI; - private interim_history: string = ''; - private assistantID: string = ''; - private threadID: string = ''; - private _oldWheel: any; - private vectorStoreID: string = ''; - private mathJaxConfig: any; - private linkedCsvIDs: string[] = []; + private vectorstore_id: string; + private vectorstore: Vectorstore; + private agent: Agent; // Add the ChatBot instance + private _oldWheel: HTMLDivElement | null = null; + private messagesRef: React.RefObject<HTMLDivElement>; public static LayoutString(fieldKey: string) { return FieldView.LayoutString(ChatBox, fieldKey); } + constructor(props: FieldViewProps) { super(props); makeObservable(this); this.openai = this.initializeOpenAI(); - this.history = []; - this.threadID = StrCast(this.dataDoc.thread_id); - this.assistantID = StrCast(this.dataDoc.assistant_id); - this.vectorStoreID = StrCast(this.dataDoc.vector_store_id); - this.openai = this.initializeOpenAI(); - if (this.assistantID === '' || this.threadID === '' || this.vectorStoreID === '') { - this.createAssistant(); + if (StrCast(this.dataDoc.vectorstore_id) == '') { + console.log('new_id'); + this.vectorstore_id = uuidv4(); + this.dataDoc.vectorstore_id = this.vectorstore_id; } else { - this.retrieveCsvUrls(); - this.isInitializing = false; + this.vectorstore_id = StrCast(this.dataDoc.vectorstore_id); } - this.mathJaxConfig = { - loader: { load: ['input/asciimath'] }, - tex: { - inlineMath: [ - ['$', '$'], - ['\\(', '\\)'], - ], - displayMath: [ - ['$$', '$$'], - ['[', ']'], - ], - }, - }; + this.vectorstore = new Vectorstore(this.vectorstore_id, this.retrieveDocIds); + this.agent = new Agent(this.vectorstore, this.retrieveSummaries, this.retrieveFormattedHistory, this.retrieveCSVData, this.addLinkedUrlDoc, this.createCSVInDash); + this.messagesRef = React.createRef<HTMLDivElement>(); + reaction( - () => this.history.map((msg: AssistantMessage) => ({ role: msg.role, text: msg.text, image: msg.image, tool_logs: msg.tool_logs, links: msg.links })), + () => this.history.map((msg: AssistantMessage) => ({ role: msg.role, content: msg.content, follow_up_questions: msg.follow_up_questions, citations: msg.citations })), serializableHistory => { this.dataDoc.data = JSON.stringify(serializableHistory); } ); } - toggleToolLogs = (index: number) => { - this.expandedLogIndex = this.expandedLogIndex === index ? null : index; + @action + addDocToVectorstore = async (newLinkedDoc: Doc) => { + this.uploadProgress = 0; + this.currentStep = 'Initializing...'; + this.isUploadingDocs = true; + + try { + await this.vectorstore.addAIDoc(newLinkedDoc, this.updateProgress); + } catch (error) { + console.error('Error uploading document:', error); + this.currentStep = 'Error during upload'; + } finally { + this.isUploadingDocs = false; + this.uploadProgress = 0; + this.currentStep = ''; + } }; - retrieveCsvUrls() { - const linkedDocs = LinkManager.Instance.getAllRelatedLinks(this.Document) - .map(d => DocCast(LinkManager.getOppositeAnchor(d, this.Document))) - .map(d => DocCast(d?.annotationOn, d)) - .filter(d => d); + @action + updateProgress = (progress: number, step: string) => { + console.log('Progress:', progress, step); + this.uploadProgress = progress; + this.currentStep = step; + }; - linkedDocs.forEach(doc => { - const aiFieldId = StrCast(doc[this.Document[Id] + '_ai_field_id']); - if (CsvCast(doc.data)) { - this.linkedCsvIDs.push(StrCast(aiFieldId)); - console.log(this.linkedCsvIDs); - } - }); - } + @action + addCSVForAnalysis = async (newLinkedDoc: Doc, id?: string) => { + console.log('adding csv file for analysis'); + if (!newLinkedDoc.chunk_simpl) { + const csvData: string = StrCast(newLinkedDoc.text); + console.log('CSV Data:', csvData); + const completion = await this.openai.chat.completions.create({ + messages: [ + { + role: 'system', + content: + 'You are an AI assistant tasked with summarizing the content of a CSV file. You will be provided with the data from the CSV file and your goal is to generate a concise summary that captures the main themes, trends, and key points represented in the data.', + }, + { + role: 'user', + content: `Please provide a comprehensive summary of the CSV file based on the provided data. Ensure the summary highlights the most important information, patterns, and insights. Your response should be in paragraph form and be concise. + + CSV Data: + + ${csvData} + + ********** + Summary:`, + }, + ], + model: 'gpt-3.5-turbo', + }); + console.log('CSV Data:', csvData); + const csvId = id ?? uuidv4(); + + this.linked_csv_files.push({ + filename: CsvCast(newLinkedDoc.data).url.pathname, + id: csvId, + text: csvData, + }); + + console.log(this.linked_csv_files); + const chunkToAdd = { + chunkId: csvId, + chunkType: CHUNK_TYPE.CSV, + }; + newLinkedDoc.chunk_simpl = JSON.stringify({ chunks: [chunkToAdd] }); + newLinkedDoc.summary = completion.choices[0].message.content!; + } + }; + + @action + toggleToolLogs = (index: number) => { + this.expandedScratchpadIndex = this.expandedScratchpadIndex === index ? null : index; + }; initializeOpenAI() { + console.log(process.env.OPENAI_KEY); const configuration: ClientOptions = { apiKey: process.env.OPENAI_KEY, dangerouslyAllowBrowser: true, @@ -108,396 +159,203 @@ export class ChatBox extends ViewBoxAnnotatableComponent<FieldViewProps>() { return new OpenAI(configuration); } - onPassiveWheel = (e: WheelEvent) => { - if (this._props.isContentActive()) { - e.stopPropagation(); + addScrollListener = () => { + if (this.messagesRef.current) { + this.messagesRef.current.addEventListener('wheel', this.onPassiveWheel, { passive: false }); } }; - createLink = (linkInfo: string, startIndex: number, endIndex: number, linkType: ANNOTATION_LINK_TYPE, annotationIndex: number = 0) => { - const text = this.interim_history; - const subString = this.current_message?.text.substring(startIndex, endIndex) ?? ''; - if (!text) return; - const textToDisplay = `${annotationIndex}`; - let fileInfo = linkInfo; - const fileName = subString.split('/')[subString.split('/').length - 1]; - if (linkType === ANNOTATION_LINK_TYPE.DOWNLOAD_FILE) { - fileInfo = linkInfo + '!!!' + fileName; + removeScrollListener = () => { + if (this.messagesRef.current) { + this.messagesRef.current.removeEventListener('wheel', this.onPassiveWheel); } - - const formattedLink = `[${textToDisplay}](${fileInfo}~~~${linkType})`; - console.log(formattedLink); - const newText = text.replace(subString, formattedLink); - runInAction(() => { - this.interim_history = newText; - console.log(newText); - this.current_message?.links?.push({ - start: startIndex, - end: endIndex, - url: linkType === ANNOTATION_LINK_TYPE.DOWNLOAD_FILE ? fileName : linkInfo, - id: linkType === ANNOTATION_LINK_TYPE.DOWNLOAD_FILE ? linkInfo : undefined, - link_type: linkType, - }); - }); }; - @action - createAssistant = async () => { - this.isInitializing = true; - try { - const vectorStore = await this.openai.beta.vectorStores.create({ - name: 'Vector Store for Assistant', - }); - const assistant = await this.openai.beta.assistants.create({ - name: 'Document Analyser Assistant', - instructions: ` - You will analyse documents with which you are provided. You will answer questions and provide insights based on the information in the documents. - For writing math formulas: - You have a MathJax render environment. - - Write all in-line equations within a single dollar sign, $, to render them as TeX (this means any time you want to use a dollar sign to represent a dollar sign itself, you must escape it with a backslash: "$"); - - Use a double dollar sign, $$, to render equations on a new line; - Example: $$x^2 + 3x$$ is output for "x² + 3x" to appear as TeX.`, - model: 'gpt-4-turbo', - tools: [{ type: 'file_search' }, { type: 'code_interpreter' }], - tool_resources: { - file_search: { - vector_store_ids: [vectorStore.id], - }, - code_interpreter: { - file_ids: this.linkedCsvIDs, - }, - }, - }); - const thread = await this.openai.beta.threads.create(); + scrollToBottom = () => { + if (this.messagesRef.current) { + this.messagesRef.current.scrollTop = this.messagesRef.current.scrollHeight; + } + }; - runInAction(() => { - this.dataDoc.assistant_id = assistant.id; - this.dataDoc.thread_id = thread.id; - this.dataDoc.vector_store_id = vectorStore.id; - this.assistantID = assistant.id; - this.threadID = thread.id; - this.vectorStoreID = vectorStore.id; - this.isInitializing = false; - }); - } catch (error) { - console.error('Initialization failed:', error); - this.isInitializing = false; + onPassiveWheel = (e: WheelEvent) => { + if (this._props.isContentActive()) { + e.stopPropagation(); } }; @action - runAssistant = async (inputText: string) => { - // Ensure an assistant and thread are created - if (!this.assistantID || !this.threadID || !this.vectorStoreID) { - await this.createAssistant(); - console.log('Assistant and thread created:', this.assistantID, this.threadID); - } - let currentText: string = ''; - let currentToolCallMessage: string = ''; + askGPT = async (event: React.FormEvent<HTMLFormElement>): Promise<void> => { + event.preventDefault(); + this.inputValue = ''; - // Send the user's input to the assistant - await this.openai.beta.threads.messages.create(this.threadID, { - role: 'user', - content: inputText, - }); + const textInput = event.currentTarget.elements.namedItem('messageInput') as HTMLInputElement; + const trimmedText = textInput.value.trim(); - // Listen to the streaming responses - const stream = this.openai.beta.threads.runs - .stream(this.threadID, { - assistant_id: this.assistantID, - }) - .on('runStepCreated', (runStep: RunStep) => { - currentText = ''; - runInAction(() => { - this.current_message = { role: ASSISTANT_ROLE.ASSISTANT, text: currentText, tool_logs: '', links: [] }; - }); + if (trimmedText) { + try { + textInput.value = ''; + this.history.push({ role: ASSISTANT_ROLE.USER, content: [{ index: 0, type: TEXT_TYPE.NORMAL, text: trimmedText, citation_ids: null }], processing_info: [] }); this.isLoading = true; - }) - .on('toolCallDelta', (toolCallDelta, snapshot) => { - this.isLoading = false; - if (toolCallDelta.type === 'code_interpreter') { - if (toolCallDelta.code_interpreter?.input) { - currentToolCallMessage += toolCallDelta.code_interpreter.input; - runInAction(() => { - if (this.current_message) { - this.current_message.tool_logs = currentToolCallMessage; - } - }); - } - if (toolCallDelta.code_interpreter?.outputs) { - currentToolCallMessage += '\n Code interpreter output:'; - toolCallDelta.code_interpreter.outputs.forEach(output => { - if (output.type === 'logs') { - runInAction(() => { - if (this.current_message) { - this.current_message.tool_logs += '\n|' + output.logs; - } - }); - } - }); - } - } - }) - .on('textDelta', (textDelta, snapshot) => { - this.isLoading = false; - currentText += textDelta.value; - runInAction(() => { - if (this.current_message) { - // this.current_message = {...this.current_message, text: current_text}; - this.current_message.text = currentText; - } - }); - }) - .on('messageDone', async event => { - console.log(event); - const textItem = event.content.find(item => item.type === 'text'); - if (textItem && textItem.type === 'text') { - const { text } = textItem; - console.log(text.value); - try { - runInAction(() => { - this.interim_history = text.value; - }); - } catch (e) { - console.error('Error parsing JSON response:', e); - } + this.current_message = { role: ASSISTANT_ROLE.ASSISTANT, content: [], citations: [], processing_info: [] }; - const { annotations } = text; - console.log('Annotations: ' + annotations); - let index = 0; - annotations.forEach(async annotation => { - console.log(' ' + annotation); - console.log(' ' + annotation.text); - if (annotation.type === 'file_path') { - const { file_path: filePath } = annotation; - const fileToDownload = filePath.file_id; - console.log(fileToDownload); - if (filePath) { - console.log(filePath); - console.log(fileToDownload); - this.createLink(fileToDownload, annotation.start_index, annotation.end_index, ANNOTATION_LINK_TYPE.DOWNLOAD_FILE); - } - } else { - const { file_citation: fileCitation } = annotation; - if (fileCitation) { - const citedFile = await this.openai.files.retrieve(fileCitation.file_id); - const citationUrl = citedFile.filename; - this.createLink(citationUrl, annotation.start_index, annotation.end_index, ANNOTATION_LINK_TYPE.DASH_DOC, index); - index++; - } + const onProcessingUpdate = (processingUpdate: ProcessingInfo[]) => { + runInAction(() => { + if (this.current_message) { + this.current_message = { ...this.current_message, processing_info: processingUpdate }; } }); + this.scrollToBottom(); + }; + + const onAnswerUpdate = (answerUpdate: string) => { runInAction(() => { if (this.current_message) { - console.log('current message: ' + this.current_message.text); - this.current_message.text = this.interim_history; - this.history.push({ ...this.current_message }); - this.current_message = undefined; + this.current_message = { ...this.current_message, content: [{ text: answerUpdate, type: TEXT_TYPE.NORMAL, index: 0, citation_ids: [] }] }; } }); - } - }) - .on('toolCallDone', toolCall => { + }; + + const finalMessage = await this.agent.askAgent(trimmedText, onProcessingUpdate, onAnswerUpdate); + runInAction(() => { - if (this.current_message && currentToolCallMessage) { - this.current_message.tool_logs = currentToolCallMessage; + if (this.current_message) { + this.history.push({ ...finalMessage }); + this.current_message = undefined; + this.dataDoc.data = JSON.stringify(this.history); } }); - }) - .on('imageFileDone', (content: ImageFile, snapshot: Message) => { - console.log('Image file done:', content); - }) - .on('end', () => { - console.log('Streaming done'); - }); + } catch (err) { + console.error('Error:', err); + this.history.push({ role: ASSISTANT_ROLE.ASSISTANT, content: [{ index: 0, type: TEXT_TYPE.ERROR, text: 'Sorry, I encountered an error while processing your request.', citation_ids: null }], processing_info: [] }); + } finally { + this.isLoading = false; + this.scrollToBottom(); + } + } + this.scrollToBottom(); }; @action - goToLinkedDoc = async (link: string) => { - const linkedDocs = LinkManager.Instance.getAllRelatedLinks(this.Document) - .map(d => DocCast(LinkManager.getOppositeAnchor(d, this.Document))) - .map(d => DocCast(d?.annotationOn, d)) - .filter(d => d); - - const linkedDoc = linkedDocs.find(doc => { - const docUrl = CsvCast(doc.data, PDFCast(doc.data)).url.pathname.replace('/files/pdfs/', '').replace('/files/csvs/', ''); - console.log('URL: ' + docUrl + ' Citation URL: ' + link); - return link === docUrl; - }); - - if (linkedDoc) { - await DocumentManager.Instance.showDocument(DocCast(linkedDoc), { willZoomCentered: true }, () => {}); + updateMessageCitations = (index: number, citations: Citation[]) => { + if (this.history[index]) { + this.history[index].citations = citations; } }; @action - askGPT = async (event: React.FormEvent<HTMLFormElement>): Promise<void> => { - event.preventDefault(); + addLinkedUrlDoc = async (url: string, id: string) => { + const doc = Docs.Create.WebDocument(url); - const textInput = event.currentTarget.elements.namedItem('messageInput') as HTMLInputElement; - const trimmedText = textInput.value.trim(); + const linkDoc = Docs.Create.LinkDocument(this.Document, doc); + LinkManager.Instance.addLink(linkDoc); - if (!this.assistantID || !this.threadID) { - try { - await this.createAssistant(); - } catch (err) { - console.error('Error:', err); - } - } + const chunkToAdd = { + chunkId: id, + chunkType: CHUNK_TYPE.URL, + }; - if (trimmedText) { - try { - textInput.value = ''; - runInAction(() => { - this.history.push({ role: ASSISTANT_ROLE.USER, text: trimmedText }); - }); - await this.runAssistant(trimmedText); - this.dataDoc.data = this.history.toString(); - } catch (err) { - console.error('Error:', err); - } - } + doc.chunk_simpl = JSON.stringify({ chunks: [chunkToAdd] }); }; - @action - uploadLinks = async (linkedDocs: Doc[]) => { - if (this.isInitializing) { - console.log('Initialization in progress, upload aborted.'); - return; - } - const urls = linkedDocs.map(doc => CsvCast(doc.data, PDFCast(doc.data)).url.pathname); - const csvUrls = urls.filter(url => url.endsWith('.csv')); - console.log(this.assistantID, this.threadID, urls); + @computed + get userName() { + return ClientUtils.CurrentUserEmail; + } - const { openai_file_ids: openaiFileIds } = await Networking.PostToServer('/uploadPDFToVectorStore', { urls, threadID: this.threadID, assistantID: this.assistantID, vector_store_id: this.vectorStoreID }); + @action + createCSVInDash = async (url: string, title: string, id: string, data: string) => { + console.log('Creating CSV in Dash:', url, title); + const doc = DocCast(await DocUtils.DocumentFromType('csv', url, { title: title, text: RTFCast(data) })); - linkedDocs.forEach((doc, i) => { - doc[this.Document[Id] + '_ai_field_id'] = openaiFileIds[i]; - console.log('AI Field ID: ' + openaiFileIds[i]); - }); + const linkDoc = Docs.Create.LinkDocument(this.Document, doc); + LinkManager.Instance.addLink(linkDoc); - if (csvUrls.length > 0) { - for (let i = 0; i < csvUrls.length; i++) { - this.linkedCsvIDs.push(openaiFileIds[urls.indexOf(csvUrls[i])]); - } - console.log('linked csvs:' + this.linkedCsvIDs); - await this.openai.beta.assistants.update(this.assistantID, { - tools: [{ type: 'file_search' }, { type: 'code_interpreter' }], - tool_resources: { - file_search: { - vector_store_ids: [this.vectorStoreID], - }, - code_interpreter: { - file_ids: this.linkedCsvIDs, - }, - }, - }); - } - }; + doc && this._props.addDocument?.(doc); + await DocumentManager.Instance.showDocument(doc, { willZoomCentered: true }, () => {}); - downloadToComputer = (url: string, fileName: string) => { - fetch(url, { method: 'get', mode: 'no-cors', referrerPolicy: 'no-referrer' }) - .then(res => res.blob()) - .then(res => { - const aElement = document.createElement('a'); - aElement.setAttribute('download', fileName); - const href = URL.createObjectURL(res); - aElement.href = href; - aElement.setAttribute('target', '_blank'); - aElement.click(); - URL.revokeObjectURL(href); - }); + this.addCSVForAnalysis(doc, id); }; - createDocumentInDash = async (url: string) => { - const fileSuffix = url.substring(url.lastIndexOf('.') + 1); - console.log(fileSuffix); - let doc: Doc | null = null; - switch (fileSuffix) { - case 'pdf': - doc = DocCast(await DocUtils.DocumentFromType('pdf', url, {})); - break; - case 'csv': - doc = DocCast(await DocUtils.DocumentFromType('csv', url, {})); - break; - case 'png': - case 'jpg': - case 'jpeg': - doc = DocCast(await DocUtils.DocumentFromType('image', url, {})); - break; - default: - console.error('Unsupported file type:', fileSuffix); - break; - } - if (doc) { - doc && this._props.addDocument?.(doc); - await DocumentManager.Instance.showDocument(doc, { willZoomCentered: true }, () => {}); - } - }; + @action + handleCitationClick = (citation: Citation) => { + console.log('Citation clicked:', citation); + const currentLinkedDocs: Doc[] = this.linkedDocs; + + const chunkId = citation.chunk_id; + + for (let doc of currentLinkedDocs) { + if (doc.chunk_simpl) { + const docChunkSimpl = JSON.parse(StrCast(doc.chunk_simpl)) as { chunks: SimplifiedChunk[] }; + console.log(docChunkSimpl); + const foundChunk = docChunkSimpl.chunks.find(chunk => chunk.chunkId === chunkId); + console.log(foundChunk); + if (foundChunk) { + console.log(getChunkType(foundChunk.chunkType)); + switch (foundChunk.chunkType) { + case CHUNK_TYPE.IMAGE: + case CHUNK_TYPE.TABLE: + const values = foundChunk.location?.replace(/[\[\]]/g, '').split(','); + + if (values?.length !== 4) { + console.error('Location string must contain exactly 4 numbers'); + return; + } - downloadFile = async (fileInfo: string, downloadType: DOWNLOAD_TYPE) => { - try { - console.log(fileInfo); - const [fileId, fileName] = fileInfo.split(/!!!/); - const { file_path: filePath } = await Networking.PostToServer('/downloadFileFromOpenAI', { file_id: fileId, file_name: fileName }); - const fileLink = CsvCast(new CsvField(filePath)).url.href; - if (downloadType === DOWNLOAD_TYPE.DASH) { - this.createDocumentInDash(fileLink); - } else { - this.downloadToComputer(fileLink, fileName); + const x1 = parseFloat(values[0]) * Doc.NativeWidth(doc); + const y1 = parseFloat(values[1]) * Doc.NativeHeight(doc) + foundChunk.startPage * Doc.NativeHeight(doc); + const x2 = parseFloat(values[2]) * Doc.NativeWidth(doc); + const y2 = parseFloat(values[3]) * Doc.NativeHeight(doc) + foundChunk.startPage * Doc.NativeHeight(doc); + + const annotationKey = Doc.LayoutFieldKey(doc) + '_annotations'; + + const existingDoc = DocListCast(doc[DocData][annotationKey]).find(d => d.citation_id === citation.citation_id); + const highlightDoc = existingDoc ?? this.createImageCitationHighlight(x1, y1, x2, y2, citation, annotationKey, doc); + + DocumentManager.Instance.showDocument(highlightDoc, { willZoomCentered: true }, () => {}); + break; + case CHUNK_TYPE.TEXT: + DocumentManager.Instance.showDocument(doc, { willZoomCentered: true }, () => { + const firstView = Array.from(doc[DocViews])[0]; + firstView.ComponentView?.search?.(citation.direct_text); + }); + break; + case CHUNK_TYPE.URL: + DocumentManager.Instance.showDocument(doc, { willZoomCentered: true }, () => { + const firstView = Array.from(doc[DocViews])[0]; + }); + break; + case CHUNK_TYPE.CSV: + DocumentManager.Instance.showDocument(doc, { willZoomCentered: true }, () => { + const firstView = Array.from(doc[DocViews])[0]; + }); + break; + default: + console.log('Chunk type not supported', foundChunk.chunkType); + break; + } + } } - } catch (error) { - console.error('Error downloading file:', error); } }; - handleDownloadToDevice = () => { - this.downloadFile(this.currentFile.url, DOWNLOAD_TYPE.DEVICE); - this.modalStatus = false; // Close the modal after the action - this.currentFile = { url: '' }; // Reset the current file - }; - - handleAddToDash = () => { - // Assuming `downloadFile` is a method that handles adding to Dash - this.downloadFile(this.currentFile.url, DOWNLOAD_TYPE.DASH); - this.modalStatus = false; // Close the modal after the action - this.currentFile = { url: '' }; // Reset the current file - }; - - renderModal = () => { - if (!this.modalStatus) return null; - - return ( - <div className="modal"> - <div className="modal-content"> - <h4>File Actions</h4> - <p>Choose an action for the file:</p> - <button type="button" onClick={this.handleDownloadToDevice}> - Download to Device - </button> - <button type="button" onClick={this.handleAddToDash}> - Add to Dash - </button> - <button - type="button" - onClick={() => { - this.modalStatus = false; - }}> - Cancel - </button> - </div> - </div> - ); - }; - @action - showModal = () => { - this.modalStatus = true; + createImageCitationHighlight = (x1: number, y1: number, x2: number, y2: number, citation: Citation, annotationKey: string, pdfDoc: Doc): Doc => { + const highlight_doc = Docs.Create.FreeformDocument([], { + x: x1, + y: y1, + _width: x2 - x1, + _height: y2 - y1, + backgroundColor: 'rgba(255, 255, 0, 0.5)', + }); + highlight_doc[DocData].citation_id = citation.citation_id; + Doc.AddDocToList(pdfDoc[DocData], annotationKey, highlight_doc); + highlight_doc.annotationOn = pdfDoc; + Doc.SetContainer(highlight_doc, pdfDoc); + return highlight_doc; }; - @action - setCurrentFile = (file: { url: string }) => { - this.currentFile = file; - }; + componentDidUpdate() { + this.scrollToBottom(); + } componentDidMount() { this._props.setContentViewBox?.(this); @@ -505,17 +363,26 @@ export class ChatBox extends ViewBoxAnnotatableComponent<FieldViewProps>() { try { const storedHistory = JSON.parse(StrCast(this.dataDoc.data)); runInAction(() => { - this.history = storedHistory.map((msg: AssistantMessage) => ({ - role: msg.role, - text: msg.text, - quote: msg.quote, - tool_logs: msg.tool_logs, - image: msg.image, - })); + this.history.push( + ...storedHistory.map((msg: AssistantMessage) => ({ + role: msg.role, + content: msg.content, + follow_up_questions: msg.follow_up_questions, + citations: msg.citations, + })) + ); }); } catch (e) { console.error('Failed to parse history from dataDoc:', e); } + } else { + runInAction(() => { + this.history.push({ + role: ASSISTANT_ROLE.ASSISTANT, + content: [{ index: 0, type: TEXT_TYPE.NORMAL, text: `Hey, ${this.userName()} Welcome to the Your Friendly Assistant! Link a document or ask questions about anything to get started.`, citation_ids: null }], + processing_info: [], + }); + }); } reaction( () => { @@ -526,79 +393,150 @@ export class ChatBox extends ViewBoxAnnotatableComponent<FieldViewProps>() { return linkedDocs; }, - linked => this.linked_docs_to_add.push(...linked.filter(linkedDoc => !this.linked_docs_to_add.includes(linkedDoc))) + linked => linked.forEach(doc => this.linked_docs_to_add.add(doc)) ); - observe( - // right now this skips during initialization which is necessary because it would be blank - // However, it will upload the same link twice when it is - this.linked_docs_to_add, - change => { - // observe pushes/splices on a user link DB 'data' field (should only happen for local changes) - switch (change.type as any) { - case 'splice': - if ((change as any).addedCount > 0) { - // maybe check here if its already in the urls datadoc array so doesn't add twice - console.log((change as any).added as Doc[]); - this.uploadLinks((change as any).added as Doc[]); - } - // (change as any).removed.forEach((link: any) => remLinkFromDoc(toRealField(link))); - break; - case 'update': // let oldValue = change.oldValue; - default: + observe(this.linked_docs_to_add, change => { + if (change.type === 'add') { + if (PDFCast(change.newValue.data)) { + this.addDocToVectorstore(change.newValue); + } else if (CsvCast(change.newValue.data)) { + this.addCSVForAnalysis(change.newValue); } - }, - true + } else if (change.type === 'delete') { + console.log('Deleted docs: ', change.oldValue); + } + }); + this.addScrollListener(); + } + + componentWillUnmount() { + this.removeScrollListener(); + } + + @computed + get linkedDocs() { + return LinkManager.Instance.getAllRelatedLinks(this.Document) + .map(d => DocCast(LinkManager.getOppositeAnchor(d, this.Document))) + .map(d => DocCast(d?.annotationOn, d)) + .filter(d => d); + } + + @computed + get docIds() { + return LinkManager.Instance.getAllRelatedLinks(this.Document) + .map(d => DocCast(LinkManager.getOppositeAnchor(d, this.Document))) + .map(d => DocCast(d?.annotationOn, d)) + .filter(d => d) + .filter(d => d.ai_doc_id) + .map(d => StrCast(d.ai_doc_id)); + } + + @computed + get summaries(): string { + return ( + LinkManager.Instance.getAllRelatedLinks(this.Document) + .map(d => DocCast(LinkManager.getOppositeAnchor(d, this.Document))) + .map(d => DocCast(d?.annotationOn, d)) + .filter(d => d) + .filter(d => d.summary) + .map((doc, index) => { + if (PDFCast(doc.data)) { + return `<summary file_name="${PDFCast(doc.data).url.pathname}" applicable_tools=["rag"]>${doc.summary}</summary>`; + } else if (CsvCast(doc.data)) { + return `<summary file_name="${CsvCast(doc.data).url.pathname}" applicable_tools=["dataAnalysis"]>${doc.summary}</summary>`; + } else { + return `${index + 1}) ${doc.summary}`; + } + }) + .join('\n') + '\n' ); } + @computed + get linkedCSVs(): { filename: string; id: string; text: string }[] { + return this.linked_csv_files; + } + + @computed + get formattedHistory(): string { + let history = '<chat_history>\n'; + for (const message of this.history) { + history += `<${message.role}>${message.content.map(content => content.text).join(' ')}`; + if (message.loop_summary) { + history += `<loop_summary>${message.loop_summary}</loop_summary>`; + } + history += `</${message.role}>\n`; + } + history += '</chat_history>'; + return history; + } + + retrieveSummaries = () => { + return this.summaries; + }; + + retrieveCSVData = () => { + return this.linkedCSVs; + }; + + retrieveFormattedHistory = () => { + return this.formattedHistory; + }; + + retrieveDocIds = () => { + return this.docIds; + }; + + @action + handleFollowUpClick = (question: string) => { + console.log('Follow-up question clicked:', question); + this.inputValue = question; + }; + render() { return ( - <MathJaxContext config={this.mathJaxConfig}> - <div className="chatBox"> - {this.isInitializing && <div className="initializing-overlay">Initializing...</div>} - {this.renderModal()} - <div - className="scroll-box chat-content" - ref={r => { - this._oldWheel?.removeEventListener('wheel', this.onPassiveWheel); - this._oldWheel = r; - r?.addEventListener('wheel', this.onPassiveWheel, { passive: false }); - }}> - <div className="messages"> - {this.history.map((message, index) => ( - <MessageComponent - key={index} - message={message} - toggleToolLogs={this.toggleToolLogs} - expandedLogIndex={this.expandedLogIndex} - index={index} - showModal={this.showModal} - goToLinkedDoc={this.goToLinkedDoc} - setCurrentFile={this.setCurrentFile} - /> - ))} - {!this.current_message ? null : ( - <MessageComponent - key={this.history.length} - message={this.current_message} - toggleToolLogs={this.toggleToolLogs} - expandedLogIndex={this.expandedLogIndex} - index={this.history.length} - showModal={this.showModal} - goToLinkedDoc={this.goToLinkedDoc} - setCurrentFile={this.setCurrentFile} - isCurrent - /> - )} + <div className="chat-box"> + {this.isUploadingDocs && ( + <div className="uploading-overlay"> + <div className="progress-container"> + <ProgressBar /> + <div className="step-name">{this.currentStep}</div> </div> </div> - <form onSubmit={this.askGPT} className="chat-form"> - <input type="text" name="messageInput" autoComplete="off" placeholder="Type a message..." /> - <button type="submit">Send</button> - </form> + )} + <div className="chat-header"> + <h2>{this.userName()}'s AI Assistant</h2> </div> - </MathJaxContext> + <div className="chat-messages" ref={this.messagesRef}> + {this.history.map((message, index) => ( + <MessageComponentBox key={index} message={message} index={index} onFollowUpClick={this.handleFollowUpClick} onCitationClick={this.handleCitationClick} updateMessageCitations={this.updateMessageCitations} /> + ))} + {this.current_message && ( + <MessageComponentBox + key={this.history.length} + message={this.current_message} + index={this.history.length} + onFollowUpClick={this.handleFollowUpClick} + onCitationClick={this.handleCitationClick} + updateMessageCitations={this.updateMessageCitations} + /> + )} + </div> + <form onSubmit={this.askGPT} className="chat-input"> + <input type="text" name="messageInput" autoComplete="off" placeholder="Type your message here..." value={this.inputValue} onChange={e => (this.inputValue = e.target.value)} /> + <button className="submit-button" type="submit" disabled={this.isLoading}> + {this.isLoading ? ( + <div className="spinner"></div> + ) : ( + <svg viewBox="0 0 24 24" width="24" height="24" stroke="currentColor" strokeWidth="2" fill="none" strokeLinecap="round" strokeLinejoin="round"> + <line x1="22" y1="2" x2="11" y2="13"></line> + <polygon points="22 2 15 22 11 13 2 9 22 2"></polygon> + </svg> + )} + </button> + </form> + </div> ); } } |