import { MathJaxContext } from 'better-react-mathjax'; import { action, makeObservable, observable, observe, reaction, runInAction } from 'mobx'; import { observer } from 'mobx-react'; import OpenAI, { ClientOptions } from 'openai'; import { ImageFile, Message } from 'openai/resources/beta/threads/messages'; import { RunStep } from 'openai/resources/beta/threads/runs/steps'; import * as React from 'react'; import { Doc } from '../../../../fields/Doc'; import { Id } from '../../../../fields/FieldSymbols'; import { CsvCast, DocCast, PDFCast, StrCast } from '../../../../fields/Types'; import { CsvField } from '../../../../fields/URLField'; import { Networking } from '../../../Network'; import { DocUtils } from '../../../documents/DocUtils'; import { DocumentType } from '../../../documents/DocumentTypes'; import { Docs } from '../../../documents/Documents'; import { DocumentManager } from '../../../util/DocumentManager'; import { LinkManager } from '../../../util/LinkManager'; import { ViewBoxAnnotatableComponent } from '../../DocComponent'; import { FieldView, FieldViewProps } from '../FieldView'; import './ChatBox.scss'; import MessageComponent from './MessageComponent'; import { ANNOTATION_LINK_TYPE, ASSISTANT_ROLE, AssistantMessage, DOWNLOAD_TYPE } from './types'; import { Annotation } from 'mobx/dist/internal'; import { FormEvent } from 'react'; @observer export class ChatBox extends ViewBoxAnnotatableComponent() { @observable modalStatus = false; @observable currentFile = { url: '' }; @observable history: AssistantMessage[] = []; @observable.deep current_message: AssistantMessage | undefined = undefined; @observable isLoading: boolean = false; @observable isInitializing: boolean = true; @observable expandedLogIndex: number | null = null; @observable linked_docs_to_add: Doc[] = []; @observable inputValue: string = ''; private openai: OpenAI; private assistantID: string = ''; private threadID: string = ''; private _oldWheel: any; private vectorStoreID: string = ''; private mathJaxConfig: any; private linkedCsvIDs: string[] = []; public static LayoutString(fieldKey: string) { return FieldView.LayoutString(ChatBox, fieldKey); } constructor(props: FieldViewProps) { super(props); makeObservable(this); this.openai = this.initializeOpenAI(); this.history = [{ role: ASSISTANT_ROLE.ASSISTANT, text: 'Welcome to the Document Analyser Assistant! Link a document or ask questions to get started.' }]; this.threadID = StrCast(this.dataDoc.thread_id); this.assistantID = StrCast(this.dataDoc.assistant_id); this.vectorStoreID = StrCast(this.dataDoc.vector_store_id); this.openai = this.initializeOpenAI(); if (this.assistantID === '' || this.threadID === '' || this.vectorStoreID === '') { this.createAssistant(); } else { this.retrieveCsvUrls(); this.isInitializing = false; } this.mathJaxConfig = { loader: { load: ['input/asciimath'] }, tex: { inlineMath: [ ['$', '$'], ['\\(', '\\)'], ], displayMath: [ ['$$', '$$'], ['[', ']'], ], }, }; reaction( () => this.history.map((msg: AssistantMessage) => ({ role: msg.role, text: msg.text, image: msg.image, tool_logs: msg.tool_logs })), serializableHistory => { this.dataDoc.data = JSON.stringify(serializableHistory); } ); } @action toggleToolLogs = (index: number) => { this.expandedLogIndex = this.expandedLogIndex === index ? null : index; }; retrieveCsvUrls() { const linkedDocs = LinkManager.Instance.getAllRelatedLinks(this.Document) .map(d => DocCast(LinkManager.getOppositeAnchor(d, this.Document))) .map(d => DocCast(d?.annotationOn, d)) .filter(d => d); linkedDocs.forEach(doc => { const aiFieldId = StrCast(doc[this.Document[Id] + '_ai_field_id']); if (CsvCast(doc.data)) { this.linkedCsvIDs.push(StrCast(aiFieldId)); console.log(this.linkedCsvIDs); } }); } initializeOpenAI() { //console.log(process.env._CLIENT_OPENAI_KEY); const configuration: ClientOptions = { apiKey: process.env.OPENAI_KEY, dangerouslyAllowBrowser: true, }; return new OpenAI(configuration); } onPassiveWheel = (e: WheelEvent) => { if (this._props.isContentActive()) { e.stopPropagation(); } }; @action createAssistant = async () => { this.isInitializing = true; try { const vectorStore = await this.openai.beta.vectorStores.create({ name: 'Vector Store for Assistant', }); const assistant = await this.openai.beta.assistants.create({ name: 'Document Analyser Assistant', instructions: ` You will analyse documents with which you are provided. You will answer questions and provide insights based on the information in the documents. For writing math formulas: You have a MathJax render environment. - Write all in-line equations within a single dollar sign, $, to render them as TeX (this means any time you want to use a dollar sign to represent a dollar sign itself, you must escape it with a backslash: "$"); - Use a double dollar sign, $$, to render equations on a new line; Example: $$x^2 + 3x$$ is output for "x² + 3x" to appear as TeX.`, model: 'gpt-4-turbo', tools: [{ type: 'file_search' }, { type: 'code_interpreter' }], tool_resources: { file_search: { vector_store_ids: [vectorStore.id], }, code_interpreter: { file_ids: this.linkedCsvIDs, }, }, }); const thread = await this.openai.beta.threads.create(); runInAction(() => { this.dataDoc.assistant_id = assistant.id; this.dataDoc.thread_id = thread.id; this.dataDoc.vector_store_id = vectorStore.id; this.assistantID = assistant.id; this.threadID = thread.id; this.vectorStoreID = vectorStore.id; this.isInitializing = false; }); } catch (error) { console.error('Initialization failed:', error); this.isInitializing = false; } }; @action askGPT = async (event: React.FormEvent): Promise => { event.preventDefault(); this.inputValue = ''; const textInput = event.currentTarget.elements.namedItem('messageInput') as HTMLInputElement; const trimmedText = textInput.value.trim(); if (trimmedText) { try { textInput.value = ''; runInAction(() => { this.history.push({ role: ASSISTANT_ROLE.USER, text: trimmedText }); }); const { response } = await Networking.PostToServer('/askAgent', { input: trimmedText }); runInAction(() => { this.history.push({ role: ASSISTANT_ROLE.ASSISTANT, text: response }); }); this.dataDoc.data = JSON.stringify(this.history); } catch (err) { console.error('Error:', err); } } }; // @action // uploadLinks = async (linkedDocs: Doc[]) => { // if (this.isInitializing) { // console.log('Initialization in progress, upload aborted.'); // return; // } // const urls: string[] = linkedDocs.map(doc => CsvCast(doc.data, PDFCast(doc.data)).url.pathname); // const csvUrls: string[] = urls.filter(url => url.endsWith('.csv')); // console.log(this.assistantID, this.threadID, urls); // await Networking.PostToServer('/uploadPDFs', { file_path: urls[0] }); // // linkedDocs.forEach((doc, i) => { // // doc[this.Document[Id] + '_ai_field_id'] = openaiFileIds[i]; // // console.log('AI Field ID: ' + openaiFileIds[i]); // // }); // // if (csvUrls.length > 0) { // // for (let i = 0; i < csvUrls.length; i++) { // // this.linkedCsvIDs.push(openaiFileIds[urls.indexOf(csvUrls[i])]); // // } // // console.log('linked csvs:' + this.linkedCsvIDs); // // await this.openai.beta.assistants.update(this.assistantID, { // // tools: [{ type: 'file_search' }, { type: 'code_interpreter' }], // // tool_resources: { // // file_search: { // // vector_store_ids: [this.vectorStoreID], // // }, // // code_interpreter: { // // file_ids: this.linkedCsvIDs, // // }, // // }, // // }); // // } // }; downloadToComputer = (url: string, fileName: string) => { fetch(url, { method: 'get', mode: 'no-cors', referrerPolicy: 'no-referrer' }) .then(res => res.blob()) .then(res => { const aElement = document.createElement('a'); aElement.setAttribute('download', fileName); const href = URL.createObjectURL(res); aElement.href = href; aElement.setAttribute('target', '_blank'); aElement.click(); URL.revokeObjectURL(href); }); }; createDocumentInDash = async (url: string) => { const fileSuffix = url.substring(url.lastIndexOf('.') + 1); console.log(fileSuffix); let doc: Doc | null = null; switch (fileSuffix) { case 'pdf': doc = DocCast(await DocUtils.DocumentFromType('pdf', url, {})); break; case 'csv': doc = DocCast(await DocUtils.DocumentFromType('csv', url, {})); break; case 'png': case 'jpg': case 'jpeg': doc = DocCast(await DocUtils.DocumentFromType('image', url, {})); break; default: console.error('Unsupported file type:', fileSuffix); break; } if (doc) { doc && this._props.addDocument?.(doc); //add to overlay await DocumentManager.Instance.showDocument(doc, { willZoomCentered: true }, () => {}); } }; downloadFile = async (fileInfo: string, downloadType: DOWNLOAD_TYPE) => { try { console.log(fileInfo); const [fileId, fileName] = fileInfo.split(/!!!/); const { file_path: filePath } = await Networking.PostToServer('/downloadFileFromOpenAI', { file_id: fileId, file_name: fileName }); const fileLink = CsvCast(new CsvField(filePath)).url.href; if (downloadType === DOWNLOAD_TYPE.DASH) { this.createDocumentInDash(fileLink); } else { this.downloadToComputer(fileLink, fileName); } } catch (error) { console.error('Error downloading file:', error); } }; handleDownloadToDevice = () => { this.downloadFile(this.currentFile.url, DOWNLOAD_TYPE.DEVICE); this.modalStatus = false; // Close the modal after the action this.currentFile = { url: '' }; // Reset the current file }; handleAddToDash = () => { // Assuming `downloadFile` is a method that handles adding to Dash this.downloadFile(this.currentFile.url, DOWNLOAD_TYPE.DASH); this.modalStatus = false; // Close the modal after the action this.currentFile = { url: '' }; // Reset the current file }; renderModal = () => { if (!this.modalStatus) return null; return (

File Actions

Choose an action for the file:

); }; @action showModal = () => { this.modalStatus = true; }; @action setCurrentFile = (file: { url: string }) => { this.currentFile = file; }; componentDidMount() { this._props.setContentViewBox?.(this); if (this.dataDoc.data) { try { const storedHistory = JSON.parse(StrCast(this.dataDoc.data)); runInAction(() => { this.history.push( ...storedHistory.map((msg: AssistantMessage) => ({ role: msg.role, text: msg.text, quote: msg.quote, tool_logs: msg.tool_logs, image: msg.image, })) ); }); } catch (e) { console.error('Failed to parse history from dataDoc:', e); } } reaction( () => { const linkedDocs = LinkManager.Instance.getAllRelatedLinks(this.Document) .map(d => DocCast(LinkManager.getOppositeAnchor(d, this.Document))) .map(d => DocCast(d?.annotationOn, d)) .filter(d => d); return linkedDocs; }, linked => this.linked_docs_to_add.push(...linked.filter(linkedDoc => !this.linked_docs_to_add.includes(linkedDoc))) ); observe( // right now this skips during initialization which is necessary because it would be blank // However, it will upload the same link twice when it is this.linked_docs_to_add, change => { // observe pushes/splices on a user link DB 'data' field (should only happen for local changes) switch (change.type as any) { case 'splice': if ((change as any).addedCount > 0) { // maybe check here if its already in the urls datadoc array so doesn't add twice console.log((change as any).added as Doc[]); this.uploadLinks((change as any).added as Doc[]); } // (change as any).removed.forEach((link: any) => remLinkFromDoc(toRealField(link))); break; case 'update': // let oldValue = change.oldValue; default: } }, true ); } @action handleFollowUpClick = (question: string) => { console.log('Follow-up question clicked:', question); this.inputValue = question; }; render() { return ( /** **/
{this.isInitializing &&
Initializing...
} {this.renderModal()}
{ this._oldWheel?.removeEventListener('wheel', this.onPassiveWheel); this._oldWheel = r; r?.addEventListener('wheel', this.onPassiveWheel, { passive: false }); }}>
{this.history.map((message, index) => ( {}} setCurrentFile={this.setCurrentFile} onFollowUpClick={this.handleFollowUpClick} /> ))} {!this.current_message ? null : ( {}} setCurrentFile={this.setCurrentFile} onFollowUpClick={this.handleFollowUpClick} /> )}
(this.inputValue = e.target.value)} />
/**
**/ ); } } Docs.Prototypes.TemplateMap.set(DocumentType.CHAT, { layout: { view: ChatBox, dataField: 'data' }, options: { acl: '', chat: '', chat_history: '', chat_thread_id: '', chat_assistant_id: '', chat_vector_store_id: '' }, });