diff options
Diffstat (limited to 'src')
22 files changed, 1525 insertions, 729 deletions
diff --git a/src/client/documents/DocumentTypes.ts b/src/client/documents/DocumentTypes.ts index 8f95068db..cb1625381 100644 --- a/src/client/documents/DocumentTypes.ts +++ b/src/client/documents/DocumentTypes.ts @@ -28,6 +28,7 @@ export enum DocumentType { DATAVIZ = 'dataviz', LOADING = 'loading', SIMULATION = 'simulation', // physics simulation + MESSAGE = 'message', // chat message // special purpose wrappers that either take no data or are compositions of lower level types LINK = 'link', diff --git a/src/client/documents/Documents.ts b/src/client/documents/Documents.ts index b96fdb4bd..fb5cb27b2 100644 --- a/src/client/documents/Documents.ts +++ b/src/client/documents/Documents.ts @@ -791,6 +791,11 @@ export namespace Docs { export function RTFDocument(field: RichTextField, options: DocumentOptions = {}, fieldKey: string = 'text') { return InstanceFromProto(Prototypes.get(DocumentType.RTF), field, options, undefined, fieldKey); } + + export function MessageDocument(field: string, options: DocumentOptions = {}, fieldKey: string = 'data') { + return InstanceFromProto(Prototypes.get(DocumentType.MESSAGE), field, options, undefined, fieldKey); + } + export function TextDocument(text: string, options: DocumentOptions = {}, fieldKey: string = 'text') { const rtf = { doc: { diff --git a/src/client/views/collections/CollectionNoteTakingViewColumn.tsx b/src/client/views/collections/CollectionNoteTakingViewColumn.tsx index 44ab1968d..2c6257cf2 100644 --- a/src/client/views/collections/CollectionNoteTakingViewColumn.tsx +++ b/src/client/views/collections/CollectionNoteTakingViewColumn.tsx @@ -1,6 +1,6 @@ /* eslint-disable jsx-a11y/control-has-associated-label */ import { FontAwesomeIcon } from '@fortawesome/react-fontawesome'; -import { action, computed, makeObservable, observable } from 'mobx'; +import { action, computed, makeObservable, observable, runInAction } from 'mobx'; import { observer } from 'mobx-react'; import * as React from 'react'; import { lightOrDark, returnEmptyString } from '../../../ClientUtils'; @@ -88,12 +88,16 @@ export class CollectionNoteTakingViewColumn extends ObservableReactComponent<CSV }; componentDidMount(): void { - this._ele && this.props.refList.push(this._ele); + runInAction(() => { + this._ele && this.props.refList.push(this._ele); + }); } componentWillUnmount() { - this._ele && this.props.refList.splice(this._props.refList.indexOf(this._ele), 1); - this._ele = null; + runInAction(() => { + this._ele && this.props.refList.splice(this._props.refList.indexOf(this._ele), 1); + this._ele = null; + }); } @undoBatch diff --git a/src/client/views/nodes/ChatBox/Agent.ts b/src/client/views/nodes/ChatBox/Agent.ts new file mode 100644 index 000000000..a3b1d083c --- /dev/null +++ b/src/client/views/nodes/ChatBox/Agent.ts @@ -0,0 +1,144 @@ +import OpenAI from 'openai'; +import { Tool, AgentMessage } from './types'; +import { getReactPrompt } from './prompts'; +import { XMLParser, XMLBuilder } from 'fast-xml-parser'; +import { WikipediaTool } from './tools/WikipediaTool'; +import { CalculateTool } from './tools/CalculateTool'; +import { RAGTool } from './tools/RAGTool'; +import { NoTool } from './tools/NoTool'; +import { Vectorstore } from './vectorstore/VectorstoreUpload'; +import { ChatCompletionAssistantMessageParam, ChatCompletionMessageParam } from 'openai/resources'; +import dotenv from 'dotenv'; +import { ChatBox } from './ChatBox'; +dotenv.config(); + +export class Agent { + private client: OpenAI; + private tools: Record<string, Tool<any>>; + private messages: AgentMessage[] = []; + private interMessages: AgentMessage[] = []; + private vectorstore: Vectorstore; + private _history: () => string; + + constructor(_vectorstore: Vectorstore, summaries: () => string, history: () => string) { + this.client = new OpenAI({ apiKey: process.env.OPENAI_KEY, dangerouslyAllowBrowser: true }); + this.vectorstore = _vectorstore; + this._history = history; + this.tools = { + wikipedia: new WikipediaTool(), + calculate: new CalculateTool(), + rag: new RAGTool(this.vectorstore, summaries), + no_tool: new NoTool(), + }; + } + + async askAgent(question: string, maxTurns: number = 8): Promise<string> { + console.log(`Starting query: ${question}`); + this.messages.push({ role: 'user', content: question }); + const chatHistory = this._history(); + console.log(`Chat history: ${chatHistory}`); + const systemPrompt = getReactPrompt(Object.values(this.tools), chatHistory); + console.log(`System prompt: ${systemPrompt}`); + this.interMessages = [{ role: 'system', content: systemPrompt }]; + + this.interMessages.push({ role: 'user', content: `<query>${question}</query>` }); + + const parser = new XMLParser(); + const builder = new XMLBuilder(); + let currentAction: string | undefined; + + for (let i = 0; i < maxTurns; i++) { + console.log(`Turn ${i + 1}/${maxTurns}`); + + const result = await this.execute(); + console.log(`Bot response: ${result}`); + this.interMessages.push({ role: 'assistant', content: result }); + + let parsedResult; + try { + parsedResult = parser.parse(result); + } catch (error) { + console.log('Error: Invalid XML response from bot'); + return '<error>Invalid response format.</error>'; + } + + const step = parsedResult[Object.keys(parsedResult)[0]]; + + for (const key in step) { + if (key === 'thought') { + console.log(`Thought: ${step[key]}`); + } else if (key === 'action') { + currentAction = step[key] as string; + console.log(`Action: ${currentAction}`); + if (this.tools[currentAction]) { + const nextPrompt = [ + { + type: 'text', + text: builder.build({ action_rules: this.tools[currentAction].getActionRule() }), + }, + ]; + this.interMessages.push({ role: 'user', content: nextPrompt }); + break; + } else { + console.log('Error: No valid action'); + this.interMessages.push({ role: 'user', content: 'No valid action, try again.' }); + break; + } + } else if (key === 'action_input') { + const actionInput = builder.build({ action_input: step[key] }); + console.log(`Action input: ${actionInput}`); + if (currentAction) { + try { + const observation = await this.processAction(currentAction, step[key]); + const nextPrompt = [{ type: 'text', text: '<observation>' }, ...observation, { type: 'text', text: '</observation>' }]; + console.log(observation); + this.interMessages.push({ role: 'user', content: nextPrompt }); + break; + } catch (error) { + console.log(`Error processing action: ${error}`); + return `<error>${error}</error>`; + } + } else { + console.log('Error: Action input without a valid action'); + return '<error>Action input without a valid action</error>'; + } + } else if (key === 'answer') { + console.log('Answer found. Ending query.'); + return result; + } + } + } + console.log(this.messages); + console.log('Reached maximum turns. Ending query.'); + return '<error>Reached maximum turns without finding an answer</error>'; + } + + private async execute(): Promise<string> { + console.log(this.interMessages); + const completion = await this.client.chat.completions.create({ + model: 'gpt-4o', + messages: this.interMessages as ChatCompletionMessageParam[], + temperature: 0, + }); + if (completion.choices[0].message.content) return completion.choices[0].message.content; + else throw new Error('No completion content found'); + } + + private async processAction(action: string, actionInput: any): Promise<any> { + if (!(action in this.tools)) { + throw new Error(`Unknown action: ${action}`); + } + + const tool = this.tools[action]; + const args: Record<string, any> = {}; + for (const paramName in tool.parameters) { + if (actionInput[paramName] !== undefined) { + args[paramName] = actionInput[paramName]; + } else if (tool.parameters[paramName].required === 'true') { + throw new Error(`Missing required parameter '${paramName}' for action '${action}'`); + } + } + + return await tool.execute(args); + } +} diff --git a/src/client/views/nodes/ChatBox/AnswerParser.ts b/src/client/views/nodes/ChatBox/AnswerParser.ts new file mode 100644 index 000000000..1162d46b0 --- /dev/null +++ b/src/client/views/nodes/ChatBox/AnswerParser.ts @@ -0,0 +1,62 @@ +import { ASSISTANT_ROLE, AssistantMessage, Citation, getChunkType } from './types'; +import { v4 as uuid } from 'uuid'; + +export class AnswerParser { + static parse(xml: string): AssistantMessage { + const answerRegex = /<answer>([\s\S]*?)<\/answer>/; + const citationRegex = /<citation chunk_id="([^"]+)" type="([^"]+)">(.*?)<\/citation>/g; + const followUpQuestionsRegex = /<follow_up_questions>([\s\S]*?)<\/follow_up_questions>/; + const questionRegex = /<question>(.*?)<\/question>/g; + + const answerMatch = answerRegex.exec(xml); + const followUpQuestionsMatch = followUpQuestionsRegex.exec(xml); + + if (!answerMatch) { + throw new Error('Invalid XML: Missing <answer> tag.'); + } + + const rawTextContent = answerMatch[1].trim(); + const textContentWithCitations = rawTextContent.replace(citationRegex, ''); + const textContent = textContentWithCitations.replace(followUpQuestionsRegex, '').trim(); + + let citations: Citation[] = []; + let match: RegExpExecArray | null; + + let plainTextOffset = 0; + let citationOffset = 0; + + while ((match = citationRegex.exec(rawTextContent)) !== null) { + const [fullMatch, chunk_id, type, direct_text] = match; + const citationStartIndex = match.index; + const citationPlainStart = citationStartIndex - citationOffset; + + citations.push({ + direct_text: direct_text.trim(), + type: getChunkType(type), + chunk_id: chunk_id, + text_location: citationPlainStart, + citation_id: uuid(), + }); + + citationOffset += fullMatch.length; + } + + let followUpQuestions: string[] = []; + if (followUpQuestionsMatch) { + const questionsText = followUpQuestionsMatch[1]; + let questionMatch: RegExpExecArray | null; + + while ((questionMatch = questionRegex.exec(questionsText)) !== null) { + followUpQuestions.push(questionMatch[1].trim()); + } + } + const assistantResponse: AssistantMessage = { + role: ASSISTANT_ROLE.ASSISTANT, + text_content: textContent, + follow_up_questions: followUpQuestions, + citations: citations, + }; + + return assistantResponse; + } +} diff --git a/src/client/views/nodes/ChatBox/ChatBox.scss b/src/client/views/nodes/ChatBox/ChatBox.scss index f1ad3d074..e39938c4f 100644 --- a/src/client/views/nodes/ChatBox/ChatBox.scss +++ b/src/client/views/nodes/ChatBox/ChatBox.scss @@ -5,16 +5,18 @@ $button-color: #007bff; $button-hover-color: darken($button-color, 10%); $shadow-color: rgba(0, 0, 0, 0.075); $border-radius: 8px; +$citation-color: #ff6347; +$citation-hover-color: darken($citation-color, 10%); +$follow-up-bg-color: #e9ecef; +$follow-up-hover-bg-color: #dee2e6; .chatBox { display: flex; flex-direction: column; - width: 100%; /* Adjust the width as needed, could be in percentage */ - height: 100%; /* Adjust the height as needed, could be in percentage */ + width: 100%; + height: 100%; background-color: $background-color; font-family: 'Helvetica Neue', Arial, sans-serif; - //margin: 20px auto; - //overflow: hidden; .scroll-box { flex-grow: 1; @@ -24,15 +26,15 @@ $border-radius: 8px; padding: 10px; display: flex; flex-direction: column-reverse; - + padding-bottom: 0; + &::-webkit-scrollbar { - width: 8px; + width: 8px; } &::-webkit-scrollbar-thumb { - background-color: darken($background-color, 10%); - border-radius: $border-radius; + background-color: darken($background-color, 10%); + border-radius: $border-radius; } - .chat-content { display: flex; @@ -42,98 +44,112 @@ $border-radius: 8px; .messages { display: flex; flex-direction: column; + .message { - padding: 10px; + padding: 10px 15px; margin-bottom: 10px; border-radius: $border-radius; background-color: lighten($background-color, 5%); box-shadow: 0 2px 5px $shadow-color; - //display: flex; - align-items: center; - max-width: 70%; + align-items: flex-start; + max-width: 90%; + width: fit-content; word-break: break-word; - .message-footer { // Assuming this is the container for the toggle button - //max-width: 70%; - - - .toggle-logs-button { - margin-top: 10px; // Padding on sides to align with the text above - width: 95%; - //display: block; // Ensures the button extends the full width of its container - text-align: center; // Centers the text inside the button - //padding: 8px 0; // Adequate padding for touch targets - background-color: $button-color; - color: #fff; - border: none; - border-radius: $border-radius; - cursor: pointer; - //transition: background-color 0.3s; - //margin-top: 10px; // Adds space above the button - box-shadow: 0 2px 4px $shadow-color; // Consistent shadow with other elements - &:hover { - background-color: $button-hover-color; - } - } - .tool-logs { - width: 100%; - background-color: $input-background; - color: $text-color; - margin-top: 5px; - //padding: 10px; - //border-radius: $border-radius; - //box-shadow: inset 0 2px 4px $shadow-color; - //transition: opacity 1s ease-in-out; - font-family: monospace; - overflow-x: auto; - max-height: 150px; // Ensuring it does not grow too large - overflow-y: auto; - } - - } - - .custom-link { - color: lightblue; - text-decoration: underline; + position: relative; + + .citation-button { + background-color: $citation-color; + color: #fff; + border: none; + border-radius: 50%; cursor: pointer; - } + width: 20px; + height: 20px; + display: inline-flex; + align-items: center; + justify-content: center; + font-size: 12px; + font-weight: bold; + margin: 0 2px; + padding: 0; + transition: background-color 0.3s; + + &:hover { + background-color: $citation-hover-color; + } + } + &.user { - align-self: flex-end; - background-color: $button-color; - color: #fff; + align-self: flex-end; + background-color: $button-color; + color: #fff; } - + &.chatbot { - align-self: flex-start; - background-color: $input-background; - color: $text-color; + align-self: flex-start; + background-color: $input-background; + color: $text-color; } - + span { - flex-grow: 1; - padding-right: 10px; + flex-grow: 1; + padding-right: 10px; } - + img { - max-width: 50px; - max-height: 50px; - border-radius: 50%; + max-width: 50px; + max-height: 50px; + border-radius: 50%; + } + } + + .follow-up-questions { + margin-top: 10px; + width: 100%; + + h4 { + margin-bottom: 5px; + font-size: 14px; + } + + .follow-up-button { + background-color: $follow-up-bg-color; + border: 1px solid #ddd; + border-radius: 8px; + padding: 8px 10px; + margin: 4px 0; + cursor: pointer; + transition: background-color 0.3s; + display: block; + width: 100%; + text-align: left; + white-space: normal; + word-wrap: break-word; + font-size: 12px; + color: $text-color; + min-height: 40px; + height: auto; + line-height: 1.3; + + &:hover { + background-color: $follow-up-hover-bg-color; + } } } } - padding-bottom: 0; } .chat-form { display: flex; - flex-grow: 1; - //height: 50px; + flex-grow: 0; bottom: 0; width: 100%; padding: 10px; background-color: $input-background; box-shadow: inset 0 -1px 2px $shadow-color; - - input[type="text"] { + margin-bottom: 0; + + input[type='text'] { flex-grow: 1; border: 1px solid darken($input-background, 10%); border-radius: $border-radius; @@ -143,22 +159,22 @@ $border-radius: 8px; button { padding: 8px 16px; - background-color: $button-color; - color: #fff; - border: none; - border-radius: $border-radius; - cursor: pointer; - transition: background-color 0.3s; - - &:hover { - background-color: $button-hover-color; - } + background-color: $button-color; + color: #fff; + border: none; + border-radius: $border-radius; + cursor: pointer; + transition: background-color 0.3s; + min-width: 80px; + + &:hover { + background-color: $button-hover-color; + } } - margin-bottom: 0; } } -.initializing-overlay { +.uploading-overlay { position: absolute; top: 0; left: 0; @@ -170,15 +186,14 @@ $border-radius: 8px; align-items: center; font-size: 1.5em; color: $text-color; - z-index: 10; // Ensure it's above all other content (may be better solution) + z-index: 10; &::before { - content: 'Initializing...'; + content: 'Uploading Docs...'; font-weight: bold; } } - .modal { position: fixed; top: 0; @@ -217,7 +232,6 @@ $border-radius: 8px; border: none; border-radius: $border-radius; cursor: pointer; - margin: 5px; transition: background-color 0.3s; &:hover { diff --git a/src/client/views/nodes/ChatBox/ChatBox.tsx b/src/client/views/nodes/ChatBox/ChatBox.tsx index 880c332ac..e3a164b3e 100644 --- a/src/client/views/nodes/ChatBox/ChatBox.tsx +++ b/src/client/views/nodes/ChatBox/ChatBox.tsx @@ -1,104 +1,95 @@ -import { MathJaxContext } from 'better-react-mathjax'; -import { action, makeObservable, observable, observe, reaction, runInAction } from 'mobx'; +import { action, computed, makeObservable, observable, observe, reaction, runInAction, ObservableSet } from 'mobx'; import { observer } from 'mobx-react'; import OpenAI, { ClientOptions } from 'openai'; -import { ImageFile, Message } from 'openai/resources/beta/threads/messages'; -import { RunStep } from 'openai/resources/beta/threads/runs/steps'; import * as React from 'react'; -import { Doc } from '../../../../fields/Doc'; -import { Id } from '../../../../fields/FieldSymbols'; +import { Doc, DocListCast } from '../../../../fields/Doc'; import { CsvCast, DocCast, PDFCast, StrCast } from '../../../../fields/Types'; -import { CsvField } from '../../../../fields/URLField'; import { Networking } from '../../../Network'; -import { DocUtils } from '../../../documents/DocUtils'; import { DocumentType } from '../../../documents/DocumentTypes'; import { Docs } from '../../../documents/Documents'; -import { DocumentManager } from '../../../util/DocumentManager'; import { LinkManager } from '../../../util/LinkManager'; import { ViewBoxAnnotatableComponent } from '../../DocComponent'; import { FieldView, FieldViewProps } from '../FieldView'; import './ChatBox.scss'; -import MessageComponent from './MessageComponent'; -import { ANNOTATION_LINK_TYPE, ASSISTANT_ROLE, AssistantMessage, DOWNLOAD_TYPE } from './types'; +import MessageComponentBox from './MessageComponent'; +import { ASSISTANT_ROLE, AssistantMessage, AI_Document, Citation, CHUNK_TYPE, Chunk, getChunkType } from './types'; +import { Vectorstore } from './vectorstore/VectorstoreUpload'; +import { CollectionFreeFormDocumentView } from '../CollectionFreeFormDocumentView'; +import { CollectionFreeFormView } from '../../collections/collectionFreeForm'; +import { Agent } from './Agent'; +import dotenv from 'dotenv'; +import { DocData, DocViews } from '../../../../fields/DocSymbols'; +import { DocumentView } from '../DocumentView'; +import { AnswerParser } from './AnswerParser'; +import { DocumentManager } from '../../../util/DocumentManager'; +import { UUID } from 'bson'; +import { v4 as uuidv4 } from 'uuid'; +import { aS } from '@fullcalendar/core/internal-common'; +import { computeRect } from '@fullcalendar/core/internal'; +import { DocUtils } from '../../../documents/DocUtils'; + +dotenv.config(); @observer export class ChatBox extends ViewBoxAnnotatableComponent<FieldViewProps>() { - @observable modalStatus = false; - @observable currentFile = { url: '' }; @observable history: AssistantMessage[] = []; @observable.deep current_message: AssistantMessage | undefined = undefined; @observable isLoading: boolean = false; - @observable isInitializing: boolean = true; - @observable expandedLogIndex: number | null = null; - @observable linked_docs_to_add: Doc[] = []; - + @observable isUploadingDocs: boolean = false; + @observable expandedScratchpadIndex: number | null = null; + @observable inputValue: string = ''; + @observable private linked_docs_to_add: ObservableSet<Doc> = observable.set(); private openai: OpenAI; - private interim_history: string = ''; - private assistantID: string = ''; - private threadID: string = ''; + private vectorstore_id: string; + private documents: AI_Document[] = []; private _oldWheel: any; - private vectorStoreID: string = ''; - private mathJaxConfig: any; - private linkedCsvIDs: string[] = []; + private vectorstore: Vectorstore; + private agent: Agent; // Add the ChatBot instance public static LayoutString(fieldKey: string) { return FieldView.LayoutString(ChatBox, fieldKey); } + constructor(props: FieldViewProps) { super(props); makeObservable(this); this.openai = this.initializeOpenAI(); - this.history = []; - this.threadID = StrCast(this.dataDoc.thread_id); - this.assistantID = StrCast(this.dataDoc.assistant_id); - this.vectorStoreID = StrCast(this.dataDoc.vector_store_id); - this.openai = this.initializeOpenAI(); - if (this.assistantID === '' || this.threadID === '' || this.vectorStoreID === '') { - this.createAssistant(); + if (StrCast(this.dataDoc.vectorstore_id) == '') { + console.log('new_id'); + this.vectorstore_id = uuidv4(); + this.dataDoc.vectorstore_id = this.vectorstore_id; } else { - this.retrieveCsvUrls(); - this.isInitializing = false; + this.vectorstore_id = StrCast(this.dataDoc.vectorstore_id); } - this.mathJaxConfig = { - loader: { load: ['input/asciimath'] }, - tex: { - inlineMath: [ - ['$', '$'], - ['\\(', '\\)'], - ], - displayMath: [ - ['$$', '$$'], - ['[', ']'], - ], - }, - }; + this.vectorstore = new Vectorstore(this.vectorstore_id); + this.agent = new Agent(this.vectorstore, this.retrieveSummaries, this.retrieveFormattedHistory); + reaction( - () => this.history.map((msg: AssistantMessage) => ({ role: msg.role, text: msg.text, image: msg.image, tool_logs: msg.tool_logs, links: msg.links })), + () => this.history.map((msg: AssistantMessage) => ({ role: msg.role, text_content: msg.text_content, follow_up_questions: msg.follow_up_questions, citations: msg.citations })), serializableHistory => { this.dataDoc.data = JSON.stringify(serializableHistory); } ); } - toggleToolLogs = (index: number) => { - this.expandedLogIndex = this.expandedLogIndex === index ? null : index; + @action + addDocToVectorstore = async (newLinkedDoc: Doc) => { + await this.vectorstore.addAIDoc(newLinkedDoc); }; - retrieveCsvUrls() { - const linkedDocs = LinkManager.Instance.getAllRelatedLinks(this.Document) - .map(d => DocCast(LinkManager.getOppositeAnchor(d, this.Document))) - .map(d => DocCast(d?.annotationOn, d)) - .filter(d => d); + // @action + // uploadNewDocument = async (newDoc: Doc) => { + // const local_file_path: string = CsvCast(newDoc.data, PDFCast(newDoc.data)).url.pathname; + // const { document_json } = await Networking.PostToServer('/createDocument', { file_path: local_file_path }); + // this.documents.push(...document_json.map(convertToAIDocument)); + // //newDoc['ai_document'] = document_json; + // }; - linkedDocs.forEach(doc => { - const aiFieldId = StrCast(doc[this.Document[Id] + '_ai_field_id']); - if (CsvCast(doc.data)) { - this.linkedCsvIDs.push(StrCast(aiFieldId)); - console.log(this.linkedCsvIDs); - } - }); - } + @action + toggleToolLogs = (index: number) => { + this.expandedScratchpadIndex = this.expandedScratchpadIndex === index ? null : index; + }; initializeOpenAI() { const configuration: ClientOptions = { @@ -114,390 +105,159 @@ export class ChatBox extends ViewBoxAnnotatableComponent<FieldViewProps>() { } }; - createLink = (linkInfo: string, startIndex: number, endIndex: number, linkType: ANNOTATION_LINK_TYPE, annotationIndex: number = 0) => { - const text = this.interim_history; - const subString = this.current_message?.text.substring(startIndex, endIndex) ?? ''; - if (!text) return; - const textToDisplay = `${annotationIndex}`; - let fileInfo = linkInfo; - const fileName = subString.split('/')[subString.split('/').length - 1]; - if (linkType === ANNOTATION_LINK_TYPE.DOWNLOAD_FILE) { - fileInfo = linkInfo + '!!!' + fileName; - } - - const formattedLink = `[${textToDisplay}](${fileInfo}~~~${linkType})`; - console.log(formattedLink); - const newText = text.replace(subString, formattedLink); - runInAction(() => { - this.interim_history = newText; - console.log(newText); - this.current_message?.links?.push({ - start: startIndex, - end: endIndex, - url: linkType === ANNOTATION_LINK_TYPE.DOWNLOAD_FILE ? fileName : linkInfo, - id: linkType === ANNOTATION_LINK_TYPE.DOWNLOAD_FILE ? linkInfo : undefined, - link_type: linkType, - }); - }); - }; - - @action - createAssistant = async () => { - this.isInitializing = true; - try { - const vectorStore = await this.openai.beta.vectorStores.create({ - name: 'Vector Store for Assistant', - }); - const assistant = await this.openai.beta.assistants.create({ - name: 'Document Analyser Assistant', - instructions: ` - You will analyse documents with which you are provided. You will answer questions and provide insights based on the information in the documents. - For writing math formulas: - You have a MathJax render environment. - - Write all in-line equations within a single dollar sign, $, to render them as TeX (this means any time you want to use a dollar sign to represent a dollar sign itself, you must escape it with a backslash: "$"); - - Use a double dollar sign, $$, to render equations on a new line; - Example: $$x^2 + 3x$$ is output for "x² + 3x" to appear as TeX.`, - model: 'gpt-4-turbo', - tools: [{ type: 'file_search' }, { type: 'code_interpreter' }], - tool_resources: { - file_search: { - vector_store_ids: [vectorStore.id], - }, - code_interpreter: { - file_ids: this.linkedCsvIDs, - }, - }, - }); - const thread = await this.openai.beta.threads.create(); - - runInAction(() => { - this.dataDoc.assistant_id = assistant.id; - this.dataDoc.thread_id = thread.id; - this.dataDoc.vector_store_id = vectorStore.id; - this.assistantID = assistant.id; - this.threadID = thread.id; - this.vectorStoreID = vectorStore.id; - this.isInitializing = false; - }); - } catch (error) { - console.error('Initialization failed:', error); - this.isInitializing = false; - } - }; - - @action - runAssistant = async (inputText: string) => { - // Ensure an assistant and thread are created - if (!this.assistantID || !this.threadID || !this.vectorStoreID) { - await this.createAssistant(); - console.log('Assistant and thread created:', this.assistantID, this.threadID); - } - let currentText: string = ''; - let currentToolCallMessage: string = ''; - - // Send the user's input to the assistant - await this.openai.beta.threads.messages.create(this.threadID, { - role: 'user', - content: inputText, - }); - - // Listen to the streaming responses - const stream = this.openai.beta.threads.runs - .stream(this.threadID, { - assistant_id: this.assistantID, - }) - .on('runStepCreated', (runStep: RunStep) => { - currentText = ''; - runInAction(() => { - this.current_message = { role: ASSISTANT_ROLE.ASSISTANT, text: currentText, tool_logs: '', links: [] }; - }); - this.isLoading = true; - }) - .on('toolCallDelta', (toolCallDelta, snapshot) => { - this.isLoading = false; - if (toolCallDelta.type === 'code_interpreter') { - if (toolCallDelta.code_interpreter?.input) { - currentToolCallMessage += toolCallDelta.code_interpreter.input; - runInAction(() => { - if (this.current_message) { - this.current_message.tool_logs = currentToolCallMessage; - } - }); - } - if (toolCallDelta.code_interpreter?.outputs) { - currentToolCallMessage += '\n Code interpreter output:'; - toolCallDelta.code_interpreter.outputs.forEach(output => { - if (output.type === 'logs') { - runInAction(() => { - if (this.current_message) { - this.current_message.tool_logs += '\n|' + output.logs; - } - }); - } - }); - } - } - }) - .on('textDelta', (textDelta, snapshot) => { - this.isLoading = false; - currentText += textDelta.value; - runInAction(() => { - if (this.current_message) { - // this.current_message = {...this.current_message, text: current_text}; - this.current_message.text = currentText; - } - }); - }) - .on('messageDone', async event => { - console.log(event); - const textItem = event.content.find(item => item.type === 'text'); - if (textItem && textItem.type === 'text') { - const { text } = textItem; - console.log(text.value); - try { - runInAction(() => { - this.interim_history = text.value; - }); - } catch (e) { - console.error('Error parsing JSON response:', e); - } - - const { annotations } = text; - console.log('Annotations: ' + annotations); - let index = 0; - annotations.forEach(async annotation => { - console.log(' ' + annotation); - console.log(' ' + annotation.text); - if (annotation.type === 'file_path') { - const { file_path: filePath } = annotation; - const fileToDownload = filePath.file_id; - console.log(fileToDownload); - if (filePath) { - console.log(filePath); - console.log(fileToDownload); - this.createLink(fileToDownload, annotation.start_index, annotation.end_index, ANNOTATION_LINK_TYPE.DOWNLOAD_FILE); - } - } else { - const { file_citation: fileCitation } = annotation; - if (fileCitation) { - const citedFile = await this.openai.files.retrieve(fileCitation.file_id); - const citationUrl = citedFile.filename; - this.createLink(citationUrl, annotation.start_index, annotation.end_index, ANNOTATION_LINK_TYPE.DASH_DOC, index); - index++; - } - } - }); - runInAction(() => { - if (this.current_message) { - console.log('current message: ' + this.current_message.text); - this.current_message.text = this.interim_history; - this.history.push({ ...this.current_message }); - this.current_message = undefined; - } - }); - } - }) - .on('toolCallDone', toolCall => { - runInAction(() => { - if (this.current_message && currentToolCallMessage) { - this.current_message.tool_logs = currentToolCallMessage; - } - }); - }) - .on('imageFileDone', (content: ImageFile, snapshot: Message) => { - console.log('Image file done:', content); - }) - .on('end', () => { - console.log('Streaming done'); - }); - }; - - @action - goToLinkedDoc = async (link: string) => { - const linkedDocs = LinkManager.Instance.getAllRelatedLinks(this.Document) - .map(d => DocCast(LinkManager.getOppositeAnchor(d, this.Document))) - .map(d => DocCast(d?.annotationOn, d)) - .filter(d => d); - - const linkedDoc = linkedDocs.find(doc => { - const docUrl = CsvCast(doc.data, PDFCast(doc.data)).url.pathname.replace('/files/pdfs/', '').replace('/files/csvs/', ''); - console.log('URL: ' + docUrl + ' Citation URL: ' + link); - return link === docUrl; - }); - - if (linkedDoc) { - await DocumentManager.Instance.showDocument(DocCast(linkedDoc), { willZoomCentered: true }, () => {}); - } - }; + // getAssistantResponse() { + // return Docs.Create.MessageDocument(text, {}); + // } @action askGPT = async (event: React.FormEvent<HTMLFormElement>): Promise<void> => { event.preventDefault(); + this.inputValue = ''; const textInput = event.currentTarget.elements.namedItem('messageInput') as HTMLInputElement; const trimmedText = textInput.value.trim(); - if (!this.assistantID || !this.threadID) { - try { - await this.createAssistant(); - } catch (err) { - console.error('Error:', err); - } - } - if (trimmedText) { try { textInput.value = ''; runInAction(() => { - this.history.push({ role: ASSISTANT_ROLE.USER, text: trimmedText }); + this.history.push({ role: ASSISTANT_ROLE.USER, text_content: trimmedText }); + this.isLoading = true; }); - await this.runAssistant(trimmedText); - this.dataDoc.data = this.history.toString(); + + const response = await this.agent.askAgent(trimmedText); // Use the chatbot to get the response + runInAction(() => { + this.history.push(AnswerParser.parse(response)); + }); + this.dataDoc.data = JSON.stringify(this.history); } catch (err) { console.error('Error:', err); + runInAction(() => { + this.history.push({ role: ASSISTANT_ROLE.ASSISTANT, text_content: 'Sorry, I encountered an error while processing your request.' }); + }); + } finally { + runInAction(() => { + this.isLoading = false; + }); } } }; @action - uploadLinks = async (linkedDocs: Doc[]) => { - if (this.isInitializing) { - console.log('Initialization in progress, upload aborted.'); - return; + updateMessageCitations = (index: number, citations: Citation[]) => { + if (this.history[index]) { + this.history[index].citations = citations; } - const urls = linkedDocs.map(doc => CsvCast(doc.data, PDFCast(doc.data)).url.pathname); - const csvUrls = urls.filter(url => url.endsWith('.csv')); - console.log(this.assistantID, this.threadID, urls); + }; - const { openai_file_ids: openaiFileIds } = await Networking.PostToServer('/uploadPDFToVectorStore', { urls, threadID: this.threadID, assistantID: this.assistantID, vector_store_id: this.vectorStoreID }); + @action + handleCitationClick = (citation: Citation) => { + console.log('Citation clicked:', citation); + const currentLinkedDocs: Doc[] = this.linkedDocs; + const chunk_id = citation.chunk_id; + for (let doc of currentLinkedDocs) { + if (doc.chunk_simpl) { + //console.log(JSON.parse(StrCast(doc.chunk_simpl))); + const doc_chunk_simpl = JSON.parse(StrCast(doc.chunk_simpl)); + console.log(doc_chunk_simpl); + const text_chunks = doc_chunk_simpl.text_chunks as [{ chunk_id: string; start_page: number; end_page: number }] | []; + const image_chunks = doc_chunk_simpl.image_chunks as [{ chunk_id: string; location: string; page: number }] | []; + + const found_text_chunk = text_chunks.find(chunk => chunk.chunk_id === chunk_id); + if (found_text_chunk) { + const doc_url = CsvCast(doc.data, PDFCast(doc.data)).url.pathname; + console.log('URL: ' + doc_url); + + //const ai_field_id = doc[this.Document[Id] + '_ai_field_id']; + DocumentManager.Instance.showDocument(doc, { willZoomCentered: true }, () => { + console.log(doc.data); + //look at context path for each docview and choose the doc view that has as + //its parent the same collection view the chatbox is in + const first_view = Array.from(doc[DocViews])[0]; + first_view.ComponentView?.search?.(citation.direct_text); + }); + } - linkedDocs.forEach((doc, i) => { - doc[this.Document[Id] + '_ai_field_id'] = openaiFileIds[i]; - console.log('AI Field ID: ' + openaiFileIds[i]); - }); + const found_image_chunk = image_chunks.find(chunk => chunk.chunk_id === chunk_id); + if (found_image_chunk) { + const location_string: string = found_image_chunk.location; - if (csvUrls.length > 0) { - for (let i = 0; i < csvUrls.length; i++) { - this.linkedCsvIDs.push(openaiFileIds[urls.indexOf(csvUrls[i])]); - } - console.log('linked csvs:' + this.linkedCsvIDs); - await this.openai.beta.assistants.update(this.assistantID, { - tools: [{ type: 'file_search' }, { type: 'code_interpreter' }], - tool_resources: { - file_search: { - vector_store_ids: [this.vectorStoreID], - }, - code_interpreter: { - file_ids: this.linkedCsvIDs, - }, - }, - }); - } - }; + // Extract variables from location_string + const values = location_string.replace(/[\[\]]/g, '').split(','); - downloadToComputer = (url: string, fileName: string) => { - fetch(url, { method: 'get', mode: 'no-cors', referrerPolicy: 'no-referrer' }) - .then(res => res.blob()) - .then(res => { - const aElement = document.createElement('a'); - aElement.setAttribute('download', fileName); - const href = URL.createObjectURL(res); - aElement.href = href; - aElement.setAttribute('target', '_blank'); - aElement.click(); - URL.revokeObjectURL(href); - }); - }; + // Ensure we have exactly 4 values + if (values.length !== 4) { + console.error('Location string must contain exactly 4 numbers'); + return; // or handle this error as appropriate + } - createDocumentInDash = async (url: string) => { - const fileSuffix = url.substring(url.lastIndexOf('.') + 1); - console.log(fileSuffix); - let doc: Doc | null = null; - switch (fileSuffix) { - case 'pdf': - doc = DocCast(await DocUtils.DocumentFromType('pdf', url, {})); - break; - case 'csv': - doc = DocCast(await DocUtils.DocumentFromType('csv', url, {})); - break; - case 'png': - case 'jpg': - case 'jpeg': - doc = DocCast(await DocUtils.DocumentFromType('image', url, {})); - break; - default: - console.error('Unsupported file type:', fileSuffix); - break; - } - if (doc) { - doc && this._props.addDocument?.(doc); - await DocumentManager.Instance.showDocument(doc, { willZoomCentered: true }, () => {}); - } - }; + const x1 = parseFloat(values[0]) * Doc.NativeWidth(doc); + const y1 = parseFloat(values[1]) * Doc.NativeHeight(doc); + const x2 = parseFloat(values[2]) * Doc.NativeWidth(doc); + const y2 = parseFloat(values[3]) * Doc.NativeHeight(doc); - downloadFile = async (fileInfo: string, downloadType: DOWNLOAD_TYPE) => { - try { - console.log(fileInfo); - const [fileId, fileName] = fileInfo.split(/!!!/); - const { file_path: filePath } = await Networking.PostToServer('/downloadFileFromOpenAI', { file_id: fileId, file_name: fileName }); - const fileLink = CsvCast(new CsvField(filePath)).url.href; - if (downloadType === DOWNLOAD_TYPE.DASH) { - this.createDocumentInDash(fileLink); - } else { - this.downloadToComputer(fileLink, fileName); - } - } catch (error) { - console.error('Error downloading file:', error); - } - }; + const annotationKey = Doc.LayoutFieldKey(doc) + '_annotations'; - handleDownloadToDevice = () => { - this.downloadFile(this.currentFile.url, DOWNLOAD_TYPE.DEVICE); - this.modalStatus = false; // Close the modal after the action - this.currentFile = { url: '' }; // Reset the current file - }; + const existingDoc = DocListCast(doc[DocData][annotationKey]).find(d => d.citation_id === citation.citation_id); + const highlight_doc = existingDoc ?? this.createImageCitationHighlight(x1, y1, x2, y2, citation, annotationKey, doc); - handleAddToDash = () => { - // Assuming `downloadFile` is a method that handles adding to Dash - this.downloadFile(this.currentFile.url, DOWNLOAD_TYPE.DASH); - this.modalStatus = false; // Close the modal after the action - this.currentFile = { url: '' }; // Reset the current file + DocumentManager.Instance.showDocument(highlight_doc, { willZoomCentered: true }, () => {}); + } + } + } + // You can implement additional functionality here, such as showing a modal with the full citation content }; - renderModal = () => { - if (!this.modalStatus) return null; - - return ( - <div className="modal"> - <div className="modal-content"> - <h4>File Actions</h4> - <p>Choose an action for the file:</p> - <button type="button" onClick={this.handleDownloadToDevice}> - Download to Device - </button> - <button type="button" onClick={this.handleAddToDash}> - Add to Dash - </button> - <button - type="button" - onClick={() => { - this.modalStatus = false; - }}> - Cancel - </button> - </div> - </div> - ); - }; - @action - showModal = () => { - this.modalStatus = true; + createImageCitationHighlight = (x1: number, y1: number, x2: number, y2: number, citation: Citation, annotationKey: string, pdfDoc: Doc): Doc => { + const highlight_doc = Docs.Create.FreeformDocument([], { + x: x1, + y: y1, + _width: x2 - x1, + _height: y2 - y1, + backgroundColor: 'rgba(255, 255, 0, 0.5)', + }); + highlight_doc[DocData].citation_id = citation.citation_id; + Doc.AddDocToList(pdfDoc[DocData], annotationKey, highlight_doc); + highlight_doc.annotationOn = pdfDoc; + Doc.SetContainer(highlight_doc, pdfDoc); + return highlight_doc; }; - @action - setCurrentFile = (file: { url: string }) => { - this.currentFile = file; - }; + // @action + // uploadLinks = async (linkedDocs: Doc[]) => { + // if (this.isUploadingDocs) { + // console.log('Initialization in progress, upload aborted.'); + // return; + // } + // const urls: string[] = linkedDocs.map(doc => CsvCast(doc.data, PDFCast(doc.data)).url.pathname); + // const csvUrls: string[] = urls.filter(url => url.endsWith('.csv')); + // console.log(this.assistantID, this.threadID, urls); + + // await Networking.PostToServer('/uploadPDFs', { file_path: urls[0] }); + + // // linkedDocs.forEach((doc, i) => { + // // doc[this.Document[Id] + '_ai_field_id'] = openaiFileIds[i]; + // // console.log('AI Field ID: ' + openaiFileIds[i]); + // // }); + + // // if (csvUrls.length > 0) { + // // for (let i = 0; i < csvUrls.length; i++) { + // // this.linkedCsvIDs.push(openaiFileIds[urls.indexOf(csvUrls[i])]); + // // } + // // console.log('linked csvs:' + this.linkedCsvIDs); + // // await this.openai.beta.assistants.update(this.assistantID, { + // // tools: [{ type: 'file_search' }, { type: 'code_interpreter' }], + // // tool_resources: { + // // file_search: { + // // vector_store_ids: [this.vectorStoreID], + // // }, + // // code_interpreter: { + // // file_ids: this.linkedCsvIDs, + // // }, + // // }, + // // }); + // // } + // }; componentDidMount() { this._props.setContentViewBox?.(this); @@ -505,17 +265,20 @@ export class ChatBox extends ViewBoxAnnotatableComponent<FieldViewProps>() { try { const storedHistory = JSON.parse(StrCast(this.dataDoc.data)); runInAction(() => { - this.history = storedHistory.map((msg: AssistantMessage) => ({ - role: msg.role, - text: msg.text, - quote: msg.quote, - tool_logs: msg.tool_logs, - image: msg.image, - })); + this.history.push( + ...storedHistory.map((msg: AssistantMessage) => ({ + role: msg.role, + text_content: msg.text_content, + follow_up_questions: msg.follow_up_questions, + citations: msg.citations, + })) + ); }); } catch (e) { console.error('Failed to parse history from dataDoc:', e); } + } else { + this.history = [{ role: ASSISTANT_ROLE.ASSISTANT, text_content: 'Welcome to the Document Analyser Assistant! Link a document or ask questions to get started.' }]; } reaction( () => { @@ -526,79 +289,114 @@ export class ChatBox extends ViewBoxAnnotatableComponent<FieldViewProps>() { return linkedDocs; }, - linked => this.linked_docs_to_add.push(...linked.filter(linkedDoc => !this.linked_docs_to_add.includes(linkedDoc))) + linked => linked.forEach(doc => this.linked_docs_to_add.add(doc)) ); - observe( - // right now this skips during initialization which is necessary because it would be blank - // However, it will upload the same link twice when it is - this.linked_docs_to_add, - change => { - // observe pushes/splices on a user link DB 'data' field (should only happen for local changes) - switch (change.type as any) { - case 'splice': - if ((change as any).addedCount > 0) { - // maybe check here if its already in the urls datadoc array so doesn't add twice - console.log((change as any).added as Doc[]); - this.uploadLinks((change as any).added as Doc[]); - } - // (change as any).removed.forEach((link: any) => remLinkFromDoc(toRealField(link))); - break; - case 'update': // let oldValue = change.oldValue; - default: - } - }, - true + observe(this.linked_docs_to_add, change => { + if (change.type === 'add') { + runInAction(() => { + this.isUploadingDocs = true; + }); + this.addDocToVectorstore(change.newValue); + runInAction(() => { + this.isUploadingDocs = false; + }); + } else if (change.type === 'delete') { + console.log('Deleted docs: ', change.oldValue); + } + }); + } + + // case 'splice': + // if ((change as any).addedCount > 0) { + // // maybe check here if its already in the urls datadoc array so doesn't add twice + // console.log((change as any).added as Doc[]); + // this.addDocsToVectorstore((change as any).added as Doc[]); + // } + // // (change as any).removed.forEach((link: any) => remLinkFromDoc(toRealField(link))); + // break; + // case 'update': // let oldValue = change.oldValue; + // default: + + @computed + get linkedDocs() { + //return (CollectionFreeFormDocumentView.from(this._props.DocumentView?.())?._props.parent as CollectionFreeFormView)?.childDocs.filter(doc => doc != this.Document) ?? []; + return LinkManager.Instance.getAllRelatedLinks(this.Document) + .map(d => DocCast(LinkManager.getOppositeAnchor(d, this.Document))) + .map(d => DocCast(d?.annotationOn, d)) + .filter(d => d); + } + + @computed + get summaries(): string { + return ( + LinkManager.Instance.getAllRelatedLinks(this.Document) + .map(d => DocCast(LinkManager.getOppositeAnchor(d, this.Document))) + .map(d => DocCast(d?.annotationOn, d)) + .filter(d => d) + .map((doc, index) => `${index + 1}) ${doc.summary}`) + .join('\n') + '\n' ); } + @computed + get formattedHistory(): string { + let history = '<chat_history>\n'; + for (const message of this.history) { + history += `<${message.role}>${message.text_content}</${message.role}>\n`; + } + history += '</chat_history>'; + return history; + } + + retrieveSummaries = () => { + return this.summaries; + }; + + retrieveFormattedHistory = () => { + return this.formattedHistory; + }; + + @action + handleFollowUpClick = (question: string) => { + console.log('Follow-up question clicked:', question); + this.inputValue = question; + }; render() { return ( - <MathJaxContext config={this.mathJaxConfig}> - <div className="chatBox"> - {this.isInitializing && <div className="initializing-overlay">Initializing...</div>} - {this.renderModal()} - <div - className="scroll-box chat-content" - ref={r => { - this._oldWheel?.removeEventListener('wheel', this.onPassiveWheel); - this._oldWheel = r; - r?.addEventListener('wheel', this.onPassiveWheel, { passive: false }); - }}> - <div className="messages"> - {this.history.map((message, index) => ( - <MessageComponent - key={index} - message={message} - toggleToolLogs={this.toggleToolLogs} - expandedLogIndex={this.expandedLogIndex} - index={index} - showModal={this.showModal} - goToLinkedDoc={this.goToLinkedDoc} - setCurrentFile={this.setCurrentFile} - /> - ))} - {!this.current_message ? null : ( - <MessageComponent - key={this.history.length} - message={this.current_message} - toggleToolLogs={this.toggleToolLogs} - expandedLogIndex={this.expandedLogIndex} - index={this.history.length} - showModal={this.showModal} - goToLinkedDoc={this.goToLinkedDoc} - setCurrentFile={this.setCurrentFile} - isCurrent - /> - )} - </div> + <div className="chatBox"> + {this.isUploadingDocs && <div className="uploading-overlay"></div>} + <div + className="scroll-box chat-content" + ref={r => { + this._oldWheel?.removeEventListener('wheel', this.onPassiveWheel); + this._oldWheel = r; + r?.addEventListener('wheel', this.onPassiveWheel, { passive: false }); + }}> + <div className="messages"> + {this.history.map((message, index) => ( + //<DocumentView key={index} Document={message} index={index} onFollowUpClick={this.handleFollowUpClick} onCitationClick={this.handleCitationClick} updateMessageCitations={this.updateMessageCitations} /> + <MessageComponentBox key={index} message={message} index={index} onFollowUpClick={this.handleFollowUpClick} onCitationClick={this.handleCitationClick} updateMessageCitations={this.updateMessageCitations} /> + ))} + {this.current_message && ( + <MessageComponentBox + key={this.history.length} + message={this.current_message} + index={this.history.length} + onFollowUpClick={this.handleFollowUpClick} + onCitationClick={this.handleCitationClick} + updateMessageCitations={this.updateMessageCitations} + /> + )} </div> - <form onSubmit={this.askGPT} className="chat-form"> - <input type="text" name="messageInput" autoComplete="off" placeholder="Type a message..." /> - <button type="submit">Send</button> - </form> </div> - </MathJaxContext> + <form onSubmit={this.askGPT} className="chat-form"> + <input type="text" name="messageInput" autoComplete="off" placeholder="Type a message..." value={this.inputValue} onChange={e => (this.inputValue = e.target.value)} /> + <button type="submit" disabled={this.isLoading}> + {this.isLoading ? 'Thinking...' : 'Send'} + </button> + </form> + </div> ); } } diff --git a/src/client/views/nodes/ChatBox/MessageComponent.tsx b/src/client/views/nodes/ChatBox/MessageComponent.tsx index f27a18891..fb4a56bc3 100644 --- a/src/client/views/nodes/ChatBox/MessageComponent.tsx +++ b/src/client/views/nodes/ChatBox/MessageComponent.tsx @@ -1,82 +1,77 @@ -/* eslint-disable jsx-a11y/control-has-associated-label */ -/* eslint-disable react/require-default-props */ -import { MathJax, MathJaxContext } from 'better-react-mathjax'; -import { observer } from 'mobx-react'; import React from 'react'; -import * as Tb from 'react-icons/tb'; +import { observer } from 'mobx-react'; import ReactMarkdown from 'react-markdown'; -import './MessageComponent.scss'; -import { AssistantMessage } from './types'; +import { AssistantMessage, Citation } from './types'; -const TbCircles = [ - Tb.TbCircleNumber0Filled, - Tb.TbCircleNumber1Filled, - Tb.TbCircleNumber2Filled, - Tb.TbCircleNumber3Filled, - Tb.TbCircleNumber4Filled, - Tb.TbCircleNumber5Filled, - Tb.TbCircleNumber6Filled, - Tb.TbCircleNumber7Filled, - Tb.TbCircleNumber8Filled, - Tb.TbCircleNumber9Filled, -]; interface MessageComponentProps { message: AssistantMessage; - toggleToolLogs: (index: number) => void; - expandedLogIndex: number | null; index: number; - showModal: () => void; - goToLinkedDoc: (url: string) => void; - setCurrentFile: (file: { url: string }) => void; - isCurrent?: boolean; + onFollowUpClick: (question: string) => void; + onCitationClick: (citation: Citation) => void; + updateMessageCitations: (index: number, citations: Citation[]) => void; } -const LinkRendererWrapper = (goToLinkedDoc: (url: string) => void, showModal: () => void, setCurrentFile: (file: { url: string }) => void) => - function LinkRenderer({ href, children }: { href?: string; children?: React.ReactNode }) { - const Children = TbCircles[Number(children)]; // pascal case variable needed to convert IconType to JSX.Element tag - const [, aurl, linkType] = href?.match(/([a-zA-Z0-9_.!-]+)~~~(citation|file_path)/) ?? [undefined, href, null]; - const renderType = (content: JSX.Element | null, click: (url: string) => void):JSX.Element => ( - // eslint-disable-next-line jsx-a11y/anchor-is-valid - <a className={`MessageComponent-${linkType}`} - href="#" - onClick={e => { - e.preventDefault(); - aurl && click(aurl); - }}> - {content} - </a> - ); // prettier-ignore - switch (linkType) { - case 'citation': return renderType(<Children />, (url: string) => goToLinkedDoc(url)); - case 'file_path': return renderType(null, (url: string) => { showModal(); setCurrentFile({ url }); }); - default: return null; - } // prettier-ignore +const MessageComponentBox: React.FC<MessageComponentProps> = function ({ message, index, onFollowUpClick, onCitationClick, updateMessageCitations }) { + const renderContent = (content: string) => { + if (!message.citations || message.citations.length === 0) { + return <ReactMarkdown>{content}</ReactMarkdown>; + } + + const parts = []; + let lastIndex = 0; + + message.citations.forEach((citation, idx) => { + const location = citation.text_location; + const textBefore = content.slice(lastIndex, location); + parts.push(<ReactMarkdown key={`md-${idx}`}>{textBefore}</ReactMarkdown>); + const citationButton = ( + <button + key={`citation-${idx}`} + className="citation-button" + onClick={() => onCitationClick(citation)} + style={{ + display: 'inline-flex', + alignItems: 'center', + justifyContent: 'center', + width: '20px', + height: '20px', + borderRadius: '50%', + border: 'none', + background: '#ff6347', + color: 'white', + fontSize: '12px', + fontWeight: 'bold', + cursor: 'pointer', + margin: '0 2px', + padding: 0, + }}> + {idx + 1} + </button> + ); + parts.push(citationButton); + lastIndex = location; + }); + + parts.push(<ReactMarkdown key="md-last">{content.slice(lastIndex)}</ReactMarkdown>); + + return parts; }; -const MessageComponent: React.FC<MessageComponentProps> = function ({ message, toggleToolLogs, expandedLogIndex, goToLinkedDoc, index, showModal, setCurrentFile, isCurrent = false }) { - // const messageClass = `${message.role} ${isCurrent ? 'current-message' : ''}`; return ( <div className={`message ${message.role}`}> - <MathJaxContext> - <MathJax dynamic hideUntilTypeset="every"> - <ReactMarkdown components={{ a: LinkRendererWrapper(goToLinkedDoc, showModal, setCurrentFile) }}>{message.text}</ReactMarkdown> - </MathJax> - </MathJaxContext> - {message.image && <img src={message.image} alt="" />} - <div className="message-footer"> - {message.tool_logs && ( - <button type="button" className="toggle-logs-button" onClick={() => toggleToolLogs(index)}> - {expandedLogIndex === index ? 'Hide Code Interpreter Logs' : 'Show Code Interpreter Logs'} - </button> - )} - {expandedLogIndex === index && ( - <div className="tool-logs"> - <pre>{message.tool_logs}</pre> - </div> - )} - </div> + <div>{renderContent(message.text_content)}</div> + {message.follow_up_questions && message.follow_up_questions.length > 0 && ( + <div className="follow-up-questions"> + <h4>Follow-up Questions:</h4> + {message.follow_up_questions.map((question, idx) => ( + <button key={idx} className="follow-up-button" onClick={() => onFollowUpClick(question)}> + {question} + </button> + ))} + </div> + )} </div> ); }; -export default observer(MessageComponent); +export default observer(MessageComponentBox); diff --git a/src/client/views/nodes/ChatBox/prompts.ts b/src/client/views/nodes/ChatBox/prompts.ts new file mode 100644 index 000000000..d5eb99cb2 --- /dev/null +++ b/src/client/views/nodes/ChatBox/prompts.ts @@ -0,0 +1,150 @@ +// prompts.ts + +import { Tool } from './types'; + +export function getReactPrompt(tools: Tool[], chatHistory: string): string { + const toolDescriptions = tools.map(tool => `${tool.name}:\n${tool.briefSummary}`).join('\n*****\n'); + + return ` + You run in a loop of Thought, Action, (PAUSE), Action Input, (PAUSE), Observation. + (this Thought/Action/PAUSE/Action Input/PAUSE/Observation can repeat N times) + Contain each stage of the loop within an XML element that specifies the stage type (e.g. <thought>content of the thought</thought>). + At the end of the loop, you output an Answer with the answer content contained within an XML element with an <answer> tag. At the end of the answer should be an array of 3 potential follow-up questions for the user to ask you next, contained within a <follow_up_questions> key. + Use <thought> to describe your thoughts about the question you have been asked. + Use <action> to specify run one of the actions available to you. + Then, you will be provided with action rules within an <action_rules> element that specifies how you should structure the input to the action and what the output of that action will look like - then return another </pause> element. + Then, provide within an <action_input> element each parameter, with parameter names as element tags themselves with their values inside, following the structure defined in the action rules. + Observation, in an <observation> element will be the result of running those actions. + ********** + Your available actions are: + ***** + ${toolDescriptions} + ***** + no_tool: Use this when no external tool or action is required to answer the question. + ********** + Example: + You will be called with: + <query>What is the capital of France?</query> + + You will then output: + <step1> + <thought>I should look up France on Wikipedia</thought> + <action>wikipedia</action> + </step1> + + THEN PAUSE AND DO NOT OUTPUT ANYTHING. + + You will be called again with this: + <action_rules> + { + "wikipedia": { + "name": "wikipedia", + "description": "Search Wikipedia and return a summary", + "parameters": [ + { + "title": { + "type": "string", + "description": "The title of the Wikipedia article to search", + "required": "true" + } + } + ] + } + } + </action_rules> + + You will then output (back in valid XML with the parameters each being a tag): + <step2> + <action_input> + <title>France</title> + </action_input> + </step2> + + THEN PAUSE AND DO NOT OUTPUT ANYTHING. + + You will then be called again with this: + <observation>France is a country. The capital is Paris.</observation> + + You then output: + <step3> + <answer> + The capital of France is Paris + <follow_up_questions> + <question>Where in France is Paris located?</question> + <question>What are some major tourist attractions in Paris?</question> + <question>What are some other major cities in France?</question> + </follow_up_questions> + </answer> + </step3> + ********** + Example: +You will be called with: +<query>What is 2 + 2?</query> + +You will then output: +<step1> + <thought>This is a simple arithmetic question that doesn't require any external tool.</thought> + <action>no_tool</action> +</step1> + +THEN PAUSE AND DO NOT OUTPUT ANYTHING. + +You will be called again with this: +<action_rules> + { + "no_tool": { + "name": "no_tool", + "description": "Use when no external tool or action is required", + "parameters": [] + } + } +</action_rules> + +You will then output: +<step2> + <action_input></action_input> +</step2> + +THEN PAUSE AND DO NOT OUTPUT ANYTHING. + +You will then be called again with this: +<observation>No tool used. Proceed with answering the question.</observation> + +You then output: +<step3> + <answer> + 2 + 2 equals 4. + <follow_up_questions> + <question>What is 3 + 3?</question> + <question>Can you explain the concept of addition?</question> + <question>What is 2 * 2?</question> + </follow_up_questions> + </answer> +</step3> + ********** + Here is the history of your conversation with the user (all loop steps are ommitted, so it is just the user query and final answer): + ${chatHistory} + Use context from the past conversation if necessary. + ********** + If the response is inadequate, repeat the loop, either trying a different tool or changing the parameters for the action input. + ********** + !!!IMPORTANT Only use tools when they are absolutely necessary to answer the question. If you have enough information or knowledge to answer the question without using a tool, use the "no_tool" action instead. + !!!IMPORTANT When you have an Answer, Write your entire response inside an <answer> element (which itself should be inside the step element for the current step). After you finish the answer, provide an array of 3 follow-up questions inside a <follow_up_questions> array. These should relate to the query and the response and should aim to help the user better understand whatever they are looking for. + ********** + !!!IMPORTANT Every response, provide in full parsable and valid XML with the root element being the step number (e.g. <step1>), iterated every time you output something new. + `; +} + +export function getSummarizedChunksPrompt(chunks: string): string { + return `Please provide a comprehensive summary of what you think the document from which these chunks originated. + Ensure the summary captures the main ideas and key points from all provided chunks. Be concise and brief and only provide the summary in paragraph form. + + Text chunks: + \`\`\` + ${chunks} + \`\`\``; +} + +export function getSummarizedSystemPrompt(): string { + return 'You are an AI assistant tasked with summarizing a document. You are provided with important chunks from the document and provide a summary, as best you can, of what the document will contain overall. Be concise and brief with your response.'; +} diff --git a/src/client/views/nodes/ChatBox/tools.ts b/src/client/views/nodes/ChatBox/tools.ts new file mode 100644 index 000000000..4035280a8 --- /dev/null +++ b/src/client/views/nodes/ChatBox/tools.ts @@ -0,0 +1,26 @@ +import { DocCast } from '../../../../fields/Types'; +import { DocServer } from '../../../DocServer'; +import { Docs } from '../../../documents/Documents'; +import { DocUtils } from '../../../documents/DocUtils'; +import { TabDocView } from '../../collections/TabDocView'; +import { DocumentView } from '../DocumentView'; +import { OpenWhere } from '../OpenWhere'; + +export function retrieval(json: any): string { + return ''; +} + +export function create_collection(docView: DocumentView, document_ids: string[], title: string): string { + const docs = document_ids.map(doc_id => DocCast(DocServer.GetCachedRefField(doc_id))); + const collection = Docs.Create.FreeformDocument(docs, { title }); + docView._props.addDocTab(collection, OpenWhere.addRight); //in future, create popup prompting user where to add + return 'Collection created in Dash called ' + title; +} + +export function create_link(docView: DocumentView, document_ids: string[]): string { + //Make document_ids a size 2 array + const docs = document_ids.map(doc_id => DocCast(DocServer.GetCachedRefField(doc_id))); + const linkDoc = DocUtils.MakeLink(docs[0], docs[1], {})!; + DocumentView.linkCommonAncestor(linkDoc)?.ComponentView?.addDocument?.(linkDoc); + return 'Link created between ' + docs[0].title + ' and ' + docs[1].title; +} diff --git a/src/client/views/nodes/ChatBox/tools/BaseTool.ts b/src/client/views/nodes/ChatBox/tools/BaseTool.ts new file mode 100644 index 000000000..903161bd5 --- /dev/null +++ b/src/client/views/nodes/ChatBox/tools/BaseTool.ts @@ -0,0 +1,24 @@ +import { Tool } from '../types'; + +export abstract class BaseTool<T extends Record<string, any> = Record<string, any>> implements Tool<T> { + constructor( + public name: string, + public description: string, + public parameters: Record<string, any>, + public useRules: string, + public briefSummary: string + ) {} + + abstract execute(args: T): Promise<any>; + + getActionRule(): Record<string, any> { + return { + [this.name]: { + name: this.name, + useRules: this.useRules, + description: this.description, + parameters: this.parameters, + }, + }; + } +} diff --git a/src/client/views/nodes/ChatBox/tools/CalculateTool.ts b/src/client/views/nodes/ChatBox/tools/CalculateTool.ts new file mode 100644 index 000000000..818332c44 --- /dev/null +++ b/src/client/views/nodes/ChatBox/tools/CalculateTool.ts @@ -0,0 +1,25 @@ +import { BaseTool } from './BaseTool'; + +export class CalculateTool extends BaseTool<{ expression: string }> { + constructor() { + super( + 'calculate', + 'Perform a calculation', + { + expression: { + type: 'string', + description: 'The mathematical expression to evaluate', + required: 'true', + }, + }, + 'Provide a mathematical expression to calculate that would work with JavaScript eval().', + 'Runs a calculation and returns the number - uses JavaScript so be sure to use floating point syntax if necessary' + ); + } + + async execute(args: { expression: string }): Promise<any> { + // Note: Using eval() can be dangerous. Consider using a safer alternative. + const result = eval(args.expression); + return [{ type: 'text', text: result.toString() }]; + } +} diff --git a/src/client/views/nodes/ChatBox/tools/CreateCollectionTool.ts b/src/client/views/nodes/ChatBox/tools/CreateCollectionTool.ts new file mode 100644 index 000000000..26ac0d7cc --- /dev/null +++ b/src/client/views/nodes/ChatBox/tools/CreateCollectionTool.ts @@ -0,0 +1,35 @@ +import { DocCast } from '../../../../../fields/Types'; +import { DocServer } from '../../../../DocServer'; +import { Docs } from '../../../../documents/Documents'; +import { DocumentView } from '../../DocumentView'; +import { OpenWhere } from '../../OpenWhere'; +import { BaseTool } from './BaseTool'; + +export class GetDocsContentTool extends BaseTool<{ title: string; document_ids: string[] }> { + private _docView: DocumentView; + constructor(docView: DocumentView) { + super( + 'retrieveDocs', + 'Retrieves the contents of all Documents that the user is interacting with in Dash ', + { + title: { + type: 'string', + description: 'the title of the collection that you will be making', + required: 'true', + }, + }, + 'Provide a mathematical expression to calculate that would work with JavaScript eval().', + 'Runs a calculation and returns the number - uses JavaScript so be sure to use floating point syntax if necessary' + ); + this._docView = docView; + } + + async execute(args: { title: string; document_ids: string[] }): Promise<any> { + // Note: Using eval() can be dangerous. Consider using a safer alternative. + const docs = args.document_ids.map(doc_id => DocCast(DocServer.GetCachedRefField(doc_id))); + const collection = Docs.Create.FreeformDocument(docs, { title: args.title }); + this._docView._props.addDocTab(collection, OpenWhere.addRight); //in future, create popup prompting user where to add + return [{ type: 'text', text: 'Collection created in Dash called ' + args.title }]; + } +} +//export function create_collection(docView: DocumentView, document_ids: string[], title: string): string {} diff --git a/src/client/views/nodes/ChatBox/tools/GetDocsTool.ts b/src/client/views/nodes/ChatBox/tools/GetDocsTool.ts new file mode 100644 index 000000000..f970ca8ee --- /dev/null +++ b/src/client/views/nodes/ChatBox/tools/GetDocsTool.ts @@ -0,0 +1,29 @@ +import { DocCast } from '../../../../../fields/Types'; +import { DocServer } from '../../../../DocServer'; +import { Docs } from '../../../../documents/Documents'; +import { DocumentView } from '../../DocumentView'; +import { OpenWhere } from '../../OpenWhere'; +import { BaseTool } from './BaseTool'; + +export class GetDocsTool extends BaseTool<{ title: string; document_ids: string[] }> { + private _docView: DocumentView; + constructor(docView: DocumentView) { + super( + 'retrieveDocs', + 'Retrieves the contents of all Documents that the user is interacting with in Dash', + {}, + 'No need to provide anything. Just run the tool and it will retrieve the contents of all Documents that the user is interacting with in Dash.', + 'Returns the the documents in Dash in JSON form. This will include the title of the document, the location in the FreeFormDocument, and the content of the document, any applicable data fields, the layout of the document, etc.' + ); + this._docView = docView; + } + + async execute(args: { title: string; document_ids: string[] }): Promise<any> { + // Note: Using eval() can be dangerous. Consider using a safer alternative. + const docs = args.document_ids.map(doc_id => DocCast(DocServer.GetCachedRefField(doc_id))); + const collection = Docs.Create.FreeformDocument(docs, { title: args.title }); + this._docView._props.addDocTab(collection, OpenWhere.addRight); //in future, create popup prompting user where to add + return [{ type: 'text', text: 'Collection created in Dash called ' + args.title }]; + } +} +//export function create_collection(docView: DocumentView, document_ids: string[], title: string): string {} diff --git a/src/client/views/nodes/ChatBox/tools/NoTool.ts b/src/client/views/nodes/ChatBox/tools/NoTool.ts new file mode 100644 index 000000000..1f0830a77 --- /dev/null +++ b/src/client/views/nodes/ChatBox/tools/NoTool.ts @@ -0,0 +1,18 @@ +// tools/NoTool.ts +import { BaseTool } from './BaseTool'; + +export class NoTool extends BaseTool<{}> { + constructor() { + super( + 'no_tool', + 'Use this when no external tool or action is required to answer the question.', + {}, + 'When using the "no_tool" action, simply provide an empty <action_input> element. The observation will always be "No tool used. Proceed with answering the question."', + 'Use when no external tool or action is required to answer the question.' + ); + } + + async execute(args: {}): Promise<any> { + return [{ type: 'text', text: 'No tool used. Proceed with answering the question.' }]; + } +} diff --git a/src/client/views/nodes/ChatBox/tools/RAGTool.ts b/src/client/views/nodes/ChatBox/tools/RAGTool.ts new file mode 100644 index 000000000..0a4529974 --- /dev/null +++ b/src/client/views/nodes/ChatBox/tools/RAGTool.ts @@ -0,0 +1,74 @@ +import { BaseTool } from './BaseTool'; +import { Vectorstore } from '../vectorstore/VectorstoreUpload'; +import { Chunk } from '../types'; +import * as fs from 'fs'; +import { Networking } from '../../../../Network'; + +export class RAGTool extends BaseTool<{ hypothetical_document_chunk: string }> { + constructor( + private vectorstore: Vectorstore, + summaries: () => string + ) { + super( + 'rag', + 'Perform a RAG search on user documents', + { + hypothetical_document_chunk: { + type: 'string', + description: + "Detailed version of the prompt that is effectively a hypothetical document chunk that would be ideal to embed and compare to the vectors of real document chunks to fetch the most relevant document chunks to answer the user's query", + required: 'true', + }, + }, + `Your task is to first provide a response to the user's prompt based on the information given in the chunks and considering the chat history. Follow these steps: + + 1. Carefully read and analyze the provided chunks, which may include text, images, or tables. Each chunk has an associated chunk_id. + + 2. Review the prompt and chat history to understand the context of the user's question or request. + + 3. Formulate a response that addresses the prompt using information from the relevant chunks. Your response should be informative and directly answer the user's question or request. + + 4. Use citations to support your response. Citations should contain direct textual references to the granular, specific part of the original chunk that applies to the situation—with no text ommitted. Citations should be in the following format: + - For text: <citation chunk_id="d980c2a7-cad3-4d7e-9eae-19bd2380bd02" type="text">relevant direct text from the chunk that the citation in referencing specifically</citation> + - For images or tables: <citation chunk_id="9ef37681-b57e-4424-b877-e1ebc326ff11" type="image"></citation> + + Place citations after the sentences they apply to. You can use multiple citations in a row. + + 5. If there's insufficient information in the provided chunks to answer the prompt sufficiently, ALWAYS respond with <answer>RAG not applicable</answer> + + Write your entire response, including follow-up questions, inside <answer> tags. Remember to use the citation format for both text and image references, and maintain a conversational tone throughout your response. + + !!!IMPORTANT Before you close the tag with </answer>, within the answer tags provide a set of 3 follow-up questions inside a <follow_up_questions> tag and individually within <question> tags. These should relate to the document, the current query, and the chat_history and should aim to help the user better understand whatever they are looking for. + Also, ensure that the answer tags are wrapped with the correct step tags as well.`, + + `Performs a RAG (Retrieval-Augmented Generation) search on user documents and returns a + set of document chunks (either images or text) that can be used to provide a grounded response based on + user documents + + !!!IMPORTANT Use the RAG tool ANYTIME the question may potentially (even if you are not sure) relate to one of the user's documents. + Here are the summaries of the user's documents: + ${summaries()}` + ); + } + + async execute(args: { hypothetical_document_chunk: string }): Promise<any> { + const relevantChunks = await this.vectorstore.retrieve(args.hypothetical_document_chunk); + const formatted_chunks = await this.getFormattedChunks(relevantChunks); + return formatted_chunks; + } + + async getFormattedChunks(relevantChunks: Chunk[]): Promise<{ type: string; text?: string; image_url?: { url: string } }[]> { + try { + const { formattedChunks } = await Networking.PostToServer('/formatChunks', { relevantChunks }); + + if (!formattedChunks) { + throw new Error('Failed to format chunks'); + } + + return formattedChunks; + } catch (error) { + console.error('Error formatting chunks:', error); + throw error; + } + } +} diff --git a/src/client/views/nodes/ChatBox/tools/WikipediaTool.ts b/src/client/views/nodes/ChatBox/tools/WikipediaTool.ts new file mode 100644 index 000000000..e2c5009a1 --- /dev/null +++ b/src/client/views/nodes/ChatBox/tools/WikipediaTool.ts @@ -0,0 +1,31 @@ +import { title } from 'process'; +import { Networking } from '../../../../Network'; +import { BaseTool } from './BaseTool'; +import axios from 'axios'; + +export class WikipediaTool extends BaseTool<{ title: string }> { + constructor() { + super( + 'wikipedia', + 'Search Wikipedia and return a summary', + { + title: { + type: 'string', + description: 'The title of the Wikipedia article to search', + required: true, + }, + }, + 'Provide simply the title you want to search on Wikipedia and nothing more. If re-using this tool, try a different title for different information.', + 'Returns a summary from searching an article title on Wikipedia' + ); + } + + async execute(args: { title: string }): Promise<any> { + try { + const { text } = await Networking.PostToServer('/getWikipediaSummary', { title: args.title }); + return [{ type: 'text', text: text }]; + } catch (error) { + return [{ type: 'text', text: 'An error occurred while fetching the article.' }]; + } + } +} diff --git a/src/client/views/nodes/ChatBox/types.ts b/src/client/views/nodes/ChatBox/types.ts index 8212a7050..d702d5c41 100644 --- a/src/client/views/nodes/ChatBox/types.ts +++ b/src/client/views/nodes/ChatBox/types.ts @@ -1,23 +1,116 @@ +import { Doc } from '../../../../fields/Doc'; +import { StrCast } from '../../../../fields/Types'; + export enum ASSISTANT_ROLE { - USER = 'User', - ASSISTANT = 'Assistant', + USER = 'user', + ASSISTANT = 'assistant', } -export enum ANNOTATION_LINK_TYPE { - DASH_DOC = 'citation', - DOWNLOAD_FILE = 'file_path', +export enum CHUNK_TYPE { + TEXT = 'text', + IMAGE = 'image', + TABLE = 'table', } -export enum DOWNLOAD_TYPE { - DASH = 'dash', - DEVICE = 'device', +export function getChunkType(type: string): CHUNK_TYPE { + switch (type.toLowerCase()) { + case 'text': + return CHUNK_TYPE.TEXT; + case 'image': + return CHUNK_TYPE.IMAGE; + case 'table': + return CHUNK_TYPE.TABLE; + default: + return CHUNK_TYPE.TEXT; + } } export interface AssistantMessage { role: ASSISTANT_ROLE; - text: string; - quote?: string; - image?: string; - tool_logs?: string; - links?: { start: number; end: number; url: string; id?: string; link_type: ANNOTATION_LINK_TYPE }[]; + text_content: string; + follow_up_questions?: string[]; + citations?: Citation[]; +} + +export interface Citation { + direct_text?: string; + type: CHUNK_TYPE; + chunk_id: string; + text_location: number; + citation_id: string; +} + +export interface Chunk { + id: string; + values: number[]; + metadata: { + text: string; + type: CHUNK_TYPE; + original_document: string; + file_path: string; + location: string; + start_page: number; + end_page: number; + base64_data?: string | undefined; + page_width?: number | undefined; + page_height?: number | undefined; + }; +} + +export interface AI_Document { + purpose: string; + file_name: string; + num_pages: number; + summary: string; + chunks: Chunk[]; + type: string; } + +export interface Tool<T extends Record<string, any> = Record<string, any>> { + name: string; + description: string; + parameters: Record<string, any>; + useRules: string; + briefSummary: string; + execute: (args: T) => Promise<any>; + getActionRule: () => Record<string, any>; +} + +export interface AgentMessage { + role: 'system' | 'user' | 'assistant'; + content: string | { type: string; text?: string; image_url?: { url: string } }[]; +} + +// export function convertToAIDocument(json: any): AI_Document { +// if (!json) { +// throw new Error('Invalid JSON object'); +// } + +// const chunks: Chunk[] = json.chunks.map((chunk: any) => ({ +// id: chunk.id, +// values: chunk.values, +// metadata: { +// text: chunk.metadata.text, +// type: chunk.metadata.type as CHUNK_TYPE, // Ensure type casting +// original_document: chunk.metadata.original_document, +// file_path: chunk.metadata.file_path, +// location: chunk.metadata.location, +// start_page: chunk.metadata.start_page, +// end_page: chunk.metadata.end_page, +// base64_data: chunk.metadata.base64_data, +// width: chunk.metadata.width, +// height: chunk.metadata.height, +// }, +// })); + +// const aiDocument: AI_Document = { +// purpose: json.purpose, +// file_name: json.file_name, +// num_pages: json.num_pages, +// summary: json.summary, +// chunks: chunks, +// type: json.type, +// }; + +// return aiDocument; +// } diff --git a/src/client/views/nodes/ChatBox/vectorstore/VectorstoreUpload.ts b/src/client/views/nodes/ChatBox/vectorstore/VectorstoreUpload.ts new file mode 100644 index 000000000..ab0b6e617 --- /dev/null +++ b/src/client/views/nodes/ChatBox/vectorstore/VectorstoreUpload.ts @@ -0,0 +1,180 @@ +import { Pinecone, Index, IndexList, PineconeRecord, RecordMetadata, QueryResponse } from '@pinecone-database/pinecone'; +import { CohereClient } from 'cohere-ai'; +import { EmbedResponse } from 'cohere-ai/api'; +import dotenv from 'dotenv'; + +import { Chunk, AI_Document, CHUNK_TYPE } from '../types'; +import { Doc } from '../../../../../fields/Doc'; +import { DocData } from '../../../../../fields/DocSymbols'; +import { CsvCast, PDFCast, StrCast } from '../../../../../fields/Types'; +import { Networking } from '../../../../Network'; + +dotenv.config(); + +export class Vectorstore { + private pinecone: Pinecone; + private index!: Index; + private cohere: CohereClient; + private indexName: string = 'pdf-chatbot'; + private id: string; + documents: AI_Document[] = []; + + constructor(id: string) { + const pineconeApiKey = process.env.PINECONE_API_KEY; + if (!pineconeApiKey) { + throw new Error('PINECONE_API_KEY is not defined.'); + } + + this.pinecone = new Pinecone({ + apiKey: pineconeApiKey, + }); + this.cohere = new CohereClient({ + token: process.env.COHERE_API_KEY, + }); + this.id = id; + this.initializeIndex(); + } + + private async initializeIndex() { + const indexList: IndexList = await this.pinecone.listIndexes(); + + if (!indexList.indexes?.some(index => index.name === this.indexName)) { + await this.pinecone.createIndex({ + name: this.indexName, + dimension: 1024, + metric: 'cosine', + spec: { + serverless: { + cloud: 'aws', + region: 'us-east-1', + }, + }, + }); + } + + this.index = this.pinecone.Index(this.indexName); + } + + async addAIDoc(doc: Doc) { + console.log('Adding AI Document:', doc); + const ai_document_status: string = StrCast(doc.ai_document_status); + if (ai_document_status !== undefined && ai_document_status !== null && ai_document_status !== '' && ai_document_status !== ' ' && ai_document_status !== '{}') { + if (ai_document_status === 'IN PROGRESS') { + console.log('Already in progress.'); + return; + } + console.log(`Document already added: ${doc.file_name}`); + } else { + doc.ai_document_status = 'PROGRESS'; + console.log(doc); + console.log(PDFCast(doc.data)?.url?.pathname); + console.log(CsvCast(doc.data)?.url?.pathname); + const local_file_path: string = CsvCast(doc.data)?.url?.pathname ?? PDFCast(doc.data)?.url?.pathname; + console.log('Local File Path:', local_file_path); + if (local_file_path !== undefined || local_file_path !== null || local_file_path !== '') { + const { document_json } = await Networking.PostToServer('/createDocument', { file_path: local_file_path }); + console.log('Document JSON:', document_json); + //const ai_document: AI_Document = convertToAIDocument(document_json); + this.documents.push(document_json); + await this.indexDocument(JSON.parse(JSON.stringify(document_json, (key, value) => (value === null || value === undefined ? undefined : value)))); + console.log(`Document added: ${document_json.file_name}`); + doc.summary = document_json.summary; + doc.ai_purpose = document_json.purpose; + if (doc.vectorstore_id === undefined || doc.vectorstore_id === null || doc.vectorstore_id === '' || doc.vectorstore_id === '[]') { + doc.vectorstore_id = JSON.stringify([this.id]); + } else { + doc.vectorstore_id = JSON.stringify(JSON.parse(StrCast(doc.vectorstore_id)).concat([this.id])); + } + if (doc.chunk_simpl === undefined || doc.chunk_simpl === null || doc.chunk_simpl === '' || doc.chunk_simpl === '[]') { + doc.chunk_simpl = JSON.stringify({ text_chunks: [], image_chunks: [] }); + } + let new_chunk_simpl: { text_chunks: { chunk_id: string; start_page: number; end_page: number }[]; image_chunks: { chunk_id: string; location: string; page: number }[] } = { + text_chunks: [], + image_chunks: [], + }; + + document_json.chunks.forEach((chunk: Chunk) => { + let chunk_to_add: { chunk_id: string; start_page: number; end_page: number }[] | { chunk_id: string; location: string; page: number }[]; + switch (chunk.metadata.type) { + case CHUNK_TYPE.TEXT: + chunk_to_add = [{ chunk_id: chunk.id, start_page: chunk.metadata.start_page, end_page: chunk.metadata.end_page }]; + new_chunk_simpl = JSON.parse(StrCast(doc.chunk_simpl)); + new_chunk_simpl.text_chunks = new_chunk_simpl.text_chunks.concat(chunk_to_add); + doc.chunk_simpl = JSON.stringify(new_chunk_simpl); + break; + case CHUNK_TYPE.IMAGE: + case CHUNK_TYPE.TABLE: + console.log('Location:', chunk.metadata.location); + chunk_to_add = [{ chunk_id: chunk.id, location: chunk.metadata.location, page: chunk.metadata.start_page }]; + new_chunk_simpl = JSON.parse(StrCast(doc.chunk_simpl)); + new_chunk_simpl.image_chunks = new_chunk_simpl.image_chunks.concat(chunk_to_add); + doc.chunk_simpl = JSON.stringify(new_chunk_simpl); + break; + } + }); + + doc.ai_document_status = 'COMPLETED'; + } + } + } + + private async indexDocument(document: any) { + console.log('Uploading vectors to content namespace...'); + const pineconeRecords: PineconeRecord<RecordMetadata>[] = (document.chunks as Chunk[]).map( + chunk => + ({ + id: chunk.id, + values: chunk.values, + metadata: { ...chunk.metadata, vectorstore_id: this.id } as RecordMetadata, + }) as PineconeRecord + ); + await this.index.upsert(pineconeRecords); + } + + async retrieve(query: string, topK: number = 10): Promise<Chunk[]> { + console.log(`Retrieving chunks for query: ${query}`); + try { + const queryEmbeddingResponse: EmbedResponse = await this.cohere.embed({ + texts: [query], + model: 'embed-english-v3.0', + inputType: 'search_query', + }); + + let queryEmbedding: number[]; + + if (Array.isArray(queryEmbeddingResponse.embeddings)) { + queryEmbedding = queryEmbeddingResponse.embeddings[0]; + } else if (queryEmbeddingResponse.embeddings && 'embeddings' in queryEmbeddingResponse.embeddings) { + queryEmbedding = (queryEmbeddingResponse.embeddings as { embeddings: number[][] }).embeddings[0]; + } else { + throw new Error('Invalid embedding response format'); + } + + if (!Array.isArray(queryEmbedding)) { + throw new Error('Query embedding is not an array'); + } + + const queryResponse: QueryResponse<RecordMetadata> = await this.index.query({ + vector: queryEmbedding, + filter: { + vectorstore_id: this.id, + }, + topK, + includeValues: true, + includeMetadata: true, + }); + + return queryResponse.matches.map( + match => + ({ + id: match.id, + values: match.values as number[], + metadata: match.metadata as { text: string; type: string; original_document: string; file_path: string; location: string; start_page: number; end_page: number }, + }) as Chunk + ); + } catch (error) { + console.error(`Error retrieving chunks: ${error}`); + return []; + } + } +} diff --git a/src/client/views/pdf/PDFViewer.tsx b/src/client/views/pdf/PDFViewer.tsx index fa5e5cedb..e8bcca638 100644 --- a/src/client/views/pdf/PDFViewer.tsx +++ b/src/client/views/pdf/PDFViewer.tsx @@ -404,6 +404,30 @@ export class PDFViewer extends ObservableReactComponent<IViewerProps> { } }; + // @action + // createMarquee = (coords: [x1: number, x2: number, y1: number, y2: number]): void => { + // // const hit = document.elementFromPoint(e.clientX, e.clientY); + // // bcz: Change. drag selecting requires that preventDefault is NOT called. This used to happen in DocumentView, + // // but that's changed, so this shouldn't be needed. + // // if (hit && hit.localName === "span" && this.annotationsActive(true)) { // drag selecting text stops propagation + // // e.button === 0 && e.stopPropagation(); + // // } + // // if alt+left click, drag and annotate + // this._downX = coords[0]; + // this._downY = coords[2]; + + // if ((this._props.Document._freeform_scale || 1) !== 1) return; + // this._props.select(false); + // MarqueeAnnotator.clearAnnotations(this._savedAnnotations); + // this.isAnnotating = true; + // this._textSelecting = false; + // // if textLayer is hit, then we select text instead of using a marquee so clear out the marquee. + // this._styleRule = addStyleSheetRule(PDFViewer._annotationStyle, 'htmlAnnotation', { 'pointer-events': 'none' }); + + // this._marqueeref.current?.onInitiateSelection([coords[0], coords[2]]); + // this._marqueeref.current?.onTerminateSelection(); + // }; + @action finishMarquee = (/* x?: number, y?: number */) => { this._getAnchor = AnchorMenu.Instance?.GetAnchor; diff --git a/src/server/ApiManagers/AssistantManager.ts b/src/server/ApiManagers/AssistantManager.ts index b42314e41..36468157a 100644 --- a/src/server/ApiManagers/AssistantManager.ts +++ b/src/server/ApiManagers/AssistantManager.ts @@ -7,6 +7,8 @@ import * as uuid from 'uuid'; import { filesDirectory, publicDirectory } from '../SocketData'; import { Method } from '../RouteManager'; import ApiManager, { Registration } from './ApiManager'; +import axios from 'axios'; +import { Chunk } from '../../client/views/nodes/ChatBox/types'; export enum Directory { parsed_files = 'parsed_files', @@ -36,95 +38,153 @@ const readFileAsync = promisify(fs.readFile); export default class AssistantManager extends ApiManager { protected initialize(register: Registration): void { - const openai = new OpenAI({ apiKey: process.env.OPENAI_KEY, dangerouslyAllowBrowser: true }); + const openai = new OpenAI({ + apiKey: process.env._CLIENT_OPENAI_KEY, // Use client key so don't have to set key seperately for client and server. + dangerouslyAllowBrowser: true, + }); register({ method: Method.POST, - subscription: '/uploadPDFToVectorStore', + subscription: '/getWikipediaSummary', secureHandler: async ({ req, res }) => { - const { urls, threadID, assistantID, vector_store_id } = req.body; - - const csvFilesIds: string[] = []; - const otherFileIds: string[] = []; - const allFileIds: string[] = []; - - const fileProcesses = urls.map(async (source: string) => { - const fullPath = path.join(publicDirectory, source); - const fileData = await openai.files.create({ file: createReadStream(fullPath), purpose: 'assistants' }); - allFileIds.push(fileData.id); - if (source.endsWith('.csv')) { - console.log(source); - csvFilesIds.push(fileData.id); - } else { - openai.beta.vectorStores.files.create(vector_store_id, { file_id: fileData.id }); - otherFileIds.push(fileData.id); - } - }); + const { title } = req.body; try { - await Promise.all(fileProcesses).then(() => { - res.send({ vector_store_id: vector_store_id, openai_file_ids: allFileIds }); + const response = await axios.get('https://en.wikipedia.org/w/api.php', { + params: { + action: 'query', + list: 'search', + srsearch: title, + format: 'json', + }, }); - } catch (error) { - res.status(500).send({ error: 'Failed to process files' + error }); + const summary = response.data.query.search[0].snippet; + if (!summary || summary.length === 0 || summary === '' || summary === ' ') { + res.send({ text: 'No article found with that title.' }); + } else { + res.send({ text: summary }); + } + } catch (error: any) { + console.error('Error retrieving article summary from Wikipedia:', error); + res.status(500).send({ error: 'Error retrieving article summary from Wikipedia.', details: error.message }); } }, }); register({ method: Method.POST, - subscription: '/downloadFileFromOpenAI', + subscription: '/createDocument', secureHandler: async ({ req, res }) => { - const { file_id, file_name } = req.body; - //let files_directory: string; - let files_directory = '/files/openAIFiles/'; - switch (file_name.split('.').pop()) { - case 'pdf': - files_directory = '/files/pdfs/'; - break; - case 'csv': - files_directory = '/files/csv/'; - break; - case 'png': - case 'jpg': - case 'jpeg': - files_directory = '/files/images/'; - break; - default: - break; - } + const { file_path } = req.body; + const public_path = path.join(publicDirectory, file_path); + const file_name = path.basename(file_path); + + try { + // Read file data and convert to base64 + const file_data = fs.readFileSync(public_path, { encoding: 'base64' }); + + const response = await axios.post( + 'http://localhost:8080/createDocument', + { + file_data, + file_name, + }, + { + headers: { + 'Content-Type': 'application/json', + }, + } + ); + + const jobId = response.data.job_id; + + // Poll for results + let result; + while (!result) { + await new Promise(resolve => setTimeout(resolve, 5000)); // Wait for 1 second + const resultResponse = await axios.get(`http://localhost:8080/getResult/${jobId}`); + if (resultResponse.status === 200) { + result = resultResponse.data; + } + } + + if (result.chunks && Array.isArray(result.chunks)) { + for (const chunk of result.chunks) { + if (chunk.metadata && (chunk.metadata.type === 'image' || chunk.metadata.type === 'table')) { + let files_directory = '/files/chunk_images/'; + const directory = path.join(publicDirectory, files_directory); + + if (!fs.existsSync(directory)) { + fs.mkdirSync(directory); + } + + const fileName = path.basename(chunk.metadata.file_path); + const filePath = path.join(directory, fileName); + + // Check if base64_data exists + if (chunk.metadata.base64_data) { + // Decode Base64 and save as file + const buffer = Buffer.from(chunk.metadata.base64_data, 'base64'); + await fs.promises.writeFile(filePath, buffer); - const directory = path.join(publicDirectory, files_directory); + // Update the file path in the chunk + chunk.metadata.file_path = path.join(files_directory, fileName); + chunk.metadata.base64_data = undefined; + } else { + console.warn(`No base64_data found for chunk: ${fileName}`); + } + } + } + } else { + console.warn("Result does not contain an iterable 'chunks' property"); + } - if (!fs.existsSync(directory)) { - fs.mkdirSync(directory); + res.send({ document_json: result }); + } catch (error: any) { + console.error('Error communicating with chatbot:', error); + res.status(500).send({ error: 'Failed to communicate with the chatbot', details: error.message }); } - const file = await openai.files.content(file_id); - const new_file_name = `${uuid.v4()}-${file_name}`; - const file_path = path.join(directory, new_file_name); - const file_array_buffer = await file.arrayBuffer(); - const bufferView = new Uint8Array(file_array_buffer); - try { - const written_file = await writeFileAsync(file_path, bufferView); - console.log(written_file); - console.log(file_path); - console.log(file_array_buffer); - console.log(bufferView); - const file_object = new File([bufferView], file_name); - //DashUploadUtils.upload(file_object, 'openAIFiles'); - res.send({ file_path: path.join(files_directory, new_file_name) }); - /* res.send( { - source: "file", - result: { - accessPaths: { - agnostic: {client: path.join('/files/openAIFiles/', `${uuid.v4()}-${file_name}`)} - }, - rawText: "", - duration: 0, - }, - } ); */ - } catch (error) { - res.status(500).send({ error: 'Failed to write file' + error }); + }, + }); + + register({ + method: Method.POST, + subscription: '/formatChunks', + secureHandler: async ({ req, res }) => { + const { relevantChunks } = req.body; + const content: { type: string; text?: string; image_url?: { url: string } }[] = [{ type: 'text', text: '<chunks>' }]; + + for (const chunk of relevantChunks) { + content.push({ + type: 'text', + text: `<chunk chunk_id=${chunk.id} chunk_type=${chunk.metadata.type === 'image' || chunk.metadata.type === 'table' ? 'image' : 'text'}>`, + }); + + if (chunk.metadata.type === 'image' || chunk.metadata.type === 'table') { + try { + const filePath = serverPathToFile(Directory.parsed_files, chunk.metadata.file_path); + const imageBuffer = await readFileAsync(filePath); + const base64Image = imageBuffer.toString('base64'); + if (base64Image) { + content.push({ + type: 'image_url', + image_url: { + url: `data:image/jpeg;base64,${base64Image}`, + }, + }); + } else { + console.log(`Failed to encode image for chunk ${chunk.id}`); + } + } catch (error) { + console.error(`Error reading image file for chunk ${chunk.id}:`, error); + } + } + + content.push({ type: 'text', text: `${chunk.metadata.text}\n</chunk>\n` }); } + + content.push({ type: 'text', text: '</chunks>' }); + + res.send({ formattedChunks: content }); }, }); } diff --git a/src/server/RouteManager.ts b/src/server/RouteManager.ts index d8e0455f6..22e608868 100644 --- a/src/server/RouteManager.ts +++ b/src/server/RouteManager.ts @@ -8,6 +8,7 @@ import { DashUserModel } from './authentication/DashUserModel'; export enum Method { GET, POST, + PUT, } export interface CoreArguments { @@ -208,6 +209,9 @@ export default class RouteManager { case Method.POST: this.server.post(route, supervised); break; + case Method.PUT: + this.server.put(route, supervised); + break; default: } } |