From cab0311e2fd9a6379628c000d11ddcd805e01f64 Mon Sep 17 00:00:00 2001 From: "A.J. Shulman" Date: Wed, 10 Jul 2024 16:16:26 -0400 Subject: first attempt at integrating everything --- src/client/views/nodes/ChatBox/tools/BaseTool.ts | 24 +++++++ .../views/nodes/ChatBox/tools/CalculateTool.ts | 25 +++++++ src/client/views/nodes/ChatBox/tools/RAGTool.ts | 81 ++++++++++++++++++++++ .../views/nodes/ChatBox/tools/WikipediaTool.ts | 33 +++++++++ 4 files changed, 163 insertions(+) create mode 100644 src/client/views/nodes/ChatBox/tools/BaseTool.ts create mode 100644 src/client/views/nodes/ChatBox/tools/CalculateTool.ts create mode 100644 src/client/views/nodes/ChatBox/tools/RAGTool.ts create mode 100644 src/client/views/nodes/ChatBox/tools/WikipediaTool.ts (limited to 'src/client/views/nodes/ChatBox/tools') diff --git a/src/client/views/nodes/ChatBox/tools/BaseTool.ts b/src/client/views/nodes/ChatBox/tools/BaseTool.ts new file mode 100644 index 000000000..3511d9528 --- /dev/null +++ b/src/client/views/nodes/ChatBox/tools/BaseTool.ts @@ -0,0 +1,24 @@ +import { Tool } from '../types'; + +export abstract class BaseTool implements Tool { + constructor( + public name: string, + public description: string, + public parameters: Record, + public useRules: string, + public briefSummary: string + ) {} + + abstract execute(args: Record): Promise; + + getActionRule(): Record { + return { + [this.name]: { + name: this.name, + useRules: this.useRules, + description: this.description, + parameters: this.parameters, + }, + }; + } +} diff --git a/src/client/views/nodes/ChatBox/tools/CalculateTool.ts b/src/client/views/nodes/ChatBox/tools/CalculateTool.ts new file mode 100644 index 000000000..b881d90fa --- /dev/null +++ b/src/client/views/nodes/ChatBox/tools/CalculateTool.ts @@ -0,0 +1,25 @@ +import { BaseTool } from './BaseTool'; + +export class CalculateTool extends BaseTool { + constructor() { + super( + 'calculate', + 'Perform a calculation', + { + expression: { + type: 'string', + description: 'The mathematical expression to evaluate', + required: 'true', + }, + }, + 'Provide a mathematical expression to calculate that would work with JavaScript eval().', + 'Runs a calculation and returns the number - uses JavaScript so be sure to use floating point syntax if necessary' + ); + } + + async execute(args: { expression: string }): Promise { + // Note: Using eval() can be dangerous. Consider using a safer alternative. + const result = eval(args.expression); + return [{ type: 'text', text: result.toString() }]; + } +} diff --git a/src/client/views/nodes/ChatBox/tools/RAGTool.ts b/src/client/views/nodes/ChatBox/tools/RAGTool.ts new file mode 100644 index 000000000..84d5430e7 --- /dev/null +++ b/src/client/views/nodes/ChatBox/tools/RAGTool.ts @@ -0,0 +1,81 @@ +import { BaseTool } from './BaseTool'; +import { Vectorstore } from '../vectorstore/VectorstoreUpload'; +import { Chunk } from '../types'; + +export class RAGTool extends BaseTool { + constructor( + private vectorstore: Vectorstore, + summaries: string + ) { + super( + 'rag', + 'Perform a RAG search on user documents', + { + hypothetical_document_chunk: { + type: 'string', + description: + "Detailed version of the prompt that is effectively a hypothetical document chunk that would be ideal to embed and compare to the vectors of real document chunks to fetch the most relevant document chunks to answer the user's query", + required: 'true', + }, + }, + `Your task is to first provide a response to the user's prompt based on the information given in the chunks and considering the chat history. Follow these steps: + + 1. Carefully read and analyze the provided chunks, which may include text, images, or tables. Each chunk has an associated chunk_id. + + 2. Review the prompt and chat history to understand the context of the user's question or request. + + 3. Formulate a response that addresses the prompt using information from the relevant chunks. Your response should be informative and directly answer the user's question or request. + + 4. Use citations to support your response. Citations should contain direct textual references to the granular, specific part of the original chunk that applies to the situation—with no text ommitted. Citations should be in the following format: + - For text: relevant direct text from the chunk that the citation in referencing specifically + - For images or tables: + + Place citations after the sentences they apply to. You can use multiple citations in a row. + + 5. If there's insufficient information in the provided chunks to answer the prompt sufficiently, ALWAYS respond with RAG not applicable + + Write your entire response, including follow-up questions, inside tags. Remember to use the citation format for both text and image references, and maintain a conversational tone throughout your response. + + !!!IMPORTANT Before you close the tag with , within the answer tags provide a set of 3 follow-up questions inside a tag and individually within tags. These should relate to the document, the current query, and the chat_history and should aim to help the user better understand whatever they are looking for. + Also, ensure that the answer tags are wrapped with the correct step tags as well.`, + `Performs a RAG (Retrieval-Augmented Generation) search on user documents and returns a + set of document chunks (either images or text) that can be used to provide a grounded response based on + user documents + + !!!IMPORTANT Use the RAG tool ANYTIME the question may potentially (even if you are not sure) relate to one of the user's documents. + Here are the summaries of the user's documents: + ${summaries}` + ); + } + + async execute(args: { hypothetical_document_chunk: string }): Promise { + const relevantChunks = await this.vectorstore.retrieve(args.hypothetical_document_chunk); + return this.getFormattedChunks(relevantChunks); + } + + private getFormattedChunks(relevantChunks: Chunk[]): { type: string; text?: string; image_url?: { url: string } }[] { + const content: { type: string; text?: string; image_url?: { url: string } }[] = [{ type: 'text', text: '' }]; + + for (const chunk of relevantChunks) { + content.push({ + type: 'text', + text: ``, + }); + + if (chunk.metadata.type === 'image') { + // Implement image loading and base64 encoding here + // For now, we'll just add a placeholder + content.push({ + type: 'image_url', + image_url: { url: chunk.metadata.file_path }, + }); + } + + content.push({ type: 'text', text: `${chunk.metadata.text}\n\n` }); + } + + content.push({ type: 'text', text: '' }); + + return content; + } +} diff --git a/src/client/views/nodes/ChatBox/tools/WikipediaTool.ts b/src/client/views/nodes/ChatBox/tools/WikipediaTool.ts new file mode 100644 index 000000000..0aef58f61 --- /dev/null +++ b/src/client/views/nodes/ChatBox/tools/WikipediaTool.ts @@ -0,0 +1,33 @@ +import { BaseTool } from './BaseTool'; +import axios from 'axios'; + +export class WikipediaTool extends BaseTool { + constructor() { + super( + 'wikipedia', + 'Search Wikipedia and return a summary', + { + title: { + type: 'string', + description: 'The title of the Wikipedia article to search', + required: 'true', + }, + }, + 'Provide simply the title you want to search on Wikipedia and nothing more. If re-using this tool, try a different title for different information.', + 'Returns a summary from searching an article title on Wikipedia' + ); + } + + async execute(args: { title: string }): Promise { + const response = await axios.get('https://en.wikipedia.org/w/api.php', { + params: { + action: 'query', + list: 'search', + srsearch: args.title, + format: 'json', + }, + }); + const result = response.data.query.search[0].snippet; + return [{ type: 'text', text: result }]; + } +} -- cgit v1.2.3-70-g09d2