diff options
9 files changed, 130 insertions, 41 deletions
diff --git a/src/client/views/nodes/chatbot/agentsystem/Agent.ts b/src/client/views/nodes/chatbot/agentsystem/Agent.ts index 1eb5e3963..1cf6ca030 100644 --- a/src/client/views/nodes/chatbot/agentsystem/Agent.ts +++ b/src/client/views/nodes/chatbot/agentsystem/Agent.ts @@ -21,6 +21,7 @@ import { CreateTextDocTool } from '../tools/CreateTextDocumentTool'; import { DocumentOptions } from '../../../../documents/Documents'; import { CreateAnyDocumentTool } from '../tools/CreateAnyDocTool'; import { ImageCreationTool } from '../tools/ImageCreationTool'; +import { DictionaryTool } from '../tools/DictionaryTool'; dotenv.config(); @@ -60,7 +61,8 @@ export class Agent { csvData: () => { filename: string; id: string; text: string }[], addLinkedUrlDoc: (url: string, id: string) => void, addLinkedDoc: (doc_type: string, data: string | undefined, options: DocumentOptions, id: string) => void, - createCSVInDash: (url: string, title: string, id: string, data: string) => void + createCSVInDash: (url: string, title: string, id: string, data: string) => void, + createImage: (result: any, options: DocumentOptions) => void ) { // Initialize OpenAI client with API key from environment this.client = new OpenAI({ apiKey: process.env.OPENAI_KEY, dangerouslyAllowBrowser: true }); @@ -74,13 +76,14 @@ export class Agent { calculate: new CalculateTool(), rag: new RAGTool(this.vectorstore), dataAnalysis: new DataAnalysisTool(csvData), - //websiteInfoScraper: new WebsiteInfoScraperTool(addLinkedUrlDoc), - //searchTool: new SearchTool(addLinkedUrlDoc), + websiteInfoScraper: new WebsiteInfoScraperTool(addLinkedUrlDoc), + searchTool: new SearchTool(addLinkedUrlDoc), createCSV: new CreateCSVTool(createCSVInDash), noTool: new NoTool(), - imageCreationTool: new ImageCreationTool(addLinkedDoc), + imageCreationTool: new ImageCreationTool(createImage), //createTextDoc: new CreateTextDocTool(addLinkedDoc), createAnyDocument: new CreateAnyDocumentTool(addLinkedDoc), + dictionary: new DictionaryTool(), }; } diff --git a/src/client/views/nodes/chatbot/chatboxcomponents/ChatBox.tsx b/src/client/views/nodes/chatbot/chatboxcomponents/ChatBox.tsx index e5a90ab4a..d2931106a 100644 --- a/src/client/views/nodes/chatbot/chatboxcomponents/ChatBox.tsx +++ b/src/client/views/nodes/chatbot/chatboxcomponents/ChatBox.tsx @@ -40,6 +40,11 @@ import { DiagramBox } from '../../DiagramBox'; import { ImageField } from '../../../../../fields/URLField'; import { DashUploadUtils } from '../../../../../server/DashUploadUtils'; import { DocCreatorMenu, Field, FieldUtils } from '../../DataVizBox/DocCreatorMenu'; +import { ImageUtils } from '../../../../util/Import & Export/ImageUtils'; +import { ScriptManager } from '../../../../util/ScriptManager'; +import { CompileError, CompileScript } from '../../../../util/Scripting'; +import { ScriptField } from '../../../../../fields/ScriptField'; +import { ScriptingBox } from '../../ScriptingBox'; dotenv.config(); @@ -96,7 +101,7 @@ export class ChatBox extends ViewBoxAnnotatableComponent<FieldViewProps>() { this.vectorstore_id = StrCast(this.dataDoc.vectorstore_id); } this.vectorstore = new Vectorstore(this.vectorstore_id, this.retrieveDocIds); - this.agent = new Agent(this.vectorstore, this.retrieveSummaries, this.retrieveFormattedHistory, this.retrieveCSVData, this.addLinkedUrlDoc, this.createDocInDash, this.createCSVInDash); + this.agent = new Agent(this.vectorstore, this.retrieveSummaries, this.retrieveFormattedHistory, this.retrieveCSVData, this.addLinkedUrlDoc, this.createDocInDash, this.createCSVInDash, this.createImageInDash); this.messagesRef = React.createRef<HTMLDivElement>(); // Reaction to update dataDoc when chat history changes @@ -400,20 +405,17 @@ export class ChatBox extends ViewBoxAnnotatableComponent<FieldViewProps>() { }; @action - createImageInDash = async (url: string, title: string, id: string, data: string) => { - const doc = FieldUtils.ImageField( - { - tl: [0, 0], - br: [300, 300], - }, - 300, - 300, - title, - url ?? '', - {} - ); - - return doc; + createImageInDash = async (result: any, options: DocumentOptions) => { + const newImgSrc = + result.accessPaths.agnostic.client.indexOf('dashblobstore') === -1 // + ? ClientUtils.prepend(result.accessPaths.agnostic.client) + : result.accessPaths.agnostic.client; + const doc = Docs.Create.ImageDocument(newImgSrc, options); + this.addDocument(ImageUtils.AssignImgInfo(doc, result)); + const linkDoc = Docs.Create.LinkDocument(this.Document, doc); + LinkManager.Instance.addLink(linkDoc); + doc && this._props.addDocument?.(doc); + await DocumentManager.Instance.showDocument(doc, { willZoomCentered: true }, () => {}); }; /** @@ -431,10 +433,6 @@ export class ChatBox extends ViewBoxAnnotatableComponent<FieldViewProps>() { case 'text': doc = Docs.Create.TextDocument(data || '', options); break; - case 'image': - console.log('imageURL: ' + data); - doc = await this.createImageInDash(data || '', options.title as string, '', data || ''); - break; case 'pdf': doc = Docs.Create.PdfDocument(data || '', options); break; @@ -471,6 +469,24 @@ export class ChatBox extends ViewBoxAnnotatableComponent<FieldViewProps>() { case 'chat': doc = Docs.Create.ChatDocument(options); break; + case 'script': + const result = !data!.trim() ? ({ compiled: false, errors: [] } as CompileError) : CompileScript(data!, {}); + const script_field = result.compiled ? new ScriptField(result, undefined, data!) : undefined; + doc = Docs.Create.ScriptingDocument(script_field, options); + await DocumentManager.Instance.showDocument(doc, { willZoomCentered: true }, () => { + const firstView = Array.from(doc[DocViews])[0] as DocumentView; + (firstView.ComponentView as ScriptingBox)?.onApply?.(); + (firstView.ComponentView as ScriptingBox)?.onRun?.(); + }); + + break; + // this.dataDoc.script = this.rawScript; + + // ScriptManager.Instance.addScript(this.dataDoc); + + // this._scriptKeys = ScriptingGlobals.getGlobals(); + // this._scriptingDescriptions = ScriptingGlobals.getDescriptions(); + // this._scriptingParams = ScriptingGlobals.getParameters(); // Add more cases for other document types default: console.error('Unknown or unsupported document type:', doc_type); diff --git a/src/client/views/nodes/chatbot/tools/CreateAnyDocTool.ts b/src/client/views/nodes/chatbot/tools/CreateAnyDocTool.ts index 4c059177b..36f133503 100644 --- a/src/client/views/nodes/chatbot/tools/CreateAnyDocTool.ts +++ b/src/client/views/nodes/chatbot/tools/CreateAnyDocTool.ts @@ -7,8 +7,8 @@ import { DocumentOptions, Docs } from '../../../../documents/Documents'; /** * List of supported document types that can be created via text LLM. */ -type supportedDocumentTypesType = 'text' | 'html' | 'equation' | 'function_plot' | 'dataviz' | 'note_taking' | 'rtf' | 'message' | 'mermaid_diagram'; -const supportedDocumentTypes: supportedDocumentTypesType[] = ['text', 'html', 'equation', 'function_plot', 'dataviz', 'note_taking', 'rtf', 'message', 'mermaid_diagram']; +type supportedDocumentTypesType = 'text' | 'html' | 'equation' | 'function_plot' | 'dataviz' | 'note_taking' | 'rtf' | 'message' | 'mermaid_diagram' | 'script'; +const supportedDocumentTypes: supportedDocumentTypesType[] = ['text', 'html', 'equation', 'function_plot', 'dataviz', 'note_taking', 'rtf', 'message', 'mermaid_diagram', 'script']; /** * Description of document options and data field for each type. @@ -50,6 +50,10 @@ const documentTypesInfo = { options: ['title', 'backgroundColor', 'layout'], dataDescription: 'The Mermaid diagram content.', }, + script: { + options: ['title', 'backgroundColor', 'layout'], + dataDescription: 'The compilable JavaScript code. Use this for creating scripts.', + }, }; const createAnyDocumentToolParams = [ diff --git a/src/client/views/nodes/chatbot/tools/DictionaryTool.ts b/src/client/views/nodes/chatbot/tools/DictionaryTool.ts new file mode 100644 index 000000000..fa554e7b3 --- /dev/null +++ b/src/client/views/nodes/chatbot/tools/DictionaryTool.ts @@ -0,0 +1,64 @@ +import { Observation } from '../types/types'; +import { ParametersType, ToolInfo } from '../types/tool_types'; +import { BaseTool } from './BaseTool'; + +// Define the tool's parameters +const dictionaryToolParams = [ + { + name: 'word', + type: 'string', + description: 'The word to look up in the dictionary.', + required: true, + }, +] as const; + +type DictionaryToolParamsType = typeof dictionaryToolParams; + +// Define the tool's metadata and rules +const dictionaryToolInfo: ToolInfo<DictionaryToolParamsType> = { + name: 'dictionary', + citationRules: 'No citation needed.', + parameterRules: dictionaryToolParams, + description: 'Fetches the definition of a given word using an open dictionary API.', +}; + +export class DictionaryTool extends BaseTool<DictionaryToolParamsType> { + constructor() { + super(dictionaryToolInfo); + } + + async execute(args: ParametersType<DictionaryToolParamsType>): Promise<Observation[]> { + const url = `https://api.dictionaryapi.dev/api/v2/entries/en/${args.word}`; + + try { + const response = await fetch(url); + const data = await response.json(); + + // Handle cases where the word is not found + if (data.title === 'No Definitions Found') { + return [ + { + type: 'text', + text: `Sorry, I couldn't find a definition for the word "${args.word}".`, + }, + ]; + } + + // Extract the first definition + const definition = data[0]?.meanings[0]?.definitions[0]?.definition; + return [ + { + type: 'text', + text: `The definition of "${args.word}" is: ${definition}`, + }, + ]; + } catch (error) { + return [ + { + type: 'text', + text: `An error occurred while fetching the definition: ${error}`, + }, + ]; + } + } +} diff --git a/src/client/views/nodes/chatbot/tools/ImageCreationTool.ts b/src/client/views/nodes/chatbot/tools/ImageCreationTool.ts index 3db401b14..ba1aa987a 100644 --- a/src/client/views/nodes/chatbot/tools/ImageCreationTool.ts +++ b/src/client/views/nodes/chatbot/tools/ImageCreationTool.ts @@ -5,6 +5,8 @@ import { Observation } from '../types/types'; import { ParametersType, ToolInfo } from '../types/tool_types'; import { DocumentOptions } from '../../../../documents/Documents'; import { ClientUtils } from '../../../../../ClientUtils'; +import { DashUploadUtils } from '../../../../../server/DashUploadUtils'; +import { RTFCast, StrCast } from '../../../../../fields/Types'; const imageCreationToolParams = [ { @@ -25,10 +27,10 @@ const imageCreationToolInfo: ToolInfo<ImageCreationToolParamsType> = { }; export class ImageCreationTool extends BaseTool<ImageCreationToolParamsType> { - private _addLinkedDoc: (doc_type: string, data: string | undefined, options: DocumentOptions, id: string) => void; - constructor(addLinkedDoc: (doc_type: string, data: string | undefined, options: DocumentOptions, id: string) => void) { + private _createImage: (result: any, options: DocumentOptions) => void; + constructor(createImage: (result: any, options: DocumentOptions) => void) { super(imageCreationToolInfo); - this._addLinkedDoc = addLinkedDoc; + this._createImage = createImage; } async execute(args: ParametersType<ImageCreationToolParamsType>): Promise<Observation[]> { @@ -37,16 +39,14 @@ export class ImageCreationTool extends BaseTool<ImageCreationToolParamsType> { console.log(`Generating image for prompt: ${image_prompt}`); // Create an array of promises, each one handling a search for a query try { - const { url } = await Networking.PostToServer('/generateImage', { + const { result, url } = await Networking.PostToServer('/generateImage', { image_prompt, }); + console.log('Image generation result:', result); + this._createImage(result, { text: RTFCast(image_prompt) }); if (url) { - const result = await Networking.PostToServer('/uploadRemoteImage', { sources: [url] }); - const source = ClientUtils.prepend(result[0].accessPaths.agnostic.client); - const id = uuidv4(); - this._addLinkedDoc('image', source, {}, id); return [ { type: 'image_url', diff --git a/src/client/views/nodes/chatbot/tools/RAGTool.ts b/src/client/views/nodes/chatbot/tools/RAGTool.ts index 1f73986a7..2db61c768 100644 --- a/src/client/views/nodes/chatbot/tools/RAGTool.ts +++ b/src/client/views/nodes/chatbot/tools/RAGTool.ts @@ -23,8 +23,9 @@ const ragToolInfo: ToolInfo<RAGToolParamsType> = { 1. **Grounded Text Guidelines**: - Each <grounded_text> tag must correspond to exactly one citation, ensuring a one-to-one relationship. - Always cite a **subset** of the chunk, never the full text. The citation should be as short as possible while providing the relevant information (typically one to two sentences). - - Do not paraphrase the chunk text in the citation; use the original subset directly from the chunk. + - Do not paraphrase the chunk text in the citation; use the original subset directly from the chunk. IT MUST BE EXACT AND WORD FOR WORD FROM THE ORIGINAL CHUNK! - If multiple citations are needed for different sections of the response, create new <grounded_text> tags for each. + - !!!IMPORTANT: For video transcript citations, use a subset of the exact text from the transcript as the citation content. It should be just before the start of the section of the transcript that is relevant to the grounded_text tag. 2. **Citation Guidelines**: - The citation must include only the relevant excerpt from the chunk being referenced. @@ -56,7 +57,8 @@ const ragToolInfo: ToolInfo<RAGToolParamsType> = { ***NOTE***: - Prefer to cite visual elements (i.e. chart, image, table, etc.) over text, if they both can be used. Only if a visual element is not going to be helpful, then use text. Otherwise, use both! - Use as many citations as possible (even when one would be sufficient), thus keeping text as grounded as possible. - - Cite from as many documents as possible and always use MORE, and as granular, citations as possible.`, + - Cite from as many documents as possible and always use MORE, and as granular, citations as possible. + - CITATION TEXT MUST BE EXACTLY AS IT APPEARS IN THE CHUNK. DO NOT PARAPHRASE!`, parameterRules: ragToolParams, }; diff --git a/src/client/views/nodes/chatbot/types/types.ts b/src/client/views/nodes/chatbot/types/types.ts index 54fd7c979..995ac531d 100644 --- a/src/client/views/nodes/chatbot/types/types.ts +++ b/src/client/views/nodes/chatbot/types/types.ts @@ -19,6 +19,7 @@ export enum CHUNK_TYPE { URL = 'url', CSV = 'CSV', MEDIA = 'media', + VIDEO = 'video', } export enum PROCESSING_TYPE { diff --git a/src/client/views/nodes/chatbot/vectorstore/Vectorstore.ts b/src/client/views/nodes/chatbot/vectorstore/Vectorstore.ts index 3ed433778..d962b887f 100644 --- a/src/client/views/nodes/chatbot/vectorstore/Vectorstore.ts +++ b/src/client/views/nodes/chatbot/vectorstore/Vectorstore.ts @@ -159,7 +159,7 @@ export class Vectorstore { start_time: chunk.start, end_time: chunk.end, text: chunk.text, - chunkType: 'text', + type: CHUNK_TYPE.VIDEO, }, })), type: 'media', @@ -176,7 +176,7 @@ export class Vectorstore { start_time: chunk.metadata.start_time, end_time: chunk.metadata.end_time, indexes: chunk.metadata.indexes, - chunkType: CHUNK_TYPE.TEXT, + chunkType: CHUNK_TYPE.VIDEO, text: chunk.metadata.text, })); doc.chunk_simpl = JSON.stringify({ chunks: simplifiedChunks }); diff --git a/src/server/ApiManagers/AssistantManager.ts b/src/server/ApiManagers/AssistantManager.ts index 425365348..fbda74194 100644 --- a/src/server/ApiManagers/AssistantManager.ts +++ b/src/server/ApiManagers/AssistantManager.ts @@ -30,6 +30,7 @@ import ffmpeg from 'fluent-ffmpeg'; import OpenAI from 'openai'; import * as xmlbuilder from 'xmlbuilder'; import { last } from 'lodash'; +import { DashUploadUtils } from '../DashUploadUtils'; // Enumeration of directories where different file types are stored export enum Directory { @@ -370,9 +371,6 @@ export default class AssistantManager extends ApiManager { } // Step 5: Return the JSON result res.send({ full: originalSegments, condensed: combinedSegments, summary }); - - // Step 5: Return the JSON result - res.send({ full: originalSegments, condensed: combinedSegments, summary: summary }); } catch (error) { console.error('Error processing media file:', error); res.status(500).send({ error: 'Failed to process media file' }); @@ -428,10 +426,11 @@ export default class AssistantManager extends ApiManager { try { const image = await openai.images.generate({ model: 'dall-e-3', prompt: image_prompt, response_format: 'url' }); console.log(image); + const result = await DashUploadUtils.UploadImage(image.data[0].url!); const url = image.data[0].url; - res.send({ url }); + res.send({ result, url }); } catch (error) { console.error('Error fetching the URL:', error); res.status(500).send({ |