From 95c0d9b0ed3cf8bf50f3a3eac2f1dff146ba131c Mon Sep 17 00:00:00 2001 From: "A.J. Shulman" Date: Mon, 7 Jul 2025 15:20:10 -0400 Subject: merge issues --- .../views/nodes/chatbot/agentsystem/Agent.ts | 17 +-- .../nodes/chatbot/chatboxcomponents/ChatBox.tsx | 166 +++++++++++---------- src/client/views/nodes/chatbot/tools/RAGTool.ts | 1 + src/server/chunker/pdf_chunker.py | 41 ++--- 4 files changed, 102 insertions(+), 123 deletions(-) (limited to 'src') diff --git a/src/client/views/nodes/chatbot/agentsystem/Agent.ts b/src/client/views/nodes/chatbot/agentsystem/Agent.ts index a16794e10..8516f054b 100644 --- a/src/client/views/nodes/chatbot/agentsystem/Agent.ts +++ b/src/client/views/nodes/chatbot/agentsystem/Agent.ts @@ -7,14 +7,17 @@ import { AnswerParser } from '../response_parsers/AnswerParser'; import { StreamedAnswerParser } from '../response_parsers/StreamedAnswerParser'; import { BaseTool } from '../tools/BaseTool'; import { CalculateTool } from '../tools/CalculateTool'; +//import { CreateAnyDocumentTool } from '../tools/CreateAnyDocTool'; import { DataAnalysisTool } from '../tools/DataAnalysisTool'; import { DocumentMetadataTool } from '../tools/DocumentMetadataTool'; +import { ImageCreationTool } from '../tools/ImageCreationTool'; import { NoTool } from '../tools/NoTool'; import { SearchTool } from '../tools/SearchTool'; import { Parameter, ParametersType, TypeMap } from '../types/tool_types'; import { AgentMessage, ASSISTANT_ROLE, AssistantMessage, Observation, PROCESSING_TYPE, ProcessingInfo, TEXT_TYPE } from '../types/types'; import { Vectorstore } from '../vectorstore/Vectorstore'; import { getReactPrompt } from './prompts'; +//import { DictionaryTool } from '../tools/DictionaryTool'; import { ChatCompletionMessageParam } from 'openai/resources'; import { Doc } from '../../../../../fields/Doc'; import { ChatBox, parsedDoc } from '../chatboxcomponents/ChatBox'; @@ -27,7 +30,7 @@ import { CodebaseSummarySearchTool } from '../tools/CodebaseSummarySearchTool'; import { FileContentTool } from '../tools/FileContentTool'; import { FileNamesTool } from '../tools/FileNamesTool'; import { CreateNewTool } from '../tools/CreateNewTool'; -import { GPTTutorialTool } from '../tools/TutorialTool'; +//import { CreateTextDocTool } from '../tools/CreateTextDocumentTool'; dotenv.config(); @@ -50,7 +53,6 @@ export class Agent { private streamedAnswerParser: StreamedAnswerParser = new StreamedAnswerParser(); private tools: Record>>; private _docManager: AgentDocumentManager; - private is_dash_doc_assistant: boolean; // Dynamic tool registry for tools created at runtime private dynamicToolRegistry: Map>> = new Map(); // Callback for notifying when tools are created and need reload @@ -75,8 +77,7 @@ export class Agent { csvData: () => { filename: string; id: string; text: string }[], createImage: (result: Upload.FileInformation & Upload.InspectionResults, options: DocumentOptions) => void, createCSVInDash: (url: string, title: string, id: string, data: string) => void, - docManager: AgentDocumentManager, - isDashDocAssistant: boolean + docManager: AgentDocumentManager ) { // Initialize OpenAI client with API key from environment this.client = new OpenAI({ apiKey: process.env.OPENAI_KEY, dangerouslyAllowBrowser: true }); @@ -84,7 +85,6 @@ export class Agent { this._history = history; this._csvData = csvData; this._docManager = docManager; - this.is_dash_doc_assistant = isDashDocAssistant; // Initialize dynamic tool registry this.dynamicToolRegistry = new Map(); @@ -103,7 +103,6 @@ export class Agent { codebaseSummarySearch: new CodebaseSummarySearchTool(this.vectorstore), fileContent: new FileContentTool(this.vectorstore), fileNames: new FileNamesTool(this.vectorstore), - generateTutorialNode: new GPTTutorialTool(this._docManager), }; // Add the createNewTool after other tools are defined @@ -143,7 +142,7 @@ export class Agent { const instance: BaseTool> = new ToolClass(); - // Prefer the tool's self-declared name (matches tag) + // Prefer the tool’s self-declared name (matches tag) const key = (instance.name || '').trim() || legacyKey; // Check for duplicates @@ -298,7 +297,7 @@ export class Agent { ignoreAttributes: false, attributeNamePrefix: '@_', textNodeName: '_text', - isArray: name => name === 'url', + isArray: name => ['query', 'url'].indexOf(name) !== -1, processEntities: false, // Disable processing of entities stopNodes: ['*.entity'], // Do not process any entities }); @@ -760,7 +759,7 @@ export class Agent { const docSummaries = () => JSON.stringify(this._docManager.listDocs); const chatHistory = this._history(); - return getReactPrompt(allTools, docSummaries, chatHistory, this.is_dash_doc_assistant); + return getReactPrompt(allTools, docSummaries, chatHistory); } /** diff --git a/src/client/views/nodes/chatbot/chatboxcomponents/ChatBox.tsx b/src/client/views/nodes/chatbot/chatboxcomponents/ChatBox.tsx index 9fdbd8f58..093c1244c 100644 --- a/src/client/views/nodes/chatbot/chatboxcomponents/ChatBox.tsx +++ b/src/client/views/nodes/chatbot/chatboxcomponents/ChatBox.tsx @@ -7,46 +7,47 @@ * with support for follow-up questions and citation management. */ -import { Button, Size, Type } from '@dash/components'; +import dotenv from 'dotenv'; import { ObservableSet, action, computed, makeObservable, observable, observe, reaction, runInAction } from 'mobx'; import { observer } from 'mobx-react'; import OpenAI, { ClientOptions } from 'openai'; import * as React from 'react'; -import { AiOutlineSend } from 'react-icons/ai'; import { v4 as uuidv4 } from 'uuid'; import { ClientUtils, OmitKeys } from '../../../../../ClientUtils'; import { Doc, DocListCast, Opt } from '../../../../../fields/Doc'; -import { DocData, DocViews } from '../../../../../fields/DocSymbols'; +import { DocData, DocLayout, DocViews } from '../../../../../fields/DocSymbols'; import { Id } from '../../../../../fields/FieldSymbols'; import { RichTextField } from '../../../../../fields/RichTextField'; import { ScriptField } from '../../../../../fields/ScriptField'; -import { AudioCast, CsvCast, DocCast, NumCast, PDFCast, RTFCast, StrCast, VideoCast } from '../../../../../fields/Types'; -import { Upload } from '../../../../../server/SharedMediaTypes'; -import { DocServer } from '../../../../DocServer'; +import { CsvCast, DocCast, NumCast, PDFCast, RTFCast, StrCast, VideoCast, AudioCast } from '../../../../../fields/Types'; import { DocUtils } from '../../../../documents/DocUtils'; import { CollectionViewType, DocumentType } from '../../../../documents/DocumentTypes'; import { Docs, DocumentOptions } from '../../../../documents/Documents'; +import { DocServer } from '../../../../DocServer'; import { DocumentManager } from '../../../../util/DocumentManager'; import { ImageUtils } from '../../../../util/Import & Export/ImageUtils'; import { LinkManager } from '../../../../util/LinkManager'; import { CompileError, CompileScript } from '../../../../util/Scripting'; -import { SnappingManager } from '../../../../util/SnappingManager'; import { DictationButton } from '../../../DictationButton'; import { ViewBoxAnnotatableComponent } from '../../../DocComponent'; import { AudioBox } from '../../AudioBox'; import { DocumentView, DocumentViewInternal } from '../../DocumentView'; import { FieldView, FieldViewProps } from '../../FieldView'; -import { OpenWhere } from '../../OpenWhere'; import { PDFBox } from '../../PDFBox'; import { ScriptingBox } from '../../ScriptingBox'; import { VideoBox } from '../../VideoBox'; import { Agent } from '../agentsystem/Agent'; import { supportedDocTypes } from '../types/tool_types'; import { ASSISTANT_ROLE, AssistantMessage, CHUNK_TYPE, Citation, ProcessingInfo, SimplifiedChunk, TEXT_TYPE } from '../types/types'; -import { AgentDocumentManager } from '../utils/AgentDocumentManager'; import { Vectorstore } from '../vectorstore/Vectorstore'; import './ChatBox.scss'; import MessageComponentBox from './MessageComponent'; +import { OpenWhere } from '../../OpenWhere'; +import { Upload } from '../../../../../server/SharedMediaTypes'; +import { DocumentMetadataTool } from '../tools/DocumentMetadataTool'; +import { AgentDocumentManager } from '../utils/AgentDocumentManager'; + +dotenv.config(); export type parsedDocData = { doc_type: string; @@ -57,7 +58,6 @@ export type parsedDocData = { data_useCors?: boolean; }; export type parsedDoc = DocumentOptions & parsedDocData; - /** * ChatBox is the main class responsible for managing the interaction between the user and the assistant, * handling documents, and integrating with OpenAI for tasks such as document analysis, chat functionality, @@ -124,15 +124,7 @@ export class ChatBox extends ViewBoxAnnotatableComponent() { this.vectorstore = new Vectorstore(this.vectorstore_id, this.docManager); // Create an agent with the vectorstore - this.agent = new Agent( - this.vectorstore, - this.retrieveFormattedHistory.bind(this), - this.retrieveCSVData.bind(this), - this.createImageInDash.bind(this), - this.createCSVInDash.bind(this), - this.docManager, - this.dataDoc.is_dash_doc_assistant === 'true' - ); + this.agent = new Agent(this.vectorstore, this.retrieveFormattedHistory.bind(this), this.retrieveCSVData.bind(this), this.createImageInDash.bind(this), this.createCSVInDash.bind(this), this.docManager); // Set up the tool created callback this.agent.setToolCreatedCallback(this.handleToolCreated); @@ -382,6 +374,7 @@ export class ChatBox extends ViewBoxAnnotatableComponent() { }; } }); + this.scrollToBottom(); }; const onAnswerUpdate = (answerUpdate: string) => { @@ -389,33 +382,41 @@ export class ChatBox extends ViewBoxAnnotatableComponent() { if (this._current_message) { this._current_message = { ...this._current_message, - content: [{ index: 0, type: TEXT_TYPE.NORMAL, text: answerUpdate, citation_ids: null }], + content: [{ text: answerUpdate, type: TEXT_TYPE.NORMAL, index: 0, citation_ids: [] }], }; } }); }; - // Get the response from the agent - let userQuery = trimmedText; - if (this.dataDoc.is_dash_doc_assistant) { - userQuery = `The user is asking a question about Dash functionality. Their question is: "${trimmedText}". You should use the generateTutorialNode tool to answer this question.`; - } - const response = await this.agent.askAgent(userQuery, onProcessingUpdate, onAnswerUpdate); + // Send the user's question to the assistant and get the final message + const finalMessage = await this.agent.askAgent(trimmedText, onProcessingUpdate, onAnswerUpdate); - // Push the final message to history + // Update the history with the final assistant message runInAction(() => { - this._history.push(response); - this._isLoading = false; - this._current_message = undefined; + if (this._current_message) { + this._history.push({ ...finalMessage }); + this._current_message = undefined; + this.dataDoc.data = JSON.stringify(this._history); + } }); - } catch (error) { - console.error('Error in askGPT:', error); + } catch (err) { + console.error('Error:', err); + // Handle error in processing + runInAction(() => + this._history.push({ + role: ASSISTANT_ROLE.ASSISTANT, + content: [{ index: 0, type: TEXT_TYPE.ERROR, text: `Sorry, I encountered an error while processing your request: ${err} `, citation_ids: null }], + processing_info: [], + }) + ); + } finally { runInAction(() => { this._isLoading = false; - this._current_message = undefined; }); + this.scrollToBottom(); } } + this.scrollToBottom(); }; /** @@ -488,7 +489,7 @@ export class ChatBox extends ViewBoxAnnotatableComponent() { const data = (doc as parsedDocData).data; const ndoc = (() => { switch (doc.doc_type) { - default: + default: case supportedDocTypes.note: return Docs.Create.TextDocument(data as string, options); case supportedDocTypes.comparison: return this.createComparison(JSON.parse(data as string) as parsedDoc[], options); case supportedDocTypes.flashcard: return this.createFlashcard(JSON.parse(data as string) as parsedDoc[], options); @@ -496,25 +497,25 @@ export class ChatBox extends ViewBoxAnnotatableComponent() { case supportedDocTypes.image: return Docs.Create.ImageDocument(data as string, options); case supportedDocTypes.equation: return Docs.Create.EquationDocument(data as string, options); case supportedDocTypes.notetaking: return Docs.Create.NoteTakingDocument([], options); - case supportedDocTypes.web: + case supportedDocTypes.web: // Create web document with enhanced safety options const webOptions = { - ...options, + ...options, data_useCors: true }; - + // If iframe_sandbox was passed from AgentDocumentManager, add it to the options if ('_iframe_sandbox' in options) { - webOptions._iframe_sandbox = options._iframe_sandbox; + (webOptions as any)._iframe_sandbox = options._iframe_sandbox; } - + return Docs.Create.WebDocument(data as string, webOptions); case supportedDocTypes.dataviz: case supportedDocTypes.table: return Docs.Create.DataVizDocument('/Users/ajshul/Dash-Web/src/server/public/files/csv/0d237e7c-98c9-44d0-aa61-5285fdbcf96c-random_sample.csv.csv', options); case supportedDocTypes.pdf: return Docs.Create.PdfDocument(data as string, options); case supportedDocTypes.video: return Docs.Create.VideoDocument(data as string, options); case supportedDocTypes.diagram: return Docs.Create.DiagramDocument(undefined, { text: data as unknown as RichTextField, ...options}); // text: can take a string or RichTextField but it's typed for RichTextField. - - // case supportedDocumentTypes.dataviz: + + // case supportedDocumentTypes.dataviz: // { // const { fileUrl, id } = await Networking.PostToServer('/createCSV', { // filename: (options.title as string).replace(/\s+/g, '') + '.csv', @@ -539,7 +540,7 @@ export class ChatBox extends ViewBoxAnnotatableComponent() { const arr = this.createCollectionWithChildren(JSON.parse(data as string) as parsedDoc[], true).filter(d=>d).map(d => d!); const collOpts = { _width:300, _height: 300, _layout_fitWidth: true, _freeform_backgroundGrid: true, ...options, }; return (() => { - switch (options.type_collection) { + switch (options.type_collection) { case CollectionViewType.Tree: return Docs.Create.TreeDocument(arr, collOpts); case CollectionViewType.Stacking: return Docs.Create.StackingDocument(arr, collOpts); case CollectionViewType.Masonry: return Docs.Create.MasonryDocument(arr, collOpts); @@ -548,9 +549,9 @@ export class ChatBox extends ViewBoxAnnotatableComponent() { case CollectionViewType.Carousel3D: return Docs.Create.Carousel3DDocument(arr, collOpts); case CollectionViewType.Multicolumn: return Docs.Create.CarouselDocument(arr, collOpts); default: return Docs.Create.FreeformDocument(arr, collOpts); - } - })(); - } + } + })(); + } // case supportedDocumentTypes.map: return Docs.Create.MapDocument([], options); // case supportedDocumentTypes.button: return Docs.Create.ButtonDocument(options); // case supportedDocumentTypes.trail: return Docs.Create.PresDocument(options); @@ -558,8 +559,8 @@ export class ChatBox extends ViewBoxAnnotatableComponent() { })(); if (ndoc) { - ndoc.x = ((options.x as number) ?? 0) + (insideCol ? 0 : NumCast(this.layoutDoc.x) + NumCast(this.layoutDoc.width)) + 100; - ndoc.y = ((options.y as number) ?? 0) + (insideCol ? 0 : NumCast(this.layoutDoc.y)); + ndoc.x = NumCast((options.x as number) ?? 0) + (insideCol ? 0 : NumCast(this.layoutDoc.x) + NumCast(this.layoutDoc.width)) + 100; + ndoc.y = NumCast(options.y as number) + (insideCol ? 0 : NumCast(this.layoutDoc.y)); } return ndoc; }; @@ -639,7 +640,7 @@ export class ChatBox extends ViewBoxAnnotatableComponent() { handleCitationClick = async (citation: Citation) => { try { // Extract values from MobX proxy object if needed - const chunkId = typeof citation.chunk_id === 'object' ? (citation.chunk_id as unknown as object).toString() : citation.chunk_id; + const chunkId = typeof citation.chunk_id === 'object' ? (citation.chunk_id as any).toString() : citation.chunk_id; // For debugging console.log('Citation clicked:', { @@ -681,7 +682,7 @@ export class ChatBox extends ViewBoxAnnotatableComponent() { } this.handleOtherChunkTypes(foundChunk, citation, doc, dataDoc); // Show the chunk text in citation popup - const chunkText = citation.direct_text || 'Text content not available'; + let chunkText = citation.direct_text || 'Text content not available'; this.showCitationPopup(chunkText); // Also navigate to the document @@ -713,7 +714,7 @@ export class ChatBox extends ViewBoxAnnotatableComponent() { // If specific indexes are provided, filter segments by those indexes if (indexesOfSegments && indexesOfSegments.length > 0) { - segments = original_segments.filter(segment => indexesOfSegments.includes(segment.index)); + segments = original_segments.filter((segment: any) => indexesOfSegments.includes(segment.index)); } // If no segments match the indexes, use all segments @@ -722,7 +723,7 @@ export class ChatBox extends ViewBoxAnnotatableComponent() { } // First try to find an exact match - const exactMatch = segments.find(segment => segment.text && segment.text.includes(citationText)); + const exactMatch = segments.find((segment: any) => segment.text && segment.text.includes(citationText)); if (exactMatch) { return exactMatch.start; @@ -831,26 +832,22 @@ export class ChatBox extends ViewBoxAnnotatableComponent() { existingDoc._width = x2 - x1; existingDoc._height = y2 - y1; } - // const highlightDoc = - existingDoc ?? this.createImageCitationHighlight(x1, y1, x2, y2, citation, annotationKey, doc); + const highlightDoc = existingDoc ?? this.createImageCitationHighlight(x1, y1, x2, y2, citation, annotationKey, doc); //doc.layout_scroll = y1; doc._layout_curPage = foundChunk.startPage + 1; - DocumentManager.Instance.showDocument(doc, { willZoomCentered: true }, () => {}); - //DocumentManager.Instance.showDocument(highlightDoc, { willZoomCentered: true }, () => {}); + DocumentManager.Instance.showDocument(highlightDoc, { willZoomCentered: true }, () => {}); } break; case CHUNK_TYPE.TEXT: - { - this._citationPopup = { text: citation.direct_text ?? 'No text available', visible: true }; - this.startCitationPopupTimer(); + this._citationPopup = { text: citation.direct_text ?? 'No text available', visible: true }; + this.startCitationPopupTimer(); - // Check if the document is a PDF (has a PDF viewer component) - const isPDF = PDFCast(dataDoc!.data) !== null || dataDoc!.type === DocumentType.PDF; + // Check if the document is a PDF (has a PDF viewer component) + const isPDF = PDFCast(dataDoc!.data) !== null || dataDoc!.type === DocumentType.PDF; - // First ensure document is fully visible before trying to access its views - this.ensureDocumentIsVisible(dataDoc!, isPDF, citation, foundChunk, doc); - } + // First ensure document is fully visible before trying to access its views + this.ensureDocumentIsVisible(dataDoc!, isPDF, citation, foundChunk, doc); break; case CHUNK_TYPE.CSV: case CHUNK_TYPE.URL: @@ -1068,9 +1065,7 @@ export class ChatBox extends ViewBoxAnnotatableComponent() { { index: 0, type: TEXT_TYPE.NORMAL, - text: this.dataDoc.is_dash_doc_assistant - ? 'Welcome to your help assistant for Dash. Ask any Dash-related questions to get started.' - : `Hey, ${this.userName()}! Welcome to Your Friendly Assistant. Link a document or ask questions to get started.`, + text: `Hey, ${this.userName()}! Welcome to Your Friendly Assistant. Link a document or ask questions to get started.`, citation_ids: null, }, ], @@ -1167,9 +1162,6 @@ export class ChatBox extends ViewBoxAnnotatableComponent() { this._inputValue = question; }; - _dictation: DictationButton | null = null; - setInputRef = (r: HTMLInputElement) => (this._textInputRef = r); - setDictationRef = (r: DictationButton) => (this._dictation = r); /** * Handles tool creation notification and shows the reload modal * @param toolName The name of the tool that was created @@ -1220,6 +1212,8 @@ export class ChatBox extends ViewBoxAnnotatableComponent() { }, 100); }; + _dictation: DictationButton | null = null; + /** * Toggles the font size modal visibility */ @@ -1411,7 +1405,7 @@ export class ChatBox extends ViewBoxAnnotatableComponent() { )}
-

{StrCast(this.dataDoc.title) || `${this.userName()}'s AI Assistant`}

+

{this.userName()}'s AI Assistant

{this.renderFontSizeIcon()}
@@ -1448,7 +1442,9 @@ export class ChatBox extends ViewBoxAnnotatableComponent() {
{ + this._textInputRef = r; + }} type="text" name="messageInput" autoComplete="off" @@ -1458,17 +1454,23 @@ export class ChatBox extends ViewBoxAnnotatableComponent() { disabled={this._isLoading} />
- + { + this._dictation = r; + }} + setInput={this.setChatInput} + inputRef={this._textInputRef} /> - {/* Popup for citation */} {this._citationPopup.visible && ( @@ -1498,7 +1500,7 @@ export class ChatBox extends ViewBoxAnnotatableComponent() { The tool {this._toolReloadModal.toolName} has been created and saved successfully.

To make the tool available for future use, the page needs to be reloaded to rebuild the application bundle.

-

Click "Reload Page" to complete the tool installation.

+

Click "Reload Page" to complete the tool installation.