From 393b7f8286422c933102449eba1ba82874a48896 Mon Sep 17 00:00:00 2001 From: "A.J. Shulman" Date: Sun, 27 Apr 2025 14:57:39 -0400 Subject: improved consistency across doc types and parsing --- src/client/documents/Documents.ts | 1 + .../views/nodes/chatbot/agentsystem/Agent.ts | 15 +- .../nodes/chatbot/chatboxcomponents/ChatBox.tsx | 176 +++++++++------- .../chatbot/chatboxcomponents/ProgressBar.scss | 40 +++- .../nodes/chatbot/utils/AgentDocumentManager.ts | 234 ++++++++++++++++++++- .../views/nodes/chatbot/vectorstore/Vectorstore.ts | 49 ++--- 6 files changed, 390 insertions(+), 125 deletions(-) (limited to 'src') diff --git a/src/client/documents/Documents.ts b/src/client/documents/Documents.ts index 317bb7feb..f87bd7092 100644 --- a/src/client/documents/Documents.ts +++ b/src/client/documents/Documents.ts @@ -273,6 +273,7 @@ export class DocumentOptions { _layout_reflowHorizontal?: BOOLt = new BoolInfo('permit horizontal resizing with content reflow'); _layout_noSidebar?: BOOLt = new BoolInfo('whether to display the sidebar toggle button'); layout_boxShadow?: string; // box-shadow css string OR "standard" to use dash standard box shadow + _iframe_sandbox?: STRt = new StrInfo('sandbox attributes for iframes in web documents (e.g., allow-scripts, allow-same-origin)'); layout_maxShown?: NUMt = new NumInfo('maximum number of children to display at one time (see multicolumnview)'); _layout_columnWidth?: NUMt = new NumInfo('width of table column', false); _layout_columnCount?: NUMt = new NumInfo('number of columns in a masonry view'); diff --git a/src/client/views/nodes/chatbot/agentsystem/Agent.ts b/src/client/views/nodes/chatbot/agentsystem/Agent.ts index 80fdb6533..24471bf5b 100644 --- a/src/client/views/nodes/chatbot/agentsystem/Agent.ts +++ b/src/client/views/nodes/chatbot/agentsystem/Agent.ts @@ -41,7 +41,6 @@ export class Agent { private interMessages: AgentMessage[] = []; private vectorstore: Vectorstore; private _history: () => string; - private _summaries: () => string; private _csvData: () => { filename: string; id: string; text: string }[]; private actionNumber: number = 0; private thoughtNumber: number = 0; @@ -54,11 +53,13 @@ export class Agent { /** * The constructor initializes the agent with the vector store and toolset, and sets up the OpenAI client. * @param _vectorstore Vector store instance for document storage and retrieval. - * @param summaries A function to retrieve document summaries. + * @param summaries A function to retrieve document summaries (deprecated, now using docManager directly). * @param history A function to retrieve chat history. * @param csvData A function to retrieve CSV data linked to the assistant. - * @param addLinkedUrlDoc A function to add a linked document from a URL. + * @param getLinkedUrlDocId A function to get document IDs from URLs. + * @param createImage A function to create images in the dashboard. * @param createCSVInDash A function to create a CSV document in the dashboard. + * @param docManager The document manager instance. */ constructor( _vectorstore: Vectorstore, @@ -74,7 +75,6 @@ export class Agent { this.client = new OpenAI({ apiKey: process.env.OPENAI_KEY, dangerouslyAllowBrowser: true }); this.vectorstore = _vectorstore; this._history = history; - this._summaries = summaries; this._csvData = csvData; this._docManager = docManager; @@ -124,7 +124,12 @@ export class Agent { // Retrieve chat history and generate system prompt const chatHistory = this._history(); - const systemPrompt = getReactPrompt(Object.values(this.tools), this._summaries, chatHistory); + // Get document summaries directly from document manager + const documentSummaries = this._docManager.getAllDocumentSummaries(); + // Create a function that returns document summaries for the prompt + const getSummaries = () => documentSummaries; + // Generate the system prompt with the summaries + const systemPrompt = getReactPrompt(Object.values(this.tools), getSummaries, chatHistory); // Initialize intermediate messages this.interMessages = [{ role: 'system', content: systemPrompt }]; diff --git a/src/client/views/nodes/chatbot/chatboxcomponents/ChatBox.tsx b/src/client/views/nodes/chatbot/chatboxcomponents/ChatBox.tsx index b11bf7405..ba30cb42b 100644 --- a/src/client/views/nodes/chatbot/chatboxcomponents/ChatBox.tsx +++ b/src/client/views/nodes/chatbot/chatboxcomponents/ChatBox.tsx @@ -18,7 +18,7 @@ import { Doc, DocListCast, Opt } from '../../../../../fields/Doc'; import { DocData, DocViews } from '../../../../../fields/DocSymbols'; import { RichTextField } from '../../../../../fields/RichTextField'; import { ScriptField } from '../../../../../fields/ScriptField'; -import { CsvCast, DocCast, NumCast, PDFCast, RTFCast, StrCast } from '../../../../../fields/Types'; +import { CsvCast, DocCast, NumCast, PDFCast, RTFCast, StrCast, VideoCast, AudioCast } from '../../../../../fields/Types'; import { DocUtils } from '../../../../documents/DocUtils'; import { CollectionViewType, DocumentType } from '../../../../documents/DocumentTypes'; import { Docs, DocumentOptions } from '../../../../documents/Documents'; @@ -48,7 +48,14 @@ import { AgentDocumentManager } from '../utils/AgentDocumentManager'; dotenv.config(); -export type parsedDocData = { doc_type: string; data: unknown }; +export type parsedDocData = { + doc_type: string; + data: unknown; + _disable_resource_loading?: boolean; + _sandbox_iframe?: boolean; + _iframe_sandbox?: string; + data_useCors?: boolean; +}; export type parsedDoc = DocumentOptions & parsedDocData; /** * ChatBox is the main class responsible for managing the interaction between the user and the assistant, @@ -150,7 +157,14 @@ export class ChatBox extends ViewBoxAnnotatableComponent() { @action addDocToVectorstore = async (newLinkedDoc: Doc) => { try { - this._isUploadingDocs = true; + const isAudioOrVideo = VideoCast(newLinkedDoc.data)?.url?.pathname || AudioCast(newLinkedDoc.data)?.url?.pathname; + + // Set UI state to show the processing overlay + runInAction(() => { + this._isUploadingDocs = true; + this._uploadProgress = 0; + this._currentStep = isAudioOrVideo ? 'Preparing media file...' : 'Processing document...'; + }); // Process the document first to ensure it has a valid ID this.docManager.processDocument(newLinkedDoc); @@ -158,15 +172,36 @@ export class ChatBox extends ViewBoxAnnotatableComponent() { // Add the document to the vectorstore which will also register chunks await this.vectorstore.addAIDoc(newLinkedDoc, this.updateProgress); - // No longer needed as documents are tracked by the AgentDocumentManager - // this._linked_docs_to_add.add(newLinkedDoc); + // Give a slight delay to show the completion message + if (this._uploadProgress === 100) { + await new Promise(resolve => setTimeout(resolve, 1000)); + } - this._isUploadingDocs = false; + // Reset UI state + runInAction(() => { + this._isUploadingDocs = false; + this._uploadProgress = 0; + this._currentStep = ''; + }); return true; } catch (err) { console.error('Error adding document to vectorstore:', err); - this._isUploadingDocs = false; + + // Show error in UI + runInAction(() => { + this._currentStep = `Error: ${err instanceof Error ? err.message : 'Failed to process document'}`; + }); + + await new Promise(resolve => setTimeout(resolve, 2000)); + + // Reset UI state + runInAction(() => { + this._isUploadingDocs = false; + this._uploadProgress = 0; + this._currentStep = ''; + }); + return false; } }; @@ -178,8 +213,15 @@ export class ChatBox extends ViewBoxAnnotatableComponent() { */ @action updateProgress = (progress: number, step: string) => { - this._uploadProgress = progress; + // Ensure progress is within expected bounds + const validProgress = Math.min(Math.max(0, progress), 100); + this._uploadProgress = validProgress; this._currentStep = step; + + // Force UI update + if (process.env.NODE_ENV !== 'production') { + console.log(`Progress: ${validProgress}%, Step: ${step}`); + } }; /** @@ -453,7 +495,19 @@ export class ChatBox extends ViewBoxAnnotatableComponent() { case supportedDocTypes.image: return Docs.Create.ImageDocument(data as string, options); case supportedDocTypes.equation: return Docs.Create.EquationDocument(data as string, options); case supportedDocTypes.notetaking: return Docs.Create.NoteTakingDocument([], options); - case supportedDocTypes.web: return Docs.Create.WebDocument(data as string, { ...options, data_useCors: true }); + case supportedDocTypes.web: + // Create web document with enhanced safety options + const webOptions = { + ...options, + data_useCors: true + }; + + // If iframe_sandbox was passed from AgentDocumentManager, add it to the options + if ('_iframe_sandbox' in options) { + (webOptions as any)._iframe_sandbox = options._iframe_sandbox; + } + + return Docs.Create.WebDocument(data as string, webOptions); case supportedDocTypes.dataviz: return Docs.Create.DataVizDocument('/users/rz/Downloads/addresses.csv', options); case supportedDocTypes.pdf: return Docs.Create.PdfDocument(data as string, options); case supportedDocTypes.video: return Docs.Create.VideoDocument(data as string, options); @@ -607,65 +661,36 @@ export class ChatBox extends ViewBoxAnnotatableComponent() { return; } - // Process the chunk data - let docChunkSimpl: { chunks: SimplifiedChunk[] } = { chunks: [] }; - try { - docChunkSimpl = JSON.parse(StrCast(doc.chunk_simpl) || '{"chunks":[]}'); - } catch (e) { - console.error(`Error parsing chunk_simpl for the found document:`, e); + // Get the simplified chunk using the document manager + const foundChunk = this.docManager.getSimplifiedChunkById(doc, chunkId); + if (!foundChunk) { + console.warn(`Chunk not found in document for chunk ID: ${chunkId}`); + DocumentManager.Instance.showDocument(doc, { willZoomCentered: true }, () => {}); return; } - const foundChunk = docChunkSimpl.chunks.find((chunk: SimplifiedChunk) => chunk.chunkId === chunkId); + console.log(`Found chunk in document:`, foundChunk); // Handle different chunk types - if (foundChunk) { - console.log(`Found chunk in document:`, foundChunk); - if (foundChunk.chunkType === CHUNK_TYPE.AUDIO || foundChunk.chunkType === CHUNK_TYPE.VIDEO) { - const directMatchSegmentStart = this.getDirectMatchingSegmentStart(doc, citation.direct_text || '', foundChunk.indexes || []); - if (directMatchSegmentStart) { - await this.goToMediaTimestamp(doc, directMatchSegmentStart, foundChunk.chunkType); - } else { - console.error('No direct matching segment found for the citation.'); - } - } else if (foundChunk.chunkType === CHUNK_TYPE.TABLE || foundChunk.chunkType === CHUNK_TYPE.IMAGE) { - this.handleOtherChunkTypes(foundChunk, citation, doc); - } else if (foundChunk.chunkType === CHUNK_TYPE.TEXT) { - // Find text from the document's chunks metadata - let chunkText = ''; - - try { - // We already parsed the chunks earlier, so use that - const matchingChunk = docChunkSimpl.chunks.find(c => c.chunkId === foundChunk.chunkId); - if (matchingChunk && 'text' in matchingChunk) { - // If the text property exists on the chunk (even though it's not in the type) - chunkText = String(matchingChunk['text'] || ''); - } - } catch (e) { - console.error('Error getting chunk text:', e); - } - - // Default text if none found - if (!chunkText) { - chunkText = 'Text content not available'; - } - - this._citationPopup = { - text: chunkText, - visible: true, - }; - } - // Handle URL chunks - else if (foundChunk.chunkType === CHUNK_TYPE.URL) { - if (foundChunk.url) { - DocumentManager.Instance.showDocument(doc, { willZoomCentered: true }, () => {}); - console.log(`Navigated to web document with URL: ${foundChunk.url}`); - } else { - console.warn('URL chunk missing URL:', foundChunk); - } + if (foundChunk.chunkType === CHUNK_TYPE.AUDIO || foundChunk.chunkType === CHUNK_TYPE.VIDEO) { + const directMatchSegmentStart = this.getDirectMatchingSegmentStart(doc, citation.direct_text || '', foundChunk.indexes || []); + if (directMatchSegmentStart) { + await this.goToMediaTimestamp(doc, directMatchSegmentStart, foundChunk.chunkType); + } else { + console.error('No direct matching segment found for the citation.'); } + } else if (foundChunk.chunkType === CHUNK_TYPE.TABLE || foundChunk.chunkType === CHUNK_TYPE.IMAGE) { + this.handleOtherChunkTypes(foundChunk, citation, doc); } else { - console.warn('Navigating to doc. Unable to find chunk or segments for citation', citation); + // Show the chunk text in citation popup + let chunkText = foundChunk.text || 'Text content not available'; + + this._citationPopup = { + text: chunkText, + visible: true, + }; + + // Also navigate to the document DocumentManager.Instance.showDocument(doc, { willZoomCentered: true }, () => {}); } } catch (error) { @@ -683,8 +708,8 @@ export class ChatBox extends ViewBoxAnnotatableComponent() { getDirectMatchingSegmentStart = (doc: Doc, citationText: string, indexesOfSegments: string[]): number => { if (!doc || !citationText) return -1; - // Get original segments from the document - const original_segments = doc.original_segments ? JSON.parse(StrCast(doc.original_segments)) : []; + // Get original segments using document manager + const original_segments = this.docManager.getOriginalSegments(doc); if (!original_segments || !Array.isArray(original_segments) || original_segments.length === 0) { return -1; @@ -993,18 +1018,8 @@ export class ChatBox extends ViewBoxAnnotatableComponent() { */ @computed get summaries(): string { - const linkedDocs = Array.from(this.docManager.listDocs()) - .map(id => { - const doc = this.docManager.extractDocumentMetadata(id); - if (doc && doc.fields && (doc.fields.layout.summary || doc.fields.data.summary)) { - return doc.fields.layout.summary || doc.fields.data.summary; - } - return null; - }) - .filter(Boolean) - .join('\n\n'); - - return linkedDocs; + // Use the document manager to get all summaries + return this.docManager.getAllDocumentSummaries(); } /** @@ -1033,7 +1048,7 @@ export class ChatBox extends ViewBoxAnnotatableComponent() { // Other helper methods for retrieving document data and processing retrieveSummaries = (): string => { - return this.summaries; + return this.docManager.getAllDocumentSummaries(); }; retrieveCSVData = () => { @@ -1068,8 +1083,13 @@ export class ChatBox extends ViewBoxAnnotatableComponent() { {this._isUploadingDocs && (
- -
{this._currentStep}
+
+
+
+
+
{Math.round(this._uploadProgress)}%
+
{this._currentStep}
+
)} diff --git a/src/client/views/nodes/chatbot/chatboxcomponents/ProgressBar.scss b/src/client/views/nodes/chatbot/chatboxcomponents/ProgressBar.scss index ff5be4a38..3a8334695 100644 --- a/src/client/views/nodes/chatbot/chatboxcomponents/ProgressBar.scss +++ b/src/client/views/nodes/chatbot/chatboxcomponents/ProgressBar.scss @@ -58,12 +58,48 @@ flex-direction: column; align-items: center; text-align: center; + width: 80%; + max-width: 400px; + background-color: white; + padding: 20px; + border-radius: 8px; + box-shadow: 0 4px 12px rgba(0, 0, 0, 0.15); } -.step-name { +.progress-bar-wrapper { + width: 100%; + height: 12px; + background-color: #e0e0e0; + border-radius: 6px; + overflow: hidden; + margin-bottom: 10px; +} + +.progress-bar { + height: 100%; + background-color: #4a90e2; + border-radius: 6px; + transition: width 0.5s ease; +} + +.progress-details { + display: flex; + flex-direction: column; + align-items: center; + width: 100%; +} + +.progress-percentage { font-size: 18px; + font-weight: bold; color: #333; + margin-bottom: 5px; +} + +.step-name { + font-size: 16px; + color: #666; text-align: center; width: 100%; - margin-top: -10px; // Adjust to move the text closer to the spinner + margin-top: 5px; } diff --git a/src/client/views/nodes/chatbot/utils/AgentDocumentManager.ts b/src/client/views/nodes/chatbot/utils/AgentDocumentManager.ts index c3beebcde..cff8380db 100644 --- a/src/client/views/nodes/chatbot/utils/AgentDocumentManager.ts +++ b/src/client/views/nodes/chatbot/utils/AgentDocumentManager.ts @@ -14,6 +14,8 @@ import { parsedDoc } from '../chatboxcomponents/ChatBox'; import { faThumbTackSlash } from '@fortawesome/free-solid-svg-icons'; import { DocumentManager } from '../../../../util/DocumentManager'; import { DocumentView } from '../../DocumentView'; +import { RAGChunk, CHUNK_TYPE } from '../types/types'; +import { runInAction } from 'mobx'; /** * Interface representing a document in the freeform view @@ -869,20 +871,43 @@ export class AgentDocumentManager { _layout_autoHeight: true, }; - // Use the chatBox's createDocInDash method to create and link the document + // Additional handling for web documents + if (docType === 'web') { + // For web documents, don't sanitize the URL here + // Instead, set properties to handle content safely when loaded + simpleDoc._disable_resource_loading = true; + simpleDoc._sandbox_iframe = true; + simpleDoc.data_useCors = true; + + // Specify a more permissive sandbox to allow content to render properly + // but still maintain security + simpleDoc._iframe_sandbox = 'allow-same-origin allow-scripts allow-popups allow-forms'; + } + + // Use the chatBox's createDocInDash method to create the document if (!this.chatBox) { throw new Error('ChatBox instance not available for creating document'); } - const linkAndShowDoc = (doc: Opt) => { - if (doc) { - LinkManager.Instance.addLink(Docs.Create.LinkDocument(this.chatBoxDocument!, doc)); - this.chatBox._props.addDocument?.(doc); - DocumentManager.Instance.showDocument(doc, { willZoomCentered: true }, () => {}); - } - }; + const doc = this.chatBox.whichDoc(simpleDoc, false); if (doc) { - linkAndShowDoc(doc); + // Use MobX runInAction to properly modify observable state + runInAction(() => { + if (this.chatBoxDocument && doc) { + // Create link and add it to the document system + const linkDoc = Docs.Create.LinkDocument(this.chatBoxDocument, doc); + LinkManager.Instance.addLink(linkDoc); + + // Add document to view + this.chatBox._props.addDocument?.(doc); + + // Show document - defer actual display to prevent immediate resource loading + setTimeout(() => { + DocumentManager.Instance.showDocument(doc, { willZoomCentered: true }, () => {}); + }, 100); + } + }); + const id = this.processDocument(doc); return id; } else { @@ -893,6 +918,62 @@ export class AgentDocumentManager { } } + /** + * Sanitizes web content to prevent errors with external resources + * @param content The web content to sanitize + * @returns Sanitized content + */ + private sanitizeWebContent(content: string): string { + if (!content) return content; + + try { + // Replace problematic resource references that might cause errors + const sanitized = content + // Remove preload links that might cause errors + .replace(/]*rel=["']preload["'][^>]*>/gi, '') + // Remove map file references + .replace(/\/\/# sourceMappingURL=.*\.map/gi, '') + // Remove external CSS map files references + .replace(/\/\*# sourceMappingURL=.*\.css\.map.*\*\//gi, '') + // Add sandbox to iframes + .replace(/