Merge branch 'agent-paper-main' into lanyi-expanded-agent-paper-main

1 files changed, 79 insertions, 75 deletions

diff --git a/src/client/views/nodes/chatbot/chatboxcomponents/ChatBox.tsx b/src/client/views/nodes/chatbot/chatboxcomponents/ChatBox.tsx
index 0294dd5a7..732c4d637 100644
--- a/src/client/views/nodes/chatbot/chatboxcomponents/ChatBox.tsx
+++ b/src/client/views/nodes/chatbot/chatboxcomponents/ChatBox.tsx
@@ -7,46 +7,50 @@
  * with support for follow-up questions and citation management.
  */
 
-import { Button, Size, Type } from '@dash/components';
+import dotenv from 'dotenv';
 import { ObservableSet, action, computed, makeObservable, observable, observe, reaction, runInAction } from 'mobx';
 import { observer } from 'mobx-react';
 import OpenAI, { ClientOptions } from 'openai';
 import * as React from 'react';
-import { AiOutlineSend } from 'react-icons/ai';
 import { v4 as uuidv4 } from 'uuid';
 import { ClientUtils, OmitKeys } from '../../../../../ClientUtils';
 import { Doc, DocListCast, Opt } from '../../../../../fields/Doc';
-import { DocData, DocViews } from '../../../../../fields/DocSymbols';
+import { DocData, DocLayout, DocViews } from '../../../../../fields/DocSymbols';
 import { Id } from '../../../../../fields/FieldSymbols';
 import { RichTextField } from '../../../../../fields/RichTextField';
 import { ScriptField } from '../../../../../fields/ScriptField';
-import { AudioCast, CsvCast, DocCast, NumCast, PDFCast, RTFCast, StrCast, VideoCast } from '../../../../../fields/Types';
-import { Upload } from '../../../../../server/SharedMediaTypes';
-import { DocServer } from '../../../../DocServer';
+import { CsvCast, DocCast, NumCast, PDFCast, RTFCast, StrCast, VideoCast, AudioCast } from '../../../../../fields/Types';
 import { DocUtils } from '../../../../documents/DocUtils';
 import { CollectionViewType, DocumentType } from '../../../../documents/DocumentTypes';
 import { Docs, DocumentOptions } from '../../../../documents/Documents';
+import { DocServer } from '../../../../DocServer';
 import { DocumentManager } from '../../../../util/DocumentManager';
 import { ImageUtils } from '../../../../util/Import & Export/ImageUtils';
 import { LinkManager } from '../../../../util/LinkManager';
 import { CompileError, CompileScript } from '../../../../util/Scripting';
-import { SnappingManager } from '../../../../util/SnappingManager';
 import { DictationButton } from '../../../DictationButton';
 import { ViewBoxAnnotatableComponent } from '../../../DocComponent';
 import { AudioBox } from '../../AudioBox';
 import { DocumentView, DocumentViewInternal } from '../../DocumentView';
 import { FieldView, FieldViewProps } from '../../FieldView';
-import { OpenWhere } from '../../OpenWhere';
 import { PDFBox } from '../../PDFBox';
 import { ScriptingBox } from '../../ScriptingBox';
 import { VideoBox } from '../../VideoBox';
 import { Agent } from '../agentsystem/Agent';
 import { supportedDocTypes } from '../types/tool_types';
 import { ASSISTANT_ROLE, AssistantMessage, CHUNK_TYPE, Citation, ProcessingInfo, SimplifiedChunk, TEXT_TYPE } from '../types/types';
-import { AgentDocumentManager } from '../utils/AgentDocumentManager';
 import { Vectorstore } from '../vectorstore/Vectorstore';
 import './ChatBox.scss';
 import MessageComponentBox from './MessageComponent';
+import { OpenWhere } from '../../OpenWhere';
+import { Upload } from '../../../../../server/SharedMediaTypes';
+import { DocumentMetadataTool } from '../tools/DocumentMetadataTool';
+import { AgentDocumentManager } from '../utils/AgentDocumentManager';
+import { AiOutlineSend } from 'react-icons/ai';
+import { SnappingManager } from '../../../../util/SnappingManager';
+import { Button, Size, Type } from '@dash/components';
+
+dotenv.config();
 
 export type parsedDocData = {
     doc_type: string;
@@ -57,7 +61,6 @@ export type parsedDocData = {
     data_useCors?: boolean;
 };
 export type parsedDoc = DocumentOptions & parsedDocData;
-
 /**
  * ChatBox is the main class responsible for managing the interaction between the user and the assistant,
  * handling documents, and integrating with OpenAI for tasks such as document analysis, chat functionality,
@@ -130,15 +133,7 @@ export class ChatBox extends ViewBoxAnnotatableComponent<FieldViewProps>() {
         this.vectorstore = new Vectorstore(this.vectorstore_id, this.docManager);
 
         // Create an agent with the vectorstore
-        this.agent = new Agent(
-            this.vectorstore,
-            this.retrieveFormattedHistory.bind(this),
-            this.retrieveCSVData.bind(this),
-            this.createImageInDash.bind(this),
-            this.createCSVInDash.bind(this),
-            this.docManager,
-            this.dataDoc.is_dash_doc_assistant === 'true'
-        );
+        this.agent = new Agent(this.vectorstore, this.retrieveFormattedHistory.bind(this), this.retrieveCSVData.bind(this), this.createImageInDash.bind(this), this.createCSVInDash.bind(this), this.docManager);
 
         // Set up the tool created callback
         this.agent.setToolCreatedCallback(this.handleToolCreated);
@@ -411,6 +406,7 @@ export class ChatBox extends ViewBoxAnnotatableComponent<FieldViewProps>() {
                             };
                         }
                     });
+                    this.scrollToBottom();
                 };
 
                 const onAnswerUpdate = (answerUpdate: string) => {
@@ -418,33 +414,41 @@ export class ChatBox extends ViewBoxAnnotatableComponent<FieldViewProps>() {
                         if (this._current_message) {
                             this._current_message = {
                                 ...this._current_message,
-                                content: [{ index: 0, type: TEXT_TYPE.NORMAL, text: answerUpdate, citation_ids: null }],
+                                content: [{ text: answerUpdate, type: TEXT_TYPE.NORMAL, index: 0, citation_ids: [] }],
                             };
                         }
                     });
                 };
 
-                // Get the response from the agent
-                let userQuery = trimmedText;
-                if (this.dataDoc.is_dash_doc_assistant) {
-                    userQuery = `The user is asking a question about Dash functionality. Their question is: "${trimmedText}". You should use the generateTutorialNode tool to answer this question.`;
-                }
-                const response = await this.agent.askAgent(userQuery, onProcessingUpdate, onAnswerUpdate);
+                // Send the user's question to the assistant and get the final message
+                const finalMessage = await this.agent.askAgent(trimmedText, onProcessingUpdate, onAnswerUpdate);
 
-                // Push the final message to history
+                // Update the history with the final assistant message
                 runInAction(() => {
-                    this._history.push(response);
-                    this._isLoading = false;
-                    this._current_message = undefined;
+                    if (this._current_message) {
+                        this._history.push({ ...finalMessage });
+                        this._current_message = undefined;
+                        this.dataDoc.data = JSON.stringify(this._history);
+                    }
                 });
-            } catch (error) {
-                console.error('Error in askGPT:', error);
+            } catch (err) {
+                console.error('Error:', err);
+                // Handle error in processing
+                runInAction(() =>
+                    this._history.push({
+                        role: ASSISTANT_ROLE.ASSISTANT,
+                        content: [{ index: 0, type: TEXT_TYPE.ERROR, text: `Sorry, I encountered an error while processing your request: ${err} `, citation_ids: null }],
+                        processing_info: [],
+                    })
+                );
+            } finally {
                 runInAction(() => {
                     this._isLoading = false;
-                    this._current_message = undefined;
                 });
+                this.scrollToBottom();
             }
         }
+        this.scrollToBottom();
     };
 
     /**
@@ -517,7 +521,7 @@ export class ChatBox extends ViewBoxAnnotatableComponent<FieldViewProps>() {
         const data = (doc as parsedDocData).data;
         const ndoc = (() => {
             switch (doc.doc_type) {
-                default:
+                default:           
                 case supportedDocTypes.note:       return Docs.Create.TextDocument(data as string, options);
                 case supportedDocTypes.comparison: return this.createComparison(JSON.parse(data as string) as parsedDoc[], options);
                 case supportedDocTypes.flashcard:  return this.createFlashcard(JSON.parse(data as string) as  parsedDoc[], options);
@@ -525,26 +529,25 @@ export class ChatBox extends ViewBoxAnnotatableComponent<FieldViewProps>() {
                 case supportedDocTypes.image:      return Docs.Create.ImageDocument(data as string, options);
                 case supportedDocTypes.equation:   return Docs.Create.EquationDocument(data as string, options);
                 case supportedDocTypes.notetaking: return Docs.Create.NoteTakingDocument([], options);
-                case supportedDocTypes.web:       {
+                case supportedDocTypes.web:        
                     // Create web document with enhanced safety options
                     const webOptions = { 
-                        ...options,
+                        ...options, 
                         data_useCors: true
                     };
-
+                    
                     // If iframe_sandbox was passed from AgentDocumentManager, add it to the options
                     if ('_iframe_sandbox' in options) {
-                        webOptions._iframe_sandbox = options._iframe_sandbox;
+                        (webOptions as any)._iframe_sandbox = options._iframe_sandbox;
                     }
-
+                    
                     return Docs.Create.WebDocument(data as string, webOptions);
-                }
-                case supportedDocTypes.dataviz:    return Docs.Create.DataVizDocument('/users/rz/Downloads/addresses.csv', options);
+                case supportedDocTypes.dataviz: case supportedDocTypes.table: return Docs.Create.DataVizDocument('/Users/ajshul/Dash-Web/src/server/public/files/csv/0d237e7c-98c9-44d0-aa61-5285fdbcf96c-random_sample.csv.csv', options);
                 case supportedDocTypes.pdf:        return Docs.Create.PdfDocument(data as string, options);
                 case supportedDocTypes.video:      return Docs.Create.VideoDocument(data as string, options);
                 case supportedDocTypes.diagram:    return Docs.Create.DiagramDocument(undefined, { text: data as unknown as RichTextField, ...options}); // text: can take a string or RichTextField but it's typed for RichTextField.
-
-                // case supportedDocumentTypes.dataviz:
+                
+                // case supportedDocumentTypes.dataviz: 
                 // {
                 //     const { fileUrl, id } = await Networking.PostToServer('/createCSV', {
                 //         filename: (options.title as string).replace(/\s+/g, '') + '.csv',
@@ -569,7 +572,7 @@ export class ChatBox extends ViewBoxAnnotatableComponent<FieldViewProps>() {
                     const arr = this.createCollectionWithChildren(JSON.parse(data as string) as parsedDoc[], true).filter(d=>d).map(d => d!);
                     const collOpts = {  _width:300, _height: 300, _layout_fitWidth: true,  _freeform_backgroundGrid: true, ...options,  };
                     return (() => {
-                        switch (options.type_collection) {
+                            switch (options.type_collection) {
                                 case CollectionViewType.Tree:        return Docs.Create.TreeDocument(arr, collOpts);
                                 case CollectionViewType.Stacking:    return Docs.Create.StackingDocument(arr, collOpts);
                                 case CollectionViewType.Masonry:     return Docs.Create.MasonryDocument(arr, collOpts);
@@ -578,9 +581,9 @@ export class ChatBox extends ViewBoxAnnotatableComponent<FieldViewProps>() {
                                 case CollectionViewType.Carousel3D:  return Docs.Create.Carousel3DDocument(arr, collOpts);
                                 case CollectionViewType.Multicolumn: return Docs.Create.CarouselDocument(arr, collOpts);
                                 default:                             return Docs.Create.FreeformDocument(arr, collOpts);
-                        }
-                    })();
-                }
+                            }
+                        })();
+                    }
                 // case supportedDocumentTypes.map:        return Docs.Create.MapDocument([], options);
                 // case supportedDocumentTypes.button:     return Docs.Create.ButtonDocument(options);
                 // case supportedDocumentTypes.trail:      return Docs.Create.PresDocument(options);
@@ -588,8 +591,8 @@ export class ChatBox extends ViewBoxAnnotatableComponent<FieldViewProps>() {
         })();
 
         if (ndoc) {
-            ndoc.x = ((options.x as number) ?? 0) + (insideCol ? 0 : NumCast(this.layoutDoc.x) + NumCast(this.layoutDoc.width)) + 100;
-            ndoc.y = ((options.y as number) ?? 0) + (insideCol ? 0 : NumCast(this.layoutDoc.y));
+            ndoc.x = NumCast((options.x as number) ?? 0) + (insideCol ? 0 : NumCast(this.layoutDoc.x) + NumCast(this.layoutDoc.width)) + 100;
+            ndoc.y = NumCast(options.y as number) + (insideCol ? 0 : NumCast(this.layoutDoc.y));
         }
         return ndoc;
     };
@@ -669,7 +672,7 @@ export class ChatBox extends ViewBoxAnnotatableComponent<FieldViewProps>() {
     handleCitationClick = async (citation: Citation) => {
         try {
             // Extract values from MobX proxy object if needed
-            const chunkId = typeof citation.chunk_id === 'object' ? (citation.chunk_id as unknown as object).toString() : citation.chunk_id;
+            const chunkId = typeof citation.chunk_id === 'object' ? (citation.chunk_id as any).toString() : citation.chunk_id;
 
             // For debugging
             console.log('Citation clicked:', {
@@ -711,7 +714,7 @@ export class ChatBox extends ViewBoxAnnotatableComponent<FieldViewProps>() {
                 }
                 this.handleOtherChunkTypes(foundChunk, citation, doc, dataDoc);
                 // Show the chunk text in citation popup
-                const chunkText = citation.direct_text || 'Text content not available';
+                let chunkText = citation.direct_text || 'Text content not available';
                 this.showCitationPopup(chunkText);
 
                 // Also navigate to the document
@@ -743,7 +746,7 @@ export class ChatBox extends ViewBoxAnnotatableComponent<FieldViewProps>() {
 
         // If specific indexes are provided, filter segments by those indexes
         if (indexesOfSegments && indexesOfSegments.length > 0) {
-            segments = original_segments.filter(segment => indexesOfSegments.includes(segment.index));
+            segments = original_segments.filter((segment: any) => indexesOfSegments.includes(segment.index));
         }
 
         // If no segments match the indexes, use all segments
@@ -752,7 +755,7 @@ export class ChatBox extends ViewBoxAnnotatableComponent<FieldViewProps>() {
         }
 
         // First try to find an exact match
-        const exactMatch = segments.find(segment => segment.text && segment.text.includes(citationText));
+        const exactMatch = segments.find((segment: any) => segment.text && segment.text.includes(citationText));
 
         if (exactMatch) {
             return exactMatch.start;
@@ -861,26 +864,22 @@ export class ChatBox extends ViewBoxAnnotatableComponent<FieldViewProps>() {
                         existingDoc._width = x2 - x1;
                         existingDoc._height = y2 - y1;
                     }
-                    // const highlightDoc =
-                    existingDoc ?? this.createImageCitationHighlight(x1, y1, x2, y2, citation, annotationKey, doc);
+                    const highlightDoc = existingDoc ?? this.createImageCitationHighlight(x1, y1, x2, y2, citation, annotationKey, doc);
 
                     //doc.layout_scroll = y1;
                     doc._layout_curPage = foundChunk.startPage + 1;
-                    DocumentManager.Instance.showDocument(doc, { willZoomCentered: true }, () => {});
-                    //DocumentManager.Instance.showDocument(highlightDoc, { willZoomCentered: true }, () => {});
+                    DocumentManager.Instance.showDocument(highlightDoc, { willZoomCentered: true }, () => {});
                 }
                 break;
             case CHUNK_TYPE.TEXT:
-                {
-                    this._citationPopup = { text: citation.direct_text ?? 'No text available', visible: true };
-                    this.startCitationPopupTimer();
+                this._citationPopup = { text: citation.direct_text ?? 'No text available', visible: true };
+                this.startCitationPopupTimer();
 
-                    // Check if the document is a PDF (has a PDF viewer component)
-                    const isPDF = PDFCast(dataDoc!.data) !== null || dataDoc!.type === DocumentType.PDF;
+                // Check if the document is a PDF (has a PDF viewer component)
+                const isPDF = PDFCast(dataDoc!.data) !== null || dataDoc!.type === DocumentType.PDF;
 
-                    // First ensure document is fully visible before trying to access its views
-                    this.ensureDocumentIsVisible(dataDoc!, isPDF, citation, foundChunk, doc);
-                }
+                // First ensure document is fully visible before trying to access its views
+                this.ensureDocumentIsVisible(dataDoc!, isPDF, citation, foundChunk, doc);
                 break;
             case CHUNK_TYPE.CSV:
             case CHUNK_TYPE.URL:
@@ -1098,9 +1097,7 @@ export class ChatBox extends ViewBoxAnnotatableComponent<FieldViewProps>() {
                         {
                             index: 0,
                             type: TEXT_TYPE.NORMAL,
-                            text: this.dataDoc.is_dash_doc_assistant
-                                ? 'Welcome to your help assistant for Dash. Ask any Dash-related questions to get started.'
-                                : `Hey, ${this.userName()}! Welcome to Your Friendly Assistant. Link a document or ask questions to get started.`,
+                            text: `Hey, ${this.userName()}! Welcome to Your Friendly Assistant. Link a document or ask questions to get started.`,
                             citation_ids: null,
                         },
                     ],
@@ -1197,9 +1194,6 @@ export class ChatBox extends ViewBoxAnnotatableComponent<FieldViewProps>() {
         this._inputValue = question;
     };
 
-    _dictation: DictationButton | null = null;
-    setInputRef = (r: HTMLInputElement) => (this._textInputRef = r);
-    setDictationRef = (r: DictationButton) => (this._dictation = r);
     /**
      * Handles tool creation notification and shows the reload modal
      * @param toolName The name of the tool that was created
@@ -1250,6 +1244,8 @@ export class ChatBox extends ViewBoxAnnotatableComponent<FieldViewProps>() {
         }, 100);
     };
 
+    _dictation: DictationButton | null = null;
+
     /**
      * Toggles the font size modal visibility
      */
@@ -1441,7 +1437,7 @@ export class ChatBox extends ViewBoxAnnotatableComponent<FieldViewProps>() {
                     </div>
                 )}
                 <div className="chat-header">
-                    <h2>{StrCast(this.dataDoc.title) || `${this.userName()}'s AI Assistant`}</h2>
+                    <h2>{this.userName()}&apos;s AI Assistant</h2>
                     <div className="font-size-control" onClick={this.toggleFontSizeModal}>
                         {this.renderFontSizeIcon()}
                     </div>
@@ -1477,8 +1473,10 @@ export class ChatBox extends ViewBoxAnnotatableComponent<FieldViewProps>() {
 
                 <form onSubmit={this.askGPT} className="chat-input">
                     <div className="input-container">
-                        <input //
-                            ref={this.setInputRef}
+                        <input
+                            ref={r => {
+                                this._textInputRef = r;
+                            }}
                             type="text"
                             name="messageInput"
                             autoComplete="off"
@@ -1498,7 +1496,13 @@ export class ChatBox extends ViewBoxAnnotatableComponent<FieldViewProps>() {
                         icon={<AiOutlineSend />}
                         size={Size.LARGE}
                     />
-                    <DictationButton ref={this.setDictationRef} setInput={this.setChatInput} inputRef={this._textInputRef} />
+                    <DictationButton
+                        ref={r => {
+                            this._dictation = r;
+                        }}
+                        setInput={this.setChatInput}
+                        inputRef={this._textInputRef}
+                    />
                 </form>
                 {/* Popup for citation */}
                 {this._citationPopup.visible && (