changes (seeing if they work)

1 files changed, 643 insertions, 0 deletions

diff --git a/full-diff.txt b/full-diff.txt
new file mode 100644
index 000000000..e19fbe1ec
--- /dev/null
+++ b/full-diff.txt
@@ -0,0 +1,643 @@
+diff --git a/.gitignore b/.gitignore
+index 7353bc7e0..319a5fa30 100644
+--- a/.gitignore
++++ b/.gitignore
+@@ -27,3 +27,12 @@ packages/*/dist
+ src/server/ApiManagers/temp_data.txt
+ /src/server/flashcard/venv
+ /src/server/flashcard/venv
++
++summarize_dash_data.py
++tree_to_json.py
++test_dynamic_tools.py
++ts_files_with_content.txt
++ts_files_with_content.txt
++ts_files_with_summaries.txt
++ts_files_with_summaries copy.txt
++summarize_dash_ts.py
+\ No newline at end of file
+diff --git a/src/client/views/nodes/chatbot/agentsystem/Agent.ts b/src/client/views/nodes/chatbot/agentsystem/Agent.ts
+index 1a9df1a75..c3d37fd0e 100644
+--- a/src/client/views/nodes/chatbot/agentsystem/Agent.ts
++++ b/src/client/views/nodes/chatbot/agentsystem/Agent.ts
+@@ -29,6 +29,7 @@ import { CreateLinksTool } from '../tools/CreateLinksTool';
+ import { CodebaseSummarySearchTool } from '../tools/CodebaseSummarySearchTool';
+ import { FileContentTool } from '../tools/FileContentTool';
+ import { FileNamesTool } from '../tools/FileNamesTool';
++import { CreateNewTool } from '../tools/CreateNewTool';
+ //import { CreateTextDocTool } from '../tools/CreateTextDocumentTool';
+ 
+ dotenv.config();
+@@ -52,6 +53,12 @@ export class Agent {
+     private streamedAnswerParser: StreamedAnswerParser = new StreamedAnswerParser();
+     private tools: Record<string, BaseTool<ReadonlyArray<Parameter>>>;
+     private _docManager: AgentDocumentManager;
++    // Dynamic tool registry for tools created at runtime
++    private dynamicToolRegistry: Map<string, BaseTool<ReadonlyArray<Parameter>>> = new Map();
++    // Callback for notifying when tools are created and need reload
++    private onToolCreatedCallback?: (toolName: string) => void;
++    // Storage for deferred tool saving
++    private pendingToolSave?: { toolName: string; completeToolCode: string };
+ 
+     /**
+      * The constructor initializes the agent with the vector store and toolset, and sets up the OpenAI client.
+@@ -79,6 +86,9 @@ export class Agent {
+         this._csvData = csvData;
+         this._docManager = docManager;
+ 
++        // Initialize dynamic tool registry
++        this.dynamicToolRegistry = new Map();
++
+         // Define available tools for the assistant
+         this.tools = {
+             calculate: new CalculateTool(),
+@@ -94,6 +104,146 @@ export class Agent {
+             fileContent: new FileContentTool(this.vectorstore),
+             fileNames: new FileNamesTool(this.vectorstore),
+         };
++
++        // Add the createNewTool after other tools are defined
++        this.tools.createNewTool = new CreateNewTool(this.dynamicToolRegistry, this.tools, this);
++
++        // Load existing dynamic tools
++        this.loadExistingDynamicTools();
++    }
++
++    /**
++     * Loads existing dynamic tools by checking the current registry and ensuring all stored tools are available
++     */
++    private async loadExistingDynamicTools(): Promise<void> {
++        try {
++            console.log('Loading dynamic tools...');
++
++            // Since we're in a browser environment, we can't use filesystem operations
++            // Instead, we'll maintain tools in the registry and try to load known tools
++
++            // Try to manually load the known dynamic tools that exist
++            const knownDynamicTools = [
++                { name: 'CharacterCountTool', actionName: 'charactercount' },
++                { name: 'WordCountTool', actionName: 'wordcount' },
++                { name: 'TestTool', actionName: 'test' },
++            ];
++
++            let loadedCount = 0;
++            for (const toolInfo of knownDynamicTools) {
++                try {
++                    // Check if tool is already in registry
++                    if (this.dynamicToolRegistry.has(toolInfo.actionName)) {
++                        console.log(`✓ Tool ${toolInfo.actionName} already loaded`);
++                        loadedCount++;
++                        continue;
++                    }
++
++                    // Try to load the tool using require (works better in webpack environment)
++                    let toolInstance = null;
++                    try {
++                        // Use require with the relative path
++                        const toolModule = require(`../tools/dynamic/${toolInfo.name}`);
++                        const ToolClass = toolModule[toolInfo.name];
++
++                        if (ToolClass && typeof ToolClass === 'function') {
++                            toolInstance = new ToolClass();
++
++                            if (toolInstance instanceof BaseTool) {
++                                this.dynamicToolRegistry.set(toolInfo.actionName, toolInstance);
++                                loadedCount++;
++                                console.log(`✓ Loaded dynamic tool: ${toolInfo.actionName} (from ${toolInfo.name})`);
++                            }
++                        }
++                    } catch (requireError) {
++                        // Tool file doesn't exist or can't be loaded, which is fine
++                        console.log(`Tool ${toolInfo.name} not available:`, (requireError as Error).message);
++                    }
++                } catch (error) {
++                    console.warn(`⚠ Failed to load ${toolInfo.name}:`, error);
++                }
++            }
++
++            console.log(`Successfully loaded ${loadedCount} dynamic tools`);
++
++            // Log all currently registered dynamic tools
++            if (this.dynamicToolRegistry.size > 0) {
++                console.log('Currently registered dynamic tools:', Array.from(this.dynamicToolRegistry.keys()));
++            }
++        } catch (error) {
++            console.error('Error loading dynamic tools:', error);
++        }
++    }
++
++    /**
++     * Manually registers a dynamic tool instance (called by CreateNewTool)
++     */
++    public registerDynamicTool(toolName: string, toolInstance: BaseTool<ReadonlyArray<Parameter>>): void {
++        this.dynamicToolRegistry.set(toolName, toolInstance);
++        console.log(`Manually registered dynamic tool: ${toolName}`);
++    }
++
++    /**
++     * Notifies that a tool has been created and saved to disk (called by CreateNewTool)
++     */
++    public notifyToolCreated(toolName: string, completeToolCode: string): void {
++        // Store the tool data for deferred saving
++        this.pendingToolSave = { toolName, completeToolCode };
++
++        if (this.onToolCreatedCallback) {
++            this.onToolCreatedCallback(toolName);
++        }
++    }
++
++    /**
++     * Performs the deferred tool save operation (called after user confirmation)
++     */
++    public async performDeferredToolSave(): Promise<boolean> {
++        if (!this.pendingToolSave) {
++            console.warn('No pending tool save operation');
++            return false;
++        }
++
++        const { toolName, completeToolCode } = this.pendingToolSave;
++
++        try {
++            // Get the CreateNewTool instance to perform the save
++            const createNewTool = this.tools.createNewTool as any;
++            if (createNewTool && typeof createNewTool.saveToolToServerDeferred === 'function') {
++                const success = await createNewTool.saveToolToServerDeferred(toolName, completeToolCode);
++
++                if (success) {
++                    console.log(`Tool ${toolName} saved to server successfully via deferred save.`);
++                    // Clear the pending save
++                    this.pendingToolSave = undefined;
++                    return true;
++                } else {
++                    console.warn(`Tool ${toolName} could not be saved to server via deferred save.`);
++                    return false;
++                }
++            } else {
++                console.error('CreateNewTool instance not available for deferred save');
++                return false;
++            }
++        } catch (error) {
++            console.error(`Error performing deferred tool save for ${toolName}:`, error);
++            return false;
++        }
++    }
++
++    /**
++     * Sets the callback for when tools are created
++     */
++    public setToolCreatedCallback(callback: (toolName: string) => void): void {
++        this.onToolCreatedCallback = callback;
++    }
++
++    /**
++     * Public method to reload dynamic tools (called when new tools are created)
++     */
++    public reloadDynamicTools(): void {
++        console.log('Reloading dynamic tools...');
++        this.loadExistingDynamicTools();
+     }
+ 
+     /**
+@@ -126,11 +276,8 @@ export class Agent {
+         // Push sanitized user's question to message history
+         this.messages.push({ role: 'user', content: sanitizedQuestion });
+ 
+-        // Retrieve chat history and generate system prompt
+-        const chatHistory = this._history();
+-        // Get document summaries directly from document manager
+-        // Generate the system prompt with the summaries
+-        const systemPrompt = getReactPrompt(Object.values(this.tools), () => JSON.stringify(this._docManager.listDocs), chatHistory);
++        // Get system prompt with all tools (static + dynamic)
++        const systemPrompt = this.getSystemPromptWithAllTools();
+ 
+         // Initialize intermediate messages
+         this.interMessages = [{ role: 'system', content: systemPrompt }];
+@@ -193,22 +340,25 @@ export class Agent {
+                     currentAction = stage[key] as string;
+                     console.log(`Action: ${currentAction}`);
+ 
+-                    if (this.tools[currentAction]) {
++                    // Check both static tools and dynamic registry
++                    const tool = this.tools[currentAction] || this.dynamicToolRegistry.get(currentAction);
++
++                    if (tool) {
+                         // Prepare the next action based on the current tool
+                         const nextPrompt = [
+                             {
+                                 type: 'text',
+-                                text: `<stage number="${i + 1}" role="user">` + builder.build({ action_rules: this.tools[currentAction].getActionRule() }) + `</stage>`,
++                                text: `<stage number="${i + 1}" role="user">` + builder.build({ action_rules: tool.getActionRule() }) + `</stage>`,
+                             } as Observation,
+                         ];
+                         this.interMessages.push({ role: 'user', content: nextPrompt });
+                         break;
+                     } else {
+                         // Handle error in case of an invalid action
+-                        console.log('Error: No valid action');
++                        console.log(`Error: Action "${currentAction}" is not a valid tool`);
+                         this.interMessages.push({
+                             role: 'user',
+-                            content: `<stage number="${i + 1}" role="system-error-reporter">No valid action, try again.</stage>`,
++                            content: `<stage number="${i + 1}" role="system-error-reporter">Action "${currentAction}" is not a valid tool, try again.</stage>`,
+                         });
+                         break;
+                     }
+@@ -376,8 +526,8 @@ export class Agent {
+                     throw new Error('Action must be a non-empty string');
+                 }
+ 
+-                // Optional: Check if the action is among allowed actions
+-                const allowedActions = Object.keys(this.tools);
++                // Optional: Check if the action is among allowed actions (including dynamic tools)
++                const allowedActions = [...Object.keys(this.tools), ...Array.from(this.dynamicToolRegistry.keys())];
+                 if (!allowedActions.includes(stage.action)) {
+                     throw new Error(`Action "${stage.action}" is not a valid tool`);
+                 }
+@@ -482,12 +632,15 @@ export class Agent {
+      * @throws An error if the action is unknown, if required parameters are missing, or if input types don't match the expected parameter types.
+      */
+     private async processAction(action: string, actionInput: ParametersType<ReadonlyArray<Parameter>>): Promise<Observation[]> {
+-        // Check if the action exists in the tools list
+-        if (!(action in this.tools)) {
++        // Check if the action exists in the tools list or dynamic registry
++        if (!(action in this.tools) && !this.dynamicToolRegistry.has(action)) {
+             throw new Error(`Unknown action: ${action}`);
+         }
+         console.log(actionInput);
+ 
++        // Determine which tool to use - either from static tools or dynamic registry
++        const tool = this.tools[action] || this.dynamicToolRegistry.get(action);
++
+         // Special handling for documentMetadata tool with numeric or boolean fieldValue
+         if (action === 'documentMetadata') {
+             // Handle single field edit
+@@ -520,9 +673,49 @@ export class Agent {
+             }
+         }
+ 
+-        for (const param of this.tools[action].parameterRules) {
++        // Special handling for createNewTool with parsed XML toolCode
++        if (action === 'createNewTool') {
++            if ('toolCode' in actionInput && typeof actionInput.toolCode === 'object' && actionInput.toolCode !== null) {
++                try {
++                    // Convert the parsed XML object back to a string
++                    const extractText = (obj: any): string => {
++                        if (typeof obj === 'string') {
++                            return obj;
++                        } else if (obj && typeof obj === 'object') {
++                            if (obj._text) {
++                                return obj._text;
++                            }
++                            // Recursively extract text from all properties
++                            let text = '';
++                            for (const key in obj) {
++                                if (key !== '_text') {
++                                    const value = obj[key];
++                                    if (typeof value === 'string') {
++                                        text += value + '\n';
++                                    } else if (value && typeof value === 'object') {
++                                        text += extractText(value) + '\n';
++                                    }
++                                }
++                            }
++                            return text;
++                        }
++                        return '';
++                    };
++
++                    const reconstructedCode = extractText(actionInput.toolCode);
++                    actionInput.toolCode = reconstructedCode;
++                } catch (error) {
++                    console.error('Error processing toolCode:', error);
++                    // Convert to string as fallback
++                    actionInput.toolCode = String(actionInput.toolCode);
++                }
++            }
++        }
++
++        // Check parameter requirements and types for the tool
++        for (const param of tool.parameterRules) {
+             // Check if the parameter is required and missing in the input
+-            if (param.required && !(param.name in actionInput) && !this.tools[action].inputValidator(actionInput)) {
++            if (param.required && !(param.name in actionInput) && !tool.inputValidator(actionInput)) {
+                 throw new Error(`Missing required parameter: ${param.name}`);
+             }
+ 
+@@ -540,11 +733,30 @@ export class Agent {
+             }
+         }
+ 
+-        const tool = this.tools[action];
+-
++        // Execute the tool with the validated inputs
+         return await tool.execute(actionInput);
+     }
+ 
++    /**
++     * Gets a combined list of all tools, both static and dynamic
++     * @returns An array of all available tool instances
++     */
++    private getAllTools(): BaseTool<ReadonlyArray<Parameter>>[] {
++        // Combine static and dynamic tools
++        return [...Object.values(this.tools), ...Array.from(this.dynamicToolRegistry.values())];
++    }
++
++    /**
++     * Overridden method to get the React prompt with all tools (static + dynamic)
++     */
++    private getSystemPromptWithAllTools(): string {
++        const allTools = this.getAllTools();
++        const docSummaries = () => JSON.stringify(this._docManager.listDocs);
++        const chatHistory = this._history();
++
++        return getReactPrompt(allTools, docSummaries, chatHistory);
++    }
++
+     /**
+      * Reinitializes the DocumentMetadataTool with a direct reference to the ChatBox instance.
+      * This ensures that the tool can properly access the ChatBox document and find related documents.
+@@ -560,3 +772,9 @@ export class Agent {
+         }
+     }
+ }
++
++// Forward declaration to avoid circular import
++interface AgentLike {
++    registerDynamicTool(toolName: string, toolInstance: BaseTool<ReadonlyArray<Parameter>>): void;
++    notifyToolCreated(toolName: string, completeToolCode: string): void;
++}
+diff --git a/src/client/views/nodes/chatbot/chatboxcomponents/ChatBox.scss b/src/client/views/nodes/chatbot/chatboxcomponents/ChatBox.scss
+index 31f7be4c4..8e00cbdb7 100644
+--- a/src/client/views/nodes/chatbot/chatboxcomponents/ChatBox.scss
++++ b/src/client/views/nodes/chatbot/chatboxcomponents/ChatBox.scss
+@@ -950,3 +950,123 @@ $font-size-xlarge: 18px;
+         }
+     }
+ }
++
++/* Tool Reload Modal Styles */
++.tool-reload-modal-overlay {
++    position: fixed;
++    top: 0;
++    left: 0;
++    right: 0;
++    bottom: 0;
++    background-color: rgba(0, 0, 0, 0.5);
++    display: flex;
++    align-items: center;
++    justify-content: center;
++    z-index: 10000;
++    backdrop-filter: blur(4px);
++}
++
++.tool-reload-modal {
++    background: white;
++    border-radius: 12px;
++    padding: 0;
++    min-width: 400px;
++    max-width: 500px;
++    box-shadow:
++        0 20px 25px -5px rgba(0, 0, 0, 0.1),
++        0 10px 10px -5px rgba(0, 0, 0, 0.04);
++    border: 1px solid #e2e8f0;
++    animation: modalSlideIn 0.3s ease-out;
++}
++
++@keyframes modalSlideIn {
++    from {
++        opacity: 0;
++        transform: scale(0.95) translateY(-20px);
++    }
++    to {
++        opacity: 1;
++        transform: scale(1) translateY(0);
++    }
++}
++
++.tool-reload-modal-header {
++    padding: 24px 24px 16px 24px;
++    border-bottom: 1px solid #e2e8f0;
++
++    h3 {
++        margin: 0;
++        font-size: 18px;
++        font-weight: 600;
++        color: #1a202c;
++        display: flex;
++        align-items: center;
++
++        &::before {
++            content: '🛠️';
++            margin-right: 8px;
++            font-size: 20px;
++        }
++    }
++}
++
++.tool-reload-modal-content {
++    padding: 20px 24px;
++
++    p {
++        margin: 0 0 12px 0;
++        line-height: 1.5;
++        color: #4a5568;
++
++        &:last-child {
++            margin-bottom: 0;
++        }
++
++        strong {
++            color: #2d3748;
++            font-weight: 600;
++        }
++    }
++}
++
++.tool-reload-modal-actions {
++    padding: 16px 24px 24px 24px;
++    display: flex;
++    gap: 12px;
++    justify-content: flex-end;
++
++    button {
++        padding: 10px 20px;
++        border-radius: 6px;
++        font-weight: 500;
++        font-size: 14px;
++        cursor: pointer;
++        transition: all 0.2s ease;
++        border: none;
++
++        &.primary {
++            background: #3182ce;
++            color: white;
++
++            &:hover {
++                background: #2c5aa0;
++                transform: translateY(-1px);
++            }
++
++            &:active {
++                transform: translateY(0);
++            }
++        }
++
++        &.secondary {
++            background: #f7fafc;
++            color: #4a5568;
++            border: 1px solid #e2e8f0;
++
++            &:hover {
++                background: #edf2f7;
++                border-color: #cbd5e0;
++            }
++        }
++    }
++}
+diff --git a/src/client/views/nodes/chatbot/chatboxcomponents/ChatBox.tsx b/src/client/views/nodes/chatbot/chatboxcomponents/ChatBox.tsx
+index 470f94a8d..df6c5627c 100644
+--- a/src/client/views/nodes/chatbot/chatboxcomponents/ChatBox.tsx
++++ b/src/client/views/nodes/chatbot/chatboxcomponents/ChatBox.tsx
+@@ -79,6 +79,7 @@ export class ChatBox extends ViewBoxAnnotatableComponent<FieldViewProps>() {
+     @observable private _citationPopup: { text: string; visible: boolean } = { text: '', visible: false };
+     @observable private _isFontSizeModalOpen: boolean = false;
+     @observable private _fontSize: 'small' | 'normal' | 'large' | 'xlarge' = 'normal';
++    @observable private _toolReloadModal: { visible: boolean; toolName: string } = { visible: false, toolName: '' };
+ 
+     // Private properties for managing OpenAI API, vector store, agent, and UI elements
+     private openai!: OpenAI; // Using definite assignment assertion
+@@ -125,6 +126,9 @@ export class ChatBox extends ViewBoxAnnotatableComponent<FieldViewProps>() {
+         // Create an agent with the vectorstore
+         this.agent = new Agent(this.vectorstore, this.retrieveFormattedHistory.bind(this), this.retrieveCSVData.bind(this), this.createImageInDash.bind(this), this.createCSVInDash.bind(this), this.docManager);
+ 
++        // Set up the tool created callback
++        this.agent.setToolCreatedCallback(this.handleToolCreated);
++
+         // Add event listeners
+         this.addScrollListener();
+ 
+@@ -1159,6 +1163,56 @@ export class ChatBox extends ViewBoxAnnotatableComponent<FieldViewProps>() {
+         this._inputValue = question;
+     };
+ 
++    /**
++     * Handles tool creation notification and shows the reload modal
++     * @param toolName The name of the tool that was created
++     */
++    @action
++    handleToolCreated = (toolName: string) => {
++        this._toolReloadModal = {
++            visible: true,
++            toolName: toolName,
++        };
++    };
++
++    /**
++     * Closes the tool reload modal
++     */
++    @action
++    closeToolReloadModal = () => {
++        this._toolReloadModal = {
++            visible: false,
++            toolName: '',
++        };
++    };
++
++    /**
++     * Handles the reload confirmation and triggers page reload
++     */
++    @action
++    handleReloadConfirmation = async () => {
++        // Close the modal first
++        this.closeToolReloadModal();
++
++        try {
++            // Perform the deferred tool save operation
++            const saveSuccess = await this.agent.performDeferredToolSave();
++
++            if (saveSuccess) {
++                console.log('Tool saved successfully, proceeding with reload...');
++            } else {
++                console.warn('Tool save failed, but proceeding with reload anyway...');
++            }
++        } catch (error) {
++            console.error('Error during deferred tool save:', error);
++        }
++
++        // Trigger page reload to rebuild webpack and load the new tool
++        setTimeout(() => {
++            window.location.reload();
++        }, 100);
++    };
++
+     _dictation: DictationButton | null = null;
+ 
+     /**
+@@ -1434,6 +1488,32 @@ export class ChatBox extends ViewBoxAnnotatableComponent<FieldViewProps>() {
+                         <div className="citation-content">{this._citationPopup.text}</div>
+                     </div>
+                 )}
++
++                {/* Tool Reload Modal */}
++                {this._toolReloadModal.visible && (
++                    <div className="tool-reload-modal-overlay">
++                        <div className="tool-reload-modal">
++                            <div className="tool-reload-modal-header">
++                                <h3>Tool Created Successfully!</h3>
++                            </div>
++                            <div className="tool-reload-modal-content">
++                                <p>
++                                    The tool <strong>{this._toolReloadModal.toolName}</strong> has been created and saved successfully.
++                                </p>
++                                <p>To make the tool available for future use, the page needs to be reloaded to rebuild the application bundle.</p>
++                                <p>Click "Reload Page" to complete the tool installation.</p>
++                            </div>
++                            <div className="tool-reload-modal-actions">
++                                <button className="reload-button primary" onClick={this.handleReloadConfirmation}>
++                                    Reload Page
++                                </button>
++                                <button className="close-button secondary" onClick={this.closeToolReloadModal}>
++                                    Later
++                                </button>
++                            </div>
++                        </div>
++                    </div>
++                )}
+             </div>
+         );
+     }
+diff --git a/src/server/index.ts b/src/server/index.ts
+index 3b77359ec..887974ed8 100644
+--- a/src/server/index.ts
++++ b/src/server/index.ts
+@@ -2,6 +2,7 @@ import { yellow } from 'colors';
+ import * as dotenv from 'dotenv';
+ import * as mobileDetect from 'mobile-detect';
+ import * as path from 'path';
++import * as express from 'express';
+ import { logExecution } from './ActionUtilities';
+ import AssistantManager from './ApiManagers/AssistantManager';
+ import FlashcardManager from './ApiManagers/FlashcardManager';
+diff --git a/src/server/server_Initialization.ts b/src/server/server_Initialization.ts
+index 514e2ce1e..80cf977ee 100644
+--- a/src/server/server_Initialization.ts
++++ b/src/server/server_Initialization.ts
+@@ -21,6 +21,7 @@ import { Database } from './database';
+ import { WebSocket } from './websocket';
+ import axios from 'axios';
+ import { JSDOM } from 'jsdom';
++import { setupDynamicToolsAPI } from './api/dynamicTools';
+ 
+ /* RouteSetter is a wrapper around the server that prevents the server
+    from being exposed. */
+@@ -210,6 +211,10 @@ export default async function InitializeServer(routeSetter: RouteSetter) {
+     // app.use(cors({ origin: (_origin: any, callback: any) => callback(null, true) }));
+     registerAuthenticationRoutes(app); // this adds routes to authenticate a user (login, etc)
+     registerCorsProxy(app); // this adds a /corsproxy/ route to allow clients to get to urls that would otherwise be blocked by cors policies
++
++    // Set up the dynamic tools API
++    setupDynamicToolsAPI(app);
++
+     isRelease && !SSL.Loaded && SSL.exit();
+     routeSetter(new RouteManager(app, isRelease)); // this sets up all the regular supervised routes (things like /home, download/upload api's, pdf, search, session, etc)
+     isRelease && process.env.serverPort && (resolvedPorts.server = Number(process.env.serverPort));