path: root/full-diff.txt

diff --git a/.gitignore b/.gitignore
index 7353bc7e0..319a5fa30 100644
--- a/.gitignore
+++ b/.gitignore
@@ -27,3 +27,12 @@ packages/*/dist
 src/server/ApiManagers/temp_data.txt
 /src/server/flashcard/venv
 /src/server/flashcard/venv
+
+summarize_dash_data.py
+tree_to_json.py
+test_dynamic_tools.py
+ts_files_with_content.txt
+ts_files_with_content.txt
+ts_files_with_summaries.txt
+ts_files_with_summaries copy.txt
+summarize_dash_ts.py
\ No newline at end of file
diff --git a/src/client/views/nodes/chatbot/agentsystem/Agent.ts b/src/client/views/nodes/chatbot/agentsystem/Agent.ts
index 1a9df1a75..c3d37fd0e 100644
--- a/src/client/views/nodes/chatbot/agentsystem/Agent.ts
+++ b/src/client/views/nodes/chatbot/agentsystem/Agent.ts
@@ -29,6 +29,7 @@ import { CreateLinksTool } from '../tools/CreateLinksTool';
 import { CodebaseSummarySearchTool } from '../tools/CodebaseSummarySearchTool';
 import { FileContentTool } from '../tools/FileContentTool';
 import { FileNamesTool } from '../tools/FileNamesTool';
+import { CreateNewTool } from '../tools/CreateNewTool';
 //import { CreateTextDocTool } from '../tools/CreateTextDocumentTool';
 
 dotenv.config();
@@ -52,6 +53,12 @@ export class Agent {
     private streamedAnswerParser: StreamedAnswerParser = new StreamedAnswerParser();
     private tools: Record<string, BaseTool<ReadonlyArray<Parameter>>>;
     private _docManager: AgentDocumentManager;
+    // Dynamic tool registry for tools created at runtime
+    private dynamicToolRegistry: Map<string, BaseTool<ReadonlyArray<Parameter>>> = new Map();
+    // Callback for notifying when tools are created and need reload
+    private onToolCreatedCallback?: (toolName: string) => void;
+    // Storage for deferred tool saving
+    private pendingToolSave?: { toolName: string; completeToolCode: string };
 
     /**
      * The constructor initializes the agent with the vector store and toolset, and sets up the OpenAI client.
@@ -79,6 +86,9 @@ export class Agent {
         this._csvData = csvData;
         this._docManager = docManager;
 
+        // Initialize dynamic tool registry
+        this.dynamicToolRegistry = new Map();
+
         // Define available tools for the assistant
         this.tools = {
             calculate: new CalculateTool(),
@@ -94,6 +104,146 @@ export class Agent {
             fileContent: new FileContentTool(this.vectorstore),
             fileNames: new FileNamesTool(this.vectorstore),
         };
+
+        // Add the createNewTool after other tools are defined
+        this.tools.createNewTool = new CreateNewTool(this.dynamicToolRegistry, this.tools, this);
+
+        // Load existing dynamic tools
+        this.loadExistingDynamicTools();
+    }
+
+    /**
+     * Loads existing dynamic tools by checking the current registry and ensuring all stored tools are available
+     */
+    private async loadExistingDynamicTools(): Promise<void> {
+        try {
+            console.log('Loading dynamic tools...');
+
+            // Since we're in a browser environment, we can't use filesystem operations
+            // Instead, we'll maintain tools in the registry and try to load known tools
+
+            // Try to manually load the known dynamic tools that exist
+            const knownDynamicTools = [
+                { name: 'CharacterCountTool', actionName: 'charactercount' },
+                { name: 'WordCountTool', actionName: 'wordcount' },
+                { name: 'TestTool', actionName: 'test' },
+            ];
+
+            let loadedCount = 0;
+            for (const toolInfo of knownDynamicTools) {
+                try {
+                    // Check if tool is already in registry
+                    if (this.dynamicToolRegistry.has(toolInfo.actionName)) {
+                        console.log(`✓ Tool ${toolInfo.actionName} already loaded`);
+                        loadedCount++;
+                        continue;
+                    }
+
+                    // Try to load the tool using require (works better in webpack environment)
+                    let toolInstance = null;
+                    try {
+                        // Use require with the relative path
+                        const toolModule = require(`../tools/dynamic/${toolInfo.name}`);
+                        const ToolClass = toolModule[toolInfo.name];
+
+                        if (ToolClass && typeof ToolClass === 'function') {
+                            toolInstance = new ToolClass();
+
+                            if (toolInstance instanceof BaseTool) {
+                                this.dynamicToolRegistry.set(toolInfo.actionName, toolInstance);
+                                loadedCount++;
+                                console.log(`✓ Loaded dynamic tool: ${toolInfo.actionName} (from ${toolInfo.name})`);
+                            }
+                        }
+                    } catch (requireError) {
+                        // Tool file doesn't exist or can't be loaded, which is fine
+                        console.log(`Tool ${toolInfo.name} not available:`, (requireError as Error).message);
+                    }
+                } catch (error) {
+                    console.warn(`⚠ Failed to load ${toolInfo.name}:`, error);
+                }
+            }
+
+            console.log(`Successfully loaded ${loadedCount} dynamic tools`);
+
+            // Log all currently registered dynamic tools
+            if (this.dynamicToolRegistry.size > 0) {
+                console.log('Currently registered dynamic tools:', Array.from(this.dynamicToolRegistry.keys()));
+            }
+        } catch (error) {
+            console.error('Error loading dynamic tools:', error);
+        }
+    }
+
+    /**
+     * Manually registers a dynamic tool instance (called by CreateNewTool)
+     */
+    public registerDynamicTool(toolName: string, toolInstance: BaseTool<ReadonlyArray<Parameter>>): void {
+        this.dynamicToolRegistry.set(toolName, toolInstance);
+        console.log(`Manually registered dynamic tool: ${toolName}`);
+    }
+
+    /**
+     * Notifies that a tool has been created and saved to disk (called by CreateNewTool)
+     */
+    public notifyToolCreated(toolName: string, completeToolCode: string): void {
+        // Store the tool data for deferred saving
+        this.pendingToolSave = { toolName, completeToolCode };
+
+        if (this.onToolCreatedCallback) {
+            this.onToolCreatedCallback(toolName);
+        }
+    }
+
+    /**
+     * Performs the deferred tool save operation (called after user confirmation)
+     */
+    public async performDeferredToolSave(): Promise<boolean> {
+        if (!this.pendingToolSave) {
+            console.warn('No pending tool save operation');
+            return false;
+        }
+
+        const { toolName, completeToolCode } = this.pendingToolSave;
+
+        try {
+            // Get the CreateNewTool instance to perform the save
+            const createNewTool = this.tools.createNewTool as any;
+            if (createNewTool && typeof createNewTool.saveToolToServerDeferred === 'function') {
+                const success = await createNewTool.saveToolToServerDeferred(toolName, completeToolCode);
+
+                if (success) {
+                    console.log(`Tool ${toolName} saved to server successfully via deferred save.`);
+                    // Clear the pending save
+                    this.pendingToolSave = undefined;
+                    return true;
+                } else {
+                    console.warn(`Tool ${toolName} could not be saved to server via deferred save.`);
+                    return false;
+                }
+            } else {
+                console.error('CreateNewTool instance not available for deferred save');
+                return false;
+            }
+        } catch (error) {
+            console.error(`Error performing deferred tool save for ${toolName}:`, error);
+            return false;
+        }
+    }
+
+    /**
+     * Sets the callback for when tools are created
+     */
+    public setToolCreatedCallback(callback: (toolName: string) => void): void {
+        this.onToolCreatedCallback = callback;
+    }
+
+    /**
+     * Public method to reload dynamic tools (called when new tools are created)
+     */
+    public reloadDynamicTools(): void {
+        console.log('Reloading dynamic tools...');
+        this.loadExistingDynamicTools();
     }
 
     /**
@@ -126,11 +276,8 @@ export class Agent {
         // Push sanitized user's question to message history
         this.messages.push({ role: 'user', content: sanitizedQuestion });
 
-        // Retrieve chat history and generate system prompt
-        const chatHistory = this._history();
-        // Get document summaries directly from document manager
-        // Generate the system prompt with the summaries
-        const systemPrompt = getReactPrompt(Object.values(this.tools), () => JSON.stringify(this._docManager.listDocs), chatHistory);
+        // Get system prompt with all tools (static + dynamic)
+        const systemPrompt = this.getSystemPromptWithAllTools();
 
         // Initialize intermediate messages
         this.interMessages = [{ role: 'system', content: systemPrompt }];
@@ -193,22 +340,25 @@ export class Agent {
                     currentAction = stage[key] as string;
                     console.log(`Action: ${currentAction}`);
 
-                    if (this.tools[currentAction]) {
+                    // Check both static tools and dynamic registry
+                    const tool = this.tools[currentAction] || this.dynamicToolRegistry.get(currentAction);
+
+                    if (tool) {
                         // Prepare the next action based on the current tool
                         const nextPrompt = [
                             {
                                 type: 'text',
-                                text: `<stage number="${i + 1}" role="user">` + builder.build({ action_rules: this.tools[currentAction].getActionRule() }) + `</stage>`,
+                                text: `<stage number="${i + 1}" role="user">` + builder.build({ action_rules: tool.getActionRule() }) + `</stage>`,
                             } as Observation,
                         ];
                         this.interMessages.push({ role: 'user', content: nextPrompt });
                         break;
                     } else {
                         // Handle error in case of an invalid action
-                        console.log('Error: No valid action');
+                        console.log(`Error: Action "${currentAction}" is not a valid tool`);
                         this.interMessages.push({
                             role: 'user',
-                            content: `<stage number="${i + 1}" role="system-error-reporter">No valid action, try again.</stage>`,
+                            content: `<stage number="${i + 1}" role="system-error-reporter">Action "${currentAction}" is not a valid tool, try again.</stage>`,
                         });
                         break;
                     }
@@ -376,8 +526,8 @@ export class Agent {
                     throw new Error('Action must be a non-empty string');
                 }
 
-                // Optional: Check if the action is among allowed actions
-                const allowedActions = Object.keys(this.tools);
+                // Optional: Check if the action is among allowed actions (including dynamic tools)
+                const allowedActions = [...Object.keys(this.tools), ...Array.from(this.dynamicToolRegistry.keys())];
                 if (!allowedActions.includes(stage.action)) {
                     throw new Error(`Action "${stage.action}" is not a valid tool`);
                 }
@@ -482,12 +632,15 @@ export class Agent {
      * @throws An error if the action is unknown, if required parameters are missing, or if input types don't match the expected parameter types.
      */
     private async processAction(action: string, actionInput: ParametersType<ReadonlyArray<Parameter>>): Promise<Observation[]> {
-        // Check if the action exists in the tools list
-        if (!(action in this.tools)) {
+        // Check if the action exists in the tools list or dynamic registry
+        if (!(action in this.tools) && !this.dynamicToolRegistry.has(action)) {
             throw new Error(`Unknown action: ${action}`);
         }
         console.log(actionInput);
 
+        // Determine which tool to use - either from static tools or dynamic registry
+        const tool = this.tools[action] || this.dynamicToolRegistry.get(action);
+
         // Special handling for documentMetadata tool with numeric or boolean fieldValue
         if (action === 'documentMetadata') {
             // Handle single field edit
@@ -520,9 +673,49 @@ export class Agent {
             }
         }
 
-        for (const param of this.tools[action].parameterRules) {
+        // Special handling for createNewTool with parsed XML toolCode
+        if (action === 'createNewTool') {
+            if ('toolCode' in actionInput && typeof actionInput.toolCode === 'object' && actionInput.toolCode !== null) {
+                try {
+                    // Convert the parsed XML object back to a string
+                    const extractText = (obj: any): string => {
+                        if (typeof obj === 'string') {
+                            return obj;
+                        } else if (obj && typeof obj === 'object') {
+                            if (obj._text) {
+                                return obj._text;
+                            }
+                            // Recursively extract text from all properties
+                            let text = '';
+                            for (const key in obj) {
+                                if (key !== '_text') {
+                                    const value = obj[key];
+                                    if (typeof value === 'string') {
+                                        text += value + '\n';
+                                    } else if (value && typeof value === 'object') {
+                                        text += extractText(value) + '\n';
+                                    }
+                                }
+                            }
+                            return text;
+                        }
+                        return '';
+                    };
+
+                    const reconstructedCode = extractText(actionInput.toolCode);
+                    actionInput.toolCode = reconstructedCode;
+                } catch (error) {
+                    console.error('Error processing toolCode:', error);
+                    // Convert to string as fallback
+                    actionInput.toolCode = String(actionInput.toolCode);
+                }
+            }
+        }
+
+        // Check parameter requirements and types for the tool
+        for (const param of tool.parameterRules) {
             // Check if the parameter is required and missing in the input
-            if (param.required && !(param.name in actionInput) && !this.tools[action].inputValidator(actionInput)) {
+            if (param.required && !(param.name in actionInput) && !tool.inputValidator(actionInput)) {
                 throw new Error(`Missing required parameter: ${param.name}`);
             }
 
@@ -540,11 +733,30 @@ export class Agent {
             }
         }
 
-        const tool = this.tools[action];
-
+        // Execute the tool with the validated inputs
         return await tool.execute(actionInput);
     }
 
+    /**
+     * Gets a combined list of all tools, both static and dynamic
+     * @returns An array of all available tool instances
+     */
+    private getAllTools(): BaseTool<ReadonlyArray<Parameter>>[] {
+        // Combine static and dynamic tools
+        return [...Object.values(this.tools), ...Array.from(this.dynamicToolRegistry.values())];
+    }
+
+    /**
+     * Overridden method to get the React prompt with all tools (static + dynamic)
+     */
+    private getSystemPromptWithAllTools(): string {
+        const allTools = this.getAllTools();
+        const docSummaries = () => JSON.stringify(this._docManager.listDocs);
+        const chatHistory = this._history();
+
+        return getReactPrompt(allTools, docSummaries, chatHistory);
+    }
+
     /**
      * Reinitializes the DocumentMetadataTool with a direct reference to the ChatBox instance.
      * This ensures that the tool can properly access the ChatBox document and find related documents.
@@ -560,3 +772,9 @@ export class Agent {
         }
     }
 }
+
+// Forward declaration to avoid circular import
+interface AgentLike {
+    registerDynamicTool(toolName: string, toolInstance: BaseTool<ReadonlyArray<Parameter>>): void;
+    notifyToolCreated(toolName: string, completeToolCode: string): void;
+}
diff --git a/src/client/views/nodes/chatbot/chatboxcomponents/ChatBox.scss b/src/client/views/nodes/chatbot/chatboxcomponents/ChatBox.scss
index 31f7be4c4..8e00cbdb7 100644
--- a/src/client/views/nodes/chatbot/chatboxcomponents/ChatBox.scss
+++ b/src/client/views/nodes/chatbot/chatboxcomponents/ChatBox.scss
@@ -950,3 +950,123 @@ $font-size-xlarge: 18px;
         }
     }
 }
+
+/* Tool Reload Modal Styles */
+.tool-reload-modal-overlay {
+    position: fixed;
+    top: 0;
+    left: 0;
+    right: 0;
+    bottom: 0;
+    background-color: rgba(0, 0, 0, 0.5);
+    display: flex;
+    align-items: center;
+    justify-content: center;
+    z-index: 10000;
+    backdrop-filter: blur(4px);
+}
+
+.tool-reload-modal {
+    background: white;
+    border-radius: 12px;
+    padding: 0;
+    min-width: 400px;
+    max-width: 500px;
+    box-shadow:
+        0 20px 25px -5px rgba(0, 0, 0, 0.1),
+        0 10px 10px -5px rgba(0, 0, 0, 0.04);
+    border: 1px solid #e2e8f0;
+    animation: modalSlideIn 0.3s ease-out;
+}
+
+@keyframes modalSlideIn {
+    from {
+        opacity: 0;
+        transform: scale(0.95) translateY(-20px);
+    }
+    to {
+        opacity: 1;
+        transform: scale(1) translateY(0);
+    }
+}
+
+.tool-reload-modal-header {
+    padding: 24px 24px 16px 24px;
+    border-bottom: 1px solid #e2e8f0;
+
+    h3 {
+        margin: 0;
+        font-size: 18px;
+        font-weight: 600;
+        color: #1a202c;
+        display: flex;
+        align-items: center;
+
+        &::before {
+            content: '🛠️';
+            margin-right: 8px;
+            font-size: 20px;
+        }
+    }
+}
+
+.tool-reload-modal-content {
+    padding: 20px 24px;
+
+    p {
+        margin: 0 0 12px 0;
+        line-height: 1.5;
+        color: #4a5568;
+
+        &:last-child {
+            margin-bottom: 0;
+        }
+
+        strong {
+            color: #2d3748;
+            font-weight: 600;
+        }
+    }
+}
+
+.tool-reload-modal-actions {
+    padding: 16px 24px 24px 24px;
+    display: flex;
+    gap: 12px;
+    justify-content: flex-end;
+
+    button {
+        padding: 10px 20px;
+        border-radius: 6px;
+        font-weight: 500;
+        font-size: 14px;
+        cursor: pointer;
+        transition: all 0.2s ease;
+        border: none;
+
+        &.primary {
+            background: #3182ce;
+            color: white;
+
+            &:hover {
+                background: #2c5aa0;
+                transform: translateY(-1px);
+            }
+
+            &:active {
+                transform: translateY(0);
+            }
+        }
+
+        &.secondary {
+            background: #f7fafc;
+            color: #4a5568;
+            border: 1px solid #e2e8f0;
+
+            &:hover {
+                background: #edf2f7;
+                border-color: #cbd5e0;
+            }
+        }
+    }
+}
diff --git a/src/client/views/nodes/chatbot/chatboxcomponents/ChatBox.tsx b/src/client/views/nodes/chatbot/chatboxcomponents/ChatBox.tsx
index 470f94a8d..df6c5627c 100644
--- a/src/client/views/nodes/chatbot/chatboxcomponents/ChatBox.tsx
+++ b/src/client/views/nodes/chatbot/chatboxcomponents/ChatBox.tsx
@@ -79,6 +79,7 @@ export class ChatBox extends ViewBoxAnnotatableComponent<FieldViewProps>() {
     @observable private _citationPopup: { text: string; visible: boolean } = { text: '', visible: false };
     @observable private _isFontSizeModalOpen: boolean = false;
     @observable private _fontSize: 'small' | 'normal' | 'large' | 'xlarge' = 'normal';
+    @observable private _toolReloadModal: { visible: boolean; toolName: string } = { visible: false, toolName: '' };
 
     // Private properties for managing OpenAI API, vector store, agent, and UI elements
     private openai!: OpenAI; // Using definite assignment assertion
@@ -125,6 +126,9 @@ export class ChatBox extends ViewBoxAnnotatableComponent<FieldViewProps>() {
         // Create an agent with the vectorstore
         this.agent = new Agent(this.vectorstore, this.retrieveFormattedHistory.bind(this), this.retrieveCSVData.bind(this), this.createImageInDash.bind(this), this.createCSVInDash.bind(this), this.docManager);
 
+        // Set up the tool created callback
+        this.agent.setToolCreatedCallback(this.handleToolCreated);
+
         // Add event listeners
         this.addScrollListener();
 
@@ -1159,6 +1163,56 @@ export class ChatBox extends ViewBoxAnnotatableComponent<FieldViewProps>() {
         this._inputValue = question;
     };
 
+    /**
+     * Handles tool creation notification and shows the reload modal
+     * @param toolName The name of the tool that was created
+     */
+    @action
+    handleToolCreated = (toolName: string) => {
+        this._toolReloadModal = {
+            visible: true,
+            toolName: toolName,
+        };
+    };
+
+    /**
+     * Closes the tool reload modal
+     */
+    @action
+    closeToolReloadModal = () => {
+        this._toolReloadModal = {
+            visible: false,
+            toolName: '',
+        };
+    };
+
+    /**
+     * Handles the reload confirmation and triggers page reload
+     */
+    @action
+    handleReloadConfirmation = async () => {
+        // Close the modal first
+        this.closeToolReloadModal();
+
+        try {
+            // Perform the deferred tool save operation
+            const saveSuccess = await this.agent.performDeferredToolSave();
+
+            if (saveSuccess) {
+                console.log('Tool saved successfully, proceeding with reload...');
+            } else {
+                console.warn('Tool save failed, but proceeding with reload anyway...');
+            }
+        } catch (error) {
+            console.error('Error during deferred tool save:', error);
+        }
+
+        // Trigger page reload to rebuild webpack and load the new tool
+        setTimeout(() => {
+            window.location.reload();
+        }, 100);
+    };
+
     _dictation: DictationButton | null = null;
 
     /**
@@ -1434,6 +1488,32 @@ export class ChatBox extends ViewBoxAnnotatableComponent<FieldViewProps>() {
                         <div className="citation-content">{this._citationPopup.text}</div>
                     </div>
                 )}
+
+                {/* Tool Reload Modal */}
+                {this._toolReloadModal.visible && (
+                    <div className="tool-reload-modal-overlay">
+                        <div className="tool-reload-modal">
+                            <div className="tool-reload-modal-header">
+                                <h3>Tool Created Successfully!</h3>
+                            </div>
+                            <div className="tool-reload-modal-content">
+                                <p>
+                                    The tool <strong>{this._toolReloadModal.toolName}</strong> has been created and saved successfully.
+                                </p>
+                                <p>To make the tool available for future use, the page needs to be reloaded to rebuild the application bundle.</p>
+                                <p>Click "Reload Page" to complete the tool installation.</p>
+                            </div>
+                            <div className="tool-reload-modal-actions">
+                                <button className="reload-button primary" onClick={this.handleReloadConfirmation}>
+                                    Reload Page
+                                </button>
+                                <button className="close-button secondary" onClick={this.closeToolReloadModal}>
+                                    Later
+                                </button>
+                            </div>
+                        </div>
+                    </div>
+                )}
             </div>
         );
     }
diff --git a/src/server/index.ts b/src/server/index.ts
index 3b77359ec..887974ed8 100644
--- a/src/server/index.ts
+++ b/src/server/index.ts
@@ -2,6 +2,7 @@ import { yellow } from 'colors';
 import * as dotenv from 'dotenv';
 import * as mobileDetect from 'mobile-detect';
 import * as path from 'path';
+import * as express from 'express';
 import { logExecution } from './ActionUtilities';
 import AssistantManager from './ApiManagers/AssistantManager';
 import FlashcardManager from './ApiManagers/FlashcardManager';
diff --git a/src/server/server_Initialization.ts b/src/server/server_Initialization.ts
index 514e2ce1e..80cf977ee 100644
--- a/src/server/server_Initialization.ts
+++ b/src/server/server_Initialization.ts
@@ -21,6 +21,7 @@ import { Database } from './database';
 import { WebSocket } from './websocket';
 import axios from 'axios';
 import { JSDOM } from 'jsdom';
+import { setupDynamicToolsAPI } from './api/dynamicTools';
 
 /* RouteSetter is a wrapper around the server that prevents the server
    from being exposed. */
@@ -210,6 +211,10 @@ export default async function InitializeServer(routeSetter: RouteSetter) {
     // app.use(cors({ origin: (_origin: any, callback: any) => callback(null, true) }));
     registerAuthenticationRoutes(app); // this adds routes to authenticate a user (login, etc)
     registerCorsProxy(app); // this adds a /corsproxy/ route to allow clients to get to urls that would otherwise be blocked by cors policies
+
+    // Set up the dynamic tools API
+    setupDynamicToolsAPI(app);
+
     isRelease && !SSL.Loaded && SSL.exit();
     routeSetter(new RouteManager(app, isRelease)); // this sets up all the regular supervised routes (things like /home, download/upload api's, pdf, search, session, etc)
     isRelease && process.env.serverPort && (resolvedPorts.server = Number(process.env.serverPort));