diff options
| -rw-r--r-- | .gitignore | 2 | ||||
| -rw-r--r-- | src/client/views/nodes/chatbot/agentsystem/Agent.ts | 91 | ||||
| -rw-r--r-- | src/client/views/nodes/chatbot/tools/dynamic/CharacterCountTool.ts | 33 | ||||
| -rw-r--r-- | src/client/views/nodes/chatbot/tools/dynamic/InspirationalQuotesTool.ts | 39 | ||||
| -rw-r--r-- | summarize_dash_ts.py | 248 |
5 files changed, 336 insertions, 77 deletions
diff --git a/.gitignore b/.gitignore index 7353bc7e0..8ffa03e80 100644 --- a/.gitignore +++ b/.gitignore @@ -26,4 +26,4 @@ packages/*/dist /src/server/flashcard/venv src/server/ApiManagers/temp_data.txt /src/server/flashcard/venv -/src/server/flashcard/venv +/src/server/flashcard/venv
\ No newline at end of file diff --git a/src/client/views/nodes/chatbot/agentsystem/Agent.ts b/src/client/views/nodes/chatbot/agentsystem/Agent.ts index c3d37fd0e..8516f054b 100644 --- a/src/client/views/nodes/chatbot/agentsystem/Agent.ts +++ b/src/client/views/nodes/chatbot/agentsystem/Agent.ts @@ -113,65 +113,62 @@ export class Agent { } /** - * Loads existing dynamic tools by checking the current registry and ensuring all stored tools are available + * Loads every dynamic tool that the server reports via /getDynamicTools. + * • Uses dynamic `import()` so webpack/vite will code-split each tool automatically. + * • Registers the tool in `dynamicToolRegistry` under the name it advertises via + * `toolInfo.name`; also registers the legacy camel-case key if different. */ private async loadExistingDynamicTools(): Promise<void> { try { - console.log('Loading dynamic tools...'); + console.log('Loading dynamic tools from server…'); + const toolFiles = await this.fetchDynamicToolList(); - // Since we're in a browser environment, we can't use filesystem operations - // Instead, we'll maintain tools in the registry and try to load known tools + let loaded = 0; + for (const { name: className, path } of toolFiles) { + // Legacy key (e.g., CharacterCountTool → characterCountTool) + const legacyKey = className.replace(/^[A-Z]/, m => m.toLowerCase()); - // Try to manually load the known dynamic tools that exist - const knownDynamicTools = [ - { name: 'CharacterCountTool', actionName: 'charactercount' }, - { name: 'WordCountTool', actionName: 'wordcount' }, - { name: 'TestTool', actionName: 'test' }, - ]; + // Skip if we already have the legacy key + if (this.dynamicToolRegistry.has(legacyKey)) continue; - let loadedCount = 0; - for (const toolInfo of knownDynamicTools) { try { - // Check if tool is already in registry - if (this.dynamicToolRegistry.has(toolInfo.actionName)) { - console.log(`✓ Tool ${toolInfo.actionName} already loaded`); - loadedCount++; + // @vite-ignore keeps Vite/Webpack from trying to statically analyse the variable part + const ToolClass = require(`../tools/${path}`)[className]; + + if (!ToolClass || !(ToolClass.prototype instanceof BaseTool)) { + console.warn(`File ${path} does not export a valid BaseTool subclass`); continue; } - // Try to load the tool using require (works better in webpack environment) - let toolInstance = null; - try { - // Use require with the relative path - const toolModule = require(`../tools/dynamic/${toolInfo.name}`); - const ToolClass = toolModule[toolInfo.name]; + const instance: BaseTool<ReadonlyArray<Parameter>> = new ToolClass(); - if (ToolClass && typeof ToolClass === 'function') { - toolInstance = new ToolClass(); + // Prefer the tool’s self-declared name (matches <action> tag) + const key = (instance.name || '').trim() || legacyKey; - if (toolInstance instanceof BaseTool) { - this.dynamicToolRegistry.set(toolInfo.actionName, toolInstance); - loadedCount++; - console.log(`✓ Loaded dynamic tool: ${toolInfo.actionName} (from ${toolInfo.name})`); - } - } - } catch (requireError) { - // Tool file doesn't exist or can't be loaded, which is fine - console.log(`Tool ${toolInfo.name} not available:`, (requireError as Error).message); + // Check for duplicates + if (this.dynamicToolRegistry.has(key)) { + console.warn(`Dynamic tool key '${key}' already registered – keeping existing instance`); + continue; } - } catch (error) { - console.warn(`⚠ Failed to load ${toolInfo.name}:`, error); - } - } - console.log(`Successfully loaded ${loadedCount} dynamic tools`); + // ✅ register under the preferred key + this.dynamicToolRegistry.set(key, instance); - // Log all currently registered dynamic tools - if (this.dynamicToolRegistry.size > 0) { - console.log('Currently registered dynamic tools:', Array.from(this.dynamicToolRegistry.keys())); + // optional: also register the legacy key for safety + if (key !== legacyKey && !this.dynamicToolRegistry.has(legacyKey)) { + this.dynamicToolRegistry.set(legacyKey, instance); + } + + loaded++; + console.info(`✓ Loaded dynamic tool '${key}' from '${path}'`); + } catch (err) { + console.error(`✗ Failed to load '${path}':`, err); + } } - } catch (error) { - console.error('Error loading dynamic tools:', error); + + console.log(`Dynamic-tool load complete – ${loaded}/${toolFiles.length} added`); + } catch (err) { + console.error('Dynamic-tool bootstrap failed:', err); } } @@ -246,6 +243,14 @@ export class Agent { this.loadExistingDynamicTools(); } + private async fetchDynamicToolList(): Promise<{ name: string; path: string }[]> { + const res = await fetch('/getDynamicTools'); + if (!res.ok) throw new Error(`Failed to fetch dynamic tool list – ${res.statusText}`); + const json = await res.json(); + console.log('Dynamic tools fetched:', json.tools); + return json.tools ?? []; + } + /** * This method handles the conversation flow with the assistant, processes user queries, * and manages the assistant's decision-making process, including tool actions. diff --git a/src/client/views/nodes/chatbot/tools/dynamic/CharacterCountTool.ts b/src/client/views/nodes/chatbot/tools/dynamic/CharacterCountTool.ts deleted file mode 100644 index 38fed231c..000000000 --- a/src/client/views/nodes/chatbot/tools/dynamic/CharacterCountTool.ts +++ /dev/null @@ -1,33 +0,0 @@ -import { Observation } from '../../types/types'; -import { ParametersType, ToolInfo } from '../../types/tool_types'; -import { BaseTool } from '../BaseTool'; - -const characterCountParams = [ - { - name: 'text', - type: 'string', - description: 'The text to count characters in', - required: true - } - ] as const; - - type CharacterCountParamsType = typeof characterCountParams; - - const characterCountInfo: ToolInfo<CharacterCountParamsType> = { - name: 'charactercount', - description: 'Counts characters in text, excluding spaces', - citationRules: 'No citation needed.', - parameterRules: characterCountParams - }; - - export class CharacterCountTool extends BaseTool<CharacterCountParamsType> { - constructor() { - super(characterCountInfo); - } - - async execute(args: ParametersType<CharacterCountParamsType>): Promise<Observation[]> { - const { text } = args; - const count = text ? text.replace(/\s/g, '').length : 0; - return [{ type: 'text', text: `Character count (excluding spaces): ${count}` }]; - } - }
\ No newline at end of file diff --git a/src/client/views/nodes/chatbot/tools/dynamic/InspirationalQuotesTool.ts b/src/client/views/nodes/chatbot/tools/dynamic/InspirationalQuotesTool.ts new file mode 100644 index 000000000..23bbe1d76 --- /dev/null +++ b/src/client/views/nodes/chatbot/tools/dynamic/InspirationalQuotesTool.ts @@ -0,0 +1,39 @@ +import { Observation } from '../../types/types'; +import { ParametersType, ToolInfo } from '../../types/tool_types'; +import { BaseTool } from '../BaseTool'; + +const inspirationalQuotesParams = [ + { + name: 'category', + type: 'string', + description: 'The category of inspirational quotes to retrieve', + required: false + } + ] as const; + + type InspirationalQuotesParamsType = typeof inspirationalQuotesParams; + + const inspirationalQuotesInfo: ToolInfo<InspirationalQuotesParamsType> = { + name: 'inspirationalquotestool', + description: 'Provides a random inspirational quote from a predefined list.', + citationRules: 'No citation needed.', + parameterRules: inspirationalQuotesParams + }; + + export class InspirationalQuotesTool extends BaseTool<InspirationalQuotesParamsType> { + constructor() { + super(inspirationalQuotesInfo); + } + + async execute(args: ParametersType<InspirationalQuotesParamsType>): Promise<Observation[]> { + const quotes = [ + "The only way to do great work is to love what you do. - Steve Jobs", + "The best time to plant a tree was 20 years ago. The second best time is now. - Chinese Proverb", + "Your time is limited, so don’t waste it living someone else’s life. - Steve Jobs", + "Not everything that is faced can be changed, but nothing can be changed until it is faced. - James Baldwin", + "The purpose of our lives is to be happy. - Dalai Lama" + ]; + const randomQuote = quotes[Math.floor(Math.random() * quotes.length)]; + return [{ type: 'text', text: randomQuote }]; + } + }
\ No newline at end of file diff --git a/summarize_dash_ts.py b/summarize_dash_ts.py new file mode 100644 index 000000000..69f80fde5 --- /dev/null +++ b/summarize_dash_ts.py @@ -0,0 +1,248 @@ +#!/usr/bin/env python3 +""" +summarize_dash_ts.py – v4 (periodic-save edition) + +• Dumps every .ts/.tsx file (skipping node_modules, etc.) +• Calls GPT-4o with Structured Outputs (JSON-schema “const” on filename) +• Prints each raw JSON reply (unless --quiet) +• Flushes the growing summary file to disk every N files (default 10) + +pip install openai tqdm rich +""" + +from __future__ import annotations + +import argparse +import json +import os +import pathlib +import sys +from textwrap import dedent +from typing import Dict, Iterable, List + +import openai +from rich.console import Console +from rich.tree import Tree +from tqdm import tqdm + +PERIODIC_SAVE_EVERY = 10 # ← change here if you want finer or coarser saves + + +# ───────────────────────── CLI ────────────────────────── +def parse_args() -> argparse.Namespace: + p = argparse.ArgumentParser(prog="summarize_dash_ts.py") + p.add_argument("-r", "--root", type=pathlib.Path, default=".", help="Repo root") + p.add_argument("--model", default="gpt-4o-2024-08-06") + p.add_argument("--api-key", help="OpenAI API key (else env var)") + p.add_argument("--max-tokens", type=int, default=512) + p.add_argument( + "--skip-dirs", + nargs="*", + default=["node_modules", ".git", "dist", "build", ".next"], + ) + p.add_argument( + "--preview", type=int, default=5, help="How many summaries to echo at the end" + ) + p.add_argument( + "--quiet", + action="store_true", + help="Suppress the per-file raw JSON spam once you trust the run", + ) + return p.parse_args() + + +# ────────────────── helpers ────────────────── +def iter_ts(root: pathlib.Path, skip: List[str]) -> Iterable[pathlib.Path]: + for dpath, dnames, fnames in os.walk(root): + dnames[:] = [d for d in dnames if d not in skip] + for fn in fnames: + if fn.endswith((".ts", ".tsx")): + yield pathlib.Path(dpath) / fn + + +def safe_open(p: pathlib.Path): + try: + return p.open(encoding="utf-8") + except UnicodeDecodeError: + return p.open(encoding="utf-8", errors="replace") + + +def make_tree(paths: list[pathlib.Path], root: pathlib.Path) -> Tree: + t = Tree(str(root)) + nodes: dict[pathlib.Path, Tree] = {root: t} + for p in sorted(paths): + cur = root + for part in p.relative_to(root).parts: + cur = cur / part + if cur not in nodes: + nodes[cur] = nodes[cur.parent].add(part) + return t + + +def write_tree_with_summaries(*, tree: Tree, summaries: dict[pathlib.Path, str], + root: pathlib.Path, out_path: pathlib.Path) -> None: + tmp = out_path.with_suffix(".tmp") + with tmp.open("w", encoding="utf-8") as f: + + def walk(node: Tree, rel_path: pathlib.Path = pathlib.Path("."), indent: str = ""): + last = node.children[-1] if node.children else None + for child in node.children: + marker = "└── " if child is last else "├── " + new_indent = indent + (" " if child is last else "│ ") + child_rel = rel_path / child.label # ← **the missing bit** + + # absolute path used as dict-key during summarization loop + abs_path = root / child_rel + if abs_path in summaries: + f.write(f"{indent}{marker}{child.label} – {summaries[abs_path]}\n") + else: + f.write(f"{indent}{marker}{child.label}\n") + + walk(child, child_rel, new_indent) + + walk(tree) + tmp.replace(out_path) + + +# ────────────────── prompt bits ────────────────── +SYSTEM = """ +You are an expert TypeScript code summarizer for the Dash hypermedia code-base. + +You will be given ONE complete file and its **exact** relative path. + +Return ONLY JSON matching this shape: + +{ + "filename": "<EXACT path you were given>", + "summary": "<3–5 sentences, <80 words>" +} + +No markdown, no extra keys. +""".strip() + +OVERVIEW = dedent( + """ + Dash is a browser-based hypermedia system from Brown University that lets users + mix PDFs, web pages, audio, video, ink and rich-text on a free-form canvas, + create Vannevar-Bush-style “trails”, and tag/spatially arrange docs for + nonlinear workflows. 99 % of the code-base is TypeScript/React. + """ +).strip() + +SCHEMA_BASE = { + "type": "object", + "properties": { + "filename": {"type": "string"}, + "summary": {"type": "string"}, + }, + "required": ["filename", "summary"], + "additionalProperties": False, +} + + +def ask_llm( + client: openai.OpenAI, + model: str, + rel_path: str, + code: str, + max_tokens: int, + verbose: bool = True, +) -> str: + schema = { + "name": "dash_file_summary", + "strict": True, + "schema": dict( + SCHEMA_BASE, + properties=dict( + SCHEMA_BASE["properties"], filename={"type": "string", "const": rel_path} + ), + ), + } + + messages = [ + {"role": "system", "content": SYSTEM}, + { + "role": "user", + "content": f"{OVERVIEW}\n\n(PATH = {rel_path})\n\n===== BEGIN FILE =====\n{code}\n===== END FILE =====", + }, + ] + + comp = client.chat.completions.create( + model=model, + messages=messages, + response_format={"type": "json_schema", "json_schema": schema}, + max_tokens=max_tokens, + ) + + raw = comp.choices[0].message.content + if verbose: + print(f"\n📝 Raw JSON for {rel_path}:\n{raw}\n") + + data = json.loads(raw) + if data["filename"] != rel_path: + Console().print( + f"[red]⚠︎ Filename mismatch – model said {data['filename']!r}[/red]" + ) + data["filename"] = rel_path + return data["summary"].strip() + + +# ────────────────── main ────────────────── +def main() -> None: + args = parse_args() + openai.api_key = args.api_key or os.getenv("OPENAI_API_KEY") or sys.exit( + "Need OPENAI_API_KEY" + ) + + root = args.root.resolve() + con = Console() + con.print(f":mag: [bold]Scanning[/bold] {root}") + + files = list(iter_ts(root, args.skip_dirs)) + if not files: + con.print("[yellow]No TS/TSX files found[/yellow]") + return + + # 1. full dump of file contents (unchanged) + tree = make_tree(files, root) + (root / "ts_files_with_content.txt").write_text( + Console(record=True, width=120).print(tree, end="") or "" + ) + with (root / "ts_files_with_content.txt").open("a", encoding="utf-8") as fp: + for p in tqdm(files, desc="Dumping source"): + fp.write(f"{p.relative_to(root)}\n{'-'*80}\n") + fp.write(safe_open(p).read()) + fp.write(f"\n{'='*80}\n\n") + + # 2. summaries (periodic save) + client = openai.OpenAI() + summaries: Dict[pathlib.Path, str] = {} + out_file = root / "ts_files_with_summaries.txt" + + for idx, p in enumerate(tqdm(files, desc="GPT-4o summarizing"), 1): + summaries[p] = ask_llm( + client, + args.model, + str(p.relative_to(root)), + safe_open(p).read(), + args.max_tokens, + verbose=not args.quiet, + ) + + if idx % PERIODIC_SAVE_EVERY == 0: + write_tree_with_summaries(tree=tree, summaries=summaries, root=root, out_path=out_file) + con.print(f"[green]✔ Flushed after {idx} files[/green]") + + # final flush + write_tree_with_summaries(tree=tree, summaries=summaries, root=root, out_path=out_file) + + # preview + con.print("\n[cyan]Sample summaries:[/cyan]") + for i, (p, s) in enumerate(list(summaries.items())[: args.preview], 1): + con.print(f"{i}. {p.relative_to(root)} → {s}") + + con.print(f":sparkles: Done – wrote [bold]{out_file}[/bold]") + + +if __name__ == "__main__": + main() |