diff options
Diffstat (limited to 'src/client')
10 files changed, 951 insertions, 125 deletions
diff --git a/src/client/views/nodes/chatbot/agentsystem/Agent.ts b/src/client/views/nodes/chatbot/agentsystem/Agent.ts index 6a15d0c1d..0f7738703 100644 --- a/src/client/views/nodes/chatbot/agentsystem/Agent.ts +++ b/src/client/views/nodes/chatbot/agentsystem/Agent.ts @@ -35,6 +35,7 @@ import { WebsiteInfoScraperTool } from '../tools/WebsiteInfoScraperTool'; import { AgentDocumentManager } from '../utils/AgentDocumentManager'; import { FilterDocsTool } from '../tools/FilterDocsTool'; import { CanvasDocsTool } from '../tools/CanvasDocsTool'; +import { UIControlTool } from '../tools/UIControlTool'; dotenv.config(); @@ -117,6 +118,7 @@ export class Agent { filterDocs: new FilterDocsTool(this._docManager, this.parentView), takeQuiz: new TakeQuizTool(this._docManager), canvasDocs: new CanvasDocsTool(), + uiControl: new UIControlTool(), }; diff --git a/src/client/views/nodes/chatbot/tools/CanvasDocsTool.ts b/src/client/views/nodes/chatbot/tools/CanvasDocsTool.ts index daf6ed941..090b9f5c9 100644 --- a/src/client/views/nodes/chatbot/tools/CanvasDocsTool.ts +++ b/src/client/views/nodes/chatbot/tools/CanvasDocsTool.ts @@ -283,7 +283,20 @@ export class CanvasDocsTool extends BaseTool<typeof parameterRules> { try { const includeSystemDocs = args.includeSystemDocs || false; - const canvasDocs = this.getAllCanvasDocuments(includeSystemDocs); + const allCanvasDocs = this.getAllCanvasDocuments(includeSystemDocs); + + // Filter out container/dashboard documents to only show actual content + const canvasDocs = allCanvasDocs.filter(doc => { + // Filter out collection containers and docking views (dashboards) + const isContainer = doc.type === 'collection' || doc._type_collection === 'Docking'; + if (isContainer) { + console.log(`[CanvasDocsTool] Filtering out container document: ${doc.title || 'Untitled'} (type: ${doc.type}, _type_collection: ${doc._type_collection})`); + return false; + } + return true; + }); + + console.log(`[CanvasDocsTool] Found ${allCanvasDocs.length} total documents, ${canvasDocs.length} content documents after filtering containers`); switch (args.action) { case 'list': { diff --git a/src/client/views/nodes/chatbot/tools/DocumentMetadataTool.ts b/src/client/views/nodes/chatbot/tools/DocumentMetadataTool.ts index 6568766c5..941b875a2 100644 --- a/src/client/views/nodes/chatbot/tools/DocumentMetadataTool.ts +++ b/src/client/views/nodes/chatbot/tools/DocumentMetadataTool.ts @@ -491,7 +491,7 @@ export class DocumentMetadataTool extends BaseTool<DocumentMetadataToolParamsTyp constructor(docManager: AgentDocumentManager) { super(documentMetadataToolInfo); this._docManager = docManager; - this._docManager.initializeFindDocsFreeform(); + this._docManager.initializeDocuments(); } /** @@ -502,8 +502,8 @@ export class DocumentMetadataTool extends BaseTool<DocumentMetadataToolParamsTyp async execute(args: ParametersType<DocumentMetadataToolParamsType>): Promise<Observation[]> { console.log('DocumentMetadataTool: Executing with args:', args); - // Find all documents in the Freeform view - this._docManager.initializeFindDocsFreeform(); + // Find all documents based on current mode (canvas or linked) + this._docManager.initializeDocuments(); try { // Validate required input parameters based on action diff --git a/src/client/views/nodes/chatbot/tools/FilterDocsTool.ts b/src/client/views/nodes/chatbot/tools/FilterDocsTool.ts index 2eebaf8d0..b160badde 100644 --- a/src/client/views/nodes/chatbot/tools/FilterDocsTool.ts +++ b/src/client/views/nodes/chatbot/tools/FilterDocsTool.ts @@ -43,7 +43,7 @@ export class FilterDocsTool extends BaseTool<typeof parameterRules> { constructor(docManager: AgentDocumentManager, collectionView: DocumentView) { super(toolInfo); this._docManager = docManager; - this._docManager.initializeFindDocsFreeform(); + this._docManager.initializeDocuments(); this._collectionView = collectionView; this._initializeDocumentContext(); } diff --git a/src/client/views/nodes/chatbot/tools/ImageCreationTool.ts b/src/client/views/nodes/chatbot/tools/ImageCreationTool.ts index c5b1e028b..fa98e2472 100644 --- a/src/client/views/nodes/chatbot/tools/ImageCreationTool.ts +++ b/src/client/views/nodes/chatbot/tools/ImageCreationTool.ts @@ -6,6 +6,13 @@ import { Observation } from '../types/types'; import { BaseTool } from './BaseTool'; import { Upload } from '../../../../../server/SharedMediaTypes'; import { List } from '../../../../../fields/List'; +import { SmartDrawHandler } from '../../../smartdraw/SmartDrawHandler'; +import { FireflyImageDimensions } from '../../../smartdraw/FireflyConstants'; +import { gptImageCall } from '../../../../apis/gpt/GPT'; +import { ClientUtils } from '../../../../../ClientUtils'; +import { Doc } from '../../../../../fields/Doc'; +import { DocumentViewInternal } from '../../DocumentView'; +import { OpenWhere } from '../../OpenWhere'; const imageCreationToolParams = [ { @@ -14,6 +21,18 @@ const imageCreationToolParams = [ description: 'The prompt for the image to be created. This should be a string that describes the image to be created in extreme detail for an AI image generator.', required: true, }, + { + name: 'engine', + type: 'string', + description: 'The image generation engine to use. Options: "firefly" (default), "dalle". If not specified, defaults to "firefly".', + required: false, + }, + { + name: 'aspect_ratio', + type: 'string', + description: 'Aspect ratio for the image (Firefly only). Options: "square" (default), "landscape", "portrait", "widescreen".', + required: false, + }, ] as const; type ImageCreationToolParamsType = typeof imageCreationToolParams; @@ -22,7 +41,7 @@ const imageCreationToolInfo: ToolInfo<ImageCreationToolParamsType> = { name: 'imageCreationTool', citationRules: 'No citation needed. Cannot cite image generation for a response.', parameterRules: imageCreationToolParams, - description: 'Create an image of any style, content, or design, based on a prompt. The prompt should be a detailed description of the image to be created.', + description: 'Create an image of any style, content, or design, based on a prompt. Uses Firefly by default for better quality and control. Use "dalle" engine explicitly only if requested. The prompt should be a detailed description of the image to be created.', }; export class ImageCreationTool extends BaseTool<ImageCreationToolParamsType> { @@ -35,37 +54,109 @@ export class ImageCreationTool extends BaseTool<ImageCreationToolParamsType> { } async execute(args: ParametersType<ImageCreationToolParamsType>): Promise<Observation[]> { - const image_prompt = args.image_prompt; + const { image_prompt, engine = 'firefly', aspect_ratio = 'square' } = args; + + console.log(`Generating image with ${engine} for prompt: ${image_prompt}`); - console.log(`Generating image for prompt: ${image_prompt}`); - // Create an array of promises, each one handling a search for a query try { - const { result, url } = (await Networking.PostToServer('/generateImage', { - image_prompt, - })) as { result: Upload.FileInformation & Upload.InspectionResults; url: string }; - console.log('Image generation result:', result); - this._createImage(result, { text: RTFCast(image_prompt), ai: 'dall-e-3', tags: new List<string>(['@ai']) }); - return url - ? [ - { - type: 'image_url', - image_url: { url }, - }, - ] - : [ - { - type: 'text', - text: `An error occurred while generating image.`, - }, - ]; + if (engine.toLowerCase() === 'dalle') { + // Use DALL-E for image generation + return await this.generateWithDalle(image_prompt); + } else { + // Default to Firefly + return await this.generateWithFirefly(image_prompt, aspect_ratio); + } } catch (error) { - console.log(error); + console.error('ImageCreationTool error:', error); return [ { type: 'text', - text: `An error occurred while generating image.`, + text: `Error generating image: ${error}`, }, ]; } } + + private async generateWithDalle(prompt: string): Promise<Observation[]> { + try { + // Call GPT image API directly + const imageUrls = await gptImageCall(prompt); + + if (imageUrls && imageUrls[0]) { + // Upload the remote image to our server + const uploadRes = await Networking.PostToServer('/uploadRemoteImage', { sources: [imageUrls[0]] }); + const fileInfo = (uploadRes as Upload.FileInformation[])[0]; + const source = ClientUtils.prepend(fileInfo.accessPaths.agnostic.client); + + // Create image document with DALL-E metadata + this._createImage(fileInfo as Upload.FileInformation & Upload.InspectionResults, { + text: RTFCast(prompt), + ai: 'dall-e-3', + tags: new List<string>(['@ai']), + title: prompt, + _width: 400, + _height: 400, + }); + + return [ + { + type: 'image_url', + image_url: { url: source }, + }, + ]; + } else { + return [ + { + type: 'text', + text: 'Failed to generate image with DALL-E', + }, + ]; + } + } catch (error) { + console.error('DALL-E generation error:', error); + throw error; + } + } + + private async generateWithFirefly(prompt: string, aspectRatio: string): Promise<Observation[]> { + try { + // Map aspect ratio string to FireflyImageDimensions enum + const dimensionMap: Record<string, FireflyImageDimensions> = { + 'square': FireflyImageDimensions.Square, + 'landscape': FireflyImageDimensions.Landscape, + 'portrait': FireflyImageDimensions.Portrait, + 'widescreen': FireflyImageDimensions.Widescreen, + }; + + const dimensions = dimensionMap[aspectRatio.toLowerCase()] || FireflyImageDimensions.Square; + + // Use SmartDrawHandler to create Firefly image + const doc = await SmartDrawHandler.CreateWithFirefly(prompt, dimensions); + + if (doc instanceof Doc) { + // Open the document in a new tab + DocumentViewInternal.addDocTabFunc(doc, OpenWhere.addRight); + + // Get the image URL from the document + const imageUrl = doc.image || doc.url || ''; + + return [ + { + type: 'text', + text: `Created image with Firefly: "${prompt}". The image has been opened in a new tab.`, + }, + ]; + } else { + return [ + { + type: 'text', + text: 'Failed to generate image with Firefly', + }, + ]; + } + } catch (error) { + console.error('Firefly generation error:', error); + throw error; + } + } } diff --git a/src/client/views/nodes/chatbot/tools/SortDocsTool.ts b/src/client/views/nodes/chatbot/tools/SortDocsTool.ts index 1944f0bc1..9eb0cf4d1 100644 --- a/src/client/views/nodes/chatbot/tools/SortDocsTool.ts +++ b/src/client/views/nodes/chatbot/tools/SortDocsTool.ts @@ -37,7 +37,7 @@ export class SortDocsTool extends BaseTool<typeof parameterRules> { // We assume the ChatBox itself is currently selected in its parent view. this._collectionView = collectionView; this._docManager = docManager; - this._docManager.initializeFindDocsFreeform(); + this._docManager.initializeDocuments(); } async execute(args: ParametersType<typeof parameterRules>): Promise<Observation[]> { diff --git a/src/client/views/nodes/chatbot/tools/TagDocsTool.ts b/src/client/views/nodes/chatbot/tools/TagDocsTool.ts index c88c32e50..de824ec0d 100644 --- a/src/client/views/nodes/chatbot/tools/TagDocsTool.ts +++ b/src/client/views/nodes/chatbot/tools/TagDocsTool.ts @@ -1,25 +1,27 @@ -import { BaseTool } from './BaseTool'; -import { Observation } from '../types/types'; -import { ParametersType, ToolInfo } from '../types/tool_types'; -import { AgentDocumentManager } from '../utils/AgentDocumentManager'; -import { gptAPICall, GPTCallType, DescriptionSeperator, DataSeperator } from '../../../../apis/gpt/GPT'; import { v4 as uuidv4 } from 'uuid'; +import { Doc } from '../../../../../fields/Doc'; +import { Id } from '../../../../../fields/FieldSymbols'; import { TagItem } from '../../../TagsView'; +import { gptAPICall, GPTCallType, DescriptionSeperator, DataSeperator } from '../../../../apis/gpt/GPT'; +import { ParametersType, ToolInfo } from '../types/tool_types'; +import { Observation } from '../types/types'; +import { BaseTool } from './BaseTool'; +import { AgentDocumentManager } from '../utils/AgentDocumentManager'; const parameterRules = [ { name: 'taggingCriteria', type: 'string', - description: 'Natural‐language criteria for tagging documents.', + description: 'Natural language description of how to tag the documents (e.g., "tag by subject", "categorize by theme")', required: true, }, ] as const; const toolInfo: ToolInfo<typeof parameterRules> = { name: 'tagDocs', - description: 'Automatically generate and apply tags to docs based on criteria.', + description: 'Tag documents in the collection based on natural language criteria. Uses GPT to analyze document content and apply appropriate tags.', parameterRules, - citationRules: 'No citation needed for tagging operations.', + citationRules: 'Citation not required for tagging operations.', }; export class TagDocsTool extends BaseTool<typeof parameterRules> { @@ -28,77 +30,69 @@ export class TagDocsTool extends BaseTool<typeof parameterRules> { constructor(docManager: AgentDocumentManager) { super(toolInfo); this._docManager = docManager; - // ensure our manager has scanned all freeform docs - this._docManager.initializeFindDocsFreeform(); + this._docManager.initializeDocuments(); } async execute(args: ParametersType<typeof parameterRules>): Promise<Observation[]> { const chunkId = uuidv4(); try { - // 1) Build description→ID map and the prompt string - const textToId = new Map<string, string>(); + // Build the textToDocMap exactly like GPTPopup does + this._docManager.initializeDocuments(); + const textToDocMap = new Map<string, Doc>(); const descriptionsArray: string[] = []; - // We assume getDocDescription returns a Promise<string> with the text + // Build descriptions exactly like GPTPopup for (const id of this._docManager.docIds) { + const doc = this._docManager.getDocument(id); + if (!doc) continue; + const desc = (await this._docManager.getDocDescription(id)).replace(/\n/g, ' ').trim(); if (!desc) continue; - textToId.set(desc, id); - // wrap in separators exactly as GPTPopup does + + textToDocMap.set(desc, doc); descriptionsArray.push(`${DescriptionSeperator}${desc}${DescriptionSeperator}`); } const promptDescriptions = descriptionsArray.join(''); + console.log('[TagDocsTool] Sending descriptions to GPT:', promptDescriptions.substring(0, 200) + '...'); - // 2) Call GPT - const raw = await gptAPICall( + // Call GPT with the same method as GPTPopup + const gptOutput = await gptAPICall( args.taggingCriteria, GPTCallType.TAGDOCS, promptDescriptions ); - console.log('[TagDocsTool] GPT raw:', raw); + console.log('[TagDocsTool] GPT raw:', gptOutput); - // 3) Parse GPT’s response, look up each description, and apply tags + // Use the same parsing logic as GPTPopup's processGptResponse for AssignTags const appliedTags: Record<string, string[]> = {}; - - raw - .split(DescriptionSeperator) // split into blocks - .filter(block => block.trim() !== '') // remove empties - .map(block => block.replace(/\n/g, ' ').trim()) // flatten whitespace - .forEach(block => { - // each block should look like: "<desc_text>>>>><tag1,tag2>" - const [descText, tagsText = ''] = block.split(DataSeperator).map(s => s.trim()); - if (!descText || !tagsText) { - console.warn('[TagDocsTool] skipping invalid block:', block); - return; - } - const docId = textToId.get(descText); - if (!docId) { - console.warn('[TagDocsTool] no docId for description:', descText); - return; + + gptOutput + .split(DescriptionSeperator) + .filter(item => item.trim() !== '') + .map(docContentRaw => docContentRaw.replace(/\n/g, ' ').trim()) + .map(docContentRaw => ({ + doc: textToDocMap.get(docContentRaw.split(DataSeperator)[0]), + data: docContentRaw.split(DataSeperator)[1] + })) + .filter(({doc}) => doc) + .map(({doc, data}) => ({doc: doc!, data})) + .forEach(({doc, data}) => { + if (data) { + // Apply tag exactly like GPTPopup does + const tag = data.startsWith('#') ? data : '#' + data[0].toLowerCase() + data.slice(1); + TagItem.addTagToDoc(doc, tag); + + const docId = doc[Id] || 'unknown'; + appliedTags[docId] = appliedTags[docId] || []; + appliedTags[docId].push(tag); + + console.log(`[TagDocsTool] Applied tag "${tag}" to document ${docId}`); } - const doc = this._docManager.getDocument(docId); - if (!doc) { - console.warn('[TagDocsTool] no Doc instance for ID:', docId); - return; - } - - // split/normalize tags - const normalized = tagsText - .split(',') - .map(tag => tag.trim()) - .filter(tag => tag.length > 0) - .map(tag => (tag.startsWith('#') ? tag : `#${tag}`)); - - // apply tags - normalized.forEach(tag => TagItem.addTagToDoc(doc, tag)); - - // record for our summary - appliedTags[docId] = normalized; }); - // 4) Build a summary observation + // Build summary const summary = Object.entries(appliedTags) .map(([id, tags]) => `${id}: ${tags.join(', ')}`) .join('; '); @@ -107,7 +101,8 @@ export class TagDocsTool extends BaseTool<typeof parameterRules> { { type: 'text', text: `<chunk chunk_id="${chunkId}" chunk_type="tagging_status"> -Successfully tagged documents based on "${args.taggingCriteria}". Tags applied: ${summary} +Successfully tagged ${Object.keys(appliedTags).length} documents based on "${args.taggingCriteria}". +Tags applied: ${summary || '(none)'} </chunk>`, }, ]; diff --git a/src/client/views/nodes/chatbot/tools/TakeQuizTool.ts b/src/client/views/nodes/chatbot/tools/TakeQuizTool.ts index 78d9859b8..12b2d1e91 100644 --- a/src/client/views/nodes/chatbot/tools/TakeQuizTool.ts +++ b/src/client/views/nodes/chatbot/tools/TakeQuizTool.ts @@ -4,85 +4,229 @@ import { ParametersType, ToolInfo } from '../types/tool_types'; import { AgentDocumentManager } from '../utils/AgentDocumentManager'; import { GPTCallType, gptAPICall } from '../../../../apis/gpt/GPT'; import { v4 as uuidv4 } from 'uuid'; +import { Doc } from '../../../../../fields/Doc'; +import { StrCast } from '../../../../../fields/Types'; +import { DocumentView } from '../../DocumentView'; const parameterRules = [ { - name: 'userAnswer', + name: 'action', type: 'string', - description: 'User-provided answer to the quiz question.', + description: 'Quiz action to perform: "start" to begin quiz with random document, "answer" to submit answer to current quiz question, "next" to get next random question', required: true, }, + { + name: 'userAnswer', + type: 'string', + description: 'User-provided answer to the quiz question (required when action is "answer")', + required: false, + }, ] as const; const toolInfo: ToolInfo<typeof parameterRules> = { name: 'takeQuiz', - description: - 'Tests the user\'s knowledge and evaluates a user\'s answer for a randomly selected document.', + description: 'Interactive quiz system that tests user knowledge. Use "start" when user says "take quiz" or "start quiz". Use "answer" when user provides their answer to the quiz question. Use "next" when user asks for "next question". After starting a quiz, the agent should WAIT for the user to provide their answer naturally in conversation before calling this tool again.', parameterRules, citationRules: 'No citation needed for quiz operations.', }; export class TakeQuizTool extends BaseTool<typeof parameterRules> { private _docManager: AgentDocumentManager; + private _currentQuizDoc: Doc | null = null; + private _currentQuizDescription: string = ''; constructor(docManager: AgentDocumentManager) { super(toolInfo); this._docManager = docManager; - this._docManager.initializeFindDocsFreeform(); + this._docManager.initializeDocuments(); } - private async generateRubric(docId: string, description: string): Promise<string> { - const docMeta = this._docManager.extractDocumentMetadata(docId); - if (docMeta && docMeta.fields.layout.gptRubric) { - return docMeta.fields.layout.gptRubric; - } else { + /** + * Generate or retrieve a rubric for evaluating the user's description of a document + */ + private async generateRubric(doc: Doc, description: string): Promise<string> { + // Check if we already have a cached rubric + const existingRubric = StrCast(doc.gptRubric); + if (existingRubric) { + return existingRubric; + } + + // Generate new rubric using GPT + try { const rubric = await gptAPICall(description, GPTCallType.MAKERUBRIC); if (rubric) { - await this._docManager.editDocumentField(docId, 'layout.gptRubric', rubric); + // Cache the rubric on the document + doc.gptRubric = rubric; } return rubric || ''; + } catch (error) { + console.error('Failed to generate rubric:', error); + return ''; + } + } + + /** + * Randomly select a document for the quiz and highlight it (like GPTPopup does) + */ + private selectRandomDocument(): Doc | null { + // Ensure we have documents initialized + this._docManager.initializeDocuments(); + const allDocIds = this._docManager.docIds; + + if (allDocIds.length === 0) { + return null; } + + // Select a random document ID + const randomIndex = Math.floor(Math.random() * allDocIds.length); + const randomDocId = allDocIds[randomIndex]; + + // Get the actual document + const randomDoc = this._docManager.getDocument(randomDocId) || null; + + if (randomDoc) { + // Highlight the selected document, exactly like GPTPopup does + const docView = DocumentView.getDocumentView(randomDoc); + if (docView) { + docView.select(false); // false means don't extend selection, replaces current selection + console.log(`[TakeQuizTool] Selected and highlighted document: ${randomDoc.title || 'Untitled'}`); + } + } + + return randomDoc; } async execute(args: ParametersType<typeof parameterRules>): Promise<Observation[]> { const chunkId = uuidv4(); try { - const allDocIds = this._docManager.docIds; - const randomDocId = allDocIds[Math.floor(Math.random() * allDocIds.length)]; - const docMeta = this._docManager.extractDocumentMetadata(randomDocId); + switch (args.action) { + case 'start': + case 'next': + return await this.startQuiz(chunkId); + + case 'answer': + if (!args.userAnswer) { + return [{ + type: 'text', + text: `<chunk chunk_id="${chunkId}" chunk_type="error"> +userAnswer is required when action is "answer" +</chunk>` + }]; + } + return await this.evaluateAnswer(chunkId, args.userAnswer); + + default: + return [{ + type: 'text', + text: `<chunk chunk_id="${chunkId}" chunk_type="error"> +Unknown action: ${args.action}. Use "start", "answer", or "next" +</chunk>` + }]; + } + } catch (err) { + return [{ + type: 'text', + text: `<chunk chunk_id="${chunkId}" chunk_type="error"> +Quiz operation failed: ${err instanceof Error ? err.message : err} +</chunk>` + }]; + } + } - if (!docMeta) throw new Error('Randomly selected document metadata is undefined'); + /** + * Start a new quiz with a random document + */ + private async startQuiz(chunkId: string): Promise<Observation[]> { + const doc = this.selectRandomDocument(); + + if (!doc) { + return [{ + type: 'text', + text: `<chunk chunk_id="${chunkId}" chunk_type="error"> +No documents available for quiz. Please ensure you have documents in your collection. +</chunk>` + }]; + } - const description = docMeta.fields.layout.description.replace(/\n/g, ' ').trim(); - const rubric = await this.generateRubric(randomDocId, description); + this._currentQuizDoc = doc; + + // Get document description for the quiz question + try { + this._currentQuizDescription = await Doc.getDescription(doc); + + return [{ + type: 'text', + text: `<chunk chunk_id="${chunkId}" chunk_type="quiz_question"> +🎯 **Quiz Question** + +I've randomly selected a document for you to describe. Please provide your answer when you're ready. + +**Document:** ${doc.title || 'Untitled'} + +**Content to describe:** ${this._currentQuizDescription} + +**Instructions:** Please provide your description or explanation of this document content. I'll evaluate your answer once you submit it. - const prompt = ` - Question: ${description}; - UserAnswer: ${args.userAnswer}; - Rubric: ${rubric} - `; +**Status:** Waiting for your answer... +</chunk>` + }]; + } catch (error) { + return [{ + type: 'text', + text: `<chunk chunk_id="${chunkId}" chunk_type="error"> +Failed to get description for document "${doc.title}": ${error} +</chunk>` + }]; + } + } + + /** + * Evaluate the user's answer against the current quiz document + */ + private async evaluateAnswer(chunkId: string, userAnswer: string): Promise<Observation[]> { + if (!this._currentQuizDoc || !this._currentQuizDescription) { + return [{ + type: 'text', + text: `<chunk chunk_id="${chunkId}" chunk_type="error"> +No active quiz question. Please use action "start" to begin a quiz. +</chunk>` + }]; + } + try { + // Generate or get rubric for evaluation + const rubric = await this.generateRubric(this._currentQuizDoc, this._currentQuizDescription); + + // Prepare prompt for GPT evaluation + const prompt = `Question: ${this._currentQuizDescription}; +UserAnswer: ${userAnswer}; +Rubric: ${rubric}`; + + // Get GPT evaluation const evaluation = await gptAPICall(prompt, GPTCallType.QUIZDOC); - return [ - { - type: 'text', - text: `<chunk chunk_id="${chunkId}" chunk_type="quiz_result"> -Evaluation result: ${evaluation || 'GPT provided no answer'}. -Document evaluated: "${docMeta.title}" -</chunk>`, - }, - ]; - } catch (err) { - return [ - { - type: 'text', - text: `<chunk chunk_id="${chunkId}" chunk_type="error"> -Quiz evaluation failed: ${err instanceof Error ? err.message : err} -</chunk>`, - }, - ]; + return [{ + type: 'text', + text: `<chunk chunk_id="${chunkId}" chunk_type="quiz_evaluation"> +📝 **Quiz Evaluation Complete** + +**Document:** ${this._currentQuizDoc.title || 'Untitled'} + +**Your Answer:** ${userAnswer} + +**Evaluation:** ${evaluation || 'GPT provided no evaluation'} + +**Status:** Quiz answer evaluated. If you'd like another quiz question, please ask me to give you the next question. +</chunk>` + }]; + } catch (error) { + return [{ + type: 'text', + text: `<chunk chunk_id="${chunkId}" chunk_type="error"> +Failed to evaluate answer: ${error} +</chunk>` + }]; } } } diff --git a/src/client/views/nodes/chatbot/tools/UIControlTool.ts b/src/client/views/nodes/chatbot/tools/UIControlTool.ts new file mode 100644 index 000000000..252e77956 --- /dev/null +++ b/src/client/views/nodes/chatbot/tools/UIControlTool.ts @@ -0,0 +1,566 @@ +import { Doc, DocListCast } from '../../../../../fields/Doc'; +import { ScriptField } from '../../../../../fields/ScriptField'; +import { Cast, PromiseValue } from '../../../../../fields/Types'; +import { InkInkTool, InkTool } from '../../../../../fields/InkField'; +import { MainView } from '../../../MainView'; +import { DocumentView } from '../../DocumentView'; +import { RichTextMenu } from '../../formattedText/RichTextMenu'; +import { PropertiesView } from '../../../PropertiesView'; +import { CollectionViewType } from '../../../../documents/DocumentTypes'; +import { ParametersType, ToolInfo } from '../types/tool_types'; +import { Observation } from '../types/types'; +import { BaseTool } from './BaseTool'; + +const uiControlParams = [ + { + name: 'action', + type: 'string', + description: 'The UI action to perform. Options: "open_tab" (Files, Tools, Imports), "close_tab" (closes current sidebar tab), "select_tool" (pen, highlighter, eraser, text, circle, etc.), "change_font_size", "select_font", "switch_view", "toggle_tags", "toggle_properties_submenu", "toggle_properties", "toggle_header"', + required: true, + }, + { + name: 'target', + type: 'string', + description: 'The target of the action. For open_tab: "Files", "Tools", "Imports", "Trails", "Search", "Properties". For select_tool: "pen", "highlighter", "write", "math", "eraser", "text", "ink", "none". For select_font: "Arial", "Comic Sans MS", etc. For change_font_size: number as string. For switch_view: "freeform", "card", "carousel", "stacking", etc. For toggle_properties_submenu: "options", "fields", "appearance", "layout", "sharing", etc.', + required: false, + }, +] as const; + +type UIControlToolParamsType = typeof uiControlParams; + +const uiControlToolInfo: ToolInfo<UIControlToolParamsType> = { + name: 'uiControl', + citationRules: 'No citation needed for UI control actions.', + parameterRules: uiControlParams, + description: 'Control the Dash UI by opening/closing tabs, selecting tools, changing fonts, switching views, toggling tags, and managing properties sub-menus. Supports: tab management (Files, Tools, Properties, etc.), tool dropdowns (ink button opens pen/highlighter/write/math options, text button opens font/size/color options), font changes, view switching (freeform, card, etc.), tag visibility toggle, and properties panel sub-menu control (options, fields, appearance, etc.).', +}; + +export class UIControlTool extends BaseTool<UIControlToolParamsType> { + constructor() { + super(uiControlToolInfo); + } + + async execute(args: ParametersType<UIControlToolParamsType>): Promise<Observation[]> { + const { action, target } = args; + + try { + let result = ''; + + switch (action) { + case 'open_tab': + result = await this.openTab(target || ''); + break; + case 'close_tab': + result = this.closeTab(); + break; + case 'select_tool': + result = await this.selectTool(target || ''); + break; + case 'change_font_size': + result = await this.changeFontSize(String(target || '')); + break; + case 'select_font': + result = await this.selectFont(target || ''); + break; + case 'toggle_properties': + result = this.toggleProperties(); + break; + case 'toggle_header': + result = this.toggleHeader(); + break; + case 'switch_view': + result = this.switchView(target || ''); + break; + case 'toggle_tags': + result = this.toggleTags(); + break; + case 'toggle_properties_submenu': + result = this.togglePropertiesSubmenu(target || ''); + break; + default: + result = `Unknown action: ${action}`; + } + + return [ + { + type: 'text', + text: result, + }, + ]; + } catch (error) { + console.error('UIControlTool error:', error); + return [ + { + type: 'text', + text: `Error performing UI action: ${error}`, + }, + ]; + } + } + + private async openTab(tab: string): Promise<string> { + console.log(`[UIControlTool] Attempting to open tab: ${tab}`); + + const mainView = MainView.Instance; + if (!mainView) { + console.error('[UIControlTool] MainView.Instance is not available'); + return 'MainView not available'; + } + + const normalizedTab = tab.toLowerCase(); + console.log(`[UIControlTool] Normalized tab name: ${normalizedTab}`); + + try { + switch (normalizedTab) { + case 'files': + case 'filesystem': + console.log('[UIControlTool] Trying to open Files tab'); + const sidebarMenu = Doc.MyLeftSidebarMenu; + console.log('[UIControlTool] MyLeftSidebarMenu:', sidebarMenu); + + if (sidebarMenu?.data) { + const menuItems = DocListCast(sidebarMenu.data); + console.log('[UIControlTool] Menu items count:', menuItems.length); + + const filesBtn = menuItems.find(d => d.target === Doc.MyFilesystem); + console.log('[UIControlTool] Files button found:', !!filesBtn); + + if (filesBtn) { + mainView.selectLeftSidebarButton(filesBtn); + return 'Opened Files tab'; + } else { + return 'Files button not found in sidebar menu'; + } + } else { + return 'Sidebar menu data not available'; + } + + case 'tools': + console.log('[UIControlTool] Trying to open Tools tab'); + const toolsBtn = DocListCast(Doc.MyLeftSidebarMenu?.data).find(d => d.target === Doc.MyTools); + console.log('[UIControlTool] Tools button found:', !!toolsBtn); + + if (toolsBtn) { + mainView.selectLeftSidebarButton(toolsBtn); + return 'Opened Tools tab'; + } else { + return 'Tools button not found in sidebar menu'; + } + + case 'imports': + console.log('[UIControlTool] Trying to open Imports tab'); + const importBtn = DocListCast(Doc.MyLeftSidebarMenu?.data).find(d => d.target === Doc.MyImports); + console.log('[UIControlTool] Import button found:', !!importBtn); + + if (importBtn) { + mainView.selectLeftSidebarButton(importBtn); + return 'Opened Imports tab'; + } else { + return 'Imports button not found in sidebar menu'; + } + + case 'trails': + case 'presentations': + console.log('[UIControlTool] Trying to open Trails tab'); + const trailsBtn = DocListCast(Doc.MyLeftSidebarMenu?.data).find(d => d.target === Doc.MyTrails); + console.log('[UIControlTool] Trails button found:', !!trailsBtn); + + if (trailsBtn) { + mainView.selectLeftSidebarButton(trailsBtn); + return 'Opened Trails/Presentations tab'; + } else { + return 'Trails button not found in sidebar menu'; + } + + case 'search': + console.log('[UIControlTool] Trying to open Search tab'); + const searchBtn = DocListCast(Doc.MyLeftSidebarMenu?.data).find(d => d.target === Doc.MySearcher); + console.log('[UIControlTool] Search button found:', !!searchBtn); + + if (searchBtn) { + mainView.selectLeftSidebarButton(searchBtn); + return 'Opened Search tab'; + } else { + return 'Search button not found in sidebar menu'; + } + + case 'properties': + case 'property': + console.log('[UIControlTool] Trying to open Properties panel'); + // Properties is not a sidebar tab, it's a right-side panel + mainView.togglePropertiesFlyout(); + return 'Opened Properties panel'; + + default: + return `Unknown tab: ${tab}. Available: Files, Tools, Imports, Trails, Search, Properties`; + } + } catch (error) { + console.error(`[UIControlTool] Error opening tab ${tab}:`, error); + return `Error opening tab ${tab}: ${error}`; + } + } + + private closeTab(): string { + const mainView = MainView.Instance; + if (!mainView) { + return 'MainView not available'; + } + + try { + mainView.closeFlyout(); + return 'Closed sidebar tab'; + } catch (error) { + console.error('[UIControlTool] Error closing tab:', error); + return `Error closing tab: ${error}`; + } + } + + private async selectTool(tool: string): Promise<string> { + const normalizedTool = tool.toLowerCase(); + console.log(`[UIControlTool] Selecting tool: ${normalizedTool}`); + + try { + switch (normalizedTool) { + case 'pen': + case 'pen-nib': + Doc.ActiveInk = InkInkTool.Pen; + Doc.ActiveTool = InkTool.Ink; + console.log(`[UIControlTool] Set ActiveTool to ${Doc.ActiveTool}, ActiveInk to ${Doc.ActiveInk}`); + return 'Selected pen tool'; + + case 'highlighter': + case 'highlight': + Doc.ActiveInk = InkInkTool.Highlight; + Doc.ActiveTool = InkTool.Ink; + console.log(`[UIControlTool] Set ActiveTool to ${Doc.ActiveTool}, ActiveInk to ${Doc.ActiveInk}`); + return 'Selected highlighter tool'; + + case 'write': + case 'writing': + case 'handwriting': + Doc.ActiveInk = InkInkTool.Write; + Doc.ActiveTool = InkTool.Ink; + console.log(`[UIControlTool] Set ActiveTool to ${Doc.ActiveTool}, ActiveInk to ${Doc.ActiveInk}`); + return 'Selected handwriting tool'; + + case 'math': + case 'calculator': + Doc.ActiveInk = InkInkTool.Math; + Doc.ActiveTool = InkTool.Ink; + console.log(`[UIControlTool] Set ActiveTool to ${Doc.ActiveTool}, ActiveInk to ${Doc.ActiveInk}`); + return 'Selected math tool'; + + case 'eraser': + Doc.ActiveTool = InkTool.Eraser; + console.log(`[UIControlTool] Set ActiveTool to ${Doc.ActiveTool}`); + return 'Selected eraser tool'; + + case 'text': + case 'text tool': + case 'text button': + // Text button should open the text formatting dropdown, not just select text tool + // First ensure Tools panel is open + const mainView = MainView.Instance; + if (mainView) { + const toolsBtn = DocListCast(Doc.MyLeftSidebarMenu?.data).find(d => d.target === Doc.MyTools); + if (toolsBtn) { + mainView.selectLeftSidebarButton(toolsBtn); + console.log(`[UIControlTool] Opened Tools panel for text button`); + + // The text dropdown should open automatically when Tools panel is opened + // and text documents are selected. This matches existing UI behavior. + return 'Opened text button dropdown in Tools panel. You can now access font, size, color, and other text formatting options.'; + } + } + return 'Could not open text button dropdown - Tools panel not available'; + + case 'ink': + case 'ink tool': + case 'ink button': + // Ink button should open the ink tools dropdown with pen, highlighter, write, math options + const mainViewInk = MainView.Instance; + if (mainViewInk) { + const toolsBtnInk = DocListCast(Doc.MyLeftSidebarMenu?.data).find(d => d.target === Doc.MyTools); + if (toolsBtnInk) { + mainViewInk.selectLeftSidebarButton(toolsBtnInk); + console.log(`[UIControlTool] Opened Tools panel for ink button`); + + // The ink dropdown should open automatically when Tools panel is opened + // This matches existing UI behavior for MultiToggleButton types + return 'Opened ink button dropdown in Tools panel. You can now access pen, highlighter, write, and math tools.'; + } + } + return 'Could not open ink button dropdown - Tools panel not available'; + + case 'none': + case 'select': + case 'selection': + Doc.ActiveTool = InkTool.None; + console.log(`[UIControlTool] Set ActiveTool to ${Doc.ActiveTool} (selection tool)`); + return 'Selected selection tool'; + + case 'circle': + case 'shape': + // TODO: Implement shape tool selection when we find the API + return 'Shape tool selection not yet implemented'; + + default: + return `Unknown tool: ${tool}. Available tools: pen, highlighter, write, math, eraser, text, circle`; + } + } catch (error) { + console.error('[UIControlTool] Error selecting tool:', error); + return `Error selecting tool: ${error}`; + } + } + + private async selectFont(fontName: string): Promise<string> { + if (!fontName) { + return 'Font name is required'; + } + + try { + // First ensure text tool is selected + Doc.ActiveTool = InkTool.None; + + // Validate font name + const validFonts = ['Roboto', 'Roboto Mono', 'Nunito', 'Times New Roman', 'Arial', 'Georgia', 'Comic Sans MS', 'Tahoma', 'Impact', 'Crimson Text', 'Math']; + const normalizedFont = validFonts.find(font => + font.toLowerCase() === fontName.toLowerCase() || + font.toLowerCase().includes(fontName.toLowerCase()) + ); + + if (!normalizedFont) { + return `Font "${fontName}" not found. Available fonts: ${validFonts.join(', ')}`; + } + + // Try to set font using RichTextMenu + if (RichTextMenu.Instance) { + RichTextMenu.Instance.setFontField(normalizedFont, 'fontFamily'); + return `Selected font: ${normalizedFont}`; + } else { + // Fallback: Set on user document + Doc.UserDoc().fontFamily = normalizedFont; + return `Set default font to: ${normalizedFont}`; + } + } catch (error) { + console.error('[UIControlTool] Error selecting font:', error); + return `Error selecting font: ${error}`; + } + } + + private changeFontSize(size: string): string { + const fontSize = parseInt(size); + if (isNaN(fontSize) || fontSize < 1 || fontSize > 200) { + return 'Invalid font size. Please provide a number between 1 and 200.'; + } + + // Get selected text documents + const selectedViews = DocumentView.Selected(); + if (selectedViews.length === 0) { + return 'No text document selected. Please select a text document first.'; + } + + let changedCount = 0; + selectedViews.forEach(view => { + const doc = view.Document; + if (doc.type === 'rich text' || doc.type === 'text') { + // TODO: Find the correct property for font size + // This is a placeholder - need to find actual implementation + doc.$fontSize = fontSize; + changedCount++; + } + }); + + if (changedCount > 0) { + return `Changed font size to ${fontSize} for ${changedCount} document(s)`; + } else { + return 'No text documents selected to change font size'; + } + } + + private toggleProperties(): string { + MainView.Instance?.togglePropertiesFlyout(); + return 'Toggled properties panel'; + } + + private toggleHeader(): string { + MainView.Instance?.toggleTopBar(); + return 'Toggled header bar'; + } + + private switchView(viewType: string): string { + try { + // Use the existing setView function from globalScripts, which handles view switching properly + const normalizedView = viewType.toLowerCase(); + let mappedViewType: string; + + // Map common view names to CollectionViewType values + switch (normalizedView) { + case 'freeform': + case 'free form': + mappedViewType = CollectionViewType.Freeform; + break; + case 'card': + case 'card view': + mappedViewType = CollectionViewType.Card; + break; + case 'carousel': + mappedViewType = CollectionViewType.Carousel; + break; + case '3d carousel': + case 'carousel3d': + mappedViewType = CollectionViewType.Carousel3D; + break; + case 'stacking': + case 'stack': + mappedViewType = CollectionViewType.Stacking; + break; + case 'grid': + mappedViewType = CollectionViewType.Grid; + break; + case 'tree': + mappedViewType = CollectionViewType.Tree; + break; + case 'masonry': + mappedViewType = CollectionViewType.Masonry; + break; + case 'notetaking': + case 'note taking': + mappedViewType = CollectionViewType.NoteTaking; + break; + case 'schema': + mappedViewType = CollectionViewType.Schema; + break; + default: + return `Unknown view type: ${viewType}. Available: freeform, card, carousel, 3d carousel, stacking, grid, tree, masonry, notetaking, schema`; + } + + // Apply view change directly to selected document, mirroring setView function logic + const selected = DocumentView.Selected().lastElement(); + if (!selected) { + return 'No documents selected to switch view'; + } + + // Apply the view change (like setView function does) + if (selected.Document.type === 'collection') { + selected.Document._type_collection = mappedViewType; + return `Successfully switched to ${viewType} view`; + } else { + return 'Selected document is not a collection, cannot switch view'; + } + } catch (error) { + console.error('[UIControlTool] Error switching view:', error); + return `Error switching view: ${error}`; + } + } + + private toggleTags(): string { + try { + // Use the exact same logic as DocumentButtonBar keywordButton + const selectedDocs = DocumentView.Selected(); + if (selectedDocs.length === 0) { + return 'No documents selected to toggle tags'; + } + + // Check if ANY document is currently showing tags (like DocumentButtonBar does) + const showing = selectedDocs.some(dv => dv.showTags); + + // Set ALL documents to the OPPOSITE state (like DocumentButtonBar does) + selectedDocs.forEach(dv => { + dv.layoutDoc._layout_showTags = !showing; + }); + + const newState = !showing; + return `${newState ? 'Enabled' : 'Disabled'} tag display for ${selectedDocs.length} document(s)`; + } catch (error) { + console.error('[UIControlTool] Error toggling tags:', error); + return `Error toggling tags: ${error}`; + } + } + + private togglePropertiesSubmenu(submenuName: string): string { + try { + const propertiesView = PropertiesView.Instance; + if (!propertiesView) { + return 'Properties panel is not available'; + } + + const normalizedName = submenuName.toLowerCase(); + let toggledSubmenu = ''; + + switch (normalizedName) { + case 'options': + propertiesView.openOptions = !propertiesView.openOptions; + toggledSubmenu = 'Options'; + break; + case 'fields': + case 'fields & tags': + propertiesView.openFields = !propertiesView.openFields; + toggledSubmenu = 'Fields & Tags'; + break; + case 'appearance': + propertiesView.openAppearance = !propertiesView.openAppearance; + toggledSubmenu = 'Appearance'; + break; + case 'layout': + propertiesView.openLayout = !propertiesView.openLayout; + toggledSubmenu = 'Layout'; + break; + case 'sharing': + propertiesView.openSharing = !propertiesView.openSharing; + toggledSubmenu = 'Sharing'; + break; + case 'links': + propertiesView.openLinks = !propertiesView.openLinks; + toggledSubmenu = 'Links'; + break; + case 'contexts': + case 'other contexts': + propertiesView.openContexts = !propertiesView.openContexts; + toggledSubmenu = 'Other Contexts'; + break; + case 'filters': + propertiesView.openFilters = !propertiesView.openFilters; + toggledSubmenu = 'Filters'; + break; + case 'transform': + propertiesView.openTransform = !propertiesView.openTransform; + toggledSubmenu = 'Transform'; + break; + case 'firefly': + propertiesView.openFirefly = !propertiesView.openFirefly; + toggledSubmenu = 'Firefly'; + break; + case 'styling': + propertiesView.openStyling = !propertiesView.openStyling; + toggledSubmenu = 'Styling'; + break; + default: + return `Unknown submenu: ${submenuName}. Available: options, fields, appearance, layout, sharing, links, contexts, filters, transform, firefly, styling`; + } + + const currentState = this.getSubmenuState(propertiesView, normalizedName); + return `${currentState ? 'Opened' : 'Closed'} ${toggledSubmenu} submenu in Properties panel`; + } catch (error) { + console.error('[UIControlTool] Error toggling properties submenu:', error); + return `Error toggling properties submenu: ${error}`; + } + } + + private getSubmenuState(propertiesView: PropertiesView, submenuName: string): boolean { + switch (submenuName) { + case 'options': return propertiesView.openOptions; + case 'fields': return propertiesView.openFields; + case 'appearance': return propertiesView.openAppearance; + case 'layout': return propertiesView.openLayout; + case 'sharing': return propertiesView.openSharing; + case 'links': return propertiesView.openLinks; + case 'contexts': return propertiesView.openContexts; + case 'filters': return propertiesView.openFilters; + case 'transform': return propertiesView.openTransform; + case 'firefly': return propertiesView.openFirefly; + case 'styling': return propertiesView.openStyling; + default: return false; + } + } +}
\ No newline at end of file diff --git a/src/client/views/nodes/chatbot/utils/AgentDocumentManager.ts b/src/client/views/nodes/chatbot/utils/AgentDocumentManager.ts index a96d93a25..485430403 100644 --- a/src/client/views/nodes/chatbot/utils/AgentDocumentManager.ts +++ b/src/client/views/nodes/chatbot/utils/AgentDocumentManager.ts @@ -165,6 +165,21 @@ export class AgentDocumentManager { } /** + * Initialize documents based on current mode (canvas or linked) + * This should be called by tools instead of hardcoding initializeFindDocsFreeform + */ + @action + public initializeDocuments() { + if (this._useCanvasMode) { + console.log('[AgentDocumentManager] Initializing canvas documents (canvas mode enabled)'); + this.initializeCanvasDocuments(); + } else { + console.log('[AgentDocumentManager] Initializing linked documents (canvas mode disabled)'); + this.initializeFindDocsFreeform(); + } + } + + /** * Initialize documents from the entire canvas */ @action |