diff options
5 files changed, 945 insertions, 5 deletions
diff --git a/src/client/views/nodes/chatbot/agentsystem/Agent.ts b/src/client/views/nodes/chatbot/agentsystem/Agent.ts index e93fb87db..b5cdb8cf1 100644 --- a/src/client/views/nodes/chatbot/agentsystem/Agent.ts +++ b/src/client/views/nodes/chatbot/agentsystem/Agent.ts @@ -10,6 +10,7 @@ import { CalculateTool } from '../tools/CalculateTool'; //import { CreateAnyDocumentTool } from '../tools/CreateAnyDocTool'; import { CreateDocTool } from '../tools/CreateDocumentTool'; import { DataAnalysisTool } from '../tools/DataAnalysisTool'; +import { DocumentMetadataTool } from '../tools/DocumentMetadataTool'; import { ImageCreationTool } from '../tools/ImageCreationTool'; import { NoTool } from '../tools/NoTool'; import { SearchTool } from '../tools/SearchTool'; @@ -89,6 +90,7 @@ export class Agent { createDoc: new CreateDocTool(addLinkedDoc), // createAnyDocument: new CreateAnyDocumentTool(addLinkedDoc), // dictionary: new DictionaryTool(), + documentMetadata: new DocumentMetadataTool(this), }; } @@ -459,6 +461,14 @@ export class Agent { } console.log(actionInput); + // Special handling for documentMetadata tool with numeric fieldValue + if (action === 'documentMetadata' && + 'fieldValue' in actionInput && + typeof actionInput.fieldValue === 'number') { + // Convert number to string to pass validation + actionInput.fieldValue = String(actionInput.fieldValue); + } + for (const param of this.tools[action].parameterRules) { // Check if the parameter is required and missing in the input if (param.required && !(param.name in actionInput) && !this.tools[action].inputValidator(actionInput)) { @@ -483,4 +493,19 @@ export class Agent { return await tool.execute(actionInput); } + + /** + * Reinitializes the DocumentMetadataTool with a direct reference to the ChatBox instance. + * This ensures that the tool can properly access the ChatBox document and find related documents. + * + * @param chatBox The ChatBox instance to pass to the DocumentMetadataTool + */ + public reinitializeDocumentMetadataTool(chatBox: any): void { + if (this.tools && this.tools.documentMetadata) { + this.tools.documentMetadata = new DocumentMetadataTool(chatBox); + console.log('Agent: Reinitialized DocumentMetadataTool with ChatBox instance'); + } else { + console.warn('Agent: Could not reinitialize DocumentMetadataTool - tool not found'); + } + } } diff --git a/src/client/views/nodes/chatbot/agentsystem/prompts.ts b/src/client/views/nodes/chatbot/agentsystem/prompts.ts index dda6d44ef..e551ef830 100644 --- a/src/client/views/nodes/chatbot/agentsystem/prompts.ts +++ b/src/client/views/nodes/chatbot/agentsystem/prompts.ts @@ -210,7 +210,7 @@ export function getReactPrompt(tools: BaseTool<ReadonlyArray<Parameter>>[], summ <answer> <grounded_text citation_index="1">**The 2022 World Cup** saw Argentina crowned champions, with **Lionel Messi** leading his team to victory, marking a historic moment in sports.</grounded_text> <grounded_text citation_index="2">**Qatar** experienced a **40% increase in tourism** during the World Cup, welcoming over **1.5 million visitors**, significantly boosting its economy.</grounded_text> - <normal_text>Moments like **Messi’s triumph** often become ingrained in the legacy of World Cups, immortalizing these tournaments in both sports and cultural memory. The **long-term implications** of the World Cup on Qatar's **economy, tourism**, and **global image** remain important areas of interest as the country continues to build on the momentum generated by hosting this prestigious event.</normal_text> + <normal_text>Moments like **Messi's triumph** often become ingrained in the legacy of World Cups, immortalizing these tournaments in both sports and cultural memory. The **long-term implications** of the World Cup on Qatar's **economy, tourism**, and **global image** remain important areas of interest as the country continues to build on the momentum generated by hosting this prestigious event.</normal_text> <citations> <citation index="1" chunk_id="1234" type="text">Key moments from the 2022 World Cup.</citation> <citation index="2" chunk_id="5678" type="url"></citation> @@ -218,7 +218,7 @@ export function getReactPrompt(tools: BaseTool<ReadonlyArray<Parameter>>[], summ <follow_up_questions> <question>What long-term effects has the World Cup had on Qatar's economy and infrastructure?</question> <question>Can you compare Qatar's tourism numbers with previous World Cup hosts?</question> - <question>How has Qatar’s image on the global stage evolved post-World Cup?</question> + <question>How has Qatar's image on the global stage evolved post-World Cup?</question> </follow_up_questions> <loop_summary> The assistant first used the RAG tool to extract key moments from the user documents about the 2022 World Cup. Then, the assistant utilized the website scraping tool to gather data on Qatar's tourism impact. Both tools provided valuable information, and no additional tools were needed. diff --git a/src/client/views/nodes/chatbot/chatboxcomponents/ChatBox.tsx b/src/client/views/nodes/chatbot/chatboxcomponents/ChatBox.tsx index 6e9307d37..065c2780c 100644 --- a/src/client/views/nodes/chatbot/chatboxcomponents/ChatBox.tsx +++ b/src/client/views/nodes/chatbot/chatboxcomponents/ChatBox.tsx @@ -43,6 +43,7 @@ import MessageComponentBox from './MessageComponent'; import { ProgressBar } from './ProgressBar'; import { OpenWhere } from '../../OpenWhere'; import { Upload } from '../../../../../server/SharedMediaTypes'; +import { DocumentMetadataTool } from '../tools/DocumentMetadataTool'; dotenv.config(); @@ -106,7 +107,21 @@ export class ChatBox extends ViewBoxAnnotatableComponent<FieldViewProps>() { this.vectorstore_id = StrCast(this.dataDoc.vectorstore_id); } this.vectorstore = new Vectorstore(this.vectorstore_id, this.retrieveDocIds); - this.agent = new Agent(this.vectorstore, this.retrieveSummaries, this.retrieveFormattedHistory, this.retrieveCSVData, this.addLinkedUrlDoc, this.createImageInDash, this.createDocInDash, this.createCSVInDash); + this.agent = new Agent( + this.vectorstore, + this.retrieveSummaries, + this.retrieveFormattedHistory, + this.retrieveCSVData, + this.addLinkedUrlDoc, + this.createImageInDash, + this.createDocInDash, + this.createCSVInDash + ); + + // Reinitialize the DocumentMetadataTool with a direct reference to this ChatBox instance + // This ensures the tool can properly access documents in the same Freeform view + this.agent.reinitializeDocumentMetadataTool(this); + this.messagesRef = React.createRef<HTMLDivElement>(); // Reaction to update dataDoc when chat history changes @@ -408,7 +423,7 @@ export class ChatBox extends ViewBoxAnnotatableComponent<FieldViewProps>() { if (doc) { LinkManager.Instance.addLink(Docs.Create.LinkDocument(this.Document, doc)); this._props.addDocument?.(doc); - DocumentManager.Instance.showDocument(doc, { willZoomCentered: true }, () => {}).then(() => this.addCSVForAnalysis(doc, id)); + DocumentManager.Instance.showDocument(doc, { willZoomCentered: true }, () => this.addCSVForAnalysis(doc, id)); } }); diff --git a/src/client/views/nodes/chatbot/tools/DocumentMetadataTool.ts b/src/client/views/nodes/chatbot/tools/DocumentMetadataTool.ts new file mode 100644 index 000000000..9c3a1fbb5 --- /dev/null +++ b/src/client/views/nodes/chatbot/tools/DocumentMetadataTool.ts @@ -0,0 +1,900 @@ +import { Doc, FieldType } from '../../../../../fields/Doc'; +import { DocData } from '../../../../../fields/DocSymbols'; +import { Observation } from '../types/types'; +import { ParametersType, ToolInfo } from '../types/tool_types'; +import { BaseTool } from './BaseTool'; +import { DocumentOptions } from '../../../../documents/Documents'; +import { CollectionFreeFormDocumentView } from '../../../nodes/CollectionFreeFormDocumentView'; +import { v4 as uuidv4 } from 'uuid'; +import { LinkManager } from '../../../../util/LinkManager'; +import { DocCast, StrCast } from '../../../../../fields/Types'; + +// Parameter definitions for DocumentMetadataTool +const documentMetadataToolParams = [ + { + name: 'action', + type: 'string', + description: 'The action to perform: "get" for retrieving metadata, "edit" for modifying document metadata, or "list" for a simple document list', + required: true, + }, + { + name: 'documentId', + type: 'string', + description: 'The ID of the document to get metadata from or edit. If not provided when getting metadata, information about all documents will be returned.', + required: false, + }, + { + name: 'fieldName', + type: 'string', + description: 'When editing, the name of the field to modify (with or without leading underscore).', + required: false, + }, + { + name: 'fieldValue', + type: 'string', + description: 'When editing, the new value to set for the specified field. Numeric values will be converted to strings automatically.', + required: false, + }, + { + name: 'documentLocation', + type: 'string', + description: 'When editing, specify where to modify the field: "layout" (default), "data", or "auto" (determines automatically based on existing fields).', + required: false, + } +] as const; + +type DocumentMetadataToolParamsType = typeof documentMetadataToolParams; + +// Detailed description with usage guidelines for the DocumentMetadataTool +const toolDescription = `Extracts and modifies metadata from documents in the same Freeform view as the ChatBox. +This tool helps you work with document properties, understand available fields, and edit document metadata. + +The Dash document system organizes fields in two locations: +1. Layout documents: contain visual properties like position, dimensions, and appearance +2. Data documents: contain the actual content and document-specific data + +This tool provides the following capabilities: +- Get metadata from all documents in the current Freeform view +- Get metadata from a specific document +- Edit metadata fields on documents (in either layout or data documents) +- List all available documents in the current view +- Understand which fields are stored where (layout vs data document) +- Get detailed information about all available document fields`; + +// Extensive usage guidelines for the tool +const citationRules = `USAGE GUIDELINES: +To GET document metadata: +- Use action="get" with optional documentId to return metadata for one or all documents +- Returns field values, field definitions, and location information (layout vs data document) + +To EDIT document metadata: +- Use action="edit" with required documentId, fieldName, and fieldValue parameters +- Optionally specify documentLocation="layout"|"data"|"auto" (default is "auto") +- The tool will determine the correct document location automatically unless specified +- Field names can be provided with or without leading underscores (e.g., both "width" and "_width" work) +- Common fields like "width" and "height" are automatically mapped to "_width" and "_height" +- Numeric values are accepted for appropriate fields (width, height, etc.) +- The tool will apply the edit to the correct document (layout or data) + +To LIST available documents: +- Use action="list" to get a simple list of all documents in the current view +- This is useful when you need to identify documents before getting details or editing them + +Editing fields follows these rules: +1. If documentLocation is specified, the field is modified in that location +2. If documentLocation="auto", the tool checks if the field exists on layout or data document first +3. If the field doesn't exist in either document, it's added to the location specified or layout document by default +4. Fields with leading underscores are automatically handled correctly + +Examples: +- To list all documents: { action: "list" } +- To get all document metadata: { action: "get" } +- To get metadata for a specific document: { action: "get", documentId: "doc123" } +- To edit a field: { action: "edit", documentId: "doc123", fieldName: "backgroundColor", fieldValue: "#ff0000" } +- To edit a width property: { action: "edit", documentId: "doc123", fieldName: "width", fieldValue: 300 } +- To edit a field in the data document: { action: "edit", documentId: "doc123", fieldName: "text", fieldValue: "New content", documentLocation: "data" }`; + +const documentMetadataToolInfo: ToolInfo<DocumentMetadataToolParamsType> = { + name: 'documentMetadata', + description: toolDescription, + parameterRules: documentMetadataToolParams, + citationRules: citationRules, +}; + +/** + * A tool for extracting and modifying metadata from documents in a Freeform view. + * This tool collects metadata from both layout and data documents in a Freeform view + * and allows for editing document fields in the correct location. + */ +export class DocumentMetadataTool extends BaseTool<DocumentMetadataToolParamsType> { + private freeformView: any; + private chatBox: any; + private chatBoxDocument: Doc | null = null; + private documentsById: Map<string, Doc> = new Map(); + private layoutDocsById: Map<string, Doc> = new Map(); + private dataDocsById: Map<string, Doc> = new Map(); + private fieldMetadata: Record<string, any> = {}; + private readonly DOCUMENT_ID_FIELD = '_dash_document_id'; + + constructor(chatBox: any) { + super(documentMetadataToolInfo); + this.chatBox = chatBox; + + // Store a direct reference to the ChatBox document + if (chatBox && chatBox.Document) { + this.chatBoxDocument = chatBox.Document; + if (this.chatBoxDocument && this.chatBoxDocument.id) { + console.log('DocumentMetadataTool initialized with ChatBox Document:', this.chatBoxDocument.id); + } else { + console.log('DocumentMetadataTool initialized with ChatBox Document (no ID)'); + } + } else if (chatBox && chatBox.props && chatBox.props.Document) { + this.chatBoxDocument = chatBox.props.Document; + if (this.chatBoxDocument && this.chatBoxDocument.id) { + console.log('DocumentMetadataTool initialized with ChatBox props.Document:', this.chatBoxDocument.id); + } else { + console.log('DocumentMetadataTool initialized with ChatBox props.Document (no ID)'); + } + } else { + console.warn('DocumentMetadataTool initialized without valid ChatBox Document reference'); + } + + this.initializeFieldMetadata(); + } + + /** + * Extracts field metadata from DocumentOptions class + */ + private initializeFieldMetadata() { + // Parse DocumentOptions to extract field definitions + const documentOptionsInstance = new DocumentOptions(); + const documentOptionsEntries = Object.entries(documentOptionsInstance); + + for (const [fieldName, fieldInfo] of documentOptionsEntries) { + // Extract field information + const fieldData: Record<string, any> = { + name: fieldName, + withoutUnderscore: fieldName.startsWith('_') ? fieldName.substring(1) : fieldName, + description: '', + type: 'unknown', + required: false, + defaultValue: undefined, + possibleValues: [], + }; + + // Check if fieldInfo has description property (it's likely a FInfo instance) + if (fieldInfo && typeof fieldInfo === 'object' && 'description' in fieldInfo) { + fieldData.description = fieldInfo.description; + + // Extract field type if available + if ('fieldType' in fieldInfo) { + fieldData.type = fieldInfo.fieldType; + } + + // Extract possible values if available + if ('values' in fieldInfo && Array.isArray(fieldInfo.values)) { + fieldData.possibleValues = fieldInfo.values; + } + } + + this.fieldMetadata[fieldName] = fieldData; + } + } + + /** + * Gets all documents in the same Freeform view as the ChatBox + * Uses the LinkManager to get all linked documents, similar to how ChatBox does it + */ + private findDocumentsInFreeformView() { + // Reset collections + this.documentsById.clear(); + this.layoutDocsById.clear(); + this.dataDocsById.clear(); + + try { + // Use the LinkManager approach which is proven to work in ChatBox + if (this.chatBoxDocument) { + console.log('Finding documents linked to ChatBox document with ID:', this.chatBoxDocument.id); + + // Get directly linked documents via LinkManager + const linkedDocs = LinkManager.Instance.getAllRelatedLinks(this.chatBoxDocument) + .map(d => DocCast(LinkManager.getOppositeAnchor(d, this.chatBoxDocument!))) + .map(d => DocCast(d?.annotationOn, d)) + .filter(d => d); + + console.log(`Found ${linkedDocs.length} linked documents via LinkManager`); + + // Process the linked documents + linkedDocs.forEach((doc: Doc) => { + if (doc) { + this.processDocument(doc); + } + }); + + // Include the ChatBox document itself + this.processDocument(this.chatBoxDocument); + + // If we have access to the Document's parent, try to find sibling documents + if (this.chatBoxDocument.parent) { + const parent = this.chatBoxDocument.parent; + console.log('Found parent document, checking for siblings'); + + // Check if parent is a Doc type and has a childDocs function + if (parent && typeof parent === 'object' && 'childDocs' in parent && typeof parent.childDocs === 'function') { + try { + const siblingDocs = parent.childDocs(); + if (Array.isArray(siblingDocs)) { + console.log(`Found ${siblingDocs.length} sibling documents via parent.childDocs()`); + siblingDocs.forEach((doc: Doc) => { + if (doc) { + this.processDocument(doc); + } + }); + } + } catch (e) { + console.warn('Error accessing parent.childDocs:', e); + } + } + } + } else if (this.chatBox && this.chatBox.linkedDocs) { + // If we have direct access to the linkedDocs computed property from ChatBox + console.log('Using ChatBox.linkedDocs directly'); + const linkedDocs = this.chatBox.linkedDocs; + if (Array.isArray(linkedDocs)) { + console.log(`Found ${linkedDocs.length} documents via ChatBox.linkedDocs`); + linkedDocs.forEach((doc: Doc) => { + if (doc) { + this.processDocument(doc); + } + }); + } + + // Process the ChatBox document if available + if (this.chatBox.Document) { + this.processDocument(this.chatBox.Document); + } + } else { + console.warn('No ChatBox document reference available for finding linked documents'); + } + + console.log(`DocumentMetadataTool found ${this.documentsById.size} total documents`); + + // If we didn't find any documents, try a fallback method + if (this.documentsById.size === 0 && this.chatBox) { + console.log('No documents found, trying fallback method'); + + // Try to access any field that might contain documents + if (this.chatBox.props && this.chatBox.props.documents) { + const documents = this.chatBox.props.documents; + if (Array.isArray(documents)) { + console.log(`Found ${documents.length} documents via ChatBox.props.documents`); + documents.forEach((doc: Doc) => { + if (doc) { + this.processDocument(doc); + } + }); + } + } + } + } catch (error) { + console.error('Error finding documents in Freeform view:', error); + } + } + + /** + * Process a document by ensuring it has an ID and adding it to the appropriate collections + * @param doc The document to process + */ + private processDocument(doc: Doc) { + // Ensure document has a persistent ID + const docId = this.ensureDocumentId(doc); + + // Only add if we haven't already processed this document + if (!this.documentsById.has(docId)) { + this.documentsById.set(docId, doc); + + // Get layout doc (the document itself or its layout) + const layoutDoc = Doc.Layout(doc); + if (layoutDoc) { + this.layoutDocsById.set(docId, layoutDoc); + } + + // Get data doc + const dataDoc = doc[DocData]; + if (dataDoc) { + this.dataDocsById.set(docId, dataDoc); + } + } + } + + /** + * Ensures a document has a persistent ID stored in its metadata + * @param doc The document to ensure has an ID + * @returns The document's ID + */ + private ensureDocumentId(doc: Doc): string { + let docId: string | undefined; + + // First try to get the ID from our custom field + if (doc[this.DOCUMENT_ID_FIELD]) { + docId = String(doc[this.DOCUMENT_ID_FIELD]); + return docId; + } + + // Try different ways to get a document ID + + // 1. Try the direct id property if it exists + if (doc.id && typeof doc.id === 'string') { + docId = doc.id; + } + // 2. Try doc._id if it exists + else if (doc._id && typeof doc._id === 'string') { + docId = doc._id; + } + // 3. Try doc.data?.id if it exists + else if (doc.data && typeof doc.data === 'object' && 'id' in doc.data && typeof doc.data.id === 'string') { + docId = doc.data.id; + } + // 4. If none of the above work, generate a UUID + else { + docId = uuidv4(); + console.log(`Generated new UUID for document with title: ${doc.title || 'Untitled'}`); + } + + // Store the ID in the document's metadata so it persists + try { + doc[this.DOCUMENT_ID_FIELD] = docId; + } catch (e) { + console.warn(`Could not assign ID to document property`, e); + } + + return docId; + } + + /** + * Extracts metadata from a specific document + * @param docId The ID of the document to extract metadata from + * @returns An object containing the document's metadata + */ + private extractDocumentMetadata(docId: string) { + const doc = this.documentsById.get(docId); + if (!doc) { + return null; + } + + const layoutDoc = this.layoutDocsById.get(docId); + const dataDoc = this.dataDocsById.get(docId); + + const metadata: Record<string, any> = { + id: docId, + title: doc.title || '', + type: doc.type || '', + fields: { + layout: {}, + data: {}, + }, + fieldLocationMap: {}, + }; + + // Process all known field definitions + Object.keys(this.fieldMetadata).forEach(fieldName => { + const fieldDef = this.fieldMetadata[fieldName]; + const strippedName = fieldName.startsWith('_') ? fieldName.substring(1) : fieldName; + + // Check if field exists on layout document + let layoutValue = undefined; + if (layoutDoc) { + layoutValue = layoutDoc[fieldName]; + if (layoutValue !== undefined) { + // Field exists on layout document + metadata.fields.layout[fieldName] = this.formatFieldValue(layoutValue); + metadata.fieldLocationMap[strippedName] = 'layout'; + } + } + + // Check if field exists on data document + let dataValue = undefined; + if (dataDoc) { + dataValue = dataDoc[fieldName]; + if (dataValue !== undefined) { + // Field exists on data document + metadata.fields.data[fieldName] = this.formatFieldValue(dataValue); + if (!metadata.fieldLocationMap[strippedName]) { + metadata.fieldLocationMap[strippedName] = 'data'; + } + } + } + + // For fields with stripped names (without leading underscore), + // also check if they exist on documents without the underscore + if (fieldName.startsWith('_')) { + const nonUnderscoreFieldName = fieldName.substring(1); + + if (layoutDoc) { + const nonUnderscoreLayoutValue = layoutDoc[nonUnderscoreFieldName]; + if (nonUnderscoreLayoutValue !== undefined) { + metadata.fields.layout[nonUnderscoreFieldName] = this.formatFieldValue(nonUnderscoreLayoutValue); + metadata.fieldLocationMap[nonUnderscoreFieldName] = 'layout'; + } + } + + if (dataDoc) { + const nonUnderscoreDataValue = dataDoc[nonUnderscoreFieldName]; + if (nonUnderscoreDataValue !== undefined) { + metadata.fields.data[nonUnderscoreFieldName] = this.formatFieldValue(nonUnderscoreDataValue); + if (!metadata.fieldLocationMap[nonUnderscoreFieldName]) { + metadata.fieldLocationMap[nonUnderscoreFieldName] = 'data'; + } + } + } + } + }); + + // Add common field aliases for easier discovery + // This helps users understand both width and _width refer to the same property + if (metadata.fields.layout._width !== undefined && metadata.fields.layout.width === undefined) { + metadata.fields.layout.width = metadata.fields.layout._width; + metadata.fieldLocationMap.width = 'layout'; + } + + if (metadata.fields.layout._height !== undefined && metadata.fields.layout.height === undefined) { + metadata.fields.layout.height = metadata.fields.layout._height; + metadata.fieldLocationMap.height = 'layout'; + } + + return metadata; + } + + /** + * Edits a specific field on a document + * @param docId The ID of the document to edit + * @param fieldName The name of the field to edit + * @param fieldValue The new value for the field + * @param documentLocation Where to edit the field: 'layout', 'data', or 'auto' + * @returns Object with success status, message, and additional information + */ + private editDocumentField(docId: string, fieldName: string, fieldValue: string, documentLocation: string = 'auto'): { + success: boolean; + message: string; + fieldName?: string; + originalFieldName?: string; + newValue?: any; + } { + // Normalize field name (handle with/without underscore) + let normalizedFieldName = fieldName.startsWith('_') ? fieldName : fieldName; + const strippedFieldName = fieldName.startsWith('_') ? fieldName.substring(1) : fieldName; + + // Handle common field name aliases (width → _width, height → _height) + // Many document fields use '_' prefix for layout properties + if (fieldName === 'width') { + normalizedFieldName = '_width'; + } else if (fieldName === 'height') { + normalizedFieldName = '_height'; + } + + // Get the documents + const doc = this.documentsById.get(docId); + if (!doc) { + return { success: false, message: `Document with ID ${docId} not found` }; + } + + const layoutDoc = this.layoutDocsById.get(docId); + const dataDoc = this.dataDocsById.get(docId); + + if (!layoutDoc && !dataDoc) { + return { success: false, message: `Could not find layout or data document for document with ID ${docId}` }; + } + + try { + // Convert the field value to the appropriate type based on field metadata + const convertedValue = this.convertFieldValue(normalizedFieldName, fieldValue); + + // Determine where to set the field value + let targetDoc: Doc | undefined; + let targetLocation: string; + + if (documentLocation === 'layout') { + // Explicitly set on layout document + targetDoc = layoutDoc; + targetLocation = 'layout'; + } else if (documentLocation === 'data') { + // Explicitly set on data document + targetDoc = dataDoc; + targetLocation = 'data'; + } else { + // Auto-detect where to set the field + + // Check if field exists on layout document (with or without underscore) + const existsOnLayout = layoutDoc && + (layoutDoc[normalizedFieldName] !== undefined || + layoutDoc[strippedFieldName] !== undefined); + + // Check if field exists on data document (with or without underscore) + const existsOnData = dataDoc && + (dataDoc[normalizedFieldName] !== undefined || + dataDoc[strippedFieldName] !== undefined); + + if (existsOnLayout) { + targetDoc = layoutDoc; + targetLocation = 'layout'; + } else if (existsOnData) { + targetDoc = dataDoc; + targetLocation = 'data'; + } else { + // Field doesn't exist on either document, default to layout document + targetDoc = layoutDoc || dataDoc; + targetLocation = layoutDoc ? 'layout' : 'data'; + } + } + + if (!targetDoc) { + return { success: false, message: `Target document (${documentLocation}) not available` }; + } + + // Set the field value on the target document + targetDoc[normalizedFieldName] = convertedValue; + + return { + success: true, + message: `Successfully updated field '${normalizedFieldName}' on ${targetLocation} document (ID: ${docId})`, + fieldName: normalizedFieldName, + originalFieldName: fieldName, + newValue: convertedValue + }; + } catch (error) { + console.error('Error editing document field:', error); + return { + success: false, + message: `Error updating field: ${error instanceof Error ? error.message : String(error)}` + }; + } + } + + /** + * Converts a string field value to the appropriate type based on field metadata + * @param fieldName The name of the field + * @param fieldValue The string value to convert + * @returns The converted value with the appropriate type + */ + private convertFieldValue(fieldName: string, fieldValue: any): any { + // If fieldValue is already a number, we don't need to convert it from string + if (typeof fieldValue === 'number') { + return fieldValue; + } + + // If fieldValue is not a string (and not a number), convert it to string + if (typeof fieldValue !== 'string') { + fieldValue = String(fieldValue); + } + + // Get field metadata + const normalizedFieldName = fieldName.startsWith('_') ? fieldName : `_${fieldName}`; + const strippedFieldName = fieldName.startsWith('_') ? fieldName.substring(1) : fieldName; + + // Check both versions of the field name in metadata + const fieldMeta = this.fieldMetadata[normalizedFieldName] || this.fieldMetadata[strippedFieldName]; + + // Special handling for width and height without metadata + if (!fieldMeta && (fieldName === '_width' || fieldName === '_height' || fieldName === 'width' || fieldName === 'height')) { + const num = Number(fieldValue); + return isNaN(num) ? fieldValue : num; + } + + if (!fieldMeta) { + // If no metadata found, just return the string value + return fieldValue; + } + + // Convert based on field type + const fieldType = fieldMeta.type; + + if (fieldType === 'boolean') { + // Convert to boolean + return fieldValue.toLowerCase() === 'true'; + } else if (fieldType === 'number') { + // Convert to number + const num = Number(fieldValue); + return isNaN(num) ? fieldValue : num; + } else if (fieldType === 'date') { + // Try to convert to date (stored as number timestamp) + try { + return new Date(fieldValue).getTime(); + } catch (e) { + return fieldValue; + } + } else if (fieldType.includes('list') || fieldType.includes('array')) { + // Try to parse as JSON array + try { + return JSON.parse(fieldValue); + } catch (e) { + return fieldValue; + } + } else if (fieldType === 'json' || fieldType === 'object') { + // Try to parse as JSON object + try { + return JSON.parse(fieldValue); + } catch (e) { + return fieldValue; + } + } + + // Default to string + return fieldValue; + } + + /** + * Formats a field value for JSON output + * @param value The field value to format + * @returns A JSON-friendly representation of the field value + */ + private formatFieldValue(value: any): any { + if (value === undefined || value === null) { + return null; + } + + // Handle Doc objects + if (value instanceof Doc) { + return { + type: 'Doc', + id: value.id || this.ensureDocumentId(value), + title: value.title || '', + docType: value.type || '', + }; + } + + // Handle arrays and complex objects + if (typeof value === 'object') { + // If the object has a toString method, use it + if (value.toString && value.toString !== Object.prototype.toString) { + return value.toString(); + } + + try { + // Try to convert to JSON string + return JSON.stringify(value); + } catch (e) { + return '[Complex Object]'; + } + } + + // Return primitive values as is + return value; + } + + /** + * Executes the DocumentMetadataTool to extract or edit metadata from documents in the Freeform view + * @param args The parameters for the tool execution + * @returns A promise that resolves to an array of Observation objects + */ + async execute(args: ParametersType<DocumentMetadataToolParamsType>): Promise<Observation[]> { + console.log('DocumentMetadataTool executing with args:', args); + + // Add diagnostic information about the ChatBox instance + if (this.chatBox) { + console.log('ChatBox instance available:', { + hasDocument: !!this.chatBoxDocument, + chatBoxProps: this.chatBox.props ? Object.keys(this.chatBox.props) : 'No props', + linkedDocsMethod: typeof this.chatBox.linkedDocs === 'function' ? 'Exists (function)' : + typeof this.chatBox.linkedDocs === 'object' ? 'Exists (object)' : 'Not available' + }); + } else { + console.warn('No ChatBox instance available'); + } + + // Find all documents in the Freeform view + this.findDocumentsInFreeformView(); + + // Debug diagnostic information about all documents found + if (this.documentsById.size > 0) { + console.log('Documents found:', Array.from(this.documentsById.entries()).map(([id, doc]) => ({ + id, + title: doc.title || 'Untitled', + type: doc.type || 'Unknown', + hasLayout: !!this.layoutDocsById.get(id), + hasData: !!this.dataDocsById.get(id) + }))); + } + + // Check if we found any documents + if (this.documentsById.size === 0) { + console.error('No documents found in Freeform view'); + return [{ + type: 'text', + text: JSON.stringify({ + success: false, + message: 'No documents found in the current view. Unable to extract or edit metadata.', + chatBoxDocumentId: this.chatBoxDocument ? this.chatBoxDocument.id : 'No ChatBox document', + diagnostics: { + chatBoxAvailable: !!this.chatBox, + documentAvailable: !!this.chatBoxDocument, + toolInitializedWith: this.chatBox ? + (this.chatBox.constructor && this.chatBox.constructor.name) || 'Unknown constructor' + : 'No ChatBox' + } + }, null, 2) + }]; + } + + let result: Record<string, any>; + + // Determine which action to perform + const action = args.action?.toLowerCase(); + + if (action === 'list') { + // Just return a simple list of available documents for quick reference + const documentList = Array.from(this.documentsById.entries()).map(([id, doc]) => ({ + id, + title: doc.title || 'Untitled', + type: doc.type || 'Unknown' + })); + + result = { + success: true, + message: `Found ${documentList.length} documents in the current view`, + documents: documentList + }; + } + else if (action === 'edit') { + // Edit document metadata + if (!args.documentId) { + return [{ + type: 'text', + text: JSON.stringify({ + success: false, + message: 'Document ID is required for edit operations', + availableDocuments: Array.from(this.documentsById.keys()) + }, null, 2) + }]; + } + + if (!args.fieldName) { + return [{ + type: 'text', + text: JSON.stringify({ + success: false, + message: 'Field name is required for edit operations', + }, null, 2) + }]; + } + + if (args.fieldValue === undefined) { + return [{ + type: 'text', + text: JSON.stringify({ + success: false, + message: 'Field value is required for edit operations', + }, null, 2) + }]; + } + + // Check if the document exists + if (!this.documentsById.has(args.documentId)) { + return [{ + type: 'text', + text: JSON.stringify({ + success: false, + message: `Document with ID ${args.documentId} not found`, + availableDocuments: Array.from(this.documentsById.keys()) + }, null, 2) + }]; + } + + // Convert fieldValue to string if it's not already + // This is to support the Agent passing numeric values directly + const fieldValue = typeof args.fieldValue === 'string' ? + args.fieldValue : + String(args.fieldValue); + + // Perform the edit + const editResult = this.editDocumentField( + args.documentId, + args.fieldName, + fieldValue, + args.documentLocation || 'auto' + ); + + // If successful, also get the updated metadata + if (editResult.success) { + const documentMetadata = this.extractDocumentMetadata(args.documentId); + result = { + ...editResult, + document: documentMetadata, + }; + } else { + result = editResult; + } + } else if (action === 'get') { + // Get document metadata + if (args.documentId) { + // Check if the document exists + if (!this.documentsById.has(args.documentId)) { + return [{ + type: 'text', + text: JSON.stringify({ + success: false, + message: `Document with ID ${args.documentId} not found`, + availableDocuments: Array.from(this.documentsById.keys()) + }, null, 2) + }]; + } + + // Get metadata for a specific document + const documentMetadata = this.extractDocumentMetadata(args.documentId); + result = { + success: !!documentMetadata, + message: documentMetadata ? 'Document metadata retrieved successfully' : `Document with ID ${args.documentId} not found`, + document: documentMetadata, + fieldDefinitions: this.fieldMetadata, + }; + } else { + // Get metadata for all documents + const documentsMetadata: Record<string, any> = {}; + for (const docId of this.documentsById.keys()) { + documentsMetadata[docId] = this.extractDocumentMetadata(docId); + } + + result = { + success: true, + message: `Retrieved metadata for ${this.documentsById.size} documents`, + documents: documentsMetadata, + fieldDefinitions: this.fieldMetadata, + documentCount: this.documentsById.size, + fieldMetadataCount: Object.keys(this.fieldMetadata).length, + }; + } + } else { + // Invalid action + result = { + success: false, + message: `Invalid action: ${action}. Valid actions are "list", "get", and "edit".`, + availableActions: ["list", "get", "edit"] + }; + } + + // Log the result to the console + console.log('Document Metadata Tool Result:', result); + + // Return the result as an observation + // Convert to string format as the Observation type only supports 'text' or 'image_url' types + return [{ + type: 'text', + text: JSON.stringify(result, null, 2) + }]; + } + + /** + * Validates the input parameters for the DocumentMetadataTool + * This custom validator allows numbers to be passed for fieldValue while maintaining + * compatibility with the standard validation + * + * @param params The parameters to validate + * @returns True if the parameters are valid, false otherwise + */ + inputValidator(params: ParametersType<DocumentMetadataToolParamsType>): boolean { + // Default validation for required fields + if (params.action === undefined) { + return false; + } + + // For edit action, documentId, fieldName, and fieldValue are required + if (params.action === 'edit') { + if (!params.documentId || !params.fieldName || params.fieldValue === undefined) { + return false; + } + } + + // For get action with documentId, documentId is required + if (params.action === 'get' && params.documentId === '') { + return false; + } + + // Allow for numeric fieldValue even though the type is defined as string + if (params.fieldValue !== undefined && typeof params.fieldValue === 'number') { + console.log('Numeric fieldValue detected, will be converted to string'); + // We'll convert it later, so don't fail validation + return true; + } + + return true; + } +}
\ No newline at end of file diff --git a/src/server/chunker/pdf_chunker.py b/src/server/chunker/pdf_chunker.py index 697550f2e..9d4cfb1b2 100644 --- a/src/server/chunker/pdf_chunker.py +++ b/src/server/chunker/pdf_chunker.py @@ -153,7 +153,7 @@ class ElementExtractor: xref = img_info[0] # XREF of the image in the PDF base_image = page.parent.extract_image(xref) # Extract the image by its XREF image_bytes = base_image["image"] - image = Image.open(io.BytesIO(image_bytes)) # Convert bytes to PIL image + image = Image.open(io.BytesIO(image_bytes)).convert("RGB") # Ensure it's RGB before saving as PNG width_ratio = img.width / page.rect.width # Scale factor for width height_ratio = img.height / page.rect.height # Scale factor for height |