path: root/src/client/views/nodes/chatbot/utils/AgentDocumentManager.ts

import { action, computed, makeObservable, observable, ObservableMap, reaction, runInAction } from 'mobx';
import { Doc, FieldResult, StrListCast } from '../../../../../fields/Doc';
import { DocData } from '../../../../../fields/DocSymbols';
import { Id } from '../../../../../fields/FieldSymbols';
import { List } from '../../../../../fields/List';
import { DocCast, StrCast } from '../../../../../fields/Types';
import { DocServer } from '../../../../DocServer';
import { Docs, DocumentOptions } from '../../../../documents/Documents';
import { DocumentManager } from '../../../../util/DocumentManager';
import { LinkManager, UPDATE_SERVER_CACHE } from '../../../../util/LinkManager';
import { DocumentView } from '../../DocumentView';
import { ChatBox, parsedDoc } from '../chatboxcomponents/ChatBox';
import { CanvasDocsTool } from '../tools/CanvasDocsTool';
import { supportedDocTypes } from '../types/tool_types';
import { CHUNK_TYPE, RAGChunk, SimplifiedChunk } from '../types/types';

/**
 * Interface representing a document in the freeform view
 */
interface AgentDocument {
    layoutDoc: Doc;
    dataDoc: Doc;
}

/**
 * Class to manage documents in a freeform view
 */
export class AgentDocumentManager {
    @observable private documentsById: ObservableMap<string, AgentDocument>;
    private chatBox: ChatBox;
    private parentView: DocumentView;
    private chatBoxDocument: Doc | null = null;
    @observable private _useCanvasMode: boolean = false;
    private fieldMetadata: Record<string, any> = {}; // bcz: CHANGE any to a proper type!
    @observable private simplifiedChunks: ObservableMap<string, SimplifiedChunk>;

    /**
     * Creates a new DocumentManager
     * @param templateDocument The document that serves as a template for new documents
     */
    constructor(chatBox: ChatBox, parentView: DocumentView) {
        makeObservable(this);
        this.parentView = parentView;
        const agentDoc = DocCast(chatBox.Document.agentDocument) ?? new Doc();
        const chunk_simpl = DocCast(agentDoc.chunk_simpl) ?? new Doc();

        agentDoc.title = chatBox.Document.title + '_agentDocument';
        chunk_simpl.title = '_chunk_simpl';
        chatBox.Document.agentDocument = agentDoc;
        DocCast(chatBox.Document.agentDocument)!.chunk_simpl = chunk_simpl;

        this.simplifiedChunks = StrListCast(chunk_simpl.mapping).reduce((mapping, chunks) => {
            StrListCast(chunks).forEach(chunk => {
                const parsed = JSON.parse(StrCast(chunk));
                mapping.set(parsed.chunkId, parsed);
            });
            return mapping;
        }, new ObservableMap<string, SimplifiedChunk>());

        this.documentsById = StrListCast(agentDoc.mapping).reduce((mapping, content) => {
            const [id, layoutId, docId] = content.split(':');
            const layoutDoc = DocServer.GetCachedRefField(layoutId);
            const dataDoc = DocServer.GetCachedRefField(docId);
            if (!layoutDoc || !dataDoc) {
                console.warn(`Document with ID ${id} not found in mapping`);
            } else {
                mapping.set(id, { layoutDoc, dataDoc });
            }
            return mapping;
        }, new ObservableMap<string, AgentDocument>());
        console.log(`AgentDocumentManager initialized with ${this.documentsById.size} documents`);
        this.chatBox = chatBox;
        this.chatBoxDocument = chatBox.Document;

        reaction(
            () => this.documentsById.values(),
            () => {
                if (this.chatBoxDocument && DocCast(this.chatBoxDocument.agentDocument)) {
                    DocCast(this.chatBoxDocument.agentDocument)!.mapping = new List<string>(Array.from(this.documentsById.entries()).map(([id, agent]) => `${id}:${agent.dataDoc[Id]}:${agent.layoutDoc[Id]}`));
                }
            }
            //{ fireImmediately: true }
        );
        reaction(
            () => this.simplifiedChunks.values(),
            () => {
                if (this.chatBoxDocument && DocCast(this.chatBoxDocument.agentDocument)) {
                    DocCast(DocCast(this.chatBoxDocument.agentDocument)!.chunk_simpl)!.mapping = new List<string>(Array.from(this.simplifiedChunks.values()).map(chunk => JSON.stringify(chunk)));
                }
            }
            //{ fireImmediately: true }
        );
        this.processDocument(this.chatBoxDocument);
        this.initializeFieldMetadata();
    }

    /**
     * Extracts field metadata from DocumentOptions class
     */
    private initializeFieldMetadata() {
        // Parse DocumentOptions to extract field definitions
        const documentOptionsInstance = new DocumentOptions();
        const documentOptionsEntries = Object.entries(documentOptionsInstance);

        for (const [fieldName, fieldInfo] of documentOptionsEntries) {
            // Extract field information
            const fieldData: Record<string, any> = {
                // bcz: CHANGE any to a proper type!
                name: fieldName,
                withoutUnderscore: fieldName.startsWith('_') ? fieldName.substring(1) : fieldName,
                description: '',
                type: 'unknown',
                required: false,
                defaultValue: undefined,
                possibleValues: [],
            };

            // Check if fieldInfo has description property (it's likely a FInfo instance)
            if (fieldInfo && typeof fieldInfo === 'object' && 'description' in fieldInfo) {
                fieldData.description = fieldInfo.description;

                // Extract field type if available
                if ('fieldType' in fieldInfo) {
                    fieldData.type = fieldInfo.fieldType;
                }

                // Extract possible values if available
                if ('values' in fieldInfo && Array.isArray(fieldInfo.values)) {
                    fieldData.possibleValues = fieldInfo.values;
                }
            }

            this.fieldMetadata[fieldName] = fieldData;
        }
    }

    /**
     * Toggle between linked documents mode and canvas mode
     */
    @action
    public setCanvasMode(useCanvas: boolean) {
        this._useCanvasMode = useCanvas;
        console.log(`[AgentDocumentManager] Canvas mode ${useCanvas ? 'enabled' : 'disabled'}`);
        
        // Reinitialize documents based on new mode
        if (useCanvas) {
            this.initializeCanvasDocuments();
        } else {
            this.initializeFindDocsFreeform();
        }
    }

    /**
     * Get current canvas mode status
     */
    public get useCanvasMode(): boolean {
        return this._useCanvasMode;
    }

    /**
     * Get current canvas mode status (for external access)
     */
    public getCanvasMode(): boolean {
        return this._useCanvasMode;
    }

    /**
     * Initialize documents from the entire canvas
     */
    @action
    public initializeCanvasDocuments() {
        try {
            console.log('[AgentDocumentManager] Finding all documents on canvas...');
            console.log('[AgentDocumentManager] Canvas mode enabled, looking for all documents...');
            
            // Get all canvas documents using CanvasDocsTool
            const canvasDocs = CanvasDocsTool.getAllCanvasDocuments(false);
            console.log(`[AgentDocumentManager] Found ${canvasDocs.length} documents on canvas`);

            if (canvasDocs.length === 0) {
                console.warn('[AgentDocumentManager] No documents found on canvas. This might indicate:');
                console.warn('  1. No documents are currently rendered/visible');
                console.warn('  2. All documents are considered "system" documents'); 
                console.warn('  3. DocumentView.allViews() is not returning expected results');
                
                // Let's also try including system docs to see if that's the issue
                const canvasDocsWithSystem = CanvasDocsTool.getAllCanvasDocuments(true);
                console.log(`[AgentDocumentManager] With system docs included: ${canvasDocsWithSystem.length} documents`);
            }

            // Process each canvas document
            canvasDocs.forEach(async (doc: Doc) => {
                if (doc && doc !== this.chatBoxDocument) { // Don't process the chatbox itself
                    console.log('[AgentDocumentManager] Processing canvas document:', doc.id, doc.title, doc.type);
                    await this.processDocument(doc);
                    console.log('[AgentDocumentManager] Completed processing document:', doc.id);
                }
            });

        } catch (error) {
            console.error('[AgentDocumentManager] Error finding documents on canvas:', error);
        }
    }

    /**
     * Gets all documents in the same Freeform view as the ChatBox
     * Uses the LinkManager to get all linked documents, similar to how ChatBox does it
     */
    public initializeFindDocsFreeform() {
        // Reset collections
        //this.documentsById.clear();

        try {
            // Use the LinkManager approach which is proven to work in ChatBox
            if (this.chatBoxDocument) {
                console.log('Finding documents linked to ChatBox document with ID:', this.chatBoxDocument[Id]);

                // Get directly linked documents via LinkManager
                const linkedDocs = LinkManager.Instance.getAllRelatedLinks(this.chatBoxDocument)
                    .map(d => DocCast(LinkManager.getOppositeAnchor(d, this.chatBoxDocument!)))
                    .map(d => DocCast(d?.annotationOn, d))
                    .filter(d => d);

                console.log(`Found ${linkedDocs.length} linked documents via LinkManager`);

                // Process the linked documents
                linkedDocs.forEach(async (doc: Doc | undefined) => {
                    if (doc) {
                        await this.processDocument(doc);
                        console.log('Processed linked document:', doc[Id], doc.title, doc.type);
                    }
                });
            }
        } catch (error) {
            console.error('Error finding documents in Freeform view:', error);
        }
    }

    public get parentViewDocument(): DocumentView {
        return this.parentView;
    }

    /**
     * Process a document by ensuring it has an ID and adding it to the appropriate collections
     * @param doc The document to process
     */
    @action
    public async processDocument(doc: Doc): Promise<string> {
        // Ensure document has a persistent ID
        const docId = this.ensureDocumentId(doc);
        if (doc.chunk_simplified) {
            const newChunks: SimplifiedChunk[] = [];
            for (const chunk of JSON.parse(StrCast(doc.chunk_simplified))) {
                console.log('chunk', chunk);
                newChunks.push(chunk as SimplifiedChunk);
            }
            console.log('Added simplified chunks to simplifiedChunks:', docId, newChunks);
            this.addSimplifiedChunks(newChunks);
            //DocCast(DocCast(this.chatBoxDocument!.agentDocument)!.chunk_simpl)!.mapping = new List<string>(Array.from(this.simplifiedChunks.values()).map(chunk => JSON.stringify(chunk)));
        }
        // Only add if we haven't already processed this document
        if (!this.documentsById.has(docId)) {
            this.documentsById.set(docId, { layoutDoc: doc, dataDoc: doc[DocData] });
            console.log('Added document to documentsById:', doc[Id], docId, doc[Id], doc[DocData][Id]);
        }
        return docId;
    }

    /**
     * Ensures a document has a persistent ID stored in its metadata
     * @param doc The document to ensure has an ID
     * @returns The document's ID
     */
    private ensureDocumentId(doc: Doc): string {
        let docId: string | undefined;

        // 1. Try the direct id property if it exists
        if (doc[Id]) {
            console.log('Found document ID (normal):', doc[Id]);
            docId = doc[Id];
        } else {
            throw new Error('No document ID found');
        }

        return docId;
    }

    /**
     * Extracts metadata from a specific document
     * @param docId The ID of the document to extract metadata from
     * @returns An object containing the document's metadata
     */
    public extractDocumentMetadata(id: string) {
        if (!id) return null;
        const agentDoc = this.documentsById.get(id);
        if (!agentDoc) return null;
        const layoutDoc = agentDoc.layoutDoc;
        const dataDoc = agentDoc.dataDoc;

        const metadata: Record<string, any> = {
            // bcz: CHANGE any to a proper type!
            id: layoutDoc[Id] || dataDoc[Id] || '',
            title: layoutDoc.title || '',
            type: layoutDoc.type || '',
            fields: {
                layout: {},
                data: {},
            },
            fieldLocationMap: {},
        };

        // Process all known field definitions
        Object.keys(this.fieldMetadata).forEach(fieldName => {
            // const fieldDef = this.fieldMetadata[fieldName];
            const strippedName = fieldName.startsWith('_') ? fieldName.substring(1) : fieldName;

            // Check if field exists on layout document
            let layoutValue = undefined;
            if (layoutDoc) {
                layoutValue = layoutDoc[fieldName];
                if (layoutValue !== undefined) {
                    // Field exists on layout document
                    metadata.fields.layout[fieldName] = this.formatFieldValue(layoutValue);
                    metadata.fieldLocationMap[strippedName] = 'layout';
                }
            }

            // Check if field exists on data document
            let dataValue = undefined;
            if (dataDoc) {
                dataValue = dataDoc[fieldName];
                if (dataValue !== undefined) {
                    // Field exists on data document
                    metadata.fields.data[fieldName] = this.formatFieldValue(dataValue);
                    if (!metadata.fieldLocationMap[strippedName]) {
                        metadata.fieldLocationMap[strippedName] = 'data';
                    }
                }
            }

            // For fields with stripped names (without leading underscore),
            // also check if they exist on documents without the underscore
            if (fieldName.startsWith('_')) {
                const nonUnderscoreFieldName = fieldName.substring(1);

                if (layoutDoc) {
                    const nonUnderscoreLayoutValue = layoutDoc[nonUnderscoreFieldName];
                    if (nonUnderscoreLayoutValue !== undefined) {
                        metadata.fields.layout[nonUnderscoreFieldName] = this.formatFieldValue(nonUnderscoreLayoutValue);
                        metadata.fieldLocationMap[nonUnderscoreFieldName] = 'layout';
                    }
                }

                if (dataDoc) {
                    const nonUnderscoreDataValue = dataDoc[nonUnderscoreFieldName];
                    if (nonUnderscoreDataValue !== undefined) {
                        metadata.fields.data[nonUnderscoreFieldName] = this.formatFieldValue(nonUnderscoreDataValue);
                        if (!metadata.fieldLocationMap[nonUnderscoreFieldName]) {
                            metadata.fieldLocationMap[nonUnderscoreFieldName] = 'data';
                        }
                    }
                }
            }
        });

        // Add common field aliases for easier discovery
        // This helps users understand both width and _width refer to the same property
        if (metadata.fields.layout._width !== undefined && metadata.fields.layout.width === undefined) {
            metadata.fields.layout.width = metadata.fields.layout._width;
            metadata.fieldLocationMap.width = 'layout';
        }

        if (metadata.fields.layout._height !== undefined && metadata.fields.layout.height === undefined) {
            metadata.fields.layout.height = metadata.fields.layout._height;
            metadata.fieldLocationMap.height = 'layout';
        }

        return metadata;
    }

    /**
     * Formats a field value for JSON output
     * @param value The field value to format
     * @returns A JSON-friendly representation of the field value
     */
    private formatFieldValue(value: FieldResult | undefined) {
        if (value === undefined || value === null) {
            return null;
        }

        // Handle Doc objects
        if (value instanceof Doc) {
            return {
                type: 'Doc',
                id: value[Id] || this.ensureDocumentId(value),
                title: value.title || '',
                docType: value.type || '',
            };
        }

        // Handle RichTextField (try to extract plain text)
        if (typeof value === 'string' && value.includes('"type":"doc"') && value.includes('"content":')) {
            try {
                const rtfObj = JSON.parse(value);
                // If this looks like a rich text field structure
                if (rtfObj.doc && rtfObj.doc.content) {
                    // Recursively extract text from the content
                    let plainText = '';
                    const extractText = (node: { text: string; content?: unknown[] }) => {
                        if (node.text) {
                            plainText += node.text;
                        }
                        if (node.content && Array.isArray(node.content)) {
                            node.content.forEach(child => extractText(child as { text: string; content?: unknown[] }));
                        }
                    };

                    extractText(rtfObj.doc);

                    // If we successfully extracted text, show it, but also preserve the original value
                    if (plainText) {
                        return {
                            type: 'RichText',
                            text: plainText,
                            length: plainText.length,
                            // Don't include the full value as it can be very large
                        };
                    }
                }
            } catch {
                // If parsing fails, just treat as a regular string
            }
        }

        // Handle arrays and complex objects
        if (typeof value === 'object') {
            // If the object has a toString method, use it
            if (value.toString && value.toString !== Object.prototype.toString) {
                return value.toString();
            }

            try {
                // Try to convert to JSON string
                return JSON.stringify(value);
            } catch {
                return '[Complex Object]';
            }
        }

        // Return primitive values as is
        return value;
    }

    /**
     * Converts a string field value to the appropriate type based on field metadata
     * @param fieldName The name of the field
     * @param fieldValue The string value to convert
     * @returns The converted value with the appropriate type
     */
    private convertFieldValue(fieldName: string, fieldValueIn: string | number | boolean): FieldResult | undefined {
        // If fieldValue is already a number or boolean, we don't need to convert it from string
        if (typeof fieldValueIn === 'number' || typeof fieldValueIn === 'boolean') {
            return fieldValueIn;
        }

        // If fieldValue is a string "true" or "false", convert to boolean
        if (typeof fieldValueIn === 'string') {
            if (fieldValueIn.toLowerCase() === 'true') {
                return true;
            }
            if (fieldValueIn.toLowerCase() === 'false') {
                return false;
            }
        }

        // coerce fieldvValue to a string
        const fieldValue = typeof fieldValueIn !== 'string' ? String(fieldValueIn) : fieldValueIn;

        // Special handling for text field - convert to proper RichTextField format
        if (fieldName === 'text') {
            try {
                // Check if it's already a valid JSON RichTextField
                JSON.parse(fieldValue);
                return fieldValue;
            } catch {
                // It's a plain text string, so convert it to RichTextField format
                const rtf = {
                    doc: {
                        type: 'doc',
                        content: [
                            {
                                type: 'paragraph',
                                content: [
                                    {
                                        type: 'text',
                                        text: fieldValue,
                                    },
                                ],
                            },
                        ],
                    },
                };
                return JSON.stringify(rtf);
            }
        }

        // Get field metadata
        const normalizedFieldName = fieldName.startsWith('_') ? fieldName : `_${fieldName}`;
        const strippedFieldName = fieldName.startsWith('_') ? fieldName.substring(1) : fieldName;

        // Check both versions of the field name in metadata
        const fieldMeta = this.fieldMetadata[normalizedFieldName] || this.fieldMetadata[strippedFieldName];

        // Special handling for width and height without metadata
        if (!fieldMeta && (fieldName === '_width' || fieldName === '_height' || fieldName === 'width' || fieldName === 'height')) {
            const num = Number(fieldValue);
            return isNaN(num) ? fieldValue : num;
        }

        if (!fieldMeta) {
            // If no metadata found, just return the string value
            return fieldValue;
        }

        // Convert based on field type
        const fieldType = fieldMeta.type;

        if (fieldType === 'boolean') {
            // Convert to boolean
            return fieldValue.toLowerCase() === 'true';
        } else if (fieldType === 'number') {
            // Convert to number
            const num = Number(fieldValue);
            return isNaN(num) ? fieldValue : num;
        } else if (fieldType === 'date') {
            // Try to convert to date (stored as number timestamp)
            try {
                return new Date(fieldValue).getTime();
            } catch {
                return fieldValue;
            }
        } else if (fieldType.includes('list') || fieldType.includes('array')) {
            // Try to parse as JSON array
            try {
                return JSON.parse(fieldValue) as FieldResult; //  bcz: this needs to be typed properly.  Dash fields can't accept a generic 'objext'
            } catch {
                return fieldValue;
            }
        } else if (fieldType === 'json' || fieldType === 'object') {
            // Try to parse as JSON object
            try {
                return JSON.parse(fieldValue) as FieldResult; //  bcz: this needs to be typed properly.  Dash fields can't accept a generic 'objext'
            } catch {
                return fieldValue;
            }
        }

        // Default to string
        return fieldValue;
    }

    /**
     * Extracts all field metadata from DocumentOptions
     * @returns A structured object containing metadata about all available document fields
     */
    public getAllFieldMetadata() {
        // Start with our already populated fieldMetadata from the DocumentOptions class
        const result: Record<string, any> = {
            // bcz: CHANGE any to a proper type!
            fieldCount: Object.keys(this.fieldMetadata).length,
            fields: {},
            fieldsByType: {
                string: [],
                number: [],
                boolean: [],
                //doc: [],
                //list: [],
                //date: [],
                //enumeration: [],
                //other: [],
            },
            fieldNameMappings: {},
            commonFields: {
                appearance: [],
                position: [],
                size: [],
                content: [],
                behavior: [],
                layout: [],
            },
        };

        // Process each field in the metadata
        Object.entries(this.fieldMetadata).forEach(([fieldName, fieldInfo]) => {
            const strippedName = fieldName.startsWith('_') ? fieldName.substring(1) : fieldName;

            // Add to fieldNameMappings
            if (fieldName.startsWith('_')) {
                result.fieldNameMappings[strippedName] = fieldName;
            }

            // Create structured field metadata
            const fieldData: Record<string, any> = {
                // bcz: CHANGE any to a proper type!
                name: fieldName,
                displayName: strippedName,
                description: fieldInfo.description || '',
                type: fieldInfo.fieldType || 'unknown',
                possibleValues: fieldInfo.values || [],
            };

            // Add field to fields collection
            result.fields[fieldName] = fieldData;

            // Categorize by field type
            const type = fieldInfo.fieldType?.toLowerCase() || 'unknown';
            if (type === 'string') {
                result.fieldsByType.string.push(fieldName);
            } else if (type === 'number') {
                result.fieldsByType.number.push(fieldName);
            } else if (type === 'boolean') {
                result.fieldsByType.boolean.push(fieldName);
            } else if (type === 'doc') {
                //result.fieldsByType.doc.push(fieldName);
            } else if (type === 'list') {
                //result.fieldsByType.list.push(fieldName);
            } else if (type === 'date') {
                //result.fieldsByType.date.push(fieldName);
            } else if (type === 'enumeration') {
                //result.fieldsByType.enumeration.push(fieldName);
            } else {
                //result.fieldsByType.other.push(fieldName);
            }

            // Categorize by field purpose
            if (fieldName.includes('width') || fieldName.includes('height') || fieldName.includes('size')) {
                result.commonFields.size.push(fieldName);
            } else if (fieldName.includes('color') || fieldName.includes('background') || fieldName.includes('border')) {
                result.commonFields.appearance.push(fieldName);
            } else if (fieldName.includes('x') || fieldName.includes('y') || fieldName.includes('position') || fieldName.includes('pan')) {
                result.commonFields.position.push(fieldName);
            } else if (fieldName.includes('text') || fieldName.includes('title') || fieldName.includes('data')) {
                result.commonFields.content.push(fieldName);
            } else if (fieldName.includes('action') || fieldName.includes('click') || fieldName.includes('event')) {
                result.commonFields.behavior.push(fieldName);
            } else if (fieldName.includes('layout')) {
                result.commonFields.layout.push(fieldName);
            }
        });

        // Add special section for auto-sizing related fields
        result.autoSizingFields = {
            height: {
                autoHeightField: '_layout_autoHeight',
                heightField: '_height',
                displayName: 'height',
                usage: 'To manually set height, first set layout_autoHeight to false',
            },
            width: {
                autoWidthField: '_layout_autoWidth',
                widthField: '_width',
                displayName: 'width',
                usage: 'To manually set width, first set layout_autoWidth to false',
            },
        };

        // Add special section for text field format
        result.specialFields = {
            text: {
                name: 'text',
                description: 'Document text content',
                format: 'RichTextField',
                note: 'When setting text, provide plain text - it will be automatically converted to the correct format',
                example: 'For setting: "Hello world" (plain text); For getting: Will be converted to plaintext for display',
            },
        };

        return result;
    }

    /**
     * Edits a specific field on a document
     * @param docId The ID of the document to edit
     * @param fieldName The name of the field to edit
     * @param fieldValue The new value for the field (string, number, or boolean)
     * @returns Object with success status, message, and additional information
     */
    public editDocumentField(
        docId: string,
        fieldName: string,
        fieldValue: string | number | boolean
    ): {
        success: boolean;
        message: string;
        fieldName?: string;
        originalFieldName?: string;
        newValue?: string | number | boolean | object;
        warning?: string;
    } {
        // Normalize field name (handle with/without underscore)
        let normalizedFieldName = fieldName.startsWith('_') ? fieldName : fieldName;
        // const strippedFieldName = fieldName.startsWith('_') ? fieldName.substring(1) : fieldName;

        // Handle common field name aliases (width → _width, height → _height)
        // Many document fields use '_' prefix for layout properties
        if (fieldName === 'width') {
            normalizedFieldName = '_width';
        } else if (fieldName === 'height') {
            normalizedFieldName = '_height';
        }

        // Get the documents
        const doc = this.documentsById.get(docId);
        if (!doc) {
            return { success: false, message: `Document with ID ${docId} not found` };
        }

        const { layoutDoc, dataDoc } = this.documentsById.get(docId) ?? { layoutDoc: null, dataDoc: null };

        if (!layoutDoc && !dataDoc) {
            return { success: false, message: `Could not find layout or data document for document with ID ${docId}` };
        }

        try {
            // Convert the field value to the appropriate type based on field metadata
            const convertedValue = this.convertFieldValue(normalizedFieldName, fieldValue);

            let targetDoc: Doc | undefined;
            let targetLocation: string;

            // First, check if field exists on layout document using Doc.Get
            if (layoutDoc) {
                const fieldExistsOnLayout = Doc.Get(layoutDoc, normalizedFieldName, true) !== undefined;

                // If it exists on layout document, update it there
                if (fieldExistsOnLayout) {
                    targetDoc = layoutDoc;
                    targetLocation = 'layout';
                }
                // If it has an underscore prefix, it's likely a layout property even if not yet set
                else if (normalizedFieldName.startsWith('_')) {
                    targetDoc = layoutDoc;
                    targetLocation = 'layout';
                }
                // Otherwise, look for or create on data document
                else if (dataDoc) {
                    targetDoc = dataDoc;
                    targetLocation = 'data';
                }
                // If no data document available, default to layout
                else {
                    targetDoc = layoutDoc;
                    targetLocation = 'layout';
                }
            }
            // If no layout document, use data document
            else if (dataDoc) {
                targetDoc = dataDoc;
                targetLocation = 'data';
            } else {
                return { success: false, message: `No valid document found for editing` };
            }

            if (!targetDoc) {
                return { success: false, message: `Target document not available` };
            }

            // Set the field value on the target document
            targetDoc[normalizedFieldName] = convertedValue; //  bcz: converteValue needs to be typed properly.  Dash fields can't accept a generic 'objext'

            return {
                success: true,
                message: `Successfully updated field '${normalizedFieldName}' on ${targetLocation} document (ID: ${docId})`,
                fieldName: normalizedFieldName,
                originalFieldName: fieldName,
                newValue: convertedValue,
            };
        } catch (error) {
            console.error('Error editing document field:', error);
            return {
                success: false,
                message: `Error updating field: ${error instanceof Error ? error.message : String(error)}`,
            };
        }
    }
    /**
     * Gets metadata for a specific document or all documents
     * @param documentId Optional ID of a specific document to get metadata for
     * @returns Document metadata or metadata for all documents
     */
    public getDocumentMetadata(documentId?: string) {
        if (documentId) {
            console.log(`Returning document metadata for docID, ${documentId}:`, this.extractDocumentMetadata(documentId));
            return this.extractDocumentMetadata(documentId);
        } else {
            // Get metadata for all documents
            const documentsMetadata: Record<string, Record<string, any>> = {}; // bcz: CHANGE any to a proper type!
            for (const docid of this.documentsById.keys()) {
                const metadata = this.extractDocumentMetadata(docid);
                if (metadata) {
                    documentsMetadata[docid] = metadata;
                } else {
                    console.warn(`No metadata found for document with ID: ${docid}`);
                }
            }
            return {
                documentCount: this.documentsById.size,
                documents: documentsMetadata,
                //fieldDefinitions: this.fieldMetadata, // TODO: remove this, if fieldDefinitions are not needed.
            };
        }
    }

    /**
     * Adds links between documents based on their IDs
     * @param docIds Array of document IDs to link
     * @param relationship Optional relationship type for the links
     * @returns Array of created link documents
     */
    public addLinks(docIds: string[]): Doc[] {
        const createdLinks: Doc[] = [];
        // Use string keys for Set instead of arrays which don't work as expected as keys
        const alreadyLinked = new Set<string>();

        // Iterate over the document IDs and add links
        docIds.forEach(docId1 => {
            const doc1 = this.documentsById.get(docId1);
            docIds.forEach(docId2 => {
                if (docId1 === docId2) return; // Skip self-linking

                // Create a consistent key regardless of document order
                const linkKey = [docId1, docId2].sort().join('_');
                if (alreadyLinked.has(linkKey)) return;

                const doc2 = this.documentsById.get(docId2);
                if (doc1?.layoutDoc && doc2?.layoutDoc) {
                    try {
                        // Create a link document between doc1 and doc2
                        const linkDoc = Docs.Create.LinkDocument(doc1.layoutDoc, doc2.layoutDoc);

                        // Set a default color if relationship doesn't specify one
                        if (!linkDoc.color) {
                            linkDoc.color = 'lightBlue'; // Default blue color
                        }

                        // Ensure link is visible by setting essential properties
                        linkDoc.link_visible = true;
                        linkDoc.link_enabled = true;
                        linkDoc.link_autoMove = true;
                        linkDoc.link_showDirected = true;

                        // Set the embedContainer to ensure visibility
                        // This is shown in the image as a key difference between visible/non-visible links
                        if (this.chatBoxDocument && this.chatBoxDocument.parent && typeof this.chatBoxDocument.parent === 'object' && 'title' in this.chatBoxDocument.parent) {
                            linkDoc.embedContainer = String(this.chatBoxDocument.parent.title);
                        } else if (doc1.layoutDoc.parent && typeof doc1.layoutDoc.parent === 'object' && 'title' in doc1.layoutDoc.parent) {
                            linkDoc.embedContainer = String(doc1.layoutDoc.parent.title);
                        } else {
                            // Default to a tab name if we can't find one
                            linkDoc.embedContainer = 'Untitled Tab 1';
                        }

                        // Add the link to the document system
                        LinkManager.Instance.addLink(linkDoc);

                        const ancestor = DocumentView.linkCommonAncestor(linkDoc);
                        ancestor?.ComponentView?.addDocument?.(linkDoc);
                        // Add to user document list to make it visible in the UI
                        Doc.AddDocToList(Doc.UserDoc(), 'links', linkDoc);

                        // Create a visual link for display
                        if (this.chatBoxDocument) {
                            // Make sure the docs are visible in the UI
                            this.chatBox._props.addDocument?.(doc1.layoutDoc);
                            this.chatBox._props.addDocument?.(doc2.layoutDoc);

                            // Use DocumentManager to ensure documents are visible
                            DocumentManager.Instance.showDocument(doc1.layoutDoc, { willZoomCentered: false });
                            DocumentManager.Instance.showDocument(doc2.layoutDoc, { willZoomCentered: false });
                        }

                        createdLinks.push(linkDoc);
                        alreadyLinked.add(linkKey);
                    } catch (error) {
                        console.error('Error creating link between documents:', error);
                    }
                }
            });
        });

        // Force update of the UI to show new links
        setTimeout(() => {
            try {
                // Update server cache to ensure links are persisted
                UPDATE_SERVER_CACHE && typeof UPDATE_SERVER_CACHE === 'function' && UPDATE_SERVER_CACHE();
            } catch (e) {
                console.warn('Could not update server cache after creating links:', e);
            }
        }, 100);

        return createdLinks;
    }
    /**
     * Helper method to validate a document type and ensure it's a valid supportedDocType
     * @param docType The document type to validate
     * @returns True if the document type is valid, false otherwise
     */
    private isValidDocType(docType: string): boolean {
        return Object.values(supportedDocTypes).includes(docType as supportedDocTypes);
    }
    /**
     * Creates a document in the dashboard and returns its ID.
     * This is a public API used by tools like SearchTool.
     *
     * @param docType The type of document to create
     * @param data The data for the document
     * @param options Optional configuration options
     * @returns The ID of the created document
     */

    public async createDocInDash(docType: string, data: string, options?: DocumentOptions): Promise<string> {
        // Validate doc_type
        if (!this.isValidDocType(docType)) {
            throw new Error(`Invalid document type: ${docType}`);
        }

        try {
            // Create simple document with just title and data
            const simpleDoc: parsedDoc = {
                ...(options as parsedDoc), // bcz: hack .. why do we need parsedDoc and not DocumentOptions here?
                doc_type: docType,
                title: options?.title ?? `Untitled Document ${this.documentsById.size + 1}`,
                data: data,
                x: options?.x ?? 0,
                y: options?.y ?? 0,
                _width: 300,
                _height: 300,
                _layout_fitWidth: false,
                _layout_autoHeight: true,
            };

            // Additional handling for web documents
            if (docType === 'web') {
                // For web documents, don't sanitize the URL here
                // Instead, set properties to handle content safely when loaded
                simpleDoc._disable_resource_loading = true;
                simpleDoc._sandbox_iframe = true;
                simpleDoc.data_useCors = true;

                // Specify a more permissive sandbox to allow content to render properly
                // but still maintain security
                simpleDoc._iframe_sandbox = 'allow-same-origin allow-scripts allow-popups allow-forms';
            }

            // Use the chatBox's createDocInDash method to create the document
            if (!this.chatBox) {
                throw new Error('ChatBox instance not available for creating document');
            }

            const doc = this.chatBox.whichDoc(simpleDoc, false);
            if (doc) {
                // Use MobX runInAction to properly modify observable state
                runInAction(() => {
                    if (this.chatBoxDocument && doc) {
                        // Create link and add it to the document system
                        const linkDoc = Docs.Create.LinkDocument(this.chatBoxDocument, doc);
                        LinkManager.Instance.addLink(linkDoc);
                        if (doc.type !== 'web') {
                            // Add document to view
                            this.chatBox._props.addDocument?.(doc);

                            // Show document - defer actual display to prevent immediate resource loading
                            setTimeout(() => {
                                DocumentManager.Instance.showDocument(doc, { willZoomCentered: true }, () => {});
                            }, 100);
                        }
                    }
                });

                const id = await this.processDocument(doc);
                return id;
            } else {
                throw new Error(`Error creating document. Created document not found.`);
            }
        } catch (error) {
            throw new Error(`Error creating document: ${error}`);
        }
    }

    /**
     * Sanitizes web content to prevent errors with external resources
     * @param content The web content to sanitize
     * @returns Sanitized content
     */
    private sanitizeWebContent(content: string): string {
        if (!content) return content;

        try {
            // Replace problematic resource references that might cause errors
            const sanitized = content
                // Remove preload links that might cause errors
                .replace(/<link[^>]*rel=["']preload["'][^>]*>/gi, '')
                // Remove map file references
                .replace(/\/\/# sourceMappingURL=.*\.map/gi, '')
                // Remove external CSS map files references
                .replace(/\/\*# sourceMappingURL=.*\.css\.map.*\*\//gi, '')
                // Add sandbox to iframes
                .replace(/<iframe/gi, '<iframe sandbox="allow-same-origin" loading="lazy"')
                // Prevent automatic resource loading for images
                .replace(/<img/gi, '<img loading="lazy"')
                // Prevent automatic resource loading for scripts
                .replace(/<script/gi, '<script type="text/disabled"')
                // Handle invalid URIs by converting relative URLs to absolute ones
                .replace(/href=["'](\/[^"']+)["']/gi, (match, p1) => {
                    // Only handle relative URLs starting with /
                    if (p1.startsWith('/')) {
                        return `href="#disabled-link"`;
                    }
                    return match;
                })
                // Prevent automatic loading of CSS
                .replace(/<link[^>]*rel=["']stylesheet["'][^>]*href=["']([^"']+)["']/gi, (match, href) => `<link rel="prefetch" data-original-href="${href}" />`);

            // Wrap the content in a sandboxed container
            return `
            <div class="sandboxed-web-content">
                <style>
                /* Override styles to prevent external resource loading */
                @font-face { font-family: 'disabled'; src: local('Arial'); }
                * { font-family: system-ui, -apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, Oxygen, Ubuntu, Cantarell, 'Open Sans', 'Helvetica Neue', sans-serif !important; }
                img, iframe, frame, embed, object { max-width: 100%; }
                </style>
                ${sanitized}
            </div>`;
        } catch (e) {
            console.warn('Error sanitizing web content:', e);
            // Fall back to a safe container with the content as text
            return `
            <div class="sandboxed-web-content">
                <p>Content could not be safely displayed. Raw content:</p>
                <pre>${content.replace(/</g, '&lt;').replace(/>/g, '&gt;')}</pre>
            </div>`;
        }
    }

    public has(docId: string) {
        return this.documentsById.has(docId);
    }

    /**
     * Returns a list of all document IDs in the manager.
     * @returns An array of document IDs (strings).
     */
    @computed
    public get listDocs(): string {
        const xmlDocs = Array.from(this.documentsById.entries()).map(([id, agentDoc]) => {
            return `<document>
  <id>${id}</id>
  <title>${this.escapeXml(StrCast(agentDoc.layoutDoc.title))}</title>
  <type>${this.escapeXml(StrCast(agentDoc.layoutDoc.type))}</type>
  <summary>${this.escapeXml(StrCast(agentDoc.layoutDoc.summary))}</summary>
</document>`;
        });

        return xmlDocs.join('\n');
    }

    private escapeXml(str: string): string {
        return str.replace(/&/g, '&amp;').replace(/</g, '&lt;').replace(/>/g, '&gt;').replace(/"/g, '&quot;').replace(/'/g, '&apos;');
    }

    @computed
    public get docIds(): string[] {
        return Array.from(this.documentsById.keys());
    }

    /**
     * Gets a document by its ID
     * @param docId The ID of the document to retrieve
     * @returns The document if found, undefined otherwise
     */
    public getDocument(docId: string): Doc | undefined {
        const docInfo = this.documentsById.get(docId);
        return docInfo?.layoutDoc;
    }

    public getDataDocument(docId: string): Doc | undefined {
        const docInfo = this.documentsById.get(docId);
        return docInfo?.dataDoc;
    }

    // In AgentDocumentManager
    private descriptionCache = new Map<string, string>();

    public async getDocDescription(id: string): Promise<string> {
        if (!this.descriptionCache.has(id)) {
            const doc = this.getDocument(id)!;
            const desc = await Doc.getDescription(doc);
            this.descriptionCache.set(id, desc.replace(/\n/g, ' ').trim());
        }
        return this.descriptionCache.get(id)!;
    }

    /**
     * Adds simplified chunks to a document for citation handling
     * @param doc The document to add simplified chunks to
     * @param chunks Array of full RAG chunks to simplify
     * @param docType The type of document (e.g., 'pdf', 'video', 'audio', etc.)
     * @returns The updated document with simplified chunks
     */
    @action
    public addSimplifiedChunks(simplifiedChunks: SimplifiedChunk[]) {
        simplifiedChunks.forEach(chunk => {
            this.simplifiedChunks.set(chunk.chunkId, chunk);
        });
    }

    public getSimplifiedChunks(chunks: RAGChunk[], docType: string): SimplifiedChunk[] {
        console.log('chunks', chunks, 'simplifiedChunks', this.simplifiedChunks);
        const simplifiedChunks: SimplifiedChunk[] = [];
        // Create array of simplified chunks based on document type
        for (const chunk of chunks) {
            // Common properties across all chunk types
            const baseChunk: SimplifiedChunk = {
                chunkId: chunk.id,
                //text: chunk.metadata.text,
                doc_id: chunk.metadata.doc_id,
                chunkType: chunk.metadata.type || CHUNK_TYPE.TEXT,
            };

            // Add type-specific properties
            if (docType === 'video' || docType === 'audio') {
                simplifiedChunks.push({
                    ...baseChunk,
                    start_time: chunk.metadata.start_time,
                    end_time: chunk.metadata.end_time,
                    indexes: chunk.metadata.indexes,
                    chunkType: docType === 'video' ? CHUNK_TYPE.VIDEO : CHUNK_TYPE.AUDIO,
                } as SimplifiedChunk);
            } else if (docType === 'pdf') {
                simplifiedChunks.push({
                    ...baseChunk,
                    startPage: chunk.metadata.start_page,
                    endPage: chunk.metadata.end_page,
                    location: chunk.metadata.location,
                } as SimplifiedChunk);
            } else if (docType === 'csv' && 'row_start' in chunk.metadata && 'row_end' in chunk.metadata && 'col_start' in chunk.metadata && 'col_end' in chunk.metadata) {
                simplifiedChunks.push({
                    ...baseChunk,
                    rowStart: chunk.metadata.row_start,
                    rowEnd: chunk.metadata.row_end,
                    colStart: chunk.metadata.col_start,
                    colEnd: chunk.metadata.col_end,
                } as SimplifiedChunk);
            } else {
                // Default for other document types
                simplifiedChunks.push(baseChunk as SimplifiedChunk);
            }
        }
        return simplifiedChunks;
    }

    /**
     * Gets a specific simplified chunk by ID
     * @param doc The document containing chunks
     * @param chunkId The ID of the chunk to retrieve
     * @returns The simplified chunk if found, undefined otherwise
     */
    @action
    public getSimplifiedChunkById(chunkId: string) {
        return { foundChunk: this.simplifiedChunks.get(chunkId), doc: this.getDocument(this.simplifiedChunks.get(chunkId)?.doc_id || chunkId), dataDoc: this.getDataDocument(this.simplifiedChunks.get(chunkId)?.doc_id || chunkId) };
    }

    public getChunkIdsFromDocIds(docIds: string[]): string[] {
        return docIds
            .map(docId => {
                for (const chunk of this.simplifiedChunks.values()) {
                    if (chunk.doc_id === docId) {
                        return chunk.chunkId;
                    }
                }
            })
            .filter(chunkId => chunkId !== undefined) as string[];
    }

    /**
     * Gets the original segments from a media document
     * @param doc The document containing original media segments
     * @returns Array of media segments or empty array if none exist
     */
    public getOriginalSegments(doc: Doc): { text: string; index: string; start: number }[] {
        if (!doc || !doc.original_segments) {
            return [];
        }

        try {
            return JSON.parse(StrCast(doc.original_segments)) || [];
        } catch (e) {
            console.error('Error parsing original segments:', e);
            return [];
        }
    }
}