diff options
Diffstat (limited to 'src/client/views/nodes/chatbot/utils/AgentDocumentManager.ts')
| -rw-r--r-- | src/client/views/nodes/chatbot/utils/AgentDocumentManager.ts | 1218 |
1 files changed, 1218 insertions, 0 deletions
diff --git a/src/client/views/nodes/chatbot/utils/AgentDocumentManager.ts b/src/client/views/nodes/chatbot/utils/AgentDocumentManager.ts new file mode 100644 index 000000000..485430403 --- /dev/null +++ b/src/client/views/nodes/chatbot/utils/AgentDocumentManager.ts @@ -0,0 +1,1218 @@ +import { action, computed, makeObservable, observable, ObservableMap, reaction, runInAction } from 'mobx'; +import { Doc, FieldResult, StrListCast } from '../../../../../fields/Doc'; +import { DocData } from '../../../../../fields/DocSymbols'; +import { Id } from '../../../../../fields/FieldSymbols'; +import { List } from '../../../../../fields/List'; +import { DocCast, StrCast } from '../../../../../fields/Types'; +import { DocServer } from '../../../../DocServer'; +import { Docs, DocumentOptions } from '../../../../documents/Documents'; +import { DocumentManager } from '../../../../util/DocumentManager'; +import { LinkManager, UPDATE_SERVER_CACHE } from '../../../../util/LinkManager'; +import { DocumentView } from '../../DocumentView'; +import { ChatBox, parsedDoc } from '../chatboxcomponents/ChatBox'; +import { CanvasDocsTool } from '../tools/CanvasDocsTool'; +import { supportedDocTypes } from '../types/tool_types'; +import { CHUNK_TYPE, RAGChunk, SimplifiedChunk } from '../types/types'; + +/** + * Interface representing a document in the freeform view + */ +interface AgentDocument { + layoutDoc: Doc; + dataDoc: Doc; +} + +/** + * Class to manage documents in a freeform view + */ +export class AgentDocumentManager { + @observable private documentsById: ObservableMap<string, AgentDocument>; + private chatBox: ChatBox; + private parentView: DocumentView; + private chatBoxDocument: Doc | null = null; + @observable private _useCanvasMode: boolean = false; + private fieldMetadata: Record<string, any> = {}; // bcz: CHANGE any to a proper type! + @observable private simplifiedChunks: ObservableMap<string, SimplifiedChunk>; + + /** + * Creates a new DocumentManager + * @param templateDocument The document that serves as a template for new documents + */ + constructor(chatBox: ChatBox, parentView: DocumentView) { + makeObservable(this); + this.parentView = parentView; + const agentDoc = DocCast(chatBox.Document.agentDocument) ?? new Doc(); + const chunk_simpl = DocCast(agentDoc.chunk_simpl) ?? new Doc(); + + agentDoc.title = chatBox.Document.title + '_agentDocument'; + chunk_simpl.title = '_chunk_simpl'; + chatBox.Document.agentDocument = agentDoc; + DocCast(chatBox.Document.agentDocument)!.chunk_simpl = chunk_simpl; + + this.simplifiedChunks = StrListCast(chunk_simpl.mapping).reduce((mapping, chunks) => { + StrListCast(chunks).forEach(chunk => { + const parsed = JSON.parse(StrCast(chunk)); + mapping.set(parsed.chunkId, parsed); + }); + return mapping; + }, new ObservableMap<string, SimplifiedChunk>()); + + this.documentsById = StrListCast(agentDoc.mapping).reduce((mapping, content) => { + const [id, layoutId, docId] = content.split(':'); + const layoutDoc = DocServer.GetCachedRefField(layoutId); + const dataDoc = DocServer.GetCachedRefField(docId); + if (!layoutDoc || !dataDoc) { + console.warn(`Document with ID ${id} not found in mapping`); + } else { + mapping.set(id, { layoutDoc, dataDoc }); + } + return mapping; + }, new ObservableMap<string, AgentDocument>()); + console.log(`AgentDocumentManager initialized with ${this.documentsById.size} documents`); + this.chatBox = chatBox; + this.chatBoxDocument = chatBox.Document; + + reaction( + () => this.documentsById.values(), + () => { + if (this.chatBoxDocument && DocCast(this.chatBoxDocument.agentDocument)) { + DocCast(this.chatBoxDocument.agentDocument)!.mapping = new List<string>(Array.from(this.documentsById.entries()).map(([id, agent]) => `${id}:${agent.dataDoc[Id]}:${agent.layoutDoc[Id]}`)); + } + } + //{ fireImmediately: true } + ); + reaction( + () => this.simplifiedChunks.values(), + () => { + if (this.chatBoxDocument && DocCast(this.chatBoxDocument.agentDocument)) { + DocCast(DocCast(this.chatBoxDocument.agentDocument)!.chunk_simpl)!.mapping = new List<string>(Array.from(this.simplifiedChunks.values()).map(chunk => JSON.stringify(chunk))); + } + } + //{ fireImmediately: true } + ); + this.processDocument(this.chatBoxDocument); + this.initializeFieldMetadata(); + } + + /** + * Extracts field metadata from DocumentOptions class + */ + private initializeFieldMetadata() { + // Parse DocumentOptions to extract field definitions + const documentOptionsInstance = new DocumentOptions(); + const documentOptionsEntries = Object.entries(documentOptionsInstance); + + for (const [fieldName, fieldInfo] of documentOptionsEntries) { + // Extract field information + const fieldData: Record<string, any> = { + // bcz: CHANGE any to a proper type! + name: fieldName, + withoutUnderscore: fieldName.startsWith('_') ? fieldName.substring(1) : fieldName, + description: '', + type: 'unknown', + required: false, + defaultValue: undefined, + possibleValues: [], + }; + + // Check if fieldInfo has description property (it's likely a FInfo instance) + if (fieldInfo && typeof fieldInfo === 'object' && 'description' in fieldInfo) { + fieldData.description = fieldInfo.description; + + // Extract field type if available + if ('fieldType' in fieldInfo) { + fieldData.type = fieldInfo.fieldType; + } + + // Extract possible values if available + if ('values' in fieldInfo && Array.isArray(fieldInfo.values)) { + fieldData.possibleValues = fieldInfo.values; + } + } + + this.fieldMetadata[fieldName] = fieldData; + } + } + + /** + * Toggle between linked documents mode and canvas mode + */ + @action + public setCanvasMode(useCanvas: boolean) { + this._useCanvasMode = useCanvas; + console.log(`[AgentDocumentManager] Canvas mode ${useCanvas ? 'enabled' : 'disabled'}`); + + // Reinitialize documents based on new mode + if (useCanvas) { + this.initializeCanvasDocuments(); + } else { + this.initializeFindDocsFreeform(); + } + } + + /** + * Get current canvas mode status + */ + public get useCanvasMode(): boolean { + return this._useCanvasMode; + } + + /** + * Get current canvas mode status (for external access) + */ + public getCanvasMode(): boolean { + return this._useCanvasMode; + } + + /** + * Initialize documents based on current mode (canvas or linked) + * This should be called by tools instead of hardcoding initializeFindDocsFreeform + */ + @action + public initializeDocuments() { + if (this._useCanvasMode) { + console.log('[AgentDocumentManager] Initializing canvas documents (canvas mode enabled)'); + this.initializeCanvasDocuments(); + } else { + console.log('[AgentDocumentManager] Initializing linked documents (canvas mode disabled)'); + this.initializeFindDocsFreeform(); + } + } + + /** + * Initialize documents from the entire canvas + */ + @action + public initializeCanvasDocuments() { + try { + console.log('[AgentDocumentManager] Finding all documents on canvas...'); + console.log('[AgentDocumentManager] Canvas mode enabled, looking for all documents...'); + + // Get all canvas documents using CanvasDocsTool + const canvasDocs = CanvasDocsTool.getAllCanvasDocuments(false); + console.log(`[AgentDocumentManager] Found ${canvasDocs.length} documents on canvas`); + + if (canvasDocs.length === 0) { + console.warn('[AgentDocumentManager] No documents found on canvas. This might indicate:'); + console.warn(' 1. No documents are currently rendered/visible'); + console.warn(' 2. All documents are considered "system" documents'); + console.warn(' 3. DocumentView.allViews() is not returning expected results'); + + // Let's also try including system docs to see if that's the issue + const canvasDocsWithSystem = CanvasDocsTool.getAllCanvasDocuments(true); + console.log(`[AgentDocumentManager] With system docs included: ${canvasDocsWithSystem.length} documents`); + } + + // Process each canvas document + canvasDocs.forEach(async (doc: Doc) => { + if (doc && doc !== this.chatBoxDocument) { // Don't process the chatbox itself + console.log('[AgentDocumentManager] Processing canvas document:', doc.id, doc.title, doc.type); + await this.processDocument(doc); + console.log('[AgentDocumentManager] Completed processing document:', doc.id); + } + }); + + } catch (error) { + console.error('[AgentDocumentManager] Error finding documents on canvas:', error); + } + } + + /** + * Gets all documents in the same Freeform view as the ChatBox + * Uses the LinkManager to get all linked documents, similar to how ChatBox does it + */ + public initializeFindDocsFreeform() { + // Reset collections + //this.documentsById.clear(); + + try { + // Use the LinkManager approach which is proven to work in ChatBox + if (this.chatBoxDocument) { + console.log('Finding documents linked to ChatBox document with ID:', this.chatBoxDocument[Id]); + + // Get directly linked documents via LinkManager + const linkedDocs = LinkManager.Instance.getAllRelatedLinks(this.chatBoxDocument) + .map(d => DocCast(LinkManager.getOppositeAnchor(d, this.chatBoxDocument!))) + .map(d => DocCast(d?.annotationOn, d)) + .filter(d => d); + + console.log(`Found ${linkedDocs.length} linked documents via LinkManager`); + + // Process the linked documents + linkedDocs.forEach(async (doc: Doc | undefined) => { + if (doc) { + await this.processDocument(doc); + console.log('Processed linked document:', doc[Id], doc.title, doc.type); + } + }); + } + } catch (error) { + console.error('Error finding documents in Freeform view:', error); + } + } + + public get parentViewDocument(): DocumentView { + return this.parentView; + } + + /** + * Process a document by ensuring it has an ID and adding it to the appropriate collections + * @param doc The document to process + */ + @action + public async processDocument(doc: Doc): Promise<string> { + // Ensure document has a persistent ID + const docId = this.ensureDocumentId(doc); + if (doc.chunk_simplified) { + const newChunks: SimplifiedChunk[] = []; + for (const chunk of JSON.parse(StrCast(doc.chunk_simplified))) { + console.log('chunk', chunk); + newChunks.push(chunk as SimplifiedChunk); + } + console.log('Added simplified chunks to simplifiedChunks:', docId, newChunks); + this.addSimplifiedChunks(newChunks); + //DocCast(DocCast(this.chatBoxDocument!.agentDocument)!.chunk_simpl)!.mapping = new List<string>(Array.from(this.simplifiedChunks.values()).map(chunk => JSON.stringify(chunk))); + } + // Only add if we haven't already processed this document + if (!this.documentsById.has(docId)) { + this.documentsById.set(docId, { layoutDoc: doc, dataDoc: doc[DocData] }); + console.log('Added document to documentsById:', doc[Id], docId, doc[Id], doc[DocData][Id]); + } + return docId; + } + + /** + * Ensures a document has a persistent ID stored in its metadata + * @param doc The document to ensure has an ID + * @returns The document's ID + */ + private ensureDocumentId(doc: Doc): string { + let docId: string | undefined; + + // 1. Try the direct id property if it exists + if (doc[Id]) { + console.log('Found document ID (normal):', doc[Id]); + docId = doc[Id]; + } else { + throw new Error('No document ID found'); + } + + return docId; + } + + /** + * Extracts metadata from a specific document + * @param docId The ID of the document to extract metadata from + * @returns An object containing the document's metadata + */ + public extractDocumentMetadata(id: string) { + if (!id) return null; + const agentDoc = this.documentsById.get(id); + if (!agentDoc) return null; + const layoutDoc = agentDoc.layoutDoc; + const dataDoc = agentDoc.dataDoc; + + const metadata: Record<string, any> = { + // bcz: CHANGE any to a proper type! + id: layoutDoc[Id] || dataDoc[Id] || '', + title: layoutDoc.title || '', + type: layoutDoc.type || '', + fields: { + layout: {}, + data: {}, + }, + fieldLocationMap: {}, + }; + + // Process all known field definitions + Object.keys(this.fieldMetadata).forEach(fieldName => { + // const fieldDef = this.fieldMetadata[fieldName]; + const strippedName = fieldName.startsWith('_') ? fieldName.substring(1) : fieldName; + + // Check if field exists on layout document + let layoutValue = undefined; + if (layoutDoc) { + layoutValue = layoutDoc[fieldName]; + if (layoutValue !== undefined) { + // Field exists on layout document + metadata.fields.layout[fieldName] = this.formatFieldValue(layoutValue); + metadata.fieldLocationMap[strippedName] = 'layout'; + } + } + + // Check if field exists on data document + let dataValue = undefined; + if (dataDoc) { + dataValue = dataDoc[fieldName]; + if (dataValue !== undefined) { + // Field exists on data document + metadata.fields.data[fieldName] = this.formatFieldValue(dataValue); + if (!metadata.fieldLocationMap[strippedName]) { + metadata.fieldLocationMap[strippedName] = 'data'; + } + } + } + + // For fields with stripped names (without leading underscore), + // also check if they exist on documents without the underscore + if (fieldName.startsWith('_')) { + const nonUnderscoreFieldName = fieldName.substring(1); + + if (layoutDoc) { + const nonUnderscoreLayoutValue = layoutDoc[nonUnderscoreFieldName]; + if (nonUnderscoreLayoutValue !== undefined) { + metadata.fields.layout[nonUnderscoreFieldName] = this.formatFieldValue(nonUnderscoreLayoutValue); + metadata.fieldLocationMap[nonUnderscoreFieldName] = 'layout'; + } + } + + if (dataDoc) { + const nonUnderscoreDataValue = dataDoc[nonUnderscoreFieldName]; + if (nonUnderscoreDataValue !== undefined) { + metadata.fields.data[nonUnderscoreFieldName] = this.formatFieldValue(nonUnderscoreDataValue); + if (!metadata.fieldLocationMap[nonUnderscoreFieldName]) { + metadata.fieldLocationMap[nonUnderscoreFieldName] = 'data'; + } + } + } + } + }); + + // Add common field aliases for easier discovery + // This helps users understand both width and _width refer to the same property + if (metadata.fields.layout._width !== undefined && metadata.fields.layout.width === undefined) { + metadata.fields.layout.width = metadata.fields.layout._width; + metadata.fieldLocationMap.width = 'layout'; + } + + if (metadata.fields.layout._height !== undefined && metadata.fields.layout.height === undefined) { + metadata.fields.layout.height = metadata.fields.layout._height; + metadata.fieldLocationMap.height = 'layout'; + } + + return metadata; + } + + /** + * Formats a field value for JSON output + * @param value The field value to format + * @returns A JSON-friendly representation of the field value + */ + private formatFieldValue(value: FieldResult | undefined) { + if (value === undefined || value === null) { + return null; + } + + // Handle Doc objects + if (value instanceof Doc) { + return { + type: 'Doc', + id: value[Id] || this.ensureDocumentId(value), + title: value.title || '', + docType: value.type || '', + }; + } + + // Handle RichTextField (try to extract plain text) + if (typeof value === 'string' && value.includes('"type":"doc"') && value.includes('"content":')) { + try { + const rtfObj = JSON.parse(value); + // If this looks like a rich text field structure + if (rtfObj.doc && rtfObj.doc.content) { + // Recursively extract text from the content + let plainText = ''; + const extractText = (node: { text: string; content?: unknown[] }) => { + if (node.text) { + plainText += node.text; + } + if (node.content && Array.isArray(node.content)) { + node.content.forEach(child => extractText(child as { text: string; content?: unknown[] })); + } + }; + + extractText(rtfObj.doc); + + // If we successfully extracted text, show it, but also preserve the original value + if (plainText) { + return { + type: 'RichText', + text: plainText, + length: plainText.length, + // Don't include the full value as it can be very large + }; + } + } + } catch { + // If parsing fails, just treat as a regular string + } + } + + // Handle arrays and complex objects + if (typeof value === 'object') { + // If the object has a toString method, use it + if (value.toString && value.toString !== Object.prototype.toString) { + return value.toString(); + } + + try { + // Try to convert to JSON string + return JSON.stringify(value); + } catch { + return '[Complex Object]'; + } + } + + // Return primitive values as is + return value; + } + + /** + * Converts a string field value to the appropriate type based on field metadata + * @param fieldName The name of the field + * @param fieldValue The string value to convert + * @returns The converted value with the appropriate type + */ + private convertFieldValue(fieldName: string, fieldValueIn: string | number | boolean): FieldResult | undefined { + // If fieldValue is already a number or boolean, we don't need to convert it from string + if (typeof fieldValueIn === 'number' || typeof fieldValueIn === 'boolean') { + return fieldValueIn; + } + + // If fieldValue is a string "true" or "false", convert to boolean + if (typeof fieldValueIn === 'string') { + if (fieldValueIn.toLowerCase() === 'true') { + return true; + } + if (fieldValueIn.toLowerCase() === 'false') { + return false; + } + } + + // coerce fieldvValue to a string + const fieldValue = typeof fieldValueIn !== 'string' ? String(fieldValueIn) : fieldValueIn; + + // Special handling for text field - convert to proper RichTextField format + if (fieldName === 'text') { + try { + // Check if it's already a valid JSON RichTextField + JSON.parse(fieldValue); + return fieldValue; + } catch { + // It's a plain text string, so convert it to RichTextField format + const rtf = { + doc: { + type: 'doc', + content: [ + { + type: 'paragraph', + content: [ + { + type: 'text', + text: fieldValue, + }, + ], + }, + ], + }, + }; + return JSON.stringify(rtf); + } + } + + // Get field metadata + const normalizedFieldName = fieldName.startsWith('_') ? fieldName : `_${fieldName}`; + const strippedFieldName = fieldName.startsWith('_') ? fieldName.substring(1) : fieldName; + + // Check both versions of the field name in metadata + const fieldMeta = this.fieldMetadata[normalizedFieldName] || this.fieldMetadata[strippedFieldName]; + + // Special handling for width and height without metadata + if (!fieldMeta && (fieldName === '_width' || fieldName === '_height' || fieldName === 'width' || fieldName === 'height')) { + const num = Number(fieldValue); + return isNaN(num) ? fieldValue : num; + } + + if (!fieldMeta) { + // If no metadata found, just return the string value + return fieldValue; + } + + // Convert based on field type + const fieldType = fieldMeta.type; + + if (fieldType === 'boolean') { + // Convert to boolean + return fieldValue.toLowerCase() === 'true'; + } else if (fieldType === 'number') { + // Convert to number + const num = Number(fieldValue); + return isNaN(num) ? fieldValue : num; + } else if (fieldType === 'date') { + // Try to convert to date (stored as number timestamp) + try { + return new Date(fieldValue).getTime(); + } catch { + return fieldValue; + } + } else if (fieldType.includes('list') || fieldType.includes('array')) { + // Try to parse as JSON array + try { + return JSON.parse(fieldValue) as FieldResult; // bcz: this needs to be typed properly. Dash fields can't accept a generic 'objext' + } catch { + return fieldValue; + } + } else if (fieldType === 'json' || fieldType === 'object') { + // Try to parse as JSON object + try { + return JSON.parse(fieldValue) as FieldResult; // bcz: this needs to be typed properly. Dash fields can't accept a generic 'objext' + } catch { + return fieldValue; + } + } + + // Default to string + return fieldValue; + } + + /** + * Extracts all field metadata from DocumentOptions + * @returns A structured object containing metadata about all available document fields + */ + public getAllFieldMetadata() { + // Start with our already populated fieldMetadata from the DocumentOptions class + const result: Record<string, any> = { + // bcz: CHANGE any to a proper type! + fieldCount: Object.keys(this.fieldMetadata).length, + fields: {}, + fieldsByType: { + string: [], + number: [], + boolean: [], + //doc: [], + //list: [], + //date: [], + //enumeration: [], + //other: [], + }, + fieldNameMappings: {}, + commonFields: { + appearance: [], + position: [], + size: [], + content: [], + behavior: [], + layout: [], + }, + }; + + // Process each field in the metadata + Object.entries(this.fieldMetadata).forEach(([fieldName, fieldInfo]) => { + const strippedName = fieldName.startsWith('_') ? fieldName.substring(1) : fieldName; + + // Add to fieldNameMappings + if (fieldName.startsWith('_')) { + result.fieldNameMappings[strippedName] = fieldName; + } + + // Create structured field metadata + const fieldData: Record<string, any> = { + // bcz: CHANGE any to a proper type! + name: fieldName, + displayName: strippedName, + description: fieldInfo.description || '', + type: fieldInfo.fieldType || 'unknown', + possibleValues: fieldInfo.values || [], + }; + + // Add field to fields collection + result.fields[fieldName] = fieldData; + + // Categorize by field type + const type = fieldInfo.fieldType?.toLowerCase() || 'unknown'; + if (type === 'string') { + result.fieldsByType.string.push(fieldName); + } else if (type === 'number') { + result.fieldsByType.number.push(fieldName); + } else if (type === 'boolean') { + result.fieldsByType.boolean.push(fieldName); + } else if (type === 'doc') { + //result.fieldsByType.doc.push(fieldName); + } else if (type === 'list') { + //result.fieldsByType.list.push(fieldName); + } else if (type === 'date') { + //result.fieldsByType.date.push(fieldName); + } else if (type === 'enumeration') { + //result.fieldsByType.enumeration.push(fieldName); + } else { + //result.fieldsByType.other.push(fieldName); + } + + // Categorize by field purpose + if (fieldName.includes('width') || fieldName.includes('height') || fieldName.includes('size')) { + result.commonFields.size.push(fieldName); + } else if (fieldName.includes('color') || fieldName.includes('background') || fieldName.includes('border')) { + result.commonFields.appearance.push(fieldName); + } else if (fieldName.includes('x') || fieldName.includes('y') || fieldName.includes('position') || fieldName.includes('pan')) { + result.commonFields.position.push(fieldName); + } else if (fieldName.includes('text') || fieldName.includes('title') || fieldName.includes('data')) { + result.commonFields.content.push(fieldName); + } else if (fieldName.includes('action') || fieldName.includes('click') || fieldName.includes('event')) { + result.commonFields.behavior.push(fieldName); + } else if (fieldName.includes('layout')) { + result.commonFields.layout.push(fieldName); + } + }); + + // Add special section for auto-sizing related fields + result.autoSizingFields = { + height: { + autoHeightField: '_layout_autoHeight', + heightField: '_height', + displayName: 'height', + usage: 'To manually set height, first set layout_autoHeight to false', + }, + width: { + autoWidthField: '_layout_autoWidth', + widthField: '_width', + displayName: 'width', + usage: 'To manually set width, first set layout_autoWidth to false', + }, + }; + + // Add special section for text field format + result.specialFields = { + text: { + name: 'text', + description: 'Document text content', + format: 'RichTextField', + note: 'When setting text, provide plain text - it will be automatically converted to the correct format', + example: 'For setting: "Hello world" (plain text); For getting: Will be converted to plaintext for display', + }, + }; + + return result; + } + + /** + * Edits a specific field on a document + * @param docId The ID of the document to edit + * @param fieldName The name of the field to edit + * @param fieldValue The new value for the field (string, number, or boolean) + * @returns Object with success status, message, and additional information + */ + public editDocumentField( + docId: string, + fieldName: string, + fieldValue: string | number | boolean + ): { + success: boolean; + message: string; + fieldName?: string; + originalFieldName?: string; + newValue?: string | number | boolean | object; + warning?: string; + } { + // Normalize field name (handle with/without underscore) + let normalizedFieldName = fieldName.startsWith('_') ? fieldName : fieldName; + // const strippedFieldName = fieldName.startsWith('_') ? fieldName.substring(1) : fieldName; + + // Handle common field name aliases (width → _width, height → _height) + // Many document fields use '_' prefix for layout properties + if (fieldName === 'width') { + normalizedFieldName = '_width'; + } else if (fieldName === 'height') { + normalizedFieldName = '_height'; + } + + // Get the documents + const doc = this.documentsById.get(docId); + if (!doc) { + return { success: false, message: `Document with ID ${docId} not found` }; + } + + const { layoutDoc, dataDoc } = this.documentsById.get(docId) ?? { layoutDoc: null, dataDoc: null }; + + if (!layoutDoc && !dataDoc) { + return { success: false, message: `Could not find layout or data document for document with ID ${docId}` }; + } + + try { + // Convert the field value to the appropriate type based on field metadata + const convertedValue = this.convertFieldValue(normalizedFieldName, fieldValue); + + let targetDoc: Doc | undefined; + let targetLocation: string; + + // First, check if field exists on layout document using Doc.Get + if (layoutDoc) { + const fieldExistsOnLayout = Doc.Get(layoutDoc, normalizedFieldName, true) !== undefined; + + // If it exists on layout document, update it there + if (fieldExistsOnLayout) { + targetDoc = layoutDoc; + targetLocation = 'layout'; + } + // If it has an underscore prefix, it's likely a layout property even if not yet set + else if (normalizedFieldName.startsWith('_')) { + targetDoc = layoutDoc; + targetLocation = 'layout'; + } + // Otherwise, look for or create on data document + else if (dataDoc) { + targetDoc = dataDoc; + targetLocation = 'data'; + } + // If no data document available, default to layout + else { + targetDoc = layoutDoc; + targetLocation = 'layout'; + } + } + // If no layout document, use data document + else if (dataDoc) { + targetDoc = dataDoc; + targetLocation = 'data'; + } else { + return { success: false, message: `No valid document found for editing` }; + } + + if (!targetDoc) { + return { success: false, message: `Target document not available` }; + } + + // Set the field value on the target document + targetDoc[normalizedFieldName] = convertedValue; // bcz: converteValue needs to be typed properly. Dash fields can't accept a generic 'objext' + + return { + success: true, + message: `Successfully updated field '${normalizedFieldName}' on ${targetLocation} document (ID: ${docId})`, + fieldName: normalizedFieldName, + originalFieldName: fieldName, + newValue: convertedValue, + }; + } catch (error) { + console.error('Error editing document field:', error); + return { + success: false, + message: `Error updating field: ${error instanceof Error ? error.message : String(error)}`, + }; + } + } + /** + * Gets metadata for a specific document or all documents + * @param documentId Optional ID of a specific document to get metadata for + * @returns Document metadata or metadata for all documents + */ + public getDocumentMetadata(documentId?: string) { + if (documentId) { + console.log(`Returning document metadata for docID, ${documentId}:`, this.extractDocumentMetadata(documentId)); + return this.extractDocumentMetadata(documentId); + } else { + // Get metadata for all documents + const documentsMetadata: Record<string, Record<string, any>> = {}; // bcz: CHANGE any to a proper type! + for (const docid of this.documentsById.keys()) { + const metadata = this.extractDocumentMetadata(docid); + if (metadata) { + documentsMetadata[docid] = metadata; + } else { + console.warn(`No metadata found for document with ID: ${docid}`); + } + } + return { + documentCount: this.documentsById.size, + documents: documentsMetadata, + //fieldDefinitions: this.fieldMetadata, // TODO: remove this, if fieldDefinitions are not needed. + }; + } + } + + /** + * Adds links between documents based on their IDs + * @param docIds Array of document IDs to link + * @param relationship Optional relationship type for the links + * @returns Array of created link documents + */ + public addLinks(docIds: string[]): Doc[] { + const createdLinks: Doc[] = []; + // Use string keys for Set instead of arrays which don't work as expected as keys + const alreadyLinked = new Set<string>(); + + // Iterate over the document IDs and add links + docIds.forEach(docId1 => { + const doc1 = this.documentsById.get(docId1); + docIds.forEach(docId2 => { + if (docId1 === docId2) return; // Skip self-linking + + // Create a consistent key regardless of document order + const linkKey = [docId1, docId2].sort().join('_'); + if (alreadyLinked.has(linkKey)) return; + + const doc2 = this.documentsById.get(docId2); + if (doc1?.layoutDoc && doc2?.layoutDoc) { + try { + // Create a link document between doc1 and doc2 + const linkDoc = Docs.Create.LinkDocument(doc1.layoutDoc, doc2.layoutDoc); + + // Set a default color if relationship doesn't specify one + if (!linkDoc.color) { + linkDoc.color = 'lightBlue'; // Default blue color + } + + // Ensure link is visible by setting essential properties + linkDoc.link_visible = true; + linkDoc.link_enabled = true; + linkDoc.link_autoMove = true; + linkDoc.link_showDirected = true; + + // Set the embedContainer to ensure visibility + // This is shown in the image as a key difference between visible/non-visible links + if (this.chatBoxDocument && this.chatBoxDocument.parent && typeof this.chatBoxDocument.parent === 'object' && 'title' in this.chatBoxDocument.parent) { + linkDoc.embedContainer = String(this.chatBoxDocument.parent.title); + } else if (doc1.layoutDoc.parent && typeof doc1.layoutDoc.parent === 'object' && 'title' in doc1.layoutDoc.parent) { + linkDoc.embedContainer = String(doc1.layoutDoc.parent.title); + } else { + // Default to a tab name if we can't find one + linkDoc.embedContainer = 'Untitled Tab 1'; + } + + // Add the link to the document system + LinkManager.Instance.addLink(linkDoc); + + const ancestor = DocumentView.linkCommonAncestor(linkDoc); + ancestor?.ComponentView?.addDocument?.(linkDoc); + // Add to user document list to make it visible in the UI + Doc.AddDocToList(Doc.UserDoc(), 'links', linkDoc); + + // Create a visual link for display + if (this.chatBoxDocument) { + // Make sure the docs are visible in the UI + this.chatBox._props.addDocument?.(doc1.layoutDoc); + this.chatBox._props.addDocument?.(doc2.layoutDoc); + + // Use DocumentManager to ensure documents are visible + DocumentManager.Instance.showDocument(doc1.layoutDoc, { willZoomCentered: false }); + DocumentManager.Instance.showDocument(doc2.layoutDoc, { willZoomCentered: false }); + } + + createdLinks.push(linkDoc); + alreadyLinked.add(linkKey); + } catch (error) { + console.error('Error creating link between documents:', error); + } + } + }); + }); + + // Force update of the UI to show new links + setTimeout(() => { + try { + // Update server cache to ensure links are persisted + UPDATE_SERVER_CACHE && typeof UPDATE_SERVER_CACHE === 'function' && UPDATE_SERVER_CACHE(); + } catch (e) { + console.warn('Could not update server cache after creating links:', e); + } + }, 100); + + return createdLinks; + } + /** + * Helper method to validate a document type and ensure it's a valid supportedDocType + * @param docType The document type to validate + * @returns True if the document type is valid, false otherwise + */ + private isValidDocType(docType: string): boolean { + return Object.values(supportedDocTypes).includes(docType as supportedDocTypes); + } + /** + * Creates a document in the dashboard and returns its ID. + * This is a public API used by tools like SearchTool. + * + * @param docType The type of document to create + * @param data The data for the document + * @param options Optional configuration options + * @returns The ID of the created document + */ + + public async createDocInDash(docType: string, data: string, options?: DocumentOptions): Promise<string> { + // Validate doc_type + if (!this.isValidDocType(docType)) { + throw new Error(`Invalid document type: ${docType}`); + } + + try { + // Create simple document with just title and data + const simpleDoc: parsedDoc = { + ...(options as parsedDoc), // bcz: hack .. why do we need parsedDoc and not DocumentOptions here? + doc_type: docType, + title: options?.title ?? `Untitled Document ${this.documentsById.size + 1}`, + data: data, + x: options?.x ?? 0, + y: options?.y ?? 0, + _width: 300, + _height: 300, + _layout_fitWidth: false, + _layout_autoHeight: true, + }; + + // Additional handling for web documents + if (docType === 'web') { + // For web documents, don't sanitize the URL here + // Instead, set properties to handle content safely when loaded + simpleDoc._disable_resource_loading = true; + simpleDoc._sandbox_iframe = true; + simpleDoc.data_useCors = true; + + // Specify a more permissive sandbox to allow content to render properly + // but still maintain security + simpleDoc._iframe_sandbox = 'allow-same-origin allow-scripts allow-popups allow-forms'; + } + + // Use the chatBox's createDocInDash method to create the document + if (!this.chatBox) { + throw new Error('ChatBox instance not available for creating document'); + } + + const doc = this.chatBox.whichDoc(simpleDoc, false); + if (doc) { + // Use MobX runInAction to properly modify observable state + runInAction(() => { + if (this.chatBoxDocument && doc) { + // Create link and add it to the document system + const linkDoc = Docs.Create.LinkDocument(this.chatBoxDocument, doc); + LinkManager.Instance.addLink(linkDoc); + if (doc.type !== 'web') { + // Add document to view + this.chatBox._props.addDocument?.(doc); + + // Show document - defer actual display to prevent immediate resource loading + setTimeout(() => { + DocumentManager.Instance.showDocument(doc, { willZoomCentered: true }, () => {}); + }, 100); + } + } + }); + + const id = await this.processDocument(doc); + return id; + } else { + throw new Error(`Error creating document. Created document not found.`); + } + } catch (error) { + throw new Error(`Error creating document: ${error}`); + } + } + + /** + * Sanitizes web content to prevent errors with external resources + * @param content The web content to sanitize + * @returns Sanitized content + */ + private sanitizeWebContent(content: string): string { + if (!content) return content; + + try { + // Replace problematic resource references that might cause errors + const sanitized = content + // Remove preload links that might cause errors + .replace(/<link[^>]*rel=["']preload["'][^>]*>/gi, '') + // Remove map file references + .replace(/\/\/# sourceMappingURL=.*\.map/gi, '') + // Remove external CSS map files references + .replace(/\/\*# sourceMappingURL=.*\.css\.map.*\*\//gi, '') + // Add sandbox to iframes + .replace(/<iframe/gi, '<iframe sandbox="allow-same-origin" loading="lazy"') + // Prevent automatic resource loading for images + .replace(/<img/gi, '<img loading="lazy"') + // Prevent automatic resource loading for scripts + .replace(/<script/gi, '<script type="text/disabled"') + // Handle invalid URIs by converting relative URLs to absolute ones + .replace(/href=["'](\/[^"']+)["']/gi, (match, p1) => { + // Only handle relative URLs starting with / + if (p1.startsWith('/')) { + return `href="#disabled-link"`; + } + return match; + }) + // Prevent automatic loading of CSS + .replace(/<link[^>]*rel=["']stylesheet["'][^>]*href=["']([^"']+)["']/gi, (match, href) => `<link rel="prefetch" data-original-href="${href}" />`); + + // Wrap the content in a sandboxed container + return ` + <div class="sandboxed-web-content"> + <style> + /* Override styles to prevent external resource loading */ + @font-face { font-family: 'disabled'; src: local('Arial'); } + * { font-family: system-ui, -apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, Oxygen, Ubuntu, Cantarell, 'Open Sans', 'Helvetica Neue', sans-serif !important; } + img, iframe, frame, embed, object { max-width: 100%; } + </style> + ${sanitized} + </div>`; + } catch (e) { + console.warn('Error sanitizing web content:', e); + // Fall back to a safe container with the content as text + return ` + <div class="sandboxed-web-content"> + <p>Content could not be safely displayed. Raw content:</p> + <pre>${content.replace(/</g, '<').replace(/>/g, '>')}</pre> + </div>`; + } + } + + public has(docId: string) { + return this.documentsById.has(docId); + } + + /** + * Returns a list of all document IDs in the manager. + * @returns An array of document IDs (strings). + */ + @computed + public get listDocs(): string { + const xmlDocs = Array.from(this.documentsById.entries()).map(([id, agentDoc]) => { + return `<document> + <id>${id}</id> + <title>${this.escapeXml(StrCast(agentDoc.layoutDoc.title))}</title> + <type>${this.escapeXml(StrCast(agentDoc.layoutDoc.type))}</type> + <summary>${this.escapeXml(StrCast(agentDoc.layoutDoc.summary))}</summary> +</document>`; + }); + + return xmlDocs.join('\n'); + } + + private escapeXml(str: string): string { + return str.replace(/&/g, '&').replace(/</g, '<').replace(/>/g, '>').replace(/"/g, '"').replace(/'/g, '''); + } + + @computed + public get docIds(): string[] { + return Array.from(this.documentsById.keys()); + } + + /** + * Gets a document by its ID + * @param docId The ID of the document to retrieve + * @returns The document if found, undefined otherwise + */ + public getDocument(docId: string): Doc | undefined { + const docInfo = this.documentsById.get(docId); + return docInfo?.layoutDoc; + } + + public getDataDocument(docId: string): Doc | undefined { + const docInfo = this.documentsById.get(docId); + return docInfo?.dataDoc; + } + + // In AgentDocumentManager + private descriptionCache = new Map<string, string>(); + + public async getDocDescription(id: string): Promise<string> { + if (!this.descriptionCache.has(id)) { + const doc = this.getDocument(id)!; + const desc = await Doc.getDescription(doc); + this.descriptionCache.set(id, desc.replace(/\n/g, ' ').trim()); + } + return this.descriptionCache.get(id)!; + } + + /** + * Adds simplified chunks to a document for citation handling + * @param doc The document to add simplified chunks to + * @param chunks Array of full RAG chunks to simplify + * @param docType The type of document (e.g., 'pdf', 'video', 'audio', etc.) + * @returns The updated document with simplified chunks + */ + @action + public addSimplifiedChunks(simplifiedChunks: SimplifiedChunk[]) { + simplifiedChunks.forEach(chunk => { + this.simplifiedChunks.set(chunk.chunkId, chunk); + }); + } + + public getSimplifiedChunks(chunks: RAGChunk[], docType: string): SimplifiedChunk[] { + console.log('chunks', chunks, 'simplifiedChunks', this.simplifiedChunks); + const simplifiedChunks: SimplifiedChunk[] = []; + // Create array of simplified chunks based on document type + for (const chunk of chunks) { + // Common properties across all chunk types + const baseChunk: SimplifiedChunk = { + chunkId: chunk.id, + //text: chunk.metadata.text, + doc_id: chunk.metadata.doc_id, + chunkType: chunk.metadata.type || CHUNK_TYPE.TEXT, + }; + + // Add type-specific properties + if (docType === 'video' || docType === 'audio') { + simplifiedChunks.push({ + ...baseChunk, + start_time: chunk.metadata.start_time, + end_time: chunk.metadata.end_time, + indexes: chunk.metadata.indexes, + chunkType: docType === 'video' ? CHUNK_TYPE.VIDEO : CHUNK_TYPE.AUDIO, + } as SimplifiedChunk); + } else if (docType === 'pdf') { + simplifiedChunks.push({ + ...baseChunk, + startPage: chunk.metadata.start_page, + endPage: chunk.metadata.end_page, + location: chunk.metadata.location, + } as SimplifiedChunk); + } else if (docType === 'csv' && 'row_start' in chunk.metadata && 'row_end' in chunk.metadata && 'col_start' in chunk.metadata && 'col_end' in chunk.metadata) { + simplifiedChunks.push({ + ...baseChunk, + rowStart: chunk.metadata.row_start, + rowEnd: chunk.metadata.row_end, + colStart: chunk.metadata.col_start, + colEnd: chunk.metadata.col_end, + } as SimplifiedChunk); + } else { + // Default for other document types + simplifiedChunks.push(baseChunk as SimplifiedChunk); + } + } + return simplifiedChunks; + } + + /** + * Gets a specific simplified chunk by ID + * @param doc The document containing chunks + * @param chunkId The ID of the chunk to retrieve + * @returns The simplified chunk if found, undefined otherwise + */ + @action + public getSimplifiedChunkById(chunkId: string) { + return { foundChunk: this.simplifiedChunks.get(chunkId), doc: this.getDocument(this.simplifiedChunks.get(chunkId)?.doc_id || chunkId), dataDoc: this.getDataDocument(this.simplifiedChunks.get(chunkId)?.doc_id || chunkId) }; + } + + public getChunkIdsFromDocIds(docIds: string[]): string[] { + return docIds + .map(docId => { + for (const chunk of this.simplifiedChunks.values()) { + if (chunk.doc_id === docId) { + return chunk.chunkId; + } + } + }) + .filter(chunkId => chunkId !== undefined) as string[]; + } + + /** + * Gets the original segments from a media document + * @param doc The document containing original media segments + * @returns Array of media segments or empty array if none exist + */ + public getOriginalSegments(doc: Doc): { text: string; index: string; start: number }[] { + if (!doc || !doc.original_segments) { + return []; + } + + try { + return JSON.parse(StrCast(doc.original_segments)) || []; + } catch (e) { + console.error('Error parsing original segments:', e); + return []; + } + } +} |