diff options
Diffstat (limited to 'src/client/views/nodes/chatbot/utils/AgentDocumentManager.ts')
| -rw-r--r-- | src/client/views/nodes/chatbot/utils/AgentDocumentManager.ts | 1237 |
1 files changed, 1237 insertions, 0 deletions
diff --git a/src/client/views/nodes/chatbot/utils/AgentDocumentManager.ts b/src/client/views/nodes/chatbot/utils/AgentDocumentManager.ts new file mode 100644 index 000000000..a27220898 --- /dev/null +++ b/src/client/views/nodes/chatbot/utils/AgentDocumentManager.ts @@ -0,0 +1,1237 @@ +import { action, makeObservable, observable, ObservableMap, reaction, runInAction } from 'mobx'; +import { observer } from 'mobx-react'; +import { v4 as uuidv4 } from 'uuid'; +import { Doc, StrListCast } from '../../../../../fields/Doc'; +import { DocData } from '../../../../../fields/DocSymbols'; +import { Id } from '../../../../../fields/FieldSymbols'; +import { List } from '../../../../../fields/List'; +import { DocCast, StrCast } from '../../../../../fields/Types'; +import { DocServer } from '../../../../DocServer'; +import { Docs, DocumentOptions } from '../../../../documents/Documents'; +import { DocumentManager } from '../../../../util/DocumentManager'; +import { LinkManager, UPDATE_SERVER_CACHE } from '../../../../util/LinkManager'; +import { DocumentView } from '../../DocumentView'; +import { ChatBox, parsedDoc } from '../chatboxcomponents/ChatBox'; +import { supportedDocTypes } from '../types/tool_types'; +import { CHUNK_TYPE, RAGChunk } from '../types/types'; + +/** + * Interface representing a document in the freeform view + */ +interface AgentDocument { + layoutDoc: Doc; + dataDoc: Doc; +} + +/** + * Class to manage documents in a freeform view + */ +export class AgentDocumentManager { + @observable private documentsById: ObservableMap<string, AgentDocument>; + private chatBox: ChatBox; + private chatBoxDocument: Doc | null = null; + private fieldMetadata: Record<string, any> = {}; + private readonly DOCUMENT_ID_FIELD = '_dash_document_id'; + + /** + * Creates a new DocumentManager + * @param templateDocument The document that serves as a template for new documents + */ + constructor(chatBox: ChatBox) { + makeObservable(this); + const agentDoc = DocCast(chatBox.Document.agentDocument) ?? new Doc(); + agentDoc.title = chatBox.Document.title + '_agentDocument'; + chatBox.Document.agentDocument = agentDoc; + this.documentsById = StrListCast(agentDoc.mapping).reduce((mapping, content) => { + const [id, layoutId, docId] = content.split(':'); + const layoutDoc = DocServer.GetCachedRefField(layoutId); + const dataDoc = DocServer.GetCachedRefField(docId); + if (!layoutDoc || !dataDoc) { + console.warn(`Document with ID ${id} not found in mapping`); + } else { + mapping.set(id, { layoutDoc, dataDoc }); + } + return mapping; + }, new ObservableMap<string, AgentDocument>()); + console.log(`AgentDocumentManager initialized with ${this.documentsById.size} documents`); + this.chatBox = chatBox; + this.chatBoxDocument = chatBox.Document; + + reaction( + () => this.documentsById.values(), + () => { + if (this.chatBoxDocument && DocCast(this.chatBoxDocument.agentDocument)) { + DocCast(this.chatBoxDocument.agentDocument)!.mapping = new List<string>(Array.from(this.documentsById.entries()).map(([id, agent]) => `${id}:${agent.dataDoc[Id]}:${agent.layoutDoc[Id]}`)); + } + } + //{ fireImmediately: true } + ); + this.processDocument(this.chatBoxDocument); + this.initializeFieldMetadata(); + } + + /** + * Extracts field metadata from DocumentOptions class + */ + private initializeFieldMetadata() { + // Parse DocumentOptions to extract field definitions + const documentOptionsInstance = new DocumentOptions(); + const documentOptionsEntries = Object.entries(documentOptionsInstance); + + for (const [fieldName, fieldInfo] of documentOptionsEntries) { + // Extract field information + const fieldData: Record<string, any> = { + name: fieldName, + withoutUnderscore: fieldName.startsWith('_') ? fieldName.substring(1) : fieldName, + description: '', + type: 'unknown', + required: false, + defaultValue: undefined, + possibleValues: [], + }; + + // Check if fieldInfo has description property (it's likely a FInfo instance) + if (fieldInfo && typeof fieldInfo === 'object' && 'description' in fieldInfo) { + fieldData.description = fieldInfo.description; + + // Extract field type if available + if ('fieldType' in fieldInfo) { + fieldData.type = fieldInfo.fieldType; + } + + // Extract possible values if available + if ('values' in fieldInfo && Array.isArray(fieldInfo.values)) { + fieldData.possibleValues = fieldInfo.values; + } + } + + this.fieldMetadata[fieldName] = fieldData; + } + } + + /** + * Gets all documents in the same Freeform view as the ChatBox + * Uses the LinkManager to get all linked documents, similar to how ChatBox does it + */ + public initializeFindDocsFreeform() { + // Reset collections + //this.documentsById.clear(); + + try { + // Use the LinkManager approach which is proven to work in ChatBox + if (this.chatBoxDocument) { + console.log('Finding documents linked to ChatBox document with ID:', this.chatBoxDocument.id); + + // Get directly linked documents via LinkManager + const linkedDocs = LinkManager.Instance.getAllRelatedLinks(this.chatBoxDocument) + .map(d => DocCast(LinkManager.getOppositeAnchor(d, this.chatBoxDocument!))) + .map(d => DocCast(d?.annotationOn, d)) + .filter(d => d); + + console.log(`Found ${linkedDocs.length} linked documents via LinkManager`); + + // Process the linked documents + linkedDocs.forEach((doc: Doc) => { + if (doc) { + this.processDocument(doc); + console.log('Processed linked document:', doc.id, doc.title, doc.type); + } + }); + + // Include the ChatBox document itself + this.processDocument(this.chatBoxDocument); + + // If we have access to the Document's parent, try to find sibling documents + if (this.chatBoxDocument.parent) { + const parent = this.chatBoxDocument.parent; + console.log('Found parent document, checking for siblings'); + + // Check if parent is a Doc type and has a childDocs function + if (parent && typeof parent === 'object' && 'childDocs' in parent && typeof parent.childDocs === 'function') { + try { + const siblingDocs = parent.childDocs(); + if (Array.isArray(siblingDocs)) { + console.log(`Found ${siblingDocs.length} sibling documents via parent.childDocs()`); + siblingDocs.forEach((doc: Doc) => { + if (doc) { + this.processDocument(doc); + } + }); + } + } catch (e) { + console.warn('Error accessing parent.childDocs:', e); + } + } + } + } else if (this.chatBox && this.chatBox.linkedDocs) { + // If we have direct access to the linkedDocs computed property from ChatBox + console.log('Using ChatBox.linkedDocs directly'); + const linkedDocs = this.chatBox.linkedDocs; + if (Array.isArray(linkedDocs)) { + console.log(`Found ${linkedDocs.length} documents via ChatBox.linkedDocs`); + linkedDocs.forEach((doc: Doc) => { + if (doc) { + this.processDocument(doc); + } + }); + } + + // Process the ChatBox document if available + if (this.chatBox.Document) { + this.processDocument(this.chatBox.Document); + } + } else { + console.warn('No ChatBox document reference available for finding linked documents'); + } + + console.log(`DocumentMetadataTool found ${this.documentsById.size} total documents`); + } catch (error) { + console.error('Error finding documents in Freeform view:', error); + } + } + + /** + * Process a document by ensuring it has an ID and adding it to the appropriate collections + * @param doc The document to process + */ + @action + public processDocument(doc: Doc): string { + // Ensure document has a persistent ID + const docId = this.ensureDocumentId(doc); + // Only add if we haven't already processed this document + if (!this.documentsById.has(docId)) { + this.documentsById.set(docId, { layoutDoc: doc, dataDoc: doc[DocData] }); + } + return docId; + } + + /** + * Ensures a document has a persistent ID stored in its metadata + * @param doc The document to ensure has an ID + * @returns The document's ID + */ + private ensureDocumentId(doc: Doc): string { + let docId: string | undefined; + + // First try to get the ID from our custom field + if (doc[this.DOCUMENT_ID_FIELD]) { + docId = String(doc[this.DOCUMENT_ID_FIELD]); + return docId; + } + + // Try different ways to get a document ID + + // 1. Try the direct id property if it exists + if (doc.id && typeof doc.id === 'string') { + docId = doc.id; + } + // 2. Try doc._id if it exists + else if (doc._id && typeof doc._id === 'string') { + docId = doc._id; + } + // 3. Try doc.data?.id if it exists + else if (doc.data && typeof doc.data === 'object' && 'id' in doc.data && typeof doc.data.id === 'string') { + docId = doc.data.id; + } + // 4. If none of the above work, generate a UUID + else { + docId = uuidv4(); + console.log(`Generated new UUID for document with title: ${doc.title || 'Untitled'}`); + } + + // Store the ID in the document's metadata so it persists + try { + doc[this.DOCUMENT_ID_FIELD] = docId; + } catch (e) { + console.warn(`Could not assign ID to document property`, e); + } + + return docId; + } + + /** + * Extracts metadata from a specific document + * @param docId The ID of the document to extract metadata from + * @returns An object containing the document's metadata + */ + public extractDocumentMetadata(id: string) { + if (!id) return null; + const doc = this.documentsById.get(id); + if (!doc) return null; + const layoutDoc = doc.layoutDoc; + const dataDoc = doc.dataDoc; + + const metadata: Record<string, any> = { + id: layoutDoc.dash_document_id || layoutDoc.id || '', + title: layoutDoc.title || '', + type: layoutDoc.type || '', + fields: { + layout: {}, + data: {}, + }, + fieldLocationMap: {}, + }; + + // Process all known field definitions + Object.keys(this.fieldMetadata).forEach(fieldName => { + const fieldDef = this.fieldMetadata[fieldName]; + const strippedName = fieldName.startsWith('_') ? fieldName.substring(1) : fieldName; + + // Check if field exists on layout document + let layoutValue = undefined; + if (layoutDoc) { + layoutValue = layoutDoc[fieldName]; + if (layoutValue !== undefined) { + // Field exists on layout document + metadata.fields.layout[fieldName] = this.formatFieldValue(layoutValue); + metadata.fieldLocationMap[strippedName] = 'layout'; + } + } + + // Check if field exists on data document + let dataValue = undefined; + if (dataDoc) { + dataValue = dataDoc[fieldName]; + if (dataValue !== undefined) { + // Field exists on data document + metadata.fields.data[fieldName] = this.formatFieldValue(dataValue); + if (!metadata.fieldLocationMap[strippedName]) { + metadata.fieldLocationMap[strippedName] = 'data'; + } + } + } + + // For fields with stripped names (without leading underscore), + // also check if they exist on documents without the underscore + if (fieldName.startsWith('_')) { + const nonUnderscoreFieldName = fieldName.substring(1); + + if (layoutDoc) { + const nonUnderscoreLayoutValue = layoutDoc[nonUnderscoreFieldName]; + if (nonUnderscoreLayoutValue !== undefined) { + metadata.fields.layout[nonUnderscoreFieldName] = this.formatFieldValue(nonUnderscoreLayoutValue); + metadata.fieldLocationMap[nonUnderscoreFieldName] = 'layout'; + } + } + + if (dataDoc) { + const nonUnderscoreDataValue = dataDoc[nonUnderscoreFieldName]; + if (nonUnderscoreDataValue !== undefined) { + metadata.fields.data[nonUnderscoreFieldName] = this.formatFieldValue(nonUnderscoreDataValue); + if (!metadata.fieldLocationMap[nonUnderscoreFieldName]) { + metadata.fieldLocationMap[nonUnderscoreFieldName] = 'data'; + } + } + } + } + }); + + // Add common field aliases for easier discovery + // This helps users understand both width and _width refer to the same property + if (metadata.fields.layout._width !== undefined && metadata.fields.layout.width === undefined) { + metadata.fields.layout.width = metadata.fields.layout._width; + metadata.fieldLocationMap.width = 'layout'; + } + + if (metadata.fields.layout._height !== undefined && metadata.fields.layout.height === undefined) { + metadata.fields.layout.height = metadata.fields.layout._height; + metadata.fieldLocationMap.height = 'layout'; + } + + return metadata; + } + + /** + * Formats a field value for JSON output + * @param value The field value to format + * @returns A JSON-friendly representation of the field value + */ + private formatFieldValue(value: any): any { + if (value === undefined || value === null) { + return null; + } + + // Handle Doc objects + if (value instanceof Doc) { + return { + type: 'Doc', + id: value.id || this.ensureDocumentId(value), + title: value.title || '', + docType: value.type || '', + }; + } + + // Handle RichTextField (try to extract plain text) + if (typeof value === 'string' && value.includes('"type":"doc"') && value.includes('"content":')) { + try { + const rtfObj = JSON.parse(value); + // If this looks like a rich text field structure + if (rtfObj.doc && rtfObj.doc.content) { + // Recursively extract text from the content + let plainText = ''; + const extractText = (node: any) => { + if (node.text) { + plainText += node.text; + } + if (node.content && Array.isArray(node.content)) { + node.content.forEach((child: any) => extractText(child)); + } + }; + + extractText(rtfObj.doc); + + // If we successfully extracted text, show it, but also preserve the original value + if (plainText) { + return { + type: 'RichText', + text: plainText, + length: plainText.length, + // Don't include the full value as it can be very large + }; + } + } + } catch (e) { + // If parsing fails, just treat as a regular string + } + } + + // Handle arrays and complex objects + if (typeof value === 'object') { + // If the object has a toString method, use it + if (value.toString && value.toString !== Object.prototype.toString) { + return value.toString(); + } + + try { + // Try to convert to JSON string + return JSON.stringify(value); + } catch (e) { + return '[Complex Object]'; + } + } + + // Return primitive values as is + return value; + } + + /** + * Converts a string field value to the appropriate type based on field metadata + * @param fieldName The name of the field + * @param fieldValue The string value to convert + * @returns The converted value with the appropriate type + */ + private convertFieldValue(fieldName: string, fieldValue: any): any { + // If fieldValue is already a number or boolean, we don't need to convert it from string + if (typeof fieldValue === 'number' || typeof fieldValue === 'boolean') { + return fieldValue; + } + + // If fieldValue is a string "true" or "false", convert to boolean + if (typeof fieldValue === 'string') { + if (fieldValue.toLowerCase() === 'true') { + return true; + } + if (fieldValue.toLowerCase() === 'false') { + return false; + } + } + + // If fieldValue is not a string (and not a number or boolean), convert it to string + if (typeof fieldValue !== 'string') { + fieldValue = String(fieldValue); + } + + // Special handling for text field - convert to proper RichTextField format + if (fieldName === 'text') { + try { + // Check if it's already a valid JSON RichTextField + JSON.parse(fieldValue); + return fieldValue; + } catch (e) { + // It's a plain text string, so convert it to RichTextField format + const rtf = { + doc: { + type: 'doc', + content: [ + { + type: 'paragraph', + content: [ + { + type: 'text', + text: fieldValue, + }, + ], + }, + ], + }, + }; + return JSON.stringify(rtf); + } + } + + // Get field metadata + const normalizedFieldName = fieldName.startsWith('_') ? fieldName : `_${fieldName}`; + const strippedFieldName = fieldName.startsWith('_') ? fieldName.substring(1) : fieldName; + + // Check both versions of the field name in metadata + const fieldMeta = this.fieldMetadata[normalizedFieldName] || this.fieldMetadata[strippedFieldName]; + + // Special handling for width and height without metadata + if (!fieldMeta && (fieldName === '_width' || fieldName === '_height' || fieldName === 'width' || fieldName === 'height')) { + const num = Number(fieldValue); + return isNaN(num) ? fieldValue : num; + } + + if (!fieldMeta) { + // If no metadata found, just return the string value + return fieldValue; + } + + // Convert based on field type + const fieldType = fieldMeta.type; + + if (fieldType === 'boolean') { + // Convert to boolean + return fieldValue.toLowerCase() === 'true'; + } else if (fieldType === 'number') { + // Convert to number + const num = Number(fieldValue); + return isNaN(num) ? fieldValue : num; + } else if (fieldType === 'date') { + // Try to convert to date (stored as number timestamp) + try { + return new Date(fieldValue).getTime(); + } catch (e) { + return fieldValue; + } + } else if (fieldType.includes('list') || fieldType.includes('array')) { + // Try to parse as JSON array + try { + return JSON.parse(fieldValue); + } catch (e) { + return fieldValue; + } + } else if (fieldType === 'json' || fieldType === 'object') { + // Try to parse as JSON object + try { + return JSON.parse(fieldValue); + } catch (e) { + return fieldValue; + } + } + + // Default to string + return fieldValue; + } + + /** + * Extracts all field metadata from DocumentOptions + * @returns A structured object containing metadata about all available document fields + */ + public getAllFieldMetadata() { + // Start with our already populated fieldMetadata from the DocumentOptions class + const result: Record<string, any> = { + fieldCount: Object.keys(this.fieldMetadata).length, + fields: {}, + fieldsByType: { + string: [], + number: [], + boolean: [], + //doc: [], + //list: [], + //date: [], + //enumeration: [], + //other: [], + }, + fieldNameMappings: {}, + commonFields: { + appearance: [], + position: [], + size: [], + content: [], + behavior: [], + layout: [], + }, + }; + + // Process each field in the metadata + Object.entries(this.fieldMetadata).forEach(([fieldName, fieldInfo]) => { + const strippedName = fieldName.startsWith('_') ? fieldName.substring(1) : fieldName; + + // Add to fieldNameMappings + if (fieldName.startsWith('_')) { + result.fieldNameMappings[strippedName] = fieldName; + } + + // Create structured field metadata + const fieldData: Record<string, any> = { + name: fieldName, + displayName: strippedName, + description: fieldInfo.description || '', + type: fieldInfo.fieldType || 'unknown', + possibleValues: fieldInfo.values || [], + }; + + // Add field to fields collection + result.fields[fieldName] = fieldData; + + // Categorize by field type + const type = fieldInfo.fieldType?.toLowerCase() || 'unknown'; + if (type === 'string') { + result.fieldsByType.string.push(fieldName); + } else if (type === 'number') { + result.fieldsByType.number.push(fieldName); + } else if (type === 'boolean') { + result.fieldsByType.boolean.push(fieldName); + } else if (type === 'doc') { + //result.fieldsByType.doc.push(fieldName); + } else if (type === 'list') { + //result.fieldsByType.list.push(fieldName); + } else if (type === 'date') { + //result.fieldsByType.date.push(fieldName); + } else if (type === 'enumeration') { + //result.fieldsByType.enumeration.push(fieldName); + } else { + //result.fieldsByType.other.push(fieldName); + } + + // Categorize by field purpose + if (fieldName.includes('width') || fieldName.includes('height') || fieldName.includes('size')) { + result.commonFields.size.push(fieldName); + } else if (fieldName.includes('color') || fieldName.includes('background') || fieldName.includes('border')) { + result.commonFields.appearance.push(fieldName); + } else if (fieldName.includes('x') || fieldName.includes('y') || fieldName.includes('position') || fieldName.includes('pan')) { + result.commonFields.position.push(fieldName); + } else if (fieldName.includes('text') || fieldName.includes('title') || fieldName.includes('data')) { + result.commonFields.content.push(fieldName); + } else if (fieldName.includes('action') || fieldName.includes('click') || fieldName.includes('event')) { + result.commonFields.behavior.push(fieldName); + } else if (fieldName.includes('layout')) { + result.commonFields.layout.push(fieldName); + } + }); + + // Add special section for auto-sizing related fields + result.autoSizingFields = { + height: { + autoHeightField: '_layout_autoHeight', + heightField: '_height', + displayName: 'height', + usage: 'To manually set height, first set layout_autoHeight to false', + }, + width: { + autoWidthField: '_layout_autoWidth', + widthField: '_width', + displayName: 'width', + usage: 'To manually set width, first set layout_autoWidth to false', + }, + }; + + // Add special section for text field format + result.specialFields = { + text: { + name: 'text', + description: 'Document text content', + format: 'RichTextField', + note: 'When setting text, provide plain text - it will be automatically converted to the correct format', + example: 'For setting: "Hello world" (plain text); For getting: Will be converted to plaintext for display', + }, + }; + + return result; + } + + /** + * Edits a specific field on a document + * @param docId The ID of the document to edit + * @param fieldName The name of the field to edit + * @param fieldValue The new value for the field (string, number, or boolean) + * @returns Object with success status, message, and additional information + */ + public editDocumentField( + docId: string, + fieldName: string, + fieldValue: string | number | boolean + ): { + success: boolean; + message: string; + fieldName?: string; + originalFieldName?: string; + newValue?: any; + warning?: string; + } { + // Normalize field name (handle with/without underscore) + let normalizedFieldName = fieldName.startsWith('_') ? fieldName : fieldName; + const strippedFieldName = fieldName.startsWith('_') ? fieldName.substring(1) : fieldName; + + // Handle common field name aliases (width → _width, height → _height) + // Many document fields use '_' prefix for layout properties + if (fieldName === 'width') { + normalizedFieldName = '_width'; + } else if (fieldName === 'height') { + normalizedFieldName = '_height'; + } + + // Get the documents + const doc = this.documentsById.get(docId); + if (!doc) { + return { success: false, message: `Document with ID ${docId} not found` }; + } + + const { layoutDoc, dataDoc } = this.documentsById.get(docId) ?? { layoutDoc: null, dataDoc: null }; + + if (!layoutDoc && !dataDoc) { + return { success: false, message: `Could not find layout or data document for document with ID ${docId}` }; + } + + try { + // Convert the field value to the appropriate type based on field metadata + const convertedValue = this.convertFieldValue(normalizedFieldName, fieldValue); + + let targetDoc: Doc | undefined; + let targetLocation: string; + + // First, check if field exists on layout document using Doc.Get + if (layoutDoc) { + const fieldExistsOnLayout = Doc.Get(layoutDoc, normalizedFieldName, true) !== undefined; + + // If it exists on layout document, update it there + if (fieldExistsOnLayout) { + targetDoc = layoutDoc; + targetLocation = 'layout'; + } + // If it has an underscore prefix, it's likely a layout property even if not yet set + else if (normalizedFieldName.startsWith('_')) { + targetDoc = layoutDoc; + targetLocation = 'layout'; + } + // Otherwise, look for or create on data document + else if (dataDoc) { + targetDoc = dataDoc; + targetLocation = 'data'; + } + // If no data document available, default to layout + else { + targetDoc = layoutDoc; + targetLocation = 'layout'; + } + } + // If no layout document, use data document + else if (dataDoc) { + targetDoc = dataDoc; + targetLocation = 'data'; + } else { + return { success: false, message: `No valid document found for editing` }; + } + + if (!targetDoc) { + return { success: false, message: `Target document not available` }; + } + + // Set the field value on the target document + targetDoc[normalizedFieldName] = convertedValue; + + return { + success: true, + message: `Successfully updated field '${normalizedFieldName}' on ${targetLocation} document (ID: ${docId})`, + fieldName: normalizedFieldName, + originalFieldName: fieldName, + newValue: convertedValue, + }; + } catch (error) { + console.error('Error editing document field:', error); + return { + success: false, + message: `Error updating field: ${error instanceof Error ? error.message : String(error)}`, + }; + } + } + /** + * Gets metadata for a specific document or all documents + * @param documentId Optional ID of a specific document to get metadata for + * @returns Document metadata or metadata for all documents + */ + public getDocumentMetadata(documentId?: string): any { + if (documentId) { + console.log(`Returning document metadata for docID, ${documentId}:`, this.extractDocumentMetadata(documentId)); + return this.extractDocumentMetadata(documentId); + } else { + // Get metadata for all documents + const documentsMetadata: Record<string, Record<string, any>> = {}; + for (const documentId of this.documentsById.keys()) { + const metadata = this.extractDocumentMetadata(documentId); + if (metadata) { + documentsMetadata[documentId] = metadata; + } else { + console.warn(`No metadata found for document with ID: ${documentId}`); + } + } + return { + documentCount: this.documentsById.size, + documents: documentsMetadata, + //fieldDefinitions: this.fieldMetadata, // TODO: remove this, if fieldDefinitions are not needed. + }; + } + } + + /** + * Adds links between documents based on their IDs + * @param docIds Array of document IDs to link + * @param relationship Optional relationship type for the links + * @returns Array of created link documents + */ + public addLinks(docIds: string[]): Doc[] { + const createdLinks: Doc[] = []; + // Use string keys for Set instead of arrays which don't work as expected as keys + const alreadyLinked = new Set<string>(); + + // Iterate over the document IDs and add links + docIds.forEach(docId1 => { + const doc1 = this.documentsById.get(docId1); + docIds.forEach(docId2 => { + if (docId1 === docId2) return; // Skip self-linking + + // Create a consistent key regardless of document order + const linkKey = [docId1, docId2].sort().join('_'); + if (alreadyLinked.has(linkKey)) return; + + const doc2 = this.documentsById.get(docId2); + if (doc1?.layoutDoc && doc2?.layoutDoc) { + try { + // Create a link document between doc1 and doc2 + const linkDoc = Docs.Create.LinkDocument(doc1.layoutDoc, doc2.layoutDoc); + + // Set a default color if relationship doesn't specify one + if (!linkDoc.color) { + linkDoc.color = 'lightBlue'; // Default blue color + } + + // Ensure link is visible by setting essential properties + linkDoc.link_visible = true; + linkDoc.link_enabled = true; + linkDoc.link_autoMove = true; + linkDoc.link_showDirected = true; + + // Set the embedContainer to ensure visibility + // This is shown in the image as a key difference between visible/non-visible links + if (this.chatBoxDocument && this.chatBoxDocument.parent && typeof this.chatBoxDocument.parent === 'object' && 'title' in this.chatBoxDocument.parent) { + linkDoc.embedContainer = String(this.chatBoxDocument.parent.title); + } else if (doc1.layoutDoc.parent && typeof doc1.layoutDoc.parent === 'object' && 'title' in doc1.layoutDoc.parent) { + linkDoc.embedContainer = String(doc1.layoutDoc.parent.title); + } else { + // Default to a tab name if we can't find one + linkDoc.embedContainer = 'Untitled Tab 1'; + } + + // Add the link to the document system + LinkManager.Instance.addLink(linkDoc); + + const ancestor = DocumentView.linkCommonAncestor(linkDoc); + ancestor?.ComponentView?.addDocument?.(linkDoc); + // Add to user document list to make it visible in the UI + Doc.AddDocToList(Doc.UserDoc(), 'links', linkDoc); + + // Create a visual link for display + if (this.chatBoxDocument) { + // Make sure the docs are visible in the UI + this.chatBox._props.addDocument?.(doc1.layoutDoc); + this.chatBox._props.addDocument?.(doc2.layoutDoc); + + // Use DocumentManager to ensure documents are visible + DocumentManager.Instance.showDocument(doc1.layoutDoc, { willZoomCentered: false }); + DocumentManager.Instance.showDocument(doc2.layoutDoc, { willZoomCentered: false }); + } + + createdLinks.push(linkDoc); + alreadyLinked.add(linkKey); + } catch (error) { + console.error('Error creating link between documents:', error); + } + } + }); + }); + + // Force update of the UI to show new links + setTimeout(() => { + try { + // Update server cache to ensure links are persisted + UPDATE_SERVER_CACHE && typeof UPDATE_SERVER_CACHE === 'function' && UPDATE_SERVER_CACHE(); + } catch (e) { + console.warn('Could not update server cache after creating links:', e); + } + }, 100); + + return createdLinks; + } + /** + * Helper method to validate a document type and ensure it's a valid supportedDocType + * @param docType The document type to validate + * @returns True if the document type is valid, false otherwise + */ + private isValidDocType(docType: string): boolean { + return Object.values(supportedDocTypes).includes(docType as supportedDocTypes); + } + /** + * Creates a document in the dashboard and returns its ID. + * This is a public API used by tools like SearchTool. + * + * @param docType The type of document to create + * @param data The data for the document + * @param options Optional configuration options + * @returns The ID of the created document + */ + public createDocInDash(docType: string, data: string, options?: any): string { + // Validate doc_type + if (!this.isValidDocType(docType)) { + throw new Error(`Invalid document type: ${docType}`); + } + + try { + // Create simple document with just title and data + const simpleDoc: parsedDoc = { + doc_type: docType, + title: options?.title ?? `Untitled Document ${this.documentsById.size + 1}`, + data: data, + x: options?.x ?? 0, + y: options?.y ?? 0, + _width: 300, + _height: 300, + _layout_fitWidth: false, + _layout_autoHeight: true, + }; + + // Additional handling for web documents + if (docType === 'web') { + // For web documents, don't sanitize the URL here + // Instead, set properties to handle content safely when loaded + simpleDoc._disable_resource_loading = true; + simpleDoc._sandbox_iframe = true; + simpleDoc.data_useCors = true; + + // Specify a more permissive sandbox to allow content to render properly + // but still maintain security + simpleDoc._iframe_sandbox = 'allow-same-origin allow-scripts allow-popups allow-forms'; + } + + // Use the chatBox's createDocInDash method to create the document + if (!this.chatBox) { + throw new Error('ChatBox instance not available for creating document'); + } + + const doc = this.chatBox.whichDoc(simpleDoc, false); + if (doc) { + // Use MobX runInAction to properly modify observable state + runInAction(() => { + if (this.chatBoxDocument && doc) { + // Create link and add it to the document system + const linkDoc = Docs.Create.LinkDocument(this.chatBoxDocument, doc); + LinkManager.Instance.addLink(linkDoc); + + // Add document to view + this.chatBox._props.addDocument?.(doc); + + // Show document - defer actual display to prevent immediate resource loading + setTimeout(() => { + DocumentManager.Instance.showDocument(doc, { willZoomCentered: true }, () => {}); + }, 100); + } + }); + + const id = this.processDocument(doc); + return id; + } else { + throw new Error(`Error creating document. Created document not found.`); + } + } catch (error) { + throw new Error(`Error creating document: ${error}`); + } + } + + /** + * Sanitizes web content to prevent errors with external resources + * @param content The web content to sanitize + * @returns Sanitized content + */ + private sanitizeWebContent(content: string): string { + if (!content) return content; + + try { + // Replace problematic resource references that might cause errors + const sanitized = content + // Remove preload links that might cause errors + .replace(/<link[^>]*rel=["']preload["'][^>]*>/gi, '') + // Remove map file references + .replace(/\/\/# sourceMappingURL=.*\.map/gi, '') + // Remove external CSS map files references + .replace(/\/\*# sourceMappingURL=.*\.css\.map.*\*\//gi, '') + // Add sandbox to iframes + .replace(/<iframe/gi, '<iframe sandbox="allow-same-origin" loading="lazy"') + // Prevent automatic resource loading for images + .replace(/<img/gi, '<img loading="lazy"') + // Prevent automatic resource loading for scripts + .replace(/<script/gi, '<script type="text/disabled"') + // Handle invalid URIs by converting relative URLs to absolute ones + .replace(/href=["'](\/[^"']+)["']/gi, (match, p1) => { + // Only handle relative URLs starting with / + if (p1.startsWith('/')) { + return `href="#disabled-link"`; + } + return match; + }) + // Prevent automatic loading of CSS + .replace(/<link[^>]*rel=["']stylesheet["'][^>]*href=["']([^"']+)["']/gi, (match, href) => `<link rel="prefetch" data-original-href="${href}" />`); + + // Wrap the content in a sandboxed container + return ` + <div class="sandboxed-web-content"> + <style> + /* Override styles to prevent external resource loading */ + @font-face { font-family: 'disabled'; src: local('Arial'); } + * { font-family: system-ui, -apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, Oxygen, Ubuntu, Cantarell, 'Open Sans', 'Helvetica Neue', sans-serif !important; } + img, iframe, frame, embed, object { max-width: 100%; } + </style> + ${sanitized} + </div>`; + } catch (e) { + console.warn('Error sanitizing web content:', e); + // Fall back to a safe container with the content as text + return ` + <div class="sandboxed-web-content"> + <p>Content could not be safely displayed. Raw content:</p> + <pre>${content.replace(/</g, '<').replace(/>/g, '>')}</pre> + </div>`; + } + } + + public has(docId: string) { + return this.documentsById.has(docId); + } + + /** + * Returns a list of all document IDs in the manager. + * @returns An array of document IDs (strings). + */ + public listDocs(): string[] { + return Array.from(this.documentsById.keys()); + } + + /** + * Adds a document with a custom ID to the manager + * @param doc The document to add + * @param customId The custom ID to assign to the document + * @returns The customId that was assigned + */ + @action + public addCustomId(doc: Doc, customId: string): string { + if (!doc) { + console.error('Cannot add null document with custom ID'); + return ''; + } + + // Set the custom ID in the document's metadata + doc[this.DOCUMENT_ID_FIELD] = customId; + + // Store the document in our map + this.documentsById.set(customId, { + layoutDoc: doc, + dataDoc: doc, + }); + + return customId; + } + + /** + * Gets a document by its ID + * @param docId The ID of the document to retrieve + * @returns The document if found, undefined otherwise + */ + public getDocument(docId: string): Doc | undefined { + const docInfo = this.documentsById.get(docId); + return docInfo?.layoutDoc; + } + + /** + * Registers chunk IDs associated with a document in the manager + * @param docId The parent document ID + * @param chunkIds Array of chunk IDs associated with this document + */ + @action + public registerChunkIds(docId: string, chunkIds: string[]): void { + // Get the document if it exists + const docInfo = this.documentsById.get(docId); + if (!docInfo) { + console.warn(`Cannot register chunks for unknown document ID: ${docId}`); + return; + } + + // Store chunk IDs on the document for future reference + const doc = docInfo.layoutDoc; + if (!doc.chunk_ids) { + doc.chunk_ids = JSON.stringify(chunkIds); + } else { + // Merge with existing chunk IDs if they exist + const existingIds = JSON.parse(doc.chunk_ids as string); + const updatedIds = [...new Set([...existingIds, ...chunkIds])]; // Remove duplicates + doc.chunk_ids = JSON.stringify(updatedIds); + } + for (const chunkId of chunkIds) { + // Ensure each chunk ID can be linked back to its parent document + // Store a mapping from chunk ID to parent document ID + // This allows us to easily find a document by any of its chunk IDs + if (!this.documentsById.has(chunkId)) { + this.documentsById.set(chunkId, { + layoutDoc: doc, + dataDoc: docInfo.dataDoc, + }); + } + } + } + + /** + * Gets a document ID by a chunk ID + * @param chunkId The chunk ID to look up + * @returns The parent document ID if found + */ + public getDocIdByChunkId(chunkId: string): string | undefined { + const docInfo = this.documentsById.get(chunkId); + if (docInfo) { + return docInfo.layoutDoc[this.DOCUMENT_ID_FIELD] as string; + } + return undefined; + } + + /** + * Adds simplified chunks to a document for citation handling + * @param doc The document to add simplified chunks to + * @param chunks Array of full RAG chunks to simplify + * @param docType The type of document (e.g., 'pdf', 'video', 'audio', etc.) + * @returns The updated document with simplified chunks + */ + public addSimplifiedChunks(doc: Doc, chunks: RAGChunk[], docType: string): Doc { + if (!doc) { + console.error('Cannot add simplified chunks to null document'); + return doc; + } + + // Initialize empty chunks array if not exists + if (!doc.chunk_simpl) { + doc.chunk_simpl = JSON.stringify({ chunks: [] }); + } + + // Create array of simplified chunks based on document type + const simplifiedChunks = chunks.map(chunk => { + // Common properties across all chunk types + const baseChunk = { + chunkId: chunk.id, + text: chunk.metadata.text, + doc_id: chunk.metadata.doc_id, + chunkType: chunk.metadata.type || CHUNK_TYPE.TEXT, + }; + + // Add type-specific properties + if (docType === 'video' || docType === 'audio') { + return { + ...baseChunk, + start_time: chunk.metadata.start_time, + end_time: chunk.metadata.end_time, + indexes: chunk.metadata.indexes, + chunkType: docType === 'video' ? CHUNK_TYPE.VIDEO : CHUNK_TYPE.AUDIO, + }; + } else if (docType === 'pdf') { + return { + ...baseChunk, + startPage: chunk.metadata.start_page, + endPage: chunk.metadata.end_page, + location: chunk.metadata.location, + }; + } else if (docType === 'csv') { + return { + ...baseChunk, + rowStart: (chunk.metadata as any).row_start, + rowEnd: (chunk.metadata as any).row_end, + colStart: (chunk.metadata as any).col_start, + colEnd: (chunk.metadata as any).col_end, + }; + } else { + // Default for other document types + return baseChunk; + } + }); + + // Update the document with all simplified chunks at once + doc.chunk_simpl = JSON.stringify({ chunks: simplifiedChunks }); + + return doc; + } + + /** + * Gets the simplified chunks from a document + * @param doc The document to get simplified chunks from + * @returns Array of simplified chunks or empty array if none exist + */ + public getSimplifiedChunks(doc: Doc): any[] { + if (!doc || !doc.chunk_simpl) { + return []; + } + + try { + const parsed = JSON.parse(StrCast(doc.chunk_simpl)); + return parsed.chunks || []; + } catch (e) { + console.error('Error parsing simplified chunks:', e); + return []; + } + } + + /** + * Gets a specific simplified chunk by ID + * @param doc The document containing chunks + * @param chunkId The ID of the chunk to retrieve + * @returns The simplified chunk if found, undefined otherwise + */ + public getSimplifiedChunkById(doc: Doc, chunkId: string): any | undefined { + const chunks = this.getSimplifiedChunks(doc); + return chunks.find(chunk => chunk.chunkId === chunkId); + } + + /** + * Gets the original segments from a media document + * @param doc The document containing original media segments + * @returns Array of media segments or empty array if none exist + */ + public getOriginalSegments(doc: Doc): any[] { + if (!doc || !doc.original_segments) { + return []; + } + + try { + return JSON.parse(StrCast(doc.original_segments)) || []; + } catch (e) { + console.error('Error parsing original segments:', e); + return []; + } + } + + /** + * Gets all document summaries combined into a single string + * @returns String containing all document summaries + */ + public getAllDocumentSummaries(): string { + const summaries = Array.from(this.documentsById.keys()) + .map(id => { + const doc = this.getDocument(id); + if (doc) { + // Try to get summary from either the document or its data document + const summary = doc.summary || (doc[DocData] && doc[DocData].summary); + if (summary) { + return StrCast(summary); + } + } + return null; + }) + .filter(Boolean) + .join('\n\n'); + + return summaries; + } +} |