diff options
| author | A.J. Shulman <Shulman.aj@gmail.com> | 2025-05-11 17:18:18 -0400 |
|---|---|---|
| committer | A.J. Shulman <Shulman.aj@gmail.com> | 2025-05-11 17:18:18 -0400 |
| commit | e141307dbd9b951f76c908610e7b89e296ad92b8 (patch) | |
| tree | a48d7cae7a7702519d2099dfff5a503fcfc7875f /src/client/views/nodes/chatbot/vectorstore | |
| parent | e5cb67b92d9b3c84dc90b1e64cc7128621523801 (diff) | |
chanegd everything to be more consistent
- made both web related tools use doc manager and chunk Ids
Diffstat (limited to 'src/client/views/nodes/chatbot/vectorstore')
| -rw-r--r-- | src/client/views/nodes/chatbot/vectorstore/Vectorstore.ts | 21 |
1 files changed, 2 insertions, 19 deletions
diff --git a/src/client/views/nodes/chatbot/vectorstore/Vectorstore.ts b/src/client/views/nodes/chatbot/vectorstore/Vectorstore.ts index 1349df483..f1fae6f11 100644 --- a/src/client/views/nodes/chatbot/vectorstore/Vectorstore.ts +++ b/src/client/views/nodes/chatbot/vectorstore/Vectorstore.ts @@ -148,10 +148,6 @@ export class Vectorstore { // Generate chunk IDs upfront so we can register them const chunkIds = segmentedTranscript.map(() => uuidv4()); - - // Register all chunk IDs with the document manager - this.docManager.registerChunkIds(doc_id, chunkIds); - // Add transcript and embeddings to metadata result = { doc_id, @@ -185,7 +181,7 @@ export class Vectorstore { doc.segmented_transcript = JSON.stringify(segmentedTranscript); // Use doc manager to add simplified chunks const docType = local_file_path.endsWith('.mp3') ? 'audio' : 'video'; - this.docManager.addSimplifiedChunks(doc, result.chunks, docType); + this.docManager.addSimplifiedChunks(result.chunks, docType); } else { // Process regular document console.log('Processing regular document...'); @@ -216,13 +212,10 @@ export class Vectorstore { console.log('doc_id in vectorstore', result.doc_id, 'does not match doc_id in doc', doc[Id]); } - // Register chunks with the document manager - this.docManager.registerChunkIds(result.doc_id, chunkIds); - // Use doc manager to add simplified chunks - determine document type from file extension const fileExt = path.extname(local_file_path).toLowerCase(); const docType = fileExt === '.pdf' ? 'pdf' : fileExt === '.csv' ? 'csv' : 'text'; - this.docManager.addSimplifiedChunks(doc, result.chunks, docType); + this.docManager.addSimplifiedChunks(result.chunks, docType); doc.summary = result.summary; doc.ai_purpose = result.purpose; @@ -351,16 +344,6 @@ export class Vectorstore { }, } as RAGChunk; - // Ensure the document manager knows about this chunk - // This is important for maintaining backwards compatibility - if (chunk.id && !this.docManager.getDocByChunkId(chunk.id)) { - // If the chunk ID isn't registered but we have a doc_id in metadata - if (chunk.metadata.doc_id && this.docManager.has(chunk.metadata.doc_id)) { - // Register the chunk with its parent document - this.docManager.registerChunkIds(chunk.metadata.doc_id, [chunk.id]); - } - } - return chunk; }); |
