diff options
| -rw-r--r-- | src/client/views/nodes/chatbot/agentsystem/Agent.ts | 17 | ||||
| -rw-r--r-- | src/client/views/nodes/chatbot/chatboxcomponents/ChatBox.tsx | 166 | ||||
| -rw-r--r-- | src/client/views/nodes/chatbot/tools/RAGTool.ts | 1 | ||||
| -rw-r--r-- | src/server/chunker/pdf_chunker.py | 41 |
4 files changed, 102 insertions, 123 deletions
diff --git a/src/client/views/nodes/chatbot/agentsystem/Agent.ts b/src/client/views/nodes/chatbot/agentsystem/Agent.ts index a16794e10..8516f054b 100644 --- a/src/client/views/nodes/chatbot/agentsystem/Agent.ts +++ b/src/client/views/nodes/chatbot/agentsystem/Agent.ts @@ -7,14 +7,17 @@ import { AnswerParser } from '../response_parsers/AnswerParser'; import { StreamedAnswerParser } from '../response_parsers/StreamedAnswerParser'; import { BaseTool } from '../tools/BaseTool'; import { CalculateTool } from '../tools/CalculateTool'; +//import { CreateAnyDocumentTool } from '../tools/CreateAnyDocTool'; import { DataAnalysisTool } from '../tools/DataAnalysisTool'; import { DocumentMetadataTool } from '../tools/DocumentMetadataTool'; +import { ImageCreationTool } from '../tools/ImageCreationTool'; import { NoTool } from '../tools/NoTool'; import { SearchTool } from '../tools/SearchTool'; import { Parameter, ParametersType, TypeMap } from '../types/tool_types'; import { AgentMessage, ASSISTANT_ROLE, AssistantMessage, Observation, PROCESSING_TYPE, ProcessingInfo, TEXT_TYPE } from '../types/types'; import { Vectorstore } from '../vectorstore/Vectorstore'; import { getReactPrompt } from './prompts'; +//import { DictionaryTool } from '../tools/DictionaryTool'; import { ChatCompletionMessageParam } from 'openai/resources'; import { Doc } from '../../../../../fields/Doc'; import { ChatBox, parsedDoc } from '../chatboxcomponents/ChatBox'; @@ -27,7 +30,7 @@ import { CodebaseSummarySearchTool } from '../tools/CodebaseSummarySearchTool'; import { FileContentTool } from '../tools/FileContentTool'; import { FileNamesTool } from '../tools/FileNamesTool'; import { CreateNewTool } from '../tools/CreateNewTool'; -import { GPTTutorialTool } from '../tools/TutorialTool'; +//import { CreateTextDocTool } from '../tools/CreateTextDocumentTool'; dotenv.config(); @@ -50,7 +53,6 @@ export class Agent { private streamedAnswerParser: StreamedAnswerParser = new StreamedAnswerParser(); private tools: Record<string, BaseTool<ReadonlyArray<Parameter>>>; private _docManager: AgentDocumentManager; - private is_dash_doc_assistant: boolean; // Dynamic tool registry for tools created at runtime private dynamicToolRegistry: Map<string, BaseTool<ReadonlyArray<Parameter>>> = new Map(); // Callback for notifying when tools are created and need reload @@ -75,8 +77,7 @@ export class Agent { csvData: () => { filename: string; id: string; text: string }[], createImage: (result: Upload.FileInformation & Upload.InspectionResults, options: DocumentOptions) => void, createCSVInDash: (url: string, title: string, id: string, data: string) => void, - docManager: AgentDocumentManager, - isDashDocAssistant: boolean + docManager: AgentDocumentManager ) { // Initialize OpenAI client with API key from environment this.client = new OpenAI({ apiKey: process.env.OPENAI_KEY, dangerouslyAllowBrowser: true }); @@ -84,7 +85,6 @@ export class Agent { this._history = history; this._csvData = csvData; this._docManager = docManager; - this.is_dash_doc_assistant = isDashDocAssistant; // Initialize dynamic tool registry this.dynamicToolRegistry = new Map(); @@ -103,7 +103,6 @@ export class Agent { codebaseSummarySearch: new CodebaseSummarySearchTool(this.vectorstore), fileContent: new FileContentTool(this.vectorstore), fileNames: new FileNamesTool(this.vectorstore), - generateTutorialNode: new GPTTutorialTool(this._docManager), }; // Add the createNewTool after other tools are defined @@ -143,7 +142,7 @@ export class Agent { const instance: BaseTool<ReadonlyArray<Parameter>> = new ToolClass(); - // Prefer the tool's self-declared name (matches <action> tag) + // Prefer the tool’s self-declared name (matches <action> tag) const key = (instance.name || '').trim() || legacyKey; // Check for duplicates @@ -298,7 +297,7 @@ export class Agent { ignoreAttributes: false, attributeNamePrefix: '@_', textNodeName: '_text', - isArray: name => name === 'url', + isArray: name => ['query', 'url'].indexOf(name) !== -1, processEntities: false, // Disable processing of entities stopNodes: ['*.entity'], // Do not process any entities }); @@ -760,7 +759,7 @@ export class Agent { const docSummaries = () => JSON.stringify(this._docManager.listDocs); const chatHistory = this._history(); - return getReactPrompt(allTools, docSummaries, chatHistory, this.is_dash_doc_assistant); + return getReactPrompt(allTools, docSummaries, chatHistory); } /** diff --git a/src/client/views/nodes/chatbot/chatboxcomponents/ChatBox.tsx b/src/client/views/nodes/chatbot/chatboxcomponents/ChatBox.tsx index 9fdbd8f58..093c1244c 100644 --- a/src/client/views/nodes/chatbot/chatboxcomponents/ChatBox.tsx +++ b/src/client/views/nodes/chatbot/chatboxcomponents/ChatBox.tsx @@ -7,46 +7,47 @@ * with support for follow-up questions and citation management. */ -import { Button, Size, Type } from '@dash/components'; +import dotenv from 'dotenv'; import { ObservableSet, action, computed, makeObservable, observable, observe, reaction, runInAction } from 'mobx'; import { observer } from 'mobx-react'; import OpenAI, { ClientOptions } from 'openai'; import * as React from 'react'; -import { AiOutlineSend } from 'react-icons/ai'; import { v4 as uuidv4 } from 'uuid'; import { ClientUtils, OmitKeys } from '../../../../../ClientUtils'; import { Doc, DocListCast, Opt } from '../../../../../fields/Doc'; -import { DocData, DocViews } from '../../../../../fields/DocSymbols'; +import { DocData, DocLayout, DocViews } from '../../../../../fields/DocSymbols'; import { Id } from '../../../../../fields/FieldSymbols'; import { RichTextField } from '../../../../../fields/RichTextField'; import { ScriptField } from '../../../../../fields/ScriptField'; -import { AudioCast, CsvCast, DocCast, NumCast, PDFCast, RTFCast, StrCast, VideoCast } from '../../../../../fields/Types'; -import { Upload } from '../../../../../server/SharedMediaTypes'; -import { DocServer } from '../../../../DocServer'; +import { CsvCast, DocCast, NumCast, PDFCast, RTFCast, StrCast, VideoCast, AudioCast } from '../../../../../fields/Types'; import { DocUtils } from '../../../../documents/DocUtils'; import { CollectionViewType, DocumentType } from '../../../../documents/DocumentTypes'; import { Docs, DocumentOptions } from '../../../../documents/Documents'; +import { DocServer } from '../../../../DocServer'; import { DocumentManager } from '../../../../util/DocumentManager'; import { ImageUtils } from '../../../../util/Import & Export/ImageUtils'; import { LinkManager } from '../../../../util/LinkManager'; import { CompileError, CompileScript } from '../../../../util/Scripting'; -import { SnappingManager } from '../../../../util/SnappingManager'; import { DictationButton } from '../../../DictationButton'; import { ViewBoxAnnotatableComponent } from '../../../DocComponent'; import { AudioBox } from '../../AudioBox'; import { DocumentView, DocumentViewInternal } from '../../DocumentView'; import { FieldView, FieldViewProps } from '../../FieldView'; -import { OpenWhere } from '../../OpenWhere'; import { PDFBox } from '../../PDFBox'; import { ScriptingBox } from '../../ScriptingBox'; import { VideoBox } from '../../VideoBox'; import { Agent } from '../agentsystem/Agent'; import { supportedDocTypes } from '../types/tool_types'; import { ASSISTANT_ROLE, AssistantMessage, CHUNK_TYPE, Citation, ProcessingInfo, SimplifiedChunk, TEXT_TYPE } from '../types/types'; -import { AgentDocumentManager } from '../utils/AgentDocumentManager'; import { Vectorstore } from '../vectorstore/Vectorstore'; import './ChatBox.scss'; import MessageComponentBox from './MessageComponent'; +import { OpenWhere } from '../../OpenWhere'; +import { Upload } from '../../../../../server/SharedMediaTypes'; +import { DocumentMetadataTool } from '../tools/DocumentMetadataTool'; +import { AgentDocumentManager } from '../utils/AgentDocumentManager'; + +dotenv.config(); export type parsedDocData = { doc_type: string; @@ -57,7 +58,6 @@ export type parsedDocData = { data_useCors?: boolean; }; export type parsedDoc = DocumentOptions & parsedDocData; - /** * ChatBox is the main class responsible for managing the interaction between the user and the assistant, * handling documents, and integrating with OpenAI for tasks such as document analysis, chat functionality, @@ -124,15 +124,7 @@ export class ChatBox extends ViewBoxAnnotatableComponent<FieldViewProps>() { this.vectorstore = new Vectorstore(this.vectorstore_id, this.docManager); // Create an agent with the vectorstore - this.agent = new Agent( - this.vectorstore, - this.retrieveFormattedHistory.bind(this), - this.retrieveCSVData.bind(this), - this.createImageInDash.bind(this), - this.createCSVInDash.bind(this), - this.docManager, - this.dataDoc.is_dash_doc_assistant === 'true' - ); + this.agent = new Agent(this.vectorstore, this.retrieveFormattedHistory.bind(this), this.retrieveCSVData.bind(this), this.createImageInDash.bind(this), this.createCSVInDash.bind(this), this.docManager); // Set up the tool created callback this.agent.setToolCreatedCallback(this.handleToolCreated); @@ -382,6 +374,7 @@ export class ChatBox extends ViewBoxAnnotatableComponent<FieldViewProps>() { }; } }); + this.scrollToBottom(); }; const onAnswerUpdate = (answerUpdate: string) => { @@ -389,33 +382,41 @@ export class ChatBox extends ViewBoxAnnotatableComponent<FieldViewProps>() { if (this._current_message) { this._current_message = { ...this._current_message, - content: [{ index: 0, type: TEXT_TYPE.NORMAL, text: answerUpdate, citation_ids: null }], + content: [{ text: answerUpdate, type: TEXT_TYPE.NORMAL, index: 0, citation_ids: [] }], }; } }); }; - // Get the response from the agent - let userQuery = trimmedText; - if (this.dataDoc.is_dash_doc_assistant) { - userQuery = `The user is asking a question about Dash functionality. Their question is: "${trimmedText}". You should use the generateTutorialNode tool to answer this question.`; - } - const response = await this.agent.askAgent(userQuery, onProcessingUpdate, onAnswerUpdate); + // Send the user's question to the assistant and get the final message + const finalMessage = await this.agent.askAgent(trimmedText, onProcessingUpdate, onAnswerUpdate); - // Push the final message to history + // Update the history with the final assistant message runInAction(() => { - this._history.push(response); - this._isLoading = false; - this._current_message = undefined; + if (this._current_message) { + this._history.push({ ...finalMessage }); + this._current_message = undefined; + this.dataDoc.data = JSON.stringify(this._history); + } }); - } catch (error) { - console.error('Error in askGPT:', error); + } catch (err) { + console.error('Error:', err); + // Handle error in processing + runInAction(() => + this._history.push({ + role: ASSISTANT_ROLE.ASSISTANT, + content: [{ index: 0, type: TEXT_TYPE.ERROR, text: `Sorry, I encountered an error while processing your request: ${err} `, citation_ids: null }], + processing_info: [], + }) + ); + } finally { runInAction(() => { this._isLoading = false; - this._current_message = undefined; }); + this.scrollToBottom(); } } + this.scrollToBottom(); }; /** @@ -488,7 +489,7 @@ export class ChatBox extends ViewBoxAnnotatableComponent<FieldViewProps>() { const data = (doc as parsedDocData).data; const ndoc = (() => { switch (doc.doc_type) { - default: + default: case supportedDocTypes.note: return Docs.Create.TextDocument(data as string, options); case supportedDocTypes.comparison: return this.createComparison(JSON.parse(data as string) as parsedDoc[], options); case supportedDocTypes.flashcard: return this.createFlashcard(JSON.parse(data as string) as parsedDoc[], options); @@ -496,25 +497,25 @@ export class ChatBox extends ViewBoxAnnotatableComponent<FieldViewProps>() { case supportedDocTypes.image: return Docs.Create.ImageDocument(data as string, options); case supportedDocTypes.equation: return Docs.Create.EquationDocument(data as string, options); case supportedDocTypes.notetaking: return Docs.Create.NoteTakingDocument([], options); - case supportedDocTypes.web: + case supportedDocTypes.web: // Create web document with enhanced safety options const webOptions = { - ...options, + ...options, data_useCors: true }; - + // If iframe_sandbox was passed from AgentDocumentManager, add it to the options if ('_iframe_sandbox' in options) { - webOptions._iframe_sandbox = options._iframe_sandbox; + (webOptions as any)._iframe_sandbox = options._iframe_sandbox; } - + return Docs.Create.WebDocument(data as string, webOptions); case supportedDocTypes.dataviz: case supportedDocTypes.table: return Docs.Create.DataVizDocument('/Users/ajshul/Dash-Web/src/server/public/files/csv/0d237e7c-98c9-44d0-aa61-5285fdbcf96c-random_sample.csv.csv', options); case supportedDocTypes.pdf: return Docs.Create.PdfDocument(data as string, options); case supportedDocTypes.video: return Docs.Create.VideoDocument(data as string, options); case supportedDocTypes.diagram: return Docs.Create.DiagramDocument(undefined, { text: data as unknown as RichTextField, ...options}); // text: can take a string or RichTextField but it's typed for RichTextField. - - // case supportedDocumentTypes.dataviz: + + // case supportedDocumentTypes.dataviz: // { // const { fileUrl, id } = await Networking.PostToServer('/createCSV', { // filename: (options.title as string).replace(/\s+/g, '') + '.csv', @@ -539,7 +540,7 @@ export class ChatBox extends ViewBoxAnnotatableComponent<FieldViewProps>() { const arr = this.createCollectionWithChildren(JSON.parse(data as string) as parsedDoc[], true).filter(d=>d).map(d => d!); const collOpts = { _width:300, _height: 300, _layout_fitWidth: true, _freeform_backgroundGrid: true, ...options, }; return (() => { - switch (options.type_collection) { + switch (options.type_collection) { case CollectionViewType.Tree: return Docs.Create.TreeDocument(arr, collOpts); case CollectionViewType.Stacking: return Docs.Create.StackingDocument(arr, collOpts); case CollectionViewType.Masonry: return Docs.Create.MasonryDocument(arr, collOpts); @@ -548,9 +549,9 @@ export class ChatBox extends ViewBoxAnnotatableComponent<FieldViewProps>() { case CollectionViewType.Carousel3D: return Docs.Create.Carousel3DDocument(arr, collOpts); case CollectionViewType.Multicolumn: return Docs.Create.CarouselDocument(arr, collOpts); default: return Docs.Create.FreeformDocument(arr, collOpts); - } - })(); - } + } + })(); + } // case supportedDocumentTypes.map: return Docs.Create.MapDocument([], options); // case supportedDocumentTypes.button: return Docs.Create.ButtonDocument(options); // case supportedDocumentTypes.trail: return Docs.Create.PresDocument(options); @@ -558,8 +559,8 @@ export class ChatBox extends ViewBoxAnnotatableComponent<FieldViewProps>() { })(); if (ndoc) { - ndoc.x = ((options.x as number) ?? 0) + (insideCol ? 0 : NumCast(this.layoutDoc.x) + NumCast(this.layoutDoc.width)) + 100; - ndoc.y = ((options.y as number) ?? 0) + (insideCol ? 0 : NumCast(this.layoutDoc.y)); + ndoc.x = NumCast((options.x as number) ?? 0) + (insideCol ? 0 : NumCast(this.layoutDoc.x) + NumCast(this.layoutDoc.width)) + 100; + ndoc.y = NumCast(options.y as number) + (insideCol ? 0 : NumCast(this.layoutDoc.y)); } return ndoc; }; @@ -639,7 +640,7 @@ export class ChatBox extends ViewBoxAnnotatableComponent<FieldViewProps>() { handleCitationClick = async (citation: Citation) => { try { // Extract values from MobX proxy object if needed - const chunkId = typeof citation.chunk_id === 'object' ? (citation.chunk_id as unknown as object).toString() : citation.chunk_id; + const chunkId = typeof citation.chunk_id === 'object' ? (citation.chunk_id as any).toString() : citation.chunk_id; // For debugging console.log('Citation clicked:', { @@ -681,7 +682,7 @@ export class ChatBox extends ViewBoxAnnotatableComponent<FieldViewProps>() { } this.handleOtherChunkTypes(foundChunk, citation, doc, dataDoc); // Show the chunk text in citation popup - const chunkText = citation.direct_text || 'Text content not available'; + let chunkText = citation.direct_text || 'Text content not available'; this.showCitationPopup(chunkText); // Also navigate to the document @@ -713,7 +714,7 @@ export class ChatBox extends ViewBoxAnnotatableComponent<FieldViewProps>() { // If specific indexes are provided, filter segments by those indexes if (indexesOfSegments && indexesOfSegments.length > 0) { - segments = original_segments.filter(segment => indexesOfSegments.includes(segment.index)); + segments = original_segments.filter((segment: any) => indexesOfSegments.includes(segment.index)); } // If no segments match the indexes, use all segments @@ -722,7 +723,7 @@ export class ChatBox extends ViewBoxAnnotatableComponent<FieldViewProps>() { } // First try to find an exact match - const exactMatch = segments.find(segment => segment.text && segment.text.includes(citationText)); + const exactMatch = segments.find((segment: any) => segment.text && segment.text.includes(citationText)); if (exactMatch) { return exactMatch.start; @@ -831,26 +832,22 @@ export class ChatBox extends ViewBoxAnnotatableComponent<FieldViewProps>() { existingDoc._width = x2 - x1; existingDoc._height = y2 - y1; } - // const highlightDoc = - existingDoc ?? this.createImageCitationHighlight(x1, y1, x2, y2, citation, annotationKey, doc); + const highlightDoc = existingDoc ?? this.createImageCitationHighlight(x1, y1, x2, y2, citation, annotationKey, doc); //doc.layout_scroll = y1; doc._layout_curPage = foundChunk.startPage + 1; - DocumentManager.Instance.showDocument(doc, { willZoomCentered: true }, () => {}); - //DocumentManager.Instance.showDocument(highlightDoc, { willZoomCentered: true }, () => {}); + DocumentManager.Instance.showDocument(highlightDoc, { willZoomCentered: true }, () => {}); } break; case CHUNK_TYPE.TEXT: - { - this._citationPopup = { text: citation.direct_text ?? 'No text available', visible: true }; - this.startCitationPopupTimer(); + this._citationPopup = { text: citation.direct_text ?? 'No text available', visible: true }; + this.startCitationPopupTimer(); - // Check if the document is a PDF (has a PDF viewer component) - const isPDF = PDFCast(dataDoc!.data) !== null || dataDoc!.type === DocumentType.PDF; + // Check if the document is a PDF (has a PDF viewer component) + const isPDF = PDFCast(dataDoc!.data) !== null || dataDoc!.type === DocumentType.PDF; - // First ensure document is fully visible before trying to access its views - this.ensureDocumentIsVisible(dataDoc!, isPDF, citation, foundChunk, doc); - } + // First ensure document is fully visible before trying to access its views + this.ensureDocumentIsVisible(dataDoc!, isPDF, citation, foundChunk, doc); break; case CHUNK_TYPE.CSV: case CHUNK_TYPE.URL: @@ -1068,9 +1065,7 @@ export class ChatBox extends ViewBoxAnnotatableComponent<FieldViewProps>() { { index: 0, type: TEXT_TYPE.NORMAL, - text: this.dataDoc.is_dash_doc_assistant - ? 'Welcome to your help assistant for Dash. Ask any Dash-related questions to get started.' - : `Hey, ${this.userName()}! Welcome to Your Friendly Assistant. Link a document or ask questions to get started.`, + text: `Hey, ${this.userName()}! Welcome to Your Friendly Assistant. Link a document or ask questions to get started.`, citation_ids: null, }, ], @@ -1167,9 +1162,6 @@ export class ChatBox extends ViewBoxAnnotatableComponent<FieldViewProps>() { this._inputValue = question; }; - _dictation: DictationButton | null = null; - setInputRef = (r: HTMLInputElement) => (this._textInputRef = r); - setDictationRef = (r: DictationButton) => (this._dictation = r); /** * Handles tool creation notification and shows the reload modal * @param toolName The name of the tool that was created @@ -1220,6 +1212,8 @@ export class ChatBox extends ViewBoxAnnotatableComponent<FieldViewProps>() { }, 100); }; + _dictation: DictationButton | null = null; + /** * Toggles the font size modal visibility */ @@ -1411,7 +1405,7 @@ export class ChatBox extends ViewBoxAnnotatableComponent<FieldViewProps>() { </div> )} <div className="chat-header"> - <h2>{StrCast(this.dataDoc.title) || `${this.userName()}'s AI Assistant`}</h2> + <h2>{this.userName()}'s AI Assistant</h2> <div className="font-size-control" onClick={this.toggleFontSizeModal}> {this.renderFontSizeIcon()} </div> @@ -1448,7 +1442,9 @@ export class ChatBox extends ViewBoxAnnotatableComponent<FieldViewProps>() { <form onSubmit={this.askGPT} className="chat-input"> <div className="input-container"> <input - ref={this.setInputRef} + ref={r => { + this._textInputRef = r; + }} type="text" name="messageInput" autoComplete="off" @@ -1458,17 +1454,23 @@ export class ChatBox extends ViewBoxAnnotatableComponent<FieldViewProps>() { disabled={this._isLoading} /> </div> - <Button - // className="submit-button" - onClick={() => this._dictation?.stopDictation()} - type={Type.PRIM} - tooltip="Send to AI" - color={SnappingManager.userVariantColor} - inactive={this._isLoading || !this._inputValue.trim()} - icon={<AiOutlineSend />} - size={Size.LARGE} + <button className="submit-button" onClick={() => this._dictation?.stopDictation()} type="submit" disabled={this._isLoading || !this._inputValue.trim()}> + {this._isLoading ? ( + <div className="spinner"></div> + ) : ( + <svg viewBox="0 0 24 24" width="24" height="24" stroke="currentColor" strokeWidth="2" fill="none" strokeLinecap="round" strokeLinejoin="round"> + <line x1="22" y1="2" x2="11" y2="13"></line> + <polygon points="22 2 15 22 11 13 2 9 22 2"></polygon> + </svg> + )} + </button> + <DictationButton + ref={r => { + this._dictation = r; + }} + setInput={this.setChatInput} + inputRef={this._textInputRef} /> - <DictationButton ref={this.setDictationRef} setInput={this.setChatInput} inputRef={this._textInputRef} /> </form> {/* Popup for citation */} {this._citationPopup.visible && ( @@ -1498,7 +1500,7 @@ export class ChatBox extends ViewBoxAnnotatableComponent<FieldViewProps>() { The tool <strong>{this._toolReloadModal.toolName}</strong> has been created and saved successfully. </p> <p>To make the tool available for future use, the page needs to be reloaded to rebuild the application bundle.</p> - <p>Click "Reload Page" to complete the tool installation.</p> + <p>Click "Reload Page" to complete the tool installation.</p> </div> <div className="tool-reload-modal-actions"> <button className="reload-button primary" onClick={this.handleReloadConfirmation}> diff --git a/src/client/views/nodes/chatbot/tools/RAGTool.ts b/src/client/views/nodes/chatbot/tools/RAGTool.ts index b0150868e..7394e175c 100644 --- a/src/client/views/nodes/chatbot/tools/RAGTool.ts +++ b/src/client/views/nodes/chatbot/tools/RAGTool.ts @@ -66,6 +66,7 @@ const ragToolInfo: ToolInfo<RAGToolParamsType> = { - Use as many citations as possible (even when one would be sufficient), thus keeping text as grounded as possible. - When using text citations, keep the EXACT TEXT FROM THE CHUNK—WORD FOR WORD—DO NOT EMIT ANYTHING OR ADD ANYTHING. DO NOT PARAPHRASE! DO NOT CITE TEXT CONTENT FROM A TABLE OR IMAGE—INSTEAD CITE THE TABLE OR IMAGE ITSELF! - Cite from as many documents as possible and always use MORE, and as granular, citations as possible. + - If you see a table in an image of another table (it is on the same page, but not the table that is circled in red), use the RAG tool again and specifically try to search for that table by using, as the hypothetical_document_chunk, a summary of the table you are trying to find. DO NOT JUST CITE THE TABLE IN RED ON THE SAME PAGE! - CITATION TEXT MUST BE EXACTLY AS IT APPEARS IN THE CHUNK. DO NOT PARAPHRASE!`, parameterRules: ragToolParams, }; diff --git a/src/server/chunker/pdf_chunker.py b/src/server/chunker/pdf_chunker.py index 914594f1e..7cb7d077c 100644 --- a/src/server/chunker/pdf_chunker.py +++ b/src/server/chunker/pdf_chunker.py @@ -701,43 +701,20 @@ class Document: :return: The generated summary of the document. """ - # num_clusters = min(10, len(self.chunks)) # Set number of clusters for KMeans, capped at 10 - # kmeans = KMeans(n_clusters=num_clusters, random_state=42) # Initialize KMeans with 10 clusters - # doc_chunks = [chunk['values'] for chunk in self.chunks if 'values' in chunk] # Extract embeddings - # cluster_labels = kmeans.fit_predict(doc_chunks) # Assign each chunk to a cluster - - doc_chunks = [chunk['values'] for chunk in self.chunks if 'values' in chunk] - if not doc_chunks: - raise ValueError("No valid embedded chunks to summarize.") - - # Remove duplicates (e.g., from OCR-ed blank pages or repeated captions) - unique_chunks = np.unique(np.array(doc_chunks), axis=0) - - # Dynamically scale number of clusters to available signal - num_clusters = min(10, len(unique_chunks)) - kmeans = KMeans(n_clusters=num_clusters, random_state=42).fit(unique_chunks) - - # Predict cluster labels for original chunks (not just unique ones) - cluster_labels = kmeans.predict(np.array(doc_chunks)) - + num_clusters = min(10, len(self.chunks)) # Set number of clusters for KMeans, capped at 10 + kmeans = KMeans(n_clusters=num_clusters, random_state=42) # Initialize KMeans with 10 clusters + doc_chunks = [chunk['values'] for chunk in self.chunks if 'values' in chunk] # Extract embeddings + cluster_labels = kmeans.fit_predict(doc_chunks) # Assign each chunk to a cluster # Select representative chunks from each cluster selected_chunks = [] for i in range(num_clusters): - # cluster_chunks = [chunk for chunk, label in zip(self.chunks, cluster_labels) if label == i] # Get all chunks in this cluster - # cluster_embs = [emb for emb, label in zip(doc_chunks, cluster_labels) if label == i] # Get embeddings for this cluster - - cluster_idxs = np.where(cluster_labels == i)[0] - if len(cluster_idxs) == 0: - continue # skip empty clusters (shouldn't happen after downsizing) - + cluster_chunks = [chunk for chunk, label in zip(self.chunks, cluster_labels) if label == i] # Get all chunks in this cluster + cluster_embs = [emb for emb, label in zip(doc_chunks, cluster_labels) if label == i] # Get embeddings for this cluster centroid = kmeans.cluster_centers_[i] # Get the centroid of the cluster - distances = [np.linalg.norm(doc_chunks[idx] - centroid) for idx in cluster_idxs] - closest_idx = cluster_idxs[int(np.argmin(distances))] - selected_chunks.append(self.chunks[closest_idx]) - # distances = [np.linalg.norm(np.array(emb) - centroid) for emb in cluster_embs] # Compute distance to centroid - # closest_chunk = cluster_chunks[np.argmin(distances)] # Select chunk closest to the centroid - # selected_chunks.append(closest_chunk) + distances = [np.linalg.norm(np.array(emb) - centroid) for emb in cluster_embs] # Compute distance to centroid + closest_chunk = cluster_chunks[np.argmin(distances)] # Select chunk closest to the centroid + selected_chunks.append(closest_chunk) # Combine selected chunks into a summary combined_text = "\n\n".join([chunk['metadata']['text'] for chunk in selected_chunks]) # Concatenate chunk texts |