diff options
Diffstat (limited to 'src/client/views/nodes/chatbot/chatboxcomponents/ChatBox.tsx')
| -rw-r--r-- | src/client/views/nodes/chatbot/chatboxcomponents/ChatBox.tsx | 610 |
1 files changed, 452 insertions, 158 deletions
diff --git a/src/client/views/nodes/chatbot/chatboxcomponents/ChatBox.tsx b/src/client/views/nodes/chatbot/chatboxcomponents/ChatBox.tsx index 44c231c87..6e9307d37 100644 --- a/src/client/views/nodes/chatbot/chatboxcomponents/ChatBox.tsx +++ b/src/client/views/nodes/chatbot/chatboxcomponents/ChatBox.tsx @@ -13,29 +13,41 @@ import { observer } from 'mobx-react'; import OpenAI, { ClientOptions } from 'openai'; import * as React from 'react'; import { v4 as uuidv4 } from 'uuid'; -import { ClientUtils } from '../../../../../ClientUtils'; -import { Doc, DocListCast } from '../../../../../fields/Doc'; +import { ClientUtils, OmitKeys } from '../../../../../ClientUtils'; +import { Doc, DocListCast, Opt } from '../../../../../fields/Doc'; import { DocData, DocViews } from '../../../../../fields/DocSymbols'; -import { CsvCast, DocCast, PDFCast, RTFCast, StrCast } from '../../../../../fields/Types'; -import { Networking } from '../../../../Network'; +import { RichTextField } from '../../../../../fields/RichTextField'; +import { ScriptField } from '../../../../../fields/ScriptField'; +import { CsvCast, DocCast, NumCast, PDFCast, RTFCast, StrCast } from '../../../../../fields/Types'; import { DocUtils } from '../../../../documents/DocUtils'; -import { DocumentType } from '../../../../documents/DocumentTypes'; -import { Docs } from '../../../../documents/Documents'; +import { CollectionViewType, DocumentType } from '../../../../documents/DocumentTypes'; +import { Docs, DocumentOptions } from '../../../../documents/Documents'; import { DocumentManager } from '../../../../util/DocumentManager'; +import { ImageUtils } from '../../../../util/Import & Export/ImageUtils'; import { LinkManager } from '../../../../util/LinkManager'; +import { CompileError, CompileScript } from '../../../../util/Scripting'; +import { DictationButton } from '../../../DictationButton'; import { ViewBoxAnnotatableComponent } from '../../../DocComponent'; -import { DocumentView } from '../../DocumentView'; +import { AudioBox } from '../../AudioBox'; +import { DocumentView, DocumentViewInternal } from '../../DocumentView'; import { FieldView, FieldViewProps } from '../../FieldView'; import { PDFBox } from '../../PDFBox'; +import { ScriptingBox } from '../../ScriptingBox'; +import { VideoBox } from '../../VideoBox'; import { Agent } from '../agentsystem/Agent'; +import { supportedDocTypes } from '../tools/CreateDocumentTool'; import { ASSISTANT_ROLE, AssistantMessage, CHUNK_TYPE, Citation, ProcessingInfo, SimplifiedChunk, TEXT_TYPE } from '../types/types'; import { Vectorstore } from '../vectorstore/Vectorstore'; import './ChatBox.scss'; import MessageComponentBox from './MessageComponent'; import { ProgressBar } from './ProgressBar'; +import { OpenWhere } from '../../OpenWhere'; +import { Upload } from '../../../../../server/SharedMediaTypes'; dotenv.config(); +export type parsedDocData = { doc_type: string; data: unknown }; +export type parsedDoc = DocumentOptions & parsedDocData; /** * ChatBox is the main class responsible for managing the interaction between the user and the assistant, * handling documents, and integrating with OpenAI for tasks such as document analysis, chat functionality, @@ -44,17 +56,17 @@ dotenv.config(); @observer export class ChatBox extends ViewBoxAnnotatableComponent<FieldViewProps>() { // MobX observable properties to track UI state and data - @observable history: AssistantMessage[] = []; - @observable.deep current_message: AssistantMessage | undefined = undefined; - @observable isLoading: boolean = false; - @observable uploadProgress: number = 0; - @observable currentStep: string = ''; - @observable expandedScratchpadIndex: number | null = null; - @observable inputValue: string = ''; - @observable private linked_docs_to_add: ObservableSet = observable.set(); - @observable private linked_csv_files: { filename: string; id: string; text: string }[] = []; - @observable private isUploadingDocs: boolean = false; - @observable private citationPopup: { text: string; visible: boolean } = { text: '', visible: false }; + @observable private _history: AssistantMessage[] = []; + @observable.deep private _current_message: AssistantMessage | undefined = undefined; + @observable private _isLoading: boolean = false; + @observable private _uploadProgress: number = 0; + @observable private _currentStep: string = ''; + @observable private _expandedScratchpadIndex: number | null = null; + @observable private _inputValue: string = ''; + @observable private _linked_docs_to_add: ObservableSet = observable.set(); + @observable private _linked_csv_files: { filename: string; id: string; text: string }[] = []; + @observable private _isUploadingDocs: boolean = false; + @observable private _citationPopup: { text: string; visible: boolean } = { text: '', visible: false }; // Private properties for managing OpenAI API, vector store, agent, and UI elements private openai: OpenAI; @@ -62,6 +74,7 @@ export class ChatBox extends ViewBoxAnnotatableComponent<FieldViewProps>() { private vectorstore: Vectorstore; private agent: Agent; private messagesRef: React.RefObject<HTMLDivElement>; + private _textInputRef: HTMLInputElement | undefined | null; /** * Static method that returns the layout string for the field. @@ -71,6 +84,10 @@ export class ChatBox extends ViewBoxAnnotatableComponent<FieldViewProps>() { return FieldView.LayoutString(ChatBox, fieldKey); } + setChatInput = action((input: string) => { + this._inputValue = input; + }); + /** * Constructor initializes the component, sets up OpenAI, vector store, and agent instances, * and observes changes in the chat history to save the state in dataDoc. @@ -89,13 +106,13 @@ export class ChatBox extends ViewBoxAnnotatableComponent<FieldViewProps>() { this.vectorstore_id = StrCast(this.dataDoc.vectorstore_id); } this.vectorstore = new Vectorstore(this.vectorstore_id, this.retrieveDocIds); - this.agent = new Agent(this.vectorstore, this.retrieveSummaries, this.retrieveFormattedHistory, this.retrieveCSVData, this.addLinkedUrlDoc, this.createCSVInDash); + this.agent = new Agent(this.vectorstore, this.retrieveSummaries, this.retrieveFormattedHistory, this.retrieveCSVData, this.addLinkedUrlDoc, this.createImageInDash, this.createDocInDash, this.createCSVInDash); this.messagesRef = React.createRef<HTMLDivElement>(); // Reaction to update dataDoc when chat history changes reaction( () => - this.history.map((msg: AssistantMessage) => ({ + this._history.map((msg: AssistantMessage) => ({ role: msg.role, content: msg.content, follow_up_questions: msg.follow_up_questions, @@ -114,20 +131,22 @@ export class ChatBox extends ViewBoxAnnotatableComponent<FieldViewProps>() { */ @action addDocToVectorstore = async (newLinkedDoc: Doc) => { - this.uploadProgress = 0; - this.currentStep = 'Initializing...'; - this.isUploadingDocs = true; + this._uploadProgress = 0; + this._currentStep = 'Initializing...'; + this._isUploadingDocs = true; try { // Add the document to the vectorstore await this.vectorstore.addAIDoc(newLinkedDoc, this.updateProgress); } catch (error) { console.error('Error uploading document:', error); - this.currentStep = 'Error during upload'; + this._currentStep = 'Error during upload'; } finally { - this.isUploadingDocs = false; - this.uploadProgress = 0; - this.currentStep = ''; + runInAction(() => { + this._isUploadingDocs = false; + this._uploadProgress = 0; + this._currentStep = ''; + }); } }; @@ -138,8 +157,8 @@ export class ChatBox extends ViewBoxAnnotatableComponent<FieldViewProps>() { */ @action updateProgress = (progress: number, step: string) => { - this.uploadProgress = progress; - this.currentStep = step; + this._uploadProgress = progress; + this._currentStep = step; }; /** @@ -176,7 +195,7 @@ export class ChatBox extends ViewBoxAnnotatableComponent<FieldViewProps>() { const csvId = id ?? uuidv4(); // Add CSV details to linked files - this.linked_csv_files.push({ + this._linked_csv_files.push({ filename: CsvCast(newLinkedDoc.data).url.pathname, id: csvId, text: csvData, @@ -198,7 +217,7 @@ export class ChatBox extends ViewBoxAnnotatableComponent<FieldViewProps>() { */ @action toggleToolLogs = (index: number) => { - this.expandedScratchpadIndex = this.expandedScratchpadIndex === index ? null : index; + this._expandedScratchpadIndex = this._expandedScratchpadIndex === index ? null : index; }; /** @@ -257,7 +276,7 @@ export class ChatBox extends ViewBoxAnnotatableComponent<FieldViewProps>() { @action askGPT = async (event: React.FormEvent): Promise<void> => { event.preventDefault(); - this.inputValue = ''; + this._inputValue = ''; // Extract the user's message const textInput = (event.currentTarget as HTMLFormElement).elements.namedItem('messageInput') as HTMLInputElement; @@ -267,13 +286,13 @@ export class ChatBox extends ViewBoxAnnotatableComponent<FieldViewProps>() { try { textInput.value = ''; // Add the user's message to the history - this.history.push({ + this._history.push({ role: ASSISTANT_ROLE.USER, content: [{ index: 0, type: TEXT_TYPE.NORMAL, text: trimmedText, citation_ids: null }], processing_info: [], }); - this.isLoading = true; - this.current_message = { + this._isLoading = true; + this._current_message = { role: ASSISTANT_ROLE.ASSISTANT, content: [], citations: [], @@ -283,9 +302,9 @@ export class ChatBox extends ViewBoxAnnotatableComponent<FieldViewProps>() { // Define callbacks for real-time processing updates const onProcessingUpdate = (processingUpdate: ProcessingInfo[]) => { runInAction(() => { - if (this.current_message) { - this.current_message = { - ...this.current_message, + if (this._current_message) { + this._current_message = { + ...this._current_message, processing_info: processingUpdate, }; } @@ -295,9 +314,9 @@ export class ChatBox extends ViewBoxAnnotatableComponent<FieldViewProps>() { const onAnswerUpdate = (answerUpdate: string) => { runInAction(() => { - if (this.current_message) { - this.current_message = { - ...this.current_message, + if (this._current_message) { + this._current_message = { + ...this._current_message, content: [{ text: answerUpdate, type: TEXT_TYPE.NORMAL, index: 0, citation_ids: [] }], }; } @@ -309,22 +328,26 @@ export class ChatBox extends ViewBoxAnnotatableComponent<FieldViewProps>() { // Update the history with the final assistant message runInAction(() => { - if (this.current_message) { - this.history.push({ ...finalMessage }); - this.current_message = undefined; - this.dataDoc.data = JSON.stringify(this.history); + if (this._current_message) { + this._history.push({ ...finalMessage }); + this._current_message = undefined; + this.dataDoc.data = JSON.stringify(this._history); } }); } catch (err) { console.error('Error:', err); // Handle error in processing - this.history.push({ - role: ASSISTANT_ROLE.ASSISTANT, - content: [{ index: 0, type: TEXT_TYPE.ERROR, text: 'Sorry, I encountered an error while processing your request.', citation_ids: null }], - processing_info: [], - }); + runInAction(() => + this._history.push({ + role: ASSISTANT_ROLE.ASSISTANT, + content: [{ index: 0, type: TEXT_TYPE.ERROR, text: `Sorry, I encountered an error while processing your request: ${err} `, citation_ids: null }], + processing_info: [], + }) + ); } finally { - this.isLoading = false; + runInAction(() => { + this._isLoading = false; + }); this.scrollToBottom(); } } @@ -338,8 +361,8 @@ export class ChatBox extends ViewBoxAnnotatableComponent<FieldViewProps>() { */ @action updateMessageCitations = (index: number, citations: Citation[]) => { - if (this.history[index]) { - this.history[index].citations = citations; + if (this._history[index]) { + this._history[index].citations = citations; } }; @@ -354,29 +377,11 @@ export class ChatBox extends ViewBoxAnnotatableComponent<FieldViewProps>() { const linkDoc = Docs.Create.LinkDocument(this.Document, doc); LinkManager.Instance.addLink(linkDoc); - let canDisplay; - - try { - // Fetch the URL content through the proxy - const { data } = await Networking.PostToServer('/proxyFetch', { url }); - - // Simulating header behavior since we can't fetch headers via proxy - const xFrameOptions = data.headers?.['x-frame-options']; - - if (xFrameOptions && xFrameOptions.toUpperCase() === 'SAMEORIGIN') { - canDisplay = false; - } else { - canDisplay = true; - } - } catch (error) { - console.error('Error fetching the URL from the server:', error); - } const chunkToAdd = { chunkId: id, chunkType: CHUNK_TYPE.URL, url: url, - canDisplay: canDisplay, }; doc.chunk_simpl = JSON.stringify({ chunks: [chunkToAdd] }); @@ -398,89 +403,364 @@ export class ChatBox extends ViewBoxAnnotatableComponent<FieldViewProps>() { * @param data The CSV data content. */ @action - createCSVInDash = async (url: string, title: string, id: string, data: string) => { - const doc = DocCast(await DocUtils.DocumentFromType('csv', url, { title: title, text: RTFCast(data) })); + createCSVInDash = (url: string, title: string, id: string, data: string) => + DocUtils.DocumentFromType('csv', url, { title: title, text: RTFCast(data) }).then(doc => { + if (doc) { + LinkManager.Instance.addLink(Docs.Create.LinkDocument(this.Document, doc)); + this._props.addDocument?.(doc); + DocumentManager.Instance.showDocument(doc, { willZoomCentered: true }, () => {}).then(() => this.addCSVForAnalysis(doc, id)); + } + }); + @action + createImageInDash = async (result: Upload.FileInformation & Upload.InspectionResults, options: DocumentOptions) => { + const newImgSrc = + result.accessPaths.agnostic.client.indexOf('dashblobstore') === -1 // + ? ClientUtils.prepend(result.accessPaths.agnostic.client) + : result.accessPaths.agnostic.client; + const doc = Docs.Create.ImageDocument(newImgSrc, options); + this.addDocument(ImageUtils.AssignImgInfo(doc, result)); const linkDoc = Docs.Create.LinkDocument(this.Document, doc); LinkManager.Instance.addLink(linkDoc); - - doc && this._props.addDocument?.(doc); + if (doc) { + if (this._props.addDocument) this._props.addDocument(doc); + else DocumentViewInternal.addDocTabFunc(doc, OpenWhere.addRight); + } await DocumentManager.Instance.showDocument(doc, { willZoomCentered: true }, () => {}); + }; + + /** + * Creates a text document in the dashboard and adds it for analysis. + * @param title The title of the doc. + * @param text_content The text of the document. + * @param options Other optional document options (e.g. color) + * @param id The unique ID for the document. + */ + @action + private createCollectionWithChildren = (data: parsedDoc[], insideCol: boolean): Opt<Doc>[] => data.map(doc => this.whichDoc(doc, insideCol)); + + @action + whichDoc = (doc: parsedDoc, insideCol: boolean): Opt<Doc> => { + const options = OmitKeys(doc, ['doct_type', 'data']).omit as DocumentOptions; + const data = (doc as parsedDocData).data; + const ndoc = (() => { + switch (doc.doc_type) { + default: + case supportedDocTypes.text: return Docs.Create.TextDocument(data as string, options); + case supportedDocTypes.comparison: return this.createComparison(JSON.parse(data as string) as parsedDoc[], options); + case supportedDocTypes.flashcard: return this.createFlashcard(JSON.parse(data as string) as parsedDoc[], options); + case supportedDocTypes.deck: return this.createDeck(JSON.parse(data as string) as parsedDoc[], options); + case supportedDocTypes.image: return Docs.Create.ImageDocument(data as string, options); + case supportedDocTypes.equation: return Docs.Create.EquationDocument(data as string, options); + case supportedDocTypes.notetaking: return Docs.Create.NoteTakingDocument([], options); + case supportedDocTypes.web: return Docs.Create.WebDocument(data as string, { ...options, data_useCors: true }); + case supportedDocTypes.dataviz: return Docs.Create.DataVizDocument('/users/rz/Downloads/addresses.csv', options); + case supportedDocTypes.pdf: return Docs.Create.PdfDocument(data as string, options); + case supportedDocTypes.video: return Docs.Create.VideoDocument(data as string, options); + case supportedDocTypes.diagram: return Docs.Create.DiagramDocument(undefined, { text: data as unknown as RichTextField, ...options}); // text: can take a string or RichTextField but it's typed for RichTextField. + + // case supportedDocumentTypes.dataviz: + // { + // const { fileUrl, id } = await Networking.PostToServer('/createCSV', { + // filename: (options.title as string).replace(/\s+/g, '') + '.csv', + // data: data, + // }); + // const doc = Docs.Create.DataVizDocument(fileUrl, { ...options, text: RTFCast(data as string) }); + // this.addCSVForAnalysis(doc, id); + // return doc; + // } + case supportedDocTypes.script: { + const result = !(data as string).trim() ? ({ compiled: false, errors: [] } as CompileError) : CompileScript(data as string, {}); + const script_field = result.compiled ? new ScriptField(result, undefined, data as string) : undefined; + const sdoc = Docs.Create.ScriptingDocument(script_field, options); + DocumentManager.Instance.showDocument(sdoc, { willZoomCentered: true }, () => { + const firstView = Array.from(sdoc[DocViews])[0] as DocumentView; + (firstView.ComponentView as ScriptingBox)?.onApply?.(); + (firstView.ComponentView as ScriptingBox)?.onRun?.(); + }); + return sdoc; + } + case supportedDocTypes.collection: { + const arr = this.createCollectionWithChildren(JSON.parse(data as string) as parsedDoc[], true).filter(d=>d).map(d => d!); + const collOpts = { _width:300, _height: 300, _layout_fitWidth: true, _freeform_backgroundGrid: true, ...options, }; + return (() => { + switch (options.type_collection) { + case CollectionViewType.Tree: return Docs.Create.TreeDocument(arr, collOpts); + case CollectionViewType.Stacking: return Docs.Create.StackingDocument(arr, collOpts); + case CollectionViewType.Masonry: return Docs.Create.MasonryDocument(arr, collOpts); + case CollectionViewType.Card: return Docs.Create.CardDeckDocument(arr, collOpts); + case CollectionViewType.Carousel: return Docs.Create.CarouselDocument(arr, collOpts); + case CollectionViewType.Carousel3D: return Docs.Create.Carousel3DDocument(arr, collOpts); + case CollectionViewType.Multicolumn: return Docs.Create.CarouselDocument(arr, collOpts); + default: return Docs.Create.FreeformDocument(arr, collOpts); + } + })(); + } + // case supportedDocumentTypes.map: return Docs.Create.MapDocument([], options); + // case supportedDocumentTypes.button: return Docs.Create.ButtonDocument(options); + // case supportedDocumentTypes.trail: return Docs.Create.PresDocument(options); + } // prettier-ignore + })(); + + if (ndoc) { + ndoc.x = NumCast((options.x as number) ?? 0) + (insideCol ? 0 : NumCast(this.layoutDoc.x) + NumCast(this.layoutDoc.width)) + 100; + ndoc.y = NumCast(options.y as number) + (insideCol ? 0 : NumCast(this.layoutDoc.y)); + } + return ndoc; + }; + + /** + * Creates a document in the dashboard. + * + * @param {string} doc_type - The type of document to create. + * @param {string} data - The data used to generate the document. + * @param {DocumentOptions} options - Configuration options for the document. + * @returns {Promise<void>} A promise that resolves once the document is created and displayed. + */ + @action + createDocInDash = (pdoc: parsedDoc) => { + const linkAndShowDoc = (doc: Opt<Doc>) => { + if (doc) { + LinkManager.Instance.addLink(Docs.Create.LinkDocument(this.Document, doc)); + this._props.addDocument?.(doc); + DocumentManager.Instance.showDocument(doc, { willZoomCentered: true }, () => {}); + } + }; + const doc = this.whichDoc(pdoc, false); + if (doc) linkAndShowDoc(doc); + return doc; + }; + + /** + * Creates a deck of flashcards. + * + * @param {any} data - The data used to generate the flashcards. Can be a string or an object. + * @param {DocumentOptions} options - Configuration options for the flashcard deck. + * @returns {Doc} A carousel document containing the flashcard deck. + */ + @action + createDeck = (data: parsedDoc[], options: DocumentOptions) => { + const flashcardDeck: Doc[] = []; + // Process each flashcard document in the `deckData` array + if (data.length == 2 && data[0].doc_type == 'text' && data[1].doc_type == 'text') { + this.createFlashcard(data, options); + } else { + data.forEach(doc => { + const flashcardDoc = this.createFlashcard((doc as parsedDocData).data as parsedDoc[] | string[], options); + if (flashcardDoc) flashcardDeck.push(flashcardDoc); + }); + } + + // Create a carousel to contain the flashcard deck + return Docs.Create.CarouselDocument(flashcardDeck, { + title: options.title || 'Flashcard Deck', + _width: options._width || 300, + _height: options._height || 300, + _layout_fitWidth: false, + _layout_autoHeight: true, + }); + }; + + /** + * Creates a single flashcard document. + * + * @param {any} data - The data used to generate the flashcard. Can be a string or an object. + * @param {any} options - Configuration options for the flashcard. + * @returns {Doc | undefined} The created flashcard document, or undefined if the flashcard cannot be created. + */ + @action + createFlashcard = (data: parsedDoc[] | string[], options: DocumentOptions) => { + const [front, back] = data; + const sideOptions = { _height: 300, ...options }; + + // Create front and back text documents + const side1 = typeof front === 'string' ? Docs.Create.CenteredTextCreator('question', front as string, sideOptions) : this.whichDoc(front, false); + const side2 = typeof back === 'string' ? Docs.Create.CenteredTextCreator('answer', back as string, sideOptions) : this.whichDoc(back, false); - this.addCSVForAnalysis(doc, id); + // Create the flashcard document with both sides + return Docs.Create.FlashcardDocument('flashcard', side1, side2, sideOptions); }; /** + * Creates a comparison document. + * + * @param {any} doc - The document data containing left and right components for comparison. + * @param {any} options - Configuration options for the comparison document. + * @returns {Doc} The created comparison document. + */ + @action + createComparison = (doc: parsedDoc[], options: DocumentOptions) => + Docs.Create.ComparisonDocument(options.title as string, { + data_back: this.whichDoc(doc[0], false), + data_front: this.whichDoc(doc[1], false), + _width: options._width, + _height: options._height || 300, + backgroundColor: options.backgroundColor, + }); + + /** * Event handler to manage citations click in the message components. * @param citation The citation object clicked by the user. */ @action - handleCitationClick = (citation: Citation) => { + handleCitationClick = async (citation: Citation) => { const currentLinkedDocs: Doc[] = this.linkedDocs; - const chunkId = citation.chunk_id; - // Loop through the linked documents to find the matching chunk and handle its display for (const doc of currentLinkedDocs) { if (doc.chunk_simpl) { const docChunkSimpl = JSON.parse(StrCast(doc.chunk_simpl)) as { chunks: SimplifiedChunk[] }; const foundChunk = docChunkSimpl.chunks.find(chunk => chunk.chunkId === chunkId); + if (foundChunk) { - // Handle different types of chunks (image, text, table, etc.) - switch (foundChunk.chunkType) { - case CHUNK_TYPE.IMAGE: - case CHUNK_TYPE.TABLE: - { - const values = foundChunk.location?.replace(/[[\]]/g, '').split(','); + // Handle media chunks specifically + + if (doc.ai_type == 'video' || doc.ai_type == 'audio') { + const directMatchSegmentStart = this.getDirectMatchingSegmentStart(doc, citation.direct_text || '', foundChunk.indexes || []); + + if (directMatchSegmentStart) { + // Navigate to the segment's start time in the media player + await this.goToMediaTimestamp(doc, directMatchSegmentStart, doc.ai_type); + } else { + console.error('No direct matching segment found for the citation.'); + } + } else { + // Handle other chunk types as before + this.handleOtherChunkTypes(foundChunk, citation, doc); + } + } + } + } + }; - if (values?.length !== 4) { - console.error('Location string must contain exactly 4 numbers'); - return; - } + getDirectMatchingSegmentStart = (doc: Doc, citationText: string, indexesOfSegments: string[]): number => { + const originalSegments = JSON.parse(StrCast(doc.original_segments!)).map((segment: any, index: number) => ({ + index: index.toString(), + text: segment.text, + start: segment.start, + end: segment.end, + })); - const x1 = parseFloat(values[0]) * Doc.NativeWidth(doc); - const y1 = parseFloat(values[1]) * Doc.NativeHeight(doc) + foundChunk.startPage * Doc.NativeHeight(doc); - const x2 = parseFloat(values[2]) * Doc.NativeWidth(doc); - const y2 = parseFloat(values[3]) * Doc.NativeHeight(doc) + foundChunk.startPage * Doc.NativeHeight(doc); + if (!Array.isArray(originalSegments) || originalSegments.length === 0 || !Array.isArray(indexesOfSegments)) { + return 0; + } - const annotationKey = Doc.LayoutFieldKey(doc) + '_annotations'; + // Create itemsToSearch array based on indexesOfSegments + const itemsToSearch = indexesOfSegments.map((indexStr: string) => { + const index = parseInt(indexStr, 10); + const segment = originalSegments[index]; + return { text: segment.text, start: segment.start }; + }); - const existingDoc = DocListCast(doc[DocData][annotationKey]).find(d => d.citation_id === citation.citation_id); - const highlightDoc = existingDoc ?? this.createImageCitationHighlight(x1, y1, x2, y2, citation, annotationKey, doc); + console.log('Constructed itemsToSearch:', itemsToSearch); - DocumentManager.Instance.showDocument(highlightDoc, { willZoomCentered: true }, () => {}); - } - break; - case CHUNK_TYPE.TEXT: - this.citationPopup = { text: citation.direct_text ?? 'No text available', visible: true }; - setTimeout(() => (this.citationPopup.visible = false), 3000); // Hide after 3 seconds - - DocumentManager.Instance.showDocument(doc, { willZoomCentered: true }, () => { - const firstView = Array.from(doc[DocViews])[0] as DocumentView; - (firstView.ComponentView as PDFBox)?.gotoPage?.(foundChunk.startPage); - (firstView.ComponentView as PDFBox)?.search?.(citation.direct_text ?? ''); - }); - break; - case CHUNK_TYPE.URL: - if (!foundChunk.canDisplay) { - window.open(StrCast(doc.displayUrl), '_blank'); - } else if (foundChunk.canDisplay) { - DocumentManager.Instance.showDocument(doc, { willZoomCentered: true }, () => {}); - } - break; - case CHUNK_TYPE.CSV: - DocumentManager.Instance.showDocument(doc, { willZoomCentered: true }, () => {}); - break; - default: - console.error('Chunk type not recognized:', foundChunk.chunkType); - break; - } - } + // Helper function to calculate word overlap score + const calculateWordOverlap = (text1: string, text2: string): number => { + const words1 = new Set(text1.toLowerCase().split(/\W+/)); + const words2 = new Set(text2.toLowerCase().split(/\W+/)); + const intersection = new Set([...words1].filter(word => words2.has(word))); + return intersection.size / Math.max(words1.size, words2.size); // Jaccard similarity + }; + + // Search for the best matching segment + let bestMatchStart = 0; + let bestScore = 0; + + console.log(`Searching for best match for query: "${citationText}"`); + itemsToSearch.forEach(item => { + const score = calculateWordOverlap(citationText, item.text); + console.log(`Comparing query to segment: "${item.text}" | Score: ${score}`); + if (score > bestScore) { + bestScore = score; + bestMatchStart = item.start; } + }); + + console.log('Best match found with score:', bestScore, '| Start time:', bestMatchStart); + + // Return the start time of the best match + return bestMatchStart; + }; + + /** + * Navigates to the given timestamp in the media player. + * @param doc The document containing the media file. + * @param timestamp The timestamp to navigate to. + */ + goToMediaTimestamp = async (doc: Doc, timestamp: number, type: 'video' | 'audio') => { + try { + // Show the media document in the viewer + if (type == 'video') { + DocumentManager.Instance.showDocument(doc, { willZoomCentered: true }, () => { + const firstView = Array.from(doc[DocViews])[0] as DocumentView; + (firstView.ComponentView as VideoBox)?.Seek?.(timestamp); + }); + } else { + DocumentManager.Instance.showDocument(doc, { willZoomCentered: true }, () => { + const firstView = Array.from(doc[DocViews])[0] as DocumentView; + (firstView.ComponentView as AudioBox)?.playFrom?.(timestamp); + }); + } + console.log(`Navigated to timestamp: ${timestamp}s in document ${doc.id}`); + } catch (error) { + console.error('Error navigating to media timestamp:', error); } }; /** + * Handles non-media chunk types as before. + * @param foundChunk The chunk object. + * @param citation The citation object. + * @param doc The document containing the chunk. + */ + handleOtherChunkTypes = (foundChunk: SimplifiedChunk, citation: Citation, doc: Doc) => { + switch (foundChunk.chunkType) { + case CHUNK_TYPE.IMAGE: + case CHUNK_TYPE.TABLE: + { + const values = foundChunk.location?.replace(/[[\]]/g, '').split(','); + + if (values?.length !== 4) { + console.error('Location string must contain exactly 4 numbers'); + return; + } + if (foundChunk.startPage === undefined || foundChunk.endPage === undefined) { + DocumentManager.Instance.showDocument(doc, { willZoomCentered: true }, () => {}); + return; + } + const x1 = parseFloat(values[0]) * Doc.NativeWidth(doc); + const y1 = parseFloat(values[1]) * Doc.NativeHeight(doc) + foundChunk.startPage * Doc.NativeHeight(doc); + const x2 = parseFloat(values[2]) * Doc.NativeWidth(doc); + const y2 = parseFloat(values[3]) * Doc.NativeHeight(doc) + foundChunk.startPage * Doc.NativeHeight(doc); + + const annotationKey = Doc.LayoutFieldKey(doc) + '_annotations'; + + const existingDoc = DocListCast(doc[DocData][annotationKey]).find(d => d.citation_id === citation.citation_id); + const highlightDoc = existingDoc ?? this.createImageCitationHighlight(x1, y1, x2, y2, citation, annotationKey, doc); + + DocumentManager.Instance.showDocument(highlightDoc, { willZoomCentered: true }, () => {}); + } + break; + case CHUNK_TYPE.TEXT: + this._citationPopup = { text: citation.direct_text ?? 'No text available', visible: true }; + setTimeout(() => (this._citationPopup.visible = false), 3000); + + DocumentManager.Instance.showDocument(doc, { willZoomCentered: true }, () => { + const firstView = Array.from(doc[DocViews])[0] as DocumentView; + (firstView.ComponentView as PDFBox)?.gotoPage?.(foundChunk.startPage ?? 0); + (firstView.ComponentView as PDFBox)?.search?.(citation.direct_text ?? ''); + }); + break; + case CHUNK_TYPE.CSV: + case CHUNK_TYPE.URL: + DocumentManager.Instance.showDocument(doc, { willZoomCentered: true }); + break; + default: + console.error('Unhandled chunk type:', foundChunk.chunkType); + break; + } + }; + /** * Creates an annotation highlight on a PDF document for image citations. * @param x1 X-coordinate of the top-left corner of the highlight. * @param y1 Y-coordinate of the top-left corner of the highlight. @@ -524,7 +804,7 @@ export class ChatBox extends ViewBoxAnnotatableComponent<FieldViewProps>() { try { const storedHistory = JSON.parse(StrCast(this.dataDoc.data)); runInAction(() => { - this.history.push( + this._history.push( ...storedHistory.map((msg: AssistantMessage) => ({ role: msg.role, content: msg.content, @@ -539,7 +819,7 @@ export class ChatBox extends ViewBoxAnnotatableComponent<FieldViewProps>() { } else { // Default welcome message runInAction(() => { - this.history.push({ + this._history.push({ role: ASSISTANT_ROLE.ASSISTANT, content: [ { @@ -563,16 +843,16 @@ export class ChatBox extends ViewBoxAnnotatableComponent<FieldViewProps>() { .filter(d => d); return linkedDocs; }, - linked => linked.forEach(doc => this.linked_docs_to_add.add(doc)) + linked => linked.forEach(doc => this._linked_docs_to_add.add(doc)) ); // Observe changes to linked documents and handle document addition - observe(this.linked_docs_to_add, change => { + observe(this._linked_docs_to_add, change => { if (change.type === 'add') { - if (PDFCast(change.newValue.data)) { - this.addDocToVectorstore(change.newValue); - } else if (CsvCast(change.newValue.data)) { + if (CsvCast(change.newValue.data)) { this.addCSVForAnalysis(change.newValue); + } else { + this.addDocToVectorstore(change.newValue); } } else if (change.type === 'delete') { // Handle document removal @@ -609,7 +889,10 @@ export class ChatBox extends ViewBoxAnnotatableComponent<FieldViewProps>() { .map(d => DocCast(LinkManager.getOppositeAnchor(d, this.Document))) .map(d => DocCast(d?.annotationOn, d)) .filter(d => d) - .filter(d => d.ai_doc_id) + .filter(d => { + console.log(d.ai_doc_id); + return d.ai_doc_id; + }) .map(d => StrCast(d.ai_doc_id)); } @@ -640,18 +923,16 @@ export class ChatBox extends ViewBoxAnnotatableComponent<FieldViewProps>() { /** * Getter that retrieves all linked CSV files for analysis. */ - @computed - get linkedCSVs(): { filename: string; id: string; text: string }[] { - return this.linked_csv_files; + @computed get linkedCSVs(): { filename: string; id: string; text: string }[] { + return this._linked_csv_files; } /** * Getter that formats the entire chat history as a string for the agent's system message. */ - @computed - get formattedHistory(): string { + @computed get formattedHistory(): string { let history = '<chat_history>\n'; - for (const message of this.history) { + for (const message of this._history) { history += `<${message.role}>${message.content.map(content => content.text).join(' ')}`; if (message.loop_summary) { history += `<loop_summary>${message.loop_summary}</loop_summary>`; @@ -687,20 +968,21 @@ export class ChatBox extends ViewBoxAnnotatableComponent<FieldViewProps>() { */ @action handleFollowUpClick = (question: string) => { - this.inputValue = question; + this._inputValue = question; }; + _dictation: DictationButton | null = null; /** * Renders the chat interface, including the message list, input field, and other UI elements. */ render() { return ( <div className="chat-box"> - {this.isUploadingDocs && ( + {this._isUploadingDocs && ( <div className="uploading-overlay"> <div className="progress-container"> <ProgressBar /> - <div className="step-name">{this.currentStep}</div> + <div className="step-name">{this._currentStep}</div> </div> </div> )} @@ -708,24 +990,29 @@ export class ChatBox extends ViewBoxAnnotatableComponent<FieldViewProps>() { <h2>{this.userName()}'s AI Assistant</h2> </div> <div className="chat-messages" ref={this.messagesRef}> - {this.history.map((message, index) => ( - <MessageComponentBox key={index} message={message} index={index} onFollowUpClick={this.handleFollowUpClick} onCitationClick={this.handleCitationClick} updateMessageCitations={this.updateMessageCitations} /> + {this._history.map((message, index) => ( + <MessageComponentBox key={index} message={message} onFollowUpClick={this.handleFollowUpClick} onCitationClick={this.handleCitationClick} updateMessageCitations={this.updateMessageCitations} /> ))} - {this.current_message && ( - <MessageComponentBox - key={this.history.length} - message={this.current_message} - index={this.history.length} - onFollowUpClick={this.handleFollowUpClick} - onCitationClick={this.handleCitationClick} - updateMessageCitations={this.updateMessageCitations} - /> + {this._current_message && ( + <MessageComponentBox key={this._history.length} message={this._current_message} onFollowUpClick={this.handleFollowUpClick} onCitationClick={this.handleCitationClick} updateMessageCitations={this.updateMessageCitations} /> )} </div> + <form onSubmit={this.askGPT} className="chat-input"> - <input type="text" name="messageInput" autoComplete="off" placeholder="Type your message here..." value={this.inputValue} onChange={e => (this.inputValue = e.target.value)} /> - <button className="submit-button" type="submit" disabled={this.isLoading}> - {this.isLoading ? ( + <input + ref={r => { + this._textInputRef = r; + }} + type="text" + name="messageInput" + autoComplete="off" + placeholder="Type your message here..." + value={this._inputValue} + onChange={action(e => (this._inputValue = e.target.value))} + disabled={this._isLoading} + /> + <button className="submit-button" onClick={() => this._dictation?.stopDictation()} type="submit" disabled={this._isLoading || !this._inputValue.trim()}> + {this._isLoading ? ( <div className="spinner"></div> ) : ( <svg viewBox="0 0 24 24" width="24" height="24" stroke="currentColor" strokeWidth="2" fill="none" strokeLinecap="round" strokeLinejoin="round"> @@ -734,12 +1021,19 @@ export class ChatBox extends ViewBoxAnnotatableComponent<FieldViewProps>() { </svg> )} </button> + <DictationButton + ref={r => { + this._dictation = r; + }} + setInput={this.setChatInput} + inputRef={this._textInputRef} + /> </form> {/* Popup for citation */} - {this.citationPopup.visible && ( + {this._citationPopup.visible && ( <div className="citation-popup"> <p> - <strong>Text from your document: </strong> {this.citationPopup.text} + <strong>Text from your document: </strong> {this._citationPopup.text} </p> </div> )} @@ -753,5 +1047,5 @@ export class ChatBox extends ViewBoxAnnotatableComponent<FieldViewProps>() { */ Docs.Prototypes.TemplateMap.set(DocumentType.CHAT, { layout: { view: ChatBox, dataField: 'data' }, - options: { acl: '', chat: '', chat_history: '', chat_thread_id: '', chat_assistant_id: '', chat_vector_store_id: '' }, + options: { acl: '', _layout_fitWidth: true, chat: '', chat_history: '', chat_thread_id: '', chat_assistant_id: '', chat_vector_store_id: '' }, }); |