diff options
Diffstat (limited to 'src/client/views/nodes')
8 files changed, 342 insertions, 127 deletions
diff --git a/src/client/views/nodes/ImageBox.tsx b/src/client/views/nodes/ImageBox.tsx index 78bacdcac..fb2346bd1 100644 --- a/src/client/views/nodes/ImageBox.tsx +++ b/src/client/views/nodes/ImageBox.tsx @@ -49,7 +49,6 @@ import { gptImageLabel } from '../../apis/gpt/GPT'; const DefaultPath = '/assets/unknown-file-icon-hi.png'; export class ImageEditorData { - // eslint-disable-next-line no-use-before-define private static _instance: ImageEditorData; private static get imageData() { return (ImageEditorData._instance ?? new ImageEditorData()).imageData; } // prettier-ignore @observable imageData: { rootDoc: Doc | undefined; open: boolean; source: string; addDoc: Opt<(doc: Doc | Doc[], annotationKey?: string) => boolean> } = observable({ rootDoc: undefined, open: false, source: '', addDoc: undefined }); diff --git a/src/client/views/nodes/chatbot/agentsystem/Agent.ts b/src/client/views/nodes/chatbot/agentsystem/Agent.ts index d1248d098..3acdc6aa8 100644 --- a/src/client/views/nodes/chatbot/agentsystem/Agent.ts +++ b/src/client/views/nodes/chatbot/agentsystem/Agent.ts @@ -7,17 +7,14 @@ import { AnswerParser } from '../response_parsers/AnswerParser'; import { StreamedAnswerParser } from '../response_parsers/StreamedAnswerParser'; import { BaseTool } from '../tools/BaseTool'; import { CalculateTool } from '../tools/CalculateTool'; -//import { CreateAnyDocumentTool } from '../tools/CreateAnyDocTool'; import { DataAnalysisTool } from '../tools/DataAnalysisTool'; import { DocumentMetadataTool } from '../tools/DocumentMetadataTool'; -import { ImageCreationTool } from '../tools/ImageCreationTool'; import { NoTool } from '../tools/NoTool'; import { SearchTool } from '../tools/SearchTool'; import { Parameter, ParametersType, TypeMap } from '../types/tool_types'; import { AgentMessage, ASSISTANT_ROLE, AssistantMessage, Observation, PROCESSING_TYPE, ProcessingInfo, TEXT_TYPE } from '../types/types'; import { Vectorstore } from '../vectorstore/Vectorstore'; import { getReactPrompt } from './prompts'; -//import { DictionaryTool } from '../tools/DictionaryTool'; import { ChatCompletionMessageParam } from 'openai/resources'; import { Doc } from '../../../../../fields/Doc'; import { ChatBox, parsedDoc } from '../chatboxcomponents/ChatBox'; @@ -32,7 +29,7 @@ import { FileNamesTool } from '../tools/FileNamesTool'; import { CreateNewTool } from '../tools/CreateNewTool'; import { SortDocsTool} from '../tools/SortDocsTool'; import { TagDocsTool } from '../tools/TagDocsTool'; -//import { CreateTextDocTool } from '../tools/CreateTextDocumentTool'; +import { GPTTutorialTool } from '../tools/TutorialTool'; dotenv.config(); @@ -55,6 +52,7 @@ export class Agent { private streamedAnswerParser: StreamedAnswerParser = new StreamedAnswerParser(); private tools: Record<string, BaseTool<ReadonlyArray<Parameter>>>; private _docManager: AgentDocumentManager; + private is_dash_doc_assistant: boolean; // Dynamic tool registry for tools created at runtime private dynamicToolRegistry: Map<string, BaseTool<ReadonlyArray<Parameter>>> = new Map(); // Callback for notifying when tools are created and need reload @@ -79,7 +77,8 @@ export class Agent { csvData: () => { filename: string; id: string; text: string }[], createImage: (result: Upload.FileInformation & Upload.InspectionResults, options: DocumentOptions) => void, createCSVInDash: (url: string, title: string, id: string, data: string) => void, - docManager: AgentDocumentManager + docManager: AgentDocumentManager, + isDashDocAssistant: boolean ) { // Initialize OpenAI client with API key from environment this.client = new OpenAI({ apiKey: process.env.OPENAI_KEY, dangerouslyAllowBrowser: true }); @@ -87,6 +86,7 @@ export class Agent { this._history = history; this._csvData = csvData; this._docManager = docManager; + this.is_dash_doc_assistant = isDashDocAssistant; // Initialize dynamic tool registry this.dynamicToolRegistry = new Map(); @@ -105,6 +105,7 @@ export class Agent { codebaseSummarySearch: new CodebaseSummarySearchTool(this.vectorstore), fileContent: new FileContentTool(this.vectorstore), fileNames: new FileNamesTool(this.vectorstore), + generateTutorialNode: new GPTTutorialTool(this._docManager), sortDocs: new SortDocsTool(this._docManager), tagDocs: new TagDocsTool(this._docManager), }; @@ -146,7 +147,7 @@ export class Agent { const instance: BaseTool<ReadonlyArray<Parameter>> = new ToolClass(); - // Prefer the tool’s self-declared name (matches <action> tag) + // Prefer the tool's self-declared name (matches <action> tag) const key = (instance.name || '').trim() || legacyKey; // Check for duplicates @@ -301,7 +302,7 @@ export class Agent { ignoreAttributes: false, attributeNamePrefix: '@_', textNodeName: '_text', - isArray: name => ['query', 'url'].indexOf(name) !== -1, + isArray: name => name === 'url', processEntities: false, // Disable processing of entities stopNodes: ['*.entity'], // Do not process any entities }); @@ -763,7 +764,7 @@ export class Agent { const docSummaries = () => JSON.stringify(this._docManager.listDocs); const chatHistory = this._history(); - return getReactPrompt(allTools, docSummaries, chatHistory); + return getReactPrompt(allTools, docSummaries, chatHistory, this.is_dash_doc_assistant); } /** diff --git a/src/client/views/nodes/chatbot/agentsystem/prompts.ts b/src/client/views/nodes/chatbot/agentsystem/prompts.ts index fcb4ab450..b7678bd08 100644 --- a/src/client/views/nodes/chatbot/agentsystem/prompts.ts +++ b/src/client/views/nodes/chatbot/agentsystem/prompts.ts @@ -10,7 +10,7 @@ import { BaseTool } from '../tools/BaseTool'; import { Parameter } from '../types/tool_types'; -export function getReactPrompt(tools: BaseTool<ReadonlyArray<Parameter>>[], summaries: () => string, chatHistory: string): string { +export function getReactPrompt(tools: BaseTool<ReadonlyArray<Parameter>>[], summaries: () => string, chatHistory: string, isDashDocAssistant?: boolean): string { const toolDescriptions = tools .map( tool => ` @@ -21,11 +21,21 @@ export function getReactPrompt(tools: BaseTool<ReadonlyArray<Parameter>>[], summ ) .join('\n'); + const dashDocContext = isDashDocAssistant + ? ` + <dash_doc_assistant_context> + <point>You are acting as a help assistant for a software application called Dash.</point> + <point>All user queries, unless otherwise specified, should be interpreted as questions about how to use Dash or about Dash's functionality.</point> + <point>You should prioritize using the 'generateTutorialNode' tool to answer user questions about Dash.</point> + </dash_doc_assistant_context> + ` + : ''; + return `<system_message> <task> You are an advanced AI assistant equipped with tools to answer user queries efficiently. You operate in a loop that is RIGIDLY structured and requires the use of specific tags and formats for your responses. Your goal is to provide accurate and well-structured answers to user queries. Below are the guidelines and information you can use to structure your approach to accomplishing this task. </task> - + ${dashDocContext} <critical_points> <point>**STRUCTURE**: Always use the correct stage tags (e.g., <stage number="2" role="assistant">) for every response. Use only even-numbered assisntant stages for your responses.</point> <point>**STOP after every stage and wait for input. Do not combine multiple stages in one response.**</point> @@ -189,7 +199,7 @@ export function getReactPrompt(tools: BaseTool<ReadonlyArray<Parameter>>[], summ <action_input> <action_input_description>Getting information from the relevant websites about Qatar's tourism impact during the World Cup.</action_input_description> <inputs> - <urls>[***URLS to search elided, but they will be comma seperated double quoted strings"]</urls> + <chunk_ids>[***CHUNK IDS to search elided, but they will be comma separated double quoted strings"]</chunk_ids> </inputs> </action_input> </stage> diff --git a/src/client/views/nodes/chatbot/chatboxcomponents/ChatBox.scss b/src/client/views/nodes/chatbot/chatboxcomponents/ChatBox.scss index 4a916e86c..0bacc70c2 100644 --- a/src/client/views/nodes/chatbot/chatboxcomponents/ChatBox.scss +++ b/src/client/views/nodes/chatbot/chatboxcomponents/ChatBox.scss @@ -183,7 +183,6 @@ $font-size-xlarge: 18px; box-shadow: 0 -4px 20px rgba(0, 0, 0, 0.05); position: relative; align-items: center; - gap: 12px; z-index: 5; transition: padding 0.2s ease; diff --git a/src/client/views/nodes/chatbot/chatboxcomponents/ChatBox.tsx b/src/client/views/nodes/chatbot/chatboxcomponents/ChatBox.tsx index 6e6ef6212..636b77b38 100644 --- a/src/client/views/nodes/chatbot/chatboxcomponents/ChatBox.tsx +++ b/src/client/views/nodes/chatbot/chatboxcomponents/ChatBox.tsx @@ -7,11 +7,12 @@ * with support for follow-up questions and citation management. */ -import dotenv from 'dotenv'; +import { Button, Size, Type } from '@dash/components'; import { ObservableSet, action, computed, makeObservable, observable, observe, reaction, runInAction } from 'mobx'; import { observer } from 'mobx-react'; import OpenAI, { ClientOptions } from 'openai'; import * as React from 'react'; +import { AiOutlineSend } from 'react-icons/ai'; import { v4 as uuidv4 } from 'uuid'; import { ClientUtils, OmitKeys } from '../../../../../ClientUtils'; import { Doc, DocListCast, Opt } from '../../../../../fields/Doc'; @@ -19,34 +20,33 @@ import { DocData, DocViews } from '../../../../../fields/DocSymbols'; import { Id } from '../../../../../fields/FieldSymbols'; import { RichTextField } from '../../../../../fields/RichTextField'; import { ScriptField } from '../../../../../fields/ScriptField'; -import { CsvCast, DocCast, NumCast, PDFCast, RTFCast, StrCast, VideoCast, AudioCast } from '../../../../../fields/Types'; +import { AudioCast, CsvCast, DocCast, NumCast, PDFCast, RTFCast, StrCast, VideoCast } from '../../../../../fields/Types'; +import { Upload } from '../../../../../server/SharedMediaTypes'; +import { DocServer } from '../../../../DocServer'; import { DocUtils } from '../../../../documents/DocUtils'; import { CollectionViewType, DocumentType } from '../../../../documents/DocumentTypes'; import { Docs, DocumentOptions } from '../../../../documents/Documents'; -import { DocServer } from '../../../../DocServer'; import { DocumentManager } from '../../../../util/DocumentManager'; import { ImageUtils } from '../../../../util/Import & Export/ImageUtils'; import { LinkManager } from '../../../../util/LinkManager'; import { CompileError, CompileScript } from '../../../../util/Scripting'; +import { SnappingManager } from '../../../../util/SnappingManager'; import { DictationButton } from '../../../DictationButton'; import { ViewBoxAnnotatableComponent } from '../../../DocComponent'; import { AudioBox } from '../../AudioBox'; import { DocumentView, DocumentViewInternal } from '../../DocumentView'; import { FieldView, FieldViewProps } from '../../FieldView'; +import { OpenWhere } from '../../OpenWhere'; import { PDFBox } from '../../PDFBox'; import { ScriptingBox } from '../../ScriptingBox'; import { VideoBox } from '../../VideoBox'; import { Agent } from '../agentsystem/Agent'; import { supportedDocTypes } from '../types/tool_types'; import { ASSISTANT_ROLE, AssistantMessage, CHUNK_TYPE, Citation, ProcessingInfo, SimplifiedChunk, TEXT_TYPE } from '../types/types'; +import { AgentDocumentManager } from '../utils/AgentDocumentManager'; import { Vectorstore } from '../vectorstore/Vectorstore'; import './ChatBox.scss'; import MessageComponentBox from './MessageComponent'; -import { OpenWhere } from '../../OpenWhere'; -import { Upload } from '../../../../../server/SharedMediaTypes'; -import { AgentDocumentManager } from '../utils/AgentDocumentManager'; - -dotenv.config(); export type parsedDocData = { doc_type: string; @@ -57,6 +57,7 @@ export type parsedDocData = { data_useCors?: boolean; }; export type parsedDoc = DocumentOptions & parsedDocData; + /** * ChatBox is the main class responsible for managing the interaction between the user and the assistant, * handling documents, and integrating with OpenAI for tasks such as document analysis, chat functionality, @@ -123,7 +124,15 @@ export class ChatBox extends ViewBoxAnnotatableComponent<FieldViewProps>() { this.vectorstore = new Vectorstore(this.vectorstore_id, this.docManager); // Create an agent with the vectorstore - this.agent = new Agent(this.vectorstore, this.retrieveFormattedHistory.bind(this), this.retrieveCSVData.bind(this), this.createImageInDash.bind(this), this.createCSVInDash.bind(this), this.docManager); + this.agent = new Agent( + this.vectorstore, + this.retrieveFormattedHistory.bind(this), + this.retrieveCSVData.bind(this), + this.createImageInDash.bind(this), + this.createCSVInDash.bind(this), + this.docManager, + this.dataDoc.is_dash_doc_assistant === 'true' + ); // Set up the tool created callback this.agent.setToolCreatedCallback(this.handleToolCreated); @@ -373,7 +382,6 @@ export class ChatBox extends ViewBoxAnnotatableComponent<FieldViewProps>() { }; } }); - this.scrollToBottom(); }; const onAnswerUpdate = (answerUpdate: string) => { @@ -381,41 +389,33 @@ export class ChatBox extends ViewBoxAnnotatableComponent<FieldViewProps>() { if (this._current_message) { this._current_message = { ...this._current_message, - content: [{ text: answerUpdate, type: TEXT_TYPE.NORMAL, index: 0, citation_ids: [] }], + content: [{ index: 0, type: TEXT_TYPE.NORMAL, text: answerUpdate, citation_ids: null }], }; } }); }; - // Send the user's question to the assistant and get the final message - const finalMessage = await this.agent.askAgent(trimmedText, onProcessingUpdate, onAnswerUpdate); + // Get the response from the agent + let userQuery = trimmedText; + if (this.dataDoc.is_dash_doc_assistant) { + userQuery = `The user is asking a question about Dash functionality. Their question is: "${trimmedText}". You should use the generateTutorialNode tool to answer this question.`; + } + const response = await this.agent.askAgent(userQuery, onProcessingUpdate, onAnswerUpdate); - // Update the history with the final assistant message + // Push the final message to history runInAction(() => { - if (this._current_message) { - this._history.push({ ...finalMessage }); - this._current_message = undefined; - this.dataDoc.data = JSON.stringify(this._history); - } + this._history.push(response); + this._isLoading = false; + this._current_message = undefined; }); - } catch (err) { - console.error('Error:', err); - // Handle error in processing - runInAction(() => - this._history.push({ - role: ASSISTANT_ROLE.ASSISTANT, - content: [{ index: 0, type: TEXT_TYPE.ERROR, text: `Sorry, I encountered an error while processing your request: ${err} `, citation_ids: null }], - processing_info: [], - }) - ); - } finally { + } catch (error) { + console.error('Error in askGPT:', error); runInAction(() => { this._isLoading = false; + this._current_message = undefined; }); - this.scrollToBottom(); } } - this.scrollToBottom(); }; /** @@ -488,7 +488,7 @@ export class ChatBox extends ViewBoxAnnotatableComponent<FieldViewProps>() { const data = (doc as parsedDocData).data; const ndoc = (() => { switch (doc.doc_type) { - default: + default: case supportedDocTypes.note: return Docs.Create.TextDocument(data as string, options); case supportedDocTypes.comparison: return this.createComparison(JSON.parse(data as string) as parsedDoc[], options); case supportedDocTypes.flashcard: return this.createFlashcard(JSON.parse(data as string) as parsedDoc[], options); @@ -499,23 +499,23 @@ export class ChatBox extends ViewBoxAnnotatableComponent<FieldViewProps>() { case supportedDocTypes.web: { // Create web document with enhanced safety options const webOptions = { - ...options, + ...options, data_useCors: true }; - + // If iframe_sandbox was passed from AgentDocumentManager, add it to the options if ('_iframe_sandbox' in options) { webOptions._iframe_sandbox = options._iframe_sandbox; } - + return Docs.Create.WebDocument(data as string, webOptions); } case supportedDocTypes.dataviz: return Docs.Create.DataVizDocument('/users/rz/Downloads/addresses.csv', options); case supportedDocTypes.pdf: return Docs.Create.PdfDocument(data as string, options); case supportedDocTypes.video: return Docs.Create.VideoDocument(data as string, options); case supportedDocTypes.diagram: return Docs.Create.DiagramDocument(undefined, { text: data as unknown as RichTextField, ...options}); // text: can take a string or RichTextField but it's typed for RichTextField. - - // case supportedDocumentTypes.dataviz: + + // case supportedDocumentTypes.dataviz: // { // const { fileUrl, id } = await Networking.PostToServer('/createCSV', { // filename: (options.title as string).replace(/\s+/g, '') + '.csv', @@ -540,7 +540,7 @@ export class ChatBox extends ViewBoxAnnotatableComponent<FieldViewProps>() { const arr = this.createCollectionWithChildren(JSON.parse(data as string) as parsedDoc[], true).filter(d=>d).map(d => d!); const collOpts = { _width:300, _height: 300, _layout_fitWidth: true, _freeform_backgroundGrid: true, ...options, }; return (() => { - switch (options.type_collection) { + switch (options.type_collection) { case CollectionViewType.Tree: return Docs.Create.TreeDocument(arr, collOpts); case CollectionViewType.Stacking: return Docs.Create.StackingDocument(arr, collOpts); case CollectionViewType.Masonry: return Docs.Create.MasonryDocument(arr, collOpts); @@ -549,9 +549,9 @@ export class ChatBox extends ViewBoxAnnotatableComponent<FieldViewProps>() { case CollectionViewType.Carousel3D: return Docs.Create.Carousel3DDocument(arr, collOpts); case CollectionViewType.Multicolumn: return Docs.Create.CarouselDocument(arr, collOpts); default: return Docs.Create.FreeformDocument(arr, collOpts); - } - })(); - } + } + })(); + } // case supportedDocumentTypes.map: return Docs.Create.MapDocument([], options); // case supportedDocumentTypes.button: return Docs.Create.ButtonDocument(options); // case supportedDocumentTypes.trail: return Docs.Create.PresDocument(options); @@ -559,8 +559,8 @@ export class ChatBox extends ViewBoxAnnotatableComponent<FieldViewProps>() { })(); if (ndoc) { - ndoc.x = NumCast((options.x as number) ?? 0) + (insideCol ? 0 : NumCast(this.layoutDoc.x) + NumCast(this.layoutDoc.width)) + 100; - ndoc.y = NumCast(options.y as number) + (insideCol ? 0 : NumCast(this.layoutDoc.y)); + ndoc.x = ((options.x as number) ?? 0) + (insideCol ? 0 : NumCast(this.layoutDoc.x) + NumCast(this.layoutDoc.width)) + 100; + ndoc.y = ((options.y as number) ?? 0) + (insideCol ? 0 : NumCast(this.layoutDoc.y)); } return ndoc; }; @@ -714,7 +714,7 @@ export class ChatBox extends ViewBoxAnnotatableComponent<FieldViewProps>() { // If specific indexes are provided, filter segments by those indexes if (indexesOfSegments && indexesOfSegments.length > 0) { - segments = original_segments.filter((segment: any) => indexesOfSegments.includes(segment.index)); + segments = original_segments.filter(segment => indexesOfSegments.includes(segment.index)); } // If no segments match the indexes, use all segments @@ -723,7 +723,7 @@ export class ChatBox extends ViewBoxAnnotatableComponent<FieldViewProps>() { } // First try to find an exact match - const exactMatch = segments.find((segment: any) => segment.text && segment.text.includes(citationText)); + const exactMatch = segments.find(segment => segment.text && segment.text.includes(citationText)); if (exactMatch) { return exactMatch.start; @@ -832,7 +832,8 @@ export class ChatBox extends ViewBoxAnnotatableComponent<FieldViewProps>() { existingDoc._width = x2 - x1; existingDoc._height = y2 - y1; } - const highlightDoc = existingDoc ?? this.createImageCitationHighlight(x1, y1, x2, y2, citation, annotationKey, doc); + // const highlightDoc = + existingDoc ?? this.createImageCitationHighlight(x1, y1, x2, y2, citation, annotationKey, doc); //doc.layout_scroll = y1; doc._layout_curPage = foundChunk.startPage + 1; @@ -1068,7 +1069,9 @@ export class ChatBox extends ViewBoxAnnotatableComponent<FieldViewProps>() { { index: 0, type: TEXT_TYPE.NORMAL, - text: `Hey, ${this.userName()}! Welcome to Your Friendly Assistant. Link a document or ask questions to get started.`, + text: this.dataDoc.is_dash_doc_assistant + ? 'Welcome to your help assistant for Dash. Ask any Dash-related questions to get started.' + : `Hey, ${this.userName()}! Welcome to Your Friendly Assistant. Link a document or ask questions to get started.`, citation_ids: null, }, ], @@ -1409,7 +1412,7 @@ export class ChatBox extends ViewBoxAnnotatableComponent<FieldViewProps>() { </div> )} <div className="chat-header"> - <h2>{this.userName()}'s AI Assistant</h2> + <h2>{StrCast(this.dataDoc.title) || `${this.userName()}'s AI Assistant`}</h2> <div className="font-size-control" onClick={this.toggleFontSizeModal}> {this.renderFontSizeIcon()} </div> @@ -1456,16 +1459,16 @@ export class ChatBox extends ViewBoxAnnotatableComponent<FieldViewProps>() { disabled={this._isLoading} /> </div> - <button className="submit-button" onClick={() => this._dictation?.stopDictation()} type="submit" disabled={this._isLoading || !this._inputValue.trim()}> - {this._isLoading ? ( - <div className="spinner"></div> - ) : ( - <svg viewBox="0 0 24 24" width="24" height="24" stroke="currentColor" strokeWidth="2" fill="none" strokeLinecap="round" strokeLinejoin="round"> - <line x1="22" y1="2" x2="11" y2="13"></line> - <polygon points="22 2 15 22 11 13 2 9 22 2"></polygon> - </svg> - )} - </button> + <Button + // className="submit-button" + onClick={() => this._dictation?.stopDictation()} + type={Type.PRIM} + tooltip="Send to AI" + color={SnappingManager.userVariantColor} + inactive={this._isLoading || !this._inputValue.trim()} + icon={<AiOutlineSend />} + size={Size.LARGE} + /> <DictationButton ref={this.setDictationRef} setInput={this.setChatInput} inputRef={this._textInputRef} /> </form> {/* Popup for citation */} diff --git a/src/client/views/nodes/chatbot/tools/DocumentMetadataTool.ts b/src/client/views/nodes/chatbot/tools/DocumentMetadataTool.ts index a55f901e1..da4a4ae29 100644 --- a/src/client/views/nodes/chatbot/tools/DocumentMetadataTool.ts +++ b/src/client/views/nodes/chatbot/tools/DocumentMetadataTool.ts @@ -1,16 +1,7 @@ -import { Doc, FieldType } from '../../../../../fields/Doc'; -import { DocData } from '../../../../../fields/DocSymbols'; +import { Parameter, ParametersType, supportedDocTypes, ToolInfo } from '../types/tool_types'; import { Observation } from '../types/types'; -import { ParametersType, ToolInfo, Parameter } from '../types/tool_types'; -import { BaseTool } from './BaseTool'; -import { DocumentOptions } from '../../../../documents/Documents'; -import { CollectionFreeFormDocumentView } from '../../../nodes/CollectionFreeFormDocumentView'; -import { v4 as uuidv4 } from 'uuid'; -import { LinkManager } from '../../../../util/LinkManager'; -import { DocCast, StrCast } from '../../../../../fields/Types'; -import { supportedDocTypes } from '../types/tool_types'; -import { parsedDoc } from '../chatboxcomponents/ChatBox'; import { AgentDocumentManager } from '../utils/AgentDocumentManager'; +import { BaseTool } from './BaseTool'; // Define the parameters for the DocumentMetadataTool const parameterDefinitions: ReadonlyArray<Parameter> = [ @@ -598,7 +589,7 @@ export class DocumentMetadataTool extends BaseTool<DocumentMetadataToolParamsTyp message: string; fieldName?: string; originalFieldName?: string; - newValue?: any; + newValue?: string | number | boolean | object; warning?: string; }[] = []; diff --git a/src/client/views/nodes/chatbot/tools/TutorialTool.ts b/src/client/views/nodes/chatbot/tools/TutorialTool.ts new file mode 100644 index 000000000..1624f0439 --- /dev/null +++ b/src/client/views/nodes/chatbot/tools/TutorialTool.ts @@ -0,0 +1,212 @@ +import { BaseTool } from './BaseTool'; +import { Observation } from '../types/types'; +import { ParametersType, ToolInfo } from '../types/tool_types'; +import { schema } from '../../../../views/nodes/formattedText/schema_rts'; +import { v4 as uuidv4 } from 'uuid'; +import { gptTutorialAPICall } from '../../../../apis/gpt/TutorialGPT'; +import { parsedDoc } from '../chatboxcomponents/ChatBox'; +import { Id } from '../../../../../fields/FieldSymbols'; +import { Doc } from '../../../../../fields/Doc'; +import { RichTextField } from '../../../../../fields/RichTextField'; +import { DocumentViewInternal } from '../../DocumentView'; +import { Docs } from '../../../../documents/Documents'; +import { OpenWhere } from '../../OpenWhere'; +import { CollectionFreeFormView } from '../../../collections/collectionFreeForm/CollectionFreeFormView'; +import { AgentDocumentManager } from '../utils/AgentDocumentManager'; +import { Node as ProseMirrorNode } from 'prosemirror-model'; + +const generateTutorialNodeToolParams = [ + { + name: 'query', + type: 'string', + description: 'The user query that asks how to use the environment', + required: true, + }, +] as const; + +const generateTutorialNodeToolInfo: ToolInfo<typeof generateTutorialNodeToolParams> = { + name: 'generateTutorialNode', + description: "Generates a tutorial text node based on the user's query about Dash functionality. Use this when the user asks for help or tutorials on how to use Dash features.", + parameterRules: generateTutorialNodeToolParams, + citationRules: "No citation needed for this tool's output.", +}; + +interface FormattedDocument { + doc: ProseMirrorNode; + plainText: string; +} + +const applyFormatting = (markdownText: string): FormattedDocument => { + const lines = markdownText.split('\n'); + const nodes: ProseMirrorNode[] = []; + let plainText = ''; + let i = 0; + let currentListItems: ProseMirrorNode[] = []; + let currentParagraph: ProseMirrorNode[] = []; + let currentOrderedListItems: ProseMirrorNode[] = []; + let inOrderedList = false; + let inBulletList = false; + + const processBoldText = (text: string): ProseMirrorNode[] => { + const boldRegex = /\*\*(.*?)\*\*/g; + const parts: ProseMirrorNode[] = []; + let lastIndex = 0; + let match; + + while ((match = boldRegex.exec(text)) !== null) { + if (match.index > lastIndex) { + parts.push(schema.text(text.substring(lastIndex, match.index))); + } + parts.push(schema.text(match[1], [schema.marks.strong.create()])); + lastIndex = match.index + match[0].length; + } + if (lastIndex < text.length) { + parts.push(schema.text(text.substring(lastIndex))); + } + return parts.length > 0 ? parts : [schema.text(text)]; + }; + + const flushListItems = (): void => { + if (currentListItems.length > 0) { + nodes.push(schema.nodes.ordered_list.create({ mapStyle: 'bullet' }, currentListItems)); + nodes.push(schema.nodes.paragraph.create()); + currentListItems = []; + inBulletList = false; + } + if (currentOrderedListItems.length > 0) { + nodes.push(schema.nodes.ordered_list.create({ mapStyle: 'number' }, currentOrderedListItems)); + nodes.push(schema.nodes.paragraph.create()); + currentOrderedListItems = []; + inOrderedList = false; + } + }; + + const flushParagraph = (): void => { + if (currentParagraph.length > 0) { + nodes.push(schema.nodes.paragraph.create({}, currentParagraph)); + currentParagraph = []; + } + }; + + const processHeader = (line: string): boolean => { + const headerMatch = line.match(/^(#{1,6})\s+(.+)$/); + if (headerMatch) { + const level = Math.min(headerMatch[1].length, 6); // Cap at h6 + const textContent = headerMatch[2]; + flushParagraph(); + nodes.push(schema.nodes.heading.create({ level }, processBoldText(textContent))); + plainText += textContent + '\n'; + return true; + } + return false; + }; + + while (i < lines.length) { + const line = lines[i].trim(); + if (line) { + if (processHeader(line)) { + flushListItems(); + flushParagraph(); + } else if (line.startsWith('- ')) { + flushParagraph(); + if (!inBulletList) { + flushListItems(); + inBulletList = true; + } + const textContent = line.replace('- ', ''); + currentListItems.push(schema.nodes.list_item.create({}, schema.nodes.paragraph.create({}, processBoldText(textContent)))); + plainText += textContent + '\n'; + } else if (/^\d+\.\s+/.test(line)) { + flushParagraph(); + if (!inOrderedList) { + flushListItems(); + inOrderedList = true; + } + const textContent = line.replace(/^\d+\.\s+/, ''); + currentOrderedListItems.push(schema.nodes.list_item.create({}, schema.nodes.paragraph.create({}, processBoldText(textContent)))); + plainText += textContent + '\n'; + } else { + flushListItems(); + currentParagraph = currentParagraph.concat(processBoldText(line)); + plainText += line + '\n'; + } + } else { + flushListItems(); + flushParagraph(); + nodes.push(schema.nodes.paragraph.create()); + plainText += '\n'; + } + i++; + } + flushListItems(); + flushParagraph(); + + const doc = schema.nodes.doc.create({}, nodes); + return { doc, plainText: plainText.trim() }; +}; + +export class GPTTutorialTool extends BaseTool<typeof generateTutorialNodeToolParams> { + private _docManager: AgentDocumentManager; + + constructor(docManager: AgentDocumentManager) { + super(generateTutorialNodeToolInfo); + this._docManager = docManager; + } + + async execute(args: ParametersType<typeof generateTutorialNodeToolParams>): Promise<Observation[]> { + const chunkId = uuidv4(); + try { + const query = (args.query || '').trim(); + if (!query) { + return [{ type: 'text', text: `<chunk chunk_id="${chunkId}" chunk_type="error">Please provide a query.</chunk>` }]; + } + const markdown = await gptTutorialAPICall(query); + const { doc, plainText } = applyFormatting(markdown); + + // Build the ProseMirror‐in‐JSON + plain-text for RichTextField + const rtfData = { + doc: doc.toJSON ? doc.toJSON() : doc, + selection: { type: 'text', anchor: 0, head: 0 }, + storedMarks: [], + }; + const rtf = new RichTextField(JSON.stringify(rtfData), plainText); + + // Create and show the TextDocument directly: + const formattedDoc = Docs.Create.TextDocument(rtf, { + title: 'Tutorial Node', + _width: 600, + _layout_fitWidth: true, + _layout_autoHeight: true, + text_fontSize: '16px', + }); + DocumentViewInternal.addDocTabFunc(formattedDoc, OpenWhere.addRight); + + // If user asked about linking/pinning/presentation, also fire the in-app tutorial: + const q = query.toLowerCase(); + if (q.includes('link')) { + Doc.IsInfoUIDisabled = false; + CollectionFreeFormView.showTutorial('links'); + } else if (q.includes('presentation')) { + Doc.IsInfoUIDisabled = false; + CollectionFreeFormView.showTutorial('presentation'); + } else if (q.includes('pin')) { + Doc.IsInfoUIDisabled = false; + CollectionFreeFormView.showTutorial('pins'); + } + + return [ + { + type: 'text', + text: `<chunk chunk_id="${chunkId}" chunk_type="tutorial_node_creation">Created tutorial node with ID ${formattedDoc[Id]}.</chunk>`, + }, + ]; + } catch (error) { + return [ + { + type: 'text', + text: `<chunk chunk_id="${chunkId}" chunk_type="error">Error generating tutorial node: ${error}</chunk>`, + }, + ]; + } + } +} diff --git a/src/client/views/nodes/chatbot/utils/AgentDocumentManager.ts b/src/client/views/nodes/chatbot/utils/AgentDocumentManager.ts index 3c8b49f33..dcb708450 100644 --- a/src/client/views/nodes/chatbot/utils/AgentDocumentManager.ts +++ b/src/client/views/nodes/chatbot/utils/AgentDocumentManager.ts @@ -1,7 +1,5 @@ import { action, computed, makeObservable, observable, ObservableMap, reaction, runInAction } from 'mobx'; -import { observer } from 'mobx-react'; -import { v4 as uuidv4 } from 'uuid'; -import { Doc, StrListCast } from '../../../../../fields/Doc'; +import { Doc, FieldResult, StrListCast } from '../../../../../fields/Doc'; import { DocData } from '../../../../../fields/DocSymbols'; import { Id } from '../../../../../fields/FieldSymbols'; import { List } from '../../../../../fields/List'; @@ -30,7 +28,7 @@ export class AgentDocumentManager { @observable private documentsById: ObservableMap<string, AgentDocument>; private chatBox: ChatBox; private chatBoxDocument: Doc | null = null; - private fieldMetadata: Record<string, any> = {}; + private fieldMetadata: Record<string, any> = {}; // bcz: CHANGE any to a proper type! @observable private simplifiedChunks: ObservableMap<string, SimplifiedChunk>; /** @@ -103,6 +101,7 @@ export class AgentDocumentManager { for (const [fieldName, fieldInfo] of documentOptionsEntries) { // Extract field information const fieldData: Record<string, any> = { + // bcz: CHANGE any to a proper type! name: fieldName, withoutUnderscore: fieldName.startsWith('_') ? fieldName.substring(1) : fieldName, description: '', @@ -223,6 +222,7 @@ export class AgentDocumentManager { const dataDoc = agentDoc.dataDoc; const metadata: Record<string, any> = { + // bcz: CHANGE any to a proper type! id: layoutDoc[Id] || dataDoc[Id] || '', title: layoutDoc.title || '', type: layoutDoc.type || '', @@ -235,7 +235,7 @@ export class AgentDocumentManager { // Process all known field definitions Object.keys(this.fieldMetadata).forEach(fieldName => { - const fieldDef = this.fieldMetadata[fieldName]; + // const fieldDef = this.fieldMetadata[fieldName]; const strippedName = fieldName.startsWith('_') ? fieldName.substring(1) : fieldName; // Check if field exists on layout document @@ -307,7 +307,7 @@ export class AgentDocumentManager { * @param value The field value to format * @returns A JSON-friendly representation of the field value */ - private formatFieldValue(value: any): any { + private formatFieldValue(value: FieldResult | undefined) { if (value === undefined || value === null) { return null; } @@ -330,12 +330,12 @@ export class AgentDocumentManager { if (rtfObj.doc && rtfObj.doc.content) { // Recursively extract text from the content let plainText = ''; - const extractText = (node: any) => { + const extractText = (node: { text: string; content?: unknown[] }) => { if (node.text) { plainText += node.text; } if (node.content && Array.isArray(node.content)) { - node.content.forEach((child: any) => extractText(child)); + node.content.forEach(child => extractText(child as { text: string; content?: unknown[] })); } }; @@ -351,7 +351,7 @@ export class AgentDocumentManager { }; } } - } catch (e) { + } catch { // If parsing fails, just treat as a regular string } } @@ -366,7 +366,7 @@ export class AgentDocumentManager { try { // Try to convert to JSON string return JSON.stringify(value); - } catch (e) { + } catch { return '[Complex Object]'; } } @@ -381,26 +381,24 @@ export class AgentDocumentManager { * @param fieldValue The string value to convert * @returns The converted value with the appropriate type */ - private convertFieldValue(fieldName: string, fieldValue: any): any { + private convertFieldValue(fieldName: string, fieldValueIn: string | number | boolean): FieldResult | undefined { // If fieldValue is already a number or boolean, we don't need to convert it from string - if (typeof fieldValue === 'number' || typeof fieldValue === 'boolean') { - return fieldValue; + if (typeof fieldValueIn === 'number' || typeof fieldValueIn === 'boolean') { + return fieldValueIn; } // If fieldValue is a string "true" or "false", convert to boolean - if (typeof fieldValue === 'string') { - if (fieldValue.toLowerCase() === 'true') { + if (typeof fieldValueIn === 'string') { + if (fieldValueIn.toLowerCase() === 'true') { return true; } - if (fieldValue.toLowerCase() === 'false') { + if (fieldValueIn.toLowerCase() === 'false') { return false; } } - // If fieldValue is not a string (and not a number or boolean), convert it to string - if (typeof fieldValue !== 'string') { - fieldValue = String(fieldValue); - } + // coerce fieldvValue to a string + const fieldValue = typeof fieldValueIn !== 'string' ? String(fieldValueIn) : fieldValueIn; // Special handling for text field - convert to proper RichTextField format if (fieldName === 'text') { @@ -408,7 +406,7 @@ export class AgentDocumentManager { // Check if it's already a valid JSON RichTextField JSON.parse(fieldValue); return fieldValue; - } catch (e) { + } catch { // It's a plain text string, so convert it to RichTextField format const rtf = { doc: { @@ -462,21 +460,21 @@ export class AgentDocumentManager { // Try to convert to date (stored as number timestamp) try { return new Date(fieldValue).getTime(); - } catch (e) { + } catch { return fieldValue; } } else if (fieldType.includes('list') || fieldType.includes('array')) { // Try to parse as JSON array try { - return JSON.parse(fieldValue); - } catch (e) { + return JSON.parse(fieldValue) as FieldResult; // bcz: this needs to be typed properly. Dash fields can't accept a generic 'objext' + } catch { return fieldValue; } } else if (fieldType === 'json' || fieldType === 'object') { // Try to parse as JSON object try { - return JSON.parse(fieldValue); - } catch (e) { + return JSON.parse(fieldValue) as FieldResult; // bcz: this needs to be typed properly. Dash fields can't accept a generic 'objext' + } catch { return fieldValue; } } @@ -492,6 +490,7 @@ export class AgentDocumentManager { public getAllFieldMetadata() { // Start with our already populated fieldMetadata from the DocumentOptions class const result: Record<string, any> = { + // bcz: CHANGE any to a proper type! fieldCount: Object.keys(this.fieldMetadata).length, fields: {}, fieldsByType: { @@ -526,6 +525,7 @@ export class AgentDocumentManager { // Create structured field metadata const fieldData: Record<string, any> = { + // bcz: CHANGE any to a proper type! name: fieldName, displayName: strippedName, description: fieldInfo.description || '', @@ -618,12 +618,12 @@ export class AgentDocumentManager { message: string; fieldName?: string; originalFieldName?: string; - newValue?: any; + newValue?: string | number | boolean | object; warning?: string; } { // Normalize field name (handle with/without underscore) let normalizedFieldName = fieldName.startsWith('_') ? fieldName : fieldName; - const strippedFieldName = fieldName.startsWith('_') ? fieldName.substring(1) : fieldName; + // const strippedFieldName = fieldName.startsWith('_') ? fieldName.substring(1) : fieldName; // Handle common field name aliases (width → _width, height → _height) // Many document fields use '_' prefix for layout properties @@ -690,7 +690,7 @@ export class AgentDocumentManager { } // Set the field value on the target document - targetDoc[normalizedFieldName] = convertedValue; + targetDoc[normalizedFieldName] = convertedValue; // bcz: converteValue needs to be typed properly. Dash fields can't accept a generic 'objext' return { success: true, @@ -712,19 +712,19 @@ export class AgentDocumentManager { * @param documentId Optional ID of a specific document to get metadata for * @returns Document metadata or metadata for all documents */ - public getDocumentMetadata(documentId?: string): any { + public getDocumentMetadata(documentId?: string) { if (documentId) { console.log(`Returning document metadata for docID, ${documentId}:`, this.extractDocumentMetadata(documentId)); return this.extractDocumentMetadata(documentId); } else { // Get metadata for all documents - const documentsMetadata: Record<string, Record<string, any>> = {}; - for (const documentId of this.documentsById.keys()) { - const metadata = this.extractDocumentMetadata(documentId); + const documentsMetadata: Record<string, Record<string, any>> = {}; // bcz: CHANGE any to a proper type! + for (const docid of this.documentsById.keys()) { + const metadata = this.extractDocumentMetadata(docid); if (metadata) { - documentsMetadata[documentId] = metadata; + documentsMetadata[docid] = metadata; } else { - console.warn(`No metadata found for document with ID: ${documentId}`); + console.warn(`No metadata found for document with ID: ${docid}`); } } return { @@ -842,7 +842,7 @@ export class AgentDocumentManager { * @returns The ID of the created document */ - public async createDocInDash(docType: string, data: string, options?: any): Promise<string> { + public async createDocInDash(docType: string, data: string, options?: DocumentOptions): Promise<string> { // Validate doc_type if (!this.isValidDocType(docType)) { throw new Error(`Invalid document type: ${docType}`); @@ -1054,13 +1054,13 @@ export class AgentDocumentManager { endPage: chunk.metadata.end_page, location: chunk.metadata.location, } as SimplifiedChunk); - } else if (docType === 'csv') { + } else if (docType === 'csv' && 'row_start' in chunk.metadata && 'row_end' in chunk.metadata && 'col_start' in chunk.metadata && 'col_end' in chunk.metadata) { simplifiedChunks.push({ ...baseChunk, - rowStart: (chunk.metadata as any).row_start, - rowEnd: (chunk.metadata as any).row_end, - colStart: (chunk.metadata as any).col_start, - colEnd: (chunk.metadata as any).col_end, + rowStart: chunk.metadata.row_start, + rowEnd: chunk.metadata.row_end, + colStart: chunk.metadata.col_start, + colEnd: chunk.metadata.col_end, } as SimplifiedChunk); } else { // Default for other document types @@ -1077,7 +1077,7 @@ export class AgentDocumentManager { * @returns The simplified chunk if found, undefined otherwise */ @action - public getSimplifiedChunkById(chunkId: string): any | undefined { + public getSimplifiedChunkById(chunkId: string) { return { foundChunk: this.simplifiedChunks.get(chunkId), doc: this.getDocument(this.simplifiedChunks.get(chunkId)?.doc_id || chunkId), dataDoc: this.getDataDocument(this.simplifiedChunks.get(chunkId)?.doc_id || chunkId) }; } @@ -1098,7 +1098,7 @@ export class AgentDocumentManager { * @param doc The document containing original media segments * @returns Array of media segments or empty array if none exist */ - public getOriginalSegments(doc: Doc): any[] { + public getOriginalSegments(doc: Doc): { text: string; index: string; start: number }[] { if (!doc || !doc.original_segments) { return []; } |