diff options
Diffstat (limited to 'src')
| -rw-r--r-- | src/client/documents/DocumentTypes.ts | 2 | ||||
| -rw-r--r-- | src/client/views/nodes/chatbot/agentsystem/Agent.ts | 32 | ||||
| -rw-r--r-- | src/client/views/nodes/chatbot/chatboxcomponents/ChatBox.tsx | 402 | ||||
| -rw-r--r-- | src/client/views/nodes/chatbot/tools/CreateAnyDocTool.ts | 145 | ||||
| -rw-r--r-- | src/client/views/nodes/chatbot/tools/CreateDocumentTool.ts | 393 | ||||
| -rw-r--r-- | src/client/views/nodes/chatbot/tools/CreateTextDocumentTool.ts | 61 | ||||
| -rw-r--r-- | src/client/views/nodes/chatbot/vectorstore/Vectorstore.ts | 4 | ||||
| -rw-r--r-- | src/server/flashcard/labels.py | 285 | ||||
| -rw-r--r-- | src/server/flashcard/requirements.txt | 12 | ||||
| -rw-r--r-- | src/server/flashcard/venv/pyvenv.cfg | 3 |
10 files changed, 1051 insertions, 288 deletions
diff --git a/src/client/documents/DocumentTypes.ts b/src/client/documents/DocumentTypes.ts index efe73fbbe..8aa844c0b 100644 --- a/src/client/documents/DocumentTypes.ts +++ b/src/client/documents/DocumentTypes.ts @@ -26,7 +26,7 @@ export enum DocumentType { SCRIPTING = 'script', // script editor CHAT = 'chat', // chat with GPT about files EQUATION = 'equation', // equation editor - FUNCPLOT = 'funcplot', // function plotter + FUNCPLOT = 'function plot', // function plotter MAP = 'map', DATAVIZ = 'dataviz', ANNOPALETTE = 'annopalette', diff --git a/src/client/views/nodes/chatbot/agentsystem/Agent.ts b/src/client/views/nodes/chatbot/agentsystem/Agent.ts index c58f009d4..ee91ccb92 100644 --- a/src/client/views/nodes/chatbot/agentsystem/Agent.ts +++ b/src/client/views/nodes/chatbot/agentsystem/Agent.ts @@ -1,25 +1,22 @@ import dotenv from 'dotenv'; import { XMLBuilder, XMLParser } from 'fast-xml-parser'; +import { escape } from 'lodash'; // Imported escape from lodash import OpenAI from 'openai'; import { ChatCompletionMessageParam } from 'openai/resources'; -import { escape } from 'lodash'; // Imported escape from lodash +import { DocumentOptions } from '../../../../documents/Documents'; import { AnswerParser } from '../response_parsers/AnswerParser'; import { StreamedAnswerParser } from '../response_parsers/StreamedAnswerParser'; +import { BaseTool } from '../tools/BaseTool'; import { CalculateTool } from '../tools/CalculateTool'; -import { CreateCSVTool } from '../tools/CreateCSVTool'; +import { CreateAnyDocumentTool, supportedDocumentTypes } from '../tools/CreateAnyDocTool'; +import { CreateDocTool } from '../tools/CreateDocumentTool'; import { DataAnalysisTool } from '../tools/DataAnalysisTool'; import { NoTool } from '../tools/NoTool'; -import { RAGTool } from '../tools/RAGTool'; import { SearchTool } from '../tools/SearchTool'; -import { WebsiteInfoScraperTool } from '../tools/WebsiteInfoScraperTool'; +import { Parameter, ParametersType, TypeMap } from '../types/tool_types'; import { AgentMessage, ASSISTANT_ROLE, AssistantMessage, Observation, PROCESSING_TYPE, ProcessingInfo, TEXT_TYPE } from '../types/types'; import { Vectorstore } from '../vectorstore/Vectorstore'; import { getReactPrompt } from './prompts'; -import { BaseTool } from '../tools/BaseTool'; -import { Parameter, ParametersType, TypeMap } from '../types/tool_types'; -import { CreateTextDocTool } from '../tools/CreateTextDocumentTool'; -import { DocumentOptions } from '../../../../documents/Documents'; -import { CreateAnyDocumentTool } from '../tools/CreateAnyDocTool'; dotenv.config(); @@ -58,7 +55,8 @@ export class Agent { history: () => string, csvData: () => { filename: string; id: string; text: string }[], addLinkedUrlDoc: (url: string, id: string) => void, - addLinkedDoc: (doc_type: string, data: string | undefined, options: DocumentOptions, id: string) => void, + addLinkedDoc: (doc_type: supportedDocumentTypes, data: unknown, options: DocumentOptions, id: string) => void, + // eslint-disable-next-line @typescript-eslint/no-unused-vars createCSVInDash: (url: string, title: string, id: string, data: string) => void ) { // Initialize OpenAI client with API key from environment @@ -71,12 +69,13 @@ export class Agent { // Define available tools for the assistant this.tools = { calculate: new CalculateTool(), - rag: new RAGTool(this.vectorstore), + // rag: new RAGTool(this.vectorstore), dataAnalysis: new DataAnalysisTool(csvData), - websiteInfoScraper: new WebsiteInfoScraperTool(addLinkedUrlDoc), + // websiteInfoScraper: new WebsiteInfoScraperTool(addLinkedUrlDoc), searchTool: new SearchTool(addLinkedUrlDoc), //createCSV: new CreateCSVTool(createCSVInDash), noTool: new NoTool(), + createDoc: new CreateDocTool(addLinkedDoc), //createTextDoc: new CreateTextDocTool(addLinkedDoc), createAnyDocument: new CreateAnyDocumentTool(addLinkedDoc), }; @@ -136,6 +135,7 @@ export class Agent { console.log(this.interMessages); console.log(`Turn ${i}/${maxTurns}`); + // eslint-disable-next-line no-await-in-loop const result = await this.execute(onProcessingUpdate, onAnswerUpdate); this.interMessages.push({ role: 'assistant', content: result }); @@ -191,11 +191,13 @@ export class Agent { } else if (key === 'action_input') { // Handle action input stage const actionInput = stage[key]; + console.log(`Action input full:`, actionInput); console.log(`Action input:`, actionInput.inputs); if (currentAction) { try { // Process the action with its input + // eslint-disable-next-line no-await-in-loop const observation = (await this.processAction(currentAction, actionInput.inputs)) as Observation[]; const nextPrompt = [{ type: 'text', text: `<stage number="${i + 1}" role="user"> <observation>` }, ...observation, { type: 'text', text: '</observation></stage>' }] as Observation[]; console.log(observation); @@ -300,7 +302,7 @@ export class Agent { * @param response The parsed XML response from the assistant. * @throws An error if the response does not meet the expected structure. */ - private validateAssistantResponse(response: any) { + private validateAssistantResponse(response: { stage: { [key: string]: object | string } }) { if (!response.stage) { throw new Error('Response does not contain a <stage> element'); } @@ -343,7 +345,7 @@ export class Agent { // If 'action_input' is present, validate its structure if ('action_input' in stage) { - const actionInput = stage.action_input; + const actionInput = stage.action_input as object; if (!('action_input_description' in actionInput) || typeof actionInput.action_input_description !== 'string') { throw new Error('action_input must contain an action_input_description string'); @@ -358,7 +360,7 @@ export class Agent { // If 'answer' is present, validate its structure if ('answer' in stage) { - const answer = stage.answer; + const answer = stage.answer as object; // Ensure answer contains at least one of the required elements if (!('grounded_text' in answer || 'normal_text' in answer)) { diff --git a/src/client/views/nodes/chatbot/chatboxcomponents/ChatBox.tsx b/src/client/views/nodes/chatbot/chatboxcomponents/ChatBox.tsx index 3ef6bdd8b..076f49831 100644 --- a/src/client/views/nodes/chatbot/chatboxcomponents/ChatBox.tsx +++ b/src/client/views/nodes/chatbot/chatboxcomponents/ChatBox.tsx @@ -14,12 +14,12 @@ import OpenAI, { ClientOptions } from 'openai'; import * as React from 'react'; import { v4 as uuidv4 } from 'uuid'; import { ClientUtils } from '../../../../../ClientUtils'; -import { Doc, DocListCast } from '../../../../../fields/Doc'; +import { Doc, DocListCast, FieldType, Opt } from '../../../../../fields/Doc'; import { DocData, DocViews } from '../../../../../fields/DocSymbols'; -import { CsvCast, DocCast, PDFCast, RTFCast, StrCast } from '../../../../../fields/Types'; +import { CsvCast, DocCast, NumCast, PDFCast, RTFCast, StrCast } from '../../../../../fields/Types'; import { Networking } from '../../../../Network'; import { DocUtils } from '../../../../documents/DocUtils'; -import { DocumentType } from '../../../../documents/DocumentTypes'; +import { CollectionViewType, DocumentType } from '../../../../documents/DocumentTypes'; import { Docs, DocumentOptions } from '../../../../documents/Documents'; import { DocumentManager } from '../../../../util/DocumentManager'; import { LinkManager } from '../../../../util/LinkManager'; @@ -33,7 +33,6 @@ import { Vectorstore } from '../vectorstore/Vectorstore'; import './ChatBox.scss'; import MessageComponentBox from './MessageComponent'; import { ProgressBar } from './ProgressBar'; -import { RichTextField } from '../../../../../fields/RichTextField'; dotenv.config(); @@ -45,17 +44,17 @@ dotenv.config(); @observer export class ChatBox extends ViewBoxAnnotatableComponent<FieldViewProps>() { // MobX observable properties to track UI state and data - @observable history: AssistantMessage[] = []; - @observable.deep current_message: AssistantMessage | undefined = undefined; - @observable isLoading: boolean = false; - @observable uploadProgress: number = 0; - @observable currentStep: string = ''; - @observable expandedScratchpadIndex: number | null = null; - @observable inputValue: string = ''; - @observable private linked_docs_to_add: ObservableSet = observable.set(); - @observable private linked_csv_files: { filename: string; id: string; text: string }[] = []; - @observable private isUploadingDocs: boolean = false; - @observable private citationPopup: { text: string; visible: boolean } = { text: '', visible: false }; + @observable private _history: AssistantMessage[] = []; + @observable.deep private _current_message: AssistantMessage | undefined = undefined; + @observable private _isLoading: boolean = false; + @observable private _uploadProgress: number = 0; + @observable private _currentStep: string = ''; + @observable private _expandedScratchpadIndex: number | null = null; + @observable private _inputValue: string = ''; + @observable private _linked_docs_to_add: ObservableSet = observable.set(); + @observable private _linked_csv_files: { filename: string; id: string; text: string }[] = []; + @observable private _isUploadingDocs: boolean = false; + @observable private _citationPopup: { text: string; visible: boolean } = { text: '', visible: false }; // Private properties for managing OpenAI API, vector store, agent, and UI elements private openai: OpenAI; @@ -96,7 +95,7 @@ export class ChatBox extends ViewBoxAnnotatableComponent<FieldViewProps>() { // Reaction to update dataDoc when chat history changes reaction( () => - this.history.map((msg: AssistantMessage) => ({ + this._history.map((msg: AssistantMessage) => ({ role: msg.role, content: msg.content, follow_up_questions: msg.follow_up_questions, @@ -115,20 +114,20 @@ export class ChatBox extends ViewBoxAnnotatableComponent<FieldViewProps>() { */ @action addDocToVectorstore = async (newLinkedDoc: Doc) => { - this.uploadProgress = 0; - this.currentStep = 'Initializing...'; - this.isUploadingDocs = true; + this._uploadProgress = 0; + this._currentStep = 'Initializing...'; + this._isUploadingDocs = true; try { // Add the document to the vectorstore await this.vectorstore.addAIDoc(newLinkedDoc, this.updateProgress); } catch (error) { console.error('Error uploading document:', error); - this.currentStep = 'Error during upload'; + this._currentStep = 'Error during upload'; } finally { - this.isUploadingDocs = false; - this.uploadProgress = 0; - this.currentStep = ''; + this._isUploadingDocs = false; + this._uploadProgress = 0; + this._currentStep = ''; } }; @@ -139,8 +138,8 @@ export class ChatBox extends ViewBoxAnnotatableComponent<FieldViewProps>() { */ @action updateProgress = (progress: number, step: string) => { - this.uploadProgress = progress; - this.currentStep = step; + this._uploadProgress = progress; + this._currentStep = step; }; /** @@ -177,7 +176,7 @@ export class ChatBox extends ViewBoxAnnotatableComponent<FieldViewProps>() { const csvId = id ?? uuidv4(); // Add CSV details to linked files - this.linked_csv_files.push({ + this._linked_csv_files.push({ filename: CsvCast(newLinkedDoc.data).url.pathname, id: csvId, text: csvData, @@ -199,7 +198,7 @@ export class ChatBox extends ViewBoxAnnotatableComponent<FieldViewProps>() { */ @action toggleToolLogs = (index: number) => { - this.expandedScratchpadIndex = this.expandedScratchpadIndex === index ? null : index; + this._expandedScratchpadIndex = this._expandedScratchpadIndex === index ? null : index; }; /** @@ -258,7 +257,7 @@ export class ChatBox extends ViewBoxAnnotatableComponent<FieldViewProps>() { @action askGPT = async (event: React.FormEvent): Promise<void> => { event.preventDefault(); - this.inputValue = ''; + this._inputValue = ''; // Extract the user's message const textInput = (event.currentTarget as HTMLFormElement).elements.namedItem('messageInput') as HTMLInputElement; @@ -268,13 +267,13 @@ export class ChatBox extends ViewBoxAnnotatableComponent<FieldViewProps>() { try { textInput.value = ''; // Add the user's message to the history - this.history.push({ + this._history.push({ role: ASSISTANT_ROLE.USER, content: [{ index: 0, type: TEXT_TYPE.NORMAL, text: trimmedText, citation_ids: null }], processing_info: [], }); - this.isLoading = true; - this.current_message = { + this._isLoading = true; + this._current_message = { role: ASSISTANT_ROLE.ASSISTANT, content: [], citations: [], @@ -284,9 +283,9 @@ export class ChatBox extends ViewBoxAnnotatableComponent<FieldViewProps>() { // Define callbacks for real-time processing updates const onProcessingUpdate = (processingUpdate: ProcessingInfo[]) => { runInAction(() => { - if (this.current_message) { - this.current_message = { - ...this.current_message, + if (this._current_message) { + this._current_message = { + ...this._current_message, processing_info: processingUpdate, }; } @@ -296,9 +295,9 @@ export class ChatBox extends ViewBoxAnnotatableComponent<FieldViewProps>() { const onAnswerUpdate = (answerUpdate: string) => { runInAction(() => { - if (this.current_message) { - this.current_message = { - ...this.current_message, + if (this._current_message) { + this._current_message = { + ...this._current_message, content: [{ text: answerUpdate, type: TEXT_TYPE.NORMAL, index: 0, citation_ids: [] }], }; } @@ -310,22 +309,24 @@ export class ChatBox extends ViewBoxAnnotatableComponent<FieldViewProps>() { // Update the history with the final assistant message runInAction(() => { - if (this.current_message) { - this.history.push({ ...finalMessage }); - this.current_message = undefined; - this.dataDoc.data = JSON.stringify(this.history); + if (this._current_message) { + this._history.push({ ...finalMessage }); + this._current_message = undefined; + this.dataDoc.data = JSON.stringify(this._history); } }); } catch (err) { console.error('Error:', err); // Handle error in processing - this.history.push({ + this._history.push({ role: ASSISTANT_ROLE.ASSISTANT, content: [{ index: 0, type: TEXT_TYPE.ERROR, text: 'Sorry, I encountered an error while processing your request.', citation_ids: null }], processing_info: [], }); } finally { - this.isLoading = false; + runInAction(() => { + this._isLoading = false; + }); this.scrollToBottom(); } } @@ -339,8 +340,8 @@ export class ChatBox extends ViewBoxAnnotatableComponent<FieldViewProps>() { */ @action updateMessageCitations = (index: number, citations: Citation[]) => { - if (this.history[index]) { - this.history[index].citations = citations; + if (this._history[index]) { + this._history[index].citations = citations; } }; @@ -381,17 +382,14 @@ export class ChatBox extends ViewBoxAnnotatableComponent<FieldViewProps>() { * @param data The CSV data content. */ @action - createCSVInDash = async (url: string, title: string, id: string, data: string) => { - const doc = DocCast(await DocUtils.DocumentFromType('csv', url, { title: title, text: RTFCast(data) })); - - const linkDoc = Docs.Create.LinkDocument(this.Document, doc); - LinkManager.Instance.addLink(linkDoc); - - doc && this._props.addDocument?.(doc); - await DocumentManager.Instance.showDocument(doc, { willZoomCentered: true }, () => {}); - - this.addCSVForAnalysis(doc, id); - }; + createCSVInDash = (url: string, title: string, id: string, data: string) => + DocUtils.DocumentFromType('csv', url, { title: title, text: RTFCast(data) }).then(doc => { + if (doc) { + LinkManager.Instance.addLink(Docs.Create.LinkDocument(this.Document, doc)); + this._props.addDocument?.(doc); + DocumentManager.Instance.showDocument(doc, { willZoomCentered: true }, () => {}).then(() => this.addCSVForAnalysis(doc, id)); + } + }); /** * Creates a text document in the dashboard and adds it for analysis. @@ -401,68 +399,216 @@ export class ChatBox extends ViewBoxAnnotatableComponent<FieldViewProps>() { * @param id The unique ID for the document. */ @action - createDocInDash = async (doc_type: string, data: string | undefined, options: DocumentOptions, id: string) => { - let doc; - - switch (doc_type.toLowerCase()) { - case 'text': - doc = Docs.Create.TextDocument(data || '', options); - break; - case 'image': - doc = Docs.Create.ImageDocument(data || '', options); - break; - case 'pdf': - doc = Docs.Create.PdfDocument(data || '', options); - break; - case 'video': - doc = Docs.Create.VideoDocument(data || '', options); - break; - case 'audio': - doc = Docs.Create.AudioDocument(data || '', options); - break; - case 'web': - doc = Docs.Create.WebDocument(data || '', options); - break; - case 'equation': - doc = Docs.Create.EquationDocument(data || '', options); - break; - case 'functionplot': - case 'function_plot': - doc = Docs.Create.FunctionPlotDocument([], options); - break; - case 'dataviz': - case 'data_viz': { - const { fileUrl, id } = await Networking.PostToServer('/createCSV', { - filename: (options.title as string).replace(/\s+/g, '') + '.csv', - data: data, - }); - doc = Docs.Create.DataVizDocument(fileUrl, { ...options, text: RTFCast(data) }); - this.addCSVForAnalysis(doc, id); - break; + private createCollectionWithChildren = (data: { doc_type: string; id: string; data: any; title: string; width: number; height: number; backgroundColor: string }[], insideCol: boolean): Promise<Doc[]> => + Promise.all( + data.map(doc => + doc.doc_type !== 'collection' // Handle non-collection documents + ? this.whichDoc(doc.doc_type, doc.data, { backgroundColor: doc.backgroundColor, _width: doc.width, _height: doc.height }, doc.id, insideCol) + : this.createCollectionWithChildren(doc.data, true).then(nestedDocs => + Docs.Create.FreeformDocument(nestedDocs, { + title: doc.title, + backgroundColor: doc.backgroundColor, + _width: doc.width, + _height: doc.height, + _layout_fitWidth: true, + _freeform_backgroundGrid: true, + }) + ) + ) // prettier-ignore + ).then(childDocs => childDocs.filter(doc => doc).map(doc => doc!)); + // .then(nestedResults => { + // // Flatten any nested arrays from recursive collection calls + // const childDocs = nestedResults.flat() as Doc[]; + // childDocs.forEach(doc => { + // console.log(DocCast(doc)); + // console.log(DocCast(doc)[DocData].data); + // console.log(DocCast(doc)[DocData].data); + // }); + // return childDocs; + // }); + + @action + whichDoc = (doc_type: string, data: unknown, options: DocumentOptions, id: string, insideCol: boolean): Promise<Opt<Doc>> => + (async () => { + switch (doc_type) { + case 'text': return Docs.Create.TextDocument(data as string, options); + case 'flashcard': return this.createFlashcard(data as string[], options); + case 'deck': return this.createDeck(data as string, options); + case 'image': return Docs.Create.ImageDocument(data as string, options); + case 'equation': return Docs.Create.EquationDocument(data as string, options); + case 'noteboard': return Docs.Create.NoteTakingDocument([], options); + case 'simulation': return Docs.Create.SimulationDocument(options); + case 'collection': return this.createCollectionWithChildren(data as { doc_type: string; id: string; data: any; title: string; width: number; height: number; backgroundColor: string }[] , true). + then((arr, collOpts = { ...options, _layout_fitWidth: true, _freeform_backgroundGrid: true }) => + (() => { + switch (options.type_collection) { + case CollectionViewType.Tree: return Docs.Create.TreeDocument(arr, collOpts); + case CollectionViewType.Masonry: return Docs.Create.MasonryDocument(arr, collOpts); + case CollectionViewType.Card: return Docs.Create.CardDeckDocument(arr, collOpts); + case CollectionViewType.Carousel: return Docs.Create.CarouselDocument(arr, collOpts); + case CollectionViewType.Carousel3D: return Docs.Create.Carousel3DDocument(arr, collOpts); + case CollectionViewType.Multicolumn: return Docs.Create.CarouselDocument(arr, collOpts); + default: return Docs.Create.FreeformDocument(arr, collOpts); + } + })() + ); + case 'web': return Docs.Create.WebDocument(data as string, { ...options, data_useCors: true }); + case 'comparison': return this.createComparison(data as {left: {width:number ,height: number, backgroundColor: string, data: string}, right: {width:number ,height: number, backgroundColor: string, data: string}}, options); + case 'diagram': return Docs.Create.DiagramDocument(options); + case 'audio': return Docs.Create.AudioDocument(data as string, options); + case 'map': return Docs.Create.MapDocument([], options); + case 'screengrab': return Docs.Create.ScreenshotDocument(options); + case 'webcam': return Docs.Create.WebCamDocument('', options); + case 'button': return Docs.Create.ButtonDocument(options); + case 'script': return Docs.Create.ScriptingDocument(null, options); + case 'dataviz': return Docs.Create.DataVizDocument('/users/rz/Downloads/addresses.csv', options); + case 'chat': return Docs.Create.ChatDocument(options); + case 'trail': return Docs.Create.PresDocument(options); + case 'tab': return Docs.Create.FreeformDocument([], options); + case 'slide': return Docs.Create.TreeDocument([], options); + default: return Docs.Create.TextDocument(data as string, options); + } // prettier-ignore + })().then(doc => { + if (doc) { + doc.x = NumCast((options.x as number) ?? 0) + (insideCol ? 0 : NumCast(this.layoutDoc.x) + NumCast(this.layoutDoc.width)) + 100; + doc.y = NumCast(options.y as number) + (insideCol ? 0 : NumCast(this.layoutDoc.y)); + } + return doc; + }); + + /** + * Creates a document in the dashboard. + * + * @param {string} doc_type - The type of document to create. + * @param {string} data - The data used to generate the document. + * @param {DocumentOptions} options - Configuration options for the document. + * @param {string} id - Unique identifier for the document. + * @returns {Promise<void>} A promise that resolves once the document is created and displayed. + */ + @action + createDocInDash = (doc_type: string, data: unknown, options: DocumentOptions /*, id: string */) => { + const linkAndShowDoc = (doc: Opt<Doc>) => { + if (doc) { + LinkManager.Instance.addLink(Docs.Create.LinkDocument(this.Document, doc)); + this._props.addDocument?.(doc); + DocumentManager.Instance.showDocument(doc, { willZoomCentered: true }, () => {}); } - case 'chat': - doc = Docs.Create.ChatDocument(options); - break; - // Add more cases for other document types - default: - console.error('Unknown or unsupported document type:', doc_type); - return; + }; + const doc = (() => { + switch (doc_type.toLowerCase()) { + case 'flashcard': return this.createFlashcard(data as string[], options); + case 'text': return Docs.Create.TextDocument(data as string || '', options); + case 'image': return Docs.Create.ImageDocument(data as string || '', options); + case 'pdf': return Docs.Create.PdfDocument(data as string || '', options); + case 'video': return Docs.Create.VideoDocument(data as string || '', options); + case 'audio': return Docs.Create.AudioDocument(data as string || '', options); + case 'web': return Docs.Create.WebDocument(data as string || '', options); + case 'equation': return Docs.Create.EquationDocument(data as string || '', options); + case 'chat': return Docs.Create.ChatDocument(options); + case 'functionplot': return Docs.Create.FunctionPlotDocument([], options); + case 'dataviz': Networking.PostToServer('/createCSV', { + filename: (options.title as string).replace(/\s+/g, '') + '.csv', + data: data, + })?.then(({ fileUrl, id }) => { + const vdoc = Docs.Create.DataVizDocument(fileUrl, { ...options, text: RTFCast(data as FieldType) }); + this.addCSVForAnalysis(vdoc, id); + linkAndShowDoc(vdoc); + }); + return undefined; + // Add more cases for other document types + default: console.error('Unknown or unsupported document type:', doc_type); + return undefined; + } // prettier-ignore + })(); + if (doc) linkAndShowDoc(doc); + }; + + /** + * Creates a deck of flashcards. + * + * @param {any} data - The data used to generate the flashcards. Can be a string or an object. + * @param {DocumentOptions} options - Configuration options for the flashcard deck. + * @returns {Doc} A carousel document containing the flashcard deck. + */ + @action + createDeck = (data: string | unknown[], options: DocumentOptions) => { + const flashcardDeck: Doc[] = []; + // Parse `data` only if it’s a string + const deckData = typeof data === 'string' ? (JSON.parse(data) as unknown) : data; + const flashcardArray = Array.isArray(deckData) ? deckData : Object.values(deckData as object); + // Process each flashcard document in the `deckData` array + if (flashcardArray.length == 2 && flashcardArray[0].doc_type == 'text' && flashcardArray[1].doc_type == 'text') { + this.createFlashcard(flashcardArray as string[], options); + } else { + flashcardArray.forEach(doc => { + const flashcardDoc = this.createFlashcard(doc, options); + if (flashcardDoc) flashcardDeck.push(flashcardDoc); + }); } - const linkDoc = Docs.Create.LinkDocument(this.Document, doc); - LinkManager.Instance.addLink(linkDoc); - doc && this._props.addDocument?.(doc); - await DocumentManager.Instance.showDocument(doc, { willZoomCentered: true }, () => {}); + // Create a carousel to contain the flashcard deck + return Docs.Create.CarouselDocument(flashcardDeck, { + title: options.title || 'Flashcard Deck', + _width: options._width || 300, + _height: options._height || 300, + _layout_fitWidth: false, + _layout_autoHeight: true, + }); + }; + + /** + * Creates a single flashcard document. + * + * @param {any} data - The data used to generate the flashcard. Can be a string or an object. + * @param {any} options - Configuration options for the flashcard. + * @returns {Doc | undefined} The created flashcard document, or undefined if the flashcard cannot be created. + */ + @action + createFlashcard = (data: string[], options: DocumentOptions) => { + const deckData = typeof data === 'string' ? JSON.parse(data) : data; + const flashcardArray = Array.isArray(deckData) ? deckData : (Object.values(deckData)[2] as string[]); + const [front, back] = flashcardArray; + + if (typeof front === 'string' && typeof back === 'string') { + const sideOptions: DocumentOptions = { + backgroundColor: options.backgroundColor, + _width: options._width, + _height: options._height || 300, + }; + + // Create front and back text documents + const side1 = Docs.Create.CenteredTextCreator('question', front, sideOptions); + const side2 = Docs.Create.CenteredTextCreator('answer', back, sideOptions); + + // Create the flashcard document with both sides + return Docs.Create.FlashcardDocument('flashcard', side1, side2, sideOptions); + } }; /** + * Creates a comparison document. + * + * @param {any} doc - The document data containing left and right components for comparison. + * @param {any} options - Configuration options for the comparison document. + * @returns {Doc} The created comparison document. + */ + @action + createComparison = (doc: { left: { width: number; height: number; backgroundColor: string; data: string }; right: { width: number; height: number; backgroundColor: string; data: string } }, options: DocumentOptions) => + Docs.Create.ComparisonDocument(options.title as string, { + data_back: Docs.Create.TextDocument(doc.left.data, { backgroundColor: doc.left.backgroundColor, _width: doc.left.width, _height: doc.left.height }), + data_front: Docs.Create.TextDocument(doc.right.data, { backgroundColor: doc.right.backgroundColor, _width: doc.right.width, _height: doc.right.height }), + _width: options._width, + _height: options._height || 300, + backgroundColor: options.backgroundColor, + }); + + /** * Event handler to manage citations click in the message components. * @param citation The citation object clicked by the user. */ @action handleCitationClick = (citation: Citation) => { const currentLinkedDocs: Doc[] = this.linkedDocs; - const chunkId = citation.chunk_id; // Loop through the linked documents to find the matching chunk and handle its display @@ -497,8 +643,8 @@ export class ChatBox extends ViewBoxAnnotatableComponent<FieldViewProps>() { } break; case CHUNK_TYPE.TEXT: - this.citationPopup = { text: citation.direct_text ?? 'No text available', visible: true }; - setTimeout(() => (this.citationPopup.visible = false), 3000); // Hide after 3 seconds + this._citationPopup = { text: citation.direct_text ?? 'No text available', visible: true }; + setTimeout(() => (this._citationPopup.visible = false), 3000); // Hide after 3 seconds DocumentManager.Instance.showDocument(doc, { willZoomCentered: true }, () => { const firstView = Array.from(doc[DocViews])[0] as DocumentView; @@ -566,7 +712,7 @@ export class ChatBox extends ViewBoxAnnotatableComponent<FieldViewProps>() { try { const storedHistory = JSON.parse(StrCast(this.dataDoc.data)); runInAction(() => { - this.history.push( + this._history.push( ...storedHistory.map((msg: AssistantMessage) => ({ role: msg.role, content: msg.content, @@ -581,7 +727,7 @@ export class ChatBox extends ViewBoxAnnotatableComponent<FieldViewProps>() { } else { // Default welcome message runInAction(() => { - this.history.push({ + this._history.push({ role: ASSISTANT_ROLE.ASSISTANT, content: [ { @@ -605,11 +751,11 @@ export class ChatBox extends ViewBoxAnnotatableComponent<FieldViewProps>() { .filter(d => d); return linkedDocs; }, - linked => linked.forEach(doc => this.linked_docs_to_add.add(doc)) + linked => linked.forEach(doc => this._linked_docs_to_add.add(doc)) ); // Observe changes to linked documents and handle document addition - observe(this.linked_docs_to_add, change => { + observe(this._linked_docs_to_add, change => { if (change.type === 'add') { if (PDFCast(change.newValue.data)) { this.addDocToVectorstore(change.newValue); @@ -682,18 +828,16 @@ export class ChatBox extends ViewBoxAnnotatableComponent<FieldViewProps>() { /** * Getter that retrieves all linked CSV files for analysis. */ - @computed - get linkedCSVs(): { filename: string; id: string; text: string }[] { - return this.linked_csv_files; + @computed get linkedCSVs(): { filename: string; id: string; text: string }[] { + return this._linked_csv_files; } /** * Getter that formats the entire chat history as a string for the agent's system message. */ - @computed - get formattedHistory(): string { + @computed get formattedHistory(): string { let history = '<chat_history>\n'; - for (const message of this.history) { + for (const message of this._history) { history += `<${message.role}>${message.content.map(content => content.text).join(' ')}`; if (message.loop_summary) { history += `<loop_summary>${message.loop_summary}</loop_summary>`; @@ -729,7 +873,7 @@ export class ChatBox extends ViewBoxAnnotatableComponent<FieldViewProps>() { */ @action handleFollowUpClick = (question: string) => { - this.inputValue = question; + this._inputValue = question; }; /** @@ -738,11 +882,11 @@ export class ChatBox extends ViewBoxAnnotatableComponent<FieldViewProps>() { render() { return ( <div className="chat-box"> - {this.isUploadingDocs && ( + {this._isUploadingDocs && ( <div className="uploading-overlay"> <div className="progress-container"> <ProgressBar /> - <div className="step-name">{this.currentStep}</div> + <div className="step-name">{this._currentStep}</div> </div> </div> )} @@ -750,18 +894,18 @@ export class ChatBox extends ViewBoxAnnotatableComponent<FieldViewProps>() { <h2>{this.userName()}'s AI Assistant</h2> </div> <div className="chat-messages" ref={this.messagesRef}> - {this.history.map((message, index) => ( + {this._history.map((message, index) => ( <MessageComponentBox key={index} message={message} onFollowUpClick={this.handleFollowUpClick} onCitationClick={this.handleCitationClick} updateMessageCitations={this.updateMessageCitations} /> ))} - {this.current_message && ( - <MessageComponentBox key={this.history.length} message={this.current_message} onFollowUpClick={this.handleFollowUpClick} onCitationClick={this.handleCitationClick} updateMessageCitations={this.updateMessageCitations} /> + {this._current_message && ( + <MessageComponentBox key={this._history.length} message={this._current_message} onFollowUpClick={this.handleFollowUpClick} onCitationClick={this.handleCitationClick} updateMessageCitations={this.updateMessageCitations} /> )} </div> <form onSubmit={this.askGPT} className="chat-input"> - <input type="text" name="messageInput" autoComplete="off" placeholder="Type your message here..." value={this.inputValue} onChange={e => (this.inputValue = e.target.value)} disabled={this.isLoading} /> - <button className="submit-button" type="submit" disabled={this.isLoading || !this.inputValue.trim()}> - {this.isLoading ? ( + <input type="text" name="messageInput" autoComplete="off" placeholder="Type your message here..." value={this._inputValue} onChange={action(e => (this._inputValue = e.target.value))} disabled={this._isLoading} /> + <button className="submit-button" type="submit" disabled={this._isLoading || !this._inputValue.trim()}> + {this._isLoading ? ( <div className="spinner"></div> ) : ( <svg viewBox="0 0 24 24" width="24" height="24" stroke="currentColor" strokeWidth="2" fill="none" strokeLinecap="round" strokeLinejoin="round"> @@ -772,10 +916,10 @@ export class ChatBox extends ViewBoxAnnotatableComponent<FieldViewProps>() { </button> </form> {/* Popup for citation */} - {this.citationPopup.visible && ( + {this._citationPopup.visible && ( <div className="citation-popup"> <p> - <strong>Text from your document: </strong> {this.citationPopup.text} + <strong>Text from your document: </strong> {this._citationPopup.text} </p> </div> )} diff --git a/src/client/views/nodes/chatbot/tools/CreateAnyDocTool.ts b/src/client/views/nodes/chatbot/tools/CreateAnyDocTool.ts index 6f61b77d4..0049612fd 100644 --- a/src/client/views/nodes/chatbot/tools/CreateAnyDocTool.ts +++ b/src/client/views/nodes/chatbot/tools/CreateAnyDocTool.ts @@ -1,49 +1,61 @@ import { v4 as uuidv4 } from 'uuid'; import { BaseTool } from './BaseTool'; import { Observation } from '../types/types'; -import { ParametersType, Parameter } from '../types/tool_types'; -import { DocumentOptions, Docs } from '../../../../documents/Documents'; +import { ParametersType } from '../types/tool_types'; +import { DocumentOptions } from '../../../../documents/Documents'; +import { toLower } from 'lodash'; -/** - * List of supported document types that can be created via text LLM. - */ -type supportedDocumentTypesType = 'text' | 'html' | 'equation' | 'functionPlot' | 'dataviz' | 'noteTaking' | 'rtf' | 'message'; -const supportedDocumentTypes: supportedDocumentTypesType[] = ['text', 'html', 'equation', 'functionPlot', 'dataviz', 'noteTaking', 'rtf', 'message']; +export enum supportedDocumentTypes { + flashcard = 'flashcard', + text = 'text', + html = 'html', + equation = 'equation', + functionplot = 'functionplot', + dataviz = 'dataviz', + notetaking = 'notetaking', + rtf = 'rtf', + message = 'message', +} +const standardOptions = ['title', 'backgroundColor', 'layout']; /** * Description of document options and data field for each type. */ -const documentTypesInfo = { - text: { - options: ['title', 'backgroundColor', 'fontColor', 'text_align', 'layout'], +const documentTypesInfo: { [key in supportedDocumentTypes]: { options: string[]; dataDescription: string } } = { + [supportedDocumentTypes.flashcard]: { + options: [...standardOptions, 'fontColor', 'text_align'], + dataDescription: 'an array of two strings. the first string contains a question, and the second string contains an answer', + }, + [supportedDocumentTypes.text]: { + options: [...standardOptions, 'fontColor', 'text_align'], dataDescription: 'The text content of the document.', }, - html: { - options: ['title', 'backgroundColor', 'layout'], + [supportedDocumentTypes.html]: { + options: [], dataDescription: 'The HTML-formatted text content of the document.', }, - equation: { - options: ['title', 'backgroundColor', 'fontColor', 'layout'], + [supportedDocumentTypes.equation]: { + options: [...standardOptions, 'fontColor'], dataDescription: 'The equation content as a string.', }, - functionPlot: { - options: ['title', 'backgroundColor', 'layout', 'function_definition'], + [supportedDocumentTypes.functionplot]: { + options: [...standardOptions, 'function_definition'], dataDescription: 'The function definition(s) for plotting. Provide as a string or array of function definitions.', }, - dataviz: { - options: ['title', 'backgroundColor', 'layout', 'chartType'], + [supportedDocumentTypes.dataviz]: { + options: [...standardOptions, 'chartType'], dataDescription: 'A string of comma-separated values representing the CSV data.', }, - noteTaking: { - options: ['title', 'backgroundColor', 'layout'], + [supportedDocumentTypes.notetaking]: { + options: standardOptions, dataDescription: 'The initial content or structure for note-taking.', }, - rtf: { - options: ['title', 'backgroundColor', 'layout'], + [supportedDocumentTypes.rtf]: { + options: standardOptions, dataDescription: 'The rich text content in RTF format.', }, - message: { - options: ['title', 'backgroundColor', 'layout'], + [supportedDocumentTypes.message]: { + options: standardOptions, dataDescription: 'The message content of the document.', }, }; @@ -52,7 +64,7 @@ const createAnyDocumentToolParams = [ { name: 'document_type', type: 'string', - description: `The type of the document to create. Supported types are: ${supportedDocumentTypes.join(', ')}`, + description: `The type of the document to create. Supported types are: ${Object.values(supportedDocumentTypes).join(', ')}`, required: true, }, { @@ -64,90 +76,63 @@ const createAnyDocumentToolParams = [ { name: 'options', type: 'string', - description: `A JSON string representing the document options. Available options depend on the document type. For example: -${supportedDocumentTypes - .map( - docType => ` -- For '${docType}' documents, options include: ${documentTypesInfo[docType].options.join(', ')}` - ) - .join('\n')}`, required: false, + description: `A JSON string representing the document options. Available options depend on the document type. For example: + ${Object.entries(documentTypesInfo).map( ([doc_type, info]) => ` +- For '${doc_type}' documents, options include: ${info.options.join(', ')}`) + .join('\n')}`, // prettier-ignore }, ] as const; type CreateAnyDocumentToolParamsType = typeof createAnyDocumentToolParams; export class CreateAnyDocumentTool extends BaseTool<CreateAnyDocumentToolParamsType> { - private _addLinkedDoc: (doc_type: string, data: string | undefined, options: DocumentOptions, id: string) => void; + private _addLinkedDoc: (doc_type: supportedDocumentTypes, data: unknown, options: DocumentOptions, id: string) => void; - constructor(addLinkedDoc: (doc_type: string, data: string | undefined, options: DocumentOptions, id: string) => void) { + constructor(addLinkedDoc: (doc_type: supportedDocumentTypes, data: unknown, options: DocumentOptions, id: string) => void) { + // prettier-ignore super( 'createAnyDocument', - `Creates any type of document with the provided options and data. Supported document types are: ${supportedDocumentTypes.join(', ')}. dataviz is a csv table tool, so for CSVs, use dataviz. Here are the options for each type: + `Creates any type of document with the provided options and data. Supported document types are: ${Object.values(supportedDocumentTypes).join(', ')}. dataviz is a csv table tool, so for CSVs, use dataviz. Here are the options for each type: <supported_document_types> - ${supportedDocumentTypes - .map( - docType => ` - <document_type name="${docType}"> - <data_description>${documentTypesInfo[docType].dataDescription}</data_description> - <options> - ${documentTypesInfo[docType].options.map(option => `<option>${option}</option>`).join('\n')} - </options> - </document_type> - ` - ) - .join('\n')} - </supported_document_types>`, + ${Object.entries(documentTypesInfo).map(([doc_type, info]) => ` + <document_type name="${doc_type}"> + <data_description>${info.dataDescription}</data_description> + <options> + ${info.options.map(option => `<option>${option}</option>`).join('\n')} + </options> + </document_type> + `).join('\n')} + </supported_document_types>`, createAnyDocumentToolParams, 'Provide the document type, data, and options for the document. Options should be a valid JSON string containing the document options specific to the document type.', - `Creates any type of document with the provided options and data. Supported document types are: ${supportedDocumentTypes.join(', ')}.` + `Creates any type of document with the provided options and data. Supported document types are: ${Object.values(supportedDocumentTypes).join(', ')}.` ); this._addLinkedDoc = addLinkedDoc; } async execute(args: ParametersType<CreateAnyDocumentToolParamsType>): Promise<Observation[]> { try { - const documentType: supportedDocumentTypesType = args.document_type.toLowerCase() as supportedDocumentTypesType; - let options: DocumentOptions = {}; + const documentType = toLower(args.document_type) as unknown as supportedDocumentTypes; + const info = documentTypesInfo[documentType]; - if (!supportedDocumentTypes.includes(documentType)) { - throw new Error(`Unsupported document type: ${documentType}. Supported types are: ${supportedDocumentTypes.join(', ')}.`); + if (info === undefined) { + throw new Error(`Unsupported document type: ${documentType}. Supported types are: ${Object.values(supportedDocumentTypes).join(', ')}.`); } if (!args.data) { - throw new Error(`Data is required for ${documentType} documents. ${documentTypesInfo[documentType].dataDescription}`); - } - - if (args.options) { - try { - options = JSON.parse(args.options as string) as DocumentOptions; - } catch (e) { - throw new Error('Options must be a valid JSON string.'); - } + throw new Error(`Data is required for ${documentType} documents. ${info.dataDescription}`); } - const data = args.data as string; const id = uuidv4(); + const options: DocumentOptions = !args.options ? {} : JSON.parse(args.options); - // Set default options if not provided - options.title = options.title || `New ${documentType.charAt(0).toUpperCase() + documentType.slice(1)} Document`; - - // Call the function to add the linked document - this._addLinkedDoc(documentType, data, options, id); + // Call the function to add the linked document (add default title that can be overriden if set in options) + this._addLinkedDoc(documentType, args.data, { title: `New ${documentType.charAt(0).toUpperCase() + documentType.slice(1)} Document`, ...options }, id); - return [ - { - type: 'text', - text: `Created ${documentType} document with ID ${id}.`, - }, - ]; + return [{ type: 'text', text: `Created ${documentType} document with ID ${id}.` }]; } catch (error) { - return [ - { - type: 'text', - text: 'Error creating document: ' + (error as Error).message, - }, - ]; + return [{ type: 'text', text: 'Error creating document: ' + (error as Error).message }]; } } } diff --git a/src/client/views/nodes/chatbot/tools/CreateDocumentTool.ts b/src/client/views/nodes/chatbot/tools/CreateDocumentTool.ts new file mode 100644 index 000000000..f0f2fe703 --- /dev/null +++ b/src/client/views/nodes/chatbot/tools/CreateDocumentTool.ts @@ -0,0 +1,393 @@ +import { v4 as uuidv4 } from 'uuid'; +import { BaseTool } from './BaseTool'; +import { Observation } from '../types/types'; +import { ParametersType } from '../types/tool_types'; +import { DocumentOptions } from '../../../../documents/Documents'; +import { supportedDocumentTypes } from './CreateAnyDocTool'; + +/** + * Tthe CreateDocTool class is responsible for creating + * documents of various types (e.g., text, flashcards, collections) and organizing them in a + * structured manner. The tool supports creating dashboards with diverse document types and + * ensures proper placement of documents without overlap. + */ + +// Example document structure for various document types +const example = [ + { + doc_type: 'equation', + title: 'quadratic', + data: 'x^2 + y^2 = 3', + width: 300, + height: 300, + x: 0, + y: 0, + }, + { + doc_type: 'collection', + title: 'Advanced Biology', + data: [ + { + doc_type: 'text', + title: 'Cell Structure', + data: 'Cells are the basic building blocks of all living organisms.', + width: 300, + height: 300, + x: 500, + y: 0, + }, + ], + backgroundColor: '#00ff00', + width: 600, + height: 600, + x: 600, + y: 0, + type_collection: 'tree', + }, + { + doc_type: 'image', + title: 'experiment', + data: 'https://plus.unsplash.com/premium_photo-1694819488591-a43907d1c5cc?q=80&w=2628&auto=format&fit=crop&ixlib=rb-4.0.3&ixid=M3wxMjA3fDB8MHxwaG90by1wYWdlfHx8fGVufDB8fHx8fA%3D%3D', + width: 300, + height: 300, + x: 600, + y: 300, + }, + { + doc_type: 'deck', + title: 'Chemistry', + data: [ + { + doc_type: 'flashcard', + title: 'Photosynthesis', + data: [ + { + doc_type: 'text', + title: 'front_Photosynthesis', + data: 'What is photosynthesis?', + width: 300, + height: 300, + x: 100, + y: 600, + }, + { + doc_type: 'text', + title: 'back_photosynthesis', + data: 'The process by which plants make food.', + width: 300, + height: 300, + x: 100, + y: 700, + }, + ], + backgroundColor: '#00ff00', + width: 300, + height: 300, + x: 300, + y: 1000, + }, + { + doc_type: 'flashcard', + title: 'Photosynthesis', + data: [ + { + doc_type: 'text', + title: 'front_Photosynthesis', + data: 'What is photosynthesis?', + width: 300, + height: 300, + x: 200, + y: 800, + }, + { + doc_type: 'text', + title: 'back_photosynthesis', + data: 'The process by which plants make food.', + width: 300, + height: 300, + x: 100, + y: -100, + }, + ], + backgroundColor: '#00ff00', + width: 300, + height: 300, + x: 10, + y: 70, + }, + ], + backgroundColor: '#00ff00', + width: 600, + height: 600, + x: 200, + y: 800, + }, + { + doc_type: 'web', + title: 'Brown University Wikipedia', + data: 'https://en.wikipedia.org/wiki/Brown_University', + width: 300, + height: 300, + x: 1000, + y: 2000, + }, + { + doc_type: 'simulation', + title: 'Physics simulation', + data: '', + width: 300, + height: 300, + x: 100, + y: 100, + }, + { + doc_type: 'comparison', + title: 'WWI vs. WWII', + data: [ + { + doc_type: 'text', + title: 'WWI', + data: 'From 1914 to 1918, fighting took place across several continents, at sea and, for the first time, in the air.', + width: 300, + height: 300, + x: 100, + y: 100, + }, + { + doc_type: 'text', + title: 'WWII', + data: 'A devastating global conflict spanning from 1939 to 1945, saw the Allied powers fight against the Axis powers.', + width: 300, + height: 300, + x: 100, + y: 100, + }, + ], + width: 300, + height: 300, + x: 100, + y: 100, + }, + { + doc_type: 'collection', + title: 'Science Collection', + data: [ + { + doc_type: 'flashcard', + title: 'Photosynthesis', + data: [ + { + doc_type: 'text', + title: 'front_Photosynthesis', + data: 'What is photosynthesis?', + width: 300, + height: 300, + }, + { + doc_type: 'text', + title: 'back_photosynthesis', + data: 'The process by which plants make food.', + width: 300, + height: 300, + }, + ], + backgroundColor: '#00ff00', + width: 300, + height: 300, + }, + { + doc_type: 'web', + title: 'Brown University Wikipedia', + data: 'https://en.wikipedia.org/wiki/Brown_University', + width: 300, + height: 300, + x: 1100, + y: 1100, + }, + { + doc_type: 'text', + title: 'Water Cycle', + data: 'The continuous movement of water on, above, and below the Earth’s surface.', + width: 300, + height: 300, + x: 1500, + y: 500, + }, + { + doc_type: 'collection', + title: 'Advanced Biology', + data: [ + { + doc_type: 'text', + title: 'Cell Structure', + data: 'Cells are the basic building blocks of all living organisms.', + width: 300, + height: 300, + }, + ], + backgroundColor: '#00ff00', + width: 600, + height: 600, + x: 1100, + y: 500, + type_collection: 'freeform', + }, + ], + width: 600, + height: 600, + x: 500, + y: 500, + type_collection: 'freeform', + }, +]; + +// Stringify the entire structure for transmission if needed +const finalJsonString = JSON.stringify(example); + +// Instructions for creating various document types +const docInstructions = { + collection: { + description: + 'A recursive collection of documents as a stringified array. Each document can be a "text", "deck", "flashcard", "image", "web", "image", "comparison", "equation", "noteboard", "simulation", "diagram", "map", "screengrab", "webcam", "button", or another "collection".', + example: finalJsonString, + }, + text: 'Provide text content as a plain string. Example: "This is a standalone text document."', + flashcard: 'Two text documents with content for the front and back.', + deck: 'A decks data is an array of flashcards.', + web: 'A URL to a webpage. Example: https://en.wikipedia.org/wiki/Brown_University', + equation: 'Create an equation document, not a text document. Data is math equation.', + noteboard: 'Create a noteboard document', + simulation: 'Create a simulation document', + audio: 'A url to an audio recording. Example: ', +} as const; + +// Parameters for creating individual documents +const createDocToolParams = [ + { + name: 'data', + type: 'string', // Accepts either string or array, supporting individual and nested data + description: docInstructions, + required: true, + }, + { + name: 'doc_type', + type: 'string', + description: 'The type of the document. Options: "collection", "text", "flashcard", "web".', + required: true, + }, + { + name: 'title', + type: 'string', + description: 'The title of the document.', + required: true, + }, + { + name: 'x', + type: 'number', + description: 'The x location of the document; 0 <= x.', + required: true, + }, + { + name: 'y', + type: 'number', + description: 'The y location of the document; 0 <= y.', + required: true, + }, + { + name: 'background_color', + type: 'string', + description: 'The background color of the document as a hex string.', + required: false, + }, + { + name: 'font_color', + type: 'string', + description: 'The font color of the document as a hex string.', + required: false, + }, + { + name: 'width', + type: 'number', + description: 'The width of the document in pixels.', + required: true, + }, + { + name: 'height', + type: 'number', + description: 'The height of the document in pixels.', + required: true, + }, + { + name: 'type_collection', + type: 'string', + description: 'Either freeform, card, carousel, 3d-carousel, multicolumn, multirow, linear, map, notetaking, schema, stacking, grid, tree, or masonry.', + required: false, + }, +] as const; + +// Parameters for creating a list of documents +const createListDocToolParams = [ + { + name: 'docs', + type: 'string', + description: + 'Array of documents in stringified JSON format. Each item in the array should be an individual stringified JSON object. Each document can be of type "text", "flashcard", "web", or "collection" (for nested documents). ' + + 'Use this structure for nesting collections within collections. Each document should follow the structure in ' + + createDocToolParams + + '. Example: ' + + finalJsonString, + required: true, + }, +] as const; + +type CreateListDocToolParamsType = typeof createListDocToolParams; + +// Tool class for creating documents +export class CreateDocTool extends BaseTool<CreateListDocToolParamsType> { + private _addLinkedDoc: (doc_type: supportedDocumentTypes, data: unknown, options: DocumentOptions, id: string) => void; + + constructor(addLinkedDoc: (doc_type: supportedDocumentTypes, data: unknown, options: DocumentOptions, id: string) => void) { + super( + 'createDoc', + 'Creates one or more documents that best fit the user’s request. If the user requests a "dashboard," first call the search tool and then generate a variety of document types individually, with absolutely a minimum of 20 documents with two stacks of flashcards that are small and it should have a couple nested freeform collections of things, each with different content and color schemes. For example, create multiple individual documents like "text," "deck," "web", "equation," and "comparison." Use decks instead of flashcards for dashboards. Decks should have at least three flashcards. Really think about what documents are useful to the user. If they ask for a dashboard about the skeletal system, include flashcards, as they would be helpful. Arrange the documents in a grid layout, ensuring that the x and y coordinates are calculated so no documents overlap but they should be directly next to each other with 20 padding in between. Take into account the width and height of each document, spacing them appropriately to prevent collisions. Use a systematic approach, such as placing each document in a grid cell based on its order, where cell dimensions match the document dimensions plus a fixed margin for spacing. Do not nest all documents within a single collection unless explicitly requested by the user. Instead, create a set of independent documents with diverse document types. Each type should appear separately unless specified otherwise.', + createListDocToolParams, + 'Use the "data" parameter for document content and include title, color, and document dimensions. Ensure web documents use URLs from the search tool if relevant. Each document in a dashboard should be unique and well-differentiated in type and content, without repetition of similar types in any single collection.', + 'When creating a dashboard, ensure that it consists of a broad range of document types. Include a variety of documents, such as text, web, deck, comparison, image, simulation, and equation documents, each with distinct titles and colors, following the user’s preferences. ' + + 'Do not overuse collections or nest all document types within a single collection; instead, represent document types individually. Use this example for reference: ' + + finalJsonString + + '. Which documents are created should be random with different numbers of each document type and different for each dashboard. Must use search tool before creating a dashboard.' + ); + this._addLinkedDoc = addLinkedDoc; + } + + // Executes the tool logic for creating documents + async execute(args: ParametersType<CreateListDocToolParamsType>): Promise<Observation[]> { + try { + console.log('EXE' + args.docs); + const parsedDoc = JSON.parse(args.docs) as ({ doc_type: supportedDocumentTypes; data: unknown } & DocumentOptions)[]; + console.log('parsed' + parsedDoc); + parsedDoc.forEach(doc => + this._addLinkedDoc( + doc.doc_type, + doc.data, + { + title: doc.title, + backgroundColor: doc.backgroundColor, + text_fontColor: doc.text_fontColor, + _width: doc._width, + _height: doc._height, + type_collection: doc.type_collection, + _layout_fitWidth: false, + _layout_autoHeight: true, + x: doc.x, + y: doc.y, + }, + uuidv4() + ) + ); + return [{ type: 'text', text: 'Created document.' }]; + } catch (error) { + return [{ type: 'text', text: 'Error creating text document, ' + error }]; + } + } +} diff --git a/src/client/views/nodes/chatbot/tools/CreateTextDocumentTool.ts b/src/client/views/nodes/chatbot/tools/CreateTextDocumentTool.ts deleted file mode 100644 index fae78aa49..000000000 --- a/src/client/views/nodes/chatbot/tools/CreateTextDocumentTool.ts +++ /dev/null @@ -1,61 +0,0 @@ -import { v4 as uuidv4 } from 'uuid'; -import { Networking } from '../../../../Network'; -import { BaseTool } from './BaseTool'; -import { Observation } from '../types/types'; -import { ParametersType } from '../types/tool_types'; -import { DocumentOptions } from '../../../../documents/Documents'; -import { RTFCast, StrCast } from '../../../../../fields/Types'; - -const createTextDocToolParams = [ - { - name: 'text_content', - type: 'string', - description: 'The text content that the document will display', - required: true, - }, - { - name: 'title', - type: 'string', - description: 'The title of the document', - required: true, - }, - { - name: 'background_color', - type: 'string', - description: 'The background color of the document as a hex string', - required: false, - }, - { - name: 'font_color', - type: 'string', - description: 'The font color of the document as a hex string', - required: false, - }, -] as const; - -type CreateTextDocToolParamsType = typeof createTextDocToolParams; - -export class CreateTextDocTool extends BaseTool<CreateTextDocToolParamsType> { - private _addLinkedDoc: (doc_type: string, data: string, options: DocumentOptions, id: string) => void; - - constructor(addLinkedDoc: (text_content: string, data: string, options: DocumentOptions, id: string) => void) { - super( - 'createTextDoc', - 'Creates a text document with the provided content and title (and of specified other options if wanted)', - createTextDocToolParams, - 'Provide the text content and title (and optionally color) for the document.', - 'Creates a text document with the provided content and title (and of specified other options if wanted). Use if the user wants to create a textbox or text document of some sort. Can use after a search or other tool to save information.' - ); - this._addLinkedDoc = addLinkedDoc; - } - - async execute(args: ParametersType<CreateTextDocToolParamsType>): Promise<Observation[]> { - try { - console.log(RTFCast(args.text_content)); - this._addLinkedDoc('text', args.text_content, { title: args.title, backgroundColor: args.background_color, text_fontColor: args.font_color }, uuidv4()); - return [{ type: 'text', text: 'Created text document.' }]; - } catch (error) { - return [{ type: 'text', text: 'Error creating text document, ' + error }]; - } - } -} diff --git a/src/client/views/nodes/chatbot/vectorstore/Vectorstore.ts b/src/client/views/nodes/chatbot/vectorstore/Vectorstore.ts index f96f55997..cf7fa0ff3 100644 --- a/src/client/views/nodes/chatbot/vectorstore/Vectorstore.ts +++ b/src/client/views/nodes/chatbot/vectorstore/Vectorstore.ts @@ -37,14 +37,14 @@ export class Vectorstore { * @param doc_ids A function that returns a list of document IDs. */ constructor(id: string, doc_ids: () => string[]) { - const pineconeApiKey = process.env.PINECONE_API_KEY; + const pineconeApiKey = '51738e9a-bea2-4c11-b6bf-48a825e774dc'; if (!pineconeApiKey) { throw new Error('PINECONE_API_KEY is not defined.'); } // Initialize Pinecone and Cohere clients with API keys from the environment. this.pinecone = new Pinecone({ apiKey: pineconeApiKey }); - this.cohere = new CohereClient({ token: process.env.COHERE_API_KEY }); + // this.cohere = new CohereClient({ token: process.env.COHERE_API_KEY }); this._id = id; this._doc_ids = doc_ids(); this.initializeIndex(); diff --git a/src/server/flashcard/labels.py b/src/server/flashcard/labels.py new file mode 100644 index 000000000..546fc4bd3 --- /dev/null +++ b/src/server/flashcard/labels.py @@ -0,0 +1,285 @@ +import base64 +import numpy as np +import base64 +import easyocr +import sys +from PIL import Image +from io import BytesIO +import requests +import json +import numpy as np + +class BoundingBoxUtils: + """Utility class for bounding box operations and OCR result corrections.""" + + @staticmethod + def is_close(box1, box2, x_threshold=20, y_threshold=20): + """ + Determines if two bounding boxes are horizontally and vertically close. + + Parameters: + box1, box2 (list): The bounding boxes to compare. + x_threshold (int): The threshold for horizontal proximity. + y_threshold (int): The threshold for vertical proximity. + + Returns: + bool: True if boxes are close, False otherwise. + """ + horizontally_close = (abs(box1[2] - box2[0]) < x_threshold or # Right edge of box1 and left edge of box2 + abs(box2[2] - box1[0]) < x_threshold or # Right edge of box2 and left edge of box1 + abs(box1[2] - box2[2]) < x_threshold or + abs(box2[0] - box1[0]) < x_threshold) + + vertically_close = (abs(box1[3] - box2[1]) < y_threshold or # Bottom edge of box1 and top edge of box2 + abs(box2[3] - box1[1]) < y_threshold or + box1[1] == box2[1] or box1[3] == box2[3]) + + return horizontally_close and vertically_close + + @staticmethod + def adjust_bounding_box(bbox, original_text, corrected_text): + """ + Adjusts a bounding box based on differences in text length. + + Parameters: + bbox (list): The original bounding box coordinates. + original_text (str): The original text detected by OCR. + corrected_text (str): The corrected text after cleaning. + + Returns: + list: The adjusted bounding box. + """ + if not bbox or len(bbox) != 4: + return bbox + + # Adjust the x-coordinates slightly to account for text correction + x_adjustment = 5 + adjusted_bbox = [ + [bbox[0][0] + x_adjustment, bbox[0][1]], + [bbox[1][0], bbox[1][1]], + [bbox[2][0] + x_adjustment, bbox[2][1]], + [bbox[3][0], bbox[3][1]] + ] + return adjusted_bbox + + @staticmethod + def correct_ocr_results(results): + """ + Corrects common OCR misinterpretations in the detected text and adjusts bounding boxes accordingly. + + Parameters: + results (list): A list of OCR results, each containing bounding box, text, and confidence score. + + Returns: + list: Corrected OCR results with adjusted bounding boxes. + """ + corrections = { + "~": "", # Replace '~' with empty string + "-": "" # Replace '-' with empty string + } + + corrected_results = [] + for (bbox, text, prob) in results: + corrected_text = ''.join(corrections.get(char, char) for char in text) + adjusted_bbox = BoundingBoxUtils.adjust_bounding_box(bbox, text, corrected_text) + corrected_results.append((adjusted_bbox, corrected_text, prob)) + + return corrected_results + + @staticmethod + def convert_to_json_serializable(data): + """ + Converts a list containing various types, including numpy types, to a JSON-serializable format. + + Parameters: + data (list): A list containing numpy or other non-serializable types. + + Returns: + list: A JSON-serializable version of the input list. + """ + def convert_element(element): + if isinstance(element, list): + return [convert_element(e) for e in element] + elif isinstance(element, tuple): + return tuple(convert_element(e) for e in element) + elif isinstance(element, np.integer): + return int(element) + elif isinstance(element, np.floating): + return float(element) + elif isinstance(element, np.ndarray): + return element.tolist() + else: + return element + + return convert_element(data) + +class ImageLabelProcessor: + """Class to process images and perform OCR with EasyOCR.""" + + VERTICAL_THRESHOLD = 20 + HORIZONTAL_THRESHOLD = 8 + + def __init__(self, img_source, source_type, smart_mode): + self.img_source = img_source + self.source_type = source_type + self.smart_mode = smart_mode + self.img_val = self.load_image() + + def load_image(self): + """Load image from either a base64 string or URL.""" + if self.source_type == 'drag': + return self._load_base64_image() + else: + return self._load_url_image() + + def _load_base64_image(self): + """Decode and save the base64 image.""" + base64_string = self.img_source + if base64_string.startswith("data:image"): + base64_string = base64_string.split(",")[1] + + + # Decode the base64 string + image_data = base64.b64decode(base64_string) + image = Image.open(BytesIO(image_data)).convert('RGB') + image.save("temp_image.jpg") + return "temp_image.jpg" + + def _load_url_image(self): + """Download image from URL and return it in byte format.""" + url = self.img_source + response = requests.get(url) + image = Image.open(BytesIO(response.content)).convert('RGB') + + image_bytes = BytesIO() + image.save(image_bytes, format='PNG') + return image_bytes.getvalue() + + def process_image(self): + """Process the image and return the OCR results.""" + if self.smart_mode: + return self._process_smart_mode() + else: + return self._process_standard_mode() + + def _process_smart_mode(self): + """Process the image in smart mode using EasyOCR.""" + reader = easyocr.Reader(['en']) + result = reader.readtext(self.img_val, detail=1, paragraph=True) + + all_boxes = [bbox for bbox, text in result] + all_texts = [text for bbox, text in result] + + response_data = { + 'status': 'success', + 'message': 'Data received', + 'boxes': BoundingBoxUtils.convert_to_json_serializable(all_boxes), + 'text': BoundingBoxUtils.convert_to_json_serializable(all_texts), + } + + return response_data + + def _process_standard_mode(self): + """Process the image in standard mode using EasyOCR.""" + reader = easyocr.Reader(['en']) + results = reader.readtext(self.img_val) + + filtered_results = BoundingBoxUtils.correct_ocr_results([ + (bbox, text, prob) for bbox, text, prob in results if prob >= 0.7 + ]) + + return self._merge_and_prepare_response(filtered_results) + + def are_vertically_close(self, box1, box2): + """Check if two bounding boxes are vertically close.""" + box1_bottom = max(box1[2][1], box1[3][1]) + box2_top = min(box2[0][1], box2[1][1]) + vertical_distance = box2_top - box1_bottom + + box1_left = box1[0][0] + box2_left = box2[0][0] + box1_right = box1[1][0] + box2_right = box2[1][0] + hori_close = abs(box2_left - box1_left) <= self.HORIZONTAL_THRESHOLD or abs(box2_right - box1_right) <= self.HORIZONTAL_THRESHOLD + + return vertical_distance <= self.VERTICAL_THRESHOLD and hori_close + + def merge_boxes(self, boxes, texts): + """Merge multiple bounding boxes and their associated text.""" + x_coords = [] + y_coords = [] + + # Collect all x and y coordinates + for box in boxes: + for point in box: + x_coords.append(point[0]) + y_coords.append(point[1]) + + # Create the merged bounding box + merged_box = [ + [min(x_coords), min(y_coords)], + [max(x_coords), min(y_coords)], + [max(x_coords), max(y_coords)], + [min(x_coords), max(y_coords)] + ] + + # Combine the texts + merged_text = ' '.join(texts) + + return merged_box, merged_text + + def _merge_and_prepare_response(self, filtered_results): + """Merge vertically close boxes and prepare the final response.""" + current_boxes, current_texts = [], [] + all_boxes, all_texts = [], [] + + for ind in range(len(filtered_results) - 1): + if not current_boxes: + current_boxes.append(filtered_results[ind][0]) + current_texts.append(filtered_results[ind][1]) + + if self.are_vertically_close(filtered_results[ind][0], filtered_results[ind + 1][0]): + current_boxes.append(filtered_results[ind + 1][0]) + current_texts.append(filtered_results[ind + 1][1]) + else: + merged = self.merge_boxes(current_boxes, current_texts) + all_boxes.append(merged[0]) + all_texts.append(merged[1]) + current_boxes, current_texts = [], [] + + if current_boxes: + merged = self.merge_boxes(current_boxes, current_texts) + all_boxes.append(merged[0]) + all_texts.append(merged[1]) + + if not current_boxes and filtered_results: + merged = self.merge_boxes([filtered_results[-1][0]], [filtered_results[-1][1]]) + all_boxes.append(merged[0]) + all_texts.append(merged[1]) + + response = { + 'status': 'success', + 'message': 'Data received', + 'boxes': BoundingBoxUtils.convert_to_json_serializable(all_boxes), + 'text': BoundingBoxUtils.convert_to_json_serializable(all_texts), + } + + return response + +# Main execution function +def labels(): + """Main function to handle image OCR processing based on input arguments.""" + source_type = sys.argv[2] + smart_mode = (sys.argv[3] == 'smart') + with open(sys.argv[1], 'r') as f: + img_source = f.read() + # Create ImageLabelProcessor instance + processor = ImageLabelProcessor(img_source, source_type, smart_mode) + response = processor.process_image() + + # Print and return the response + print(response) + return response + + +labels() diff --git a/src/server/flashcard/requirements.txt b/src/server/flashcard/requirements.txt new file mode 100644 index 000000000..eb92a819b --- /dev/null +++ b/src/server/flashcard/requirements.txt @@ -0,0 +1,12 @@ +easyocr==1.7.1 +requests==2.32.3 +pillow==10.4.0 +numpy==1.26.4 +tqdm==4.66.4 +Werkzeug==3.0.3 +python-dateutil==2.9.0.post0 +six==1.16.0 +certifi==2024.6.2 +charset-normalizer==3.3.2 +idna==3.7 +urllib3==1.26.19
\ No newline at end of file diff --git a/src/server/flashcard/venv/pyvenv.cfg b/src/server/flashcard/venv/pyvenv.cfg new file mode 100644 index 000000000..740014e00 --- /dev/null +++ b/src/server/flashcard/venv/pyvenv.cfg @@ -0,0 +1,3 @@ +home = /Library/Frameworks/Python.framework/Versions/3.10/bin +include-system-site-packages = false +version = 3.10.11 |