From 7847717fcb18684950aa9f7640c7f2264ff3f4a1 Mon Sep 17 00:00:00 2001 From: alyssaf16 Date: Thu, 24 Oct 2024 14:23:44 -0400 Subject: create documents --- src/client/views/nodes/chatbot/agentsystem/Agent.ts | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) (limited to 'src/client/views/nodes/chatbot/agentsystem') diff --git a/src/client/views/nodes/chatbot/agentsystem/Agent.ts b/src/client/views/nodes/chatbot/agentsystem/Agent.ts index 9253175d5..23e7d4a9d 100644 --- a/src/client/views/nodes/chatbot/agentsystem/Agent.ts +++ b/src/client/views/nodes/chatbot/agentsystem/Agent.ts @@ -16,7 +16,7 @@ import { Vectorstore } from '../vectorstore/Vectorstore'; import { getReactPrompt } from './prompts'; import { BaseTool } from '../tools/BaseTool'; import { Parameter, ParametersType } from '../types/tool_types'; -import { CreateTextDocTool } from '../tools/CreateTextDocumentTool'; +import { CreateDocTool } from '../tools/CreateDocumentTool'; import { DocumentOptions } from '../../../../documents/Documents'; dotenv.config(); @@ -75,7 +75,7 @@ export class Agent { searchTool: new SearchTool(addLinkedUrlDoc), createCSV: new CreateCSVTool(createCSVInDash), noTool: new NoTool(), - createTextDoc: new CreateTextDocTool(addLinkedDoc), + createDoc: new CreateDocTool(addLinkedDoc), }; } @@ -168,6 +168,7 @@ export class Agent { } else if (key === 'action_input') { // Handle action input stage const actionInput = stage[key]; + console.log(`Action input full:`, actionInput); console.log(`Action input:`, actionInput.inputs); if (currentAction) { -- cgit v1.2.3-70-g09d2 From c358fba1ee2aa54a97373d07e7b218c74dfd9bf0 Mon Sep 17 00:00:00 2001 From: alyssaf16 Date: Tue, 12 Nov 2024 01:01:23 -0500 Subject: flashcards w assistant finally workgit add -A --- .../views/nodes/chatbot/agentsystem/Agent.ts | 4 +- .../nodes/chatbot/chatboxcomponents/ChatBox.tsx | 89 +++++-- .../nodes/chatbot/tools/CreateDocumentTool.ts | 82 +++++- .../views/nodes/chatbot/vectorstore/Vectorstore.ts | 2 +- src/server/flashcard/labels.py | 285 +++++++++++++++++++++ src/server/flashcard/requirements.txt | 12 + src/server/flashcard/venv/pyvenv.cfg | 3 + temp_image 2.jpg | Bin 0 -> 195492 bytes temp_image.jpg | Bin 0 -> 196370 bytes 9 files changed, 450 insertions(+), 27 deletions(-) create mode 100644 src/server/flashcard/labels.py create mode 100644 src/server/flashcard/requirements.txt create mode 100644 src/server/flashcard/venv/pyvenv.cfg create mode 100644 temp_image 2.jpg create mode 100644 temp_image.jpg (limited to 'src/client/views/nodes/chatbot/agentsystem') diff --git a/src/client/views/nodes/chatbot/agentsystem/Agent.ts b/src/client/views/nodes/chatbot/agentsystem/Agent.ts index 05d13d1db..0b0e211eb 100644 --- a/src/client/views/nodes/chatbot/agentsystem/Agent.ts +++ b/src/client/views/nodes/chatbot/agentsystem/Agent.ts @@ -69,9 +69,9 @@ export class Agent { // Define available tools for the assistant this.tools = { calculate: new CalculateTool(), - rag: new RAGTool(this.vectorstore), + // rag: new RAGTool(this.vectorstore), dataAnalysis: new DataAnalysisTool(csvData), - websiteInfoScraper: new WebsiteInfoScraperTool(addLinkedUrlDoc), + // websiteInfoScraper: new WebsiteInfoScraperTool(addLinkedUrlDoc), searchTool: new SearchTool(addLinkedUrlDoc), createCSV: new CreateCSVTool(createCSVInDash), noTool: new NoTool(), diff --git a/src/client/views/nodes/chatbot/chatboxcomponents/ChatBox.tsx b/src/client/views/nodes/chatbot/chatboxcomponents/ChatBox.tsx index 68d4383e7..95f3fbc5d 100644 --- a/src/client/views/nodes/chatbot/chatboxcomponents/ChatBox.tsx +++ b/src/client/views/nodes/chatbot/chatboxcomponents/ChatBox.tsx @@ -463,9 +463,11 @@ export class ChatBox extends ViewBoxAnnotatableComponent() { doc = DocCast(Docs.Create.TextDocument(data, options)); break; case 'flashcard': - // doc = this.createSingleFlashcard(data, options); doc = this.createFlashcard(data, options); break; + case 'deck': + doc = this.createDeck(data, options); + break; case 'image': doc = DocCast(Docs.Create.ImageDocument(data, options)); break; @@ -551,31 +553,82 @@ export class ChatBox extends ViewBoxAnnotatableComponent() { await DocumentManager.Instance.showDocument(doc, { willZoomCentered: true }, () => {}); }; - // TODO: DELEGATE TO DIFFERENT CLASS @action - createFlashcard = (data: string, options: DocumentOptions) => { + createDeck = (data: any, options: DocumentOptions) => { const flashcardDeck: Doc[] = []; - const parsedItems: { [key: string]: string } = JSON.parse(data); - Object.entries(parsedItems).forEach(([key, val]) => { - console.log('key' + key); - console.log('key' + val); - - const side1 = Docs.Create.CenteredTextCreator('question', key, options); - const side2 = Docs.Create.CenteredTextCreator('answer', val, options); - const doc = DocCast(Docs.Create.FlashcardDocument(data, side1, side2, { _width: 300, _height: 300 })); - this._props.addDocument?.(doc); - flashcardDeck.push(doc); + + // Parse `data` only if it’s a string + const deckData = typeof data === 'string' ? JSON.parse(data) : data; + console.log('Parsed Deck Data:', deckData); + const flashcardArray = Array.isArray(deckData) ? deckData : Object.values(deckData); + console.log(typeof flashcardArray); + // Process each flashcard document in the `deckData` array + flashcardArray.forEach(doc => { + const flashcardDoc = this.createFlashcard(doc, options); + if (flashcardDoc) flashcardDeck.push(flashcardDoc); }); - const col = DocCast( + + // Create a carousel to contain the flashcard deck + const carouselDoc = DocCast( Docs.Create.CarouselDocument(flashcardDeck, { - title: options.title, - _width: 300, - _height: 300, + title: options.title || 'Flashcard Deck', + _width: options._width || 300, + _height: options._height || 300, _layout_fitWidth: false, _layout_autoHeight: true, }) ); - return col; + + return carouselDoc; + }; + @action + createFlashcard = (data: any, options: any) => { + // const flashcardDeck: Doc[] = []; + + // Process each flashcard item in the data array + // const p = JSON.parse(data); + const deckData = typeof data === 'string' ? JSON.parse(data) : data; + const flashcardArray = Array.isArray(deckData) ? deckData : Object.values(deckData)[2]; + console.log(typeof flashcardArray); + + const [front, back] = flashcardArray; + + // Check that both front and back are text documents + console.log('DATA' + data); + console.log('front' + front); + console.log('back' + back); + console.log(front.doc_type); + console.log(back.doc_type); + if (front.doc_type === 'text' && back.doc_type === 'text') { + const sideOptions: DocumentOptions = { + backgroundColor: options.backgroundColor, + _width: options._width, + _height: options._height, + }; + + // Create front and back text documents + const side1 = Docs.Create.CenteredTextCreator(front.title, front.data, sideOptions); + const side2 = Docs.Create.CenteredTextCreator(back.title, back.data, sideOptions); + + // Create the flashcard document with both sides + const flashcardDoc = DocCast(Docs.Create.FlashcardDocument(data.title, side1, side2, sideOptions)); + return flashcardDoc; + // this._props.addDocument?.(flashcardDoc); + // flashcardDeck.push(flashcardDoc); + } + + // Create a carousel to contain the flashcard deck + // const carouselDoc = DocCast( + // Docs.Create.CarouselDocument(flashcardDeck, { + // title: options.title || data.title, + // _width: data.width || 300, + // _height: data.height || 300, + // _layout_fitWidth: false, + // _layout_autoHeight: true, + // }) + // ); + + // return carouselDoc; }; /** diff --git a/src/client/views/nodes/chatbot/tools/CreateDocumentTool.ts b/src/client/views/nodes/chatbot/tools/CreateDocumentTool.ts index b14a57779..ebe0448aa 100644 --- a/src/client/views/nodes/chatbot/tools/CreateDocumentTool.ts +++ b/src/client/views/nodes/chatbot/tools/CreateDocumentTool.ts @@ -5,17 +5,86 @@ import { ParametersType } from '../types/tool_types'; import { DocumentOptions } from '../../../../documents/Documents'; const example = [ + { + doc_type: 'deck', + title: 'Chemistry', + data: [ + { + doc_type: 'flashcard', + title: 'Photosynthesis', + data: [ + { + doc_type: 'text', + title: 'front_Photosynthesis', + data: 'What is photosynthesis?', + width: 300, + height: 300, + }, + { + doc_type: 'text', + title: 'back_photosynthesis', + data: 'The process by which plants make food.', + width: 300, + height: 300, + }, + ], + backgroundColor: '#00ff00', + width: 300, + height: 300, + }, + { + doc_type: 'flashcard', + title: 'Photosynthesis', + data: [ + { + doc_type: 'text', + title: 'front_Photosynthesis', + data: 'What is photosynthesis?', + width: 300, + height: 300, + }, + { + doc_type: 'text', + title: 'back_photosynthesis', + data: 'The process by which plants make food.', + width: 300, + height: 300, + }, + ], + backgroundColor: '#00ff00', + width: 300, + height: 300, + }, + ], + backgroundColor: '#00ff00', + width: 600, + height: 600, + }, + { + doc_type: 'web', + title: 'Brown University Wikipedia', + data: 'https://en.wikipedia.org/wiki/Brown_University', + width: 300, + height: 300, + }, { doc_type: 'collection', title: 'Science Collection', data: [ + { + doc_type: 'web', + title: 'Brown University Wikipedia', + data: 'https://en.wikipedia.org/wiki/Brown_University', + width: 300, + height: 300, + }, { doc_type: 'flashcard', title: 'Photosynthesis', data: [ { doc_type: 'text', - title: 'Front Photosynthesis', + title: 'front_Photosynthesis', data: 'What is photosynthesis?', width: 300, height: 300, @@ -72,9 +141,9 @@ const docInstructions = { }, text: 'Provide text content as a plain string. Example: "This is a standalone text document."', flashcard: 'Two text documents with content for the front and back.', - flashcardDeck: 'A collection of flashcards under a common theme.', + deck: 'A decks data is an array of flashcards.', image: 'A URL to an image for data. Example: "https://example.com/image.jpg"', - web: 'A URL to a webpage. Example: "https://example.com"', + web: 'A URL to a webpage. Example: https://en.wikipedia.org/wiki/Brown_University', equation: 'Create a equation document.', noteboard: 'Create a noteboard document', comparison: 'Create a comparison document', @@ -148,12 +217,13 @@ export class CreateDocTool extends BaseTool { constructor(addLinkedDoc: (doc_type: string, data: string, options: DocumentOptions, id: string) => void) { super( 'createDoc', - 'Creates one or more documents that best fit users request', + 'Creates one or more documents that best fit users request with input following the example below. Or creates a dashboard of many documents/collections.', createListDocToolParams, - 'Modify the data parameter and include title (and optionally color) for the document.', + 'Modify the data parameter and include title (and optionally color) for the document. Web doc data type must be url from search tool.', 'Creates one or more documents represented by an array of strings with the provided content based on the instructions ' + docInstructions + - 'Use if the user wants to create something that aligns with a document type in dash like a flashcard, flashcard deck/stack, or textbox or text document of some sort. Can use after a search or other tool to save information.' + 'Use if the user wants to create something that aligns with a document type in dash like a flashcard, flashcard deck/stack, or textbox or text document of some sort. Can use after the search tool to save information.' + + 'When user asks for dashboard, create many documents/collections with different colors and texts while listening to their preferences, after using search tool to create a dashboard.' ); this._addLinkedDoc = addLinkedDoc; } diff --git a/src/client/views/nodes/chatbot/vectorstore/Vectorstore.ts b/src/client/views/nodes/chatbot/vectorstore/Vectorstore.ts index 5ed784559..cf7fa0ff3 100644 --- a/src/client/views/nodes/chatbot/vectorstore/Vectorstore.ts +++ b/src/client/views/nodes/chatbot/vectorstore/Vectorstore.ts @@ -44,7 +44,7 @@ export class Vectorstore { // Initialize Pinecone and Cohere clients with API keys from the environment. this.pinecone = new Pinecone({ apiKey: pineconeApiKey }); - this.cohere = new CohereClient({ token: process.env.COHERE_API_KEY }); + // this.cohere = new CohereClient({ token: process.env.COHERE_API_KEY }); this._id = id; this._doc_ids = doc_ids(); this.initializeIndex(); diff --git a/src/server/flashcard/labels.py b/src/server/flashcard/labels.py new file mode 100644 index 000000000..546fc4bd3 --- /dev/null +++ b/src/server/flashcard/labels.py @@ -0,0 +1,285 @@ +import base64 +import numpy as np +import base64 +import easyocr +import sys +from PIL import Image +from io import BytesIO +import requests +import json +import numpy as np + +class BoundingBoxUtils: + """Utility class for bounding box operations and OCR result corrections.""" + + @staticmethod + def is_close(box1, box2, x_threshold=20, y_threshold=20): + """ + Determines if two bounding boxes are horizontally and vertically close. + + Parameters: + box1, box2 (list): The bounding boxes to compare. + x_threshold (int): The threshold for horizontal proximity. + y_threshold (int): The threshold for vertical proximity. + + Returns: + bool: True if boxes are close, False otherwise. + """ + horizontally_close = (abs(box1[2] - box2[0]) < x_threshold or # Right edge of box1 and left edge of box2 + abs(box2[2] - box1[0]) < x_threshold or # Right edge of box2 and left edge of box1 + abs(box1[2] - box2[2]) < x_threshold or + abs(box2[0] - box1[0]) < x_threshold) + + vertically_close = (abs(box1[3] - box2[1]) < y_threshold or # Bottom edge of box1 and top edge of box2 + abs(box2[3] - box1[1]) < y_threshold or + box1[1] == box2[1] or box1[3] == box2[3]) + + return horizontally_close and vertically_close + + @staticmethod + def adjust_bounding_box(bbox, original_text, corrected_text): + """ + Adjusts a bounding box based on differences in text length. + + Parameters: + bbox (list): The original bounding box coordinates. + original_text (str): The original text detected by OCR. + corrected_text (str): The corrected text after cleaning. + + Returns: + list: The adjusted bounding box. + """ + if not bbox or len(bbox) != 4: + return bbox + + # Adjust the x-coordinates slightly to account for text correction + x_adjustment = 5 + adjusted_bbox = [ + [bbox[0][0] + x_adjustment, bbox[0][1]], + [bbox[1][0], bbox[1][1]], + [bbox[2][0] + x_adjustment, bbox[2][1]], + [bbox[3][0], bbox[3][1]] + ] + return adjusted_bbox + + @staticmethod + def correct_ocr_results(results): + """ + Corrects common OCR misinterpretations in the detected text and adjusts bounding boxes accordingly. + + Parameters: + results (list): A list of OCR results, each containing bounding box, text, and confidence score. + + Returns: + list: Corrected OCR results with adjusted bounding boxes. + """ + corrections = { + "~": "", # Replace '~' with empty string + "-": "" # Replace '-' with empty string + } + + corrected_results = [] + for (bbox, text, prob) in results: + corrected_text = ''.join(corrections.get(char, char) for char in text) + adjusted_bbox = BoundingBoxUtils.adjust_bounding_box(bbox, text, corrected_text) + corrected_results.append((adjusted_bbox, corrected_text, prob)) + + return corrected_results + + @staticmethod + def convert_to_json_serializable(data): + """ + Converts a list containing various types, including numpy types, to a JSON-serializable format. + + Parameters: + data (list): A list containing numpy or other non-serializable types. + + Returns: + list: A JSON-serializable version of the input list. + """ + def convert_element(element): + if isinstance(element, list): + return [convert_element(e) for e in element] + elif isinstance(element, tuple): + return tuple(convert_element(e) for e in element) + elif isinstance(element, np.integer): + return int(element) + elif isinstance(element, np.floating): + return float(element) + elif isinstance(element, np.ndarray): + return element.tolist() + else: + return element + + return convert_element(data) + +class ImageLabelProcessor: + """Class to process images and perform OCR with EasyOCR.""" + + VERTICAL_THRESHOLD = 20 + HORIZONTAL_THRESHOLD = 8 + + def __init__(self, img_source, source_type, smart_mode): + self.img_source = img_source + self.source_type = source_type + self.smart_mode = smart_mode + self.img_val = self.load_image() + + def load_image(self): + """Load image from either a base64 string or URL.""" + if self.source_type == 'drag': + return self._load_base64_image() + else: + return self._load_url_image() + + def _load_base64_image(self): + """Decode and save the base64 image.""" + base64_string = self.img_source + if base64_string.startswith("data:image"): + base64_string = base64_string.split(",")[1] + + + # Decode the base64 string + image_data = base64.b64decode(base64_string) + image = Image.open(BytesIO(image_data)).convert('RGB') + image.save("temp_image.jpg") + return "temp_image.jpg" + + def _load_url_image(self): + """Download image from URL and return it in byte format.""" + url = self.img_source + response = requests.get(url) + image = Image.open(BytesIO(response.content)).convert('RGB') + + image_bytes = BytesIO() + image.save(image_bytes, format='PNG') + return image_bytes.getvalue() + + def process_image(self): + """Process the image and return the OCR results.""" + if self.smart_mode: + return self._process_smart_mode() + else: + return self._process_standard_mode() + + def _process_smart_mode(self): + """Process the image in smart mode using EasyOCR.""" + reader = easyocr.Reader(['en']) + result = reader.readtext(self.img_val, detail=1, paragraph=True) + + all_boxes = [bbox for bbox, text in result] + all_texts = [text for bbox, text in result] + + response_data = { + 'status': 'success', + 'message': 'Data received', + 'boxes': BoundingBoxUtils.convert_to_json_serializable(all_boxes), + 'text': BoundingBoxUtils.convert_to_json_serializable(all_texts), + } + + return response_data + + def _process_standard_mode(self): + """Process the image in standard mode using EasyOCR.""" + reader = easyocr.Reader(['en']) + results = reader.readtext(self.img_val) + + filtered_results = BoundingBoxUtils.correct_ocr_results([ + (bbox, text, prob) for bbox, text, prob in results if prob >= 0.7 + ]) + + return self._merge_and_prepare_response(filtered_results) + + def are_vertically_close(self, box1, box2): + """Check if two bounding boxes are vertically close.""" + box1_bottom = max(box1[2][1], box1[3][1]) + box2_top = min(box2[0][1], box2[1][1]) + vertical_distance = box2_top - box1_bottom + + box1_left = box1[0][0] + box2_left = box2[0][0] + box1_right = box1[1][0] + box2_right = box2[1][0] + hori_close = abs(box2_left - box1_left) <= self.HORIZONTAL_THRESHOLD or abs(box2_right - box1_right) <= self.HORIZONTAL_THRESHOLD + + return vertical_distance <= self.VERTICAL_THRESHOLD and hori_close + + def merge_boxes(self, boxes, texts): + """Merge multiple bounding boxes and their associated text.""" + x_coords = [] + y_coords = [] + + # Collect all x and y coordinates + for box in boxes: + for point in box: + x_coords.append(point[0]) + y_coords.append(point[1]) + + # Create the merged bounding box + merged_box = [ + [min(x_coords), min(y_coords)], + [max(x_coords), min(y_coords)], + [max(x_coords), max(y_coords)], + [min(x_coords), max(y_coords)] + ] + + # Combine the texts + merged_text = ' '.join(texts) + + return merged_box, merged_text + + def _merge_and_prepare_response(self, filtered_results): + """Merge vertically close boxes and prepare the final response.""" + current_boxes, current_texts = [], [] + all_boxes, all_texts = [], [] + + for ind in range(len(filtered_results) - 1): + if not current_boxes: + current_boxes.append(filtered_results[ind][0]) + current_texts.append(filtered_results[ind][1]) + + if self.are_vertically_close(filtered_results[ind][0], filtered_results[ind + 1][0]): + current_boxes.append(filtered_results[ind + 1][0]) + current_texts.append(filtered_results[ind + 1][1]) + else: + merged = self.merge_boxes(current_boxes, current_texts) + all_boxes.append(merged[0]) + all_texts.append(merged[1]) + current_boxes, current_texts = [], [] + + if current_boxes: + merged = self.merge_boxes(current_boxes, current_texts) + all_boxes.append(merged[0]) + all_texts.append(merged[1]) + + if not current_boxes and filtered_results: + merged = self.merge_boxes([filtered_results[-1][0]], [filtered_results[-1][1]]) + all_boxes.append(merged[0]) + all_texts.append(merged[1]) + + response = { + 'status': 'success', + 'message': 'Data received', + 'boxes': BoundingBoxUtils.convert_to_json_serializable(all_boxes), + 'text': BoundingBoxUtils.convert_to_json_serializable(all_texts), + } + + return response + +# Main execution function +def labels(): + """Main function to handle image OCR processing based on input arguments.""" + source_type = sys.argv[2] + smart_mode = (sys.argv[3] == 'smart') + with open(sys.argv[1], 'r') as f: + img_source = f.read() + # Create ImageLabelProcessor instance + processor = ImageLabelProcessor(img_source, source_type, smart_mode) + response = processor.process_image() + + # Print and return the response + print(response) + return response + + +labels() diff --git a/src/server/flashcard/requirements.txt b/src/server/flashcard/requirements.txt new file mode 100644 index 000000000..eb92a819b --- /dev/null +++ b/src/server/flashcard/requirements.txt @@ -0,0 +1,12 @@ +easyocr==1.7.1 +requests==2.32.3 +pillow==10.4.0 +numpy==1.26.4 +tqdm==4.66.4 +Werkzeug==3.0.3 +python-dateutil==2.9.0.post0 +six==1.16.0 +certifi==2024.6.2 +charset-normalizer==3.3.2 +idna==3.7 +urllib3==1.26.19 \ No newline at end of file diff --git a/src/server/flashcard/venv/pyvenv.cfg b/src/server/flashcard/venv/pyvenv.cfg new file mode 100644 index 000000000..740014e00 --- /dev/null +++ b/src/server/flashcard/venv/pyvenv.cfg @@ -0,0 +1,3 @@ +home = /Library/Frameworks/Python.framework/Versions/3.10/bin +include-system-site-packages = false +version = 3.10.11 diff --git a/temp_image 2.jpg b/temp_image 2.jpg new file mode 100644 index 000000000..05bc8db3a Binary files /dev/null and b/temp_image 2.jpg differ diff --git a/temp_image.jpg b/temp_image.jpg new file mode 100644 index 000000000..912519ce1 Binary files /dev/null and b/temp_image.jpg differ -- cgit v1.2.3-70-g09d2 From ec0ab50aad9fbb55477476998c6932488b149f45 Mon Sep 17 00:00:00 2001 From: bobzel Date: Tue, 21 Jan 2025 12:03:44 -0500 Subject: trying to cleanup chatBox code and types --- .../views/nodes/chatbot/agentsystem/Agent.ts | 15 +- .../nodes/chatbot/chatboxcomponents/ChatBox.tsx | 472 +++++++++------------ 2 files changed, 203 insertions(+), 284 deletions(-) (limited to 'src/client/views/nodes/chatbot/agentsystem') diff --git a/src/client/views/nodes/chatbot/agentsystem/Agent.ts b/src/client/views/nodes/chatbot/agentsystem/Agent.ts index a2a575f19..4d3f1e4e7 100644 --- a/src/client/views/nodes/chatbot/agentsystem/Agent.ts +++ b/src/client/views/nodes/chatbot/agentsystem/Agent.ts @@ -1,25 +1,22 @@ import dotenv from 'dotenv'; import { XMLBuilder, XMLParser } from 'fast-xml-parser'; +import { escape } from 'lodash'; // Imported escape from lodash import OpenAI from 'openai'; import { ChatCompletionMessageParam } from 'openai/resources'; -import { escape } from 'lodash'; // Imported escape from lodash +import { DocumentOptions } from '../../../../documents/Documents'; import { AnswerParser } from '../response_parsers/AnswerParser'; import { StreamedAnswerParser } from '../response_parsers/StreamedAnswerParser'; +import { BaseTool } from '../tools/BaseTool'; import { CalculateTool } from '../tools/CalculateTool'; -import { CreateCSVTool } from '../tools/CreateCSVTool'; +import { CreateAnyDocumentTool } from '../tools/CreateAnyDocTool'; +import { CreateDocTool } from '../tools/CreateDocumentTool'; import { DataAnalysisTool } from '../tools/DataAnalysisTool'; import { NoTool } from '../tools/NoTool'; -import { RAGTool } from '../tools/RAGTool'; import { SearchTool } from '../tools/SearchTool'; -import { WebsiteInfoScraperTool } from '../tools/WebsiteInfoScraperTool'; +import { Parameter, ParametersType, TypeMap } from '../types/tool_types'; import { AgentMessage, ASSISTANT_ROLE, AssistantMessage, Observation, PROCESSING_TYPE, ProcessingInfo, TEXT_TYPE } from '../types/types'; import { Vectorstore } from '../vectorstore/Vectorstore'; import { getReactPrompt } from './prompts'; -import { BaseTool } from '../tools/BaseTool'; -import { Parameter, ParametersType, TypeMap } from '../types/tool_types'; -import { CreateDocTool } from '../tools/CreateDocumentTool'; -import { DocumentOptions } from '../../../../documents/Documents'; -import { CreateAnyDocumentTool } from '../tools/CreateAnyDocTool'; dotenv.config(); diff --git a/src/client/views/nodes/chatbot/chatboxcomponents/ChatBox.tsx b/src/client/views/nodes/chatbot/chatboxcomponents/ChatBox.tsx index b89498d7d..83b50c8c6 100644 --- a/src/client/views/nodes/chatbot/chatboxcomponents/ChatBox.tsx +++ b/src/client/views/nodes/chatbot/chatboxcomponents/ChatBox.tsx @@ -14,12 +14,12 @@ import OpenAI, { ClientOptions } from 'openai'; import * as React from 'react'; import { v4 as uuidv4 } from 'uuid'; import { ClientUtils } from '../../../../../ClientUtils'; -import { Doc, DocListCast } from '../../../../../fields/Doc'; +import { Doc, DocListCast, Opt } from '../../../../../fields/Doc'; import { DocData, DocViews } from '../../../../../fields/DocSymbols'; -import { CsvCast, DocCast, PDFCast, RTFCast, StrCast, NumCast } from '../../../../../fields/Types'; +import { CsvCast, DocCast, NumCast, PDFCast, RTFCast, StrCast } from '../../../../../fields/Types'; import { Networking } from '../../../../Network'; import { DocUtils } from '../../../../documents/DocUtils'; -import { DocumentType } from '../../../../documents/DocumentTypes'; +import { CollectionViewType, DocumentType } from '../../../../documents/DocumentTypes'; import { Docs, DocumentOptions } from '../../../../documents/Documents'; import { DocumentManager } from '../../../../util/DocumentManager'; import { LinkManager } from '../../../../util/LinkManager'; @@ -33,7 +33,6 @@ import { Vectorstore } from '../vectorstore/Vectorstore'; import './ChatBox.scss'; import MessageComponentBox from './MessageComponent'; import { ProgressBar } from './ProgressBar'; -import { RichTextField } from '../../../../../fields/RichTextField'; dotenv.config(); @@ -45,17 +44,17 @@ dotenv.config(); @observer export class ChatBox extends ViewBoxAnnotatableComponent() { // MobX observable properties to track UI state and data - @observable history: AssistantMessage[] = []; - @observable.deep current_message: AssistantMessage | undefined = undefined; - @observable isLoading: boolean = false; - @observable uploadProgress: number = 0; - @observable currentStep: string = ''; - @observable expandedScratchpadIndex: number | null = null; - @observable inputValue: string = ''; - @observable private linked_docs_to_add: ObservableSet = observable.set(); - @observable private linked_csv_files: { filename: string; id: string; text: string }[] = []; - @observable private isUploadingDocs: boolean = false; - @observable private citationPopup: { text: string; visible: boolean } = { text: '', visible: false }; + @observable private _history: AssistantMessage[] = []; + @observable.deep private _current_message: AssistantMessage | undefined = undefined; + @observable private _isLoading: boolean = false; + @observable private _uploadProgress: number = 0; + @observable private _currentStep: string = ''; + @observable private _expandedScratchpadIndex: number | null = null; + @observable private _inputValue: string = ''; + @observable private _linked_docs_to_add: ObservableSet = observable.set(); + @observable private _linked_csv_files: { filename: string; id: string; text: string }[] = []; + @observable private _isUploadingDocs: boolean = false; + @observable private _citationPopup: { text: string; visible: boolean } = { text: '', visible: false }; // Private properties for managing OpenAI API, vector store, agent, and UI elements private openai: OpenAI; @@ -96,7 +95,7 @@ export class ChatBox extends ViewBoxAnnotatableComponent() { // Reaction to update dataDoc when chat history changes reaction( () => - this.history.map((msg: AssistantMessage) => ({ + this._history.map((msg: AssistantMessage) => ({ role: msg.role, content: msg.content, follow_up_questions: msg.follow_up_questions, @@ -115,20 +114,20 @@ export class ChatBox extends ViewBoxAnnotatableComponent() { */ @action addDocToVectorstore = async (newLinkedDoc: Doc) => { - this.uploadProgress = 0; - this.currentStep = 'Initializing...'; - this.isUploadingDocs = true; + this._uploadProgress = 0; + this._currentStep = 'Initializing...'; + this._isUploadingDocs = true; try { // Add the document to the vectorstore await this.vectorstore.addAIDoc(newLinkedDoc, this.updateProgress); } catch (error) { console.error('Error uploading document:', error); - this.currentStep = 'Error during upload'; + this._currentStep = 'Error during upload'; } finally { - this.isUploadingDocs = false; - this.uploadProgress = 0; - this.currentStep = ''; + this._isUploadingDocs = false; + this._uploadProgress = 0; + this._currentStep = ''; } }; @@ -139,8 +138,8 @@ export class ChatBox extends ViewBoxAnnotatableComponent() { */ @action updateProgress = (progress: number, step: string) => { - this.uploadProgress = progress; - this.currentStep = step; + this._uploadProgress = progress; + this._currentStep = step; }; /** @@ -177,7 +176,7 @@ export class ChatBox extends ViewBoxAnnotatableComponent() { const csvId = id ?? uuidv4(); // Add CSV details to linked files - this.linked_csv_files.push({ + this._linked_csv_files.push({ filename: CsvCast(newLinkedDoc.data).url.pathname, id: csvId, text: csvData, @@ -199,7 +198,7 @@ export class ChatBox extends ViewBoxAnnotatableComponent() { */ @action toggleToolLogs = (index: number) => { - this.expandedScratchpadIndex = this.expandedScratchpadIndex === index ? null : index; + this._expandedScratchpadIndex = this._expandedScratchpadIndex === index ? null : index; }; /** @@ -258,7 +257,7 @@ export class ChatBox extends ViewBoxAnnotatableComponent() { @action askGPT = async (event: React.FormEvent): Promise => { event.preventDefault(); - this.inputValue = ''; + this._inputValue = ''; // Extract the user's message const textInput = (event.currentTarget as HTMLFormElement).elements.namedItem('messageInput') as HTMLInputElement; @@ -268,13 +267,13 @@ export class ChatBox extends ViewBoxAnnotatableComponent() { try { textInput.value = ''; // Add the user's message to the history - this.history.push({ + this._history.push({ role: ASSISTANT_ROLE.USER, content: [{ index: 0, type: TEXT_TYPE.NORMAL, text: trimmedText, citation_ids: null }], processing_info: [], }); - this.isLoading = true; - this.current_message = { + this._isLoading = true; + this._current_message = { role: ASSISTANT_ROLE.ASSISTANT, content: [], citations: [], @@ -284,9 +283,9 @@ export class ChatBox extends ViewBoxAnnotatableComponent() { // Define callbacks for real-time processing updates const onProcessingUpdate = (processingUpdate: ProcessingInfo[]) => { runInAction(() => { - if (this.current_message) { - this.current_message = { - ...this.current_message, + if (this._current_message) { + this._current_message = { + ...this._current_message, processing_info: processingUpdate, }; } @@ -296,9 +295,9 @@ export class ChatBox extends ViewBoxAnnotatableComponent() { const onAnswerUpdate = (answerUpdate: string) => { runInAction(() => { - if (this.current_message) { - this.current_message = { - ...this.current_message, + if (this._current_message) { + this._current_message = { + ...this._current_message, content: [{ text: answerUpdate, type: TEXT_TYPE.NORMAL, index: 0, citation_ids: [] }], }; } @@ -310,22 +309,22 @@ export class ChatBox extends ViewBoxAnnotatableComponent() { // Update the history with the final assistant message runInAction(() => { - if (this.current_message) { - this.history.push({ ...finalMessage }); - this.current_message = undefined; - this.dataDoc.data = JSON.stringify(this.history); + if (this._current_message) { + this._history.push({ ...finalMessage }); + this._current_message = undefined; + this.dataDoc.data = JSON.stringify(this._history); } }); } catch (err) { console.error('Error:', err); // Handle error in processing - this.history.push({ + this._history.push({ role: ASSISTANT_ROLE.ASSISTANT, content: [{ index: 0, type: TEXT_TYPE.ERROR, text: 'Sorry, I encountered an error while processing your request.', citation_ids: null }], processing_info: [], }); } finally { - this.isLoading = false; + this._isLoading = false; this.scrollToBottom(); } } @@ -339,8 +338,8 @@ export class ChatBox extends ViewBoxAnnotatableComponent() { */ @action updateMessageCitations = (index: number, citations: Citation[]) => { - if (this.history[index]) { - this.history[index].citations = citations; + if (this._history[index]) { + this._history[index].citations = citations; } }; @@ -381,17 +380,14 @@ export class ChatBox extends ViewBoxAnnotatableComponent() { * @param data The CSV data content. */ @action - createCSVInDash = async (url: string, title: string, id: string, data: string) => { - const doc = DocCast(await DocUtils.DocumentFromType('csv', url, { title: title, text: RTFCast(data) })); - - const linkDoc = Docs.Create.LinkDocument(this.Document, doc); - LinkManager.Instance.addLink(linkDoc); - - doc && this._props.addDocument?.(doc); - await DocumentManager.Instance.showDocument(doc, { willZoomCentered: true }, () => {}); - - this.addCSVForAnalysis(doc, id); - }; + createCSVInDash = (url: string, title: string, id: string, data: string) => + DocUtils.DocumentFromType('csv', url, { title: title, text: RTFCast(data) }).then(doc => { + if (doc) { + LinkManager.Instance.addLink(Docs.Create.LinkDocument(this.Document, doc)); + this._props.addDocument?.(doc); + DocumentManager.Instance.showDocument(doc, { willZoomCentered: true }, () => {}).then(() => this.addCSVForAnalysis(doc, id)); + } + }); /** * Creates a text document in the dashboard and adds it for analysis. @@ -401,40 +397,35 @@ export class ChatBox extends ViewBoxAnnotatableComponent() { * @param id The unique ID for the document. */ @action - private createCollectionWithChildren = async (data: any, insideCol: boolean): Promise => { - // Create an array of promises for each document - const childDocPromises = data.map(async doc => { - const parsedDoc = doc; - if (parsedDoc.doc_type !== 'collection') { - // Handle non-collection documents - return await this.whichDoc(parsedDoc.doc_type, parsedDoc.data, { backgroundColor: parsedDoc.backgroundColor, _width: parsedDoc.width, _height: parsedDoc.height }, parsedDoc.id, insideCol); - } else { - // Recursively process collections - const nestedDocs = await this.createCollectionWithChildren(parsedDoc.data, true); - const collectionOptions: DocumentOptions = { - title: parsedDoc.title, - backgroundColor: parsedDoc.backgroundColor, - _width: parsedDoc.width, - _height: parsedDoc.height, - _layout_fitWidth: true, - _freeform_backgroundGrid: true, - }; - const collectionDoc = DocCast(Docs.Create.FreeformDocument(nestedDocs, collectionOptions)); - return collectionDoc; - } - }); - - // Await all child document creations concurrently - const nestedResults = await Promise.all(childDocPromises); - // Flatten any nested arrays from recursive collection calls - const childDocs = nestedResults.flat() as Doc[]; - childDocs.forEach(doc => { - console.log(DocCast(doc)); - console.log(DocCast(doc)[DocData].data); - console.log(DocCast(doc)[DocData].data); - }); - return childDocs; - }; + private createCollectionWithChildren = (data: { doc_type: string; id: string; data: any; title: string; width: number; height: number; backgroundColor: string }[], insideCol: boolean): Promise => + Promise.all( + data.map(doc => + doc.doc_type !== 'collection' // Handle non-collection documents + ? this.whichDoc(doc.doc_type, doc.data, { backgroundColor: doc.backgroundColor, _width: doc.width, _height: doc.height }, doc.id, insideCol) + : // Recursively process collections + this.createCollectionWithChildren(doc.data, true).then(nestedDocs => + Docs.Create.FreeformDocument(nestedDocs, { + title: doc.title, + backgroundColor: doc.backgroundColor, + _width: doc.width, + _height: doc.height, + _layout_fitWidth: true, + _freeform_backgroundGrid: true, + }) + ) + ) + .flat() // prettier-ignore + ).then(childDocs => childDocs.filter(doc => doc).map(doc => doc!)); + // .then(nestedResults => { + // // Flatten any nested arrays from recursive collection calls + // const childDocs = nestedResults.flat() as Doc[]; + // childDocs.forEach(doc => { + // console.log(DocCast(doc)); + // console.log(DocCast(doc)[DocData].data); + // console.log(DocCast(doc)[DocData].data); + // }); + // return childDocs; + // }); // @action // createSingleFlashcard = (data: any, options: DocumentOptions) => { @@ -442,101 +433,53 @@ export class ChatBox extends ViewBoxAnnotatableComponent() { // } @action - whichDoc = async (doc_type: string, data: string, options: DocumentOptions, id: string, insideCol: boolean): Promise => { - let doc; - switch (doc_type) { - case 'text': - doc = DocCast(Docs.Create.TextDocument(data, options)); - break; - case 'flashcard': - doc = this.createFlashcard(data, options); - break; - case 'deck': - doc = this.createDeck(data, options); - break; - case 'image': - doc = DocCast(Docs.Create.ImageDocument(data, options)); - break; - case 'equation': - doc = DocCast(Docs.Create.EquationDocument(data, options)); - break; - case 'noteboard': - doc = DocCast(Docs.Create.NoteTakingDocument([], options)); - break; - case 'simulation': - doc = DocCast(Docs.Create.SimulationDocument(options)); - break; - case 'collection': { - const arr = await this.createCollectionWithChildren(data, true); - options._layout_fitWidth = true; - options._freeform_backgroundGrid = true; - if (options.type_collection == 'tree') { - doc = DocCast(Docs.Create.TreeDocument(arr, options)); - } else if (options.type_collection == 'masonry') { - doc = DocCast(Docs.Create.MasonryDocument(arr, options)); - } else if (options.type_collection == 'card') { - doc = DocCast(Docs.Create.CardDeckDocument(arr, options)); - } else if (options.type_collection == 'carousel') { - doc = DocCast(Docs.Create.CarouselDocument(arr, options)); - } else if (options.type_collection == '3d-carousel') { - doc = DocCast(Docs.Create.Carousel3DDocument(arr, options)); - } else if (options.type_collection == 'multicolumn') { - doc = DocCast(Docs.Create.CarouselDocument(arr, options)); - } else { - doc = DocCast(Docs.Create.FreeformDocument(arr, options)); - } - break; + whichDoc = (doc_type: string, data: string, options: DocumentOptions, id: string, insideCol: boolean): Promise> => + (async () => { + switch (doc_type) { + case 'text': return Docs.Create.TextDocument(data, options); + case 'flashcard': return this.createFlashcard(data, options); + case 'deck': return this.createDeck(data, options); + case 'image': return Docs.Create.ImageDocument(data, options); + case 'equation': return Docs.Create.EquationDocument(data, options); + case 'noteboard': return Docs.Create.NoteTakingDocument([], options); + case 'simulation': return Docs.Create.SimulationDocument(options); + case 'collection': return this.createCollectionWithChildren(data as any, true). + then((arr, collOpts = { ...options, _layout_fitWidth: true, _freeform_backgroundGrid: true }) => + (() => { + switch (options.type_collection) { + case CollectionViewType.Tree: return Docs.Create.TreeDocument(arr, collOpts); + case CollectionViewType.Masonry: return Docs.Create.MasonryDocument(arr, collOpts); + case CollectionViewType.Card: return Docs.Create.CardDeckDocument(arr, collOpts); + case CollectionViewType.Carousel: return Docs.Create.CarouselDocument(arr, collOpts); + case CollectionViewType.Carousel3D: return Docs.Create.Carousel3DDocument(arr, collOpts); + case CollectionViewType.Multicolumn: return Docs.Create.CarouselDocument(arr, collOpts); + default: return Docs.Create.FreeformDocument(arr, collOpts); + } + })() + ); + case 'web': return Docs.Create.WebDocument(data, { ...options, data_useCors: true }); + case 'comparison': return this.createComparison(data, options); + case 'diagram': return Docs.Create.DiagramDocument(options); + case 'audio': return Docs.Create.AudioDocument(data, options); + case 'map': return Docs.Create.MapDocument([], options); + case 'screengrab': return Docs.Create.ScreenshotDocument(options); + case 'webcam': return Docs.Create.WebCamDocument('', options); + case 'button': return Docs.Create.ButtonDocument(options); + case 'script': return Docs.Create.ScriptingDocument(null, options); + case 'dataviz': return Docs.Create.DataVizDocument('/users/rz/Downloads/addresses.csv', options); + case 'chat': return Docs.Create.ChatDocument(options); + case 'trail': return Docs.Create.PresDocument(options); + case 'tab': return Docs.Create.FreeformDocument([], options); + case 'slide': return Docs.Create.TreeDocument([], options); + default: return Docs.Create.TextDocument(data, options); + } // prettier-ignore + })().then(doc => { + if (doc) { + doc.x = NumCast((options.x as number) ?? 0) + (insideCol ? 0 : NumCast(this.layoutDoc.x) + NumCast(this.layoutDoc.width)) + 100; + doc.y = NumCast(options.y as number) + (insideCol ? 0 : NumCast(this.layoutDoc.y)); } - case 'web': - options.data_useCors = true; - doc = DocCast(Docs.Create.WebDocument(data, options)); - break; - case 'comparison': - doc = this.createComparison(data, options); - break; - case 'diagram': - doc = Docs.Create.DiagramDocument(options); - break; - case 'audio': - doc = Docs.Create.AudioDocument(data, options); - break; - case 'map': - doc = Docs.Create.MapDocument([], options); - break; - case 'screengrab': - doc = Docs.Create.ScreenshotDocument(options); - break; - case 'webcam': - doc = Docs.Create.WebCamDocument('', options); - break; - case 'button': - doc = Docs.Create.ButtonDocument(options); - break; - case 'script': - doc = Docs.Create.ScriptingDocument(null, options); - break; - case 'dataviz': - doc = Docs.Create.DataVizDocument('/users/rz/Downloads/addresses.csv', options); - break; - case 'chat': - doc = Docs.Create.ChatDocument(options); - break; - case 'trail': - doc = Docs.Create.PresDocument(options); - break; - case 'tab': - doc = Docs.Create.FreeformDocument([], options); - break; - case 'slide': - doc = Docs.Create.TreeDocument([], options); - break; - default: - doc = DocCast(Docs.Create.TextDocument(data, options)); - } - doc!.x = NumCast(options.x ?? 0) + (insideCol ? 0 : NumCast(this.layoutDoc.x) + NumCast(this.layoutDoc.width)) + 100; - doc!.y = NumCast(options.y) + (insideCol ? 0 : NumCast(this.layoutDoc.y)); - return doc; - }; + return doc; + }); /** * Creates a document in the dashboard. @@ -548,57 +491,42 @@ export class ChatBox extends ViewBoxAnnotatableComponent() { * @returns {Promise} A promise that resolves once the document is created and displayed. */ @action - createDocInDash = async (doc_type: string, data: string | undefined, options: DocumentOptions, id: string) => { - let doc; - - switch (doc_type.toLowerCase()) { - case 'text': - doc = Docs.Create.TextDocument(data || '', options); - break; - case 'image': - doc = Docs.Create.ImageDocument(data || '', options); - break; - case 'pdf': - doc = Docs.Create.PdfDocument(data || '', options); - break; - case 'video': - doc = Docs.Create.VideoDocument(data || '', options); - break; - case 'audio': - doc = Docs.Create.AudioDocument(data || '', options); - break; - case 'web': - doc = Docs.Create.WebDocument(data || '', options); - break; - case 'equation': - doc = Docs.Create.EquationDocument(data || '', options); - break; - case 'functionplot': - case 'function_plot': - doc = Docs.Create.FunctionPlotDocument([], options); - break; - case 'dataviz': - case 'data_viz': { - const { fileUrl, id } = await Networking.PostToServer('/createCSV', { - filename: (options.title as string).replace(/\s+/g, '') + '.csv', - data: data, - }); - doc = Docs.Create.DataVizDocument(fileUrl, { ...options, text: RTFCast(data) }); - this.addCSVForAnalysis(doc, id); - break; + createDocInDash = (doc_type: string, data: string | undefined, options: DocumentOptions, id: string) => { + const linkAndShowDoc = (doc: Opt) => { + if (doc) { + LinkManager.Instance.addLink(Docs.Create.LinkDocument(this.Document, doc)); + this._props.addDocument?.(doc); + DocumentManager.Instance.showDocument(doc, { willZoomCentered: true }, () => {}); } - case 'chat': - doc = Docs.Create.ChatDocument(options); - break; - // Add more cases for other document types - default: - console.error('Unknown or unsupported document type:', doc_type); - return; - } - const linkDoc = Docs.Create.LinkDocument(this.Document, doc); - LinkManager.Instance.addLink(linkDoc); - doc && this._props.addDocument?.(doc); - await DocumentManager.Instance.showDocument(doc, { willZoomCentered: true }, () => {}); + }; + const doc = (() => { + switch (doc_type.toLowerCase()) { + case 'text': return Docs.Create.TextDocument(data || '', options); + case 'image': return Docs.Create.ImageDocument(data || '', options); + case 'pdf': return Docs.Create.PdfDocument(data || '', options); + case 'video': return Docs.Create.VideoDocument(data || '', options); + case 'audio': return Docs.Create.AudioDocument(data || '', options); + case 'web': return Docs.Create.WebDocument(data || '', options); + case 'equation': return Docs.Create.EquationDocument(data || '', options); + case 'chat': return Docs.Create.ChatDocument(options); + case 'functionplot': + case 'function_plot': return Docs.Create.FunctionPlotDocument([], options); + case 'dataviz': + case 'data_viz': Networking.PostToServer('/createCSV', { + filename: (options.title as string).replace(/\s+/g, '') + '.csv', + data: data, + })?.then(({ fileUrl, id }) => { + const vdoc = Docs.Create.DataVizDocument(fileUrl, { ...options, text: RTFCast(data) }); + this.addCSVForAnalysis(vdoc, id); + linkAndShowDoc(vdoc); + }); + return undefined; + // Add more cases for other document types + default: console.error('Unknown or unsupported document type:', doc_type); + return undefined; + } // prettier-ignore + })(); + if (doc) linkAndShowDoc(doc); }; /** @@ -625,16 +553,13 @@ export class ChatBox extends ViewBoxAnnotatableComponent() { } // Create a carousel to contain the flashcard deck - const carouselDoc = DocCast( - Docs.Create.CarouselDocument(flashcardDeck, { - title: options.title || 'Flashcard Deck', - _width: options._width || 300, - _height: options._height || 300, - _layout_fitWidth: false, - _layout_autoHeight: true, - }) - ); - return carouselDoc; + return Docs.Create.CarouselDocument(flashcardDeck, { + title: options.title || 'Flashcard Deck', + _width: options._width || 300, + _height: options._height || 300, + _layout_fitWidth: false, + _layout_autoHeight: true, + }); }; /** @@ -662,8 +587,7 @@ export class ChatBox extends ViewBoxAnnotatableComponent() { const side2 = Docs.Create.CenteredTextCreator(back.title, back.data, sideOptions); // Create the flashcard document with both sides - const flashcardDoc = DocCast(Docs.Create.FlashcardDocument(data.title, side1, side2, sideOptions)); - return flashcardDoc; + return Docs.Create.FlashcardDocument(data.title, side1, side2, sideOptions); } }; @@ -675,15 +599,14 @@ export class ChatBox extends ViewBoxAnnotatableComponent() { * @returns {Doc} The created comparison document. */ @action - createComparison = (doc: any, options: any) => { - const comp = Docs.Create.ComparisonDocument(options.title, { _width: options.width, _height: options.height | 300, backgroundColor: options.backgroundColor }); - const [left, right] = doc; - const docLeft = DocCast(Docs.Create.TextDocument(left.data, { backgroundColor: left.backgroundColor, _width: left.width, _height: left.height })); - const docRight = DocCast(Docs.Create.TextDocument(right.data, { backgroundColor: right.backgroundColor, _width: right.width, _height: right.height })); - comp[DocData].data_back = docLeft; - comp[DocData].data_front = docRight; - return comp; - }; + createComparison = (doc: { left: any; right: any }, options: any) => + Docs.Create.ComparisonDocument(options.title, { + data_back: Docs.Create.TextDocument(doc.left.data, { backgroundColor: doc.left.backgroundColor, _width: doc.left.width, _height: doc.left.height }), + data_front: Docs.Create.TextDocument(doc.right.data, { backgroundColor: doc.right.backgroundColor, _width: doc.right.width, _height: doc.right.height }), + _width: options.width, + _height: options.height | 300, + backgroundColor: options.backgroundColor, + }); /** * Event handler to manage citations click in the message components. @@ -692,7 +615,6 @@ export class ChatBox extends ViewBoxAnnotatableComponent() { @action handleCitationClick = (citation: Citation) => { const currentLinkedDocs: Doc[] = this.linkedDocs; - const chunkId = citation.chunk_id; // Loop through the linked documents to find the matching chunk and handle its display @@ -727,8 +649,8 @@ export class ChatBox extends ViewBoxAnnotatableComponent() { } break; case CHUNK_TYPE.TEXT: - this.citationPopup = { text: citation.direct_text ?? 'No text available', visible: true }; - setTimeout(() => (this.citationPopup.visible = false), 3000); // Hide after 3 seconds + this._citationPopup = { text: citation.direct_text ?? 'No text available', visible: true }; + setTimeout(() => (this._citationPopup.visible = false), 3000); // Hide after 3 seconds DocumentManager.Instance.showDocument(doc, { willZoomCentered: true }, () => { const firstView = Array.from(doc[DocViews])[0] as DocumentView; @@ -796,7 +718,7 @@ export class ChatBox extends ViewBoxAnnotatableComponent() { try { const storedHistory = JSON.parse(StrCast(this.dataDoc.data)); runInAction(() => { - this.history.push( + this._history.push( ...storedHistory.map((msg: AssistantMessage) => ({ role: msg.role, content: msg.content, @@ -811,7 +733,7 @@ export class ChatBox extends ViewBoxAnnotatableComponent() { } else { // Default welcome message runInAction(() => { - this.history.push({ + this._history.push({ role: ASSISTANT_ROLE.ASSISTANT, content: [ { @@ -835,11 +757,11 @@ export class ChatBox extends ViewBoxAnnotatableComponent() { .filter(d => d); return linkedDocs; }, - linked => linked.forEach(doc => this.linked_docs_to_add.add(doc)) + linked => linked.forEach(doc => this._linked_docs_to_add.add(doc)) ); // Observe changes to linked documents and handle document addition - observe(this.linked_docs_to_add, change => { + observe(this._linked_docs_to_add, change => { if (change.type === 'add') { if (PDFCast(change.newValue.data)) { this.addDocToVectorstore(change.newValue); @@ -913,7 +835,7 @@ export class ChatBox extends ViewBoxAnnotatableComponent() { * Getter that retrieves all linked CSV files for analysis. */ @computed get linkedCSVs(): { filename: string; id: string; text: string }[] { - return this.linked_csv_files; + return this._linked_csv_files; } /** @@ -921,7 +843,7 @@ export class ChatBox extends ViewBoxAnnotatableComponent() { */ @computed get formattedHistory(): string { let history = '\n'; - for (const message of this.history) { + for (const message of this._history) { history += `<${message.role}>${message.content.map(content => content.text).join(' ')}`; if (message.loop_summary) { history += `${message.loop_summary}`; @@ -957,7 +879,7 @@ export class ChatBox extends ViewBoxAnnotatableComponent() { */ @action handleFollowUpClick = (question: string) => { - this.inputValue = question; + this._inputValue = question; }; /** @@ -966,11 +888,11 @@ export class ChatBox extends ViewBoxAnnotatableComponent() { render() { return (
- {this.isUploadingDocs && ( + {this._isUploadingDocs && (
-
{this.currentStep}
+
{this._currentStep}
)} @@ -978,18 +900,18 @@ export class ChatBox extends ViewBoxAnnotatableComponent() {

{this.userName()}'s AI Assistant

- {this.history.map((message, index) => ( + {this._history.map((message, index) => ( ))} - {this.current_message && ( - + {this._current_message && ( + )}
- (this.inputValue = e.target.value)} disabled={this.isLoading} /> -
{/* Popup for citation */} - {this.citationPopup.visible && ( + {this._citationPopup.visible && (

- Text from your document: {this.citationPopup.text} + Text from your document: {this._citationPopup.text}

)} -- cgit v1.2.3-70-g09d2 From d72977ad8b67f2575cad8aea988fcfa7c04f794a Mon Sep 17 00:00:00 2001 From: bobzel Date: Tue, 21 Jan 2025 18:13:39 -0500 Subject: more attempts to cleanup typing, etc in chat box --- src/client/documents/DocumentTypes.ts | 2 +- src/client/documents/Documents.ts | 1 - .../views/nodes/chatbot/agentsystem/Agent.ts | 13 +- .../nodes/chatbot/chatboxcomponents/ChatBox.tsx | 114 ++++++++-------- .../views/nodes/chatbot/tools/CreateAnyDocTool.ts | 145 +++++++++------------ .../nodes/chatbot/tools/CreateDocumentTool.ts | 33 ++--- 6 files changed, 145 insertions(+), 163 deletions(-) (limited to 'src/client/views/nodes/chatbot/agentsystem') diff --git a/src/client/documents/DocumentTypes.ts b/src/client/documents/DocumentTypes.ts index efe73fbbe..8aa844c0b 100644 --- a/src/client/documents/DocumentTypes.ts +++ b/src/client/documents/DocumentTypes.ts @@ -26,7 +26,7 @@ export enum DocumentType { SCRIPTING = 'script', // script editor CHAT = 'chat', // chat with GPT about files EQUATION = 'equation', // equation editor - FUNCPLOT = 'funcplot', // function plotter + FUNCPLOT = 'function plot', // function plotter MAP = 'map', DATAVIZ = 'dataviz', ANNOPALETTE = 'annopalette', diff --git a/src/client/documents/Documents.ts b/src/client/documents/Documents.ts index 0bff74ac1..7f1387ff8 100644 --- a/src/client/documents/Documents.ts +++ b/src/client/documents/Documents.ts @@ -19,7 +19,6 @@ import { DocServer } from '../DocServer'; import { dropActionType } from '../util/DropActionTypes'; import { CollectionViewType, DocumentType } from './DocumentTypes'; import { Id } from '../../fields/FieldSymbols'; -import { FireflyImageData } from '../views/smartdraw/FireflyConstants'; class EmptyBox { public static LayoutString() { diff --git a/src/client/views/nodes/chatbot/agentsystem/Agent.ts b/src/client/views/nodes/chatbot/agentsystem/Agent.ts index 4d3f1e4e7..ee91ccb92 100644 --- a/src/client/views/nodes/chatbot/agentsystem/Agent.ts +++ b/src/client/views/nodes/chatbot/agentsystem/Agent.ts @@ -8,7 +8,7 @@ import { AnswerParser } from '../response_parsers/AnswerParser'; import { StreamedAnswerParser } from '../response_parsers/StreamedAnswerParser'; import { BaseTool } from '../tools/BaseTool'; import { CalculateTool } from '../tools/CalculateTool'; -import { CreateAnyDocumentTool } from '../tools/CreateAnyDocTool'; +import { CreateAnyDocumentTool, supportedDocumentTypes } from '../tools/CreateAnyDocTool'; import { CreateDocTool } from '../tools/CreateDocumentTool'; import { DataAnalysisTool } from '../tools/DataAnalysisTool'; import { NoTool } from '../tools/NoTool'; @@ -55,7 +55,8 @@ export class Agent { history: () => string, csvData: () => { filename: string; id: string; text: string }[], addLinkedUrlDoc: (url: string, id: string) => void, - addLinkedDoc: (doc_type: string, data: string | undefined, options: DocumentOptions, id: string) => void, + addLinkedDoc: (doc_type: supportedDocumentTypes, data: unknown, options: DocumentOptions, id: string) => void, + // eslint-disable-next-line @typescript-eslint/no-unused-vars createCSVInDash: (url: string, title: string, id: string, data: string) => void ) { // Initialize OpenAI client with API key from environment @@ -134,6 +135,7 @@ export class Agent { console.log(this.interMessages); console.log(`Turn ${i}/${maxTurns}`); + // eslint-disable-next-line no-await-in-loop const result = await this.execute(onProcessingUpdate, onAnswerUpdate); this.interMessages.push({ role: 'assistant', content: result }); @@ -195,6 +197,7 @@ export class Agent { if (currentAction) { try { // Process the action with its input + // eslint-disable-next-line no-await-in-loop const observation = (await this.processAction(currentAction, actionInput.inputs)) as Observation[]; const nextPrompt = [{ type: 'text', text: ` ` }, ...observation, { type: 'text', text: '' }] as Observation[]; console.log(observation); @@ -299,7 +302,7 @@ export class Agent { * @param response The parsed XML response from the assistant. * @throws An error if the response does not meet the expected structure. */ - private validateAssistantResponse(response: any) { + private validateAssistantResponse(response: { stage: { [key: string]: object | string } }) { if (!response.stage) { throw new Error('Response does not contain a element'); } @@ -342,7 +345,7 @@ export class Agent { // If 'action_input' is present, validate its structure if ('action_input' in stage) { - const actionInput = stage.action_input; + const actionInput = stage.action_input as object; if (!('action_input_description' in actionInput) || typeof actionInput.action_input_description !== 'string') { throw new Error('action_input must contain an action_input_description string'); @@ -357,7 +360,7 @@ export class Agent { // If 'answer' is present, validate its structure if ('answer' in stage) { - const answer = stage.answer; + const answer = stage.answer as object; // Ensure answer contains at least one of the required elements if (!('grounded_text' in answer || 'normal_text' in answer)) { diff --git a/src/client/views/nodes/chatbot/chatboxcomponents/ChatBox.tsx b/src/client/views/nodes/chatbot/chatboxcomponents/ChatBox.tsx index 83b50c8c6..076f49831 100644 --- a/src/client/views/nodes/chatbot/chatboxcomponents/ChatBox.tsx +++ b/src/client/views/nodes/chatbot/chatboxcomponents/ChatBox.tsx @@ -14,7 +14,7 @@ import OpenAI, { ClientOptions } from 'openai'; import * as React from 'react'; import { v4 as uuidv4 } from 'uuid'; import { ClientUtils } from '../../../../../ClientUtils'; -import { Doc, DocListCast, Opt } from '../../../../../fields/Doc'; +import { Doc, DocListCast, FieldType, Opt } from '../../../../../fields/Doc'; import { DocData, DocViews } from '../../../../../fields/DocSymbols'; import { CsvCast, DocCast, NumCast, PDFCast, RTFCast, StrCast } from '../../../../../fields/Types'; import { Networking } from '../../../../Network'; @@ -324,7 +324,9 @@ export class ChatBox extends ViewBoxAnnotatableComponent() { processing_info: [], }); } finally { - this._isLoading = false; + runInAction(() => { + this._isLoading = false; + }); this.scrollToBottom(); } } @@ -402,19 +404,17 @@ export class ChatBox extends ViewBoxAnnotatableComponent() { data.map(doc => doc.doc_type !== 'collection' // Handle non-collection documents ? this.whichDoc(doc.doc_type, doc.data, { backgroundColor: doc.backgroundColor, _width: doc.width, _height: doc.height }, doc.id, insideCol) - : // Recursively process collections - this.createCollectionWithChildren(doc.data, true).then(nestedDocs => - Docs.Create.FreeformDocument(nestedDocs, { - title: doc.title, - backgroundColor: doc.backgroundColor, - _width: doc.width, - _height: doc.height, - _layout_fitWidth: true, - _freeform_backgroundGrid: true, - }) - ) - ) - .flat() // prettier-ignore + : this.createCollectionWithChildren(doc.data, true).then(nestedDocs => + Docs.Create.FreeformDocument(nestedDocs, { + title: doc.title, + backgroundColor: doc.backgroundColor, + _width: doc.width, + _height: doc.height, + _layout_fitWidth: true, + _freeform_backgroundGrid: true, + }) + ) + ) // prettier-ignore ).then(childDocs => childDocs.filter(doc => doc).map(doc => doc!)); // .then(nestedResults => { // // Flatten any nested arrays from recursive collection calls @@ -427,23 +427,18 @@ export class ChatBox extends ViewBoxAnnotatableComponent() { // return childDocs; // }); - // @action - // createSingleFlashcard = (data: any, options: DocumentOptions) => { - - // } - @action - whichDoc = (doc_type: string, data: string, options: DocumentOptions, id: string, insideCol: boolean): Promise> => + whichDoc = (doc_type: string, data: unknown, options: DocumentOptions, id: string, insideCol: boolean): Promise> => (async () => { switch (doc_type) { - case 'text': return Docs.Create.TextDocument(data, options); - case 'flashcard': return this.createFlashcard(data, options); - case 'deck': return this.createDeck(data, options); - case 'image': return Docs.Create.ImageDocument(data, options); - case 'equation': return Docs.Create.EquationDocument(data, options); + case 'text': return Docs.Create.TextDocument(data as string, options); + case 'flashcard': return this.createFlashcard(data as string[], options); + case 'deck': return this.createDeck(data as string, options); + case 'image': return Docs.Create.ImageDocument(data as string, options); + case 'equation': return Docs.Create.EquationDocument(data as string, options); case 'noteboard': return Docs.Create.NoteTakingDocument([], options); case 'simulation': return Docs.Create.SimulationDocument(options); - case 'collection': return this.createCollectionWithChildren(data as any, true). + case 'collection': return this.createCollectionWithChildren(data as { doc_type: string; id: string; data: any; title: string; width: number; height: number; backgroundColor: string }[] , true). then((arr, collOpts = { ...options, _layout_fitWidth: true, _freeform_backgroundGrid: true }) => (() => { switch (options.type_collection) { @@ -457,10 +452,10 @@ export class ChatBox extends ViewBoxAnnotatableComponent() { } })() ); - case 'web': return Docs.Create.WebDocument(data, { ...options, data_useCors: true }); - case 'comparison': return this.createComparison(data, options); + case 'web': return Docs.Create.WebDocument(data as string, { ...options, data_useCors: true }); + case 'comparison': return this.createComparison(data as {left: {width:number ,height: number, backgroundColor: string, data: string}, right: {width:number ,height: number, backgroundColor: string, data: string}}, options); case 'diagram': return Docs.Create.DiagramDocument(options); - case 'audio': return Docs.Create.AudioDocument(data, options); + case 'audio': return Docs.Create.AudioDocument(data as string, options); case 'map': return Docs.Create.MapDocument([], options); case 'screengrab': return Docs.Create.ScreenshotDocument(options); case 'webcam': return Docs.Create.WebCamDocument('', options); @@ -471,7 +466,7 @@ export class ChatBox extends ViewBoxAnnotatableComponent() { case 'trail': return Docs.Create.PresDocument(options); case 'tab': return Docs.Create.FreeformDocument([], options); case 'slide': return Docs.Create.TreeDocument([], options); - default: return Docs.Create.TextDocument(data, options); + default: return Docs.Create.TextDocument(data as string, options); } // prettier-ignore })().then(doc => { if (doc) { @@ -491,7 +486,7 @@ export class ChatBox extends ViewBoxAnnotatableComponent() { * @returns {Promise} A promise that resolves once the document is created and displayed. */ @action - createDocInDash = (doc_type: string, data: string | undefined, options: DocumentOptions, id: string) => { + createDocInDash = (doc_type: string, data: unknown, options: DocumentOptions /*, id: string */) => { const linkAndShowDoc = (doc: Opt) => { if (doc) { LinkManager.Instance.addLink(Docs.Create.LinkDocument(this.Document, doc)); @@ -501,22 +496,21 @@ export class ChatBox extends ViewBoxAnnotatableComponent() { }; const doc = (() => { switch (doc_type.toLowerCase()) { - case 'text': return Docs.Create.TextDocument(data || '', options); - case 'image': return Docs.Create.ImageDocument(data || '', options); - case 'pdf': return Docs.Create.PdfDocument(data || '', options); - case 'video': return Docs.Create.VideoDocument(data || '', options); - case 'audio': return Docs.Create.AudioDocument(data || '', options); - case 'web': return Docs.Create.WebDocument(data || '', options); - case 'equation': return Docs.Create.EquationDocument(data || '', options); + case 'flashcard': return this.createFlashcard(data as string[], options); + case 'text': return Docs.Create.TextDocument(data as string || '', options); + case 'image': return Docs.Create.ImageDocument(data as string || '', options); + case 'pdf': return Docs.Create.PdfDocument(data as string || '', options); + case 'video': return Docs.Create.VideoDocument(data as string || '', options); + case 'audio': return Docs.Create.AudioDocument(data as string || '', options); + case 'web': return Docs.Create.WebDocument(data as string || '', options); + case 'equation': return Docs.Create.EquationDocument(data as string || '', options); case 'chat': return Docs.Create.ChatDocument(options); - case 'functionplot': - case 'function_plot': return Docs.Create.FunctionPlotDocument([], options); - case 'dataviz': - case 'data_viz': Networking.PostToServer('/createCSV', { + case 'functionplot': return Docs.Create.FunctionPlotDocument([], options); + case 'dataviz': Networking.PostToServer('/createCSV', { filename: (options.title as string).replace(/\s+/g, '') + '.csv', data: data, })?.then(({ fileUrl, id }) => { - const vdoc = Docs.Create.DataVizDocument(fileUrl, { ...options, text: RTFCast(data) }); + const vdoc = Docs.Create.DataVizDocument(fileUrl, { ...options, text: RTFCast(data as FieldType) }); this.addCSVForAnalysis(vdoc, id); linkAndShowDoc(vdoc); }); @@ -537,14 +531,14 @@ export class ChatBox extends ViewBoxAnnotatableComponent() { * @returns {Doc} A carousel document containing the flashcard deck. */ @action - createDeck = (data: any, options: DocumentOptions) => { + createDeck = (data: string | unknown[], options: DocumentOptions) => { const flashcardDeck: Doc[] = []; // Parse `data` only if it’s a string - const deckData = typeof data === 'string' ? JSON.parse(data) : data; - const flashcardArray = Array.isArray(deckData) ? deckData : Object.values(deckData); + const deckData = typeof data === 'string' ? (JSON.parse(data) as unknown) : data; + const flashcardArray = Array.isArray(deckData) ? deckData : Object.values(deckData as object); // Process each flashcard document in the `deckData` array if (flashcardArray.length == 2 && flashcardArray[0].doc_type == 'text' && flashcardArray[1].doc_type == 'text') { - this.createFlashcard(flashcardArray, options); + this.createFlashcard(flashcardArray as string[], options); } else { flashcardArray.forEach(doc => { const flashcardDoc = this.createFlashcard(doc, options); @@ -570,24 +564,24 @@ export class ChatBox extends ViewBoxAnnotatableComponent() { * @returns {Doc | undefined} The created flashcard document, or undefined if the flashcard cannot be created. */ @action - createFlashcard = (data: any, options: any) => { + createFlashcard = (data: string[], options: DocumentOptions) => { const deckData = typeof data === 'string' ? JSON.parse(data) : data; - const flashcardArray = Array.isArray(deckData) ? deckData : Object.values(deckData)[2]; + const flashcardArray = Array.isArray(deckData) ? deckData : (Object.values(deckData)[2] as string[]); const [front, back] = flashcardArray; - if (front.doc_type === 'text' && back.doc_type === 'text') { + if (typeof front === 'string' && typeof back === 'string') { const sideOptions: DocumentOptions = { backgroundColor: options.backgroundColor, _width: options._width, - _height: options._height, + _height: options._height || 300, }; // Create front and back text documents - const side1 = Docs.Create.CenteredTextCreator(front.title, front.data, sideOptions); - const side2 = Docs.Create.CenteredTextCreator(back.title, back.data, sideOptions); + const side1 = Docs.Create.CenteredTextCreator('question', front, sideOptions); + const side2 = Docs.Create.CenteredTextCreator('answer', back, sideOptions); // Create the flashcard document with both sides - return Docs.Create.FlashcardDocument(data.title, side1, side2, sideOptions); + return Docs.Create.FlashcardDocument('flashcard', side1, side2, sideOptions); } }; @@ -599,12 +593,12 @@ export class ChatBox extends ViewBoxAnnotatableComponent() { * @returns {Doc} The created comparison document. */ @action - createComparison = (doc: { left: any; right: any }, options: any) => - Docs.Create.ComparisonDocument(options.title, { + createComparison = (doc: { left: { width: number; height: number; backgroundColor: string; data: string }; right: { width: number; height: number; backgroundColor: string; data: string } }, options: DocumentOptions) => + Docs.Create.ComparisonDocument(options.title as string, { data_back: Docs.Create.TextDocument(doc.left.data, { backgroundColor: doc.left.backgroundColor, _width: doc.left.width, _height: doc.left.height }), data_front: Docs.Create.TextDocument(doc.right.data, { backgroundColor: doc.right.backgroundColor, _width: doc.right.width, _height: doc.right.height }), - _width: options.width, - _height: options.height | 300, + _width: options._width, + _height: options._height || 300, backgroundColor: options.backgroundColor, }); @@ -909,7 +903,7 @@ export class ChatBox extends ViewBoxAnnotatableComponent() {
- (this._inputValue = e.target.value)} disabled={this._isLoading} /> + (this._inputValue = e.target.value))} disabled={this._isLoading} />