From 7847717fcb18684950aa9f7640c7f2264ff3f4a1 Mon Sep 17 00:00:00 2001 From: alyssaf16 Date: Thu, 24 Oct 2024 14:23:44 -0400 Subject: create documents --- .../nodes/chatbot/chatboxcomponents/ChatBox.tsx | 101 ++++++++++++++++++++- 1 file changed, 100 insertions(+), 1 deletion(-) (limited to 'src/client/views/nodes/chatbot/chatboxcomponents') diff --git a/src/client/views/nodes/chatbot/chatboxcomponents/ChatBox.tsx b/src/client/views/nodes/chatbot/chatboxcomponents/ChatBox.tsx index 98f242ebf..14fdd9b8d 100644 --- a/src/client/views/nodes/chatbot/chatboxcomponents/ChatBox.tsx +++ b/src/client/views/nodes/chatbot/chatboxcomponents/ChatBox.tsx @@ -418,16 +418,88 @@ export class ChatBox extends ViewBoxAnnotatableComponent() { * @param options Other optional document options (e.g. color) * @param id The unique ID for the document. */ + + // @action + // createDocInDash = async (docs: string[]) => { + // console.log('DOCS HERE' + docs); + // docs.forEach(doc => { + // const parsedDoc = JSON.parse(doc); + // this.createIndivDocInDash(parsedDoc.doc_type, parsedDoc.data, parsedDoc.options, ''); + // }); + // }; + @action createDocInDash = async (doc_type: string, data: string, options: DocumentOptions, id: string) => { + console.log('INDIV DOC' + doc_type); let doc; switch (doc_type) { case 'text': doc = DocCast(Docs.Create.TextDocument(data, options)); + break; + case 'flashcard': + doc = this.createFlashcard(data, options); + break; + case 'image': + doc = DocCast(Docs.Create.ImageDocument(data, options)); + break; + case 'equation': + doc = DocCast(Docs.Create.EquationDocument('', options)); + break; + case 'noteboard': + doc = DocCast(Docs.Create.NoteTakingDocument([], options)); + break; + case 'simulation': + doc = DocCast(Docs.Create.SimulationDocument(options)); + break; + case 'collection': + doc = DocCast(Docs.Create.FreeformDocument([], options)); + break; + case 'web': + doc = DocCast(Docs.Create.WebDocument(data, options)); + break; + case 'comparison': + doc = Docs.Create.ComparisonDocument('', options); + break; + case 'diagram': + doc = Docs.Create.DiagramDocument(options); + break; + case 'audio': + doc = Docs.Create.AudioDocument(data, options); + break; + case 'map': + doc = Docs.Create.MapDocument([], options); + break; + case 'screengrab': + doc = Docs.Create.ScreenshotDocument(options); + break; + case 'webcam': + doc = Docs.Create.WebCamDocument('', options); + break; + case 'button': + doc = Docs.Create.ButtonDocument(options); + break; + case 'script': + doc = Docs.Create.ScriptingDocument(null, options); + break; + case 'dataviz': + doc = Docs.Create.DataVizDocument('/users/rz/Downloads/addresses.csv', options); + break; + case 'chat': + doc = Docs.Create.ChatDocument(options); + break; + case 'trail': + doc = Docs.Create.PresDocument(options); + break; + case 'tab': + doc = Docs.Create.FreeformDocument([], options); + break; + case 'slide': + doc = Docs.Create.TreeDocument([], options); + break; default: doc = DocCast(Docs.Create.TextDocument(data, options)); } - + console.log('DOC' + doc_type); const linkDoc = Docs.Create.LinkDocument(this.Document, doc); LinkManager.Instance.addLink(linkDoc); @@ -435,6 +507,33 @@ export class ChatBox extends ViewBoxAnnotatableComponent() { await DocumentManager.Instance.showDocument(doc, { willZoomCentered: true }, () => {}); }; + // TODO: DELEGATE TO DIFFERENT CLASS + @action + createFlashcard = (data: string, options: DocumentOptions) => { + const flashcardDeck: Doc[] = []; + const parsedItems: { [key: string]: string } = JSON.parse(data); + Object.entries(parsedItems).forEach(([key, val]) => { + console.log('key' + key); + console.log('key' + val); + + const side1 = Docs.Create.CenteredTextCreator('question', key, options); + const side2 = Docs.Create.CenteredTextCreator('answer', val, options); + const doc = DocCast(Docs.Create.FlashcardDocument(data, side1, side2, { _width: 300, _height: 300 })); + this._props.addDocument?.(doc); + flashcardDeck.push(doc); + }); + const col = DocCast( + Docs.Create.CarouselDocument(flashcardDeck, { + title: options.title, + _width: 300, + _height: 300, + _layout_fitWidth: false, + _layout_autoHeight: true, + }) + ); + return col; + }; + /** * Event handler to manage citations click in the message components. * @param citation The citation object clicked by the user. -- cgit v1.2.3-70-g09d2 From 44d1ed4fce9203ce0141caddf8f0cde20f0950cd Mon Sep 17 00:00:00 2001 From: alyssaf16 Date: Sun, 27 Oct 2024 20:19:17 -0400 Subject: redesigning to make structure recursive --- .../nodes/chatbot/chatboxcomponents/ChatBox.tsx | 45 +++++++++- .../nodes/chatbot/tools/CreateDocumentTool.ts | 95 +++++++++++++++++----- 2 files changed, 115 insertions(+), 25 deletions(-) (limited to 'src/client/views/nodes/chatbot/chatboxcomponents') diff --git a/src/client/views/nodes/chatbot/chatboxcomponents/ChatBox.tsx b/src/client/views/nodes/chatbot/chatboxcomponents/ChatBox.tsx index 14fdd9b8d..7b7431bbe 100644 --- a/src/client/views/nodes/chatbot/chatboxcomponents/ChatBox.tsx +++ b/src/client/views/nodes/chatbot/chatboxcomponents/ChatBox.tsx @@ -427,10 +427,35 @@ export class ChatBox extends ViewBoxAnnotatableComponent() { // this.createIndivDocInDash(parsedDoc.doc_type, parsedDoc.data, parsedDoc.options, ''); // }); // }; + @action + private createCollectionWithChildren = async (data: string[]): Promise => { + console.log('Creating collection with nested documents'); + + // Create an array of promises for each document + const childDocPromises = data.map(async doc => { + const parsedDoc = JSON.parse(doc); + + if (parsedDoc.doc_type !== 'collection') { + // Handle non-collection documents + return await this.whichDoc(parsedDoc.doc_type, parsedDoc.data, parsedDoc.options, parsedDoc.id); + } else { + // Recursively process collections + const nestedDocs = await this.createCollectionWithChildren(parsedDoc.data); + return nestedDocs; // This will return an array of Docs + } + }); + + // Await all child document creations concurrently + const nestedResults = await Promise.all(childDocPromises); + + // Flatten any nested arrays from recursive collection calls + const childDocs = nestedResults.flat(); + + return childDocs; + }; @action - createDocInDash = async (doc_type: string, data: string, options: DocumentOptions, id: string) => { - console.log('INDIV DOC' + doc_type); + whichDoc = async (doc_type: string, data: string, options: DocumentOptions, id: string): Promise => { let doc; switch (doc_type) { case 'text': @@ -451,9 +476,12 @@ export class ChatBox extends ViewBoxAnnotatableComponent() { case 'simulation': doc = DocCast(Docs.Create.SimulationDocument(options)); break; - case 'collection': - doc = DocCast(Docs.Create.FreeformDocument([], options)); + case 'collection': { + const par = JSON.parse(data); + const arr = await this.createCollectionWithChildren(par); + doc = DocCast(Docs.Create.FreeformDocument(arr, options)); break; + } case 'web': doc = DocCast(Docs.Create.WebDocument(data, options)); break; @@ -499,6 +527,15 @@ export class ChatBox extends ViewBoxAnnotatableComponent() { default: doc = DocCast(Docs.Create.TextDocument(data, options)); } + return doc; + }; + + @action + createDocInDash = async (doc_type: string, data: string, options: DocumentOptions, id: string) => { + console.log('INDIV DOC' + doc_type); + + const doc = await this.whichDoc(doc_type, data, options, id); + console.log('DOC' + doc_type); const linkDoc = Docs.Create.LinkDocument(this.Document, doc); LinkManager.Instance.addLink(linkDoc); diff --git a/src/client/views/nodes/chatbot/tools/CreateDocumentTool.ts b/src/client/views/nodes/chatbot/tools/CreateDocumentTool.ts index 992935b77..4c5d4dc7d 100644 --- a/src/client/views/nodes/chatbot/tools/CreateDocumentTool.ts +++ b/src/client/views/nodes/chatbot/tools/CreateDocumentTool.ts @@ -3,60 +3,104 @@ import { BaseTool } from './BaseTool'; import { Observation } from '../types/types'; import { ParametersType } from '../types/tool_types'; import { DocumentOptions } from '../../../../documents/Documents'; -import { RTFCast, StrCast } from '../../../../../fields/Types'; - const docInstructions = { - text: 'Provide text content as string, not dictionary', - flashcard: 'A string dictionary mapping front to back of flashcard that the document will display. Follow this example: data: {"What is photosynthesis?":"The process by which g…t and absorb water and nutrients from the soil."}', - image: 'Provide a real image url', - web: 'Only provide real url to be displayed, not text content', + collection: { + description: 'A recursive collection of documents. Each document can be a "text", "flashcard", "image", "web", or another "collection".', + example: [ + { + doc_type: 'collection', + title: 'Science Collection', + data: [ + { + doc_type: 'flashcard', + title: 'Photosynthesis', + data: { 'What is photosynthesis?': 'The process by which plants make food.' }, + width: 300, + height: 300, + }, + { + doc_type: 'text', + title: 'Water Cycle', + data: 'The continuous movement of water on, above, and below the Earth’s surface.', + width: 300, + height: 300, + }, + { + doc_type: 'collection', + title: 'Advanced Biology', + data: [ + { + doc_type: 'flashcard', + title: 'Respiration', + data: { 'What is respiration?': 'Conversion of oxygen and glucose to energy.' }, + width: 300, + height: 300, + }, + { + doc_type: 'text', + title: 'Cell Structure', + data: 'Cells are the basic building blocks of all living organisms.', + width: 300, + height: 300, + }, + ], + width: 600, + height: 600, + }, + ], + width: 600, + height: 600, + }, + ], + }, + text: 'Provide text content as a plain string. Example: "This is a standalone text document."', + flashcard: 'A dictionary mapping the front to the back of the flashcard. Example: {"Question":"Answer"}', + flashcardDeck: 'A collection of flashcards under a common theme.', + image: 'A URL to an image. Example: "https://example.com/image.jpg"', + web: 'A URL to a webpage. Example: "https://example.com"', } as const; -// have recursive structure -// get array of all documents that each have their options -// (if its a collection) - const createDocToolParams = [ { name: 'data', - type: 'string', + type: 'any', // Accepts either string or array, supporting individual and nested data description: docInstructions, required: true, }, { name: 'doc_type', type: 'string', - description: 'The type of the document', + description: 'The type of the document. Options: "collection", "text", "flashcard", "image", "web".', required: true, }, { name: 'title', type: 'string', - description: 'The title of the document', + description: 'The title of the document.', required: true, }, { name: 'background_color', type: 'string', - description: 'The background color of the document as a hex string', + description: 'The background color of the document as a hex string.', required: false, }, { name: 'font_color', type: 'string', - description: 'The font color of the document as a hex string', + description: 'The font color of the document as a hex string.', required: false, }, { name: 'width', type: 'number', - description: 'The height of the document as a number', + description: 'The width of the document in pixels.', required: true, }, { name: 'height', type: 'number', - description: 'The height of the document as a number', + description: 'The height of the document in pixels.', required: true, }, ] as const; @@ -64,11 +108,12 @@ const createDocToolParams = [ const createListDocToolParams = [ { name: 'docs', - type: 'string', // array of stringified documents + type: 'string', // Array of stringified JSON objects description: - 'docs is an array that contains stringified JSON objects representing different document types. Each item in the array is a stringified version of this: ' + + 'Array of documents in stringified JSON format. Each item in the array should be an individual stringified JSON object. Each document can be of type "text", "flashcard", "image", "web", or "collection" (for nested documents). ' + + 'Use this structure for nesting collections within collections. Each document should follow the structure in ' + createDocToolParams + - 'Each document should be individually serialized (using JSON.stringify or equivalent) so that it fits within string[]. An example is ["{"data":"Plants are living organisms that belong to the kingdom Plantae.","doc_type":"text","title":"Introduction to Plants","width":300,"height":300}", "{"data":"Photosynthesis is the process by which plants make food.","type":"text","title":"Photosynthesis","width":300,"height":300}"]', + '. Example: ["{"doc_type":"collection","title":"Science Topics","data":["{\\"doc_type\\":\\"text\\",\\"title\\":\\"Photosynthesis\\",\\"data\\":\\"Photosynthesis is the process by which plants make food.\\",\\"width\\":300,\\"height\\":300}","{\\"doc_type\\":\\"collection\\",\\"title\\":\\"Advanced Biology\\",\\"data\\":["{\\"doc_type\\":\\"flashcard\\",\\"title\\":\\"Respiration\\",\\"data\\":{\\"What is respiration?\\":\\"Conversion of oxygen and glucose to energy.\\"},\\"width\\":300,\\"height\\":300}","{\\"doc_type\\":\\"text\\",\\"title\\":\\"Cell Structure\\",\\"data\\":\\"Cells are the basic building blocks of all living organisms.\\",\\"width\\":300,\\"height\\":300}"],\\"width\\":600,\\"height\\":600}"],"width":600,"height":600}"]', required: true, }, ] as const; @@ -93,6 +138,12 @@ export class CreateDocTool extends BaseTool { } async execute(args: ParametersType): Promise { + + /** + * loop through each collection calling the + */ + + try { console.log('EXE' + args.docs); const parsedDoc = JSON.parse(args.docs); @@ -103,8 +154,10 @@ export class CreateDocTool extends BaseTool { const doc = JSON.parse(firstDoc); console.log('NEW DOC' + doc); console.log('TYPE' + doc['doc_type']); + console.log('WIDTH' + doc['width']); + console.log('HEIGHT' + doc['height']); - this._addLinkedDoc(doc['doc_type'], doc['data'], { title: doc['title'], backgroundColor: doc['background_color'], text_fontColor: doc['font_color'] }, uuidv4()); + this._addLinkedDoc(doc['doc_type'], doc['data'], { title: doc['title'], backgroundColor: doc['background_color'], text_fontColor: doc['font_color'], _layout_fitWidth: false, _layout_autoHeight: true }, uuidv4()); }); // this._addLinkedDoc(args.doc_type, args.data, { title: args.title, backgroundColor: args.background_color, text_fontColor: args.font_color }, uuidv4()); return [{ type: 'text', text: 'Created document.' }]; -- cgit v1.2.3-70-g09d2 From f37c99bb332bba274dc93b7509696c6d61c3dd21 Mon Sep 17 00:00:00 2001 From: alyssaf16 Date: Wed, 30 Oct 2024 20:06:43 -0400 Subject: version 4 of restructuring --- .../nodes/chatbot/chatboxcomponents/ChatBox.tsx | 32 +++- .../nodes/chatbot/tools/CreateDocumentTool.ts | 190 ++++++++++++++++----- 2 files changed, 172 insertions(+), 50 deletions(-) (limited to 'src/client/views/nodes/chatbot/chatboxcomponents') diff --git a/src/client/views/nodes/chatbot/chatboxcomponents/ChatBox.tsx b/src/client/views/nodes/chatbot/chatboxcomponents/ChatBox.tsx index 7b7431bbe..0939247b7 100644 --- a/src/client/views/nodes/chatbot/chatboxcomponents/ChatBox.tsx +++ b/src/client/views/nodes/chatbot/chatboxcomponents/ChatBox.tsx @@ -434,23 +434,37 @@ export class ChatBox extends ViewBoxAnnotatableComponent() { // Create an array of promises for each document const childDocPromises = data.map(async doc => { const parsedDoc = JSON.parse(doc); - + console.log('Parse #3: ' + parsedDoc); if (parsedDoc.doc_type !== 'collection') { // Handle non-collection documents return await this.whichDoc(parsedDoc.doc_type, parsedDoc.data, parsedDoc.options, parsedDoc.id); } else { // Recursively process collections - const nestedDocs = await this.createCollectionWithChildren(parsedDoc.data); - return nestedDocs; // This will return an array of Docs + const nestedDocs = await this.createCollectionWithChildren(JSON.parse(parsedDoc.data) as string[]); + const collectionOptions: DocumentOptions = { + title: parsedDoc.title, + backgroundColor: parsedDoc.background_color, + _width: parsedDoc.width, + _height: parsedDoc.height, + _layout_fitWidth: true, + _freeform_backgroundGrid: true, + }; + const collectionDoc = DocCast(Docs.Create.FreeformDocument(nestedDocs, collectionOptions)); + return collectionDoc; // Return th } }); // Await all child document creations concurrently const nestedResults = await Promise.all(childDocPromises); - + console.log('n' + nestedResults); // Flatten any nested arrays from recursive collection calls - const childDocs = nestedResults.flat(); - + const childDocs = nestedResults.flat() as Doc[]; + console.log('c' + childDocs); + childDocs.forEach(doc => { + console.log(DocCast(doc)); + console.log(DocCast(doc)[DocData].data); + console.log(DocCast(doc)[DocData].data); + }); return childDocs; }; @@ -478,8 +492,12 @@ export class ChatBox extends ViewBoxAnnotatableComponent() { break; case 'collection': { const par = JSON.parse(data); + console.log('Parse #2: ' + par); const arr = await this.createCollectionWithChildren(par); - doc = DocCast(Docs.Create.FreeformDocument(arr, options)); + options._layout_fitWidth = true; + options._freeform_backgroundGrid = true; + const opts = { _width: 500, _height: 800, _layout_fitWidth: true, _freeform_backgroundGrid: true }; + doc = DocCast(Docs.Create.FreeformDocument(arr, opts)); break; } case 'web': diff --git a/src/client/views/nodes/chatbot/tools/CreateDocumentTool.ts b/src/client/views/nodes/chatbot/tools/CreateDocumentTool.ts index 4c5d4dc7d..a1924ed82 100644 --- a/src/client/views/nodes/chatbot/tools/CreateDocumentTool.ts +++ b/src/client/views/nodes/chatbot/tools/CreateDocumentTool.ts @@ -3,55 +3,153 @@ import { BaseTool } from './BaseTool'; import { Observation } from '../types/types'; import { ParametersType } from '../types/tool_types'; import { DocumentOptions } from '../../../../documents/Documents'; -const docInstructions = { - collection: { - description: 'A recursive collection of documents. Each document can be a "text", "flashcard", "image", "web", or another "collection".', - example: [ +const jsonData = [ + { + doc_type: 'collection', + title: 'Science Collection', + data: JSON.stringify([ + { + doc_type: 'flashcard', + title: 'Photosynthesis', + data: JSON.stringify({ 'What is photosynthesis?': 'The process by which plants make food.' }), + width: 300, + height: 300, + backgroundColor: '#0000FF', + }, + { + doc_type: 'text', + title: 'Water Cycle', + data: 'The continuous movement of water on, above, and below the Earth’s surface.', + width: 300, + height: 300, + }, { doc_type: 'collection', - title: 'Science Collection', - data: [ + title: 'Advanced Biology', + data: JSON.stringify([ { doc_type: 'flashcard', - title: 'Photosynthesis', - data: { 'What is photosynthesis?': 'The process by which plants make food.' }, + title: 'Respiration', + data: JSON.stringify({ 'What is respiration?': 'Conversion of oxygen and glucose to energy.' }), width: 300, height: 300, }, { doc_type: 'text', - title: 'Water Cycle', - data: 'The continuous movement of water on, above, and below the Earth’s surface.', + title: 'Cell Structure', + data: 'Cells are the basic building blocks of all living organisms.', + width: 300, + height: 300, + }, + ]), + width: 600, + height: 600, + }, + ]), + width: 600, + height: 600, + }, + { + doc_type: 'collection', + title: 'Math Collection', + data: JSON.stringify([ + { + doc_type: 'flashcard', + title: 'Pythagorean Theorem', + data: JSON.stringify({ 'What is the Pythagorean theorem?': 'In a right triangle, a² + b² = c².' }), + width: 300, + height: 300, + backgroundColor: '#FFA500', + }, + { + doc_type: 'text', + title: 'Calculus Introduction', + data: 'Calculus is the mathematical study of continuous change.', + width: 300, + height: 300, + }, + { + doc_type: 'collection', + title: 'Algebra Concepts', + data: JSON.stringify([ + { + doc_type: 'flashcard', + title: 'Quadratic Formula', + data: JSON.stringify({ 'What is the quadratic formula?': 'x = (-b ± √(b² - 4ac)) / 2a' }), width: 300, height: 300, }, { - doc_type: 'collection', - title: 'Advanced Biology', - data: [ - { - doc_type: 'flashcard', - title: 'Respiration', - data: { 'What is respiration?': 'Conversion of oxygen and glucose to energy.' }, - width: 300, - height: 300, - }, - { - doc_type: 'text', - title: 'Cell Structure', - data: 'Cells are the basic building blocks of all living organisms.', - width: 300, - height: 300, - }, - ], - width: 600, - height: 600, + doc_type: 'text', + title: 'Linear Equations', + data: 'A linear equation is an equation between two variables that gives a straight line when plotted.', + width: 300, + height: 300, }, - ], + ]), width: 600, height: 600, }, - ], + ]), + width: 600, + height: 600, + }, +]; + +// Stringify the entire object if needed for your API +const finalJsonString = JSON.stringify(jsonData); + +const docInstructions = { + collection: { + description: 'A recursive collection of documents. Each document can be a "text", "flashcard", "image", "web", or another "collection".', + example: finalJsonString, + // example: [ + // { + // doc_type: 'collection', + // title: 'Science Collection', + // data: [ + // { + // doc_type: 'flashcard', + // title: 'Photosynthesis', + // data: { 'What is photosynthesis?': 'The process by which plants make food.' }, + // width: 300, + // height: 300, + // backgroundColor: '#0000FF', + // }, + // { + // doc_type: 'text', + // title: 'Water Cycle', + // data: 'The continuous movement of water on, above, and below the Earth’s surface.', + // width: 300, + // height: 300, + // }, + // { + // doc_type: 'collection', + // title: 'Advanced Biology', + // data: [ + // { + // doc_type: 'flashcard', + // title: 'Respiration', + // data: { 'What is respiration?': 'Conversion of oxygen and glucose to energy.' }, + // width: 300, + // height: 300, + // }, + // { + // doc_type: 'text', + // title: 'Cell Structure', + // data: 'Cells are the basic building blocks of all living organisms.', + // width: 300, + // height: 300, + // }, + // ], + // width: 600, + // height: 600, + // }, + // ], + // width: 600, + // height: 600, + // }, + // ], }, text: 'Provide text content as a plain string. Example: "This is a standalone text document."', flashcard: 'A dictionary mapping the front to the back of the flashcard. Example: {"Question":"Answer"}', @@ -63,7 +161,7 @@ const docInstructions = { const createDocToolParams = [ { name: 'data', - type: 'any', // Accepts either string or array, supporting individual and nested data + type: 'string', // Accepts either string or array, supporting individual and nested data description: docInstructions, required: true, }, @@ -113,7 +211,10 @@ const createListDocToolParams = [ 'Array of documents in stringified JSON format. Each item in the array should be an individual stringified JSON object. Each document can be of type "text", "flashcard", "image", "web", or "collection" (for nested documents). ' + 'Use this structure for nesting collections within collections. Each document should follow the structure in ' + createDocToolParams + - '. Example: ["{"doc_type":"collection","title":"Science Topics","data":["{\\"doc_type\\":\\"text\\",\\"title\\":\\"Photosynthesis\\",\\"data\\":\\"Photosynthesis is the process by which plants make food.\\",\\"width\\":300,\\"height\\":300}","{\\"doc_type\\":\\"collection\\",\\"title\\":\\"Advanced Biology\\",\\"data\\":["{\\"doc_type\\":\\"flashcard\\",\\"title\\":\\"Respiration\\",\\"data\\":{\\"What is respiration?\\":\\"Conversion of oxygen and glucose to energy.\\"},\\"width\\":300,\\"height\\":300}","{\\"doc_type\\":\\"text\\",\\"title\\":\\"Cell Structure\\",\\"data\\":\\"Cells are the basic building blocks of all living organisms.\\",\\"width\\":300,\\"height\\":300}"],\\"width\\":600,\\"height\\":600}"],"width":600,"height":600}"]', + '. Example: ' + finalJsonString, + //Example: ["{"doc_type":"collection","title":"Science Topics","data":"[\\"{\\\\"doc_type\\\\":\\\\"text\\\\",\\\\"title\\\\":\\\\"Photosynthesis\\\\",\\\\"background_color\\\\":\\\\"#0000FF\\\\",\\\\"data\\\\":\\\\"Photosynthesis is the process by which plants make food.\\\\",\\\\"width\\\\":300,\\\\"height\\\\":300}\\",\\"{\\\\"doc_type\\\\":\\\\"collection\\\\",\\\\"title\\\\":\\\\"Advanced Biology\\\\",\\\\"data\\\\":\\\\"[\\\\"{\\\\"doc_type\\\\":\\\\"flashcard\\\\",\\\\"title\\\\":\\\\"Respiration\\\\",\\\\"data\\\\":{\\\\"What is respiration?\\\\":\\\\"Conversion of oxygen and glucose to energy.\\\\"},\\\\"width\\\\":300,\\\\"height\\\\":300}\\",\\\\"{\\\\"doc_type\\\\":\\\\"text\\\\",\\\\"title\\\\":\\\\"Cell Structure\\\\",\\\\"data\\\\":\\\\"Cells are the basic building blocks of all living organisms.\\\\",\\\\"width\\\\":300,\\\\"height\\\\":300}\\"]\\\\",\\\\"width\\\\":600,\\\\"height\\\\":600}\\"]","width":600,"height":600}"]', + + //["{"doc_type":"collection","title":"Science Topics","data":["{\\"doc_type\\":\\"text\\",\\"title\\":\\"Photosynthesis\\",\\"data\\":\\"Photosynthesis is the process by which plants make food.\\",\\"width\\":300,\\"height\\":300}","{\\"doc_type\\":\\"collection\\",\\"title\\":\\"Advanced Biology\\",\\"data\\":["{\\"doc_type\\":\\"flashcard\\",\\"title\\":\\"Respiration\\",\\"data\\":{\\"What is respiration?\\":\\"Conversion of oxygen and glucose to energy.\\"},\\"width\\":300,\\"height\\":300}","{\\"doc_type\\":\\"text\\",\\"title\\":\\"Cell Structure\\",\\"data\\":\\"Cells are the basic building blocks of all living organisms.\\",\\"width\\":300,\\"height\\":300}"],\\"width\\":600,\\"height\\":600}"],"width":600,"height":600}"]', required: true, }, ] as const; @@ -138,26 +239,29 @@ export class CreateDocTool extends BaseTool { } async execute(args: ParametersType): Promise { - /** - * loop through each collection calling the + * loop through each collection calling the */ - try { - console.log('EXE' + args.docs); + // console.log('EXE' + args.docs); const parsedDoc = JSON.parse(args.docs); console.log('parsed' + parsedDoc); parsedDoc.forEach((firstDoc: string) => { - console.log('THIS DOC' + firstDoc); + // console.log('THIS DOC' + firstDoc); console.log(typeof firstDoc); const doc = JSON.parse(firstDoc); console.log('NEW DOC' + doc); - console.log('TYPE' + doc['doc_type']); - console.log('WIDTH' + doc['width']); - console.log('HEIGHT' + doc['height']); + // console.log('TYPE' + doc['doc_type']); + // console.log('WIDTH' + doc['width']); + // console.log('HEIGHT' + doc['height']); - this._addLinkedDoc(doc['doc_type'], doc['data'], { title: doc['title'], backgroundColor: doc['background_color'], text_fontColor: doc['font_color'], _layout_fitWidth: false, _layout_autoHeight: true }, uuidv4()); + this._addLinkedDoc( + doc['doc_type'], + doc['data'], + { title: doc['title'], backgroundColor: doc['background_color'], text_fontColor: doc['font_color'], _width: doc['width'], _height: doc['height'], _layout_fitWidth: false, _layout_autoHeight: true }, + uuidv4() + ); }); // this._addLinkedDoc(args.doc_type, args.data, { title: args.title, backgroundColor: args.background_color, text_fontColor: args.font_color }, uuidv4()); return [{ type: 'text', text: 'Created document.' }]; -- cgit v1.2.3-70-g09d2 From cd54cba6269dbc4e10b457fe7eddd5114a9d301e Mon Sep 17 00:00:00 2001 From: alyssaf16 Date: Wed, 30 Oct 2024 22:17:57 -0400 Subject: version 5 working --- .../nodes/chatbot/chatboxcomponents/ChatBox.tsx | 21 ++--- .../nodes/chatbot/tools/CreateDocumentTool.ts | 91 +++++++--------------- 2 files changed, 38 insertions(+), 74 deletions(-) (limited to 'src/client/views/nodes/chatbot/chatboxcomponents') diff --git a/src/client/views/nodes/chatbot/chatboxcomponents/ChatBox.tsx b/src/client/views/nodes/chatbot/chatboxcomponents/ChatBox.tsx index 0939247b7..4a39ee388 100644 --- a/src/client/views/nodes/chatbot/chatboxcomponents/ChatBox.tsx +++ b/src/client/views/nodes/chatbot/chatboxcomponents/ChatBox.tsx @@ -428,22 +428,22 @@ export class ChatBox extends ViewBoxAnnotatableComponent() { // }); // }; @action - private createCollectionWithChildren = async (data: string[]): Promise => { + private createCollectionWithChildren = async (data: any): Promise => { console.log('Creating collection with nested documents'); // Create an array of promises for each document const childDocPromises = data.map(async doc => { - const parsedDoc = JSON.parse(doc); + const parsedDoc = doc; console.log('Parse #3: ' + parsedDoc); if (parsedDoc.doc_type !== 'collection') { // Handle non-collection documents - return await this.whichDoc(parsedDoc.doc_type, parsedDoc.data, parsedDoc.options, parsedDoc.id); + return await this.whichDoc(parsedDoc.doc_type, parsedDoc.data, { backgroundColor: parsedDoc.backgroundColor, _width: parsedDoc.width, _height: parsedDoc.height }, parsedDoc.id); } else { // Recursively process collections - const nestedDocs = await this.createCollectionWithChildren(JSON.parse(parsedDoc.data) as string[]); + const nestedDocs = await this.createCollectionWithChildren(parsedDoc.data); const collectionOptions: DocumentOptions = { title: parsedDoc.title, - backgroundColor: parsedDoc.background_color, + backgroundColor: parsedDoc.backgroundColor, _width: parsedDoc.width, _height: parsedDoc.height, _layout_fitWidth: true, @@ -491,13 +491,14 @@ export class ChatBox extends ViewBoxAnnotatableComponent() { doc = DocCast(Docs.Create.SimulationDocument(options)); break; case 'collection': { - const par = JSON.parse(data); - console.log('Parse #2: ' + par); - const arr = await this.createCollectionWithChildren(par); + // const par = JSON.parse(data); + // console.log('Parse #2: ' + par); + const arr = await this.createCollectionWithChildren(data); options._layout_fitWidth = true; options._freeform_backgroundGrid = true; - const opts = { _width: 500, _height: 800, _layout_fitWidth: true, _freeform_backgroundGrid: true }; - doc = DocCast(Docs.Create.FreeformDocument(arr, opts)); + + // const opts = { _width: 500, _height: 800, _layout_fitWidth: true, _freeform_backgroundGrid: true }; + doc = DocCast(Docs.Create.FreeformDocument(arr, options)); break; } case 'web': diff --git a/src/client/views/nodes/chatbot/tools/CreateDocumentTool.ts b/src/client/views/nodes/chatbot/tools/CreateDocumentTool.ts index a1924ed82..0b83ff24f 100644 --- a/src/client/views/nodes/chatbot/tools/CreateDocumentTool.ts +++ b/src/client/views/nodes/chatbot/tools/CreateDocumentTool.ts @@ -3,18 +3,19 @@ import { BaseTool } from './BaseTool'; import { Observation } from '../types/types'; import { ParametersType } from '../types/tool_types'; import { DocumentOptions } from '../../../../documents/Documents'; -const jsonData = [ + +const example = [ { doc_type: 'collection', title: 'Science Collection', - data: JSON.stringify([ + data: [ { doc_type: 'flashcard', title: 'Photosynthesis', - data: JSON.stringify({ 'What is photosynthesis?': 'The process by which plants make food.' }), + data: { 'What is photosynthesis?': 'The process by which plants make food.' }, + backgroundColor: '#00ff00', width: 300, height: 300, - backgroundColor: '#0000FF', }, { doc_type: 'text', @@ -26,11 +27,12 @@ const jsonData = [ { doc_type: 'collection', title: 'Advanced Biology', - data: JSON.stringify([ + data: [ { doc_type: 'flashcard', title: 'Respiration', - data: JSON.stringify({ 'What is respiration?': 'Conversion of oxygen and glucose to energy.' }), + data: { 'What is respiration?': 'Conversion of oxygen and glucose to energy.' }, + backgroundColor: '#00ff00', width: 300, height: 300, }, @@ -41,67 +43,23 @@ const jsonData = [ width: 300, height: 300, }, - ]), - width: 600, - height: 600, - }, - ]), - width: 600, - height: 600, - }, - { - doc_type: 'collection', - title: 'Math Collection', - data: JSON.stringify([ - { - doc_type: 'flashcard', - title: 'Pythagorean Theorem', - data: JSON.stringify({ 'What is the Pythagorean theorem?': 'In a right triangle, a² + b² = c².' }), - width: 300, - height: 300, - backgroundColor: '#FFA500', - }, - { - doc_type: 'text', - title: 'Calculus Introduction', - data: 'Calculus is the mathematical study of continuous change.', - width: 300, - height: 300, - }, - { - doc_type: 'collection', - title: 'Algebra Concepts', - data: JSON.stringify([ - { - doc_type: 'flashcard', - title: 'Quadratic Formula', - data: JSON.stringify({ 'What is the quadratic formula?': 'x = (-b ± √(b² - 4ac)) / 2a' }), - width: 300, - height: 300, - }, - { - doc_type: 'text', - title: 'Linear Equations', - data: 'A linear equation is an equation between two variables that gives a straight line when plotted.', - width: 300, - height: 300, - }, - ]), + ], width: 600, height: 600, }, - ]), + ], width: 600, height: 600, }, ]; -// Stringify the entire object if needed for your API -const finalJsonString = JSON.stringify(jsonData); +// Stringify the entire structure for transmission if needed +const finalJsonString = JSON.stringify(example); const docInstructions = { collection: { - description: 'A recursive collection of documents. Each document can be a "text", "flashcard", "image", "web", or another "collection".', + description: + 'A recursive collection of documents as a stringified array. Each document can be a "text", "flashcard", "image", "web", "image", "comparison", "equation", "noteboard", "simulation", "diagram", "map", "screengrab", "webcam", "button", or another "collection".', example: finalJsonString, // example: [ // { @@ -156,6 +114,10 @@ const docInstructions = { flashcardDeck: 'A collection of flashcards under a common theme.', image: 'A URL to an image. Example: "https://example.com/image.jpg"', web: 'A URL to a webpage. Example: "https://example.com"', + equation: 'Create a equation document.', + noteboard: 'Create a noteboard document', + comparison: 'Create a comparison document', + simulation: 'Create a simulation document', } as const; const createDocToolParams = [ @@ -211,8 +173,9 @@ const createListDocToolParams = [ 'Array of documents in stringified JSON format. Each item in the array should be an individual stringified JSON object. Each document can be of type "text", "flashcard", "image", "web", or "collection" (for nested documents). ' + 'Use this structure for nesting collections within collections. Each document should follow the structure in ' + createDocToolParams + - '. Example: ' + finalJsonString, - //Example: ["{"doc_type":"collection","title":"Science Topics","data":"[\\"{\\\\"doc_type\\\\":\\\\"text\\\\",\\\\"title\\\\":\\\\"Photosynthesis\\\\",\\\\"background_color\\\\":\\\\"#0000FF\\\\",\\\\"data\\\\":\\\\"Photosynthesis is the process by which plants make food.\\\\",\\\\"width\\\\":300,\\\\"height\\\\":300}\\",\\"{\\\\"doc_type\\\\":\\\\"collection\\\\",\\\\"title\\\\":\\\\"Advanced Biology\\\\",\\\\"data\\\\":\\\\"[\\\\"{\\\\"doc_type\\\\":\\\\"flashcard\\\\",\\\\"title\\\\":\\\\"Respiration\\\\",\\\\"data\\\\":{\\\\"What is respiration?\\\\":\\\\"Conversion of oxygen and glucose to energy.\\\\"},\\\\"width\\\\":300,\\\\"height\\\\":300}\\",\\\\"{\\\\"doc_type\\\\":\\\\"text\\\\",\\\\"title\\\\":\\\\"Cell Structure\\\\",\\\\"data\\\\":\\\\"Cells are the basic building blocks of all living organisms.\\\\",\\\\"width\\\\":300,\\\\"height\\\\":300}\\"]\\\\",\\\\"width\\\\":600,\\\\"height\\\\":600}\\"]","width":600,"height":600}"]', + '. Example: ' + + finalJsonString, + //["{"doc_type":"collection","title":"Science Topics","data":"[\\"{\\\\"doc_type\\\\":\\\\"text\\\\",\\\\"title\\\\":\\\\"Photosynthesis\\\\",\\\\"background_color\\\\":\\\\""#0000FF"\\\\",\\\\"data\\\\":\\\\"Photosynthesis is the process by which plants make food.\\\\",\\\\"width\\\\":300,\\\\"height\\\\":300}\\",\\"{\\\\"doc_type\\\\":\\\\"collection\\\\",\\\\"title\\\\":\\\\"Advanced Biology\\\\",\\\\"data\\\\":\\\\"[\\\\"{\\\\"doc_type\\\\":\\\\"flashcard\\\\",\\\\"title\\\\":\\\\"Respiration\\\\",\\\\"data\\\\":{\\\\"What is respiration?\\\\":\\\\"Conversion of oxygen and glucose to energy.\\\\"},\\\\"width\\\\":300,\\\\"height\\\\":300}\\",\\\\"{\\\\"doc_type\\\\":\\\\"text\\\\",\\\\"title\\\\":\\\\"Cell Structure\\\\",\\\\"data\\\\":\\\\"Cells are the basic building blocks of all living organisms.\\\\",\\\\"width\\\\":300,\\\\"height\\\\":300}\\"]\\\\",\\\\"width\\\\":600,\\\\"height\\\\":600}\\"]","width":600,"height":600}"]', //["{"doc_type":"collection","title":"Science Topics","data":["{\\"doc_type\\":\\"text\\",\\"title\\":\\"Photosynthesis\\",\\"data\\":\\"Photosynthesis is the process by which plants make food.\\",\\"width\\":300,\\"height\\":300}","{\\"doc_type\\":\\"collection\\",\\"title\\":\\"Advanced Biology\\",\\"data\\":["{\\"doc_type\\":\\"flashcard\\",\\"title\\":\\"Respiration\\",\\"data\\":{\\"What is respiration?\\":\\"Conversion of oxygen and glucose to energy.\\"},\\"width\\":300,\\"height\\":300}","{\\"doc_type\\":\\"text\\",\\"title\\":\\"Cell Structure\\",\\"data\\":\\"Cells are the basic building blocks of all living organisms.\\",\\"width\\":300,\\"height\\":300}"],\\"width\\":600,\\"height\\":600}"],"width":600,"height":600}"]', required: true, @@ -244,14 +207,14 @@ export class CreateDocTool extends BaseTool { */ try { - // console.log('EXE' + args.docs); + console.log('EXE' + args.docs); const parsedDoc = JSON.parse(args.docs); console.log('parsed' + parsedDoc); - parsedDoc.forEach((firstDoc: string) => { + parsedDoc.forEach(doc => { // console.log('THIS DOC' + firstDoc); - console.log(typeof firstDoc); - const doc = JSON.parse(firstDoc); - console.log('NEW DOC' + doc); + // console.log(typeof firstDoc); + // const doc = JSON.parse(firstDoc); + // console.log('NEW DOC' + doc); // console.log('TYPE' + doc['doc_type']); // console.log('WIDTH' + doc['width']); // console.log('HEIGHT' + doc['height']); @@ -259,7 +222,7 @@ export class CreateDocTool extends BaseTool { this._addLinkedDoc( doc['doc_type'], doc['data'], - { title: doc['title'], backgroundColor: doc['background_color'], text_fontColor: doc['font_color'], _width: doc['width'], _height: doc['height'], _layout_fitWidth: false, _layout_autoHeight: true }, + { title: doc['title'], backgroundColor: doc['backgroundColor'], text_fontColor: doc['font_color'], _width: doc['width'], _height: doc['height'], _layout_fitWidth: false, _layout_autoHeight: true }, uuidv4() ); }); -- cgit v1.2.3-70-g09d2 From ff934c67fc1fac71e7b3191cc213cfe29983fe2e Mon Sep 17 00:00:00 2001 From: alyssaf16 Date: Thu, 31 Oct 2024 00:05:27 -0400 Subject: nesting works --- .../nodes/chatbot/chatboxcomponents/ChatBox.tsx | 6 +++++ .../nodes/chatbot/tools/CreateDocumentTool.ts | 29 ++++++++++++++-------- 2 files changed, 24 insertions(+), 11 deletions(-) (limited to 'src/client/views/nodes/chatbot/chatboxcomponents') diff --git a/src/client/views/nodes/chatbot/chatboxcomponents/ChatBox.tsx b/src/client/views/nodes/chatbot/chatboxcomponents/ChatBox.tsx index 4a39ee388..e2508d752 100644 --- a/src/client/views/nodes/chatbot/chatboxcomponents/ChatBox.tsx +++ b/src/client/views/nodes/chatbot/chatboxcomponents/ChatBox.tsx @@ -468,6 +468,11 @@ export class ChatBox extends ViewBoxAnnotatableComponent() { return childDocs; }; + // @action + // createSingleFlashcard = (data: any, options: DocumentOptions) => { + + // } + @action whichDoc = async (doc_type: string, data: string, options: DocumentOptions, id: string): Promise => { let doc; @@ -476,6 +481,7 @@ export class ChatBox extends ViewBoxAnnotatableComponent() { doc = DocCast(Docs.Create.TextDocument(data, options)); break; case 'flashcard': + // doc = this.createSingleFlashcard(data, options); doc = this.createFlashcard(data, options); break; case 'image': diff --git a/src/client/views/nodes/chatbot/tools/CreateDocumentTool.ts b/src/client/views/nodes/chatbot/tools/CreateDocumentTool.ts index 0b83ff24f..7d74c3b15 100644 --- a/src/client/views/nodes/chatbot/tools/CreateDocumentTool.ts +++ b/src/client/views/nodes/chatbot/tools/CreateDocumentTool.ts @@ -12,7 +12,22 @@ const example = [ { doc_type: 'flashcard', title: 'Photosynthesis', - data: { 'What is photosynthesis?': 'The process by which plants make food.' }, + data: [ + { + doc_type: 'text', + title: 'Front Photosynthesis', + data: 'What is photosynthesis?', + width: 300, + height: 300, + }, + { + doc_type: 'text', + title: 'back_photosynthesis', + data: 'The process by which plants make food.', + width: 300, + height: 300, + }, + ], backgroundColor: '#00ff00', width: 300, height: 300, @@ -28,14 +43,6 @@ const example = [ doc_type: 'collection', title: 'Advanced Biology', data: [ - { - doc_type: 'flashcard', - title: 'Respiration', - data: { 'What is respiration?': 'Conversion of oxygen and glucose to energy.' }, - backgroundColor: '#00ff00', - width: 300, - height: 300, - }, { doc_type: 'text', title: 'Cell Structure', @@ -110,9 +117,9 @@ const docInstructions = { // ], }, text: 'Provide text content as a plain string. Example: "This is a standalone text document."', - flashcard: 'A dictionary mapping the front to the back of the flashcard. Example: {"Question":"Answer"}', + flashcard: 'Two text documents with content for the front and back.', flashcardDeck: 'A collection of flashcards under a common theme.', - image: 'A URL to an image. Example: "https://example.com/image.jpg"', + image: 'A URL to an image for data. Example: "https://example.com/image.jpg"', web: 'A URL to a webpage. Example: "https://example.com"', equation: 'Create a equation document.', noteboard: 'Create a noteboard document', -- cgit v1.2.3-70-g09d2 From c358fba1ee2aa54a97373d07e7b218c74dfd9bf0 Mon Sep 17 00:00:00 2001 From: alyssaf16 Date: Tue, 12 Nov 2024 01:01:23 -0500 Subject: flashcards w assistant finally workgit add -A --- .../views/nodes/chatbot/agentsystem/Agent.ts | 4 +- .../nodes/chatbot/chatboxcomponents/ChatBox.tsx | 89 +++++-- .../nodes/chatbot/tools/CreateDocumentTool.ts | 82 +++++- .../views/nodes/chatbot/vectorstore/Vectorstore.ts | 2 +- src/server/flashcard/labels.py | 285 +++++++++++++++++++++ src/server/flashcard/requirements.txt | 12 + src/server/flashcard/venv/pyvenv.cfg | 3 + temp_image 2.jpg | Bin 0 -> 195492 bytes temp_image.jpg | Bin 0 -> 196370 bytes 9 files changed, 450 insertions(+), 27 deletions(-) create mode 100644 src/server/flashcard/labels.py create mode 100644 src/server/flashcard/requirements.txt create mode 100644 src/server/flashcard/venv/pyvenv.cfg create mode 100644 temp_image 2.jpg create mode 100644 temp_image.jpg (limited to 'src/client/views/nodes/chatbot/chatboxcomponents') diff --git a/src/client/views/nodes/chatbot/agentsystem/Agent.ts b/src/client/views/nodes/chatbot/agentsystem/Agent.ts index 05d13d1db..0b0e211eb 100644 --- a/src/client/views/nodes/chatbot/agentsystem/Agent.ts +++ b/src/client/views/nodes/chatbot/agentsystem/Agent.ts @@ -69,9 +69,9 @@ export class Agent { // Define available tools for the assistant this.tools = { calculate: new CalculateTool(), - rag: new RAGTool(this.vectorstore), + // rag: new RAGTool(this.vectorstore), dataAnalysis: new DataAnalysisTool(csvData), - websiteInfoScraper: new WebsiteInfoScraperTool(addLinkedUrlDoc), + // websiteInfoScraper: new WebsiteInfoScraperTool(addLinkedUrlDoc), searchTool: new SearchTool(addLinkedUrlDoc), createCSV: new CreateCSVTool(createCSVInDash), noTool: new NoTool(), diff --git a/src/client/views/nodes/chatbot/chatboxcomponents/ChatBox.tsx b/src/client/views/nodes/chatbot/chatboxcomponents/ChatBox.tsx index 68d4383e7..95f3fbc5d 100644 --- a/src/client/views/nodes/chatbot/chatboxcomponents/ChatBox.tsx +++ b/src/client/views/nodes/chatbot/chatboxcomponents/ChatBox.tsx @@ -463,9 +463,11 @@ export class ChatBox extends ViewBoxAnnotatableComponent() { doc = DocCast(Docs.Create.TextDocument(data, options)); break; case 'flashcard': - // doc = this.createSingleFlashcard(data, options); doc = this.createFlashcard(data, options); break; + case 'deck': + doc = this.createDeck(data, options); + break; case 'image': doc = DocCast(Docs.Create.ImageDocument(data, options)); break; @@ -551,31 +553,82 @@ export class ChatBox extends ViewBoxAnnotatableComponent() { await DocumentManager.Instance.showDocument(doc, { willZoomCentered: true }, () => {}); }; - // TODO: DELEGATE TO DIFFERENT CLASS @action - createFlashcard = (data: string, options: DocumentOptions) => { + createDeck = (data: any, options: DocumentOptions) => { const flashcardDeck: Doc[] = []; - const parsedItems: { [key: string]: string } = JSON.parse(data); - Object.entries(parsedItems).forEach(([key, val]) => { - console.log('key' + key); - console.log('key' + val); - - const side1 = Docs.Create.CenteredTextCreator('question', key, options); - const side2 = Docs.Create.CenteredTextCreator('answer', val, options); - const doc = DocCast(Docs.Create.FlashcardDocument(data, side1, side2, { _width: 300, _height: 300 })); - this._props.addDocument?.(doc); - flashcardDeck.push(doc); + + // Parse `data` only if it’s a string + const deckData = typeof data === 'string' ? JSON.parse(data) : data; + console.log('Parsed Deck Data:', deckData); + const flashcardArray = Array.isArray(deckData) ? deckData : Object.values(deckData); + console.log(typeof flashcardArray); + // Process each flashcard document in the `deckData` array + flashcardArray.forEach(doc => { + const flashcardDoc = this.createFlashcard(doc, options); + if (flashcardDoc) flashcardDeck.push(flashcardDoc); }); - const col = DocCast( + + // Create a carousel to contain the flashcard deck + const carouselDoc = DocCast( Docs.Create.CarouselDocument(flashcardDeck, { - title: options.title, - _width: 300, - _height: 300, + title: options.title || 'Flashcard Deck', + _width: options._width || 300, + _height: options._height || 300, _layout_fitWidth: false, _layout_autoHeight: true, }) ); - return col; + + return carouselDoc; + }; + @action + createFlashcard = (data: any, options: any) => { + // const flashcardDeck: Doc[] = []; + + // Process each flashcard item in the data array + // const p = JSON.parse(data); + const deckData = typeof data === 'string' ? JSON.parse(data) : data; + const flashcardArray = Array.isArray(deckData) ? deckData : Object.values(deckData)[2]; + console.log(typeof flashcardArray); + + const [front, back] = flashcardArray; + + // Check that both front and back are text documents + console.log('DATA' + data); + console.log('front' + front); + console.log('back' + back); + console.log(front.doc_type); + console.log(back.doc_type); + if (front.doc_type === 'text' && back.doc_type === 'text') { + const sideOptions: DocumentOptions = { + backgroundColor: options.backgroundColor, + _width: options._width, + _height: options._height, + }; + + // Create front and back text documents + const side1 = Docs.Create.CenteredTextCreator(front.title, front.data, sideOptions); + const side2 = Docs.Create.CenteredTextCreator(back.title, back.data, sideOptions); + + // Create the flashcard document with both sides + const flashcardDoc = DocCast(Docs.Create.FlashcardDocument(data.title, side1, side2, sideOptions)); + return flashcardDoc; + // this._props.addDocument?.(flashcardDoc); + // flashcardDeck.push(flashcardDoc); + } + + // Create a carousel to contain the flashcard deck + // const carouselDoc = DocCast( + // Docs.Create.CarouselDocument(flashcardDeck, { + // title: options.title || data.title, + // _width: data.width || 300, + // _height: data.height || 300, + // _layout_fitWidth: false, + // _layout_autoHeight: true, + // }) + // ); + + // return carouselDoc; }; /** diff --git a/src/client/views/nodes/chatbot/tools/CreateDocumentTool.ts b/src/client/views/nodes/chatbot/tools/CreateDocumentTool.ts index b14a57779..ebe0448aa 100644 --- a/src/client/views/nodes/chatbot/tools/CreateDocumentTool.ts +++ b/src/client/views/nodes/chatbot/tools/CreateDocumentTool.ts @@ -5,17 +5,86 @@ import { ParametersType } from '../types/tool_types'; import { DocumentOptions } from '../../../../documents/Documents'; const example = [ + { + doc_type: 'deck', + title: 'Chemistry', + data: [ + { + doc_type: 'flashcard', + title: 'Photosynthesis', + data: [ + { + doc_type: 'text', + title: 'front_Photosynthesis', + data: 'What is photosynthesis?', + width: 300, + height: 300, + }, + { + doc_type: 'text', + title: 'back_photosynthesis', + data: 'The process by which plants make food.', + width: 300, + height: 300, + }, + ], + backgroundColor: '#00ff00', + width: 300, + height: 300, + }, + { + doc_type: 'flashcard', + title: 'Photosynthesis', + data: [ + { + doc_type: 'text', + title: 'front_Photosynthesis', + data: 'What is photosynthesis?', + width: 300, + height: 300, + }, + { + doc_type: 'text', + title: 'back_photosynthesis', + data: 'The process by which plants make food.', + width: 300, + height: 300, + }, + ], + backgroundColor: '#00ff00', + width: 300, + height: 300, + }, + ], + backgroundColor: '#00ff00', + width: 600, + height: 600, + }, + { + doc_type: 'web', + title: 'Brown University Wikipedia', + data: 'https://en.wikipedia.org/wiki/Brown_University', + width: 300, + height: 300, + }, { doc_type: 'collection', title: 'Science Collection', data: [ + { + doc_type: 'web', + title: 'Brown University Wikipedia', + data: 'https://en.wikipedia.org/wiki/Brown_University', + width: 300, + height: 300, + }, { doc_type: 'flashcard', title: 'Photosynthesis', data: [ { doc_type: 'text', - title: 'Front Photosynthesis', + title: 'front_Photosynthesis', data: 'What is photosynthesis?', width: 300, height: 300, @@ -72,9 +141,9 @@ const docInstructions = { }, text: 'Provide text content as a plain string. Example: "This is a standalone text document."', flashcard: 'Two text documents with content for the front and back.', - flashcardDeck: 'A collection of flashcards under a common theme.', + deck: 'A decks data is an array of flashcards.', image: 'A URL to an image for data. Example: "https://example.com/image.jpg"', - web: 'A URL to a webpage. Example: "https://example.com"', + web: 'A URL to a webpage. Example: https://en.wikipedia.org/wiki/Brown_University', equation: 'Create a equation document.', noteboard: 'Create a noteboard document', comparison: 'Create a comparison document', @@ -148,12 +217,13 @@ export class CreateDocTool extends BaseTool { constructor(addLinkedDoc: (doc_type: string, data: string, options: DocumentOptions, id: string) => void) { super( 'createDoc', - 'Creates one or more documents that best fit users request', + 'Creates one or more documents that best fit users request with input following the example below. Or creates a dashboard of many documents/collections.', createListDocToolParams, - 'Modify the data parameter and include title (and optionally color) for the document.', + 'Modify the data parameter and include title (and optionally color) for the document. Web doc data type must be url from search tool.', 'Creates one or more documents represented by an array of strings with the provided content based on the instructions ' + docInstructions + - 'Use if the user wants to create something that aligns with a document type in dash like a flashcard, flashcard deck/stack, or textbox or text document of some sort. Can use after a search or other tool to save information.' + 'Use if the user wants to create something that aligns with a document type in dash like a flashcard, flashcard deck/stack, or textbox or text document of some sort. Can use after the search tool to save information.' + + 'When user asks for dashboard, create many documents/collections with different colors and texts while listening to their preferences, after using search tool to create a dashboard.' ); this._addLinkedDoc = addLinkedDoc; } diff --git a/src/client/views/nodes/chatbot/vectorstore/Vectorstore.ts b/src/client/views/nodes/chatbot/vectorstore/Vectorstore.ts index 5ed784559..cf7fa0ff3 100644 --- a/src/client/views/nodes/chatbot/vectorstore/Vectorstore.ts +++ b/src/client/views/nodes/chatbot/vectorstore/Vectorstore.ts @@ -44,7 +44,7 @@ export class Vectorstore { // Initialize Pinecone and Cohere clients with API keys from the environment. this.pinecone = new Pinecone({ apiKey: pineconeApiKey }); - this.cohere = new CohereClient({ token: process.env.COHERE_API_KEY }); + // this.cohere = new CohereClient({ token: process.env.COHERE_API_KEY }); this._id = id; this._doc_ids = doc_ids(); this.initializeIndex(); diff --git a/src/server/flashcard/labels.py b/src/server/flashcard/labels.py new file mode 100644 index 000000000..546fc4bd3 --- /dev/null +++ b/src/server/flashcard/labels.py @@ -0,0 +1,285 @@ +import base64 +import numpy as np +import base64 +import easyocr +import sys +from PIL import Image +from io import BytesIO +import requests +import json +import numpy as np + +class BoundingBoxUtils: + """Utility class for bounding box operations and OCR result corrections.""" + + @staticmethod + def is_close(box1, box2, x_threshold=20, y_threshold=20): + """ + Determines if two bounding boxes are horizontally and vertically close. + + Parameters: + box1, box2 (list): The bounding boxes to compare. + x_threshold (int): The threshold for horizontal proximity. + y_threshold (int): The threshold for vertical proximity. + + Returns: + bool: True if boxes are close, False otherwise. + """ + horizontally_close = (abs(box1[2] - box2[0]) < x_threshold or # Right edge of box1 and left edge of box2 + abs(box2[2] - box1[0]) < x_threshold or # Right edge of box2 and left edge of box1 + abs(box1[2] - box2[2]) < x_threshold or + abs(box2[0] - box1[0]) < x_threshold) + + vertically_close = (abs(box1[3] - box2[1]) < y_threshold or # Bottom edge of box1 and top edge of box2 + abs(box2[3] - box1[1]) < y_threshold or + box1[1] == box2[1] or box1[3] == box2[3]) + + return horizontally_close and vertically_close + + @staticmethod + def adjust_bounding_box(bbox, original_text, corrected_text): + """ + Adjusts a bounding box based on differences in text length. + + Parameters: + bbox (list): The original bounding box coordinates. + original_text (str): The original text detected by OCR. + corrected_text (str): The corrected text after cleaning. + + Returns: + list: The adjusted bounding box. + """ + if not bbox or len(bbox) != 4: + return bbox + + # Adjust the x-coordinates slightly to account for text correction + x_adjustment = 5 + adjusted_bbox = [ + [bbox[0][0] + x_adjustment, bbox[0][1]], + [bbox[1][0], bbox[1][1]], + [bbox[2][0] + x_adjustment, bbox[2][1]], + [bbox[3][0], bbox[3][1]] + ] + return adjusted_bbox + + @staticmethod + def correct_ocr_results(results): + """ + Corrects common OCR misinterpretations in the detected text and adjusts bounding boxes accordingly. + + Parameters: + results (list): A list of OCR results, each containing bounding box, text, and confidence score. + + Returns: + list: Corrected OCR results with adjusted bounding boxes. + """ + corrections = { + "~": "", # Replace '~' with empty string + "-": "" # Replace '-' with empty string + } + + corrected_results = [] + for (bbox, text, prob) in results: + corrected_text = ''.join(corrections.get(char, char) for char in text) + adjusted_bbox = BoundingBoxUtils.adjust_bounding_box(bbox, text, corrected_text) + corrected_results.append((adjusted_bbox, corrected_text, prob)) + + return corrected_results + + @staticmethod + def convert_to_json_serializable(data): + """ + Converts a list containing various types, including numpy types, to a JSON-serializable format. + + Parameters: + data (list): A list containing numpy or other non-serializable types. + + Returns: + list: A JSON-serializable version of the input list. + """ + def convert_element(element): + if isinstance(element, list): + return [convert_element(e) for e in element] + elif isinstance(element, tuple): + return tuple(convert_element(e) for e in element) + elif isinstance(element, np.integer): + return int(element) + elif isinstance(element, np.floating): + return float(element) + elif isinstance(element, np.ndarray): + return element.tolist() + else: + return element + + return convert_element(data) + +class ImageLabelProcessor: + """Class to process images and perform OCR with EasyOCR.""" + + VERTICAL_THRESHOLD = 20 + HORIZONTAL_THRESHOLD = 8 + + def __init__(self, img_source, source_type, smart_mode): + self.img_source = img_source + self.source_type = source_type + self.smart_mode = smart_mode + self.img_val = self.load_image() + + def load_image(self): + """Load image from either a base64 string or URL.""" + if self.source_type == 'drag': + return self._load_base64_image() + else: + return self._load_url_image() + + def _load_base64_image(self): + """Decode and save the base64 image.""" + base64_string = self.img_source + if base64_string.startswith("data:image"): + base64_string = base64_string.split(",")[1] + + + # Decode the base64 string + image_data = base64.b64decode(base64_string) + image = Image.open(BytesIO(image_data)).convert('RGB') + image.save("temp_image.jpg") + return "temp_image.jpg" + + def _load_url_image(self): + """Download image from URL and return it in byte format.""" + url = self.img_source + response = requests.get(url) + image = Image.open(BytesIO(response.content)).convert('RGB') + + image_bytes = BytesIO() + image.save(image_bytes, format='PNG') + return image_bytes.getvalue() + + def process_image(self): + """Process the image and return the OCR results.""" + if self.smart_mode: + return self._process_smart_mode() + else: + return self._process_standard_mode() + + def _process_smart_mode(self): + """Process the image in smart mode using EasyOCR.""" + reader = easyocr.Reader(['en']) + result = reader.readtext(self.img_val, detail=1, paragraph=True) + + all_boxes = [bbox for bbox, text in result] + all_texts = [text for bbox, text in result] + + response_data = { + 'status': 'success', + 'message': 'Data received', + 'boxes': BoundingBoxUtils.convert_to_json_serializable(all_boxes), + 'text': BoundingBoxUtils.convert_to_json_serializable(all_texts), + } + + return response_data + + def _process_standard_mode(self): + """Process the image in standard mode using EasyOCR.""" + reader = easyocr.Reader(['en']) + results = reader.readtext(self.img_val) + + filtered_results = BoundingBoxUtils.correct_ocr_results([ + (bbox, text, prob) for bbox, text, prob in results if prob >= 0.7 + ]) + + return self._merge_and_prepare_response(filtered_results) + + def are_vertically_close(self, box1, box2): + """Check if two bounding boxes are vertically close.""" + box1_bottom = max(box1[2][1], box1[3][1]) + box2_top = min(box2[0][1], box2[1][1]) + vertical_distance = box2_top - box1_bottom + + box1_left = box1[0][0] + box2_left = box2[0][0] + box1_right = box1[1][0] + box2_right = box2[1][0] + hori_close = abs(box2_left - box1_left) <= self.HORIZONTAL_THRESHOLD or abs(box2_right - box1_right) <= self.HORIZONTAL_THRESHOLD + + return vertical_distance <= self.VERTICAL_THRESHOLD and hori_close + + def merge_boxes(self, boxes, texts): + """Merge multiple bounding boxes and their associated text.""" + x_coords = [] + y_coords = [] + + # Collect all x and y coordinates + for box in boxes: + for point in box: + x_coords.append(point[0]) + y_coords.append(point[1]) + + # Create the merged bounding box + merged_box = [ + [min(x_coords), min(y_coords)], + [max(x_coords), min(y_coords)], + [max(x_coords), max(y_coords)], + [min(x_coords), max(y_coords)] + ] + + # Combine the texts + merged_text = ' '.join(texts) + + return merged_box, merged_text + + def _merge_and_prepare_response(self, filtered_results): + """Merge vertically close boxes and prepare the final response.""" + current_boxes, current_texts = [], [] + all_boxes, all_texts = [], [] + + for ind in range(len(filtered_results) - 1): + if not current_boxes: + current_boxes.append(filtered_results[ind][0]) + current_texts.append(filtered_results[ind][1]) + + if self.are_vertically_close(filtered_results[ind][0], filtered_results[ind + 1][0]): + current_boxes.append(filtered_results[ind + 1][0]) + current_texts.append(filtered_results[ind + 1][1]) + else: + merged = self.merge_boxes(current_boxes, current_texts) + all_boxes.append(merged[0]) + all_texts.append(merged[1]) + current_boxes, current_texts = [], [] + + if current_boxes: + merged = self.merge_boxes(current_boxes, current_texts) + all_boxes.append(merged[0]) + all_texts.append(merged[1]) + + if not current_boxes and filtered_results: + merged = self.merge_boxes([filtered_results[-1][0]], [filtered_results[-1][1]]) + all_boxes.append(merged[0]) + all_texts.append(merged[1]) + + response = { + 'status': 'success', + 'message': 'Data received', + 'boxes': BoundingBoxUtils.convert_to_json_serializable(all_boxes), + 'text': BoundingBoxUtils.convert_to_json_serializable(all_texts), + } + + return response + +# Main execution function +def labels(): + """Main function to handle image OCR processing based on input arguments.""" + source_type = sys.argv[2] + smart_mode = (sys.argv[3] == 'smart') + with open(sys.argv[1], 'r') as f: + img_source = f.read() + # Create ImageLabelProcessor instance + processor = ImageLabelProcessor(img_source, source_type, smart_mode) + response = processor.process_image() + + # Print and return the response + print(response) + return response + + +labels() diff --git a/src/server/flashcard/requirements.txt b/src/server/flashcard/requirements.txt new file mode 100644 index 000000000..eb92a819b --- /dev/null +++ b/src/server/flashcard/requirements.txt @@ -0,0 +1,12 @@ +easyocr==1.7.1 +requests==2.32.3 +pillow==10.4.0 +numpy==1.26.4 +tqdm==4.66.4 +Werkzeug==3.0.3 +python-dateutil==2.9.0.post0 +six==1.16.0 +certifi==2024.6.2 +charset-normalizer==3.3.2 +idna==3.7 +urllib3==1.26.19 \ No newline at end of file diff --git a/src/server/flashcard/venv/pyvenv.cfg b/src/server/flashcard/venv/pyvenv.cfg new file mode 100644 index 000000000..740014e00 --- /dev/null +++ b/src/server/flashcard/venv/pyvenv.cfg @@ -0,0 +1,3 @@ +home = /Library/Frameworks/Python.framework/Versions/3.10/bin +include-system-site-packages = false +version = 3.10.11 diff --git a/temp_image 2.jpg b/temp_image 2.jpg new file mode 100644 index 000000000..05bc8db3a Binary files /dev/null and b/temp_image 2.jpg differ diff --git a/temp_image.jpg b/temp_image.jpg new file mode 100644 index 000000000..912519ce1 Binary files /dev/null and b/temp_image.jpg differ -- cgit v1.2.3-70-g09d2 From f2a4c9e484340b0542b896974673ffa13fbc0ac5 Mon Sep 17 00:00:00 2001 From: alyssaf16 Date: Tue, 12 Nov 2024 07:53:22 -0500 Subject: comparison box with assistant working --- .../nodes/chatbot/chatboxcomponents/ChatBox.tsx | 17 ++++++++++++++++- .../nodes/chatbot/tools/CreateDocumentTool.ts | 22 ++++++++++++++++++++++ 2 files changed, 38 insertions(+), 1 deletion(-) (limited to 'src/client/views/nodes/chatbot/chatboxcomponents') diff --git a/src/client/views/nodes/chatbot/chatboxcomponents/ChatBox.tsx b/src/client/views/nodes/chatbot/chatboxcomponents/ChatBox.tsx index 95f3fbc5d..d85f9d82e 100644 --- a/src/client/views/nodes/chatbot/chatboxcomponents/ChatBox.tsx +++ b/src/client/views/nodes/chatbot/chatboxcomponents/ChatBox.tsx @@ -495,7 +495,7 @@ export class ChatBox extends ViewBoxAnnotatableComponent() { doc = DocCast(Docs.Create.WebDocument(data, options)); break; case 'comparison': - doc = Docs.Create.ComparisonDocument('', options); + doc = this.createComparison(data, options); break; case 'diagram': doc = Docs.Create.DiagramDocument(options); @@ -631,6 +631,21 @@ export class ChatBox extends ViewBoxAnnotatableComponent() { // return carouselDoc; }; + @action + createComparison = (doc: any, options: any) => { + const comp = Docs.Create.ComparisonDocument(options.title, { _width: options.width, _height: options.height | 300, backgroundColor: options.backgroundColor }); + const [left, right] = doc; + console.log(DocCast(comp.dataDoc)); + console.log(DocCast(comp[DocData])); + console.log(DocCast(comp[DocData].data_back)); + const docLeft = DocCast(Docs.Create.TextDocument(left.data, { backgroundColor: left.backgroundColor, _width: left.width, _height: left.height })); + const docRight = DocCast(Docs.Create.TextDocument(right.data, { backgroundColor: right.backgroundColor, _width: right.width, _height: right.height })); + comp[DocData].data_back = docLeft; + comp[DocData].data_front = docRight; + + return comp; + }; + /** * Event handler to manage citations click in the message components. * @param citation The citation object clicked by the user. diff --git a/src/client/views/nodes/chatbot/tools/CreateDocumentTool.ts b/src/client/views/nodes/chatbot/tools/CreateDocumentTool.ts index ebe0448aa..7b78ed510 100644 --- a/src/client/views/nodes/chatbot/tools/CreateDocumentTool.ts +++ b/src/client/views/nodes/chatbot/tools/CreateDocumentTool.ts @@ -67,6 +67,28 @@ const example = [ width: 300, height: 300, }, + { + doc_type: 'comparison', + title: 'WWI vs. WWII', + data: [ + { + doc_type: 'text', + title: 'WWI', + data: 'From 1914 to 1918, fighting took place across several continents, at sea and, for the first time, in the air.', + width: 300, + height: 300, + }, + { + doc_type: 'text', + title: 'WWII', + data: 'A devastating global conflict spanning from 1939 to 1945, saw the Allied powers fight against the Axis powers.', + width: 300, + height: 300, + }, + ], + width: 300, + height: 300, + }, { doc_type: 'collection', title: 'Science Collection', -- cgit v1.2.3-70-g09d2 From be60b14248de71d44e1a3195afab2c4d74c10363 Mon Sep 17 00:00:00 2001 From: alyssaf16 Date: Tue, 12 Nov 2024 08:00:33 -0500 Subject: image doc working --- src/client/views/nodes/chatbot/chatboxcomponents/ChatBox.tsx | 4 ---- src/client/views/nodes/chatbot/tools/CreateDocumentTool.ts | 7 +++++++ 2 files changed, 7 insertions(+), 4 deletions(-) (limited to 'src/client/views/nodes/chatbot/chatboxcomponents') diff --git a/src/client/views/nodes/chatbot/chatboxcomponents/ChatBox.tsx b/src/client/views/nodes/chatbot/chatboxcomponents/ChatBox.tsx index d85f9d82e..f6b284bdb 100644 --- a/src/client/views/nodes/chatbot/chatboxcomponents/ChatBox.tsx +++ b/src/client/views/nodes/chatbot/chatboxcomponents/ChatBox.tsx @@ -481,13 +481,9 @@ export class ChatBox extends ViewBoxAnnotatableComponent() { doc = DocCast(Docs.Create.SimulationDocument(options)); break; case 'collection': { - // const par = JSON.parse(data); - // console.log('Parse #2: ' + par); const arr = await this.createCollectionWithChildren(data); options._layout_fitWidth = true; options._freeform_backgroundGrid = true; - - // const opts = { _width: 500, _height: 800, _layout_fitWidth: true, _freeform_backgroundGrid: true }; doc = DocCast(Docs.Create.FreeformDocument(arr, options)); break; } diff --git a/src/client/views/nodes/chatbot/tools/CreateDocumentTool.ts b/src/client/views/nodes/chatbot/tools/CreateDocumentTool.ts index 7b78ed510..810f763af 100644 --- a/src/client/views/nodes/chatbot/tools/CreateDocumentTool.ts +++ b/src/client/views/nodes/chatbot/tools/CreateDocumentTool.ts @@ -5,6 +5,13 @@ import { ParametersType } from '../types/tool_types'; import { DocumentOptions } from '../../../../documents/Documents'; const example = [ + { + doc_type: 'image', + title: 'experiment', + data: 'https://plus.unsplash.com/premium_photo-1694819488591-a43907d1c5cc?q=80&w=2628&auto=format&fit=crop&ixlib=rb-4.0.3&ixid=M3wxMjA3fDB8MHxwaG90by1wYWdlfHx8fGVufDB8fHx8fA%3D%3D', + width: 300, + height: 300, + }, { doc_type: 'deck', title: 'Chemistry', -- cgit v1.2.3-70-g09d2 From 84cfc17b6b743a498f0f3b8680d262c3695e00fa Mon Sep 17 00:00:00 2001 From: alyssaf16 Date: Wed, 13 Nov 2024 01:43:37 -0500 Subject: 4 hours later...dashboard i think working --- .../nodes/chatbot/chatboxcomponents/ChatBox.tsx | 24 +++- .../nodes/chatbot/tools/CreateDocumentTool.ts | 122 ++++++++++++++++++--- 2 files changed, 128 insertions(+), 18 deletions(-) (limited to 'src/client/views/nodes/chatbot/chatboxcomponents') diff --git a/src/client/views/nodes/chatbot/chatboxcomponents/ChatBox.tsx b/src/client/views/nodes/chatbot/chatboxcomponents/ChatBox.tsx index f6b284bdb..594736fbc 100644 --- a/src/client/views/nodes/chatbot/chatboxcomponents/ChatBox.tsx +++ b/src/client/views/nodes/chatbot/chatboxcomponents/ChatBox.tsx @@ -472,19 +472,37 @@ export class ChatBox extends ViewBoxAnnotatableComponent() { doc = DocCast(Docs.Create.ImageDocument(data, options)); break; case 'equation': - doc = DocCast(Docs.Create.EquationDocument('', options)); + // make more advanced + doc = DocCast(Docs.Create.EquationDocument(data, options)); break; case 'noteboard': + // COME BACK doc = DocCast(Docs.Create.NoteTakingDocument([], options)); break; case 'simulation': + // make more advanced doc = DocCast(Docs.Create.SimulationDocument(options)); break; case 'collection': { + // COME BACK const arr = await this.createCollectionWithChildren(data); options._layout_fitWidth = true; options._freeform_backgroundGrid = true; - doc = DocCast(Docs.Create.FreeformDocument(arr, options)); + if (options.type_collection == 'tree') { + doc = DocCast(Docs.Create.TreeDocument(arr, options)); + } else if (options.type_collection == 'masonry') { + doc = DocCast(Docs.Create.MasonryDocument(arr, options)); + } else if (options.type_collection == 'card') { + doc = DocCast(Docs.Create.CardDeckDocument(arr, options)); + } else if (options.type_collection == 'carousel') { + doc = DocCast(Docs.Create.CarouselDocument(arr, options)); + } else if (options.type_collection == '3d-carousel') { + doc = DocCast(Docs.Create.Carousel3DDocument(arr, options)); + } else if (options.type_collection == 'multicolumn') { + doc = DocCast(Docs.Create.CarouselDocument(arr, options)); + } else { + doc = DocCast(Docs.Create.FreeformDocument(arr, options)); + } break; } case 'web': @@ -494,9 +512,11 @@ export class ChatBox extends ViewBoxAnnotatableComponent() { doc = this.createComparison(data, options); break; case 'diagram': + // come back doc = Docs.Create.DiagramDocument(options); break; case 'audio': + // come back doc = Docs.Create.AudioDocument(data, options); break; case 'map': diff --git a/src/client/views/nodes/chatbot/tools/CreateDocumentTool.ts b/src/client/views/nodes/chatbot/tools/CreateDocumentTool.ts index 810f763af..d35dd0e1d 100644 --- a/src/client/views/nodes/chatbot/tools/CreateDocumentTool.ts +++ b/src/client/views/nodes/chatbot/tools/CreateDocumentTool.ts @@ -5,6 +5,30 @@ import { ParametersType } from '../types/tool_types'; import { DocumentOptions } from '../../../../documents/Documents'; const example = [ + { + doc_type: 'equation', + title: 'quadratic', + data: 'x^2 + y^2 = 3', + width: 300, + height: 300, + }, + { + doc_type: 'collection', + title: 'Advanced Biology', + data: [ + { + doc_type: 'text', + title: 'Cell Structure', + data: 'Cells are the basic building blocks of all living organisms.', + width: 300, + height: 300, + }, + ], + backgroundColor: '#00ff00', + width: 600, + height: 600, + type_collection: 'freeform', + }, { doc_type: 'image', title: 'experiment', @@ -74,6 +98,13 @@ const example = [ width: 300, height: 300, }, + { + doc_type: 'simulation', + title: 'Physics simulation', + data: '', + width: 300, + height: 300, + }, { doc_type: 'comparison', title: 'WWI vs. WWII', @@ -100,13 +131,6 @@ const example = [ doc_type: 'collection', title: 'Science Collection', data: [ - { - doc_type: 'web', - title: 'Brown University Wikipedia', - data: 'https://en.wikipedia.org/wiki/Brown_University', - width: 300, - height: 300, - }, { doc_type: 'flashcard', title: 'Photosynthesis', @@ -130,6 +154,13 @@ const example = [ width: 300, height: 300, }, + { + doc_type: 'web', + title: 'Brown University Wikipedia', + data: 'https://en.wikipedia.org/wiki/Brown_University', + width: 300, + height: 300, + }, { doc_type: 'text', title: 'Water Cycle', @@ -152,10 +183,12 @@ const example = [ backgroundColor: '#00ff00', width: 600, height: 600, + type_collection: 'freeform', }, ], width: 600, height: 600, + type_collection: 'freeform', }, ]; @@ -173,10 +206,11 @@ const docInstructions = { deck: 'A decks data is an array of flashcards.', image: 'A URL to an image for data. Example: "https://example.com/image.jpg"', web: 'A URL to a webpage. Example: https://en.wikipedia.org/wiki/Brown_University', - equation: 'Create a equation document.', + equation: 'Create an equation document, not a text document. Data is math equation.', noteboard: 'Create a noteboard document', - comparison: 'Create a comparison document', + comparison: 'Create a comparison document - in type_collection specify the type of collection: masonry, freeform, tree, carousel.', simulation: 'Create a simulation document', + audio: 'A url to an audio recording. Example: ', } as const; const createDocToolParams = [ @@ -222,6 +256,12 @@ const createDocToolParams = [ description: 'The height of the document in pixels.', required: true, }, + { + name: 'type_collection', + type: 'string', + description: 'Either freeform, card, carousel, 3d-carousel, multicolumn, multirow, linear, map, notetaking, schema, stacking, grid, tree, or masonry.', + required: false, + }, ] as const; const createListDocToolParams = [ @@ -246,14 +286,55 @@ export class CreateDocTool extends BaseTool { constructor(addLinkedDoc: (doc_type: string, data: string, options: DocumentOptions, id: string) => void) { super( 'createDoc', - 'Creates one or more documents that best fit users request with input following the example below. Or creates a dashboard of many documents/collections.', + 'Creates one or more documents that best fit the user’s request. If the user requests a "dashboard," first call search tool and then generate a variety of document types individually, each with different content and color schemes. For example, create multiple individual documents like "text," "deck," "web," "image," "equation," and "comparison." ' + + 'Do not nest all documents within a single collection unless explicitly requested by the user. Instead, create a set of independent documents with diverse document types. Each type should appear separately unless specified otherwise.', createListDocToolParams, - 'Modify the data parameter and include title (and optionally color) for the document. Web doc data type must be url from search tool.', - 'Creates one or more documents represented by an array of strings with the provided content based on the instructions ' + - docInstructions + - 'Use if the user wants to create something that aligns with a document type in dash like a flashcard, flashcard deck/stack, or textbox or text document of some sort. Can use after the search tool to save information.' + - 'When user asks for dashboard, create many documents/collections with different colors and texts while listening to their preferences, after using search tool to create a dashboard.' + 'Use the "data" parameter for document content and include title, color, and document dimensions. Ensure web documents use URLs from the search tool if relevant. Each document in a dashboard should be unique and well-differentiated in type and content, without repetition of similar types in any single collection.', + 'When creating a dashboard, ensure that it consists of a broad range of document types. Include a variety of documents, such as text, web, deck, comparison, image, simulation, and equation documents, each with distinct titles and colors, following the user’s preferences. ' + + 'Do not overuse collections or nest all document types within a single collection; instead, represent document types individually. Use this example for reference: ' + + finalJsonString + + '. Create dashboard after search tool.' ); + + // super( + // 'createDoc', + // 'Creates one or more documents that best fit users request with input following the example below. Or creates a dashboard of many documents/collections with this as an example: ' + finalJsonString, + // createListDocToolParams, + // 'Modify the data parameter and include title (and optionally color) for the document. Web doc data type must be url from search tool.', + // 'Creates one or more documents represented by an array of strings with the provided content based on the instructions ' + + // docInstructions + + // 'Use if the user wants to create something that aligns with a document type in dash like a flashcard, flashcard deck/stack, or textbox or text document of some sort. Can use after the search tool to save information.' + + // 'When user asks for dashboard, create many documents/collections with different colors and texts while listening to their preferences, after using search tool to create a dashboard.' + // ); + // this._addLinkedDoc = addLinkedDoc; + + // super( + // 'createDoc', + // 'Creates one or more documents based on the user’s request. If the user asks for a "dashboard," generate multiple documents of different types, such as "text," "flashcard," "image," "web," "comparison," and "collection," and include no more than three web documents unless specified otherwise. For a single document request (like "a text document"), create only the requested document type, without extra templates. Follow the examples below to understand the structure of various document types and their required properties.', + // createListDocToolParams, + // 'Use the "data" parameter to specify document content and include the title (and optionally colors) for each document. For web documents, use a URL from a search tool if applicable.' + + // 'For dashboards, create a diverse array of documents including text, flashcard decks, images, comparisons, and collections, with a mix of colors and unique titles, adhering to the user’s preferences. Example structures are provided for both dashboards and individual document types.' + + // 'Examples:' + + // finalJsonString, + // '' + // ); + + // super( + // 'createDoc', + // 'Creates one or more documents that best fit users request with input following the example below. Or creates a dashboard for the user; there must be different kinds of documents. If the user asks for "a" document, only make one since it is singular - not an additional template one or more documents.', + // createListDocToolParams, + // 'Modify the data parameter and include title (and optionally color) for the document. ' + + // 'Web doc data type must be url from search tool. ' + + // 'Creates one or more documents represented by an array of strings with the provided content based on the instructions ' + + // docInstructions + + // //'. If user asks for dashboard, create many documents/collections, including text, web, image, flashcard, deck, comparison, and collection, after calling the search tool following this example but only have three websites max: ' + + // //'Use if the user wants to create something that aligns with a document type in dash like a flashcard, flashcard deck/stack, or textbox or text document of some sort. Can use after the search tool to save information. If use after the search tool, do not use than more than 3 websites unless specified if dashboard; have text documents, flashcards, and more documents.' + + // // 'When user asks for dashboard, create many documents/collections of different types, such as text, web, flashcard, with different colors and texts while listening to their preferences, after using search tool to create a dashboard. Documents, such as text and web, should not all be in a collection unless specified. Do not have too many websites, and all websites do not need to be in one collection. An example of a dashboard is ' + + // // finalJsonString + // '. If user asks for dashboard, first use search tool. Then create text, web, flashcard, collections, and comparison documents. There absolutely must be different kinds of documents.' + + // 'Do not only have web documents for dashboard; have other kinds of documents. Follow this example: ' + + // finalJsonString + // ); this._addLinkedDoc = addLinkedDoc; } @@ -266,7 +347,16 @@ export class CreateDocTool extends BaseTool { this._addLinkedDoc( doc['doc_type'], doc['data'], - { title: doc['title'], backgroundColor: doc['backgroundColor'], text_fontColor: doc['font_color'], _width: doc['width'], _height: doc['height'], _layout_fitWidth: false, _layout_autoHeight: true }, + { + title: doc['title'], + backgroundColor: doc['backgroundColor'], + text_fontColor: doc['font_color'], + _width: doc['width'], + _height: doc['height'], + type_collection: doc['type_collection'], + _layout_fitWidth: false, + _layout_autoHeight: true, + }, uuidv4() ); }); -- cgit v1.2.3-70-g09d2 From 89424e0a8efc6cf3364a2fd1ffc85c9d0d837453 Mon Sep 17 00:00:00 2001 From: bobzel Date: Fri, 22 Nov 2024 10:27:33 -0500 Subject: added initial Firefly endpoint and hanged smartDrawHandler to generate an image and an svg. --- src/client/util/bezierFit.ts | 3 +- src/client/views/MainView.tsx | 32 ++++++++++++-- .../nodes/chatbot/chatboxcomponents/ChatBox.tsx | 3 +- src/client/views/pdf/AnchorMenu.tsx | 9 ++-- src/client/views/smartdraw/SmartDrawHandler.tsx | 29 ++++++------ src/server/ApiManagers/DataVizManager.ts | 2 +- src/server/ApiManagers/FireflyManager.ts | 51 ++++++++++++++++++++++ src/server/DashUploadUtils.ts | 3 +- src/server/index.ts | 3 +- webpack.config.js | 7 ++- 10 files changed, 113 insertions(+), 29 deletions(-) create mode 100644 src/server/ApiManagers/FireflyManager.ts (limited to 'src/client/views/nodes/chatbot/chatboxcomponents') diff --git a/src/client/util/bezierFit.ts b/src/client/util/bezierFit.ts index d52460023..84b27e84c 100644 --- a/src/client/util/bezierFit.ts +++ b/src/client/util/bezierFit.ts @@ -703,7 +703,6 @@ export function SVGToBezier(name: SVGType, attributes: any): Point[] { coordList.push({ X: parseInt(match[1]), Y: parseInt(match[2]) }); coordList.push({ X: parseInt(match[1]), Y: parseInt(match[2]) }); coordList.push({ X: parseInt(match[3]), Y: parseInt(match[4]) }); - coordList.push({ X: parseInt(match[3]), Y: parseInt(match[4]) }); lastPt = { X: parseInt(match[3]), Y: parseInt(match[4]) }; } else if (match[0].startsWith('C')) { coordList.push({ X: parseInt(match[5]), Y: parseInt(match[6]) }); @@ -720,7 +719,7 @@ export function SVGToBezier(name: SVGType, attributes: any): Point[] { } }); const hasZ = attributes.d.match(/Z/); - if (hasZ) { + if (hasZ || attributes.fill) { coordList.push(lastPt); coordList.push(startPt); coordList.push(startPt); diff --git a/src/client/views/MainView.tsx b/src/client/views/MainView.tsx index 7779d339f..0d071fe4f 100644 --- a/src/client/views/MainView.tsx +++ b/src/client/views/MainView.tsx @@ -8,7 +8,7 @@ import { observer } from 'mobx-react'; import * as React from 'react'; import ResizeObserver from 'resize-observer-polyfill'; import '../../../node_modules/browndash-components/dist/styles/global.min.css'; -import { ClientUtils, lightOrDark, returnEmptyFilter, returnFalse, returnTrue, returnZero, setupMoveUpEvents } from '../../ClientUtils'; +import { ClientUtils, returnEmptyFilter, returnFalse, returnTrue, returnZero, setupMoveUpEvents } from '../../ClientUtils'; import { emptyFunction } from '../../Utils'; import { Doc, DocListCast, GetDocFromUrl, Opt, returnEmptyDoclist } from '../../fields/Doc'; import { DocData } from '../../fields/DocSymbols'; @@ -1023,10 +1023,36 @@ export class MainView extends ObservableReactComponent { {[ ...SnappingManager.HorizSnapLines.map(l => ( - + )), ...SnappingManager.VertSnapLines.map(l => ( - + )), ]} diff --git a/src/client/views/nodes/chatbot/chatboxcomponents/ChatBox.tsx b/src/client/views/nodes/chatbot/chatboxcomponents/ChatBox.tsx index a61705250..3ef6bdd8b 100644 --- a/src/client/views/nodes/chatbot/chatboxcomponents/ChatBox.tsx +++ b/src/client/views/nodes/chatbot/chatboxcomponents/ChatBox.tsx @@ -431,7 +431,7 @@ export class ChatBox extends ViewBoxAnnotatableComponent() { doc = Docs.Create.FunctionPlotDocument([], options); break; case 'dataviz': - case 'data_viz': + case 'data_viz': { const { fileUrl, id } = await Networking.PostToServer('/createCSV', { filename: (options.title as string).replace(/\s+/g, '') + '.csv', data: data, @@ -439,6 +439,7 @@ export class ChatBox extends ViewBoxAnnotatableComponent() { doc = Docs.Create.DataVizDocument(fileUrl, { ...options, text: RTFCast(data) }); this.addCSVForAnalysis(doc, id); break; + } case 'chat': doc = Docs.Create.ChatDocument(options); break; diff --git a/src/client/views/pdf/AnchorMenu.tsx b/src/client/views/pdf/AnchorMenu.tsx index 5ab9b556c..fe03f32a5 100644 --- a/src/client/views/pdf/AnchorMenu.tsx +++ b/src/client/views/pdf/AnchorMenu.tsx @@ -131,12 +131,15 @@ export class AnchorMenu extends AntimodeMenu { /** * Creates a GPT drawing based on selected text. */ - gptDraw = async (e: React.PointerEvent) => { + gptDraw = (e: React.PointerEvent) => { try { SmartDrawHandler.Instance.AddDrawing = this.createDrawingAnnotation; runInAction(() => (this._isLoading = true)); - await SmartDrawHandler.Instance.drawWithGPT({ X: e.clientX, Y: e.clientY }, this._selectedText, 5, 100, true); - runInAction(() => (this._isLoading = false)); + SmartDrawHandler.Instance.drawWithGPT({ X: e.clientX, Y: e.clientY }, this._selectedText, 5, 100, true)?.then( + action(() => { + this._isLoading = false; + }) + ); } catch (err) { console.error(err); } diff --git a/src/client/views/smartdraw/SmartDrawHandler.tsx b/src/client/views/smartdraw/SmartDrawHandler.tsx index d0f6566a5..342b91bd9 100644 --- a/src/client/views/smartdraw/SmartDrawHandler.tsx +++ b/src/client/views/smartdraw/SmartDrawHandler.tsx @@ -13,6 +13,7 @@ import { Doc, DocListCast } from '../../../fields/Doc'; import { DocData } from '../../../fields/DocSymbols'; import { InkData, InkField, InkTool } from '../../../fields/InkField'; import { BoolCast, ImageCast, NumCast, StrCast } from '../../../fields/Types'; +import { Networking } from '../../Network'; import { GPTCallType, gptAPICall, gptDrawingColor } from '../../apis/gpt/GPT'; import { Docs } from '../../documents/Documents'; import { SettingsManager } from '../../util/SettingsManager'; @@ -21,7 +22,8 @@ import { SVGToBezier, SVGType } from '../../util/bezierFit'; import { InkingStroke } from '../InkingStroke'; import { ObservableReactComponent } from '../ObservableReactComponent'; import { MarqueeView } from '../collections/collectionFreeForm'; -import { ActiveInkArrowEnd, ActiveInkArrowStart, ActiveInkDash, ActiveInkFillColor, ActiveInkBezierApprox, ActiveInkColor, ActiveInkWidth, ActiveIsInkMask, DocumentView } from '../nodes/DocumentView'; +import { ActiveInkArrowEnd, ActiveInkArrowStart, ActiveInkBezierApprox, ActiveInkColor, ActiveInkDash, ActiveInkFillColor, ActiveInkWidth, ActiveIsInkMask, DocumentView, DocumentViewInternal } from '../nodes/DocumentView'; +import { OpenWhere } from '../nodes/OpenWhere'; import './SmartDrawHandler.scss'; export interface DrawingOptions { @@ -230,20 +232,21 @@ export class SmartDrawHandler extends ObservableReactComponent { * Calls GPT API to create a drawing based on user input. */ @action - drawWithGPT = async (startPt: { X: number; Y: number }, input: string, complexity: number, size: number, autoColor: boolean) => { - if (input === '') return; - this._lastInput = { text: input, complexity: complexity, size: size, autoColor: autoColor, x: startPt.X, y: startPt.Y }; - const res = await gptAPICall(`"${input}", "${complexity}", "${size}"`, GPTCallType.DRAW, undefined, true); - if (!res) { - console.error('GPT call failed'); - return; - } - const strokeData = await this.parseSvg(res, startPt, false, autoColor); - const drawingDoc = strokeData && this.CreateDrawingDoc(strokeData.data, strokeData.lastInput, strokeData.lastRes); - drawingDoc && this.AddDrawing(drawingDoc, this._lastInput, res); + drawWithGPT = (startPt: { X: number; Y: number }, prompt: string, complexity: number, size: number, autoColor: boolean) => { + if (prompt === '') return; + this._lastInput = { text: prompt, complexity: complexity, size: size, autoColor: autoColor, x: startPt.X, y: startPt.Y }; + + Networking.PostToServer('/queryFireflyImage', { prompt }).then(img => DocumentViewInternal.addDocTabFunc(Docs.Create.ImageDocument(img, { title: prompt }), OpenWhere.addRight)); + + const result = gptAPICall(`"${prompt}", "${complexity}", "${size}"`, GPTCallType.DRAW, undefined, true).then(res => + this.parseSvg(res, startPt, false, autoColor).then(strokeData => { + const drawingDoc = strokeData && this.CreateDrawingDoc(strokeData.data, strokeData.lastInput, strokeData.lastRes); + drawingDoc && this.AddDrawing(drawingDoc, this._lastInput, res); + }) + ); this._errorOccurredOnce = false; - return strokeData; + return result; }; /** diff --git a/src/server/ApiManagers/DataVizManager.ts b/src/server/ApiManagers/DataVizManager.ts index 88f22992d..d2028f23b 100644 --- a/src/server/ApiManagers/DataVizManager.ts +++ b/src/server/ApiManagers/DataVizManager.ts @@ -9,7 +9,7 @@ export default class DataVizManager extends ApiManager { register({ method: Method.GET, subscription: '/csvData', - secureHandler: async ({ req, res }) => { + secureHandler: ({ req, res }) => { const uri = req.query.uri as string; return new Promise(resolve => { diff --git a/src/server/ApiManagers/FireflyManager.ts b/src/server/ApiManagers/FireflyManager.ts new file mode 100644 index 000000000..04fa8f065 --- /dev/null +++ b/src/server/ApiManagers/FireflyManager.ts @@ -0,0 +1,51 @@ +import { DashUploadUtils } from '../DashUploadUtils'; +import { _invalid, _success, Method } from '../RouteManager'; +import ApiManager, { Registration } from './ApiManager'; + +export default class FireflyManager extends ApiManager { + askFirefly = (prompt: string = 'a realistic illustration of a cat coding') => { + const fetched = fetch('https://ims-na1.adobelogin.com/ims/token/v3', { + method: 'POST', + headers: { + 'Content-Type': 'application/x-www-form-urlencoded', + }, + body: `grant_type=client_credentials&client_id=${process.env._CLIENT_FIREFLY_CLIENT_ID}&client_secret=${process.env._CLIENT_FIREFLY_SECRET}&scope=openid,AdobeID,session,additional_info,read_organizations,firefly_api,ff_apis`, + }) + .then(response => response.json()) + .then((data: { access_token: string }) => + fetch('https://firefly-api.adobe.io/v3/images/generate', { + method: 'POST', + headers: [ + ['Content-Type', 'application/json'], + ['Accept', 'application/json'], + ['x-api-key', process.env._CLIENT_FIREFLY_CLIENT_ID ?? ''], + ['Authorization', `Bearer ${data.access_token}`], + ], + body: `{ "prompt": "${prompt}" }`, + }) + .then(response => response.json().then(json => JSON.stringify((json.outputs?.[0] as { image: { url: string } })?.image))) + .catch(error => { + console.error('Error:', error); + return ''; + }) + ) + .catch(error => { + console.error('Error:', error); + return ''; + }); + return fetched; + }; + protected initialize(register: Registration): void { + register({ + method: Method.POST, + subscription: '/queryFireflyImage', + secureHandler: ({ req, res }) => + this.askFirefly(req.body.prompt).then(fire => + DashUploadUtils.UploadImage(JSON.parse(fire).url).then(info => { + if (info instanceof Error) _invalid(res, info.message); + else _success(res, info.accessPaths.agnostic.client); + }) + ), + }); + } +} diff --git a/src/server/DashUploadUtils.ts b/src/server/DashUploadUtils.ts index 1e55a885a..032d13d43 100644 --- a/src/server/DashUploadUtils.ts +++ b/src/server/DashUploadUtils.ts @@ -369,7 +369,8 @@ export namespace DashUploadUtils { */ export const UploadInspectedImage = async (metadata: Upload.InspectionResults, filename: string, prefix = '', cleanUp = true): Promise => { const { requestable, source, ...remaining } = metadata; - const resolved = filename || `${prefix}upload_${Utils.GenerateGuid()}.${remaining.contentType.split('/')[1].toLowerCase()}`; + const dfltSuffix = remaining.contentType.split('/')[1].toLowerCase(); + const resolved = filename || `${prefix}upload_${Utils.GenerateGuid()}.${dfltSuffix === 'xml' ? 'jpg' : dfltSuffix}`; const { images } = Directory; const information: Upload.ImageInformation = { accessPaths: { diff --git a/src/server/index.ts b/src/server/index.ts index 88dbd232d..1f9af9ee0 100644 --- a/src/server/index.ts +++ b/src/server/index.ts @@ -7,6 +7,7 @@ import AssistantManager from './ApiManagers/AssistantManager'; import DataVizManager from './ApiManagers/DataVizManager'; import DeleteManager from './ApiManagers/DeleteManager'; import DownloadManager from './ApiManagers/DownloadManager'; +import FireflyManager from './ApiManagers/FireflyManager'; import GeneralGoogleManager from './ApiManagers/GeneralGoogleManager'; import SessionManager from './ApiManagers/SessionManager'; import UploadManager from './ApiManagers/UploadManager'; @@ -71,6 +72,7 @@ function routeSetter({ addSupervisedRoute, logRegistrationOutcome }: RouteManage new GeneralGoogleManager(), /* new GooglePhotosManager(), */ new DataVizManager(), new AssistantManager(), + new FireflyManager(), ]; // initialize API Managers @@ -112,7 +114,6 @@ function routeSetter({ addSupervisedRoute, logRegistrationOutcome }: RouteManage }); const serve: PublicHandler = ({ req, res }) => { - // eslint-disable-next-line new-cap const detector = new mobileDetect(req.headers['user-agent'] || ''); const filename = detector.mobile() !== null ? 'mobile/image.html' : 'index.html'; res.sendFile(path.join(__dirname, '../../deploy/' + filename)); diff --git a/webpack.config.js b/webpack.config.js index e1afc64e5..67417fb02 100644 --- a/webpack.config.js +++ b/webpack.config.js @@ -1,4 +1,4 @@ -/* eslint-disable @typescript-eslint/no-var-requires */ +/* eslint-disable @typescript-eslint/no-require-imports */ const path = require('path'); const webpack = require('webpack'); const HtmlWebpackPlugin = require('html-webpack-plugin'); @@ -36,7 +36,6 @@ function transferEnvironmentVariables() { } const resolvedClientSide = Object.keys(parsed).reduce((mapping, envKey) => { if (envKey.startsWith(prefix)) { - // eslint-disable-next-line mapping[`process.env.${envKey.replace(prefix, '')}`] = JSON.stringify(parsed[envKey]); } return mapping; @@ -112,7 +111,7 @@ module.exports = { test: /\.scss|css$/, exclude: /\.module\.scss$/i, use: [ - { loader: 'style-loader' }, // eslint-disable-next-line prettier/prettier + { loader: 'style-loader' }, // { loader: 'css-loader' }, { loader: 'sass-loader' }, ], @@ -127,7 +126,7 @@ module.exports = { { test: /\.module\.scss$/i, use: [ - { loader: 'style-loader' }, // eslint-disable-next-line prettier/prettier + { loader: 'style-loader' }, // { loader: 'css-loader', options: { modules: true } }, { loader: 'sass-loader' }, ], -- cgit v1.2.3-70-g09d2 From 8c0a4a773433f52bd3fdf85509bbd9e3117d5dda Mon Sep 17 00:00:00 2001 From: alyssaf16 Date: Tue, 26 Nov 2024 15:31:07 -0500 Subject: website glitch fixed --- .../views/nodes/chatbot/chatboxcomponents/ChatBox.tsx | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) (limited to 'src/client/views/nodes/chatbot/chatboxcomponents') diff --git a/src/client/views/nodes/chatbot/chatboxcomponents/ChatBox.tsx b/src/client/views/nodes/chatbot/chatboxcomponents/ChatBox.tsx index 594736fbc..7859eae01 100644 --- a/src/client/views/nodes/chatbot/chatboxcomponents/ChatBox.tsx +++ b/src/client/views/nodes/chatbot/chatboxcomponents/ChatBox.tsx @@ -506,6 +506,7 @@ export class ChatBox extends ViewBoxAnnotatableComponent() { break; } case 'web': + options.data_useCors = true; doc = DocCast(Docs.Create.WebDocument(data, options)); break; case 'comparison': @@ -579,10 +580,14 @@ export class ChatBox extends ViewBoxAnnotatableComponent() { const flashcardArray = Array.isArray(deckData) ? deckData : Object.values(deckData); console.log(typeof flashcardArray); // Process each flashcard document in the `deckData` array - flashcardArray.forEach(doc => { - const flashcardDoc = this.createFlashcard(doc, options); - if (flashcardDoc) flashcardDeck.push(flashcardDoc); - }); + if (flashcardArray.length == 2 && flashcardArray[0].doc_type == 'text' && flashcardArray[1].doc_type == 'text') { + this.createFlashcard(flashcardArray, options); + } else { + flashcardArray.forEach(doc => { + const flashcardDoc = this.createFlashcard(doc, options); + if (flashcardDoc) flashcardDeck.push(flashcardDoc); + }); + } // Create a carousel to contain the flashcard deck const carouselDoc = DocCast( @@ -603,6 +608,7 @@ export class ChatBox extends ViewBoxAnnotatableComponent() { // Process each flashcard item in the data array // const p = JSON.parse(data); + const deckData = typeof data === 'string' ? JSON.parse(data) : data; const flashcardArray = Array.isArray(deckData) ? deckData : Object.values(deckData)[2]; console.log(typeof flashcardArray); -- cgit v1.2.3-70-g09d2 From 91b478e7bf69c2b4fe52185a0dd35ce8035b534b Mon Sep 17 00:00:00 2001 From: alyssaf16 Date: Mon, 13 Jan 2025 16:52:46 -0500 Subject: commit chatbox --- .../nodes/chatbot/chatboxcomponents/ChatBox.tsx | 108 ++++++++------------- 1 file changed, 40 insertions(+), 68 deletions(-) (limited to 'src/client/views/nodes/chatbot/chatboxcomponents') diff --git a/src/client/views/nodes/chatbot/chatboxcomponents/ChatBox.tsx b/src/client/views/nodes/chatbot/chatboxcomponents/ChatBox.tsx index 7859eae01..542d8ea58 100644 --- a/src/client/views/nodes/chatbot/chatboxcomponents/ChatBox.tsx +++ b/src/client/views/nodes/chatbot/chatboxcomponents/ChatBox.tsx @@ -16,7 +16,7 @@ import { v4 as uuidv4 } from 'uuid'; import { ClientUtils } from '../../../../../ClientUtils'; import { Doc, DocListCast } from '../../../../../fields/Doc'; import { DocData, DocViews } from '../../../../../fields/DocSymbols'; -import { CsvCast, DocCast, PDFCast, RTFCast, StrCast } from '../../../../../fields/Types'; +import { CsvCast, DocCast, PDFCast, RTFCast, StrCast, NumCast } from '../../../../../fields/Types'; import { Networking } from '../../../../Network'; import { DocUtils } from '../../../../documents/DocUtils'; import { DocumentType } from '../../../../documents/DocumentTypes'; @@ -400,29 +400,17 @@ export class ChatBox extends ViewBoxAnnotatableComponent() { * @param options Other optional document options (e.g. color) * @param id The unique ID for the document. */ - - // @action - // createDocInDash = async (docs: string[]) => { - // console.log('DOCS HERE' + docs); - // docs.forEach(doc => { - // const parsedDoc = JSON.parse(doc); - // this.createIndivDocInDash(parsedDoc.doc_type, parsedDoc.data, parsedDoc.options, ''); - // }); - // }; @action - private createCollectionWithChildren = async (data: any): Promise => { - console.log('Creating collection with nested documents'); - + private createCollectionWithChildren = async (data: any, insideCol: boolean): Promise => { // Create an array of promises for each document const childDocPromises = data.map(async doc => { const parsedDoc = doc; - console.log('Parse #3: ' + parsedDoc); if (parsedDoc.doc_type !== 'collection') { // Handle non-collection documents - return await this.whichDoc(parsedDoc.doc_type, parsedDoc.data, { backgroundColor: parsedDoc.backgroundColor, _width: parsedDoc.width, _height: parsedDoc.height }, parsedDoc.id); + return await this.whichDoc(parsedDoc.doc_type, parsedDoc.data, { backgroundColor: parsedDoc.backgroundColor, _width: parsedDoc.width, _height: parsedDoc.height }, parsedDoc.id, insideCol); } else { // Recursively process collections - const nestedDocs = await this.createCollectionWithChildren(parsedDoc.data); + const nestedDocs = await this.createCollectionWithChildren(parsedDoc.data, true); const collectionOptions: DocumentOptions = { title: parsedDoc.title, backgroundColor: parsedDoc.backgroundColor, @@ -432,16 +420,14 @@ export class ChatBox extends ViewBoxAnnotatableComponent() { _freeform_backgroundGrid: true, }; const collectionDoc = DocCast(Docs.Create.FreeformDocument(nestedDocs, collectionOptions)); - return collectionDoc; // Return th + return collectionDoc; } }); // Await all child document creations concurrently const nestedResults = await Promise.all(childDocPromises); - console.log('n' + nestedResults); // Flatten any nested arrays from recursive collection calls const childDocs = nestedResults.flat() as Doc[]; - console.log('c' + childDocs); childDocs.forEach(doc => { console.log(DocCast(doc)); console.log(DocCast(doc)[DocData].data); @@ -456,7 +442,7 @@ export class ChatBox extends ViewBoxAnnotatableComponent() { // } @action - whichDoc = async (doc_type: string, data: string, options: DocumentOptions, id: string): Promise => { + whichDoc = async (doc_type: string, data: string, options: DocumentOptions, id: string, insideCol: boolean): Promise => { let doc; switch (doc_type) { case 'text': @@ -472,20 +458,16 @@ export class ChatBox extends ViewBoxAnnotatableComponent() { doc = DocCast(Docs.Create.ImageDocument(data, options)); break; case 'equation': - // make more advanced doc = DocCast(Docs.Create.EquationDocument(data, options)); break; case 'noteboard': - // COME BACK doc = DocCast(Docs.Create.NoteTakingDocument([], options)); break; case 'simulation': - // make more advanced doc = DocCast(Docs.Create.SimulationDocument(options)); break; case 'collection': { - // COME BACK - const arr = await this.createCollectionWithChildren(data); + const arr = await this.createCollectionWithChildren(data, true); options._layout_fitWidth = true; options._freeform_backgroundGrid = true; if (options.type_collection == 'tree') { @@ -513,11 +495,9 @@ export class ChatBox extends ViewBoxAnnotatableComponent() { doc = this.createComparison(data, options); break; case 'diagram': - // come back doc = Docs.Create.DiagramDocument(options); break; case 'audio': - // come back doc = Docs.Create.AudioDocument(data, options); break; case 'map': @@ -553,32 +533,42 @@ export class ChatBox extends ViewBoxAnnotatableComponent() { default: doc = DocCast(Docs.Create.TextDocument(data, options)); } + doc!.x = NumCast(options.x ?? 0) + (insideCol ? 0 : NumCast(this.layoutDoc.x) + NumCast(this.layoutDoc.width)) + 100; + doc!.y = NumCast(options.y) + (insideCol ? 0 : NumCast(this.layoutDoc.y)); return doc; }; + /** + * Creates a document in the dashboard. + * + * @param {string} doc_type - The type of document to create. + * @param {string} data - The data used to generate the document. + * @param {DocumentOptions} options - Configuration options for the document. + * @param {string} id - Unique identifier for the document. + * @returns {Promise} A promise that resolves once the document is created and displayed. + */ @action createDocInDash = async (doc_type: string, data: string, options: DocumentOptions, id: string) => { - console.log('INDIV DOC' + doc_type); - const doc = await this.whichDoc(doc_type, data, options, id); - - console.log('DOC' + doc_type); const linkDoc = Docs.Create.LinkDocument(this.Document, doc); LinkManager.Instance.addLink(linkDoc); - doc && this._props.addDocument?.(doc); await DocumentManager.Instance.showDocument(doc, { willZoomCentered: true }, () => {}); }; + /** + * Creates a deck of flashcards. + * + * @param {any} data - The data used to generate the flashcards. Can be a string or an object. + * @param {DocumentOptions} options - Configuration options for the flashcard deck. + * @returns {Doc} A carousel document containing the flashcard deck. + */ @action createDeck = (data: any, options: DocumentOptions) => { const flashcardDeck: Doc[] = []; - // Parse `data` only if it’s a string const deckData = typeof data === 'string' ? JSON.parse(data) : data; - console.log('Parsed Deck Data:', deckData); const flashcardArray = Array.isArray(deckData) ? deckData : Object.values(deckData); - console.log(typeof flashcardArray); // Process each flashcard document in the `deckData` array if (flashcardArray.length == 2 && flashcardArray[0].doc_type == 'text' && flashcardArray[1].doc_type == 'text') { this.createFlashcard(flashcardArray, options); @@ -599,28 +589,22 @@ export class ChatBox extends ViewBoxAnnotatableComponent() { _layout_autoHeight: true, }) ); - return carouselDoc; }; + + /** + * Creates a single flashcard document. + * + * @param {any} data - The data used to generate the flashcard. Can be a string or an object. + * @param {any} options - Configuration options for the flashcard. + * @returns {Doc | undefined} The created flashcard document, or undefined if the flashcard cannot be created. + */ @action createFlashcard = (data: any, options: any) => { - // const flashcardDeck: Doc[] = []; - - // Process each flashcard item in the data array - // const p = JSON.parse(data); - const deckData = typeof data === 'string' ? JSON.parse(data) : data; const flashcardArray = Array.isArray(deckData) ? deckData : Object.values(deckData)[2]; - console.log(typeof flashcardArray); - const [front, back] = flashcardArray; - // Check that both front and back are text documents - console.log('DATA' + data); - console.log('front' + front); - console.log('back' + back); - console.log(front.doc_type); - console.log(back.doc_type); if (front.doc_type === 'text' && back.doc_type === 'text') { const sideOptions: DocumentOptions = { backgroundColor: options.backgroundColor, @@ -635,36 +619,24 @@ export class ChatBox extends ViewBoxAnnotatableComponent() { // Create the flashcard document with both sides const flashcardDoc = DocCast(Docs.Create.FlashcardDocument(data.title, side1, side2, sideOptions)); return flashcardDoc; - // this._props.addDocument?.(flashcardDoc); - // flashcardDeck.push(flashcardDoc); } - - // Create a carousel to contain the flashcard deck - // const carouselDoc = DocCast( - // Docs.Create.CarouselDocument(flashcardDeck, { - // title: options.title || data.title, - // _width: data.width || 300, - // _height: data.height || 300, - // _layout_fitWidth: false, - // _layout_autoHeight: true, - // }) - // ); - - // return carouselDoc; }; + /** + * Creates a comparison document. + * + * @param {any} doc - The document data containing left and right components for comparison. + * @param {any} options - Configuration options for the comparison document. + * @returns {Doc} The created comparison document. + */ @action createComparison = (doc: any, options: any) => { const comp = Docs.Create.ComparisonDocument(options.title, { _width: options.width, _height: options.height | 300, backgroundColor: options.backgroundColor }); const [left, right] = doc; - console.log(DocCast(comp.dataDoc)); - console.log(DocCast(comp[DocData])); - console.log(DocCast(comp[DocData].data_back)); const docLeft = DocCast(Docs.Create.TextDocument(left.data, { backgroundColor: left.backgroundColor, _width: left.width, _height: left.height })); const docRight = DocCast(Docs.Create.TextDocument(right.data, { backgroundColor: right.backgroundColor, _width: right.width, _height: right.height })); comp[DocData].data_back = docLeft; comp[DocData].data_front = docRight; - return comp; }; -- cgit v1.2.3-70-g09d2 From 9ef41b53aab0a5c7a3f429bb97bcd029bc06caeb Mon Sep 17 00:00:00 2001 From: bobzel Date: Tue, 21 Jan 2025 10:28:58 -0500 Subject: fixed merge error --- src/client/views/nodes/chatbot/chatboxcomponents/ChatBox.tsx | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) (limited to 'src/client/views/nodes/chatbot/chatboxcomponents') diff --git a/src/client/views/nodes/chatbot/chatboxcomponents/ChatBox.tsx b/src/client/views/nodes/chatbot/chatboxcomponents/ChatBox.tsx index 6d5290c95..b89498d7d 100644 --- a/src/client/views/nodes/chatbot/chatboxcomponents/ChatBox.tsx +++ b/src/client/views/nodes/chatbot/chatboxcomponents/ChatBox.tsx @@ -548,8 +548,6 @@ export class ChatBox extends ViewBoxAnnotatableComponent() { * @returns {Promise} A promise that resolves once the document is created and displayed. */ @action - createDocInDash = async (doc_type: string, data: string, options: DocumentOptions, id: string) => { - const doc = await this.whichDoc(doc_type, data, options, id); createDocInDash = async (doc_type: string, data: string | undefined, options: DocumentOptions, id: string) => { let doc; @@ -914,16 +912,14 @@ export class ChatBox extends ViewBoxAnnotatableComponent() { /** * Getter that retrieves all linked CSV files for analysis. */ - @computed - get linkedCSVs(): { filename: string; id: string; text: string }[] { + @computed get linkedCSVs(): { filename: string; id: string; text: string }[] { return this.linked_csv_files; } /** * Getter that formats the entire chat history as a string for the agent's system message. */ - @computed - get formattedHistory(): string { + @computed get formattedHistory(): string { let history = '\n'; for (const message of this.history) { history += `<${message.role}>${message.content.map(content => content.text).join(' ')}`; -- cgit v1.2.3-70-g09d2 From ec0ab50aad9fbb55477476998c6932488b149f45 Mon Sep 17 00:00:00 2001 From: bobzel Date: Tue, 21 Jan 2025 12:03:44 -0500 Subject: trying to cleanup chatBox code and types --- .../views/nodes/chatbot/agentsystem/Agent.ts | 15 +- .../nodes/chatbot/chatboxcomponents/ChatBox.tsx | 472 +++++++++------------ 2 files changed, 203 insertions(+), 284 deletions(-) (limited to 'src/client/views/nodes/chatbot/chatboxcomponents') diff --git a/src/client/views/nodes/chatbot/agentsystem/Agent.ts b/src/client/views/nodes/chatbot/agentsystem/Agent.ts index a2a575f19..4d3f1e4e7 100644 --- a/src/client/views/nodes/chatbot/agentsystem/Agent.ts +++ b/src/client/views/nodes/chatbot/agentsystem/Agent.ts @@ -1,25 +1,22 @@ import dotenv from 'dotenv'; import { XMLBuilder, XMLParser } from 'fast-xml-parser'; +import { escape } from 'lodash'; // Imported escape from lodash import OpenAI from 'openai'; import { ChatCompletionMessageParam } from 'openai/resources'; -import { escape } from 'lodash'; // Imported escape from lodash +import { DocumentOptions } from '../../../../documents/Documents'; import { AnswerParser } from '../response_parsers/AnswerParser'; import { StreamedAnswerParser } from '../response_parsers/StreamedAnswerParser'; +import { BaseTool } from '../tools/BaseTool'; import { CalculateTool } from '../tools/CalculateTool'; -import { CreateCSVTool } from '../tools/CreateCSVTool'; +import { CreateAnyDocumentTool } from '../tools/CreateAnyDocTool'; +import { CreateDocTool } from '../tools/CreateDocumentTool'; import { DataAnalysisTool } from '../tools/DataAnalysisTool'; import { NoTool } from '../tools/NoTool'; -import { RAGTool } from '../tools/RAGTool'; import { SearchTool } from '../tools/SearchTool'; -import { WebsiteInfoScraperTool } from '../tools/WebsiteInfoScraperTool'; +import { Parameter, ParametersType, TypeMap } from '../types/tool_types'; import { AgentMessage, ASSISTANT_ROLE, AssistantMessage, Observation, PROCESSING_TYPE, ProcessingInfo, TEXT_TYPE } from '../types/types'; import { Vectorstore } from '../vectorstore/Vectorstore'; import { getReactPrompt } from './prompts'; -import { BaseTool } from '../tools/BaseTool'; -import { Parameter, ParametersType, TypeMap } from '../types/tool_types'; -import { CreateDocTool } from '../tools/CreateDocumentTool'; -import { DocumentOptions } from '../../../../documents/Documents'; -import { CreateAnyDocumentTool } from '../tools/CreateAnyDocTool'; dotenv.config(); diff --git a/src/client/views/nodes/chatbot/chatboxcomponents/ChatBox.tsx b/src/client/views/nodes/chatbot/chatboxcomponents/ChatBox.tsx index b89498d7d..83b50c8c6 100644 --- a/src/client/views/nodes/chatbot/chatboxcomponents/ChatBox.tsx +++ b/src/client/views/nodes/chatbot/chatboxcomponents/ChatBox.tsx @@ -14,12 +14,12 @@ import OpenAI, { ClientOptions } from 'openai'; import * as React from 'react'; import { v4 as uuidv4 } from 'uuid'; import { ClientUtils } from '../../../../../ClientUtils'; -import { Doc, DocListCast } from '../../../../../fields/Doc'; +import { Doc, DocListCast, Opt } from '../../../../../fields/Doc'; import { DocData, DocViews } from '../../../../../fields/DocSymbols'; -import { CsvCast, DocCast, PDFCast, RTFCast, StrCast, NumCast } from '../../../../../fields/Types'; +import { CsvCast, DocCast, NumCast, PDFCast, RTFCast, StrCast } from '../../../../../fields/Types'; import { Networking } from '../../../../Network'; import { DocUtils } from '../../../../documents/DocUtils'; -import { DocumentType } from '../../../../documents/DocumentTypes'; +import { CollectionViewType, DocumentType } from '../../../../documents/DocumentTypes'; import { Docs, DocumentOptions } from '../../../../documents/Documents'; import { DocumentManager } from '../../../../util/DocumentManager'; import { LinkManager } from '../../../../util/LinkManager'; @@ -33,7 +33,6 @@ import { Vectorstore } from '../vectorstore/Vectorstore'; import './ChatBox.scss'; import MessageComponentBox from './MessageComponent'; import { ProgressBar } from './ProgressBar'; -import { RichTextField } from '../../../../../fields/RichTextField'; dotenv.config(); @@ -45,17 +44,17 @@ dotenv.config(); @observer export class ChatBox extends ViewBoxAnnotatableComponent() { // MobX observable properties to track UI state and data - @observable history: AssistantMessage[] = []; - @observable.deep current_message: AssistantMessage | undefined = undefined; - @observable isLoading: boolean = false; - @observable uploadProgress: number = 0; - @observable currentStep: string = ''; - @observable expandedScratchpadIndex: number | null = null; - @observable inputValue: string = ''; - @observable private linked_docs_to_add: ObservableSet = observable.set(); - @observable private linked_csv_files: { filename: string; id: string; text: string }[] = []; - @observable private isUploadingDocs: boolean = false; - @observable private citationPopup: { text: string; visible: boolean } = { text: '', visible: false }; + @observable private _history: AssistantMessage[] = []; + @observable.deep private _current_message: AssistantMessage | undefined = undefined; + @observable private _isLoading: boolean = false; + @observable private _uploadProgress: number = 0; + @observable private _currentStep: string = ''; + @observable private _expandedScratchpadIndex: number | null = null; + @observable private _inputValue: string = ''; + @observable private _linked_docs_to_add: ObservableSet = observable.set(); + @observable private _linked_csv_files: { filename: string; id: string; text: string }[] = []; + @observable private _isUploadingDocs: boolean = false; + @observable private _citationPopup: { text: string; visible: boolean } = { text: '', visible: false }; // Private properties for managing OpenAI API, vector store, agent, and UI elements private openai: OpenAI; @@ -96,7 +95,7 @@ export class ChatBox extends ViewBoxAnnotatableComponent() { // Reaction to update dataDoc when chat history changes reaction( () => - this.history.map((msg: AssistantMessage) => ({ + this._history.map((msg: AssistantMessage) => ({ role: msg.role, content: msg.content, follow_up_questions: msg.follow_up_questions, @@ -115,20 +114,20 @@ export class ChatBox extends ViewBoxAnnotatableComponent() { */ @action addDocToVectorstore = async (newLinkedDoc: Doc) => { - this.uploadProgress = 0; - this.currentStep = 'Initializing...'; - this.isUploadingDocs = true; + this._uploadProgress = 0; + this._currentStep = 'Initializing...'; + this._isUploadingDocs = true; try { // Add the document to the vectorstore await this.vectorstore.addAIDoc(newLinkedDoc, this.updateProgress); } catch (error) { console.error('Error uploading document:', error); - this.currentStep = 'Error during upload'; + this._currentStep = 'Error during upload'; } finally { - this.isUploadingDocs = false; - this.uploadProgress = 0; - this.currentStep = ''; + this._isUploadingDocs = false; + this._uploadProgress = 0; + this._currentStep = ''; } }; @@ -139,8 +138,8 @@ export class ChatBox extends ViewBoxAnnotatableComponent() { */ @action updateProgress = (progress: number, step: string) => { - this.uploadProgress = progress; - this.currentStep = step; + this._uploadProgress = progress; + this._currentStep = step; }; /** @@ -177,7 +176,7 @@ export class ChatBox extends ViewBoxAnnotatableComponent() { const csvId = id ?? uuidv4(); // Add CSV details to linked files - this.linked_csv_files.push({ + this._linked_csv_files.push({ filename: CsvCast(newLinkedDoc.data).url.pathname, id: csvId, text: csvData, @@ -199,7 +198,7 @@ export class ChatBox extends ViewBoxAnnotatableComponent() { */ @action toggleToolLogs = (index: number) => { - this.expandedScratchpadIndex = this.expandedScratchpadIndex === index ? null : index; + this._expandedScratchpadIndex = this._expandedScratchpadIndex === index ? null : index; }; /** @@ -258,7 +257,7 @@ export class ChatBox extends ViewBoxAnnotatableComponent() { @action askGPT = async (event: React.FormEvent): Promise => { event.preventDefault(); - this.inputValue = ''; + this._inputValue = ''; // Extract the user's message const textInput = (event.currentTarget as HTMLFormElement).elements.namedItem('messageInput') as HTMLInputElement; @@ -268,13 +267,13 @@ export class ChatBox extends ViewBoxAnnotatableComponent() { try { textInput.value = ''; // Add the user's message to the history - this.history.push({ + this._history.push({ role: ASSISTANT_ROLE.USER, content: [{ index: 0, type: TEXT_TYPE.NORMAL, text: trimmedText, citation_ids: null }], processing_info: [], }); - this.isLoading = true; - this.current_message = { + this._isLoading = true; + this._current_message = { role: ASSISTANT_ROLE.ASSISTANT, content: [], citations: [], @@ -284,9 +283,9 @@ export class ChatBox extends ViewBoxAnnotatableComponent() { // Define callbacks for real-time processing updates const onProcessingUpdate = (processingUpdate: ProcessingInfo[]) => { runInAction(() => { - if (this.current_message) { - this.current_message = { - ...this.current_message, + if (this._current_message) { + this._current_message = { + ...this._current_message, processing_info: processingUpdate, }; } @@ -296,9 +295,9 @@ export class ChatBox extends ViewBoxAnnotatableComponent() { const onAnswerUpdate = (answerUpdate: string) => { runInAction(() => { - if (this.current_message) { - this.current_message = { - ...this.current_message, + if (this._current_message) { + this._current_message = { + ...this._current_message, content: [{ text: answerUpdate, type: TEXT_TYPE.NORMAL, index: 0, citation_ids: [] }], }; } @@ -310,22 +309,22 @@ export class ChatBox extends ViewBoxAnnotatableComponent() { // Update the history with the final assistant message runInAction(() => { - if (this.current_message) { - this.history.push({ ...finalMessage }); - this.current_message = undefined; - this.dataDoc.data = JSON.stringify(this.history); + if (this._current_message) { + this._history.push({ ...finalMessage }); + this._current_message = undefined; + this.dataDoc.data = JSON.stringify(this._history); } }); } catch (err) { console.error('Error:', err); // Handle error in processing - this.history.push({ + this._history.push({ role: ASSISTANT_ROLE.ASSISTANT, content: [{ index: 0, type: TEXT_TYPE.ERROR, text: 'Sorry, I encountered an error while processing your request.', citation_ids: null }], processing_info: [], }); } finally { - this.isLoading = false; + this._isLoading = false; this.scrollToBottom(); } } @@ -339,8 +338,8 @@ export class ChatBox extends ViewBoxAnnotatableComponent() { */ @action updateMessageCitations = (index: number, citations: Citation[]) => { - if (this.history[index]) { - this.history[index].citations = citations; + if (this._history[index]) { + this._history[index].citations = citations; } }; @@ -381,17 +380,14 @@ export class ChatBox extends ViewBoxAnnotatableComponent() { * @param data The CSV data content. */ @action - createCSVInDash = async (url: string, title: string, id: string, data: string) => { - const doc = DocCast(await DocUtils.DocumentFromType('csv', url, { title: title, text: RTFCast(data) })); - - const linkDoc = Docs.Create.LinkDocument(this.Document, doc); - LinkManager.Instance.addLink(linkDoc); - - doc && this._props.addDocument?.(doc); - await DocumentManager.Instance.showDocument(doc, { willZoomCentered: true }, () => {}); - - this.addCSVForAnalysis(doc, id); - }; + createCSVInDash = (url: string, title: string, id: string, data: string) => + DocUtils.DocumentFromType('csv', url, { title: title, text: RTFCast(data) }).then(doc => { + if (doc) { + LinkManager.Instance.addLink(Docs.Create.LinkDocument(this.Document, doc)); + this._props.addDocument?.(doc); + DocumentManager.Instance.showDocument(doc, { willZoomCentered: true }, () => {}).then(() => this.addCSVForAnalysis(doc, id)); + } + }); /** * Creates a text document in the dashboard and adds it for analysis. @@ -401,40 +397,35 @@ export class ChatBox extends ViewBoxAnnotatableComponent() { * @param id The unique ID for the document. */ @action - private createCollectionWithChildren = async (data: any, insideCol: boolean): Promise => { - // Create an array of promises for each document - const childDocPromises = data.map(async doc => { - const parsedDoc = doc; - if (parsedDoc.doc_type !== 'collection') { - // Handle non-collection documents - return await this.whichDoc(parsedDoc.doc_type, parsedDoc.data, { backgroundColor: parsedDoc.backgroundColor, _width: parsedDoc.width, _height: parsedDoc.height }, parsedDoc.id, insideCol); - } else { - // Recursively process collections - const nestedDocs = await this.createCollectionWithChildren(parsedDoc.data, true); - const collectionOptions: DocumentOptions = { - title: parsedDoc.title, - backgroundColor: parsedDoc.backgroundColor, - _width: parsedDoc.width, - _height: parsedDoc.height, - _layout_fitWidth: true, - _freeform_backgroundGrid: true, - }; - const collectionDoc = DocCast(Docs.Create.FreeformDocument(nestedDocs, collectionOptions)); - return collectionDoc; - } - }); - - // Await all child document creations concurrently - const nestedResults = await Promise.all(childDocPromises); - // Flatten any nested arrays from recursive collection calls - const childDocs = nestedResults.flat() as Doc[]; - childDocs.forEach(doc => { - console.log(DocCast(doc)); - console.log(DocCast(doc)[DocData].data); - console.log(DocCast(doc)[DocData].data); - }); - return childDocs; - }; + private createCollectionWithChildren = (data: { doc_type: string; id: string; data: any; title: string; width: number; height: number; backgroundColor: string }[], insideCol: boolean): Promise => + Promise.all( + data.map(doc => + doc.doc_type !== 'collection' // Handle non-collection documents + ? this.whichDoc(doc.doc_type, doc.data, { backgroundColor: doc.backgroundColor, _width: doc.width, _height: doc.height }, doc.id, insideCol) + : // Recursively process collections + this.createCollectionWithChildren(doc.data, true).then(nestedDocs => + Docs.Create.FreeformDocument(nestedDocs, { + title: doc.title, + backgroundColor: doc.backgroundColor, + _width: doc.width, + _height: doc.height, + _layout_fitWidth: true, + _freeform_backgroundGrid: true, + }) + ) + ) + .flat() // prettier-ignore + ).then(childDocs => childDocs.filter(doc => doc).map(doc => doc!)); + // .then(nestedResults => { + // // Flatten any nested arrays from recursive collection calls + // const childDocs = nestedResults.flat() as Doc[]; + // childDocs.forEach(doc => { + // console.log(DocCast(doc)); + // console.log(DocCast(doc)[DocData].data); + // console.log(DocCast(doc)[DocData].data); + // }); + // return childDocs; + // }); // @action // createSingleFlashcard = (data: any, options: DocumentOptions) => { @@ -442,101 +433,53 @@ export class ChatBox extends ViewBoxAnnotatableComponent() { // } @action - whichDoc = async (doc_type: string, data: string, options: DocumentOptions, id: string, insideCol: boolean): Promise => { - let doc; - switch (doc_type) { - case 'text': - doc = DocCast(Docs.Create.TextDocument(data, options)); - break; - case 'flashcard': - doc = this.createFlashcard(data, options); - break; - case 'deck': - doc = this.createDeck(data, options); - break; - case 'image': - doc = DocCast(Docs.Create.ImageDocument(data, options)); - break; - case 'equation': - doc = DocCast(Docs.Create.EquationDocument(data, options)); - break; - case 'noteboard': - doc = DocCast(Docs.Create.NoteTakingDocument([], options)); - break; - case 'simulation': - doc = DocCast(Docs.Create.SimulationDocument(options)); - break; - case 'collection': { - const arr = await this.createCollectionWithChildren(data, true); - options._layout_fitWidth = true; - options._freeform_backgroundGrid = true; - if (options.type_collection == 'tree') { - doc = DocCast(Docs.Create.TreeDocument(arr, options)); - } else if (options.type_collection == 'masonry') { - doc = DocCast(Docs.Create.MasonryDocument(arr, options)); - } else if (options.type_collection == 'card') { - doc = DocCast(Docs.Create.CardDeckDocument(arr, options)); - } else if (options.type_collection == 'carousel') { - doc = DocCast(Docs.Create.CarouselDocument(arr, options)); - } else if (options.type_collection == '3d-carousel') { - doc = DocCast(Docs.Create.Carousel3DDocument(arr, options)); - } else if (options.type_collection == 'multicolumn') { - doc = DocCast(Docs.Create.CarouselDocument(arr, options)); - } else { - doc = DocCast(Docs.Create.FreeformDocument(arr, options)); - } - break; + whichDoc = (doc_type: string, data: string, options: DocumentOptions, id: string, insideCol: boolean): Promise> => + (async () => { + switch (doc_type) { + case 'text': return Docs.Create.TextDocument(data, options); + case 'flashcard': return this.createFlashcard(data, options); + case 'deck': return this.createDeck(data, options); + case 'image': return Docs.Create.ImageDocument(data, options); + case 'equation': return Docs.Create.EquationDocument(data, options); + case 'noteboard': return Docs.Create.NoteTakingDocument([], options); + case 'simulation': return Docs.Create.SimulationDocument(options); + case 'collection': return this.createCollectionWithChildren(data as any, true). + then((arr, collOpts = { ...options, _layout_fitWidth: true, _freeform_backgroundGrid: true }) => + (() => { + switch (options.type_collection) { + case CollectionViewType.Tree: return Docs.Create.TreeDocument(arr, collOpts); + case CollectionViewType.Masonry: return Docs.Create.MasonryDocument(arr, collOpts); + case CollectionViewType.Card: return Docs.Create.CardDeckDocument(arr, collOpts); + case CollectionViewType.Carousel: return Docs.Create.CarouselDocument(arr, collOpts); + case CollectionViewType.Carousel3D: return Docs.Create.Carousel3DDocument(arr, collOpts); + case CollectionViewType.Multicolumn: return Docs.Create.CarouselDocument(arr, collOpts); + default: return Docs.Create.FreeformDocument(arr, collOpts); + } + })() + ); + case 'web': return Docs.Create.WebDocument(data, { ...options, data_useCors: true }); + case 'comparison': return this.createComparison(data, options); + case 'diagram': return Docs.Create.DiagramDocument(options); + case 'audio': return Docs.Create.AudioDocument(data, options); + case 'map': return Docs.Create.MapDocument([], options); + case 'screengrab': return Docs.Create.ScreenshotDocument(options); + case 'webcam': return Docs.Create.WebCamDocument('', options); + case 'button': return Docs.Create.ButtonDocument(options); + case 'script': return Docs.Create.ScriptingDocument(null, options); + case 'dataviz': return Docs.Create.DataVizDocument('/users/rz/Downloads/addresses.csv', options); + case 'chat': return Docs.Create.ChatDocument(options); + case 'trail': return Docs.Create.PresDocument(options); + case 'tab': return Docs.Create.FreeformDocument([], options); + case 'slide': return Docs.Create.TreeDocument([], options); + default: return Docs.Create.TextDocument(data, options); + } // prettier-ignore + })().then(doc => { + if (doc) { + doc.x = NumCast((options.x as number) ?? 0) + (insideCol ? 0 : NumCast(this.layoutDoc.x) + NumCast(this.layoutDoc.width)) + 100; + doc.y = NumCast(options.y as number) + (insideCol ? 0 : NumCast(this.layoutDoc.y)); } - case 'web': - options.data_useCors = true; - doc = DocCast(Docs.Create.WebDocument(data, options)); - break; - case 'comparison': - doc = this.createComparison(data, options); - break; - case 'diagram': - doc = Docs.Create.DiagramDocument(options); - break; - case 'audio': - doc = Docs.Create.AudioDocument(data, options); - break; - case 'map': - doc = Docs.Create.MapDocument([], options); - break; - case 'screengrab': - doc = Docs.Create.ScreenshotDocument(options); - break; - case 'webcam': - doc = Docs.Create.WebCamDocument('', options); - break; - case 'button': - doc = Docs.Create.ButtonDocument(options); - break; - case 'script': - doc = Docs.Create.ScriptingDocument(null, options); - break; - case 'dataviz': - doc = Docs.Create.DataVizDocument('/users/rz/Downloads/addresses.csv', options); - break; - case 'chat': - doc = Docs.Create.ChatDocument(options); - break; - case 'trail': - doc = Docs.Create.PresDocument(options); - break; - case 'tab': - doc = Docs.Create.FreeformDocument([], options); - break; - case 'slide': - doc = Docs.Create.TreeDocument([], options); - break; - default: - doc = DocCast(Docs.Create.TextDocument(data, options)); - } - doc!.x = NumCast(options.x ?? 0) + (insideCol ? 0 : NumCast(this.layoutDoc.x) + NumCast(this.layoutDoc.width)) + 100; - doc!.y = NumCast(options.y) + (insideCol ? 0 : NumCast(this.layoutDoc.y)); - return doc; - }; + return doc; + }); /** * Creates a document in the dashboard. @@ -548,57 +491,42 @@ export class ChatBox extends ViewBoxAnnotatableComponent() { * @returns {Promise} A promise that resolves once the document is created and displayed. */ @action - createDocInDash = async (doc_type: string, data: string | undefined, options: DocumentOptions, id: string) => { - let doc; - - switch (doc_type.toLowerCase()) { - case 'text': - doc = Docs.Create.TextDocument(data || '', options); - break; - case 'image': - doc = Docs.Create.ImageDocument(data || '', options); - break; - case 'pdf': - doc = Docs.Create.PdfDocument(data || '', options); - break; - case 'video': - doc = Docs.Create.VideoDocument(data || '', options); - break; - case 'audio': - doc = Docs.Create.AudioDocument(data || '', options); - break; - case 'web': - doc = Docs.Create.WebDocument(data || '', options); - break; - case 'equation': - doc = Docs.Create.EquationDocument(data || '', options); - break; - case 'functionplot': - case 'function_plot': - doc = Docs.Create.FunctionPlotDocument([], options); - break; - case 'dataviz': - case 'data_viz': { - const { fileUrl, id } = await Networking.PostToServer('/createCSV', { - filename: (options.title as string).replace(/\s+/g, '') + '.csv', - data: data, - }); - doc = Docs.Create.DataVizDocument(fileUrl, { ...options, text: RTFCast(data) }); - this.addCSVForAnalysis(doc, id); - break; + createDocInDash = (doc_type: string, data: string | undefined, options: DocumentOptions, id: string) => { + const linkAndShowDoc = (doc: Opt) => { + if (doc) { + LinkManager.Instance.addLink(Docs.Create.LinkDocument(this.Document, doc)); + this._props.addDocument?.(doc); + DocumentManager.Instance.showDocument(doc, { willZoomCentered: true }, () => {}); } - case 'chat': - doc = Docs.Create.ChatDocument(options); - break; - // Add more cases for other document types - default: - console.error('Unknown or unsupported document type:', doc_type); - return; - } - const linkDoc = Docs.Create.LinkDocument(this.Document, doc); - LinkManager.Instance.addLink(linkDoc); - doc && this._props.addDocument?.(doc); - await DocumentManager.Instance.showDocument(doc, { willZoomCentered: true }, () => {}); + }; + const doc = (() => { + switch (doc_type.toLowerCase()) { + case 'text': return Docs.Create.TextDocument(data || '', options); + case 'image': return Docs.Create.ImageDocument(data || '', options); + case 'pdf': return Docs.Create.PdfDocument(data || '', options); + case 'video': return Docs.Create.VideoDocument(data || '', options); + case 'audio': return Docs.Create.AudioDocument(data || '', options); + case 'web': return Docs.Create.WebDocument(data || '', options); + case 'equation': return Docs.Create.EquationDocument(data || '', options); + case 'chat': return Docs.Create.ChatDocument(options); + case 'functionplot': + case 'function_plot': return Docs.Create.FunctionPlotDocument([], options); + case 'dataviz': + case 'data_viz': Networking.PostToServer('/createCSV', { + filename: (options.title as string).replace(/\s+/g, '') + '.csv', + data: data, + })?.then(({ fileUrl, id }) => { + const vdoc = Docs.Create.DataVizDocument(fileUrl, { ...options, text: RTFCast(data) }); + this.addCSVForAnalysis(vdoc, id); + linkAndShowDoc(vdoc); + }); + return undefined; + // Add more cases for other document types + default: console.error('Unknown or unsupported document type:', doc_type); + return undefined; + } // prettier-ignore + })(); + if (doc) linkAndShowDoc(doc); }; /** @@ -625,16 +553,13 @@ export class ChatBox extends ViewBoxAnnotatableComponent() { } // Create a carousel to contain the flashcard deck - const carouselDoc = DocCast( - Docs.Create.CarouselDocument(flashcardDeck, { - title: options.title || 'Flashcard Deck', - _width: options._width || 300, - _height: options._height || 300, - _layout_fitWidth: false, - _layout_autoHeight: true, - }) - ); - return carouselDoc; + return Docs.Create.CarouselDocument(flashcardDeck, { + title: options.title || 'Flashcard Deck', + _width: options._width || 300, + _height: options._height || 300, + _layout_fitWidth: false, + _layout_autoHeight: true, + }); }; /** @@ -662,8 +587,7 @@ export class ChatBox extends ViewBoxAnnotatableComponent() { const side2 = Docs.Create.CenteredTextCreator(back.title, back.data, sideOptions); // Create the flashcard document with both sides - const flashcardDoc = DocCast(Docs.Create.FlashcardDocument(data.title, side1, side2, sideOptions)); - return flashcardDoc; + return Docs.Create.FlashcardDocument(data.title, side1, side2, sideOptions); } }; @@ -675,15 +599,14 @@ export class ChatBox extends ViewBoxAnnotatableComponent() { * @returns {Doc} The created comparison document. */ @action - createComparison = (doc: any, options: any) => { - const comp = Docs.Create.ComparisonDocument(options.title, { _width: options.width, _height: options.height | 300, backgroundColor: options.backgroundColor }); - const [left, right] = doc; - const docLeft = DocCast(Docs.Create.TextDocument(left.data, { backgroundColor: left.backgroundColor, _width: left.width, _height: left.height })); - const docRight = DocCast(Docs.Create.TextDocument(right.data, { backgroundColor: right.backgroundColor, _width: right.width, _height: right.height })); - comp[DocData].data_back = docLeft; - comp[DocData].data_front = docRight; - return comp; - }; + createComparison = (doc: { left: any; right: any }, options: any) => + Docs.Create.ComparisonDocument(options.title, { + data_back: Docs.Create.TextDocument(doc.left.data, { backgroundColor: doc.left.backgroundColor, _width: doc.left.width, _height: doc.left.height }), + data_front: Docs.Create.TextDocument(doc.right.data, { backgroundColor: doc.right.backgroundColor, _width: doc.right.width, _height: doc.right.height }), + _width: options.width, + _height: options.height | 300, + backgroundColor: options.backgroundColor, + }); /** * Event handler to manage citations click in the message components. @@ -692,7 +615,6 @@ export class ChatBox extends ViewBoxAnnotatableComponent() { @action handleCitationClick = (citation: Citation) => { const currentLinkedDocs: Doc[] = this.linkedDocs; - const chunkId = citation.chunk_id; // Loop through the linked documents to find the matching chunk and handle its display @@ -727,8 +649,8 @@ export class ChatBox extends ViewBoxAnnotatableComponent() { } break; case CHUNK_TYPE.TEXT: - this.citationPopup = { text: citation.direct_text ?? 'No text available', visible: true }; - setTimeout(() => (this.citationPopup.visible = false), 3000); // Hide after 3 seconds + this._citationPopup = { text: citation.direct_text ?? 'No text available', visible: true }; + setTimeout(() => (this._citationPopup.visible = false), 3000); // Hide after 3 seconds DocumentManager.Instance.showDocument(doc, { willZoomCentered: true }, () => { const firstView = Array.from(doc[DocViews])[0] as DocumentView; @@ -796,7 +718,7 @@ export class ChatBox extends ViewBoxAnnotatableComponent() { try { const storedHistory = JSON.parse(StrCast(this.dataDoc.data)); runInAction(() => { - this.history.push( + this._history.push( ...storedHistory.map((msg: AssistantMessage) => ({ role: msg.role, content: msg.content, @@ -811,7 +733,7 @@ export class ChatBox extends ViewBoxAnnotatableComponent() { } else { // Default welcome message runInAction(() => { - this.history.push({ + this._history.push({ role: ASSISTANT_ROLE.ASSISTANT, content: [ { @@ -835,11 +757,11 @@ export class ChatBox extends ViewBoxAnnotatableComponent() { .filter(d => d); return linkedDocs; }, - linked => linked.forEach(doc => this.linked_docs_to_add.add(doc)) + linked => linked.forEach(doc => this._linked_docs_to_add.add(doc)) ); // Observe changes to linked documents and handle document addition - observe(this.linked_docs_to_add, change => { + observe(this._linked_docs_to_add, change => { if (change.type === 'add') { if (PDFCast(change.newValue.data)) { this.addDocToVectorstore(change.newValue); @@ -913,7 +835,7 @@ export class ChatBox extends ViewBoxAnnotatableComponent() { * Getter that retrieves all linked CSV files for analysis. */ @computed get linkedCSVs(): { filename: string; id: string; text: string }[] { - return this.linked_csv_files; + return this._linked_csv_files; } /** @@ -921,7 +843,7 @@ export class ChatBox extends ViewBoxAnnotatableComponent() { */ @computed get formattedHistory(): string { let history = '\n'; - for (const message of this.history) { + for (const message of this._history) { history += `<${message.role}>${message.content.map(content => content.text).join(' ')}`; if (message.loop_summary) { history += `${message.loop_summary}`; @@ -957,7 +879,7 @@ export class ChatBox extends ViewBoxAnnotatableComponent() { */ @action handleFollowUpClick = (question: string) => { - this.inputValue = question; + this._inputValue = question; }; /** @@ -966,11 +888,11 @@ export class ChatBox extends ViewBoxAnnotatableComponent() { render() { return (
- {this.isUploadingDocs && ( + {this._isUploadingDocs && (
-
{this.currentStep}
+
{this._currentStep}
)} @@ -978,18 +900,18 @@ export class ChatBox extends ViewBoxAnnotatableComponent() {

{this.userName()}'s AI Assistant

- {this.history.map((message, index) => ( + {this._history.map((message, index) => ( ))} - {this.current_message && ( - + {this._current_message && ( + )}
- (this.inputValue = e.target.value)} disabled={this.isLoading} /> -
{/* Popup for citation */} - {this.citationPopup.visible && ( + {this._citationPopup.visible && (

- Text from your document: {this.citationPopup.text} + Text from your document: {this._citationPopup.text}

)} -- cgit v1.2.3-70-g09d2 From d72977ad8b67f2575cad8aea988fcfa7c04f794a Mon Sep 17 00:00:00 2001 From: bobzel Date: Tue, 21 Jan 2025 18:13:39 -0500 Subject: more attempts to cleanup typing, etc in chat box --- src/client/documents/DocumentTypes.ts | 2 +- src/client/documents/Documents.ts | 1 - .../views/nodes/chatbot/agentsystem/Agent.ts | 13 +- .../nodes/chatbot/chatboxcomponents/ChatBox.tsx | 114 ++++++++-------- .../views/nodes/chatbot/tools/CreateAnyDocTool.ts | 145 +++++++++------------ .../nodes/chatbot/tools/CreateDocumentTool.ts | 33 ++--- 6 files changed, 145 insertions(+), 163 deletions(-) (limited to 'src/client/views/nodes/chatbot/chatboxcomponents') diff --git a/src/client/documents/DocumentTypes.ts b/src/client/documents/DocumentTypes.ts index efe73fbbe..8aa844c0b 100644 --- a/src/client/documents/DocumentTypes.ts +++ b/src/client/documents/DocumentTypes.ts @@ -26,7 +26,7 @@ export enum DocumentType { SCRIPTING = 'script', // script editor CHAT = 'chat', // chat with GPT about files EQUATION = 'equation', // equation editor - FUNCPLOT = 'funcplot', // function plotter + FUNCPLOT = 'function plot', // function plotter MAP = 'map', DATAVIZ = 'dataviz', ANNOPALETTE = 'annopalette', diff --git a/src/client/documents/Documents.ts b/src/client/documents/Documents.ts index 0bff74ac1..7f1387ff8 100644 --- a/src/client/documents/Documents.ts +++ b/src/client/documents/Documents.ts @@ -19,7 +19,6 @@ import { DocServer } from '../DocServer'; import { dropActionType } from '../util/DropActionTypes'; import { CollectionViewType, DocumentType } from './DocumentTypes'; import { Id } from '../../fields/FieldSymbols'; -import { FireflyImageData } from '../views/smartdraw/FireflyConstants'; class EmptyBox { public static LayoutString() { diff --git a/src/client/views/nodes/chatbot/agentsystem/Agent.ts b/src/client/views/nodes/chatbot/agentsystem/Agent.ts index 4d3f1e4e7..ee91ccb92 100644 --- a/src/client/views/nodes/chatbot/agentsystem/Agent.ts +++ b/src/client/views/nodes/chatbot/agentsystem/Agent.ts @@ -8,7 +8,7 @@ import { AnswerParser } from '../response_parsers/AnswerParser'; import { StreamedAnswerParser } from '../response_parsers/StreamedAnswerParser'; import { BaseTool } from '../tools/BaseTool'; import { CalculateTool } from '../tools/CalculateTool'; -import { CreateAnyDocumentTool } from '../tools/CreateAnyDocTool'; +import { CreateAnyDocumentTool, supportedDocumentTypes } from '../tools/CreateAnyDocTool'; import { CreateDocTool } from '../tools/CreateDocumentTool'; import { DataAnalysisTool } from '../tools/DataAnalysisTool'; import { NoTool } from '../tools/NoTool'; @@ -55,7 +55,8 @@ export class Agent { history: () => string, csvData: () => { filename: string; id: string; text: string }[], addLinkedUrlDoc: (url: string, id: string) => void, - addLinkedDoc: (doc_type: string, data: string | undefined, options: DocumentOptions, id: string) => void, + addLinkedDoc: (doc_type: supportedDocumentTypes, data: unknown, options: DocumentOptions, id: string) => void, + // eslint-disable-next-line @typescript-eslint/no-unused-vars createCSVInDash: (url: string, title: string, id: string, data: string) => void ) { // Initialize OpenAI client with API key from environment @@ -134,6 +135,7 @@ export class Agent { console.log(this.interMessages); console.log(`Turn ${i}/${maxTurns}`); + // eslint-disable-next-line no-await-in-loop const result = await this.execute(onProcessingUpdate, onAnswerUpdate); this.interMessages.push({ role: 'assistant', content: result }); @@ -195,6 +197,7 @@ export class Agent { if (currentAction) { try { // Process the action with its input + // eslint-disable-next-line no-await-in-loop const observation = (await this.processAction(currentAction, actionInput.inputs)) as Observation[]; const nextPrompt = [{ type: 'text', text: ` ` }, ...observation, { type: 'text', text: '' }] as Observation[]; console.log(observation); @@ -299,7 +302,7 @@ export class Agent { * @param response The parsed XML response from the assistant. * @throws An error if the response does not meet the expected structure. */ - private validateAssistantResponse(response: any) { + private validateAssistantResponse(response: { stage: { [key: string]: object | string } }) { if (!response.stage) { throw new Error('Response does not contain a element'); } @@ -342,7 +345,7 @@ export class Agent { // If 'action_input' is present, validate its structure if ('action_input' in stage) { - const actionInput = stage.action_input; + const actionInput = stage.action_input as object; if (!('action_input_description' in actionInput) || typeof actionInput.action_input_description !== 'string') { throw new Error('action_input must contain an action_input_description string'); @@ -357,7 +360,7 @@ export class Agent { // If 'answer' is present, validate its structure if ('answer' in stage) { - const answer = stage.answer; + const answer = stage.answer as object; // Ensure answer contains at least one of the required elements if (!('grounded_text' in answer || 'normal_text' in answer)) { diff --git a/src/client/views/nodes/chatbot/chatboxcomponents/ChatBox.tsx b/src/client/views/nodes/chatbot/chatboxcomponents/ChatBox.tsx index 83b50c8c6..076f49831 100644 --- a/src/client/views/nodes/chatbot/chatboxcomponents/ChatBox.tsx +++ b/src/client/views/nodes/chatbot/chatboxcomponents/ChatBox.tsx @@ -14,7 +14,7 @@ import OpenAI, { ClientOptions } from 'openai'; import * as React from 'react'; import { v4 as uuidv4 } from 'uuid'; import { ClientUtils } from '../../../../../ClientUtils'; -import { Doc, DocListCast, Opt } from '../../../../../fields/Doc'; +import { Doc, DocListCast, FieldType, Opt } from '../../../../../fields/Doc'; import { DocData, DocViews } from '../../../../../fields/DocSymbols'; import { CsvCast, DocCast, NumCast, PDFCast, RTFCast, StrCast } from '../../../../../fields/Types'; import { Networking } from '../../../../Network'; @@ -324,7 +324,9 @@ export class ChatBox extends ViewBoxAnnotatableComponent() { processing_info: [], }); } finally { - this._isLoading = false; + runInAction(() => { + this._isLoading = false; + }); this.scrollToBottom(); } } @@ -402,19 +404,17 @@ export class ChatBox extends ViewBoxAnnotatableComponent() { data.map(doc => doc.doc_type !== 'collection' // Handle non-collection documents ? this.whichDoc(doc.doc_type, doc.data, { backgroundColor: doc.backgroundColor, _width: doc.width, _height: doc.height }, doc.id, insideCol) - : // Recursively process collections - this.createCollectionWithChildren(doc.data, true).then(nestedDocs => - Docs.Create.FreeformDocument(nestedDocs, { - title: doc.title, - backgroundColor: doc.backgroundColor, - _width: doc.width, - _height: doc.height, - _layout_fitWidth: true, - _freeform_backgroundGrid: true, - }) - ) - ) - .flat() // prettier-ignore + : this.createCollectionWithChildren(doc.data, true).then(nestedDocs => + Docs.Create.FreeformDocument(nestedDocs, { + title: doc.title, + backgroundColor: doc.backgroundColor, + _width: doc.width, + _height: doc.height, + _layout_fitWidth: true, + _freeform_backgroundGrid: true, + }) + ) + ) // prettier-ignore ).then(childDocs => childDocs.filter(doc => doc).map(doc => doc!)); // .then(nestedResults => { // // Flatten any nested arrays from recursive collection calls @@ -427,23 +427,18 @@ export class ChatBox extends ViewBoxAnnotatableComponent() { // return childDocs; // }); - // @action - // createSingleFlashcard = (data: any, options: DocumentOptions) => { - - // } - @action - whichDoc = (doc_type: string, data: string, options: DocumentOptions, id: string, insideCol: boolean): Promise> => + whichDoc = (doc_type: string, data: unknown, options: DocumentOptions, id: string, insideCol: boolean): Promise> => (async () => { switch (doc_type) { - case 'text': return Docs.Create.TextDocument(data, options); - case 'flashcard': return this.createFlashcard(data, options); - case 'deck': return this.createDeck(data, options); - case 'image': return Docs.Create.ImageDocument(data, options); - case 'equation': return Docs.Create.EquationDocument(data, options); + case 'text': return Docs.Create.TextDocument(data as string, options); + case 'flashcard': return this.createFlashcard(data as string[], options); + case 'deck': return this.createDeck(data as string, options); + case 'image': return Docs.Create.ImageDocument(data as string, options); + case 'equation': return Docs.Create.EquationDocument(data as string, options); case 'noteboard': return Docs.Create.NoteTakingDocument([], options); case 'simulation': return Docs.Create.SimulationDocument(options); - case 'collection': return this.createCollectionWithChildren(data as any, true). + case 'collection': return this.createCollectionWithChildren(data as { doc_type: string; id: string; data: any; title: string; width: number; height: number; backgroundColor: string }[] , true). then((arr, collOpts = { ...options, _layout_fitWidth: true, _freeform_backgroundGrid: true }) => (() => { switch (options.type_collection) { @@ -457,10 +452,10 @@ export class ChatBox extends ViewBoxAnnotatableComponent() { } })() ); - case 'web': return Docs.Create.WebDocument(data, { ...options, data_useCors: true }); - case 'comparison': return this.createComparison(data, options); + case 'web': return Docs.Create.WebDocument(data as string, { ...options, data_useCors: true }); + case 'comparison': return this.createComparison(data as {left: {width:number ,height: number, backgroundColor: string, data: string}, right: {width:number ,height: number, backgroundColor: string, data: string}}, options); case 'diagram': return Docs.Create.DiagramDocument(options); - case 'audio': return Docs.Create.AudioDocument(data, options); + case 'audio': return Docs.Create.AudioDocument(data as string, options); case 'map': return Docs.Create.MapDocument([], options); case 'screengrab': return Docs.Create.ScreenshotDocument(options); case 'webcam': return Docs.Create.WebCamDocument('', options); @@ -471,7 +466,7 @@ export class ChatBox extends ViewBoxAnnotatableComponent() { case 'trail': return Docs.Create.PresDocument(options); case 'tab': return Docs.Create.FreeformDocument([], options); case 'slide': return Docs.Create.TreeDocument([], options); - default: return Docs.Create.TextDocument(data, options); + default: return Docs.Create.TextDocument(data as string, options); } // prettier-ignore })().then(doc => { if (doc) { @@ -491,7 +486,7 @@ export class ChatBox extends ViewBoxAnnotatableComponent() { * @returns {Promise} A promise that resolves once the document is created and displayed. */ @action - createDocInDash = (doc_type: string, data: string | undefined, options: DocumentOptions, id: string) => { + createDocInDash = (doc_type: string, data: unknown, options: DocumentOptions /*, id: string */) => { const linkAndShowDoc = (doc: Opt) => { if (doc) { LinkManager.Instance.addLink(Docs.Create.LinkDocument(this.Document, doc)); @@ -501,22 +496,21 @@ export class ChatBox extends ViewBoxAnnotatableComponent() { }; const doc = (() => { switch (doc_type.toLowerCase()) { - case 'text': return Docs.Create.TextDocument(data || '', options); - case 'image': return Docs.Create.ImageDocument(data || '', options); - case 'pdf': return Docs.Create.PdfDocument(data || '', options); - case 'video': return Docs.Create.VideoDocument(data || '', options); - case 'audio': return Docs.Create.AudioDocument(data || '', options); - case 'web': return Docs.Create.WebDocument(data || '', options); - case 'equation': return Docs.Create.EquationDocument(data || '', options); + case 'flashcard': return this.createFlashcard(data as string[], options); + case 'text': return Docs.Create.TextDocument(data as string || '', options); + case 'image': return Docs.Create.ImageDocument(data as string || '', options); + case 'pdf': return Docs.Create.PdfDocument(data as string || '', options); + case 'video': return Docs.Create.VideoDocument(data as string || '', options); + case 'audio': return Docs.Create.AudioDocument(data as string || '', options); + case 'web': return Docs.Create.WebDocument(data as string || '', options); + case 'equation': return Docs.Create.EquationDocument(data as string || '', options); case 'chat': return Docs.Create.ChatDocument(options); - case 'functionplot': - case 'function_plot': return Docs.Create.FunctionPlotDocument([], options); - case 'dataviz': - case 'data_viz': Networking.PostToServer('/createCSV', { + case 'functionplot': return Docs.Create.FunctionPlotDocument([], options); + case 'dataviz': Networking.PostToServer('/createCSV', { filename: (options.title as string).replace(/\s+/g, '') + '.csv', data: data, })?.then(({ fileUrl, id }) => { - const vdoc = Docs.Create.DataVizDocument(fileUrl, { ...options, text: RTFCast(data) }); + const vdoc = Docs.Create.DataVizDocument(fileUrl, { ...options, text: RTFCast(data as FieldType) }); this.addCSVForAnalysis(vdoc, id); linkAndShowDoc(vdoc); }); @@ -537,14 +531,14 @@ export class ChatBox extends ViewBoxAnnotatableComponent() { * @returns {Doc} A carousel document containing the flashcard deck. */ @action - createDeck = (data: any, options: DocumentOptions) => { + createDeck = (data: string | unknown[], options: DocumentOptions) => { const flashcardDeck: Doc[] = []; // Parse `data` only if it’s a string - const deckData = typeof data === 'string' ? JSON.parse(data) : data; - const flashcardArray = Array.isArray(deckData) ? deckData : Object.values(deckData); + const deckData = typeof data === 'string' ? (JSON.parse(data) as unknown) : data; + const flashcardArray = Array.isArray(deckData) ? deckData : Object.values(deckData as object); // Process each flashcard document in the `deckData` array if (flashcardArray.length == 2 && flashcardArray[0].doc_type == 'text' && flashcardArray[1].doc_type == 'text') { - this.createFlashcard(flashcardArray, options); + this.createFlashcard(flashcardArray as string[], options); } else { flashcardArray.forEach(doc => { const flashcardDoc = this.createFlashcard(doc, options); @@ -570,24 +564,24 @@ export class ChatBox extends ViewBoxAnnotatableComponent() { * @returns {Doc | undefined} The created flashcard document, or undefined if the flashcard cannot be created. */ @action - createFlashcard = (data: any, options: any) => { + createFlashcard = (data: string[], options: DocumentOptions) => { const deckData = typeof data === 'string' ? JSON.parse(data) : data; - const flashcardArray = Array.isArray(deckData) ? deckData : Object.values(deckData)[2]; + const flashcardArray = Array.isArray(deckData) ? deckData : (Object.values(deckData)[2] as string[]); const [front, back] = flashcardArray; - if (front.doc_type === 'text' && back.doc_type === 'text') { + if (typeof front === 'string' && typeof back === 'string') { const sideOptions: DocumentOptions = { backgroundColor: options.backgroundColor, _width: options._width, - _height: options._height, + _height: options._height || 300, }; // Create front and back text documents - const side1 = Docs.Create.CenteredTextCreator(front.title, front.data, sideOptions); - const side2 = Docs.Create.CenteredTextCreator(back.title, back.data, sideOptions); + const side1 = Docs.Create.CenteredTextCreator('question', front, sideOptions); + const side2 = Docs.Create.CenteredTextCreator('answer', back, sideOptions); // Create the flashcard document with both sides - return Docs.Create.FlashcardDocument(data.title, side1, side2, sideOptions); + return Docs.Create.FlashcardDocument('flashcard', side1, side2, sideOptions); } }; @@ -599,12 +593,12 @@ export class ChatBox extends ViewBoxAnnotatableComponent() { * @returns {Doc} The created comparison document. */ @action - createComparison = (doc: { left: any; right: any }, options: any) => - Docs.Create.ComparisonDocument(options.title, { + createComparison = (doc: { left: { width: number; height: number; backgroundColor: string; data: string }; right: { width: number; height: number; backgroundColor: string; data: string } }, options: DocumentOptions) => + Docs.Create.ComparisonDocument(options.title as string, { data_back: Docs.Create.TextDocument(doc.left.data, { backgroundColor: doc.left.backgroundColor, _width: doc.left.width, _height: doc.left.height }), data_front: Docs.Create.TextDocument(doc.right.data, { backgroundColor: doc.right.backgroundColor, _width: doc.right.width, _height: doc.right.height }), - _width: options.width, - _height: options.height | 300, + _width: options._width, + _height: options._height || 300, backgroundColor: options.backgroundColor, }); @@ -909,7 +903,7 @@ export class ChatBox extends ViewBoxAnnotatableComponent() {
- (this._inputValue = e.target.value)} disabled={this._isLoading} /> + (this._inputValue = e.target.value))} disabled={this._isLoading} /> + { + this._dictation = r; + }} + setInput={this.setChatInput} + inputRef={this._textInputRef} + /> {/* Popup for citation */} {this._citationPopup.visible && ( diff --git a/src/client/views/nodes/trails/PresBox.tsx b/src/client/views/nodes/trails/PresBox.tsx index 3ce4dc6cb..b98a7f96e 100644 --- a/src/client/views/nodes/trails/PresBox.tsx +++ b/src/client/views/nodes/trails/PresBox.tsx @@ -7,7 +7,6 @@ import { IReactionDisposer, ObservableSet, action, computed, makeObservable, obs import { observer } from 'mobx-react'; import * as React from 'react'; import { AiOutlineSend } from 'react-icons/ai'; -import { BiMicrophone } from 'react-icons/bi'; import { FaArrowDown, FaArrowLeft, FaArrowRight, FaArrowUp } from 'react-icons/fa'; import ReactLoading from 'react-loading'; import ReactTextareaAutosize from 'react-textarea-autosize'; @@ -26,12 +25,12 @@ import { DocServer } from '../../../DocServer'; import { getSlideTransitionSuggestions, gptSlideProperties, gptTrailSlideCustomization } from '../../../apis/gpt/PresCustomization'; import { CollectionViewType, DocumentType } from '../../../documents/DocumentTypes'; import { Docs } from '../../../documents/Documents'; -import { DictationManager } from '../../../util/DictationManager'; import { dropActionType } from '../../../util/DropActionTypes'; import { ScriptingGlobals } from '../../../util/ScriptingGlobals'; import { SerializationHelper } from '../../../util/SerializationHelper'; import { SnappingManager } from '../../../util/SnappingManager'; import { UndoManager, undoBatch, undoable } from '../../../util/UndoManager'; +import { DictationButton } from '../../DictationButton'; import { ViewBoxBaseComponent } from '../../DocComponent'; import { pinDataTypes as dataTypes } from '../../PinFuncs'; import { CollectionView } from '../../collections/CollectionView'; @@ -79,6 +78,8 @@ export class PresBox extends ViewBoxBaseComponent() { _keyTimer: NodeJS.Timeout | undefined; // timer for turning off transition flag when key frame change has completed. Need to clear this if you do a second navigation before first finishes, or else first timer can go off during second naviation. _unmounting = false; // flag that view is unmounting used to block RemFromMap from deleting things _presTimer: NodeJS.Timeout | undefined; + _animationDictation: DictationButton | null = null; + _slideDictation: DictationButton | null = null; // eslint-disable-next-line no-use-before-define @observable public static Instance: PresBox; @@ -108,7 +109,6 @@ export class PresBox extends ViewBoxBaseComponent() { @observable _chatActive: boolean = false; @observable _animationChat: string = ''; @observable _chatInput: string = ''; - @observable _isRecording: boolean = false; @observable _isLoading: boolean = false; @observable generatedAnimations: AnimationSettings[] = [ @@ -147,7 +147,6 @@ export class PresBox extends ViewBoxBaseComponent() { setChatInput = action((input: string) => { this._chatInput = input; }); // prettier-ignore setAnimationChat = action((input: string) => { this._animationChat = input; }); // prettier-ignore setIsLoading = action((input?: boolean) => { this._isLoading = !!input; }); // prettier-ignore - setIsRecording = action((input: boolean) => { this._isRecording = input; }); // prettier-ignore setShowAIGalleryVisibilty = action((visible: boolean) => { this._showAIGallery = visible; }); // prettier-ignore setBezierControlPoints = action((newPoints: { p1: number[]; p2: number[] }) => { this.setEaseFunc(this.activeItem, `cubic-bezier(${newPoints.p1[0]}, ${newPoints.p1[1]}, ${newPoints.p2[0]}, ${newPoints.p2[1]})`); @@ -280,24 +279,6 @@ export class PresBox extends ViewBoxBaseComponent() { } }; - // Recording for GPT customization - recordDictation = () => { - this.setIsRecording(true); - this.setChatInput(''); - DictationManager.Controls.listen({ - interimHandler: this.setDictationContent, - continuous: { indefinite: false }, - }).then(results => { - if (results && [DictationManager.Controls.Infringed].includes(results)) { - DictationManager.Controls.stop(); - } - }); - }; - stopDictation = () => { - this.setIsRecording(false); - DictationManager.Controls.stop(); - }; - setDictationContent = (value: string) => this.setChatInput(value); customizeAnimations = action(() => { @@ -310,7 +291,6 @@ export class PresBox extends ViewBoxBaseComponent() { customizeWithGPT = action((input: string) => { // const testInput = 'change title to Customized Slide, transition for 2.3s with fade in effect'; - this.setIsRecording(false); this.setIsLoading(true); const slideDefaults: { [key: string]: FieldResult } = { presentation_transition: 500, config_zoom: 1 }; const currSlideProperties = gptSlideProperties.reduce( @@ -1785,7 +1765,7 @@ export class PresBox extends ViewBoxBaseComponent() { this.setAnimationChat(e.target.value); }} onKeyDown={e => { - this.stopDictation(); + this._animationDictation?.stopDictation(); e.stopPropagation(); }} /> @@ -1799,6 +1779,12 @@ export class PresBox extends ViewBoxBaseComponent() { color={SnappingManager.userVariantColor} onClick={this.customizeAnimations} /> + { + this._animationDictation = r; + }} + setInput={this.setAnimationChat} + />
Click a box to use the effect. @@ -1878,22 +1864,15 @@ export class PresBox extends ViewBoxBaseComponent() { this.setChatInput(e.target.value); }} onKeyDown={e => { - this.stopDictation(); + this._slideDictation?.stopDictation(); e.stopPropagation(); }} /> - } - onClick={() => { - if (!this._isRecording) { - this.recordDictation(); - } else { - this.stopDictation(); - } + { + this._slideDictation = r; }} + setInput={this.setChatInput} />