From 79e4c4a3fba42b90ffa656db3ca435505f978afe Mon Sep 17 00:00:00 2001 From: "A.J. Shulman" Date: Tue, 20 Aug 2024 18:32:08 -0400 Subject: supports multiple inputs maybe also make it so web results cannot have overlap (no same url in websites returned by search) Also make sure it will cite multiple websites --- src/client/views/nodes/ChatBox/Agent.ts | 52 ++++- src/client/views/nodes/ChatBox/prompts.ts | 228 +++++++++++---------- .../views/nodes/ChatBox/tools/CalculateTool.ts | 1 + .../nodes/ChatBox/tools/CreateCollectionTool.ts | 1 + .../views/nodes/ChatBox/tools/DataAnalysisTool.ts | 36 +++- src/client/views/nodes/ChatBox/tools/SearchTool.ts | 63 +++--- .../nodes/ChatBox/tools/WebsiteInfoScraperTool.ts | 52 +++-- src/server/ApiManagers/AssistantManager.ts | 3 +- 8 files changed, 260 insertions(+), 176 deletions(-) (limited to 'src') diff --git a/src/client/views/nodes/ChatBox/Agent.ts b/src/client/views/nodes/ChatBox/Agent.ts index ae08271ee..43138bf94 100644 --- a/src/client/views/nodes/ChatBox/Agent.ts +++ b/src/client/views/nodes/ChatBox/Agent.ts @@ -55,13 +55,23 @@ export class Agent { const systemPrompt = getReactPrompt(Object.values(this.tools), this._summaries, chatHistory); this.interMessages = [{ role: 'system', content: systemPrompt }]; this.interMessages.push({ role: 'user', content: `${question}` }); - const parser = new XMLParser({ ignoreAttributes: false, attributeNamePrefix: '@_' }); + const parser = new XMLParser({ + ignoreAttributes: false, + attributeNamePrefix: '@_', + textNodeName: '_text', + isArray: (name, jpath, isLeafNode, isAttribute) => { + // Convert tags with the same name to arrays + return ['query', 'url'].indexOf(name) !== -1; + }, + }); const builder = new XMLBuilder({ ignoreAttributes: false, attributeNamePrefix: '@_' }); + let currentAction: string | undefined; this.processingInfo = []; for (let i = 2; i < maxTurns; i += 2) { + console.log(this.interMessages); console.log(`Turn ${i}/${maxTurns}`); const result = await this.execute(onUpdate); @@ -102,11 +112,14 @@ export class Agent { break; } } else if (key === 'action_input') { - const actionInput = builder.build({ action_input: stage[key] }); - console.log(`Action input: ${actionInput}`); + const actionInput = stage[key]; + console.log(`Action input:`, actionInput); if (currentAction) { try { - const observation = await this.processAction(currentAction, stage[key].inputs); + // Parse the inputs + //const parsedInputs = this.parseActionInputs(actionInput.inputs); + //console.log(`Parsed inputs:`, parsedInputs); + const observation = await this.processAction(currentAction, actionInput.inputs); const nextPrompt = [{ type: 'text', text: ` ` }, ...observation, { type: 'text', text: '' }]; console.log(observation); this.interMessages.push({ role: 'user', content: nextPrompt }); @@ -198,9 +211,26 @@ export class Agent { const tool = this.tools[action]; const args: Record = {}; + for (const paramName in tool.parameters) { if (actionInput[paramName] !== undefined) { - args[paramName] = actionInput[paramName]; + if (Array.isArray(actionInput[paramName])) { + // If the input is already an array, use it as is + args[paramName] = actionInput[paramName]; + } else if (typeof actionInput[paramName] === 'object' && actionInput[paramName] !== null) { + // If the input is an object, check if it has multiple of the same tag + const values = Object.values(actionInput[paramName]); + if (values.length > 1) { + // If there are multiple values, convert to an array + args[paramName] = values; + } else { + // If there's only one value, use it directly + args[paramName] = values[0]; + } + } else { + // For single values, use them as is + args[paramName] = actionInput[paramName]; + } } else if (tool.parameters[paramName].required === 'true') { throw new Error(`Missing required parameter '${paramName}' for action '${action}'`); } @@ -208,4 +238,16 @@ export class Agent { return await tool.execute(args); } + + private parseActionInputs(inputs: any): Record { + const parsedInputs: Record = {}; + for (const key in inputs) { + if (Array.isArray(inputs[key])) { + parsedInputs[key] = inputs[key].map((item: any) => item._text); + } else { + parsedInputs[key] = inputs[key]._text; + } + } + return parsedInputs; + } } diff --git a/src/client/views/nodes/ChatBox/prompts.ts b/src/client/views/nodes/ChatBox/prompts.ts index 200a0fa75..845b4d255 100644 --- a/src/client/views/nodes/ChatBox/prompts.ts +++ b/src/client/views/nodes/ChatBox/prompts.ts @@ -56,15 +56,15 @@ export function getReactPrompt(tools: Tool[], summaries: () => string, chatHisto Grounded text can be as short as a few words or as long as several sentences. Avoid overlapping or nesting <grounded_text> tags; use sequential tags instead. Grounded text tags should always have a citation_index attribute that references a citation index number that the text is grounded in. - Should be in Markdown format. + Content within the <grounded_text> tags should be in Markdown format. - Wrap all information that is not derived from tools or chunks in <normal_text> tags. + Wrap all text that is not derived from tools or chunks in <normal_text> tags (any text outputted in the answer that is not in a <grounded_text> tag should be within a normal text tag). Ensure that these tags are used for your reasoning, background knowledge, or general information that does not require a citation. Do not use <normal_text> tags for information that needs grounding or citation. Anything that is in any user docs should be grounded text and cited, not normal text, even if it is background or general information. - Should be in Markdown format. + Content within the <normal_text> tags should be in Markdown format. @@ -87,6 +87,7 @@ export function getReactPrompt(tools: Tool[], summaries: () => string, chatHisto Use the <action> tag to specify the tool. End your response after the <action> tag and wait for action rules to be provided. Based on the action rules, provide the necessary tool parameters within the <inputs>. The <inputs> tag should be within the <action_input> tag, and should follow an <action_description> tag that contains a brief description of what you're doing with the action. + For each input, you may provide as many different iterations of the same tag (i.e. to provide multiple inputs to the tool) as is specified in the input's max_input's field. End your response again and wait for the observation from the tool. If no tool is needed, use the 'no_tool' action but still follow the same response structure. @@ -169,7 +170,7 @@ export function getReactPrompt(tools: Tool[], summaries: () => string, chatHisto - With information on key moments from the press release, I will now analyze the medal count data to compare how different countries performed during the games. + With information on key moments from the press release, I will now analyze the medal count data and economic impact data to provide a comprehensive overview of the 2010 Vancouver Winter Olympics's impact. dataAnalysis @@ -180,9 +181,10 @@ export function getReactPrompt(tools: Tool[], summaries: () => string, chatHisto - Analyzing the medal count data for the 2010 Vancouver Winter Olympics to compare country performances. + Analyzing the medal count data and economic impact data for the 2010 Vancouver Winter Olympics to compare country performances and assess the games' overall impact. 2010_Vancouver_Olympics_Medal_Count.csv + 2010_Vancouver_Olympics_Economic_Impact.csv @@ -194,234 +196,236 @@ export function getReactPrompt(tools: Tool[], summaries: () => string, chatHisto Germany,10,13,7,30 Canada,14,7,5,26 Norway,9,8,6,23 + Russia,3,5,7,15 + South Korea,6,6,2,14 + China,5,2,4,11 + Sweden,5,2,4,11 + France,2,3,6,11 + Austria,4,6,6,16 + + + Category,Value + Total Cost (CAD),7.7 billion + Revenue Generated (CAD),2.5 billion + Jobs Created,45000 + Tourism Increase (%),35 + Infrastructure Investment (CAD),1.9 billion + Post-Olympic Annual Tourism Revenue Increase (CAD),220 million + Venues Repurposed,8 + Carbon Footprint Offset (%),118 - Now that I have gathered comprehensive information from the official press release and the medal count data, I will provide a detailed answer to the user's query. + Now that I have gathered comprehensive information from the official press release, medal count data, and economic impact data, I will provide a detailed answer to the user's query, incorporating all aspects of the 2010 Vancouver Winter Olympics's impact. The 2010 Winter Olympics, officially known as the XXI Olympic Winter Games, took place in Vancouver, British Columbia, Canada, from February 12 to 28, 2010. It featured 86 events in 15 disciplines across 7 sports, with 2,566 athletes from 82 National Olympic Committees participating. + The Opening Ceremony was one of the most spectacular in Olympic history, celebrating Canadian culture and heritage. A significant highlight for the host nation was the victory of the Canadian men's and women's ice hockey teams, both securing gold medals, which was a source of immense national pride. + - In terms of medal performance, the United States led the overall tally with 37 medals, followed by Germany with 30 medals. Canada, while placing third in total medals, achieved a record number of 14 gold medals, the highest of any country in the 2010 games. + In terms of medal performance, the United States led the overall tally with 37 medals (9 gold, 15 silver, 13 bronze), followed by Germany with 30 medals (10 gold, 13 silver, 7 bronze). Canada, while placing third in total medals with 26, achieved a record number of 14 gold medals, the highest of any country in the 2010 games. Other notable performances came from Norway (23 medals), Russia (15 medals), and South Korea (14 medals). + - However, the games were also marked by tragedy with the unfortunate death of Georgian luger Nodar Kumaritashvili during a training run, which deeply affected the Olympic community. + The economic impact of the 2010 Vancouver Winter Olympics was significant. The total cost of hosting the games was approximately 7.7 billion CAD, while generating revenue of 2.5 billion CAD. The games created about 45,000 jobs and led to a 35% increase in tourism during the event. The long-term benefits include an annual increase of 220 million CAD in tourism revenue and 1.9 billion CAD invested in infrastructure. - - - - Vancouver, BC - The 2010 Winter Olympics have been hailed as a monumental success, showcasing the best of Canadian culture and athletic prowess. The Opening Ceremony, held on February 12, 2010, was a spectacular celebration of the nation's history and diversity... - Key Moments: - - The Canadian men's and women's ice hockey teams both claimed gold medals, an achievement that brought immense pride to the host nation. - + + From an environmental perspective, the Vancouver Olympics made efforts to be sustainable, with 118% of the carbon footprint offset and 8 venues repurposed after the games, demonstrating a commitment to leaving a positive legacy. + + + + However, the games were also marked by tragedy with the unfortunate death of Georgian luger Nodar Kumaritashvili during a training run, which deeply affected the Olympic community and led to increased scrutiny of safety measures across all events. + + + + Despite this tragedy, the 2010 Vancouver Winter Olympics were generally hailed as a monumental success, showcasing the best of Canadian culture and athletic prowess. The games concluded on February 28, 2010, with a memorable Closing Ceremony that highlighted the achievements of the athletes and the spirit of unity that defined these Olympics. + + + + + - - Tragedy struck early in the games with the death of Georgian luger Nodar Kumaritashvili during a training run. - + + + + - What were the economic impacts on Vancouver after hosting the 2010 Winter Olympics? - How did the tragic accident of Nodar Kumaritashvili influence safety protocols in luge and other winter sports? - Can you provide more information on other significant performances by athletes during the 2010 Winter Olympics? + How did the 2010 Vancouver Winter Olympics compare to previous Winter Olympics in terms of economic impact and sustainability efforts? + What specific infrastructure improvements were made in Vancouver as a result of hosting the 2010 Winter Olympics? + How did the performance of Canadian athletes in the 2010 Vancouver Winter Olympics compare to their performance in previous Winter Olympics? + + ***SYSTEM MESSAGE ELIDED*** + - What are the best hiking trails in Yosemite National Park? + What are the best hiking trails in Yosemite National Park? And which are the safest? - - To provide comprehensive information about the best hiking trails in Yosemite National Park, I'll use the search tool to find relevant websites with hiking recommendations and park information. + To provide comprehensive information about the best hiking trails in Yosemite National Park, I'll use the search tool to find relevant websites with hiking recommendations, safety information, and general park information. searchTool - ***Action rules elided*** - - Searching the web for information about the best hiking trails in Yosemite National Park. + Searching the web for information about the best and safest hiking trails in Yosemite National Park. Best hiking trails Yosemite National Park + Safest hiking trails Yosemite National Park + Information about hiking in Yosemite National Park - - + https://www.nps.gov/yose/planyourvisit/hiking.htm Official National Park Service website with hiking information for Yosemite - + https://www.alltrails.com/parks/us/california/yosemite-national-park AllTrails page featuring user-rated hiking trails in Yosemite - + https://www.yosemitehikes.com/ Comprehensive guide to Yosemite hikes with detailed trail information - ***7 more search results elided*** - - I've found several relevant websites about hiking in Yosemite National Park. I'll start by scraping content from the official National Park Service website for the most reliable information. + I've found several relevant websites about hiking in Yosemite National Park. I'll start by scraping content from the official National Park Service website for the most reliable information, then move on to AllTrails for user ratings and reviews, and finally to YosemiteHikes.com for more detailed trail descriptions. websiteInfoScraper - ***Action rules elided*** - - Scraping information about hiking trails from the official Yosemite National Park website. + Scraping information from three websites about hiking trails in Yosemite National Park. https://www.nps.gov/yose/planyourvisit/hiking.htm + https://www.alltrails.com/parks/us/california/yosemite-national-park + https://www.yosemitehikes.com/ - - ***Website image elided*** - ***Scraped content from NPS website*** + ***NPS general planning website image elided*** + + + ***AllTrails website image elided*** + + + ***YosemiteHikes.com website image elided*** - - The information from the National Park Service website was helpful, but it lacks user reviews and ratings. I'll check AllTrails for this additional perspective, although it may not provide as much official information. + The information gathered from the three websites provides a good overview of the best hiking trails and safety considerations in Yosemite National Park. However, I noticed that there's limited information about seasonal trail conditions and closures, which could be crucial for planning a safe hike. To address this, I'll scrape additional information from the Yosemite National Park's current conditions page. websiteInfoScraper - ***Action rules elided*** - - Scraping user reviews and ratings for Yosemite hiking trails from AllTrails. + Scraping information about current trail conditions and closures in Yosemite National Park. - https://www.alltrails.com/parks/us/california/yosemite-national-park + https://www.nps.gov/yose/planyourvisit/conditions.htm - - - ***Website image elided*** - ***Scraped content from AllTrails*** + + ***NPS conditions website image elided*** - - The AllTrails website didn't provide as much detailed information as I had hoped. To ensure a comprehensive answer, I'll scrape YosemiteHikes.com for additional details and trail descriptions. + Now that I have additional information about current trail conditions and seasonal closures, I can provide a more comprehensive and up-to-date answer to the user's query about the best and safest hiking trails in Yosemite National Park. - websiteInfoScraper - - - - ***Action rules elided*** - + + + Yosemite National Park offers a wide range of hiking trails suitable for various skill levels. Some of the best hiking trails in Yosemite, based on the information gathered, include: - - - Gathering detailed trail descriptions and hiking information from YosemiteHikes.com. - - https://www.yosemitehikes.com/ - - - + 1. Mist Trail to Vernal Fall: A moderate 3-mile round-trip hike in Yosemite Valley, offering close-up views of waterfalls... ***REST ELIDED FOR BREVITY*** - - - - ***Website image elided*** - ***Scraped content from YosemiteHikes.com*** - - - - - - - I have successfully scraped information from all three websites. Now, I'll synthesize this information to provide a comprehensive answer about the best hiking trails in Yosemite National Park. - - - - Yosemite National Park is renowned for its breathtaking landscapes and diverse hiking opportunities. The park spans nearly 1,200 square miles of wilderness in California's Sierra Nevada mountains, offering trails suitable for all skill levels. - + + ***CONTENT ELIDED FOR BREVITY*** + + + ***CONTENT ELIDED FOR BREVITY*** + - Based on the information gathered from multiple sources, here are some of the best hiking trails in Yosemite: + Regarding safety, some of the safest trails for hikers of all skill levels include: + + 1. Lower Yosemite Fall Trail: An easy 1-mile loop in Yosemite Valley... ***REST ELIDED FOR BREVITY*** + - 1. Half Dome: This strenuous 14-16 mile round trip hike is one of Yosemite's most iconic and challenging trails, requiring a permit for the cable section. - - 2. Mist Trail to Vernal and Nevada Falls: A popular 6.4-mile round trip hike offering close-up views of two spectacular waterfalls. - - 3. Yosemite Falls Trail: A strenuous 7.2-mile round trip hike to the top of North America's tallest waterfall. - - 4. Bridalveil Fall Trail: An easy 0.5-mile round trip walk to the base of this iconic Yosemite waterfall. - - 5. Mirror Lake Trail: A moderate 2-5 mile round trip hike to a lake that reflects surrounding cliffs and Half Dome. - + + ***CONTENT ELIDED FOR BREVITY*** + - These trails offer a range of difficulties and showcase the park's diverse natural beauty, from towering waterfalls to panoramic valley views. + To ensure a safe hiking experience in Yosemite, regardless of the trail you choose, follow these safety tips: - - - It's important to note that trail conditions can vary depending on the season, and some trails may require permits or have specific regulations. Always check the official Yosemite National Park website for the most up-to-date information before planning your hike. + + 1. Stay on marked trails at all times... + ***REST ELIDED FOR BREVITY*** - - Remember to practice Leave No Trace principles, stay on designated trails, and come prepared with appropriate gear, water, and snacks for your hike. + Remember that even the "safest" trails can pose risks if proper precautions are not taken. Always hike within your ability level and be prepared for the specific conditions of the trail you choose. It's essential to check for daily updates on trail conditions before starting your hike, as they can change rapidly, especially during spring and fall. - - - - - - - - + + https://www.nps.gov/yose/planyourvisit/hiking.htm + + + https://www.alltrails.com/parks/us/california/yosemite-national-park + + + https://www.yosemitehikes.com/ + + + https://www.nps.gov/yose/planyourvisit/conditions.htm + - What is the best time of year to hike in Yosemite National Park? - Are there any beginner-friendly trails in Yosemite that offer scenic views? - What safety precautions should hikers take when exploring Yosemite's trails? + Are there any special permits required for hiking certain trails in Yosemite? + What wildlife might I encounter while hiking in Yosemite, and how should I prepare for potential encounters? - ${chatHistory} Use this history for context when appropriate. diff --git a/src/client/views/nodes/ChatBox/tools/CalculateTool.ts b/src/client/views/nodes/ChatBox/tools/CalculateTool.ts index 818332c44..74b7ca27b 100644 --- a/src/client/views/nodes/ChatBox/tools/CalculateTool.ts +++ b/src/client/views/nodes/ChatBox/tools/CalculateTool.ts @@ -10,6 +10,7 @@ export class CalculateTool extends BaseTool<{ expression: string }> { type: 'string', description: 'The mathematical expression to evaluate', required: 'true', + max_inputs: '1', }, }, 'Provide a mathematical expression to calculate that would work with JavaScript eval().', diff --git a/src/client/views/nodes/ChatBox/tools/CreateCollectionTool.ts b/src/client/views/nodes/ChatBox/tools/CreateCollectionTool.ts index 26ac0d7cc..573428179 100644 --- a/src/client/views/nodes/ChatBox/tools/CreateCollectionTool.ts +++ b/src/client/views/nodes/ChatBox/tools/CreateCollectionTool.ts @@ -16,6 +16,7 @@ export class GetDocsContentTool extends BaseTool<{ title: string; document_ids: type: 'string', description: 'the title of the collection that you will be making', required: 'true', + max_inputs: '1', }, }, 'Provide a mathematical expression to calculate that would work with JavaScript eval().', diff --git a/src/client/views/nodes/ChatBox/tools/DataAnalysisTool.ts b/src/client/views/nodes/ChatBox/tools/DataAnalysisTool.ts index b45733639..a12ee46e5 100644 --- a/src/client/views/nodes/ChatBox/tools/DataAnalysisTool.ts +++ b/src/client/views/nodes/ChatBox/tools/DataAnalysisTool.ts @@ -1,21 +1,22 @@ import { BaseTool } from './BaseTool'; -export class DataAnalysisTool extends BaseTool<{ csv_file_name: string }> { +export class DataAnalysisTool extends BaseTool<{ csv_file_name: string | string[] }> { private csv_files_function: () => { filename: string; id: string; text: string }[]; constructor(csv_files: () => { filename: string; id: string; text: string }[]) { super( 'dataAnalysis', - 'Analyzes, and provides insights, from a CSV file', + 'Analyzes, and provides insights, from one or more CSV files', { csv_file_name: { type: 'string', - description: 'Name of the CSV file to analyze', + description: 'Name(s) of the CSV file(s) to analyze', required: 'true', + max_inputs: '3', }, }, - 'Provide the name of the CSV file to analyze based on the user query and whichever available CSV file may be relevant.', - 'Provides the full CSV file text for your analysis based on the user query and the available CSV file. ' + 'Provide the name(s) of up to 3 CSV files to analyze based on the user query and whichever available CSV files may be relevant.', + 'Provides the full CSV file text for your analysis based on the user query and the available CSV file(s). ' ); this.csv_files_function = csv_files; } @@ -32,8 +33,27 @@ export class DataAnalysisTool extends BaseTool<{ csv_file_name: string }> { return file?.id; } - async execute(args: { csv_file_name: string }): Promise { - console.log(this.csv_files_function()); - return [{ type: 'text', text: `` + this.getFileContent(args.csv_file_name) + '' }]; + async execute(args: { csv_file_name: string | string[] }): Promise { + const filenames = Array.isArray(args.csv_file_name) ? args.csv_file_name : [args.csv_file_name]; + const results = []; + + for (const filename of filenames) { + const fileContent = this.getFileContent(filename); + const fileID = this.getFileID(filename); + + if (fileContent && fileID) { + results.push({ + type: 'text', + text: `${fileContent}`, + }); + } else { + results.push({ + type: 'text', + text: `File not found: ${filename}`, + }); + } + } + + return results; } } diff --git a/src/client/views/nodes/ChatBox/tools/SearchTool.ts b/src/client/views/nodes/ChatBox/tools/SearchTool.ts index 6523fd11c..b926cbadc 100644 --- a/src/client/views/nodes/ChatBox/tools/SearchTool.ts +++ b/src/client/views/nodes/ChatBox/tools/SearchTool.ts @@ -1,47 +1,54 @@ +import { max } from 'lodash'; import { Networking } from '../../../../Network'; import { BaseTool } from './BaseTool'; import { v4 as uuidv4 } from 'uuid'; -export class SearchTool extends BaseTool<{ query: string }> { +export class SearchTool extends BaseTool<{ query: string | string[] }> { private _addLinkedUrlDoc: (url: string, id: string) => void; - - constructor(addLinkedUrlDoc: (url: string, id: string) => void) { + private _max_results: number; + constructor(addLinkedUrlDoc: (url: string, id: string) => void, max_results: number = 5) { super( 'searchTool', - 'Search the web to find a wide range of websites related to a query', + 'Search the web to find a wide range of websites related to a query or multiple queries', { query: { type: 'string', - description: 'The search query to use for finding websites', - required: true, + description: 'The search query or queries to use for finding websites', + required: 'true', + max_inputs: '3', }, }, - 'Provide a search query to find a broad range of websites. This tool is intended to help you identify relevant websites, but not to be used for providing the final answer. Use this information to determine which specific website to investigate further.', - 'Returns a list of websites and their overviews based on the search query, helping to identify which website might contain the most relevant information.' + 'Provide up to 3 search queries to find a broad range of websites. This tool is intended to help you identify relevant websites, but not to be used for providing the final answer. Use this information to determine which specific website to investigate further.', + 'Returns a list of websites and their overviews based on the search queries, helping to identify which websites might contain relevant information.' ); this._addLinkedUrlDoc = addLinkedUrlDoc; + this._max_results = max_results; } - async execute(args: { query: string }): Promise { - try { - const { results } = await Networking.PostToServer('/getWebSearchResults', { query: args.query }); - console.log(results); - const data: { type: string; text: string }[] = results.map((result: { url: string; snippet: string }) => { - console.log; - const id = uuidv4(); - //this._addLinkedUrlDoc(result.url, id); //not needed right now because it shouldn't use this information to ground responses and should scrape afterwards - return { - type: 'text', - text: ` - ${result.url} - ${result.snippet} - `, - }; - }); - return data; - } catch (error) { - console.log(error); - return [{ type: 'text', text: 'An error occurred while performing the web search.' }]; + async execute(args: { query: string | string[] }): Promise { + const queries = Array.isArray(args.query) ? args.query : [args.query]; + const allResults = []; + + for (const query of queries) { + try { + const { results } = await Networking.PostToServer('/getWebSearchResults', { query, max_results: this._max_results }); + const data: { type: string; text: string }[] = results.map((result: { url: string; snippet: string }) => { + const id = uuidv4(); + return { + type: 'text', + text: ` + ${result.url} + ${result.snippet} + `, + }; + }); + allResults.push(...data); + } catch (error) { + console.log(error); + allResults.push({ type: 'text', text: `An error occurred while performing the web search for query: ${query}` }); + } } + + return allResults; } } diff --git a/src/client/views/nodes/ChatBox/tools/WebsiteInfoScraperTool.ts b/src/client/views/nodes/ChatBox/tools/WebsiteInfoScraperTool.ts index c59afefbd..739329bea 100644 --- a/src/client/views/nodes/ChatBox/tools/WebsiteInfoScraperTool.ts +++ b/src/client/views/nodes/ChatBox/tools/WebsiteInfoScraperTool.ts @@ -2,43 +2,51 @@ import { Networking } from '../../../../Network'; import { BaseTool } from './BaseTool'; import { v4 as uuidv4 } from 'uuid'; -export class WebsiteInfoScraperTool extends BaseTool<{ url: string }> { +export class WebsiteInfoScraperTool extends BaseTool<{ url: string | string[] }> { private _addLinkedUrlDoc: (url: string, id: string) => void; constructor(addLinkedUrlDoc: (url: string, id: string) => void) { super( 'websiteInfoScraper', - 'Scrape detailed information from a specific website identified as the most relevant', + 'Scrape detailed information from specific websites identified as relevant to the user query', { url: { type: 'string', - description: 'The URL of the website to scrape', - required: true, + description: 'The URL(s) of the website(s) to scrape', + required: 'true', + max_inputs: '3', }, }, - 'Provide the URL of the website that you have identified as the most relevant from the previous search. This tool will scrape and process detailed information from that specific website. It will also create a document from the scraped content for future reference.', - 'Returns the full HTML content from the provided URL and creates a document from the content for further analysis.' + 'Provide up to 3 URLs of websites that you have identified as the most relevant from the previous search. This tool will provide screenshots of those specific websites. It will also create a document from the scraped content for future reference. When providing a response to the user, ideally reference as many of the websites as possible in order to provide a well grounded result.', + 'Returns the full content of the webpages as images for analysis.' ); this._addLinkedUrlDoc = addLinkedUrlDoc; } - async execute(args: { url: string }): Promise { - try { - const { website_image_base64 } = await Networking.PostToServer('/scrapeWebsite', { url: args.url }); - const id = uuidv4(); - this._addLinkedUrlDoc(args.url, id); - return [ - { type: 'text', text: ` ` }, - { - type: 'image_url', - image_url: { - url: `data:image/jpeg;base64,${website_image_base64}`, + async execute(args: { url: string | string[] }): Promise { + const urls = Array.isArray(args.url) ? args.url : [args.url]; + const results = []; + + for (const url of urls) { + try { + const { website_image_base64 } = await Networking.PostToServer('/scrapeWebsite', { url }); + const id = uuidv4(); + this._addLinkedUrlDoc(url, id); + results.push( + { type: 'text', text: ` ` }, + { + type: 'image_url', + image_url: { + url: `data:image/jpeg;base64,${website_image_base64}`, + }, }, - }, - { type: 'text', text: `\n` }, - ]; - } catch (error) { - return [{ type: 'text', text: 'An error occurred while scraping the website.' }]; + { type: 'text', text: `\n` } + ); + } catch (error) { + results.push({ type: 'text', text: `An error occurred while scraping the website: ${url}` }); + } } + + return results; } } diff --git a/src/server/ApiManagers/AssistantManager.ts b/src/server/ApiManagers/AssistantManager.ts index 9b85dbbe8..afaeaf961 100644 --- a/src/server/ApiManagers/AssistantManager.ts +++ b/src/server/ApiManagers/AssistantManager.ts @@ -90,13 +90,14 @@ export default class AssistantManager extends ApiManager { method: Method.POST, subscription: '/getWebSearchResults', secureHandler: async ({ req, res }) => { - const { query } = req.body; + const { query, max_results } = req.body; try { const response = await customsearch.cse.list({ q: query, cx: process.env._CLIENT_GOOGLE_SEARCH_ENGINE_ID, key: process.env._CLIENT_GOOGLE_API_KEY, safe: 'active', + num: max_results, }); const results = -- cgit v1.2.3-70-g09d2