implemented search tool and other tools but scraping doesn't work

1 files changed, 35 insertions, 0 deletions

diff --git a/src/client/views/nodes/ChatBox/tools/WebsiteInfoScraperTool.ts b/src/client/views/nodes/ChatBox/tools/WebsiteInfoScraperTool.ts
new file mode 100644
index 000000000..59fd47b7a
--- /dev/null
+++ b/src/client/views/nodes/ChatBox/tools/WebsiteInfoScraperTool.ts
@@ -0,0 +1,35 @@
+import { Networking } from '../../../../Network';
+import { BaseTool } from './BaseTool';
+import { v4 as uuidv4 } from 'uuid';
+
+export class WebsiteInfoScraperTool extends BaseTool<{ url: string }> {
+    private _addLinkedUrlDoc: (url: string, id: string) => void;
+
+    constructor(addLinkedUrlDoc: (url: string, id: string) => void) {
+        super(
+            'websiteInfoScraper',
+            'Scrape detailed information from a specific website identified as the most relevant',
+            {
+                url: {
+                    type: 'string',
+                    description: 'The URL of the website to scrape',
+                    required: true,
+                },
+            },
+            'Provide the URL of the website that you have identified as the most relevant from the previous search. This tool will scrape and process detailed information from that specific website. It will also create a document from the scraped content for future reference.',
+            'Returns the full HTML content from the provided URL and creates a document from the content for further analysis.'
+        );
+        this._addLinkedUrlDoc = addLinkedUrlDoc;
+    }
+
+    async execute(args: { url: string }): Promise<any> {
+        try {
+            const { html } = await Networking.PostToServer('/scrapeWebsite', { url: args.url });
+            const id = uuidv4();
+            this._addLinkedUrlDoc(args.url, id);
+            return [{ type: 'text', text: `<chunk chunk_id=${id} chunk_type=text> ${html} </chunk>` }];
+        } catch (error) {
+            return [{ type: 'text', text: 'An error occurred while scraping the website.' }];
+        }
+    }
+}