implemented search tool and other tools but scraping doesn't work

1 files changed, 139 insertions, 14 deletions

diff --git a/package-lock.json b/package-lock.json
index 0a485dcb7..c73470775 100644
--- a/package-lock.json
+++ b/package-lock.json
@@ -74,6 +74,7 @@
                 "bson": "^6.2.0",
                 "canvas": "^2.11.2",
                 "chart.js": "^4.4.0",
+                "cheerio": "^1.0.0",
                 "child_process": "^1.0.2",
                 "class-transformer": "^0.5.1",
                 "cohere-ai": "^7.10.6",
@@ -306,7 +307,7 @@
                 "eslint-plugin-react": "^7.34.1",
                 "eslint-plugin-react-hooks": "^4.6.0",
                 "globals": "^15.1.0",
-                "jsdom": "^24.0.0",
+                "jsdom": "^24.1.1",
                 "mocha": "^10.2.0",
                 "prettier": "^3.1.0",
                 "scss-loader": "0.0.1",
@@ -16599,6 +16600,79 @@
                 "node": ">= 16"
             }
         },
+        "node_modules/cheerio": {
+            "version": "1.0.0",
+            "resolved": "https://registry.npmjs.org/cheerio/-/cheerio-1.0.0.tgz",
+            "integrity": "sha512-quS9HgjQpdaXOvsZz82Oz7uxtXiy6UIsIQcpBj7HRw2M63Skasm9qlDocAM7jNuaxdhpPU7c4kJN+gA5MCu4ww==",
+            "dependencies": {
+                "cheerio-select": "^2.1.0",
+                "dom-serializer": "^2.0.0",
+                "domhandler": "^5.0.3",
+                "domutils": "^3.1.0",
+                "encoding-sniffer": "^0.2.0",
+                "htmlparser2": "^9.1.0",
+                "parse5": "^7.1.2",
+                "parse5-htmlparser2-tree-adapter": "^7.0.0",
+                "parse5-parser-stream": "^7.1.2",
+                "undici": "^6.19.5",
+                "whatwg-mimetype": "^4.0.0"
+            },
+            "engines": {
+                "node": ">=18.17"
+            },
+            "funding": {
+                "url": "https://github.com/cheeriojs/cheerio?sponsor=1"
+            }
+        },
+        "node_modules/cheerio-select": {
+            "version": "2.1.0",
+            "resolved": "https://registry.npmjs.org/cheerio-select/-/cheerio-select-2.1.0.tgz",
+            "integrity": "sha512-9v9kG0LvzrlcungtnJtpGNxY+fzECQKhK4EGJX2vByejiMX84MFNQw4UxPJl3bFbTMw+Dfs37XaIkCwTZfLh4g==",
+            "dependencies": {
+                "boolbase": "^1.0.0",
+                "css-select": "^5.1.0",
+                "css-what": "^6.1.0",
+                "domelementtype": "^2.3.0",
+                "domhandler": "^5.0.3",
+                "domutils": "^3.0.1"
+            },
+            "funding": {
+                "url": "https://github.com/sponsors/fb55"
+            }
+        },
+        "node_modules/cheerio-select/node_modules/css-select": {
+            "version": "5.1.0",
+            "resolved": "https://registry.npmjs.org/css-select/-/css-select-5.1.0.tgz",
+            "integrity": "sha512-nwoRF1rvRRnnCqqY7updORDsuqKzqYJ28+oSMaJMMgOauh3fvwHqMS7EZpIPqK8GL+g9mKxF1vP/ZjSeNjEVHg==",
+            "dependencies": {
+                "boolbase": "^1.0.0",
+                "css-what": "^6.1.0",
+                "domhandler": "^5.0.2",
+                "domutils": "^3.0.1",
+                "nth-check": "^2.0.1"
+            },
+            "funding": {
+                "url": "https://github.com/sponsors/fb55"
+            }
+        },
+        "node_modules/cheerio/node_modules/htmlparser2": {
+            "version": "9.1.0",
+            "resolved": "https://registry.npmjs.org/htmlparser2/-/htmlparser2-9.1.0.tgz",
+            "integrity": "sha512-5zfg6mHUoaer/97TxnGpxmbR7zJtPwIYFMZ/H5ucTlPZhKvtum05yiPK3Mgai3a0DyVxv7qYqoweaEd2nrYQzQ==",
+            "funding": [
+                "https://github.com/fb55/htmlparser2?sponsor=1",
+                {
+                    "type": "github",
+                    "url": "https://github.com/sponsors/fb55"
+                }
+            ],
+            "dependencies": {
+                "domelementtype": "^2.3.0",
+                "domhandler": "^5.0.3",
+                "domutils": "^3.1.0",
+                "entities": "^4.5.0"
+            }
+        },
         "node_modules/child_process": {
             "version": "1.0.2",
             "resolved": "https://registry.npmjs.org/child_process/-/child_process-1.0.2.tgz",
@@ -18866,6 +18940,29 @@
                 "iconv-lite": "^0.6.2"
             }
         },
+        "node_modules/encoding-sniffer": {
+            "version": "0.2.0",
+            "resolved": "https://registry.npmjs.org/encoding-sniffer/-/encoding-sniffer-0.2.0.tgz",
+            "integrity": "sha512-ju7Wq1kg04I3HtiYIOrUrdfdDvkyO9s5XM8QAj/bN61Yo/Vb4vgJxy5vi4Yxk01gWHbrofpPtpxM8bKger9jhg==",
+            "dependencies": {
+                "iconv-lite": "^0.6.3",
+                "whatwg-encoding": "^3.1.1"
+            },
+            "funding": {
+                "url": "https://github.com/fb55/encoding-sniffer?sponsor=1"
+            }
+        },
+        "node_modules/encoding-sniffer/node_modules/iconv-lite": {
+            "version": "0.6.3",
+            "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.6.3.tgz",
+            "integrity": "sha512-4fCk79wshMdzMp2rH06qWrJE4iolqLhCUH+OiuIgU++RB0+94NlDL81atO7GX55uUKueo0txHNtvEyI6D7WdMw==",
+            "dependencies": {
+                "safer-buffer": ">= 2.1.2 < 3.0.0"
+            },
+            "engines": {
+                "node": ">=0.10.0"
+            }
+        },
         "node_modules/encoding/node_modules/iconv-lite": {
             "version": "0.6.3",
             "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.6.3.tgz",
@@ -24992,9 +25089,9 @@
             "integrity": "sha512-UVU9dibq2JcFWxQPA6KCqj5O42VOmAY3zQUfEKxU0KpTGXwNoCjkX1e13eHNvw/xPynt6pU0rZ1htjWTNTSXsg=="
         },
         "node_modules/jsdom": {
-            "version": "24.1.0",
-            "resolved": "https://registry.npmjs.org/jsdom/-/jsdom-24.1.0.tgz",
-            "integrity": "sha512-6gpM7pRXCwIOKxX47cgOyvyQDN/Eh0f1MeKySBV2xGdKtqJBLj8P25eY3EVCWo2mglDDzozR2r2MW4T+JiNUZA==",
+            "version": "24.1.1",
+            "resolved": "https://registry.npmjs.org/jsdom/-/jsdom-24.1.1.tgz",
+            "integrity": "sha512-5O1wWV99Jhq4DV7rCLIoZ/UIhyQeDR7wHVyZAHAshbrvZsLs+Xzz7gtwnlJTJDjleiTKh54F4dXrX70vJQTyJQ==",
             "dev": true,
             "dependencies": {
                 "cssstyle": "^4.0.1",
@@ -25003,11 +25100,11 @@
                 "form-data": "^4.0.0",
                 "html-encoding-sniffer": "^4.0.0",
                 "http-proxy-agent": "^7.0.2",
-                "https-proxy-agent": "^7.0.4",
+                "https-proxy-agent": "^7.0.5",
                 "is-potential-custom-element-name": "^1.0.1",
-                "nwsapi": "^2.2.10",
+                "nwsapi": "^2.2.12",
                 "parse5": "^7.1.2",
-                "rrweb-cssom": "^0.7.0",
+                "rrweb-cssom": "^0.7.1",
                 "saxes": "^6.0.0",
                 "symbol-tree": "^3.2.4",
                 "tough-cookie": "^4.1.4",
@@ -25016,7 +25113,7 @@
                 "whatwg-encoding": "^3.1.1",
                 "whatwg-mimetype": "^4.0.0",
                 "whatwg-url": "^14.0.0",
-                "ws": "^8.17.0",
+                "ws": "^8.18.0",
                 "xml-name-validator": "^5.0.0"
             },
             "engines": {
@@ -35598,9 +35695,9 @@
             }
         },
         "node_modules/nwsapi": {
-            "version": "2.2.10",
-            "resolved": "https://registry.npmjs.org/nwsapi/-/nwsapi-2.2.10.tgz",
-            "integrity": "sha512-QK0sRs7MKv0tKe1+5uZIQk/C8XGza4DAnztJG8iD+TpJIORARrCxczA738awHrZoHeTjSSoHqao2teO0dC/gFQ==",
+            "version": "2.2.12",
+            "resolved": "https://registry.npmjs.org/nwsapi/-/nwsapi-2.2.12.tgz",
+            "integrity": "sha512-qXDmcVlZV4XRtKFzddidpfVP4oMSGhga+xdMc25mv8kaLUHtgzCDhUxkrN8exkGdTlLNaXj7CV3GtON7zuGZ+w==",
             "dev": true
         },
         "node_modules/oauth": {
@@ -36157,6 +36254,29 @@
                 "url": "https://github.com/inikulin/parse5?sponsor=1"
             }
         },
+        "node_modules/parse5-htmlparser2-tree-adapter": {
+            "version": "7.0.0",
+            "resolved": "https://registry.npmjs.org/parse5-htmlparser2-tree-adapter/-/parse5-htmlparser2-tree-adapter-7.0.0.tgz",
+            "integrity": "sha512-B77tOZrqqfUfnVcOrUvfdLbz4pu4RopLD/4vmu3HUPswwTA8OH0EMW9BlWR2B0RCoiZRAHEUu7IxeP1Pd1UU+g==",
+            "dependencies": {
+                "domhandler": "^5.0.2",
+                "parse5": "^7.0.0"
+            },
+            "funding": {
+                "url": "https://github.com/inikulin/parse5?sponsor=1"
+            }
+        },
+        "node_modules/parse5-parser-stream": {
+            "version": "7.1.2",
+            "resolved": "https://registry.npmjs.org/parse5-parser-stream/-/parse5-parser-stream-7.1.2.tgz",
+            "integrity": "sha512-JyeQc9iwFLn5TbvvqACIF/VXG6abODeB3Fwmv/TGdLk2LfbWkaySGY72at4+Ty7EkPZj854u4CrICqNk2qIbow==",
+            "dependencies": {
+                "parse5": "^7.0.0"
+            },
+            "funding": {
+                "url": "https://github.com/inikulin/parse5?sponsor=1"
+            }
+        },
         "node_modules/parseley": {
             "version": "0.12.1",
             "resolved": "https://registry.npmjs.org/parseley/-/parseley-0.12.1.tgz",
@@ -42277,6 +42397,14 @@
             "resolved": "https://registry.npmjs.org/undefsafe/-/undefsafe-2.0.5.tgz",
             "integrity": "sha512-WxONCrssBM8TSPRqN5EmsjVrsv4A8X12J4ArBiiayv3DyyG3ZlIg6yysuuSYdZsVz3TKcTg2fd//Ujd4CHV1iA=="
         },
+        "node_modules/undici": {
+            "version": "6.19.7",
+            "resolved": "https://registry.npmjs.org/undici/-/undici-6.19.7.tgz",
+            "integrity": "sha512-HR3W/bMGPSr90i8AAp2C4DM3wChFdJPLrWYpIS++LxS8K+W535qftjt+4MyjNYHeWabMj1nvtmLIi7l++iq91A==",
+            "engines": {
+                "node": ">=18.17"
+            }
+        },
         "node_modules/undici-types": {
             "version": "5.26.5",
             "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-5.26.5.tgz",
@@ -43380,7 +43508,6 @@
             "version": "3.1.1",
             "resolved": "https://registry.npmjs.org/whatwg-encoding/-/whatwg-encoding-3.1.1.tgz",
             "integrity": "sha512-6qN4hJdMwfYBtE3YBTTHhoeuUrDBPZmbQaxWAqSALV/MeEnR5z1xd8UKud2RAkFoPkmB+hli1TZSnyi84xz1vQ==",
-            "dev": true,
             "dependencies": {
                 "iconv-lite": "0.6.3"
             },
@@ -43392,7 +43519,6 @@
             "version": "0.6.3",
             "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.6.3.tgz",
             "integrity": "sha512-4fCk79wshMdzMp2rH06qWrJE4iolqLhCUH+OiuIgU++RB0+94NlDL81atO7GX55uUKueo0txHNtvEyI6D7WdMw==",
-            "dev": true,
             "dependencies": {
                 "safer-buffer": ">= 2.1.2 < 3.0.0"
             },
@@ -43409,7 +43535,6 @@
             "version": "4.0.0",
             "resolved": "https://registry.npmjs.org/whatwg-mimetype/-/whatwg-mimetype-4.0.0.tgz",
             "integrity": "sha512-QaKxh0eNIi2mE9p2vEdzfagOKHCcj1pJ56EEHGQOVxp8r9/iszLUUV7v89x9O1p/T+NlTM5W7jW6+cz4Fq1YVg==",
-            "dev": true,
             "engines": {
                 "node": ">=18"
             }