diff options
Diffstat (limited to 'src/server/ApiManagers/AssistantManager.ts')
| -rw-r--r-- | src/server/ApiManagers/AssistantManager.ts | 178 |
1 files changed, 87 insertions, 91 deletions
diff --git a/src/server/ApiManagers/AssistantManager.ts b/src/server/ApiManagers/AssistantManager.ts index e940fb121..b4ebb1eae 100644 --- a/src/server/ApiManagers/AssistantManager.ts +++ b/src/server/ApiManagers/AssistantManager.ts @@ -1,23 +1,19 @@ +import { Readability } from '@mozilla/readability'; +import axios from 'axios'; +import { spawn } from 'child_process'; import * as fs from 'fs'; -import { createReadStream, writeFile } from 'fs'; +import { writeFile } from 'fs'; +import { google } from 'googleapis'; +import { JSDOM } from 'jsdom'; import OpenAI from 'openai'; import * as path from 'path'; +import * as puppeteer from 'puppeteer'; import { promisify } from 'util'; import * as uuid from 'uuid'; -import { filesDirectory, publicDirectory } from '../SocketData'; +import { AI_Document } from '../../client/views/nodes/chatbot/types/types'; import { Method } from '../RouteManager'; +import { filesDirectory, publicDirectory } from '../SocketData'; import ApiManager, { Registration } from './ApiManager'; -import axios from 'axios'; -import { RAGChunk } from '../../client/views/nodes/chatbot/types/types'; -import { UnstructuredClient } from 'unstructured-client'; -import { PartitionResponse } from 'unstructured-client/sdk/models/operations'; -import { ChunkingStrategy, Strategy } from 'unstructured-client/sdk/models/shared'; -import * as cheerio from 'cheerio'; -import { google } from 'googleapis'; -import * as puppeteer from 'puppeteer'; -import { JSDOM } from 'jsdom'; -import { Readability } from '@mozilla/readability'; -import { spawn } from 'child_process'; // Enumeration of directories where different file types are stored export enum Directory { @@ -34,8 +30,8 @@ export enum Directory { } // In-memory job tracking -let jobResults: { [key: string]: any } = {}; -let jobProgress: { [key: string]: any } = {}; +const jobResults: { [key: string]: unknown } = {}; +const jobProgress: { [key: string]: unknown } = {}; /** * Constructs a normalized path to a file in the server's file system. @@ -81,10 +77,10 @@ export default class AssistantManager extends ApiManager { */ protected initialize(register: Registration): void { // Initialize OpenAI API with client key - const openai = new OpenAI({ + const openai = new OpenAI({ // bcz: is this needed? variable is never used... apiKey: process.env._CLIENT_OPENAI_KEY, dangerouslyAllowBrowser: true, - }); + }); // prettier-ignore // Initialize Google Custom Search API const customsearch = google.customsearch('v1'); @@ -107,11 +103,11 @@ export default class AssistantManager extends ApiManager { }); const summary = response.data.query.search[0]?.snippet || 'No article found with that title.'; res.send({ text: summary }); - } catch (error: any) { + } catch (error) { console.error('Error retrieving Wikipedia summary:', error); res.status(500).send({ error: 'Error retrieving article summary from Wikipedia.', - details: error.message, + details: (error as { message: string }).message ?? error, // bcz: don't know what the error type contains... }); } }, @@ -134,17 +130,17 @@ export default class AssistantManager extends ApiManager { }); const results = - response.data.items?.map((item: any) => ({ + response.data.items?.map(item => ({ url: item.link, snippet: item.snippet, })) || []; res.send({ results }); - } catch (error: any) { + } catch (error) { console.error('Error performing web search:', error); res.status(500).send({ error: 'Failed to perform web search', - details: error.message, + details: (error as { message: string }).message ?? error, // bcz: don't know wha tthe error type contains... }); } }, @@ -170,16 +166,16 @@ export default class AssistantManager extends ApiManager { * @param retries The number of retry attempts. * @param backoff Initial backoff time in milliseconds. */ - const fetchWithRetry = async (url: string, retries = 3, backoff = 300) => { + const fetchWithRetry = async (url: string, retries = 3, backoff = 300): Promise<unknown> => { try { const response = await axiosInstance.get(url); return response.data; - } catch (error: any) { - if (retries > 0 && error.response?.status === 429) { + } catch (error) { + if (retries > 0 && (error as { response: { status: number } }).response?.status === 429) { // bcz: don't know the error type console.log(`Rate limited. Retrying in ${backoff}ms...`); await delay(backoff); return fetchWithRetry(url, retries - 1, backoff * 2); - } + } // prettier-ignore throw error; } }; @@ -199,11 +195,11 @@ export default class AssistantManager extends ApiManager { try { const data = await fetchWithRetry(url); res.send({ data }); - } catch (error: any) { + } catch (error) { console.error('Error fetching the URL:', error); res.status(500).send({ error: 'Failed to fetch the URL', - details: error.message, + details: (error as { message: string }).message ?? error, // bcz: don't know wha tthe error type contains... }); } }, @@ -241,11 +237,11 @@ export default class AssistantManager extends ApiManager { } else { res.status(500).send({ error: 'Failed to extract readable content' }); } - } catch (error: any) { + } catch (error) { console.error('Error scraping website:', error); res.status(500).send({ error: 'Failed to scrape website', - details: error.message, + details: (error as { message: string }).message ?? error, // bcz: don't know wha tthe error type contains... }); } }, @@ -267,15 +263,16 @@ export default class AssistantManager extends ApiManager { const jobId = uuid.v4(); // Spawn the Python process and track its progress/output + // eslint-disable-next-line no-use-before-define spawnPythonProcess(jobId, file_name, file_data); // Send the job ID back to the client for tracking res.send({ jobId }); - } catch (error: any) { + } catch (error) { console.error('Error initiating document creation:', error); res.status(500).send({ error: 'Failed to initiate document creation', - details: error.message, + details: (error as { message: string }).message ?? error, // bcz: don't know wha tthe error type contains... }); } }, @@ -307,13 +304,13 @@ export default class AssistantManager extends ApiManager { const { jobId } = req.params; // Get the job ID from the URL parameters // Check if the job result is available if (jobResults[jobId]) { - const result = jobResults[jobId]; + const result = jobResults[jobId] as AI_Document & { status: string }; // If the result contains image or table chunks, save the base64 data as image files if (result.chunks && Array.isArray(result.chunks)) { for (const chunk of result.chunks) { if (chunk.metadata && (chunk.metadata.type === 'image' || chunk.metadata.type === 'table')) { - let files_directory = '/files/chunk_images/'; + const files_directory = '/files/chunk_images/'; const directory = path.join(publicDirectory, files_directory); // Ensure the directory exists or create it @@ -338,7 +335,7 @@ export default class AssistantManager extends ApiManager { } } } - result['status'] = 'completed'; + result.status = 'completed'; } else { result.status = 'pending'; } @@ -429,11 +426,11 @@ export default class AssistantManager extends ApiManager { // Send the file URL and UUID back to the client res.send({ fileUrl, id: uuidv4 }); - } catch (error: any) { + } catch (error) { console.error('Error creating CSV file:', error); res.status(500).send({ error: 'Failed to create CSV file.', - details: error.message, + details: (error as { message: string }).message ?? error, // bcz: don't know what the error type contains... }); } }, @@ -446,59 +443,6 @@ function spawnPythonProcess(jobId: string, file_name: string, file_data: string) const requirementsPath = path.join(__dirname, '../chunker/requirements.txt'); const pythonScriptPath = path.join(__dirname, '../chunker/pdf_chunker.py'); - // Check if venv exists - if (!fs.existsSync(venvPath)) { - console.log('Virtual environment not found. Creating and setting up...'); - - // Create venv - const createVenvProcess = spawn('python', ['-m', 'venv', venvPath]); - - createVenvProcess.on('close', code => { - if (code !== 0) { - console.error(`Failed to create virtual environment. Exit code: ${code}`); - return; - } - - console.log('Virtual environment created. Installing requirements...'); - - // Determine the pip path based on the OS - const pipPath = process.platform === 'win32' ? path.join(venvPath, 'Scripts', 'pip.exe') : path.join(venvPath, 'bin', 'pip3'); // Try 'pip3' for Unix-like systems - - if (!fs.existsSync(pipPath)) { - console.error(`pip executable not found at ${pipPath}`); - return; - } - - // Install requirements - const installRequirementsProcess = spawn(pipPath, ['install', '-r', requirementsPath]); - - installRequirementsProcess.stdout.on('data', data => { - console.log(`pip stdout: ${data}`); - }); - - installRequirementsProcess.stderr.on('data', data => { - console.error(`pip stderr: ${data}`); - }); - - installRequirementsProcess.on('error', error => { - console.error(`Error starting pip process: ${error}`); - }); - - installRequirementsProcess.on('close', code => { - if (code !== 0) { - console.error(`Failed to install requirements. Exit code: ${code}`); - return; - } - - console.log('Requirements installed. Running Python script...'); - runPythonScript(); - }); - }); - } else { - console.log('Virtual environment found. Running Python script...'); - runPythonScript(); - } - function runPythonScript() { const pythonPath = process.platform === 'win32' ? path.join(venvPath, 'Scripts', 'python') : path.join(venvPath, 'bin', 'python3'); @@ -530,7 +474,7 @@ function spawnPythonProcess(jobId: string, file_name: string, file_data: string) }; } } catch (err) { - console.error('Progress log from Python:', line); + console.error('Progress log from Python:', line, err); } } }); @@ -551,4 +495,56 @@ function spawnPythonProcess(jobId: string, file_name: string, file_data: string) } }); } + // Check if venv exists + if (!fs.existsSync(venvPath)) { + console.log('Virtual environment not found. Creating and setting up...'); + + // Create venv + const createVenvProcess = spawn('python', ['-m', 'venv', venvPath]); + + createVenvProcess.on('close', code => { + if (code !== 0) { + console.error(`Failed to create virtual environment. Exit code: ${code}`); + return; + } + + console.log('Virtual environment created. Installing requirements...'); + + // Determine the pip path based on the OS + const pipPath = process.platform === 'win32' ? path.join(venvPath, 'Scripts', 'pip.exe') : path.join(venvPath, 'bin', 'pip3'); // Try 'pip3' for Unix-like systems + + if (!fs.existsSync(pipPath)) { + console.error(`pip executable not found at ${pipPath}`); + return; + } + + // Install requirements + const installRequirementsProcess = spawn(pipPath, ['install', '-r', requirementsPath]); + + installRequirementsProcess.stdout.on('data', data => { + console.log(`pip stdout: ${data}`); + }); + + installRequirementsProcess.stderr.on('data', data => { + console.error(`pip stderr: ${data}`); + }); + + installRequirementsProcess.on('error', error => { + console.error(`Error starting pip process: ${error}`); + }); + + installRequirementsProcess.on('close', closecode => { + if (closecode !== 0) { + console.error(`Failed to install requirements. Exit code: ${closecode}`); + return; + } + + console.log('Requirements installed. Running Python script...'); + runPythonScript(); + }); + }); + } else { + console.log('Virtual environment found. Running Python script...'); + runPythonScript(); + } } |