diff options
Diffstat (limited to 'src/server/ApiManagers/AssistantManager.ts')
| -rw-r--r-- | src/server/ApiManagers/AssistantManager.ts | 121 |
1 files changed, 78 insertions, 43 deletions
diff --git a/src/server/ApiManagers/AssistantManager.ts b/src/server/ApiManagers/AssistantManager.ts index d98d32d30..dfe5d747b 100644 --- a/src/server/ApiManagers/AssistantManager.ts +++ b/src/server/ApiManagers/AssistantManager.ts @@ -17,6 +17,7 @@ import { google } from 'googleapis'; import * as puppeteer from 'puppeteer'; import { JSDOM } from 'jsdom'; import { Readability } from '@mozilla/readability'; +import { spawn } from 'child_process'; // Enumeration of directories where different file types are stored export enum Directory { @@ -32,6 +33,10 @@ export enum Directory { scrape_images = 'scrape_images', } +// In-memory job tracking +let jobResults: { [key: string]: any } = {}; +let jobProgress: { [key: string]: any } = {}; + /** * Constructs a normalized path to a file in the server's file system. * @param directory The directory where the file is stored. @@ -246,7 +251,6 @@ export default class AssistantManager extends ApiManager { }, }); - // Register an API route to create documents by sending files to a chatbot register({ method: Method.POST, subscription: '/createDocument', @@ -259,30 +263,18 @@ export default class AssistantManager extends ApiManager { // Read the file data and encode it as base64 const file_data: string = fs.readFileSync(public_path, { encoding: 'base64' }); - // Send the file data to a local chatbot API for document creation - const response = await axios.post( - 'http://localhost:8080/createDocument', - { - file_data, - file_name, - }, - { - headers: { - 'Content-Type': 'application/json', - }, - } - ); + // Generate a unique job ID for tracking + const jobId = uuid.v4(); - // Retrieve the job ID from the response - const jobId = response.data['job_id']; - console.log('Job ID:', jobId); + // Spawn the Python process and track its progress/output + spawnPythonProcess(jobId, file_name, file_data); - // Send the job ID back to the client + // Send the job ID back to the client for tracking res.send({ jobId }); } catch (error: any) { - console.error('Error communicating with chatbot:', error); + console.error('Error initiating document creation:', error); res.status(500).send({ - error: 'Failed to communicate with the chatbot', + error: 'Failed to initiate document creation', details: error.message, }); } @@ -295,17 +287,11 @@ export default class AssistantManager extends ApiManager { subscription: '/getProgress/:jobId', secureHandler: async ({ req, res }) => { const { jobId } = req.params; // Get the job ID from the URL parameters - try { - // Query the local API to get the progress of the job - const progressResponse = await axios.get(`http://localhost:8080/getProgress/${jobId}`); - console.log(`Current step: ${progressResponse.data.step}, Progress within step: ${progressResponse.data.progress}%`); - res.json(progressResponse.data); // Send the progress data back to the client - } catch (error) { - console.error('Error getting progress:', error); - res.status(500).send({ - error: 'Failed to get progress', - details: error, - }); + // Check if the job progress is available + if (jobProgress[jobId]) { + res.json(jobProgress[jobId]); + } else { + res.status(404).send({ error: 'Job not found' }); } }, }); @@ -316,11 +302,9 @@ export default class AssistantManager extends ApiManager { subscription: '/getResult/:jobId', secureHandler: async ({ req, res }) => { const { jobId } = req.params; // Get the job ID from the URL parameters - try { - // Query the local API to get the final result of the job - const finalResponse = await axios.get(`http://localhost:8080/getResult/${jobId}`); - console.log('Result:', finalResponse.data); - const result = finalResponse.data; + // Check if the job result is available + if (jobResults[jobId]) { + const result = jobResults[jobId]; // If the result contains image or table chunks, save the base64 data as image files if (result.chunks && Array.isArray(result.chunks)) { @@ -353,16 +337,11 @@ export default class AssistantManager extends ApiManager { } result['status'] = 'completed'; } else { - console.warn('Not ready'); result.status = 'pending'; } res.json(result); // Send the result back to the client - } catch (error) { - console.error('Error getting result:', error); - res.status(500).send({ - error: 'Failed to get result', - details: error, - }); + } else { + res.status(202).send({ status: 'pending' }); } }, }); @@ -458,3 +437,59 @@ export default class AssistantManager extends ApiManager { }); } } + +function spawnPythonProcess(jobId: string, file_name: string, file_data: string) { + const pythonPath = + process.platform === 'win32' + ? path.join(__dirname, '../chunker/venv/Scripts/python') // Windows path + : path.join(__dirname, '../chunker/venv/bin/python3'); // Linux/Mac path + + const pythonProcess = spawn(pythonPath, [ + path.join(__dirname, '../chunker/pdf_chunker.py'), // Correct path to Python script + jobId, + file_name, + file_data, + ]); + + let pythonOutput = ''; // Accumulate stdout data + + // Handle stdout data (progress and final results) + pythonProcess.stdout.on('data', data => { + pythonOutput += data.toString(); // Accumulate data + + const lines = pythonOutput.split('\n'); // Handle multi-line JSON + lines.forEach(line => { + if (line.trim()) { + try { + const parsedOutput = JSON.parse(line); // Parse each line of JSON + + if (parsedOutput.job_id && parsedOutput.progress !== undefined) { + jobProgress[parsedOutput.job_id] = { + step: parsedOutput.step, + progress: parsedOutput.progress, + }; + } else if (parsedOutput.chunks) { + jobResults[parsedOutput.job_id] = parsedOutput; + jobProgress[parsedOutput.job_id] = { step: 'Complete', progress: 100 }; + } + } catch (err) { + console.error('Error parsing Python output:', err); + } + } + }); + }); + + // Handle stderr (error logging) + pythonProcess.stderr.on('data', data => { + console.error(`Python script error: ${data}`); + }); + + // Handle process exit + pythonProcess.on('close', code => { + if (code !== 0) { + console.error(`Python process exited with code ${code}`); + console.error(`Command: python3 ${path.join(__dirname, '../chunker/pdf_chunker.py')} ${jobId} ${file_name}`); + jobResults[jobId] = { error: 'Python process failed' }; + } + }); +} |