working much better still working on adding images thouhg

1 files changed, 74 insertions, 106 deletions

diff --git a/src/server/ApiManagers/AssistantManager.ts b/src/server/ApiManagers/AssistantManager.ts
index d5a8ebeb3..36468157a 100644
--- a/src/server/ApiManagers/AssistantManager.ts
+++ b/src/server/ApiManagers/AssistantManager.ts
@@ -8,6 +8,7 @@ import { filesDirectory, publicDirectory } from '../SocketData';
 import { Method } from '../RouteManager';
 import ApiManager, { Registration } from './ApiManager';
 import axios from 'axios';
+import { Chunk } from '../../client/views/nodes/ChatBox/types';
 
 export enum Directory {
     parsed_files = 'parsed_files',
@@ -44,112 +45,6 @@ export default class AssistantManager extends ApiManager {
 
         register({
             method: Method.POST,
-            subscription: '/uploadPDFToVectorStore',
-            secureHandler: async ({ req, res }) => {
-                const { urls, threadID, assistantID, vector_store_id } = req.body;
-
-                const csvFilesIds: string[] = [];
-                const otherFileIds: string[] = [];
-                const allFileIds: string[] = [];
-
-                const fileProcesses = urls.map(async (source: string) => {
-                    const fullPath = path.join(publicDirectory, source);
-                    const fileData = await openai.files.create({ file: createReadStream(fullPath), purpose: 'assistants' });
-                    allFileIds.push(fileData.id);
-                    if (source.endsWith('.csv')) {
-                        console.log(source);
-                        csvFilesIds.push(fileData.id);
-                    } else {
-                        openai.beta.vectorStores.files.create(vector_store_id, { file_id: fileData.id });
-                        otherFileIds.push(fileData.id);
-                    }
-                });
-                try {
-                    await Promise.all(fileProcesses).then(() => {
-                        res.send({ vector_store_id: vector_store_id, openai_file_ids: allFileIds });
-                    });
-                } catch (error) {
-                    res.status(500).send({ error: 'Failed to process files' + error });
-                }
-            },
-        });
-
-        register({
-            method: Method.POST,
-            subscription: '/downloadFileFromOpenAI',
-            secureHandler: async ({ req, res }) => {
-                const { file_id, file_name } = req.body;
-                //let files_directory: string;
-                let files_directory = '/files/openAIFiles/';
-                switch (file_name.split('.').pop()) {
-                    case 'pdf':
-                        files_directory = '/files/pdfs/';
-                        break;
-                    case 'csv':
-                        files_directory = '/files/csv/';
-                        break;
-                    case 'png':
-                    case 'jpg':
-                    case 'jpeg':
-                        files_directory = '/files/images/';
-                        break;
-                    default:
-                        break;
-                }
-
-                const directory = path.join(publicDirectory, files_directory);
-
-                if (!fs.existsSync(directory)) {
-                    fs.mkdirSync(directory);
-                }
-                const file = await openai.files.content(file_id);
-                const new_file_name = `${uuid.v4()}-${file_name}`;
-                const file_path = path.join(directory, new_file_name);
-                const file_array_buffer = await file.arrayBuffer();
-                const bufferView = new Uint8Array(file_array_buffer);
-                try {
-                    const written_file = await writeFileAsync(file_path, bufferView);
-                    console.log(written_file);
-                    console.log(file_path);
-                    console.log(file_array_buffer);
-                    console.log(bufferView);
-                    const file_object = new File([bufferView], file_name);
-                    //DashUploadUtils.upload(file_object, 'openAIFiles');
-                    res.send({ file_path: path.join(files_directory, new_file_name) });
-                    /* res.send( {
-                        source: "file",
-                        result: {
-                                  accessPaths: {
-                                      agnostic: {client: path.join('/files/openAIFiles/', `${uuid.v4()}-${file_name}`)}
-                                  },
-                                  rawText: "",
-                                  duration: 0,
-                              },
-                    } ); */
-                } catch (error) {
-                    res.status(500).send({ error: 'Failed to write file' + error });
-                }
-            },
-        });
-
-        register({
-            method: Method.POST,
-            subscription: '/askAgent',
-            secureHandler: async ({ req, res }) => {
-                const { input } = req.body;
-
-                try {
-                    const response = await axios.post('http://localhost:8080/ask', { input });
-                    res.send({ response: response.data.response });
-                } catch (error: any) {
-                    console.error('Error communicating with chatbot:', error);
-                    res.status(500).send({ error: 'Failed to communicate with the chatbot', details: error.message });
-                }
-            },
-        });
-
-        register({
-            method: Method.POST,
             subscription: '/getWikipediaSummary',
             secureHandler: async ({ req, res }) => {
                 const { title } = req.body;
@@ -212,6 +107,37 @@ export default class AssistantManager extends ApiManager {
                         }
                     }
 
+                    if (result.chunks && Array.isArray(result.chunks)) {
+                        for (const chunk of result.chunks) {
+                            if (chunk.metadata && (chunk.metadata.type === 'image' || chunk.metadata.type === 'table')) {
+                                let files_directory = '/files/chunk_images/';
+                                const directory = path.join(publicDirectory, files_directory);
+
+                                if (!fs.existsSync(directory)) {
+                                    fs.mkdirSync(directory);
+                                }
+
+                                const fileName = path.basename(chunk.metadata.file_path);
+                                const filePath = path.join(directory, fileName);
+
+                                // Check if base64_data exists
+                                if (chunk.metadata.base64_data) {
+                                    // Decode Base64 and save as file
+                                    const buffer = Buffer.from(chunk.metadata.base64_data, 'base64');
+                                    await fs.promises.writeFile(filePath, buffer);
+
+                                    // Update the file path in the chunk
+                                    chunk.metadata.file_path = path.join(files_directory, fileName);
+                                    chunk.metadata.base64_data = undefined;
+                                } else {
+                                    console.warn(`No base64_data found for chunk: ${fileName}`);
+                                }
+                            }
+                        }
+                    } else {
+                        console.warn("Result does not contain an iterable 'chunks' property");
+                    }
+
                     res.send({ document_json: result });
                 } catch (error: any) {
                     console.error('Error communicating with chatbot:', error);
@@ -219,5 +145,47 @@ export default class AssistantManager extends ApiManager {
                 }
             },
         });
+
+        register({
+            method: Method.POST,
+            subscription: '/formatChunks',
+            secureHandler: async ({ req, res }) => {
+                const { relevantChunks } = req.body;
+                const content: { type: string; text?: string; image_url?: { url: string } }[] = [{ type: 'text', text: '<chunks>' }];
+
+                for (const chunk of relevantChunks) {
+                    content.push({
+                        type: 'text',
+                        text: `<chunk chunk_id=${chunk.id} chunk_type=${chunk.metadata.type === 'image' || chunk.metadata.type === 'table' ? 'image' : 'text'}>`,
+                    });
+
+                    if (chunk.metadata.type === 'image' || chunk.metadata.type === 'table') {
+                        try {
+                            const filePath = serverPathToFile(Directory.parsed_files, chunk.metadata.file_path);
+                            const imageBuffer = await readFileAsync(filePath);
+                            const base64Image = imageBuffer.toString('base64');
+                            if (base64Image) {
+                                content.push({
+                                    type: 'image_url',
+                                    image_url: {
+                                        url: `data:image/jpeg;base64,${base64Image}`,
+                                    },
+                                });
+                            } else {
+                                console.log(`Failed to encode image for chunk ${chunk.id}`);
+                            }
+                        } catch (error) {
+                            console.error(`Error reading image file for chunk ${chunk.id}:`, error);
+                        }
+                    }
+
+                    content.push({ type: 'text', text: `${chunk.metadata.text}\n</chunk>\n` });
+                }
+
+                content.push({ type: 'text', text: '</chunks>' });
+
+                res.send({ formattedChunks: content });
+            },
+        });
     }
 }