attempt at new multimodal vector

1 files changed, 20 insertions, 18 deletions

diff --git a/src/client/views/nodes/ChatBox/vectorstore/VectorstoreUpload.ts b/src/client/views/nodes/ChatBox/vectorstore/Vectorstore.ts
index 787705bb6..25aec751f 100644
--- a/src/client/views/nodes/ChatBox/vectorstore/VectorstoreUpload.ts
+++ b/src/client/views/nodes/ChatBox/vectorstore/Vectorstore.ts
@@ -2,6 +2,7 @@ import { Pinecone, Index, IndexList, PineconeRecord, RecordMetadata, QueryRespon
 import { CohereClient } from 'cohere-ai';
 import { EmbedResponse } from 'cohere-ai/api';
 import dotenv from 'dotenv';
+import axios from 'axios';
 
 import { Chunk, AI_Document, CHUNK_TYPE } from '../types';
 import { Doc } from '../../../../../fields/Doc';
@@ -43,7 +44,7 @@ export class Vectorstore {
         if (!indexList.indexes?.some(index => index.name === this.indexName)) {
             await this.pinecone.createIndex({
                 name: this.indexName,
-                dimension: 1024,
+                dimension: 768,
                 metric: 'cosine',
                 spec: {
                     serverless: {
@@ -138,25 +139,26 @@ export class Vectorstore {
     async retrieve(query: string, topK: number = 10): Promise<Chunk[]> {
         console.log(`Retrieving chunks for query: ${query}`);
         try {
-            const queryEmbeddingResponse: EmbedResponse = await this.cohere.embed({
-                texts: [query],
-                model: 'embed-english-v3.0',
-                inputType: 'search_query',
-            });
-
-            let queryEmbedding: number[];
-
-            if (Array.isArray(queryEmbeddingResponse.embeddings)) {
-                queryEmbedding = queryEmbeddingResponse.embeddings[0];
-            } else if (queryEmbeddingResponse.embeddings && 'embeddings' in queryEmbeddingResponse.embeddings) {
-                queryEmbedding = (queryEmbeddingResponse.embeddings as { embeddings: number[][] }).embeddings[0];
-            } else {
-                throw new Error('Invalid embedding response format');
+            const url = 'https://api.jina.ai/v1/embeddings';
+            const headers = {
+                'Content-Type': 'application/json',
+                Authorization: `Bearer ${process.env.JINA_API_KEY}`,
+            };
+            const data = {
+                model: 'jina-clip-v1',
+                normalized: true,
+                embedding_type: 'float',
+                input: [{ text: query }],
+            };
+
+            const response = await axios.post(url, data, { headers });
+            const embeddings = response.data?.data?.[0]?.embedding;
+
+            if (!embeddings || !Array.isArray(embeddings)) {
+                throw new Error('Invalid embedding response format from Jina API');
             }
 
-            if (!Array.isArray(queryEmbedding)) {
-                throw new Error('Query embedding is not an array');
-            }
+            const queryEmbedding = embeddings;
 
             const queryResponse: QueryResponse<RecordMetadata> = await this.index.query({
                 vector: queryEmbedding,