From d0a3f1e3e16e0ff6d7450bec0e2d45a0748da95b Mon Sep 17 00:00:00 2001 From: Timothy Carambat Date: Tue, 16 Jan 2024 13:41:01 -0800 Subject: [PATCH] Fix present diminsions on vectorDBs to be inferred for providers who require it (#605) --- .../EmbeddingEngines/azureOpenAi/index.js | 1 - .../utils/EmbeddingEngines/localAi/index.js | 4 --- server/utils/EmbeddingEngines/native/index.js | 1 - server/utils/EmbeddingEngines/openAi/index.js | 1 - .../utils/vectorDbProviders/milvus/index.js | 20 ++++++----- .../utils/vectorDbProviders/qdrant/index.js | 34 +++++++++++++------ 6 files changed, 35 insertions(+), 26 deletions(-) diff --git a/server/utils/EmbeddingEngines/azureOpenAi/index.js b/server/utils/EmbeddingEngines/azureOpenAi/index.js index 8cde1fc7ca..e80b4b734b 100644 --- a/server/utils/EmbeddingEngines/azureOpenAi/index.js +++ b/server/utils/EmbeddingEngines/azureOpenAi/index.js @@ -13,7 +13,6 @@ class AzureOpenAiEmbedder { new AzureKeyCredential(process.env.AZURE_OPENAI_KEY) ); this.openai = openai; - this.dimensions = 1536; // Limit of how many strings we can process in a single pass to stay with resource or network limits // https://learn.microsoft.com/en-us/azure/ai-services/openai/faq#i-am-trying-to-use-embeddings-and-received-the-error--invalidrequesterror--too-many-inputs--the-max-number-of-inputs-is-1---how-do-i-fix-this-:~:text=consisting%20of%20up%20to%2016%20inputs%20per%20API%20request diff --git a/server/utils/EmbeddingEngines/localAi/index.js b/server/utils/EmbeddingEngines/localAi/index.js index 6f9d721b9b..1480755d76 100644 --- a/server/utils/EmbeddingEngines/localAi/index.js +++ b/server/utils/EmbeddingEngines/localAi/index.js @@ -16,10 +16,6 @@ class LocalAiEmbedder { : {}), }); this.openai = new OpenAIApi(config); - // We don't know this for user's set model so for vectorDB integrations that requires dimensionality - // in schema, we will throw an error. - // Applies to QDrant and Milvus. - this.dimensions = null; // Limit of how many strings we can process in a single pass to stay with resource or network limits this.maxConcurrentChunks = 50; diff --git a/server/utils/EmbeddingEngines/native/index.js b/server/utils/EmbeddingEngines/native/index.js index 789e51fe9e..fc933e1b84 100644 --- a/server/utils/EmbeddingEngines/native/index.js +++ b/server/utils/EmbeddingEngines/native/index.js @@ -13,7 +13,6 @@ class NativeEmbedder { : path.resolve(__dirname, `../../../storage/models`) ); this.modelPath = path.resolve(this.cacheDir, "Xenova", "all-MiniLM-L6-v2"); - this.dimensions = 384; // Limit of how many strings we can process in a single pass to stay with resource or network limits this.maxConcurrentChunks = 25; diff --git a/server/utils/EmbeddingEngines/openAi/index.js b/server/utils/EmbeddingEngines/openAi/index.js index 31f556e899..105be9d73a 100644 --- a/server/utils/EmbeddingEngines/openAi/index.js +++ b/server/utils/EmbeddingEngines/openAi/index.js @@ -9,7 +9,6 @@ class OpenAiEmbedder { }); const openai = new OpenAIApi(config); this.openai = openai; - this.dimensions = 1536; // Limit of how many strings we can process in a single pass to stay with resource or network limits this.maxConcurrentChunks = 500; diff --git a/server/utils/vectorDbProviders/milvus/index.js b/server/utils/vectorDbProviders/milvus/index.js index a9104784bb..cc934a9a2b 100644 --- a/server/utils/vectorDbProviders/milvus/index.js +++ b/server/utils/vectorDbProviders/milvus/index.js @@ -81,13 +81,15 @@ const Milvus = { await client.dropCollection({ collection_name: namespace }); return true; }, - getOrCreateCollection: async function (client, namespace) { + // Milvus requires a dimension aspect for collection creation + // we pass this in from the first chunk to infer the dimensions like other + // providers do. + getOrCreateCollection: async function (client, namespace, dimensions = null) { const isExists = await this.namespaceExists(client, namespace); if (!isExists) { - const embedder = getEmbeddingEngineSelection(); - if (!embedder.dimensions) + if (!dimensions) throw new Error( - `Your embedder selection has unknown dimensions output. It should be defined when using ${this.name}. Open an issue on Github for support.` + `Milvus:getOrCreateCollection Unable to infer vector dimension from input. Open an issue on Github for support.` ); await client.createCollection({ @@ -104,7 +106,7 @@ const Milvus = { name: "vector", description: "vector", data_type: DataType.FloatVector, - dim: embedder.dimensions, + dim: dimensions, }, { name: "metadata", @@ -131,6 +133,7 @@ const Milvus = { ) { const { DocumentVectors } = require("../../../models/vectors"); try { + let vectorDimension = null; const { pageContent, docId, ...metadata } = documentData; if (!pageContent || pageContent.length == 0) return false; @@ -138,11 +141,11 @@ const Milvus = { const cacheResult = await cachedVectorInformation(fullFilePath); if (cacheResult.exists) { const { client } = await this.connect(); - await this.getOrCreateCollection(client, namespace); - const { chunks } = cacheResult; const documentVectors = []; + vectorDimension = chunks[0][0].values.length || null; + await this.getOrCreateCollection(client, namespace, vectorDimension); for (const chunk of chunks) { // Before sending to Pinecone and saving the records to our db // we need to assign the id of each chunk that is stored in the cached file. @@ -182,6 +185,7 @@ const Milvus = { if (!!vectorValues && vectorValues.length > 0) { for (const [i, vector] of vectorValues.entries()) { + if (!vectorDimension) vectorDimension = vector.length; const vectorRecord = { id: uuidv4(), values: vector, @@ -202,7 +206,7 @@ const Milvus = { if (vectors.length > 0) { const chunks = []; const { client } = await this.connect(); - await this.getOrCreateCollection(client, namespace); + await this.getOrCreateCollection(client, namespace, vectorDimension); console.log("Inserting vectorized chunks into Milvus."); for (const chunk of toChunks(vectors, 100)) { diff --git a/server/utils/vectorDbProviders/qdrant/index.js b/server/utils/vectorDbProviders/qdrant/index.js index ddc3408da8..2783cde932 100644 --- a/server/utils/vectorDbProviders/qdrant/index.js +++ b/server/utils/vectorDbProviders/qdrant/index.js @@ -108,19 +108,20 @@ const QDrant = { await client.deleteCollection(namespace); return true; }, - getOrCreateCollection: async function (client, namespace) { + // QDrant requires a dimension aspect for collection creation + // we pass this in from the first chunk to infer the dimensions like other + // providers do. + getOrCreateCollection: async function (client, namespace, dimensions = null) { if (await this.namespaceExists(client, namespace)) { return await client.getCollection(namespace); } - - const embedder = getEmbeddingEngineSelection(); - if (!embedder.dimensions) + if (!dimensions) throw new Error( - `Your embedder selection has unknown dimensions output. It should be defined when using ${this.name}. Open an issue on Github for support.` + `Qdrant:getOrCreateCollection Unable to infer vector dimension from input. Open an issue on Github for support.` ); await client.createCollection(namespace, { vectors: { - size: embedder.dimensions, + size: dimensions, distance: "Cosine", }, }); @@ -133,6 +134,7 @@ const QDrant = { ) { const { DocumentVectors } = require("../../../models/vectors"); try { + let vectorDimension = null; const { pageContent, docId, ...metadata } = documentData; if (!pageContent || pageContent.length == 0) return false; @@ -140,15 +142,20 @@ const QDrant = { const cacheResult = await cachedVectorInformation(fullFilePath); if (cacheResult.exists) { const { client } = await this.connect(); - const collection = await this.getOrCreateCollection(client, namespace); + const { chunks } = cacheResult; + const documentVectors = []; + vectorDimension = chunks[0][0].vector.length || null; + + const collection = await this.getOrCreateCollection( + client, + namespace, + vectorDimension + ); if (!collection) throw new Error("Failed to create new QDrant collection!", { namespace, }); - const { chunks } = cacheResult; - const documentVectors = []; - for (const chunk of chunks) { const submission = { ids: [], @@ -210,6 +217,7 @@ const QDrant = { if (!!vectorValues && vectorValues.length > 0) { for (const [i, vector] of vectorValues.entries()) { + if (!vectorDimension) vectorDimension = vector.length; const vectorRecord = { id: uuidv4(), vector: vector, @@ -233,7 +241,11 @@ const QDrant = { } const { client } = await this.connect(); - const collection = await this.getOrCreateCollection(client, namespace); + const collection = await this.getOrCreateCollection( + client, + namespace, + vectorDimension + ); if (!collection) throw new Error("Failed to create new QDrant collection!", { namespace,