fix: better recognition of vectorisers for chunked collections

Author: dannyjameswilliams

elysia/tools/retrieval/chunk.py

@@ -194,7 +194,6 @@ async def get_vectoriser(
             self.collection_name
         ).config.get()
 
-        # see if any named vectors exclusively vectorise the content field
         if collection_config.vector_config is not None:
             for (
                 named_vector_name,
@@ -203,9 +202,15 @@ async def get_vectoriser(
 
                 named_vectorizer = named_vector_config.vectorizer
 
-                if (
+                if (  # default weaviate naming "default" or a single named vector
+                    (
+                        named_vector_name == "default"
+                        or len(collection_config.vector_config) == 1
+                    )
+                    and named_vectorizer.source_properties is None
+                ) or (  # named vector with source properties which includes the content field
                     named_vectorizer.source_properties is not None
-                    and named_vectorizer.source_properties == [content_field]
+                    and content_field in named_vectorizer.source_properties
                 ):
                     try:
                         vectorizer = getattr(
@@ -229,27 +234,20 @@ async def get_vectoriser(
                     except AttributeError as e:
                         pass
 
-        # if we haven't returned yet, try the overall vectoriser
-        if collection_config.vectorizer_config is not None:
-            try:
-                vectorizer_name = (
-                    collection_config.vectorizer_config.vectorizer.replace("-", "_")
-                )
-                vectorizer = getattr(Configure.Vectors, vectorizer_name)
-                valid_args = inspect.signature(vectorizer).parameters
-
-                return vectorizer(
-                    **{
-                        arg: collection_config.vectorizer_config.model[arg]
-                        for arg in collection_config.vectorizer_config.model
-                        if arg in valid_args and arg != "vector_index_config"
-                    },
-                    vector_index_config=Configure.VectorIndex.hnsw(
-                        quantizer=Configure.VectorIndex.Quantizer.sq()  # scalar quantization
-                    ),
-                )
-            except AttributeError as e:
-                pass
+        # check old vectorizer config
+        elif collection_config.vectorizer_config is not None:
+            vectorizer_name = collection_config.vectorizer_config.vectorizer.replace(
+                "-", "_"
+            )
+            vectorizer = getattr(Configure.Vectors, vectorizer_name)
+            valid_args = inspect.signature(vectorizer).parameters
+            return vectorizer(
+                **{
+                    arg: collection_config.vectorizer_config.model[arg]
+                    for arg in collection_config.vectorizer_config.model
+                    if arg in valid_args and arg != "vector_index_config"
+                }
+            )
 
         # otherwise use default weaviate embedding service
         return Configure.Vectors.text2vec_weaviate(

tests/requires_env/general/test_chunking.py

@@ -16,15 +16,15 @@
 
 
 @pytest.mark.asyncio
-async def test_correct_vectoriser():
+async def test_correct_vectoriser_old():
     collection_name_full = "Test_ELYSIA_collection_chunking_vectoriser_full"
     collection_name_named = "Test_ELYSIA_collection_chunking_vectoriser_named"
     client_manager = ClientManager()
 
     # example data
     data = [
         {
-            "content": (
+            "random_content_field": (
                 "Lorem ipsum dolor sit amet consectetur adipiscing elit. "
                 "Quisque faucibus ex sapien vitae pellentesque sem placerat. "
                 "In id cursus mi pretium tellus duis convallis. "
@@ -37,7 +37,7 @@ async def test_correct_vectoriser():
             "other_field": "other_value",
         },
         {
-            "content": (
+            "random_content_field": (
                 "Lorem ipsum dolor sit sit amet consectetur adipiscing elit. "
                 "Quisque faucibus ex sapien sed vitae pellentesque sem placerat. "
                 "In id cursus mi pretium tellus sed duis convallis. "
@@ -53,8 +53,8 @@ async def test_correct_vectoriser():
 
     # create collection with vectoriser
     full_vectoriser = Configure.Vectorizer.text2vec_openai(
-        model="text-embedding-3-large",
-        dimensions=256,
+        model="text-embedding-3-small",
+        dimensions=512,
     )
     async with client_manager.connect_to_async_client() as client:
         collection_full = await client.collections.create(
@@ -64,8 +64,11 @@ async def test_correct_vectoriser():
         await collection_full.data.insert_many(data)
 
     # create collection with named vectoriser
-    named_vectoriser = Configure.NamedVectors.text2vec_jinaai(
-        name="content_vector", source_properties=["content"], model="jina-embeddings-v3"
+    named_vectoriser = Configure.NamedVectors.text2vec_openai(
+        name="content_vector",
+        source_properties=["random_content_field"],
+        model="text-embedding-3-small",
+        dimensions=512,
     )
     async with client_manager.connect_to_async_client() as client:
         collection_named = await client.collections.create(
@@ -79,36 +82,172 @@ async def test_correct_vectoriser():
     try:
         # do chunking on full collection
         collection_chunker = AsyncCollectionChunker(collection_name_full)
-        await collection_chunker.create_chunked_reference("content", client_manager)
+        await collection_chunker.create_chunked_reference(
+            "random_content_field", client_manager
+        )
 
         async with client_manager.connect_to_async_client() as client:
             # check existence of collection
-            assert await client.collections.exists(collection_name_full)
+            assert await client.collections.exists(
+                f"ELYSIA_CHUNKED_{collection_name_full.lower()}__"
+            )
 
             # check vectoriser
             collection_full_config = await client.collections.get(
-                collection_name_full
+                f"ELYSIA_CHUNKED_{collection_name_full.lower()}__"
             ).config.get()
+            assert "default" in collection_full_config.vector_config
             assert (
-                collection_full_config.vectorizer_config.vectorizer
+                collection_full_config.vector_config["default"].vectorizer.vectorizer
                 == full_vectoriser.vectorizer
             )
 
         # check named vectoriser
         collection_chunker = AsyncCollectionChunker(collection_name_named)
-        await collection_chunker.create_chunked_reference("content", client_manager)
+        await collection_chunker.create_chunked_reference(
+            "random_content_field", client_manager
+        )
 
         async with client_manager.connect_to_async_client() as client:
             # check existence of collection
-            assert await client.collections.exists(collection_name_named)
+            assert await client.collections.exists(
+                f"ELYSIA_CHUNKED_{collection_name_named.lower()}__"
+            )
 
             # check vectoriser
             collection_named_config = await client.collections.get(
-                collection_name_named
+                f"ELYSIA_CHUNKED_{collection_name_named.lower()}__"
+            ).config.get()
+            assert (
+                list(collection_named_config.vector_config.keys())[0]
+                in collection_named_config.vector_config
+            )
+            assert (
+                collection_named_config.vector_config[
+                    list(collection_named_config.vector_config.keys())[0]
+                ].vectorizer.vectorizer
+                == named_vectoriser.vectorizer.vectorizer
+            )
+
+    finally:
+        async with client_manager.connect_to_async_client() as client:
+            await client.collections.delete(collection_name_full)
+            await client.collections.delete(collection_name_named)
+            await client.collections.delete(
+                f"ELYSIA_CHUNKED_{collection_name_full.lower()}__"
+            )
+            await client.collections.delete(
+                f"ELYSIA_CHUNKED_{collection_name_named.lower()}__"
+            )
+
+        await client_manager.close_clients()
+
+
+@pytest.mark.asyncio
+async def test_correct_vectoriser_new():
+    collection_name_full = "Test_ELYSIA_collection_chunking_vectoriser_full_new"
+    collection_name_named = "Test_ELYSIA_collection_chunking_vectoriser_named_new"
+    client_manager = ClientManager()
+
+    # example data
+    data = [
+        {
+            "random_content_field": (
+                "Lorem ipsum dolor sit amet consectetur adipiscing elit. "
+                "Quisque faucibus ex sapien vitae pellentesque sem placerat. "
+                "In id cursus mi pretium tellus duis convallis. "
+                "Tempus leo eu aenean sed diam urna tempor. "
+                "Pulvinar vivamus fringilla lacus nec metus bibendum egestas. "
+                "Iaculis massa nisl malesuada lacinia integer nunc posuere. "
+                "Ut hendrerit semper vel class aptent taciti sociosqu. "
+                "Ad litora torquent per conubia nostra inceptos himenaeos."
+            ),
+            "other_field": "other_value",
+        },
+        {
+            "random_content_field": (
+                "Lorem ipsum dolor sit sit amet consectetur adipiscing elit. "
+                "Quisque faucibus ex sapien sed vitae pellentesque sem placerat. "
+                "In id cursus mi pretium tellus sed duis convallis. "
+                "Tempus leo eu aenean sed diam urna tempor. "
+                "Pulvinar vivamus fringilla la cus nec metus bibendum egestas. "
+                "Iaculis massa nisl malesuada   lacinia integer nunc posuere. "
+                "Ut hendrerit semper vel class ooga aptent taciti sociosqu. "
+                "Ad litora torquent per conubia booga nostra inceptos himenaeos."
+            ),
+            "other_field": "other_value_2",
+        },
+    ]
+
+    # create collection with vectoriser
+    full_vectoriser = Configure.Vectors.text2vec_openai(
+        model="text-embedding-3-large",
+        dimensions=512,
+    )
+    async with client_manager.connect_to_async_client() as client:
+        collection_full = await client.collections.create(
+            collection_name_full,
+            vector_config=full_vectoriser,
+        )
+        await collection_full.data.insert_many(data)
+
+    named_vectoriser = Configure.Vectors.text2vec_openai(
+        name="content_vector",
+        source_properties=["random_content_field"],
+        model="text-embedding-3-small",
+    )
+    # create collection with named vectoriser
+    async with client_manager.connect_to_async_client() as client:
+        collection_named = await client.collections.create(
+            collection_name_named,
+            vector_config=[
+                named_vectoriser,
+            ],
+        )
+        await collection_named.data.insert_many(data)
+
+    try:
+        # do chunking on full collection
+        collection_chunker = AsyncCollectionChunker(collection_name_full)
+        await collection_chunker.create_chunked_reference(
+            "random_content_field", client_manager
+        )
+
+        async with client_manager.connect_to_async_client() as client:
+            # check existence of collection
+            assert await client.collections.exists(
+                f"ELYSIA_CHUNKED_{collection_name_full.lower()}__"
+            )
+
+            # check vectoriser
+            collection_full_config = await client.collections.get(
+                f"ELYSIA_CHUNKED_{collection_name_full.lower()}__"
+            ).config.get()
+            assert "default" in collection_full_config.vector_config
+            assert (
+                collection_full_config.vector_config["default"].vectorizer.vectorizer
+                == full_vectoriser.vectorizer.vectorizer
+            )
+
+        # check named vectoriser
+        collection_chunker = AsyncCollectionChunker(collection_name_named)
+        await collection_chunker.create_chunked_reference(
+            "random_content_field", client_manager
+        )
+
+        async with client_manager.connect_to_async_client() as client:
+            # check existence of collection
+            assert await client.collections.exists(
+                f"ELYSIA_CHUNKED_{collection_name_named.lower()}__"
+            )
+
+            # check vectoriser
+            collection_named_config = await client.collections.get(
+                f"ELYSIA_CHUNKED_{collection_name_named.lower()}__"
             ).config.get()
             assert collection_named_config.vector_config[
                 list(collection_named_config.vector_config.keys())[0]
-            ].vectorizer.source_properties == ["content"]
+            ].vectorizer.source_properties == ["random_content_field"]
             assert (
                 collection_named_config.vector_config[
                     list(collection_named_config.vector_config.keys())[0]
@@ -120,5 +259,11 @@ async def test_correct_vectoriser():
         async with client_manager.connect_to_async_client() as client:
             await client.collections.delete(collection_name_full)
             await client.collections.delete(collection_name_named)
+            await client.collections.delete(
+                f"ELYSIA_CHUNKED_{collection_name_full.lower()}__"
+            )
+            await client.collections.delete(
+                f"ELYSIA_CHUNKED_{collection_name_named.lower()}__"
+            )
 
         await client_manager.close_clients()