diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index b2747769..25bc8d22 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -170,7 +170,7 @@ jobs: if [[ "${{ matrix.python-version }}" > "3.9" ]]; then make test-notebooks else - poetry run test-notebooks --ignore ./docs/user_guide/09_threshold_optimization.ipynb + poetry run test-notebooks --ignore ./docs/user_guide/09_threshold_optimization.ipynb --ignore ./docs/user_guide/release_guide/0_5_0_release.ipynb fi docs: diff --git a/docs/api/index.md b/docs/api/index.md index 598d9dcd..e3dc486e 100644 --- a/docs/api/index.md +++ b/docs/api/index.md @@ -22,5 +22,6 @@ reranker cache session_manager router +threshold_optimizer ``` diff --git a/docs/api/query.rst b/docs/api/query.rst index 8086f5ca..fa92230e 100644 --- a/docs/api/query.rst +++ b/docs/api/query.rst @@ -34,6 +34,34 @@ VectorRangeQuery :show-inheritance: :exclude-members: add_filter,get_args,highlight,return_field,summarize +HybridQuery +================ + + +.. currentmodule:: redisvl.query + + +.. autoclass:: HybridQuery + :members: + :inherited-members: + :show-inheritance: + :exclude-members: add_filter,get_args,highlight,return_field,summarize + + +TextQuery +================ + + +.. currentmodule:: redisvl.query + + +.. autoclass:: TextQuery + :members: + :inherited-members: + :show-inheritance: + :exclude-members: add_filter,get_args,highlight,return_field,summarize + + FilterQuery =========== diff --git a/docs/api/session_manager.rst b/docs/api/session_manager.rst index 9b885a35..86289204 100644 --- a/docs/api/session_manager.rst +++ b/docs/api/session_manager.rst @@ -2,7 +2,6 @@ LLM Session Manager ******************* - SemanticSessionManager ====================== diff --git a/docs/api/threshold_optimizer.rst b/docs/api/threshold_optimizer.rst new file mode 100644 index 00000000..dc226cef --- /dev/null +++ b/docs/api/threshold_optimizer.rst @@ -0,0 +1,26 @@ +******************** +Threshold Optimizers +******************** + +CacheThresholdOptimizer +======================= + +.. _cachethresholdoptimizer_api: + +.. currentmodule:: redisvl.utils.optimize.cache + +.. autoclass:: CacheThresholdOptimizer + :show-inheritance: + :members: + + +RouterThresholdOptimizer +======================== + +.. _routerthresholdoptimizer_api: + +.. currentmodule:: redisvl.utils.optimize.router + +.. autoclass:: RouterThresholdOptimizer + :show-inheritance: + :members: diff --git a/docs/user_guide/index.md b/docs/user_guide/index.md index b6592e3d..30a51b8a 100644 --- a/docs/user_guide/index.md +++ b/docs/user_guide/index.md @@ -2,7 +2,7 @@ myst: html_meta: "description lang=en": | - User Guides for RedisVL + User guides for RedisVL --- # User Guides @@ -20,4 +20,6 @@ User guides provide helpful resources for using RedisVL and its different compon 06_rerankers 07_session_manager 08_semantic_router +09_threshold_optimization +release_guide/index ``` diff --git a/docs/user_guide/release_guide/0_5_0_release.ipynb b/docs/user_guide/release_guide/0_5_0_release.ipynb new file mode 100644 index 00000000..ca539f34 --- /dev/null +++ b/docs/user_guide/release_guide/0_5_0_release.ipynb @@ -0,0 +1,715 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# 0.5.0 Feature Overview\n", + "\n", + "This notebook provides an overview of what's new with the 0.5.0 release of redisvl. It also highlights changes and potential enhancements for existing usage.\n", + "\n", + "## What's new?\n", + "\n", + "- Hybrid query and text query classes\n", + "- Threshold optimizer classes\n", + "- Schema validation\n", + "- Timestamp filters\n", + "- Batched queries\n", + "- Vector normalization\n", + "- Hybrid policy on knn with filters\n", + "\n", + "## Define and load index for examples" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\u001b[32m12:44:52\u001b[0m \u001b[34mredisvl.index.index\u001b[0m \u001b[1;30mINFO\u001b[0m Index already exists, overwriting.\n" + ] + }, + { + "data": { + "text/plain": [ + "['jobs:01JR0V1SA29RVD9AAVSTBV9P5H',\n", + " 'jobs:01JR0V1SA209KMVHMD7G54P3H5',\n", + " 'jobs:01JR0V1SA23ZE7BRERXTZWC33Z']" + ] + }, + "execution_count": 9, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "from redisvl.utils.vectorize import HFTextVectorizer\n", + "from redisvl.index import SearchIndex\n", + "import datetime as dt\n", + "\n", + "import warnings\n", + "warnings.filterwarnings(\"ignore\", category=UserWarning, module=\"redis\")\n", + "\n", + "# Embedding model\n", + "emb_model = HFTextVectorizer()\n", + "\n", + "REDIS_URL = \"redis://localhost:6379/0\"\n", + "NOW = dt.datetime.now()\n", + "\n", + "job_data = [\n", + " {\n", + " \"job_title\": \"Software Engineer\",\n", + " \"job_description\": \"Develop and maintain web applications using JavaScript, React, and Node.js.\",\n", + " \"posted\": (NOW - dt.timedelta(days=1)).timestamp() # day ago\n", + " },\n", + " {\n", + " \"job_title\": \"Data Analyst\",\n", + " \"job_description\": \"Analyze large datasets to provide business insights and create data visualizations.\",\n", + " \"posted\": (NOW - dt.timedelta(days=7)).timestamp() # week ago\n", + " },\n", + " {\n", + " \"job_title\": \"Marketing Manager\",\n", + " \"job_description\": \"Develop and implement marketing strategies to drive brand awareness and customer engagement.\",\n", + " \"posted\": (NOW - dt.timedelta(days=30)).timestamp() # month ago\n", + " }\n", + "]\n", + "\n", + "job_data = [{**job, \"job_embedding\": emb_model.embed(job[\"job_description\"], as_buffer=True)} for job in job_data]\n", + "\n", + "\n", + "job_schema = {\n", + " \"index\": {\n", + " \"name\": \"jobs\",\n", + " \"prefix\": \"jobs\",\n", + " \"storage_type\": \"hash\",\n", + " },\n", + " \"fields\": [\n", + " {\"name\": \"job_title\", \"type\": \"text\"},\n", + " {\"name\": \"job_description\", \"type\": \"text\"},\n", + " {\"name\": \"posted\", \"type\": \"numeric\"},\n", + " {\n", + " \"name\": \"job_embedding\",\n", + " \"type\": \"vector\",\n", + " \"attrs\": {\n", + " \"dims\": 768,\n", + " \"distance_metric\": \"cosine\",\n", + " \"algorithm\": \"flat\",\n", + " \"datatype\": \"float32\"\n", + " }\n", + "\n", + " }\n", + " ],\n", + "}\n", + "\n", + "index = SearchIndex.from_dict(job_schema, redis_url=REDIS_URL)\n", + "index.create(overwrite=True, drop=True)\n", + "index.load(job_data)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# HybridQuery class\n", + "\n", + "Perform hybrid lexical (BM25) and vector search where results are ranked by: `hybrid_score = (1-alpha)*lexical_Score + alpha*vector_similarity`." + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "[{'vector_distance': '0.61871612072',\n", + " 'job_title': 'Software Engineer',\n", + " 'vector_similarity': '0.69064193964',\n", + " 'text_score': '49.6242910712',\n", + " 'hybrid_score': '15.3707366791'},\n", + " {'vector_distance': '0.937997639179',\n", + " 'job_title': 'Marketing Manager',\n", + " 'vector_similarity': '0.53100118041',\n", + " 'text_score': '49.6242910712',\n", + " 'hybrid_score': '15.2589881476'},\n", + " {'vector_distance': '0.859166145325',\n", + " 'job_title': 'Data Analyst',\n", + " 'vector_similarity': '0.570416927338',\n", + " 'text_score': '0',\n", + " 'hybrid_score': '0.399291849136'}]" + ] + }, + "execution_count": 16, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "from redisvl.query import HybridQuery\n", + "\n", + "text = \"Find a job as a where you develop software\"\n", + "vec = emb_model.embed(text, as_buffer=True)\n", + "\n", + "query = HybridQuery(\n", + " text=text,\n", + " text_field_name=\"job_description\",\n", + " vector=vec,\n", + " vector_field_name=\"job_embedding\",\n", + " alpha=0.7,\n", + " num_results=10,\n", + " return_fields=[\"job_title\"],\n", + ")\n", + "\n", + "results = index.query(query)\n", + "results" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# TextQueries\n", + "\n", + "TextQueries make it easy to perform pure lexical search with redisvl." + ] + }, + { + "cell_type": "code", + "execution_count": 17, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "[{'id': 'jobs:01JR0V1SA29RVD9AAVSTBV9P5H',\n", + " 'score': 49.62429107116745,\n", + " 'job_title': 'Software Engineer'},\n", + " {'id': 'jobs:01JR0V1SA23ZE7BRERXTZWC33Z',\n", + " 'score': 49.62429107116745,\n", + " 'job_title': 'Marketing Manager'}]" + ] + }, + "execution_count": 17, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "from redisvl.query import TextQuery\n", + "\n", + "text = \"Find where you develop software\"\n", + "\n", + "query = TextQuery(\n", + " text=text,\n", + " text_field_name=\"job_description\",\n", + " return_fields=[\"job_title\"],\n", + " num_results=10,\n", + ")\n", + "\n", + "results = index.query(query)\n", + "results" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Threshold optimization\n", + "\n", + "In redis 0.5.0 we added the ability to quickly configure either your semantic cache or semantic router with test data examples.\n", + "\n", + "For a step by step guide see: [09_threshold_optimization.ipynb](../09_threshold_optimization.ipynb).\n", + "\n", + "For a more advanced routing example see: [this example](https://github.com/redis-developer/redis-ai-resources/blob/main/python-recipes/semantic-router/01_routing_optimization.ipynb). " + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "Distance threshold before: 0.5 \n", + "\n", + "\n", + "Distance threshold after: 0.13050847457627118 \n", + "\n" + ] + } + ], + "source": [ + "from redisvl.utils.optimize import CacheThresholdOptimizer\n", + "from redisvl.extensions.llmcache import SemanticCache\n", + "\n", + "sem_cache = SemanticCache(\n", + " name=\"sem_cache\", # underlying search index name\n", + " redis_url=\"redis://localhost:6379\", # redis connection url string\n", + " distance_threshold=0.5 # semantic cache distance threshold\n", + ")\n", + "\n", + "paris_key = sem_cache.store(prompt=\"what is the capital of france?\", response=\"paris\")\n", + "rabat_key = sem_cache.store(prompt=\"what is the capital of morocco?\", response=\"rabat\")\n", + "\n", + "test_data = [\n", + " {\n", + " \"query\": \"What's the capital of Britain?\",\n", + " \"query_match\": \"\"\n", + " },\n", + " {\n", + " \"query\": \"What's the capital of France??\",\n", + " \"query_match\": paris_key\n", + " },\n", + " {\n", + " \"query\": \"What's the capital city of Morocco?\",\n", + " \"query_match\": rabat_key\n", + " },\n", + "]\n", + "\n", + "print(f\"\\nDistance threshold before: {sem_cache.distance_threshold} \\n\")\n", + "optimizer = CacheThresholdOptimizer(sem_cache, test_data)\n", + "optimizer.optimize()\n", + "print(f\"\\nDistance threshold after: {sem_cache.distance_threshold} \\n\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Schema validation\n", + "\n", + "This feature makes it easier to make sure your data is in the right format. To demo this we will create a new index with the `validate_on_load` flag set to `True`" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\u001b[32m16:20:25\u001b[0m \u001b[34mredisvl.index.index\u001b[0m \u001b[1;30mERROR\u001b[0m \u001b[31mSchema validation error while loading data\u001b[0m\n", + "Traceback (most recent call last):\n", + " File \"/Users/robert.shelton/.pyenv/versions/3.11.9/lib/python3.11/site-packages/redisvl/index/storage.py\", line 204, in _preprocess_and_validate_objects\n", + " processed_obj = self._validate(processed_obj)\n", + " ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n", + " File \"/Users/robert.shelton/.pyenv/versions/3.11.9/lib/python3.11/site-packages/redisvl/index/storage.py\", line 160, in _validate\n", + " return validate_object(self.index_schema, obj)\n", + " ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n", + " File \"/Users/robert.shelton/.pyenv/versions/3.11.9/lib/python3.11/site-packages/redisvl/schema/validation.py\", line 276, in validate_object\n", + " validated = model_class.model_validate(flat_obj)\n", + " ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n", + " File \"/Users/robert.shelton/.pyenv/versions/3.11.9/lib/python3.11/site-packages/pydantic/main.py\", line 627, in model_validate\n", + " return cls.__pydantic_validator__.validate_python(\n", + " ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n", + "pydantic_core._pydantic_core.ValidationError: 2 validation errors for cars__PydanticModel\n", + "mpg.int\n", + " Input should be a valid integer, unable to parse string as an integer [type=int_parsing, input_value='twenty-two', input_type=str]\n", + " For further information visit https://errors.pydantic.dev/2.10/v/int_parsing\n", + "mpg.float\n", + " Input should be a valid number, unable to parse string as a number [type=float_parsing, input_value='twenty-two', input_type=str]\n", + " For further information visit https://errors.pydantic.dev/2.10/v/float_parsing\n", + "\n", + "The above exception was the direct cause of the following exception:\n", + "\n", + "Traceback (most recent call last):\n", + " File \"/Users/robert.shelton/.pyenv/versions/3.11.9/lib/python3.11/site-packages/redisvl/index/index.py\", line 615, in load\n", + " return self._storage.write(\n", + " ^^^^^^^^^^^^^^^^^^^^\n", + " File \"/Users/robert.shelton/.pyenv/versions/3.11.9/lib/python3.11/site-packages/redisvl/index/storage.py\", line 265, in write\n", + " prepared_objects = self._preprocess_and_validate_objects(\n", + " ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n", + " File \"/Users/robert.shelton/.pyenv/versions/3.11.9/lib/python3.11/site-packages/redisvl/index/storage.py\", line 211, in _preprocess_and_validate_objects\n", + " raise SchemaValidationError(str(e), index=i) from e\n", + "redisvl.exceptions.SchemaValidationError: Validation failed for object at index 1: 2 validation errors for cars__PydanticModel\n", + "mpg.int\n", + " Input should be a valid integer, unable to parse string as an integer [type=int_parsing, input_value='twenty-two', input_type=str]\n", + " For further information visit https://errors.pydantic.dev/2.10/v/int_parsing\n", + "mpg.float\n", + " Input should be a valid number, unable to parse string as a number [type=float_parsing, input_value='twenty-two', input_type=str]\n", + " For further information visit https://errors.pydantic.dev/2.10/v/float_parsing\n", + "Error loading data: Validation failed for object at index 1: 2 validation errors for cars__PydanticModel\n", + "mpg.int\n", + " Input should be a valid integer, unable to parse string as an integer [type=int_parsing, input_value='twenty-two', input_type=str]\n", + " For further information visit https://errors.pydantic.dev/2.10/v/int_parsing\n", + "mpg.float\n", + " Input should be a valid number, unable to parse string as a number [type=float_parsing, input_value='twenty-two', input_type=str]\n", + " For further information visit https://errors.pydantic.dev/2.10/v/float_parsing\n" + ] + } + ], + "source": [ + "# NBVAL_SKIP\n", + "from redisvl.index import SearchIndex\n", + "\n", + "# sample schema\n", + "car_schema = {\n", + " \"index\": {\n", + " \"name\": \"cars\",\n", + " \"prefix\": \"cars\",\n", + " \"storage_type\": \"json\",\n", + " },\n", + " \"fields\": [\n", + " {\"name\": \"make\", \"type\": \"text\"},\n", + " {\"name\": \"model\", \"type\": \"text\"},\n", + " {\"name\": \"description\", \"type\": \"text\"},\n", + " {\"name\": \"mpg\", \"type\": \"numeric\"},\n", + " {\n", + " \"name\": \"car_embedding\",\n", + " \"type\": \"vector\",\n", + " \"attrs\": {\n", + " \"dims\": 3,\n", + " \"distance_metric\": \"cosine\",\n", + " \"algorithm\": \"flat\",\n", + " \"datatype\": \"float32\"\n", + " }\n", + "\n", + " }\n", + " ],\n", + "}\n", + "\n", + "sample_data_bad = [\n", + " {\n", + " \"make\": \"Toyota\",\n", + " \"model\": \"Camry\",\n", + " \"description\": \"A reliable sedan with great fuel economy.\",\n", + " \"mpg\": 28,\n", + " \"car_embedding\": [0.1, 0.2, 0.3]\n", + " },\n", + " {\n", + " \"make\": \"Honda\",\n", + " \"model\": \"CR-V\",\n", + " \"description\": \"A practical SUV with advanced technology.\",\n", + " # incorrect type will throw an error\n", + " \"mpg\": \"twenty-two\",\n", + " \"car_embedding\": [0.4, 0.5, 0.6]\n", + " }\n", + "]\n", + "\n", + "# this should now throw an error\n", + "car_index = SearchIndex.from_dict(car_schema, redis_url=REDIS_URL, validate_on_load=True)\n", + "car_index.create(overwrite=True)\n", + "\n", + "try:\n", + " car_index.load(sample_data_bad)\n", + "except Exception as e:\n", + " print(f\"Error loading data: {e}\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Timestamp filters\n", + "\n", + "In Redis datetime objects are stored as numeric epoch times. Timestamp filter makes it easier to handle querying by these fields by handling conversion for you." + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "[{'id': 'jobs:01JQYMYZBA6NM6DX9YW35MCHJZ',\n", + " 'job_title': 'Software Engineer',\n", + " 'job_description': 'Develop and maintain web applications using JavaScript, React, and Node.js.',\n", + " 'posted': '1743625199.9'},\n", + " {'id': 'jobs:01JQYMYZBABXYR96H96SQ99ZPS',\n", + " 'job_title': 'Data Analyst',\n", + " 'job_description': 'Analyze large datasets to provide business insights and create data visualizations.',\n", + " 'posted': '1743106799.9'},\n", + " {'id': 'jobs:01JQYMYZBAGEBDS270EZADQ1TM',\n", + " 'job_title': 'Marketing Manager',\n", + " 'job_description': 'Develop and implement marketing strategies to drive brand awareness and customer engagement.',\n", + " 'posted': '1741123199.9'}]" + ] + }, + "execution_count": 10, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "from redisvl.query import FilterQuery\n", + "from redisvl.query.filter import Timestamp\n", + "\n", + "# find all jobs\n", + "ts = Timestamp(\"posted\") < NOW # now datetime created above\n", + "\n", + "filter_query = FilterQuery(\n", + " return_fields=[\"job_title\", \"job_description\", \"posted\"], \n", + " filter_expression=ts,\n", + " num_results=10,\n", + ")\n", + "res = index.query(filter_query)\n", + "res" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "[{'id': 'jobs:01JQYMYZBA6NM6DX9YW35MCHJZ',\n", + " 'job_title': 'Software Engineer',\n", + " 'job_description': 'Develop and maintain web applications using JavaScript, React, and Node.js.',\n", + " 'posted': '1743625199.9'}]" + ] + }, + "execution_count": 11, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# jobs posted in the last 3 days => 1 job\n", + "ts = Timestamp(\"posted\") > NOW - dt.timedelta(days=3)\n", + "\n", + "filter_query = FilterQuery(\n", + " return_fields=[\"job_title\", \"job_description\", \"posted\"], \n", + " filter_expression=ts,\n", + " num_results=10,\n", + ")\n", + "res = index.query(filter_query)\n", + "res" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "[{'id': 'jobs:01JQYMYZBABXYR96H96SQ99ZPS',\n", + " 'job_title': 'Data Analyst',\n", + " 'job_description': 'Analyze large datasets to provide business insights and create data visualizations.',\n", + " 'posted': '1743106799.9'}]" + ] + }, + "execution_count": 12, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# more than 3 days ago but less than 14 days ago => 1 job\n", + "ts = Timestamp(\"posted\").between(\n", + " NOW - dt.timedelta(days=14),\n", + " NOW - dt.timedelta(days=3),\n", + ")\n", + "\n", + "filter_query = FilterQuery(\n", + " return_fields=[\"job_title\", \"job_description\", \"posted\"], \n", + " filter_expression=ts,\n", + " num_results=10,\n", + ")\n", + "\n", + "res = index.query(filter_query)\n", + "res" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Batch search\n", + "\n", + "This enhancement allows you to speed up the execution of queries by reducing the impact of network latency." + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Time taken for 200 queries: 0.11 seconds\n" + ] + } + ], + "source": [ + "import time\n", + "num_queries = 200\n", + "\n", + "start = time.time()\n", + "for i in range(num_queries):\n", + " # run the same filter query \n", + " res = index.query(filter_query)\n", + "end = time.time()\n", + "print(f\"Time taken for {num_queries} queries: {end - start:.2f} seconds\")" + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Time taken for 200 batched queries: 0.03 seconds\n" + ] + } + ], + "source": [ + "batched_queries = [filter_query] * num_queries\n", + "\n", + "start = time.time()\n", + "\n", + "index.batch_search(batched_queries, batch_size=10)\n", + "\n", + "end = time.time()\n", + "print(f\"Time taken for {num_queries} batched queries: {end - start:.2f} seconds\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Vector normalization\n", + "\n", + "By default, Redis returns the vector cosine distance when performing a search, which yields a value between 0 and 2, where 0 represents a perfect match. However, you may sometimes prefer a similarity score between 0 and 1, where 1 indicates a perfect match. When enabled, this flag performs the conversion for you. Additionally, if this flag is set to true for L2 distance, it normalizes the Euclidean distance to a value between 0 and 1 as well.\n", + " " + ] + }, + { + "cell_type": "code", + "execution_count": 17, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "[{'id': 'jobs:01JQYMYZBA6NM6DX9YW35MCHJZ',\n", + " 'vector_distance': '0.7090711295605',\n", + " 'job_title': 'Software Engineer',\n", + " 'job_description': 'Develop and maintain web applications using JavaScript, React, and Node.js.',\n", + " 'posted': '1743625199.9'},\n", + " {'id': 'jobs:01JQYMYZBABXYR96H96SQ99ZPS',\n", + " 'vector_distance': '0.6049451231955',\n", + " 'job_title': 'Data Analyst',\n", + " 'job_description': 'Analyze large datasets to provide business insights and create data visualizations.',\n", + " 'posted': '1743106799.9'},\n", + " {'id': 'jobs:01JQYMYZBAGEBDS270EZADQ1TM',\n", + " 'vector_distance': '0.553376108408',\n", + " 'job_title': 'Marketing Manager',\n", + " 'job_description': 'Develop and implement marketing strategies to drive brand awareness and customer engagement.',\n", + " 'posted': '1741123199.9'}]" + ] + }, + "execution_count": 17, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "from redisvl.query import VectorQuery\n", + "\n", + "query = VectorQuery(\n", + " vector=emb_model.embed(\"Software Engineer\", as_buffer=True),\n", + " vector_field_name=\"job_embedding\",\n", + " return_fields=[\"job_title\", \"job_description\", \"posted\"],\n", + " normalize_vector_distance=True,\n", + ")\n", + "\n", + "res = index.query(query)\n", + "res" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Hybrid policy on knn with filters\n", + "\n", + "Within the default redis client you can set the `HYBRID_POLICY` which specifies the filter mode to use during vector search with filters. It can take values `BATCHES` or `ADHOC_BF`. Previously this option was not exposed by redisvl." + ] + }, + { + "cell_type": "code", + "execution_count": 18, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "[{'id': 'jobs:01JQYMYZBA6NM6DX9YW35MCHJZ',\n", + " 'vector_distance': '0.581857740879',\n", + " 'job_title': 'Software Engineer',\n", + " 'job_description': 'Develop and maintain web applications using JavaScript, React, and Node.js.',\n", + " 'posted': '1743625199.9'},\n", + " {'id': 'jobs:01JQYMYZBAGEBDS270EZADQ1TM',\n", + " 'vector_distance': '0.893247783184',\n", + " 'job_title': 'Marketing Manager',\n", + " 'job_description': 'Develop and implement marketing strategies to drive brand awareness and customer engagement.',\n", + " 'posted': '1741123199.9'}]" + ] + }, + "execution_count": 18, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "from redisvl.query.filter import Text\n", + "\n", + "filter = Text(\"job_description\") % \"Develop\"\n", + "\n", + "query = VectorQuery(\n", + " vector=emb_model.embed(\"Software Engineer\", as_buffer=True),\n", + " vector_field_name=\"job_embedding\",\n", + " return_fields=[\"job_title\", \"job_description\", \"posted\"],\n", + " hybrid_policy=\"BATCHES\"\n", + ")\n", + "\n", + "query.set_filter(filter)\n", + "\n", + "res = index.query(query)\n", + "res" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "redisvl-56gG2io_-py3.11", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.9" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/docs/user_guide/release_guide/index.md b/docs/user_guide/release_guide/index.md new file mode 100644 index 00000000..1eba08c9 --- /dev/null +++ b/docs/user_guide/release_guide/index.md @@ -0,0 +1,17 @@ +--- +myst: + html_meta: + "description lang=en": | + Release guides for RedisVL +--- + +# Release Guides + +This section contains guidelines and information for RedisVL releases. + +```{toctree} +:caption: Release Guides +:maxdepth: 2 + +0_5_0_release +``` diff --git a/pyproject.toml b/pyproject.toml index eed0e0dd..4e3934c2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "redisvl" -version = "0.4.1" +version = "0.5.0" description = "Python client library and CLI for using Redis as a vector database" authors = ["Redis Inc. "] license = "MIT" diff --git a/redisvl/query/aggregate.py b/redisvl/query/aggregate.py index 1e5526d1..4e6f4085 100644 --- a/redisvl/query/aggregate.py +++ b/redisvl/query/aggregate.py @@ -23,6 +23,31 @@ class HybridQuery(AggregationQuery): HybridQuery combines text and vector search in Redis. It allows you to perform a hybrid search using both text and vector similarity. It scores documents based on a weighted combination of text and vector similarity. + + .. code-block:: python + + from redisvl.query import HybridQuery + from redisvl.index import SearchIndex + + index = SearchIndex.from_yaml("path/to/index.yaml") + + query = HybridQuery( + text="example text", + text_field_name="text_field", + vector=[0.1, 0.2, 0.3], + vector_field_name="vector_field", + text_scorer="BM25STD", + filter_expression=None, + alpha=0.7, + dtype="float32", + num_results=10, + return_fields=["field1", "field2"], + stopwords="english", + dialect=2, + ) + + results = index.query(query) + """ DISTANCE_ID: str = "vector_distance" @@ -72,29 +97,6 @@ def __init__( ValueError: If the text string is empty, or if the text string becomes empty after stopwords are removed. TypeError: If the stopwords are not a set, list, or tuple of strings. - - .. code-block:: python - from redisvl.query import HybridQuery - from redisvl.index import SearchIndex - - index = SearchIndex.from_yaml(index.yaml) - - query = HybridQuery( - text="example text", - text_field_name="text_field", - vector=[0.1, 0.2, 0.3], - vector_field_name="vector_field", - text_scorer="BM25STD", - filter_expression=None, - alpha=0.7, - dtype="float32", - num_results=10, - return_fields=["field1", "field2"], - stopwords="english", - dialect=2, - ) - - results = index.query(query) """ if not text.strip(): diff --git a/redisvl/query/query.py b/redisvl/query/query.py index cc4d26f0..ea09bc19 100644 --- a/redisvl/query/query.py +++ b/redisvl/query/query.py @@ -690,6 +690,30 @@ class RangeQuery(VectorRangeQuery): class TextQuery(BaseQuery): + """ + TextQuery is a query for running a full text search, along with an optional filter expression. + + .. code-block:: python + + from redisvl.query import TextQuery + from redisvl.index import SearchIndex + + index = SearchIndex.from_yaml(index.yaml) + + query = TextQuery( + text="example text", + text_field_name="text_field", + text_scorer="BM25STD", + filter_expression=None, + num_results=10, + return_fields=["field1", "field2"], + stopwords="english", + dialect=2, + ) + + results = index.query(query) + """ + def __init__( self, text: str, @@ -739,25 +763,6 @@ def __init__( Raises: ValueError: if stopwords language string cannot be loaded. TypeError: If stopwords is not a valid iterable set of strings. - - .. code-block:: python - from redisvl.query import TextQuery - from redisvl.index import SearchIndex - - index = SearchIndex.from_yaml(index.yaml) - - query = TextQuery( - text="example text", - text_field_name="text_field", - text_scorer="BM25STD", - filter_expression=None, - num_results=10, - return_fields=["field1", "field2"], - stopwords="english", - dialect=2, - ) - - results = index.query(query) """ self._text = text self._text_field = text_field_name diff --git a/redisvl/utils/optimize/cache.py b/redisvl/utils/optimize/cache.py index f88c53ff..05c99f76 100644 --- a/redisvl/utils/optimize/cache.py +++ b/redisvl/utils/optimize/cache.py @@ -71,13 +71,62 @@ def _grid_search_opt_cache( class CacheThresholdOptimizer(BaseThresholdOptimizer): + """ + Class for optimizing thresholds for a SemanticCache. + + .. code-block:: python + + from redisvl.extensions.llmcache import SemanticCache + from redisvl.utils.optimize import CacheThresholdOptimizer + + sem_cache = SemanticCache( + name="sem_cache", # underlying search index name + redis_url="redis://localhost:6379", # redis connection url string + distance_threshold=0.5 # semantic cache distance threshold + ) + + paris_key = sem_cache.store(prompt="what is the capital of france?", response="paris") + rabat_key = sem_cache.store(prompt="what is the capital of morocco?", response="rabat") + + test_data = [ + { + "query": "What's the capital of Britain?", + "query_match": "" + }, + { + "query": "What's the capital of France??", + "query_match": paris_key + }, + { + "query": "What's the capital city of Morocco?", + "query_match": rabat_key + }, + ] + + optimizer = CacheThresholdOptimizer(sem_cache, test_data) + optimizer.optimize() + """ + def __init__( self, cache: SemanticCache, - test_dict: List[Dict], + test_dict: List[Dict[str, Any]], opt_fn: Callable = _grid_search_opt_cache, eval_metric: str = "f1", ): + """Initialize the cache optimizer. + + Args: + cache (SemanticCache): The RedisVL SemanticCache instance to optimize. + test_dict (List[Dict[str, Any]]): List of test cases. + opt_fn (Callable): Function to perform optimization. Defaults to + grid search. + eval_metric (str): Evaluation metric for threshold optimization. + Defaults to "f1" score. + + Raises: + ValueError: If the test_dict not in LabeledData format. + """ super().__init__(cache, test_dict, opt_fn, eval_metric) def optimize(self, **kwargs: Any): diff --git a/redisvl/utils/optimize/router.py b/redisvl/utils/optimize/router.py index 40a7ea7d..8105dddd 100644 --- a/redisvl/utils/optimize/router.py +++ b/redisvl/utils/optimize/router.py @@ -90,16 +90,70 @@ def _random_search_opt_router( class RouterThresholdOptimizer(BaseThresholdOptimizer): + """ + Class for optimizing thresholds for a SemanticRouter. + + .. code-block:: python + + from redisvl.extensions.router import Route, SemanticRouter + from redisvl.utils.vectorize import HFTextVectorizer + from redisvl.utils.optimize import RouterThresholdOptimizer + + routes = [ + Route( + name="greeting", + references=["hello", "hi"], + metadata={"type": "greeting"}, + distance_threshold=0.5, + ), + Route( + name="farewell", + references=["bye", "goodbye"], + metadata={"type": "farewell"}, + distance_threshold=0.5, + ), + ] + + router = SemanticRouter( + name="greeting-router", + vectorizer=HFTextVectorizer(), + routes=routes, + redis_url="redis://localhost:6379", + overwrite=True # Blow away any other routing index with this name + ) + + test_data = [ + {"query": "hello", "query_match": "greeting"}, + {"query": "goodbye", "query_match": "farewell"}, + ... + ] + + optimizer = RouterThresholdOptimizer(router, test_data) + optimizer.optimize() + """ + def __init__( self, router: SemanticRouter, - test_dict: List[Dict], + test_dict: List[Dict[str, Any]], opt_fn: Callable = _random_search_opt_router, eval_metric: str = "f1", ): + """Initialize the router optimizer. + + Args: + router (SemanticRouter): The RedisVL SemanticRouter instance to optimize. + test_dict (List[Dict[str, Any]]): List of test cases. + opt_fn (Callable): Function to perform optimization. Defaults to + grid search. + eval_metric (str): Evaluation metric for threshold optimization. + Defaults to "f1" score. + Raises: + ValueError: If the test_dict not in LabeledData format. + """ super().__init__(router, test_dict, opt_fn, eval_metric) def optimize(self, **kwargs: Any): - """Optimize thresholds using the provided optimization function for router case.""" + """Optimize kicks of the optimization process for router""" qrels = _format_qrels(self.test_data) self.opt_fn(self.optimizable, self.test_data, qrels, self.eval_metric, **kwargs) diff --git a/redisvl/version.py b/redisvl/version.py index 3d26edf7..3d187266 100644 --- a/redisvl/version.py +++ b/redisvl/version.py @@ -1 +1 @@ -__version__ = "0.4.1" +__version__ = "0.5.0"