Merge pull request #3002 from liyun95/v2.5.x

deprecate SPARSE_WAND and drop_ratio_build params for sparse embeddings
milvus-io · Jan 24, 2025 · b59ecca · b59ecca
2 parents 1e33421 + ad033d1
commit b59ecca
Show file tree

Hide file tree

Showing 6 changed files with 44 additions and 63 deletions.
diff --git a/site/en/about/limitations.md b/site/en/about/limitations.md
@@ -70,16 +70,16 @@ In current release, data to be load must be under 90% of the total memory resour
 
 The following table provides an overview of the support for various search behaviors across different index types.
 
-|                                      | HNSW | DISKANN | FLAT | IVF_FLAT | IVF_SQ8 | IVF_PQ | SCANN | GPU_IFV_FLAT | GPU_IVF_PQ | GPU_CAGRA | GPU_BRUTE_FORCE | SPARSE_INVERTED_INDEX | SPARSE_WAND         | BIN_FLAT | BIN_IVF_FLAT |
-|--------------------------------------|------|---------|------|----------|---------|--------|-------|--------------|------------|-----------|-----------------|-----------------------|---------------------|----------|--------------|
-| Basic search                         | Yes  | Yes     | Yes  | Yes      | Yes     | Yes    | Yes   | Yes          | Yes        | Yes       | Yes             | Yes                   | Yes                 | Yes      | Yes          |
-| Partition search                     | Yes  | Yes     | Yes  | Yes      | Yes     | Yes    | Yes   | Yes          | Yes        | Yes       | Yes             | Yes                   | Yes                 | Yes      | Yes          |
-| Basic search with raw data retrieved | Yes  | Yes     | Yes  | Yes      | Yes     | Yes    | Yes   | Yes          | Yes        | Yes       | Yes             | Yes                   | Yes                 | Yes      | Yes          |
-| Basic search with pagination         | Yes  | Yes     | Yes  | Yes      | Yes     | Yes    | Yes   | Yes          | Yes        | Yes       | Yes             | Yes                   | Yes                 | Yes      | Yes          |
-| Filtered search                      | Yes  | Yes     | Yes  | Yes      | Yes     | Yes    | Yes   | Yes          | Yes        | Yes       | Yes             | Yes                   | Yes                 | Yes      | Yes          |
-| Range search                         | Yes  | Yes     | Yes  | Yes      | Yes     | Yes    | Yes   | No           | No         | No        | No              | No                    | No                  | Yes      | Yes          |
-| Grouping search                      | Yes  | No      | Yes  | Yes      | No      | No     | No    | No           | No         | No        | No              | No                    | No                  | No       | No           |
-| Search with iterator                 | Yes  | Yes     | Yes  | Yes      | Yes     | Yes    | Yes   | No           | No         | No        | No              | No                    | No                  | No       | No           |
-| Hybrid search                        | Yes  | Yes     | Yes  | Yes      | Yes     | Yes    | Yes   | Yes          | Yes        | Yes       | Yes             | Yes(Only RRFRanker)   | Yes(Only RRFRanker) | Yes      | Yes          |
-| Query/Get                            | Yes  | Yes     | Yes  | Yes      | Yes     | Yes    | Yes   | Yes          | Yes        | Yes       | Yes             | Yes                   | Yes                 | Yes      | Yes          |
-| Query with iterator                  | Yes  | Yes     | Yes  | Yes      | Yes     | Yes    | Yes   | No           | No         | No        | No              | Yes                   | Yes                 | Yes      | Yes          |
+|                                      | HNSW | DISKANN | FLAT | IVF_FLAT | IVF_SQ8 | IVF_PQ | SCANN | GPU_IFV_FLAT | GPU_IVF_PQ | GPU_CAGRA | GPU_BRUTE_FORCE | SPARSE_INVERTED_INDEX | BIN_FLAT | BIN_IVF_FLAT |
+|--------------------------------------|------|---------|------|----------|---------|--------|-------|--------------|------------|-----------|-----------------|-----------------------|----------|--------------|
+| Basic search                         | Yes  | Yes     | Yes  | Yes      | Yes     | Yes    | Yes   | Yes          | Yes        | Yes       | Yes             | Yes                   | Yes      | Yes          |
+| Partition search                     | Yes  | Yes     | Yes  | Yes      | Yes     | Yes    | Yes   | Yes          | Yes        | Yes       | Yes             | Yes                   | Yes      | Yes          |
+| Basic search with raw data retrieved | Yes  | Yes     | Yes  | Yes      | Yes     | Yes    | Yes   | Yes          | Yes        | Yes       | Yes             | Yes                   | Yes      | Yes          |
+| Basic search with pagination         | Yes  | Yes     | Yes  | Yes      | Yes     | Yes    | Yes   | Yes          | Yes        | Yes       | Yes             | Yes                   | Yes      | Yes          |
+| Filtered search                      | Yes  | Yes     | Yes  | Yes      | Yes     | Yes    | Yes   | Yes          | Yes        | Yes       | Yes             | Yes                   | Yes      | Yes          |
+| Range search                         | Yes  | Yes     | Yes  | Yes      | Yes     | Yes    | Yes   | No           | No         | No        | No              | No                    | Yes      | Yes          |
+| Grouping search                      | Yes  | No      | Yes  | Yes      | No      | No     | No    | No           | No         | No        | No              | No                    | No       | No           |
+| Search with iterator                 | Yes  | Yes     | Yes  | Yes      | Yes     | Yes    | Yes   | No           | No         | No        | No              | No                    | No       | No           |
+| Hybrid search                        | Yes  | Yes     | Yes  | Yes      | Yes     | Yes    | Yes   | Yes          | Yes        | Yes       | Yes             | Yes(Only RRFRanker)   | Yes      | Yes          |
+| Query/Get                            | Yes  | Yes     | Yes  | Yes      | Yes     | Yes    | Yes   | Yes          | Yes        | Yes       | Yes             | Yes                   | Yes      | Yes          |
+| Query with iterator                  | Yes  | Yes     | Yes  | Yes      | Yes     | Yes    | Yes   | No           | No         | No        | No              | Yes                   | Yes      | Yes          |
diff --git a/site/en/reference/index.md b/site/en/reference/index.md
@@ -59,7 +59,13 @@ This type of indexes include `BIN_FLAT` and `BIN_IVF_FLAT`.
 
 Indexes for sparse embeddings support the `IP` and `BM25` (for full-text search) metrics only.
 
-The types of indexes include `SPARSE_INVERTED_INDEX` and `SPARSE_WAND`.
+Index type supported for sparse embeddings: `SPARSE_INVERTED_INDEX`.
+
+<div class="alert note">
+
+From Milvus 2.5.4 onward, `SPARSE_WAND` is being deprecated. Instead, it is recommended to use `"inverted_index_algo": "DAAT_WAND"` for equivalency while maintaining compatibility. For more information, refer to [Sparse Vector](sparse_vector.md#Set-index-params-for-vector-field).
+
+</div>
 
 </div>
 
@@ -230,14 +236,6 @@ The types of indexes include `SPARSE_INVERTED_INDEX` and `SPARSE_WAND`.
       <li>Requires a 100% recall rate.</li>
     </ul></td>
   </tr>
-  <tr>
-    <td>SPARSE_WAND</td>
-    <td>Inverted index</td>
-    <td><ul>
-      <li><a href="https://dl.acm.org/doi/10.1145/956863.956944">Weak-AND</a> algorithm accelerated</li>
-      <li>Can get a significant speed improvement while only sacrificing a small amount of recall.</li>
-    </ul></td>
-  </tr>
 </tbody>
 </table>
 
@@ -529,25 +527,13 @@ Each dimension maintains a list of vectors that have a non-zero value at that di
 
   | Parameter        | Description                | Range        |
   | ---------------- | -------------------------- | ------------ |
-  | `drop_ratio_build` | The proportion of small vector values that are excluded during the indexing process. This option allows fine-tuning of the indexing process, making a trade-off between efficiency and accuracy by disregarding small values when building the index.              | [0, 1] |
-
-- Search parameters
+  | `inverted_index_algo` | The algorithm used for building and querying the index. For details, refer to [Sparse Vector](sparse_vector.md#Set-index-params-for-vector-field). | `DAAT_MAXSCORE` (default), `DAAT_WAND`, `TAAT_NAIVE`  |
 
-    | Parameter           | Description                                                                                                                                                                                                                                                                                                                                                                                                                                                                               | Range  |
-    |---------------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|--------|
-    | `drop_ratio_search` | The proportion of small vector values that are excluded during the search process. This option allows fine-tuning of the search process by specifying the ratio of the smallest values in the query vector to ignore. It helps balance search precision and performance. The smaller the value set for `drop_ratio_search`, the less these small values contribute to the final score. By ignoring some small values, search performance can be improved with minimal impact on accuracy. | [0, 1] |
-
-### SPARSE_WAND
-
-This index shares similarities with `SPARSE_INVERTED_INDEX`, while it utilizes the [Weak-AND](https://dl.acm.org/doi/10.1145/956863.956944) algorithm to further reduce the number of full IP distance evaluations during the search process.
-
-Based on our testing, `SPARSE_WAND` generally outperforms other methods in terms of speed. However, its performance can deteriorate rapidly as the density of the vectors increases. To address this issue, introducing a non-zero `drop_ratio_search` can significantly enhance performance while only incurring minimal accuracy loss. For more information, refer to [Sparse Vector](sparse_vector.md).
+  <div class="alert note">
 
-- Index building parameters
+  The `drop_ratio_build` parameter is deprecated since Milvus v2.5.4, which can still be accepted during index building, but will no longer have actual effect on the index.
 
-  | Parameter        | Description                | Range        |
-  | ---------------- | -------------------------- | ------------ |
-  | `drop_ratio_build` | The proportion of small vector values that are excluded during the indexing process. This option allows fine-tuning of the indexing process, making a trade-off between efficiency and accuracy by disregarding small values when building the index.               | [0, 1] |
+  </div>
 
 - Search parameters
 

diff --git a/site/en/userGuide/manage-indexes/index-vector-fields.md b/site/en/userGuide/manage-indexes/index-vector-fields.md
@@ -71,11 +71,17 @@ Milvus provides several index types and metrics to sort field values for efficie
 <tbody>
   <tr>
     <td class="tg-0pky">IP</td>
-    <td class="tg-0pky"><ul><li>SPARSE_INVERTED_INDEX</li><li>SPARSE_WAND</li></ul></td>
+    <td class="tg-0pky">SPARSE_INVERTED_INDEX</td>
   </tr>
 </tbody>
 </table>
 
+<div class="alert note">
+
+From Milvus 2.5.4 onward, `SPARSE_WAND` is being deprecated. Instead, it is recommended to use `"inverted_index_algo": "DAAT_WAND"` for equivalency while maintaining compatibility. For more information, refer to [Sparse Vector](sparse_vector.md#Set-index-params-for-vector-field).
+
+</div>
+
 </div>
 
 It is recommended to create indexes for both the vector field and scalar fields that are frequently accessed.

diff --git a/site/en/userGuide/schema/sparse_vector.md b/site/en/userGuide/schema/sparse_vector.md
@@ -259,7 +259,7 @@ const indexParams = await client.createIndex({
     index_name: 'sparse_inverted_index',
     field_name: 'sparse_vector',
     metric_type: MetricType.IP,
-    index_type: IndexType.SPARSE_WAND,
+    index_type: IndexType.SPARSE_INVERTED_INDEX,
     params: {
       inverted_index_algo: 'DAAT_MAXSCORE',
     },
@@ -285,7 +285,6 @@ In the example above:
 - `index_type`: The type of index to create for the sparse vector field. Valid Values:
 
   - `SPARSE_INVERTED_INDEX`: A general-purpose inverted index for sparse vectors.
-  - `SPARSE_WAND`: A specialized index type supported in Milvus v2.5.3 and earlier.
 
   <div class="alert note">
 
@@ -585,7 +584,7 @@ When using sparse vectors in Milvus, consider the following limits:
 
 - Currently, only the __IP__ and __BM25__ (for full-text search) distance metrics are supported for sparse vectors. The high dimensionality of sparse vectors makes L2 and cosine distance impractical.
 
-- For sparse vector fields, only the __SPARSE_INVERTED_INDEX__ and __SPARSE_WAND__ index types are supported.
+- For sparse vector fields, only the __SPARSE_INVERTED_INDEX__ index type is supported.
 
 - The data types supported for sparse vectors:
 
@@ -599,14 +598,6 @@ When using sparse vectors in Milvus, consider the following limits:
 
 ## FAQ
 
-- __Can you explain the difference between SPARSE_INVERTED_INDEX and SPARSE_WAND, and how do I choose between them?__
-
-    __SPARSE_INVERTED_INDEX__ is a traditional inverted index, while __SPARSE_WAND__ uses the [Weak-AND](https://dl.acm.org/doi/10.1145/956863.956944) algorithm to reduce the number of full IP distance evaluations during search. __SPARSE_WAND__ is typically faster, but its performance can decline with increasing vector density. To choose between them, conduct experiments and benchmarks based on your specific dataset and use case.
-
-- __How should I choose the drop_ratio_build and drop_ratio_search parameters?__
-
-    The choice of __drop_ratio_build__ and __drop_ratio_search__ depends on the characteristics of your data and your requirements for search latency/throughput and accuracy.
-
 - __Can the dimension of a sparse embedding be any discrete value within the uint32 space?__
 
     Yes, with one exception. The dimension of a sparse embedding can be any value in the range of `[0, maximum of uint32)`. This means you cannot use the maximum value of uint32.

diff --git a/site/en/userGuide/search-query-get/multi-vector-search.md b/site/en/userGuide/search-query-get/multi-vector-search.md
@@ -246,7 +246,7 @@ index_params.add_index(
     index_name="sparse_index",
     index_type="SPARSE_INVERTED_INDEX",  # Index type for sparse vectors
     metric_type="IP",  # Currently, only IP (Inner Product) is supported for sparse vectors
-    params={"drop_ratio_build": 0.2},  # The ratio of small vector values to be dropped during indexing
+    params={"inverted_index_algo": "DAAT_MAXSCORE"},  # The ratio of small vector values to be dropped during indexing
 )
 
 ```
@@ -266,7 +266,7 @@ IndexParam indexParamForDenseField = IndexParam.builder()
         .build();
 
 Map<String, Object> sparseParams = new HashMap<>();
-sparseParams.put("drop_ratio_build", 0.2);
+sparseParams.put("inverted_index_algo": "DAAT_MAXSCORE");
 IndexParam indexParamForSparseField = IndexParam.builder()
         .fieldName("sparse")
         .indexName("sparse_index")
@@ -504,7 +504,7 @@ search_param_2 = {
     "anns_field": "sparse",
     "param": {
         "metric_type": "IP",
-        "params": {"drop_ratio_build": 0.2}
+        "params": {}
     },
     "limit": 2
 }
@@ -544,7 +544,7 @@ searchRequests.add(AnnSearchReq.builder()
         .vectorFieldName("sparse")
         .vectors(querySparseVectors)
         .metricType(IndexParam.MetricType.IP)
-        .params("{\"drop_ratio_build\": 0.2}")
+        .params()
         .topK(2)
         .build());
 
@@ -555,20 +555,20 @@ const search_param_1 = {
     "data": query_vector, 
     "anns_field": "dense", 
     "param": {
-        "metric_type": "IP", // 参数值需要与 Collection Schema 中定义的保持一致
+        "metric_type": "IP", 
         "params": {"nprobe": 10}
     },
-    "limit": 2 // AnnSearchRequest 返还的搜索结果数量
+    "limit": 2 
 }
 
 const search_param_2 = {
     "data": query_sparse_vector, 
     "anns_field": "sparse", 
     "param": {
-        "metric_type": "IP", // 参数值需要与 Collection Schema 中定义的保持一致
-        "params": {"drop_ratio_build": 0.2}
+        "metric_type": "IP", 
+        "params": {}
     },
-    "limit": 2 // AnnSearchRequest 返还的搜索结果数量
+    "limit": 2 
 }
 
 ```
@@ -589,9 +589,7 @@ export req='[
         "data": [{"3573": 0.34701499565746674}, {"5263": 0.2639375518635271}],
         "annsField": "sparse",
         "params": {
-            "params": {
-                "drop_ratio_build": 0.2
-             }
+            "params": {}
         },
         "limit": 2
     }

diff --git a/site/en/userGuide/tools/cli_commands.md b/site/en/userGuide/tools/cli_commands.md
@@ -336,7 +336,7 @@ The name of the field to create an index for (vector): vector
 Index name: vectorIndex
 
 # Default is ''
-Index type FLAT, IVF_FLAT, IVF_SQ8, IVF_PQ, RNSG, HNSW, ANNOY, AUTOINDEX, DISKANN, GPU_IVF_FLAT, GPU_IVF_PQ, SPARSE_INVERTED_INDEX, SPARSE_WAND, SCANN, STL_SORT, Trie, INVERTED, ) []: IVF_FLAT
+Index type FLAT, IVF_FLAT, IVF_SQ8, IVF_PQ, RNSG, HNSW, ANNOY, AUTOINDEX, DISKANN, GPU_IVF_FLAT, GPU_IVF_PQ, SPARSE_INVERTED_INDEX, SCANN, STL_SORT, Trie, INVERTED, ) []: IVF_FLAT
 
 # Default is ''
 Index metric type (L2, IP, HAMMING, TANIMOTO, COSINE, ) []: