From 29dd6a2b296c6e883579dd28549aefe6a859a516 Mon Sep 17 00:00:00 2001 From: Volodymyr Shkolka Date: Sat, 13 Jul 2024 10:53:19 +0300 Subject: [PATCH 1/7] Comments fix from previous PR Updated System.Text.Json package because of compile errors (warning as errors) --- Directory.Packages.props | 2 +- Milvus.Client/MilvusCollection.Entity.cs | 70 ++++++++++++++---------- 2 files changed, 41 insertions(+), 31 deletions(-) diff --git a/Directory.Packages.props b/Directory.Packages.props index 026d961..c707df4 100644 --- a/Directory.Packages.props +++ b/Directory.Packages.props @@ -3,7 +3,7 @@ - + diff --git a/Milvus.Client/MilvusCollection.Entity.cs b/Milvus.Client/MilvusCollection.Entity.cs index 24b9e44..82ebfc8 100644 --- a/Milvus.Client/MilvusCollection.Entity.cs +++ b/Milvus.Client/MilvusCollection.Entity.cs @@ -5,6 +5,7 @@ using System.Runtime.InteropServices; using System.Text.Json; using Google.Protobuf.Collections; +using KeyValuePair = Milvus.Client.Grpc.KeyValuePair; namespace Milvus.Client; @@ -143,7 +144,7 @@ public async Task DeleteAsync( { Verify.NotNullOrWhiteSpace(expression); - var request = new DeleteRequest + DeleteRequest request = new DeleteRequest { CollectionName = Name, Expr = expression, @@ -390,7 +391,7 @@ public Task FlushAsync(CancellationToken cancellationToken = defaul public async Task> GetPersistentSegmentInfosAsync( CancellationToken cancellationToken = default) { - var request = new GetPersistentSegmentInfoRequest { CollectionName = Name }; + GetPersistentSegmentInfoRequest request = new GetPersistentSegmentInfoRequest { CollectionName = Name }; GetPersistentSegmentInfoResponse response = await _client.InvokeAsync( _client.GrpcClient.GetPersistentSegmentInfoAsync, @@ -429,7 +430,7 @@ public async Task> QueryAsync( PopulateQueryRequestFromParameters(request, parameters); - var response = await _client.InvokeAsync( + QueryResults? response = await _client.InvokeAsync( _client.GrpcClient.QueryAsync, request, static r => r.Status, @@ -460,22 +461,22 @@ public async IAsyncEnumerable> QueryWithIteratorAsync( throw new MilvusException("Not support offset when searching iteration"); } - var describeResponse = await _client.InvokeAsync( + DescribeCollectionResponse? describeResponse = await _client.InvokeAsync( _client.GrpcClient.DescribeCollectionAsync, new DescribeCollectionRequest { CollectionName = Name }, r => r.Status, cancellationToken) .ConfigureAwait(false); - var pkField = describeResponse.Schema.Fields.FirstOrDefault(x => x.IsPrimaryKey); + Grpc.FieldSchema? pkField = describeResponse.Schema.Fields.FirstOrDefault(x => x.IsPrimaryKey); if (pkField == null) { throw new MilvusException("Schema must contain pk field"); } - var isUserRequestPkField = parameters?.OutputFieldsInternal?.Contains(pkField.Name) ?? false; - var userExpression = expression; - var userLimit = parameters?.Limit ?? int.MaxValue; + bool isUserRequestPkField = parameters?.OutputFieldsInternal?.Contains(pkField.Name) ?? false; + string? userExpression = expression; + int userLimit = parameters?.Limit ?? int.MaxValue; QueryRequest request = new() { @@ -486,8 +487,10 @@ public async IAsyncEnumerable> QueryWithIteratorAsync( {userExpression: not null} => userExpression, // If user expression is null and pk field is string {pkField.DataType: DataType.VarChar} => $"{pkField.Name} != ''", - // If user expression is null and pk field is not string - _ => $"{pkField.Name} < {long.MaxValue}", + // If user expression is null and pk field is int + {pkField.DataType: DataType.Int8 or DataType.Int16 or DataType.Int32 or DataType.Int64} => $"{pkField.Name} < {long.MaxValue}", + // If user expression is null and pk field is not string and not int + _ => throw new MilvusException("Unsupported data type for primary key field") } }; @@ -497,17 +500,18 @@ public async IAsyncEnumerable> QueryWithIteratorAsync( if (!isUserRequestPkField) request.OutputFields.Add(pkField.Name); // Replace parameters required for iterator + string iterationBatchSize = Math.Min(batchSize, userLimit).ToString(CultureInfo.InvariantCulture); ReplaceKeyValueItems(request.QueryParams, new Grpc.KeyValuePair {Key = Constants.Iterator, Value = "True"}, new Grpc.KeyValuePair {Key = Constants.ReduceStopForBest, Value = "True"}, - new Grpc.KeyValuePair {Key = Constants.BatchSize, Value = batchSize.ToString(CultureInfo.InvariantCulture)}, + new Grpc.KeyValuePair {Key = Constants.BatchSize, Value = iterationBatchSize}, new Grpc.KeyValuePair {Key = Constants.Offset, Value = 0.ToString(CultureInfo.InvariantCulture)}, - new Grpc.KeyValuePair {Key = Constants.Limit, Value = Math.Min(batchSize, userLimit).ToString(CultureInfo.InvariantCulture)}); + new Grpc.KeyValuePair {Key = Constants.Limit, Value = iterationBatchSize}); - var processedItemsCount = 0; + int processedItemsCount = 0; while (!cancellationToken.IsCancellationRequested) { - var response = await _client.InvokeAsync( + QueryResults? response = await _client.InvokeAsync( _client.GrpcClient.QueryAsync, request, static r => r.Status, @@ -516,16 +520,22 @@ public async IAsyncEnumerable> QueryWithIteratorAsync( object? pkLastValue; int processedDuringIterationCount; - var pkFieldsData = response.FieldsData.Single(x => x.FieldId == pkField.FieldID); - if (pkField.DataType == DataType.VarChar) + Grpc.FieldData? pkFieldsData = response.FieldsData.Single(x => x.FieldId == pkField.FieldID); + switch (pkField.DataType) { - pkLastValue = pkFieldsData.Scalars.StringData.Data.LastOrDefault(); - processedDuringIterationCount = pkFieldsData.Scalars.StringData.Data.Count; - } - else - { - pkLastValue = pkFieldsData.Scalars.IntData.Data.LastOrDefault(); - processedDuringIterationCount = pkFieldsData.Scalars.IntData.Data.Count; + case DataType.VarChar: + pkLastValue = pkFieldsData.Scalars.StringData.Data.LastOrDefault(); + processedDuringIterationCount = pkFieldsData.Scalars.StringData.Data.Count; + break; + case DataType.Int8: + case DataType.Int16: + case DataType.Int32: + case DataType.Int64: + pkLastValue = pkFieldsData.Scalars.IntData.Data.LastOrDefault(); + processedDuringIterationCount = pkFieldsData.Scalars.IntData.Data.Count; + break; + default: + throw new MilvusException("Unsupported data type for primary key field"); } // If there are no more items to process, we should break the loop @@ -540,7 +550,7 @@ public async IAsyncEnumerable> QueryWithIteratorAsync( yield return ProcessReturnedFieldData(response.FieldsData); processedItemsCount += processedDuringIterationCount; - var leftItemsCount = userLimit - processedItemsCount; + int leftItemsCount = userLimit - processedItemsCount; // If user limit is reached, we should break the loop if(leftItemsCount <= 0) yield break; @@ -554,7 +564,7 @@ public async IAsyncEnumerable> QueryWithIteratorAsync( Value = Math.Min(batchSize, leftItemsCount).ToString(CultureInfo.InvariantCulture) }); - var nextExpression = pkField.DataType == DataType.VarChar + string nextExpression = pkField.DataType == DataType.VarChar ? $"{pkField.Name} > '{pkLastValue}'" : $"{pkField.Name} > {pkLastValue}"; @@ -577,7 +587,7 @@ public async IAsyncEnumerable> QueryWithIteratorAsync( public async Task> GetQuerySegmentInfoAsync( CancellationToken cancellationToken = default) { - var request = new GetQuerySegmentInfoRequest { CollectionName = Name }; + GetQuerySegmentInfoRequest request = new GetQuerySegmentInfoRequest { CollectionName = Name }; GetQuerySegmentInfoResponse response = await _client.InvokeAsync(_client.GrpcClient.GetQuerySegmentInfoAsync, request, static r => r.Status, @@ -780,14 +790,14 @@ ulong CalculateGuaranteeTimestamp( private static void ReplaceKeyValueItems(RepeatedField collection, params Grpc.KeyValuePair[] pairs) { - var obsoleteParameterKeys = pairs.Select(x => x.Key).Distinct().ToArray(); - var obsoleteParameters = collection.Where(x => obsoleteParameterKeys.Contains(x.Key)).ToArray(); - foreach (var field in obsoleteParameters) + string[] obsoleteParameterKeys = pairs.Select(x => x.Key).Distinct().ToArray(); + KeyValuePair[] obsoleteParameters = collection.Where(x => obsoleteParameterKeys.Contains(x.Key)).ToArray(); + foreach (KeyValuePair field in obsoleteParameters) { collection.Remove(field); } - foreach (var pair in pairs) + foreach (KeyValuePair pair in pairs) { collection.Add(pair); } From 3bb279b9c3ca81a4c6b428991d6823b4c93f1e48 Mon Sep 17 00:00:00 2001 From: Volodymyr Shkolka Date: Sat, 13 Jul 2024 13:09:25 +0300 Subject: [PATCH 2/7] Added unit tests for query iterator --- .../SearchQueryIteratorLongKeyTests.cs | 256 ++++++++++++++++++ .../SearchQueryIteratorStringKeyTests.cs | 229 ++++++++++++++++ Milvus.Client/MilvusCollection.Entity.cs | 5 +- 3 files changed, 489 insertions(+), 1 deletion(-) create mode 100644 Milvus.Client.Tests/SearchQueryIteratorLongKeyTests.cs create mode 100644 Milvus.Client.Tests/SearchQueryIteratorStringKeyTests.cs diff --git a/Milvus.Client.Tests/SearchQueryIteratorLongKeyTests.cs b/Milvus.Client.Tests/SearchQueryIteratorLongKeyTests.cs new file mode 100644 index 0000000..5dbe18c --- /dev/null +++ b/Milvus.Client.Tests/SearchQueryIteratorLongKeyTests.cs @@ -0,0 +1,256 @@ +using Xunit; + +namespace Milvus.Client.Tests; + +[Collection("Milvus")] +public class SearchQueryIteratorLongKeyTests : IClassFixture, + IAsyncLifetime +{ + private const string CollectionName = nameof(DataTests); + private readonly DataCollectionFixture _dataCollectionFixture; + private readonly MilvusClient Client; + + public SearchQueryIteratorLongKeyTests(MilvusFixture milvusFixture, DataCollectionFixture dataCollectionFixture) + { + Client = milvusFixture.CreateClient(); + _dataCollectionFixture = dataCollectionFixture; + } + + public Task InitializeAsync() => Task.CompletedTask; + + public Task DisposeAsync() + { + Client.Dispose(); + return Task.CompletedTask; + } + + private MilvusCollection Collection => _dataCollectionFixture.Collection; + + [Fact] + public async Task QueryWithIterator_NoOutputFields() + { + var items = new List + { + new(1, new[] { 10f, 20f }), + new(2, new[] { 30f, 40f }), + new(3, new[] { 50f, 60f }) + }; + + await Collection.InsertAsync( + [ + FieldData.Create("id", items.Select(x => x.Id).ToArray()), + FieldData.CreateFloatVector("float_vector", items.Select(x => x.Vector).ToArray()) + ]); + + var iterator = Collection.QueryWithIteratorAsync(); + + List> results = new(); + await foreach (var result in iterator) + { + results.Add(result); + } + + var returnedItems = results.SelectMany(ExtractItems).ToList(); + Assert.Empty(returnedItems); + } + + [Fact] + public void QueryWithIterator_OffsetNotZero() + { + var queryParameters = new QueryParameters + { + Offset = 1 + }; + + var iterator = Collection.QueryWithIteratorAsync(parameters: queryParameters); + + var exception = Assert.ThrowsAsync(async () => await iterator.GetAsyncEnumerator().MoveNextAsync()); + Assert.NotNull(exception); + } + + [Fact] + public void QueryWithIterator_LimitNotZero() + { + var queryParameters = new QueryParameters + { + Limit = 0 + }; + + var iterator = Collection.QueryWithIteratorAsync(parameters: queryParameters); + + var exception = Assert.ThrowsAsync(async () => await iterator.GetAsyncEnumerator().MoveNextAsync()); + Assert.NotNull(exception); + } + + [Theory] + [InlineData("id in [1, 2, 3]", 1, null)] + [InlineData("id in [1, 2, 3]", 1, 2)] + [InlineData("id in [1, 2, 3]", 2, null)] + [InlineData("id in [1, 2, 3]", 2, 2)] + [InlineData("id in [1, 2, 3]", 1000, null)] + [InlineData("id in [1, 2, 3]", 1000, 2)] + [InlineData(null, 1, null)] + [InlineData(null, 1, 2)] + [InlineData(null, 2, null)] + [InlineData(null, 2, 2)] + [InlineData(null, 1000, null)] + [InlineData(null, 1000, 2)] + public async Task QueryWithIterator(string? expression, int batchSize, int? limit) + { + var items = new List + { + new(1, new[] { 10f, 20f }), + new(2, new[] { 30f, 40f }), + new(3, new[] { 50f, 60f }) + }; + + await Collection.InsertAsync( + [ + FieldData.Create("id", items.Select(x => x.Id).ToArray()), + FieldData.CreateFloatVector("float_vector", items.Select(x => x.Vector).ToArray()) + ]); + + var queryParameters = new QueryParameters + { + OutputFields = { "id", "float_vector" }, + Limit = limit + }; + + var iterator = Collection.QueryWithIteratorAsync( + expression: expression, + batchSize: batchSize, + parameters: queryParameters); + + List> results = new(); + await foreach (var result in iterator) + { + results.Add(result); + } + + var returnedItems = results.SelectMany(ExtractItems).ToArray(); + var expectedItems = items.Take(limit ?? int.MaxValue).ToArray(); + Assert.Equal(expectedItems.Length, returnedItems.Length); + + foreach (var expectedItem in expectedItems) + { + Assert.Contains(expectedItem, returnedItems); + } + } + + IEnumerable ExtractItems(IReadOnlyList result) + { + long rowCount = result.Select(x => x.RowCount).FirstOrDefault(); + + var items = new Item[rowCount]; + for (int i = 0; i < rowCount; i++) + { + items[i] = new Item(); + } + + foreach (var fieldData in result) + { + switch (fieldData.FieldName) + { + case "id": + { + for (int j = 0; j < rowCount; j++) + { + items[j].Id = ((FieldData) fieldData).Data[j]; + } + + break; + } + + case "float_vector": + { + for (int j = 0; j < rowCount; j++) + { + items[j].Vector = ((FloatVectorFieldData) fieldData).Data[j]; + } + + break; + } + } + } + + return items; + } + + #region Nested type: DataCollectionFixture + + public class DataCollectionFixture : IAsyncLifetime + { + public MilvusCollection Collection; + private readonly MilvusClient Client; + + public DataCollectionFixture(MilvusFixture milvusFixture) + { + Client = milvusFixture.CreateClient(); + Collection = Client.GetCollection(CollectionName); + } + + public async Task InitializeAsync() + { + await Collection.DropAsync(); + + await Client.CreateCollectionAsync( + Collection.Name, + new[] + { + FieldSchema.Create("id", isPrimaryKey: true), + FieldSchema.CreateFloatVector("float_vector", 2) + }); + + await Collection.CreateIndexAsync("float_vector", IndexType.Flat, SimilarityMetricType.L2); + await Collection.WaitForIndexBuildAsync("float_vector"); + await Collection.LoadAsync(); + await Collection.WaitForCollectionLoadAsync(); + } + + public Task DisposeAsync() + { + Client.Dispose(); + return Task.CompletedTask; + } + } + + #endregion + + #region Nested type: Item + + public record Item + { + public Item(long id, ReadOnlyMemory vector) + { + Id = id; + Vector = vector; + } + + public Item() + { + } + + public virtual bool Equals(Item? other) + { + return other != null && Id == other.Id && Vector.Span.SequenceEqual(other.Vector.Span); + } + + public long Id { get; set; } + + public ReadOnlyMemory Vector { get; set; } + + public override int GetHashCode() + { + var hashCode = new HashCode(); + hashCode.Add(Id); + foreach (float value in Vector.ToArray()) + { + hashCode.Add(value); + } + + return hashCode.ToHashCode(); + } + } + + #endregion +} diff --git a/Milvus.Client.Tests/SearchQueryIteratorStringKeyTests.cs b/Milvus.Client.Tests/SearchQueryIteratorStringKeyTests.cs new file mode 100644 index 0000000..6093336 --- /dev/null +++ b/Milvus.Client.Tests/SearchQueryIteratorStringKeyTests.cs @@ -0,0 +1,229 @@ +using Xunit; + +namespace Milvus.Client.Tests; + +[Collection("Milvus")] +public class SearchQueryIteratorStringKeyTests : IClassFixture, + IAsyncLifetime +{ + private const string CollectionName = nameof(DataTests); + + private readonly DataCollectionFixture _dataCollectionFixture; + private readonly MilvusClient Client; + + public SearchQueryIteratorStringKeyTests(MilvusFixture milvusFixture, DataCollectionFixture dataCollectionFixture) + { + Client = milvusFixture.CreateClient(); + _dataCollectionFixture = dataCollectionFixture; + } + + public Task InitializeAsync() => Task.CompletedTask; + + public Task DisposeAsync() + { + Client.Dispose(); + return Task.CompletedTask; + } + + private MilvusCollection Collection => _dataCollectionFixture.Collection; + + [Fact] + public async Task QueryWithIterator_NoOutputFields() + { + var items = new List + { + new("1", new[] { 10f, 20f }), + new("2", new[] { 30f, 40f }), + new("3", new[] { 50f, 60f }) + }; + + await Collection.InsertAsync( + [ + FieldData.Create("id", items.Select(x => x.Id).ToArray()), + FieldData.CreateFloatVector("float_vector", items.Select(x => x.Vector).ToArray()) + ]); + + var iterator = Collection.QueryWithIteratorAsync(); + + List> results = new(); + await foreach (var result in iterator) + { + results.Add(result); + } + + var returnedItems = results.SelectMany(ExtractItems).ToList(); + Assert.Empty(returnedItems); + } + + [Theory] + [InlineData("id in ['1', '2', '3']", 1, null)] + [InlineData("id in ['1', '2', '3']", 1, 2)] + [InlineData("id in ['1', '2', '3']", 2, null)] + [InlineData("id in ['1', '2', '3']", 2, 2)] + [InlineData("id in ['1', '2', '3']", 1000, null)] + [InlineData("id in ['1', '2', '3']", 1000, 2)] + [InlineData(null, 1, null)] + [InlineData(null, 1, 2)] + [InlineData(null, 2, null)] + [InlineData(null, 2, 2)] + [InlineData(null, 1000, null)] + [InlineData(null, 1000, 2)] + public async Task QueryWithIterator(string? expression, int batchSize, int? limit) + { + var items = new List + { + new("1", new[] { 10f, 20f }), + new("2", new[] { 30f, 40f }), + new("3", new[] { 50f, 60f }) + }; + + await Collection.InsertAsync( + [ + FieldData.Create("id", items.Select(x => x.Id).ToArray()), + FieldData.CreateFloatVector("float_vector", items.Select(x => x.Vector).ToArray()) + ]); + + var queryParameters = new QueryParameters + { + OutputFields = { "id", "float_vector" }, + Limit = limit + }; + + var iterator = Collection.QueryWithIteratorAsync( + expression: expression, + batchSize: batchSize, + parameters: queryParameters); + + List> results = new(); + await foreach (var result in iterator) + { + results.Add(result); + } + + var returnedItems = results.SelectMany(ExtractItems).ToArray(); + var expectedItems = items.Take(limit ?? int.MaxValue).ToArray(); + Assert.Equal(expectedItems.Length, returnedItems.Length); + + foreach (var expectedItem in expectedItems) + { + Assert.Contains(expectedItem, returnedItems); + } + } + + IEnumerable ExtractItems(IReadOnlyList result) + { + long rowCount = result.Select(x => x.RowCount).FirstOrDefault(); + + var items = new Item[rowCount]; + for (int i = 0; i < rowCount; i++) + { + items[i] = new Item(); + } + + foreach (var fieldData in result) + { + switch (fieldData.FieldName) + { + case "id": + { + for (int j = 0; j < rowCount; j++) + { + items[j].Id = ((FieldData) fieldData).Data[j]; + } + + break; + } + + case "float_vector": + { + for (int j = 0; j < rowCount; j++) + { + items[j].Vector = ((FloatVectorFieldData) fieldData).Data[j]; + } + + break; + } + } + } + + return items; + } + + #region Nested type: DataCollectionFixture + + public class DataCollectionFixture : IAsyncLifetime + { + public MilvusCollection Collection; + private readonly MilvusClient Client; + + public DataCollectionFixture(MilvusFixture milvusFixture) + { + Client = milvusFixture.CreateClient(); + Collection = Client.GetCollection(CollectionName); + } + + public async Task InitializeAsync() + { + await Collection.DropAsync(); + + await Client.CreateCollectionAsync( + Collection.Name, + new[] + { + FieldSchema.CreateVarchar("id", 16, isPrimaryKey: true), + FieldSchema.CreateFloatVector("float_vector", 2) + }); + + await Collection.CreateIndexAsync("float_vector", IndexType.Flat, SimilarityMetricType.L2); + await Collection.WaitForIndexBuildAsync("float_vector"); + await Collection.LoadAsync(); + await Collection.WaitForCollectionLoadAsync(); + } + + public Task DisposeAsync() + { + Client.Dispose(); + return Task.CompletedTask; + } + } + + #endregion + + #region Nested type: Item + + public record Item + { + public Item(string id, ReadOnlyMemory vector) + { + Id = id; + Vector = vector; + } + + public Item() + { + } + + public virtual bool Equals(Item? other) + { + return other != null && Id == other.Id && Vector.Span.SequenceEqual(other.Vector.Span); + } + + public string? Id { get; set; } + + public ReadOnlyMemory Vector { get; set; } + + public override int GetHashCode() + { + var hashCode = new HashCode(); + hashCode.Add(Id); + foreach (float value in Vector.ToArray()) + { + hashCode.Add(value); + } + + return hashCode.ToHashCode(); + } + } + + #endregion +} diff --git a/Milvus.Client/MilvusCollection.Entity.cs b/Milvus.Client/MilvusCollection.Entity.cs index 82ebfc8..b58134a 100644 --- a/Milvus.Client/MilvusCollection.Entity.cs +++ b/Milvus.Client/MilvusCollection.Entity.cs @@ -530,10 +530,13 @@ public async IAsyncEnumerable> QueryWithIteratorAsync( case DataType.Int8: case DataType.Int16: case DataType.Int32: - case DataType.Int64: pkLastValue = pkFieldsData.Scalars.IntData.Data.LastOrDefault(); processedDuringIterationCount = pkFieldsData.Scalars.IntData.Data.Count; break; + case DataType.Int64: + pkLastValue = pkFieldsData.Scalars.LongData.Data.LastOrDefault(); + processedDuringIterationCount = pkFieldsData.Scalars.LongData.Data.Count; + break; default: throw new MilvusException("Unsupported data type for primary key field"); } From 4d719437be21b3ac8bcac0790bd3f26a98e28308 Mon Sep 17 00:00:00 2001 From: Volodymyr Shkolka Date: Sat, 13 Jul 2024 13:13:16 +0300 Subject: [PATCH 3/7] Comments fix --- Milvus.Client/MilvusCollection.Entity.cs | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/Milvus.Client/MilvusCollection.Entity.cs b/Milvus.Client/MilvusCollection.Entity.cs index b58134a..66d66b9 100644 --- a/Milvus.Client/MilvusCollection.Entity.cs +++ b/Milvus.Client/MilvusCollection.Entity.cs @@ -567,13 +567,16 @@ public async IAsyncEnumerable> QueryWithIteratorAsync( Value = Math.Min(batchSize, leftItemsCount).ToString(CultureInfo.InvariantCulture) }); - string nextExpression = pkField.DataType == DataType.VarChar - ? $"{pkField.Name} > '{pkLastValue}'" - : $"{pkField.Name} > {pkLastValue}"; + string nextExpression = pkField.DataType switch + { + DataType.VarChar => $"{pkField.Name} > '{pkLastValue}'", + DataType.Int8 or DataType.Int16 or DataType.Int32 or DataType.Int64 => $"{pkField.Name} > '{pkLastValue}'", + _ => throw new MilvusException("Unsupported data type for primary key field") + }; if (!string.IsNullOrWhiteSpace(userExpression)) { - nextExpression += $" and {userExpression}"; + nextExpression += $" and ({userExpression})"; } request.Expr = nextExpression; From 0bc8f0c92213b885ef5a0a20ca1e6954555c17dd Mon Sep 17 00:00:00 2001 From: Volodymyr Shkolka Date: Sat, 13 Jul 2024 13:15:59 +0300 Subject: [PATCH 4/7] Comments fix --- Milvus.Client/MilvusCollection.Entity.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Milvus.Client/MilvusCollection.Entity.cs b/Milvus.Client/MilvusCollection.Entity.cs index 66d66b9..84bb099 100644 --- a/Milvus.Client/MilvusCollection.Entity.cs +++ b/Milvus.Client/MilvusCollection.Entity.cs @@ -505,7 +505,7 @@ public async IAsyncEnumerable> QueryWithIteratorAsync( new Grpc.KeyValuePair {Key = Constants.Iterator, Value = "True"}, new Grpc.KeyValuePair {Key = Constants.ReduceStopForBest, Value = "True"}, new Grpc.KeyValuePair {Key = Constants.BatchSize, Value = iterationBatchSize}, - new Grpc.KeyValuePair {Key = Constants.Offset, Value = 0.ToString(CultureInfo.InvariantCulture)}, + new Grpc.KeyValuePair {Key = Constants.Offset, Value = "0"}, new Grpc.KeyValuePair {Key = Constants.Limit, Value = iterationBatchSize}); int processedItemsCount = 0; From f5dd63662a37d39d9beb92a742cdfd605372d4cf Mon Sep 17 00:00:00 2001 From: Volodymyr Shkolka Date: Sat, 13 Jul 2024 13:17:20 +0300 Subject: [PATCH 5/7] Comments fix --- Milvus.Client/MilvusCollection.Entity.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Milvus.Client/MilvusCollection.Entity.cs b/Milvus.Client/MilvusCollection.Entity.cs index 84bb099..1389c66 100644 --- a/Milvus.Client/MilvusCollection.Entity.cs +++ b/Milvus.Client/MilvusCollection.Entity.cs @@ -509,7 +509,7 @@ public async IAsyncEnumerable> QueryWithIteratorAsync( new Grpc.KeyValuePair {Key = Constants.Limit, Value = iterationBatchSize}); int processedItemsCount = 0; - while (!cancellationToken.IsCancellationRequested) + while (true) { QueryResults? response = await _client.InvokeAsync( _client.GrpcClient.QueryAsync, From 3602fe1ef8fe450ba6f85a7399f173de175e33d0 Mon Sep 17 00:00:00 2001 From: Volodymyr Shkolka Date: Sat, 13 Jul 2024 13:37:43 +0300 Subject: [PATCH 6/7] Fixed unit tests rise condition --- Milvus.Client.Tests/SearchQueryIteratorLongKeyTests.cs | 2 +- Milvus.Client.Tests/SearchQueryIteratorStringKeyTests.cs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Milvus.Client.Tests/SearchQueryIteratorLongKeyTests.cs b/Milvus.Client.Tests/SearchQueryIteratorLongKeyTests.cs index 5dbe18c..3443994 100644 --- a/Milvus.Client.Tests/SearchQueryIteratorLongKeyTests.cs +++ b/Milvus.Client.Tests/SearchQueryIteratorLongKeyTests.cs @@ -6,7 +6,7 @@ namespace Milvus.Client.Tests; public class SearchQueryIteratorLongKeyTests : IClassFixture, IAsyncLifetime { - private const string CollectionName = nameof(DataTests); + private const string CollectionName = nameof(SearchQueryIteratorLongKeyTests); private readonly DataCollectionFixture _dataCollectionFixture; private readonly MilvusClient Client; diff --git a/Milvus.Client.Tests/SearchQueryIteratorStringKeyTests.cs b/Milvus.Client.Tests/SearchQueryIteratorStringKeyTests.cs index 6093336..d9cf379 100644 --- a/Milvus.Client.Tests/SearchQueryIteratorStringKeyTests.cs +++ b/Milvus.Client.Tests/SearchQueryIteratorStringKeyTests.cs @@ -6,7 +6,7 @@ namespace Milvus.Client.Tests; public class SearchQueryIteratorStringKeyTests : IClassFixture, IAsyncLifetime { - private const string CollectionName = nameof(DataTests); + private const string CollectionName = nameof(SearchQueryIteratorStringKeyTests); private readonly DataCollectionFixture _dataCollectionFixture; private readonly MilvusClient Client; From f1009b59eed4525dff1ab416554d9c94851290a1 Mon Sep 17 00:00:00 2001 From: Volodymyr Shkolka Date: Sat, 13 Jul 2024 13:58:51 +0300 Subject: [PATCH 7/7] Fixed copy paste next iteration condition processing --- Milvus.Client/MilvusCollection.Entity.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Milvus.Client/MilvusCollection.Entity.cs b/Milvus.Client/MilvusCollection.Entity.cs index 1389c66..dd48f2f 100644 --- a/Milvus.Client/MilvusCollection.Entity.cs +++ b/Milvus.Client/MilvusCollection.Entity.cs @@ -570,7 +570,7 @@ public async IAsyncEnumerable> QueryWithIteratorAsync( string nextExpression = pkField.DataType switch { DataType.VarChar => $"{pkField.Name} > '{pkLastValue}'", - DataType.Int8 or DataType.Int16 or DataType.Int32 or DataType.Int64 => $"{pkField.Name} > '{pkLastValue}'", + DataType.Int8 or DataType.Int16 or DataType.Int32 or DataType.Int64 => $"{pkField.Name} > {pkLastValue}", _ => throw new MilvusException("Unsupported data type for primary key field") };