From a4b976f1ab372fc5e2b50711a6b76699c94aeb38 Mon Sep 17 00:00:00 2001 From: Blake Niemyjski Date: Fri, 30 Aug 2024 15:45:00 -0500 Subject: [PATCH 01/19] Use elasticsearch 8 --- docker-compose.yml | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index eb818b54..3434dfab 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,8 +1,6 @@ -version: '3.5' - services: elasticsearch: - image: docker.elastic.co/elasticsearch/elasticsearch:7.17.16 + image: docker.elastic.co/elasticsearch/elasticsearch:8.15.0 environment: discovery.type: single-node xpack.security.enabled: 'false' @@ -16,7 +14,7 @@ services: kibana: depends_on: - elasticsearch - image: docker.elastic.co/kibana/kibana:7.17.16 + image: docker.elastic.co/kibana/kibana:8.15.0 ports: - 5601:5601 networks: From c6be6c7d23a0adfd5f4638101040f4e217bf6644 Mon Sep 17 00:00:00 2001 From: Blake Niemyjski Date: Fri, 30 Aug 2024 15:54:29 -0500 Subject: [PATCH 02/19] WIP - Upgrade to the new Elasticsearch client --- .../ElasticMappingResolver.cs | 177 ++++++++++-------- .../ElasticQueryParser.cs | 18 +- .../ElasticQueryParserConfiguration.cs | 13 +- .../DefaultAggregationNodeExtensions.cs | 95 +++++----- .../Extensions/DefaultQueryNodeExtensions.cs | 53 +++--- .../Extensions/DefaultSortNodeExtensions.cs | 14 +- .../Extensions/ElasticExtensions.cs | 55 +++--- .../Extensions/ElasticMappingExtensions.cs | 17 +- .../Extensions/QueryNodeExtensions.cs | 29 +-- .../QueryVisitorContextExtensions.cs | 2 +- .../Extensions/SearchDescriptorExtensions.cs | 33 ---- .../Foundatio.Parsers.ElasticQueries.csproj | 2 +- .../Visitors/CombineAggregationsVisitor.cs | 24 +-- .../Visitors/CombineQueriesVisitor.cs | 6 +- .../Visitors/GeoVisitor.cs | 7 +- .../Visitors/GetSortFieldsVisitor.cs | 14 +- .../Visitors/NestedVisitor.cs | 2 +- .../AggregationParserTests.cs | 6 +- .../CustomVisitorTests.cs | 1 - .../ElasticMappingResolverTests.cs | 3 +- .../ElasticQueryParserTests.cs | 6 +- .../InvertQueryTests.cs | 1 - .../Utility/ElasticsearchTestBase.cs | 34 ++-- .../Utility/LoggingTracer.cs | 2 +- 24 files changed, 313 insertions(+), 301 deletions(-) delete mode 100644 src/Foundatio.Parsers.ElasticQueries/Extensions/SearchDescriptorExtensions.cs diff --git a/src/Foundatio.Parsers.ElasticQueries/ElasticMappingResolver.cs b/src/Foundatio.Parsers.ElasticQueries/ElasticMappingResolver.cs index 67584ba2..5bdf85d2 100644 --- a/src/Foundatio.Parsers.ElasticQueries/ElasticMappingResolver.cs +++ b/src/Foundatio.Parsers.ElasticQueries/ElasticMappingResolver.cs @@ -1,32 +1,35 @@ -using System; +using System; using System.Collections.Concurrent; +using System.Collections.Generic; using System.Linq; +using Elastic.Clients.Elasticsearch; +using Elastic.Clients.Elasticsearch.IndexManagement; +using Elastic.Clients.Elasticsearch.Mapping; using Exceptionless.DateTimeExtensions; using Foundatio.Parsers.ElasticQueries.Extensions; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Logging.Abstractions; -using Nest; namespace Foundatio.Parsers.ElasticQueries; public class ElasticMappingResolver { - private ITypeMapping _serverMapping; - private readonly ITypeMapping _codeMapping; + private TypeMapping _serverMapping; + private readonly TypeMapping _codeMapping; private readonly Inferrer _inferrer; private readonly ConcurrentDictionary _mappingCache = new(); private readonly ILogger _logger; public static ElasticMappingResolver NullInstance = new(() => null); - public ElasticMappingResolver(Func getMapping, Inferrer inferrer = null, ILogger logger = null) + public ElasticMappingResolver(Func getMapping, Inferrer inferrer = null, ILogger logger = null) { GetServerMappingFunc = getMapping; _inferrer = inferrer; _logger = logger ?? NullLogger.Instance; } - public ElasticMappingResolver(ITypeMapping codeMapping, Inferrer inferrer, Func getMapping, ILogger logger = null) + public ElasticMappingResolver(TypeMapping codeMapping, Inferrer inferrer, Func getMapping, ILogger logger = null) : this(getMapping, inferrer, logger) { _codeMapping = codeMapping; @@ -37,7 +40,7 @@ public ElasticMappingResolver(ITypeMapping codeMapping, Inferrer inferrer, Func< /// public void RefreshMapping() { - _logger.LogInformation("Mapping refresh triggered."); + _logger.LogInformation("Mapping refresh triggered"); _serverMapping = null; _lastMappingUpdate = null; } @@ -53,7 +56,7 @@ public FieldMapping GetMapping(string field, bool followAlias = false) if (_mappingCache.TryGetValue(field, out var mapping)) { - if (followAlias && mapping.Found && mapping.Property is IFieldAliasProperty fieldAlias) + if (followAlias && mapping.Found && mapping.Property is FieldAliasProperty fieldAlias) { _logger.LogTrace("Cached alias mapping: {Field}={FieldPath}:{FieldType}", field, mapping.FullPath, mapping.Property?.Type); return GetMapping(fieldAlias.Path.Name); @@ -67,11 +70,11 @@ public FieldMapping GetMapping(string field, bool followAlias = false) if (mapping.ServerMapTime >= _lastMappingUpdate && !GetServerMapping()) { - _logger.LogTrace("Cached mapping (not found): {field}=", field); + _logger.LogTrace("Cached mapping (not found): {Field}=", field); return mapping; } - _logger.LogTrace("Cached mapping (not found), got new server mapping."); + _logger.LogTrace("Cached mapping (not found), got new server mapping"); } string[] fieldParts = field.Split('.'); @@ -83,11 +86,11 @@ public FieldMapping GetMapping(string field, bool followAlias = false) { string fieldPart = fieldParts[depth]; IProperty fieldMapping = null; - if (currentProperties == null || !currentProperties.TryGetValue(fieldPart, out fieldMapping)) + if (currentProperties == null || !currentProperties.TryGetProperty(fieldPart, out fieldMapping)) { - // check to see if there is an name match + // check to see if there is a name match if (currentProperties != null) - fieldMapping = currentProperties.Values.FirstOrDefault(m => + fieldMapping = ((IDictionary)currentProperties).Values.FirstOrDefault(m => { string propertyName = _inferrer.PropertyName(m?.Name); return propertyName != null && propertyName.Equals(fieldPart, StringComparison.OrdinalIgnoreCase); @@ -136,19 +139,19 @@ public FieldMapping GetMapping(string field, bool followAlias = false) _mappingCache.AddOrUpdate(field, resolvedMapping, (_, _) => resolvedMapping); _logger.LogTrace("Resolved mapping: {Field}={FieldPath}:{FieldType}", field, resolvedMapping.FullPath, resolvedMapping.Property?.Type); - if (followAlias && resolvedMapping.Property is IFieldAliasProperty fieldAlias) + if (followAlias && resolvedMapping.Property is FieldAliasProperty fieldAlias) return GetMapping(fieldAlias.Path.Name); return resolvedMapping; } - if (fieldMapping is IObjectProperty objectProperty) + if (fieldMapping is ObjectProperty objectProperty) { currentProperties = objectProperty.Properties; } else { - if (fieldMapping is ITextProperty textProperty) + if (fieldMapping is TextProperty textProperty) currentProperties = textProperty.Fields; else break; @@ -229,13 +232,13 @@ public string GetNonAnalyzedFieldName(string field, string preferredSubField = n if (mapping?.Property == null || !IsPropertyAnalyzed(mapping.Property)) return field; - var multiFieldProperty = mapping.Property as ICoreProperty; + var multiFieldProperty = mapping.Property; if (multiFieldProperty?.Fields == null) return mapping.FullPath; var nonAnalyzedProperty = multiFieldProperty.Fields.OrderByDescending(kvp => kvp.Key.Name == preferredSubField).FirstOrDefault(kvp => { - if (kvp.Value is IKeywordProperty) + if (kvp.Value is KeywordProperty) return true; if (!IsPropertyAnalyzed(kvp.Value)) @@ -265,7 +268,7 @@ public bool IsPropertyAnalyzed(string field) public bool IsPropertyAnalyzed(IProperty property) { - if (property is ITextProperty textProperty) + if (property is TextProperty textProperty) return !textProperty.Index.HasValue || textProperty.Index.Value; return false; @@ -276,7 +279,7 @@ public bool IsNestedPropertyType(string field) if (String.IsNullOrEmpty(field)) return false; - return GetMappingProperty(field, true) is INestedProperty; + return GetMappingProperty(field, true) is NestedProperty; } public bool IsGeoPropertyType(string field) @@ -284,7 +287,7 @@ public bool IsGeoPropertyType(string field) if (String.IsNullOrEmpty(field)) return false; - return GetMappingProperty(field, true) is IGeoPointProperty; + return GetMappingProperty(field, true) is GeoPointProperty; } public bool IsNumericPropertyType(string field) @@ -292,7 +295,16 @@ public bool IsNumericPropertyType(string field) if (String.IsNullOrEmpty(field)) return false; - return GetMappingProperty(field, true) is INumberProperty; + var property = GetMappingProperty(field, true); + return property is ByteNumberProperty + or DoubleNumberProperty + or FloatNumberProperty + or HalfFloatNumberProperty + or IntegerNumberProperty + or LongNumberProperty + or ScaledFloatNumberProperty + or ShortNumberProperty + or UnsignedLongNumberProperty; } public bool IsBooleanPropertyType(string field) @@ -300,7 +312,7 @@ public bool IsBooleanPropertyType(string field) if (String.IsNullOrEmpty(field)) return false; - return GetMappingProperty(field, true) is IBooleanProperty; + return GetMappingProperty(field, true) is BooleanProperty; } public bool IsDatePropertyType(string field) @@ -308,7 +320,7 @@ public bool IsDatePropertyType(string field) if (String.IsNullOrEmpty(field)) return false; - return GetMappingProperty(field, true) is IDateProperty; + return GetMappingProperty(field, true) is DateProperty; } public FieldType GetFieldType(string field) @@ -323,44 +335,62 @@ public FieldType GetFieldType(string field) return property.Type switch { - "geo_point" => FieldType.GeoPoint, - "geo_shape" => FieldType.GeoShape, - "ip" => FieldType.Ip, - "binary" => FieldType.Binary, - "keyword" => FieldType.Keyword, - "string" or "text" => FieldType.Text, - "date" => FieldType.Date, - "boolean" => FieldType.Boolean, - "completion" => FieldType.Completion, - "nested" => FieldType.Nested, - "object" => FieldType.Object, - "murmur3" => FieldType.Murmur3Hash, - "token_count" => FieldType.TokenCount, - "percolator" => FieldType.Percolator, - "integer" => FieldType.Integer, - "long" => FieldType.Long, - "short" => FieldType.Short, - "byte" => FieldType.Byte, - "float" => FieldType.Float, - "half_float" => FieldType.HalfFloat, - "scaled_float" => FieldType.ScaledFloat, - "double" => FieldType.Double, - "integer_range" => FieldType.IntegerRange, - "float_range" => FieldType.FloatRange, - "long_range" => FieldType.LongRange, - "double_range" => FieldType.DoubleRange, - "date_range" => FieldType.DateRange, - "ip_range" => FieldType.IpRange, + "version"=> FieldType.Version, + "token_count"=> FieldType.TokenCount, + "text"=> FieldType.Text, + "sparse_vector"=> FieldType.SparseVector, + "short"=> FieldType.Short, + "shape"=> FieldType.Shape, + "semantic_text"=> FieldType.SemanticText, + "search_as_you_type"=> FieldType.SearchAsYouType, + "scaled_float"=> FieldType.ScaledFloat, + "rank_features"=> FieldType.RankFeatures, + "rank_feature"=> FieldType.RankFeature, + "percolator"=> FieldType.Percolator, + "object"=> FieldType.Object, + "none"=> FieldType.None, + "nested"=> FieldType.Nested, + "murmur3"=> FieldType.Murmur3, + "match_only_text"=> FieldType.MatchOnlyText, + "long_range"=> FieldType.LongRange, + "long"=> FieldType.Long, + "keyword"=> FieldType.Keyword, + "join"=> FieldType.Join, + "ip_range"=> FieldType.IpRange, + "ip"=> FieldType.Ip, + "integer_range"=> FieldType.IntegerRange, + "integer"=> FieldType.Integer, + "icu_collation_keyword"=> FieldType.IcuCollationKeyword, + "histogram"=> FieldType.Histogram, + "half_float"=> FieldType.HalfFloat, + "geo_shape"=> FieldType.GeoShape, + "geo_point"=> FieldType.GeoPoint, + "float_range"=> FieldType.FloatRange, + "float"=> FieldType.Float, + "flattened"=> FieldType.Flattened, + "double_range"=> FieldType.DoubleRange, + "double"=> FieldType.Double, + "dense_vector"=> FieldType.DenseVector, + "date_range"=> FieldType.DateRange, + "date_nanos"=> FieldType.DateNanos, + "date"=> FieldType.Date, + "constant_keyword"=> FieldType.ConstantKeyword, + "completion"=> FieldType.Completion, + "byte"=> FieldType.Byte, + "boolean"=> FieldType.Boolean, + "binary"=> FieldType.Binary, + "alias"=> FieldType.Alias, + "aggregate_metric_double"=> FieldType.AggregateMetricDouble, _ => FieldType.None, }; } - private IProperties MergeProperties(IProperties codeProperties, IProperties serverProperties) + private Properties MergeProperties(Properties codeProperties, Properties serverProperties) { if (codeProperties == null && serverProperties == null) return null; - IProperties mergedCodeProperties = null; + Properties mergedCodeProperties = null; // resolve code mapping property expressions using inferrer if (codeProperties != null) { @@ -369,7 +399,7 @@ private IProperties MergeProperties(IProperties codeProperties, IProperties serv foreach (var kvp in codeProperties) { var propertyName = kvp.Key; - if (_inferrer != null && (String.IsNullOrEmpty(kvp.Key.Name) || kvp.Value is IFieldAliasProperty)) + if (_inferrer != null && (String.IsNullOrEmpty(kvp.Key.Name) || kvp.Value is FieldAliasProperty)) propertyName = _inferrer.PropertyName(kvp.Key) ?? kvp.Key; mergedCodeProperties[propertyName] = kvp.Value; @@ -380,12 +410,12 @@ private IProperties MergeProperties(IProperties codeProperties, IProperties serv // resolve field alias foreach (var kvp in codeProperties) { - if (kvp.Value is not IFieldAliasProperty aliasProperty) + if (kvp.Value is not FieldAliasProperty aliasProperty) continue; mergedCodeProperties[kvp.Key] = new FieldAliasProperty { - LocalMetadata = aliasProperty.LocalMetadata, + Meta = aliasProperty.Meta, Path = _inferrer?.Field(aliasProperty.Path) ?? aliasProperty.Path, Name = aliasProperty.Name }; @@ -397,21 +427,21 @@ private IProperties MergeProperties(IProperties codeProperties, IProperties serv if (mergedCodeProperties == null || serverProperties == null) return mergedCodeProperties ?? serverProperties; - IProperties properties = new Properties(); + Properties properties = new Properties(); foreach (var serverProperty in serverProperties) { var merged = serverProperty.Value; - if (mergedCodeProperties.TryGetValue(serverProperty.Key, out var codeProperty)) + if (mergedCodeProperties.TryGetProperty(serverProperty.Key, out var codeProperty)) merged.LocalMetadata = codeProperty.LocalMetadata; switch (merged) { - case IObjectProperty objectProperty: - var codeObjectProperty = codeProperty as IObjectProperty; + case ObjectProperty objectProperty: + var codeObjectProperty = codeProperty as ObjectProperty; objectProperty.Properties = MergeProperties(codeObjectProperty?.Properties, objectProperty.Properties); break; - case ITextProperty textProperty: - var codeTextProperty = codeProperty as ITextProperty; + case TextProperty textProperty: + var codeTextProperty = codeProperty as TextProperty; textProperty.Fields = MergeProperties(codeTextProperty?.Fields, textProperty.Fields); break; } @@ -421,7 +451,7 @@ private IProperties MergeProperties(IProperties codeProperties, IProperties serv foreach (var codeProperty in mergedCodeProperties) { - if (properties.TryGetValue(codeProperty.Key, out _)) + if (properties.TryGetProperty(codeProperty.Key, out _)) continue; properties.Add(codeProperty.Key, codeProperty.Value); @@ -430,7 +460,7 @@ private IProperties MergeProperties(IProperties codeProperties, IProperties serv return properties; } - private Func GetServerMappingFunc { get; set; } + private Func GetServerMappingFunc { get; set; } private DateTime? _lastMappingUpdate = null; private bool GetServerMapping() { @@ -450,12 +480,12 @@ private bool GetServerMapping() } catch (Exception ex) { - _logger.LogError(ex, "Error getting server mapping: " + ex.Message); + _logger.LogError(ex, "Error getting server mapping: {Message}", ex.Message); return false; } } - public static ElasticMappingResolver Create(Func, ITypeMapping> mappingBuilder, IElasticClient client, ILogger logger = null) where T : class + public static ElasticMappingResolver Create(Func, TypeMapping> mappingBuilder, ElasticsearchClient client, ILogger logger = null) where T : class { logger ??= NullLogger.Instance; @@ -471,7 +501,7 @@ public static ElasticMappingResolver Create(Func, IT }, logger); } - public static ElasticMappingResolver Create(Func, ITypeMapping> mappingBuilder, IElasticClient client, string index, ILogger logger = null) where T : class + public static ElasticMappingResolver Create(Func, TypeMapping> mappingBuilder, ElasticsearchClient client, string index, ILogger logger = null) where T : class { logger ??= NullLogger.Instance; @@ -487,14 +517,13 @@ public static ElasticMappingResolver Create(Func, IT }, logger); } - public static ElasticMappingResolver Create(Func, ITypeMapping> mappingBuilder, Inferrer inferrer, Func getMapping, ILogger logger = null) where T : class + public static ElasticMappingResolver Create(Func, TypeMapping> mappingBuilder, Inferrer inferrer, Func getMapping, ILogger logger = null) where T : class { - var codeMapping = new TypeMappingDescriptor(); - codeMapping = mappingBuilder(codeMapping) as TypeMappingDescriptor; + var codeMapping = mappingBuilder(new TypeMappingDescriptor()); return new ElasticMappingResolver(codeMapping, inferrer, getMapping, logger: logger); } - public static ElasticMappingResolver Create(IElasticClient client, ILogger logger = null) + public static ElasticMappingResolver Create(ElasticsearchClient client, ILogger logger = null) { logger ??= NullLogger.Instance; @@ -510,7 +539,7 @@ public static ElasticMappingResolver Create(IElasticClient client, ILogger lo }, client.Infer, logger); } - public static ElasticMappingResolver Create(IElasticClient client, string index, ILogger logger = null) + public static ElasticMappingResolver Create(ElasticsearchClient client, string index, ILogger logger = null) { logger ??= NullLogger.Instance; @@ -526,7 +555,7 @@ public static ElasticMappingResolver Create(IElasticClient client, string index, }, client.Infer, logger); } - public static ElasticMappingResolver Create(Func getMapping, Inferrer inferrer, ILogger logger = null) + public static ElasticMappingResolver Create(Func getMapping, Inferrer inferrer, ILogger logger = null) { return new ElasticMappingResolver(getMapping, inferrer, logger: logger); } diff --git a/src/Foundatio.Parsers.ElasticQueries/ElasticQueryParser.cs b/src/Foundatio.Parsers.ElasticQueries/ElasticQueryParser.cs index a9abce24..4c410f4a 100644 --- a/src/Foundatio.Parsers.ElasticQueries/ElasticQueryParser.cs +++ b/src/Foundatio.Parsers.ElasticQueries/ElasticQueryParser.cs @@ -1,14 +1,16 @@ -using System; +using System; using System.Collections.Generic; using System.Linq; using System.Threading.Tasks; +using Elastic.Clients.Elasticsearch; +using Elastic.Clients.Elasticsearch.Aggregations; +using Elastic.Clients.Elasticsearch.QueryDsl; using Foundatio.Parsers.ElasticQueries.Extensions; using Foundatio.Parsers.ElasticQueries.Visitors; using Foundatio.Parsers.LuceneQueries; using Foundatio.Parsers.LuceneQueries.Extensions; using Foundatio.Parsers.LuceneQueries.Nodes; using Foundatio.Parsers.LuceneQueries.Visitors; -using Nest; using Pegasus.Common; namespace Foundatio.Parsers.ElasticQueries; @@ -158,7 +160,7 @@ public async Task ValidateQueryAsync(string query, QueryV return context.GetValidationResult(); } - public async Task BuildQueryAsync(string query, IElasticQueryVisitorContext context = null) + public async Task BuildQueryAsync(string query, IElasticQueryVisitorContext context = null) { context ??= new ElasticQueryVisitorContext(); context.QueryType = QueryTypes.Query; @@ -169,7 +171,7 @@ public async Task BuildQueryAsync(string query, IElasticQueryVis return await BuildQueryAsync(result, context).ConfigureAwait(false); } - public async Task BuildQueryAsync(IQueryNode query, IElasticQueryVisitorContext context = null) + public async Task BuildQueryAsync(IQueryNode query, IElasticQueryVisitorContext context = null) { context ??= new ElasticQueryVisitorContext(); var q = await query.GetQueryAsync() ?? new MatchAllQuery(); @@ -195,7 +197,7 @@ public async Task ValidateAggregationsAsync(string query, return context.GetValidationResult(); } - public async Task BuildAggregationsAsync(string aggregations, IElasticQueryVisitorContext context = null) + public async Task BuildAggregationsAsync(string aggregations, IElasticQueryVisitorContext context = null) { context ??= new ElasticQueryVisitorContext(); context.QueryType = QueryTypes.Aggregation; @@ -207,7 +209,7 @@ public async Task BuildAggregationsAsync(string aggregatio } #pragma warning disable IDE0060 // Remove unused parameter - public async Task BuildAggregationsAsync(IQueryNode aggregations, IElasticQueryVisitorContext context = null) + public async Task BuildAggregationsAsync(IQueryNode aggregations, IElasticQueryVisitorContext context = null) { if (aggregations == null) return null; @@ -227,7 +229,7 @@ public async Task ValidateSortAsync(string query, QueryVa return context.GetValidationResult(); } - public async Task> BuildSortAsync(string sort, IElasticQueryVisitorContext context = null) + public async Task> BuildSortAsync(string sort, IElasticQueryVisitorContext context = null) { context ??= new ElasticQueryVisitorContext(); context.QueryType = QueryTypes.Sort; @@ -238,7 +240,7 @@ public async Task> BuildSortAsync(string sort, IElasticQ return await BuildSortAsync(result, context).ConfigureAwait(false); } - public Task> BuildSortAsync(IQueryNode sort, IElasticQueryVisitorContext context = null) + public Task> BuildSortAsync(IQueryNode sort, IElasticQueryVisitorContext context = null) { context ??= new ElasticQueryVisitorContext(); return GetSortFieldsVisitor.RunAsync(sort, context); diff --git a/src/Foundatio.Parsers.ElasticQueries/ElasticQueryParserConfiguration.cs b/src/Foundatio.Parsers.ElasticQueries/ElasticQueryParserConfiguration.cs index 8b6de9cd..f4c69b2e 100644 --- a/src/Foundatio.Parsers.ElasticQueries/ElasticQueryParserConfiguration.cs +++ b/src/Foundatio.Parsers.ElasticQueries/ElasticQueryParserConfiguration.cs @@ -1,12 +1,13 @@ using System; using System.Collections.Generic; using System.Threading.Tasks; +using Elastic.Clients.Elasticsearch; +using Elastic.Clients.Elasticsearch.Mapping; using Foundatio.Parsers.ElasticQueries.Visitors; using Foundatio.Parsers.LuceneQueries; using Foundatio.Parsers.LuceneQueries.Visitors; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Logging.Abstractions; -using Nest; namespace Foundatio.Parsers.ElasticQueries; @@ -285,35 +286,35 @@ public ElasticQueryParserConfiguration AddAggregationVisitorAfter(IChainableQ #endregion - public ElasticQueryParserConfiguration UseMappings(Func, TypeMappingDescriptor> mappingBuilder, IElasticClient client, string index) where T : class + public ElasticQueryParserConfiguration UseMappings(Func, TypeMappingDescriptor> mappingBuilder, ElasticsearchClient client, string index) where T : class { MappingResolver = ElasticMappingResolver.Create(mappingBuilder, client, index, logger: _logger); return this; } - public ElasticQueryParserConfiguration UseMappings(Func, TypeMappingDescriptor> mappingBuilder, Inferrer inferrer, Func getMapping) where T : class + public ElasticQueryParserConfiguration UseMappings(Func, TypeMappingDescriptor> mappingBuilder, Inferrer inferrer, Func getMapping) where T : class { MappingResolver = ElasticMappingResolver.Create(mappingBuilder, inferrer, getMapping, logger: _logger); return this; } - public ElasticQueryParserConfiguration UseMappings(IElasticClient client) + public ElasticQueryParserConfiguration UseMappings(ElasticsearchClient client) { MappingResolver = ElasticMappingResolver.Create(client, logger: _logger); return this; } - public ElasticQueryParserConfiguration UseMappings(IElasticClient client, string index) + public ElasticQueryParserConfiguration UseMappings(ElasticsearchClient client, string index) { MappingResolver = ElasticMappingResolver.Create(client, index, logger: _logger); return this; } - public ElasticQueryParserConfiguration UseMappings(Func getMapping, Inferrer inferrer = null) + public ElasticQueryParserConfiguration UseMappings(Func getMapping, Inferrer inferrer = null) { MappingResolver = ElasticMappingResolver.Create(getMapping, inferrer, logger: _logger); diff --git a/src/Foundatio.Parsers.ElasticQueries/Extensions/DefaultAggregationNodeExtensions.cs b/src/Foundatio.Parsers.ElasticQueries/Extensions/DefaultAggregationNodeExtensions.cs index dad8290b..6450cb66 100644 --- a/src/Foundatio.Parsers.ElasticQueries/Extensions/DefaultAggregationNodeExtensions.cs +++ b/src/Foundatio.Parsers.ElasticQueries/Extensions/DefaultAggregationNodeExtensions.cs @@ -1,12 +1,13 @@ -using System; +using System; using System.Collections.Generic; using System.Threading.Tasks; +using Elastic.Clients.Elasticsearch; +using Elastic.Clients.Elasticsearch.Aggregations; using Exceptionless.DateTimeExtensions; using Foundatio.Parsers.ElasticQueries.Visitors; using Foundatio.Parsers.LuceneQueries.Extensions; using Foundatio.Parsers.LuceneQueries.Nodes; using Foundatio.Parsers.LuceneQueries.Visitors; -using Nest; namespace Foundatio.Parsers.ElasticQueries.Extensions; @@ -15,7 +16,7 @@ public static class DefaultAggregationNodeExtensions // NOTE: We may want to read this dynamically from server settings. public const int MAX_BUCKET_SIZE = 10000; - public static Task GetDefaultAggregationAsync(this IQueryNode node, IQueryVisitorContext context) + public static Task GetDefaultAggregationAsync(this IQueryNode node, IQueryVisitorContext context) { if (node is GroupNode groupNode) return groupNode.GetDefaultAggregationAsync(context); @@ -26,7 +27,7 @@ public static Task GetDefaultAggregationAsync(this IQueryNode n return null; } - public static async Task GetDefaultAggregationAsync(this GroupNode node, IQueryVisitorContext context) + public static async Task GetDefaultAggregationAsync(this GroupNode node, IQueryVisitorContext context) { if (context is not IElasticQueryVisitorContext elasticContext) throw new ArgumentException("Context must be of type IElasticQueryVisitorContext", nameof(context)); @@ -47,29 +48,34 @@ public static async Task GetDefaultAggregationAsync(this GroupN return GetHistogramAggregation("histogram_" + originalField, field, node.UnescapedProximity, node.UnescapedBoost, context); case AggregationType.GeoHashGrid: - var precision = GeoHashPrecision.Precision1; - if (!String.IsNullOrEmpty(node.UnescapedProximity)) - Enum.TryParse(node.UnescapedProximity, out precision); + var precision = new GeohashPrecision(1); + if (!String.IsNullOrEmpty(node.UnescapedProximity) && Double.TryParse(node.UnescapedProximity, out double parsedPrecision)) + { + if (parsedPrecision is < 1 or > 12) + throw new ArgumentOutOfRangeException(nameof(node.UnescapedProximity), "Precision must be between 1 and 12"); + + precision = new GeohashPrecision(parsedPrecision); + } - return new GeoHashGridAggregation("geogrid_" + originalField) + return new Aggregation("geogrid_" + originalField, new GeohashGridAggregation { Field = field, Precision = precision, Aggregations = new AverageAggregation("avg_lat", null) { - Script = new InlineScript($"doc['{node.Field}'].lat") + Script = new Script { Source = $"doc['{node.Field}'].lat" } } && new AverageAggregation("avg_lon", null) { - Script = new InlineScript($"doc['{node.Field}'].lon") + Script = new Script { Source = $"doc['{node.Field}'].lon" } } - }; + }); case AggregationType.Terms: var agg = new TermsAggregation("terms_" + originalField) { Field = field, Size = node.GetProximityAsInt32(), - MinimumDocumentCount = node.GetBoostAsInt32(), + MinDocCount = node.GetBoostAsInt32(), Meta = new Dictionary { { "@field_type", property?.Type } } }; @@ -85,7 +91,7 @@ public static async Task GetDefaultAggregationAsync(this GroupN return null; } - public static async Task GetDefaultAggregationAsync(this TermNode node, IQueryVisitorContext context) + public static async Task GetDefaultAggregationAsync(this TermNode node, IQueryVisitorContext context) { if (context is not IElasticQueryVisitorContext elasticContext) throw new ArgumentException("Context must be of type IElasticQueryVisitorContext", nameof(context)); @@ -134,20 +140,25 @@ public static async Task GetDefaultAggregationAsync(this TermNo return GetPercentilesAggregation("percentiles_" + originalField, aggField, node.UnescapedProximity, node.UnescapedBoost, context); case AggregationType.GeoHashGrid: - var precision = GeoHashPrecision.Precision1; - if (!String.IsNullOrEmpty(node.UnescapedProximity)) - Enum.TryParse(node.UnescapedProximity, out precision); + var precision = new GeohashPrecision(1); + if (!String.IsNullOrEmpty(node.UnescapedProximity) && Double.TryParse(node.UnescapedProximity, out double parsedPrecision)) + { + if (parsedPrecision is < 1 or > 12) + throw new ArgumentOutOfRangeException(nameof(node.UnescapedProximity), "Precision must be between 1 and 12"); + + precision = new GeohashPrecision(parsedPrecision); + } - return new GeoHashGridAggregation("geogrid_" + originalField) + return new GeohashGridAggregation("geogrid_" + originalField) { Field = aggField, Precision = precision, Aggregations = new AverageAggregation("avg_lat", null) { - Script = new InlineScript($"doc['{node.Field}'].lat") + Script = new Script { Source = $"doc['{node.Field}'].lat" } } && new AverageAggregation("avg_lon", null) { - Script = new InlineScript($"doc['{node.Field}'].lon") + Script = new Script { Source = $"doc['{node.Field}'].lon" } } }; @@ -156,7 +167,7 @@ public static async Task GetDefaultAggregationAsync(this TermNo { Field = aggField, Size = node.GetProximityAsInt32(), - MinimumDocumentCount = node.GetBoostAsInt32(), + MinDocCount = node.GetBoostAsInt32(), Meta = new Dictionary { { "@field_type", property?.Type } } }; @@ -169,7 +180,7 @@ public static async Task GetDefaultAggregationAsync(this TermNo return null; } - private static AggregationBase GetPercentilesAggregation(string originalField, string field, string proximity, string boost, IQueryVisitorContext context) + private static Aggregation GetPercentilesAggregation(string originalField, string field, string proximity, string boost, IQueryVisitorContext context) { List percents = null; if (!String.IsNullOrWhiteSpace(proximity)) @@ -189,7 +200,7 @@ private static AggregationBase GetPercentilesAggregation(string originalField, s }; } - private static AggregationBase GetHistogramAggregation(string originalField, string field, string proximity, string boost, IQueryVisitorContext context) + private static Aggregation GetHistogramAggregation(string originalField, string field, string proximity, string boost, IQueryVisitorContext context) { double interval = 50; if (Double.TryParse(proximity, out double prox)) @@ -198,25 +209,25 @@ private static AggregationBase GetHistogramAggregation(string originalField, str return new HistogramAggregation(originalField) { Field = field, - MinimumDocumentCount = 0, + MinDocCount = 0, Interval = interval }; } - private static AggregationBase GetDateHistogramAggregation(string originalField, string field, string proximity, string boost, IQueryVisitorContext context) + private static Aggregation GetDateHistogramAggregation(string originalField, string field, string proximity, string boost, IQueryVisitorContext context) { // NOTE: StartDate and EndDate are set in the Repositories QueryBuilderContext. var start = context.GetDate("StartDate"); var end = context.GetDate("EndDate"); bool isValidRange = start.HasValue && start.Value > DateTime.MinValue && end.HasValue && end.Value < DateTime.MaxValue && start.Value <= end.Value; - var bounds = isValidRange ? new ExtendedBounds { Minimum = start.Value, Maximum = end.Value } : null; + var bounds = isValidRange ? new ExtendedBoundsDate { Min = start.Value, Max = end.Value } : null; var interval = GetInterval(proximity, start, end); string timezone = TryConvertTimeUnitToUtcOffset(boost); var agg = new DateHistogramAggregation(originalField) { Field = field, - MinimumDocumentCount = 0, + MinDocCount = 0, Format = "date_optional_time", TimeZone = timezone, Meta = !String.IsNullOrEmpty(boost) ? new Dictionary { { "@timezone", boost } } : null, @@ -247,55 +258,55 @@ private static string TryConvertTimeUnitToUtcOffset(string boost) return "+" + timezoneOffset.Value.ToString("hh\\:mm"); } - private static Union GetInterval(string proximity, DateTime? start, DateTime? end) + private static Union GetInterval(string proximity, DateTime? start, DateTime? end) { if (String.IsNullOrEmpty(proximity)) return GetInterval(start, end); return proximity.Trim() switch { - "s" or "1s" or "second" => DateInterval.Second, - "m" or "1m" or "minute" => DateInterval.Minute, - "h" or "1h" or "hour" => DateInterval.Hour, - "d" or "1d" or "day" => DateInterval.Day, - "w" or "1w" or "week" => DateInterval.Week, - "M" or "1M" or "month" => DateInterval.Month, - "q" or "1q" or "quarter" => DateInterval.Quarter, - "y" or "1y" or "year" => DateInterval.Year, - _ => new Union(proximity), + "s" or "1s" or "second" => CalendarInterval.Second, + "m" or "1m" or "minute" => CalendarInterval.Minute, + "h" or "1h" or "hour" => CalendarInterval.Hour, + "d" or "1d" or "day" => CalendarInterval.Day, + "w" or "1w" or "week" => CalendarInterval.Week, + "M" or "1M" or "month" => CalendarInterval.Month, + "q" or "1q" or "quarter" => CalendarInterval.Quarter, + "y" or "1y" or "year" => CalendarInterval.Year, + _ => new Union(proximity), }; } - private static Union GetInterval(DateTime? utcStart, DateTime? utcEnd, int desiredDataPoints = 100) + private static Union GetInterval(DateTime? utcStart, DateTime? utcEnd, int desiredDataPoints = 100) { if (!utcStart.HasValue || !utcEnd.HasValue || utcStart.Value == DateTime.MinValue) - return DateInterval.Day; + return CalendarInterval.Day; var totalTime = utcEnd.Value - utcStart.Value; var timePerBlock = TimeSpan.FromMinutes(totalTime.TotalMinutes / desiredDataPoints); if (timePerBlock.TotalDays > 1) { timePerBlock = timePerBlock.Round(TimeSpan.FromDays(1)); - return (Time)timePerBlock; + return (Duration)timePerBlock; } if (timePerBlock.TotalHours > 1) { timePerBlock = timePerBlock.Round(TimeSpan.FromHours(1)); - return (Time)timePerBlock; + return (Duration)timePerBlock; } if (timePerBlock.TotalMinutes > 1) { timePerBlock = timePerBlock.Round(TimeSpan.FromMinutes(1)); - return (Time)timePerBlock; + return (Duration)timePerBlock; } timePerBlock = timePerBlock.Round(TimeSpan.FromSeconds(15)); if (timePerBlock.TotalSeconds < 1) timePerBlock = TimeSpan.FromSeconds(15); - return (Time)timePerBlock; + return (Duration)timePerBlock; } public static int? GetProximityAsInt32(this IFieldQueryWithProximityAndBoostNode node) diff --git a/src/Foundatio.Parsers.ElasticQueries/Extensions/DefaultQueryNodeExtensions.cs b/src/Foundatio.Parsers.ElasticQueries/Extensions/DefaultQueryNodeExtensions.cs index 4c2aba27..ab76b6f0 100644 --- a/src/Foundatio.Parsers.ElasticQueries/Extensions/DefaultQueryNodeExtensions.cs +++ b/src/Foundatio.Parsers.ElasticQueries/Extensions/DefaultQueryNodeExtensions.cs @@ -1,16 +1,16 @@ -using System; +using System; using System.Threading.Tasks; +using Elastic.Clients.Elasticsearch.QueryDsl; using Foundatio.Parsers.ElasticQueries.Visitors; using Foundatio.Parsers.LuceneQueries.Extensions; using Foundatio.Parsers.LuceneQueries.Nodes; using Foundatio.Parsers.LuceneQueries.Visitors; -using Nest; namespace Foundatio.Parsers.ElasticQueries.Extensions; public static class DefaultQueryNodeExtensions { - public static async Task GetDefaultQueryAsync(this IQueryNode node, IQueryVisitorContext context) + public static async Task GetDefaultQueryAsync(this IQueryNode node, IQueryVisitorContext context) { if (node is TermNode termNode) return termNode.GetDefaultQuery(context); @@ -27,12 +27,12 @@ public static async Task GetDefaultQueryAsync(this IQueryNode node, I return null; } - public static QueryBase GetDefaultQuery(this TermNode node, IQueryVisitorContext context) + public static Query GetDefaultQuery(this TermNode node, IQueryVisitorContext context) { if (context is not IElasticQueryVisitorContext elasticContext) throw new ArgumentException("Context must be of type IElasticQueryVisitorContext", nameof(context)); - QueryBase query; + Query query; string field = node.UnescapedField; string[] defaultFields = node.GetDefaultFields(elasticContext.DefaultFields); if (field == null && defaultFields != null && defaultFields.Length == 1) @@ -58,17 +58,15 @@ public static QueryBase GetDefaultQuery(this TermNode node, IQueryVisitorContext { if (node.IsQuotedTerm) { - query = new MatchPhraseQuery + query = new MatchPhraseQuery(fields[0]) { - Field = fields[0], Query = node.UnescapedTerm }; } else { - query = new MatchQuery + query = new MatchQuery(fields[0]) { - Field = fields[0], Query = node.UnescapedTerm }; } @@ -78,10 +76,9 @@ public static QueryBase GetDefaultQuery(this TermNode node, IQueryVisitorContext query = new MultiMatchQuery { Fields = fields, - Query = node.UnescapedTerm + Query = node.UnescapedTerm, + Type = node.IsQuotedTerm ? TextQueryType.Phrase : null }; - if (node.IsQuotedTerm) - ((MultiMatchQuery)query).Type = TextQueryType.Phrase; } } } @@ -89,17 +86,15 @@ public static QueryBase GetDefaultQuery(this TermNode node, IQueryVisitorContext { if (!node.IsQuotedTerm && node.UnescapedTerm.EndsWith("*")) { - query = new PrefixQuery + query = new PrefixQuery(field) { - Field = field, Value = node.UnescapedTerm.TrimEnd('*') }; } else { - query = new TermQuery + query = new TermQuery(field) { - Field = field, Value = node.UnescapedTerm }; } @@ -108,7 +103,7 @@ public static QueryBase GetDefaultQuery(this TermNode node, IQueryVisitorContext return query; } - public static async Task GetDefaultQueryAsync(this TermRangeNode node, IQueryVisitorContext context) + public static async Task GetDefaultQueryAsync(this TermRangeNode node, IQueryVisitorContext context) { if (context is not IElasticQueryVisitorContext elasticContext) throw new ArgumentException("Context must be of type IElasticQueryVisitorContext", nameof(context)); @@ -116,54 +111,54 @@ public static async Task GetDefaultQueryAsync(this TermRangeNode node string field = node.UnescapedField; if (elasticContext.MappingResolver.IsDatePropertyType(field)) { - var range = new DateRangeQuery { Field = field, TimeZone = node.Boost ?? node.GetTimeZone(await elasticContext.GetTimeZoneAsync()) }; + var range = new DateRangeQuery(field) { TimeZone = node.Boost ?? node.GetTimeZone(await elasticContext.GetTimeZoneAsync()) }; if (!String.IsNullOrWhiteSpace(node.UnescapedMin) && node.UnescapedMin != "*") { if (node.MinInclusive.HasValue && !node.MinInclusive.Value) - range.GreaterThan = node.UnescapedMin; + range.Gt = node.UnescapedMin; else - range.GreaterThanOrEqualTo = node.UnescapedMin; + range.Gte = node.UnescapedMin; } if (!String.IsNullOrWhiteSpace(node.UnescapedMax) && node.UnescapedMax != "*") { if (node.MaxInclusive.HasValue && !node.MaxInclusive.Value) - range.LessThan = node.UnescapedMax; + range.Lt = node.UnescapedMax; else - range.LessThanOrEqualTo = node.UnescapedMax; + range.Lte = node.UnescapedMax; } return range; } else { - var range = new TermRangeQuery { Field = field }; + var range = new TermRangeQuery(field); if (!String.IsNullOrWhiteSpace(node.UnescapedMin) && node.UnescapedMin != "*") { if (node.MinInclusive.HasValue && !node.MinInclusive.Value) - range.GreaterThan = node.UnescapedMin; + range.Gt = node.UnescapedMin; else - range.GreaterThanOrEqualTo = node.UnescapedMin; + range.Gte = node.UnescapedMin; } if (!String.IsNullOrWhiteSpace(node.UnescapedMax) && node.UnescapedMax != "*") { if (node.MaxInclusive.HasValue && !node.MaxInclusive.Value) - range.LessThan = node.UnescapedMax; + range.Lt = node.UnescapedMax; else - range.LessThanOrEqualTo = node.UnescapedMax; + range.Lte = node.UnescapedMax; } return range; } } - public static QueryBase GetDefaultQuery(this ExistsNode node, IQueryVisitorContext context) + public static Query GetDefaultQuery(this ExistsNode node, IQueryVisitorContext context) { return new ExistsQuery { Field = node.UnescapedField }; } - public static QueryBase GetDefaultQuery(this MissingNode node, IQueryVisitorContext context) + public static Query GetDefaultQuery(this MissingNode node, IQueryVisitorContext context) { return new BoolQuery { diff --git a/src/Foundatio.Parsers.ElasticQueries/Extensions/DefaultSortNodeExtensions.cs b/src/Foundatio.Parsers.ElasticQueries/Extensions/DefaultSortNodeExtensions.cs index 3775c69e..cdcd4147 100644 --- a/src/Foundatio.Parsers.ElasticQueries/Extensions/DefaultSortNodeExtensions.cs +++ b/src/Foundatio.Parsers.ElasticQueries/Extensions/DefaultSortNodeExtensions.cs @@ -1,15 +1,16 @@ using System; +using Elastic.Clients.Elasticsearch; +using Elastic.Clients.Elasticsearch.Mapping; using Foundatio.Parsers.ElasticQueries.Visitors; using Foundatio.Parsers.LuceneQueries.Extensions; using Foundatio.Parsers.LuceneQueries.Nodes; using Foundatio.Parsers.LuceneQueries.Visitors; -using Nest; namespace Foundatio.Parsers.ElasticQueries.Extensions; public static class DefaultSortNodeExtensions { - public static IFieldSort GetDefaultSort(this TermNode node, IQueryVisitorContext context) + public static SortOptions GetDefaultSort(this TermNode node, IQueryVisitorContext context) { if (context is not IElasticQueryVisitorContext elasticContext) throw new ArgumentException("Context must be of type IElasticQueryVisitorContext", nameof(context)); @@ -17,13 +18,10 @@ public static IFieldSort GetDefaultSort(this TermNode node, IQueryVisitorContext string field = elasticContext.MappingResolver.GetSortFieldName(node.UnescapedField); var fieldType = elasticContext.MappingResolver.GetFieldType(field); - var sort = new FieldSort + return SortOptions.Field(field, new FieldSort { - Field = field, UnmappedType = fieldType == FieldType.None ? FieldType.Keyword : fieldType, - Order = node.IsNodeOrGroupNegated() ? SortOrder.Descending : SortOrder.Ascending - }; - - return sort; + Order = node.IsNodeOrGroupNegated() ? SortOrder.Desc : SortOrder.Asc + }); } } diff --git a/src/Foundatio.Parsers.ElasticQueries/Extensions/ElasticExtensions.cs b/src/Foundatio.Parsers.ElasticQueries/Extensions/ElasticExtensions.cs index e12e54cf..aa442120 100644 --- a/src/Foundatio.Parsers.ElasticQueries/Extensions/ElasticExtensions.cs +++ b/src/Foundatio.Parsers.ElasticQueries/Extensions/ElasticExtensions.cs @@ -6,9 +6,10 @@ using System.Text.Json; using System.Threading; using System.Threading.Tasks; -using Elasticsearch.Net; +using Elastic.Clients.Elasticsearch; +using Elastic.Clients.Elasticsearch.Aggregations; +using Elastic.Transport.Products.Elasticsearch; using Microsoft.Extensions.Logging; -using Nest; namespace Foundatio.Parsers.ElasticQueries.Extensions; @@ -37,7 +38,7 @@ public static TermsExclude AddValue(this TermsExclude exclude, string value) } // TODO: Handle IFailureReason/BulkIndexByScrollFailure and other bulk response types. - public static string GetErrorMessage(this IElasticsearchResponse elasticResponse, string message = null, bool normalize = false, bool includeResponse = false, bool includeDebugInformation = false) + public static string GetErrorMessage(this ElasticsearchResponse elasticResponse, string message = null, bool normalize = false, bool includeResponse = false, bool includeDebugInformation = false) { if (elasticResponse == null) return String.Empty; @@ -47,31 +48,30 @@ public static string GetErrorMessage(this IElasticsearchResponse elasticResponse if (!String.IsNullOrEmpty(message)) sb.AppendLine(message); - var response = elasticResponse as IResponse; - if (includeDebugInformation && response?.DebugInformation != null) - sb.AppendLine(response.DebugInformation); + if (includeDebugInformation && elasticResponse?.DebugInformation != null) + sb.AppendLine(elasticResponse.DebugInformation); - if (response?.OriginalException != null) - sb.AppendLine($"Original: [{response.OriginalException.GetType().Name}] {response.OriginalException.Message}"); + if (elasticResponse.TryGetOriginalException(out var exception) && exception is not null) + sb.AppendLine($"Original: [{exception.GetType().Name}] {exception.Message}"); - if (response?.ServerError?.Error != null) - sb.AppendLine($"Server Error (Index={response.ServerError.Error?.Index}): {response.ServerError.Error.Reason}"); + if (elasticResponse.ElasticsearchServerError?.Error != null) + sb.AppendLine($"Server Error (Index={elasticResponse.ElasticsearchServerError.Error?.Index}): {elasticResponse.ElasticsearchServerError.Error.Reason}"); if (elasticResponse is BulkResponse bulkResponse) sb.AppendLine($"Bulk: {String.Join("\r\n", bulkResponse.ItemsWithErrors.Select(i => i.Error))}"); - if (elasticResponse.ApiCall != null) - sb.AppendLine($"[{elasticResponse.ApiCall.HttpStatusCode}] {elasticResponse.ApiCall.HttpMethod} {elasticResponse.ApiCall.Uri?.PathAndQuery}"); + var apiCall = elasticResponse.ApiCallDetails; + if (apiCall is not null) + sb.AppendLine($"[{apiCall.HttpStatusCode}] {apiCall.HttpMethod} {apiCall.Uri?.PathAndQuery}"); - if (elasticResponse.ApiCall?.RequestBodyInBytes != null) + if (apiCall?.RequestBodyInBytes is not null) { - string body = Encoding.UTF8.GetString(elasticResponse.ApiCall?.RequestBodyInBytes); + string body = Encoding.UTF8.GetString(apiCall.RequestBodyInBytes); if (normalize) body = JsonUtility.Normalize(body); sb.AppendLine(body); } - var apiCall = response.ApiCall; if (includeResponse && apiCall.ResponseBodyInBytes != null && apiCall.ResponseBodyInBytes.Length > 0 && apiCall.ResponseBodyInBytes.Length < 20000) { string body = Encoding.UTF8.GetString(apiCall?.ResponseBodyInBytes); @@ -88,52 +88,52 @@ public static string GetErrorMessage(this IElasticsearchResponse elasticResponse return sb.ToString(); } - public static string GetRequest(this IElasticsearchResponse elasticResponse, bool normalize = false, bool includeResponse = false, bool includeDebugInformation = false) + public static string GetRequest(this ElasticsearchResponse elasticResponse, bool normalize = false, bool includeResponse = false, bool includeDebugInformation = false) { return GetErrorMessage(elasticResponse, null, normalize, includeResponse, includeDebugInformation); } - public static async Task WaitForReadyAsync(this IElasticClient client, CancellationToken cancellationToken, ILogger logger = null) + public static async Task WaitForReadyAsync(this ElasticsearchClient client, CancellationToken cancellationToken, ILogger logger = null) { - var nodes = client.ConnectionSettings.ConnectionPool.Nodes.Select(n => n.Uri.ToString()); + var nodes = client.ElasticsearchClientSettings.NodePool.Nodes.Select(n => n.Uri.ToString()); var startTime = DateTime.UtcNow; while (!cancellationToken.IsCancellationRequested) { - var pingResponse = await client.PingAsync(ct: cancellationToken); - if (pingResponse.IsValid) + var pingResponse = await client.PingAsync(cancellationToken); + if (pingResponse.IsValidResponse) return true; - if (logger != null && logger.IsEnabled(Microsoft.Extensions.Logging.LogLevel.Information)) + if (logger != null && logger.IsEnabled(LogLevel.Information)) logger?.LogInformation("Waiting for Elasticsearch to be ready {Server} after {Duration:g}...", nodes, DateTime.UtcNow.Subtract(startTime)); await Task.Delay(1000, cancellationToken); } - if (logger != null && logger.IsEnabled(Microsoft.Extensions.Logging.LogLevel.Error)) + if (logger != null && logger.IsEnabled(LogLevel.Error)) logger?.LogError("Unable to connect to Elasticsearch {Server} after attempting for {Duration:g}", nodes, DateTime.UtcNow.Subtract(startTime)); return false; } - public static bool WaitForReady(this IElasticClient client, CancellationToken cancellationToken, ILogger logger = null) + public static bool WaitForReady(this ElasticsearchClient client, CancellationToken cancellationToken, ILogger logger = null) { - var nodes = client.ConnectionSettings.ConnectionPool.Nodes.Select(n => n.Uri.ToString()); + var nodes = client.ElasticsearchClientSettings.NodePool.Nodes.Select(n => n.Uri.ToString()); var startTime = DateTime.UtcNow; while (!cancellationToken.IsCancellationRequested) { var pingResponse = client.Ping(); - if (pingResponse.IsValid) + if (pingResponse.IsValidResponse) return true; - if (logger != null && logger.IsEnabled(Microsoft.Extensions.Logging.LogLevel.Information)) + if (logger != null && logger.IsEnabled(LogLevel.Information)) logger?.LogInformation("Waiting for Elasticsearch to be ready {Server} after {Duration:g}...", nodes, DateTime.UtcNow.Subtract(startTime)); Thread.Sleep(1000); } - if (logger != null && logger.IsEnabled(Microsoft.Extensions.Logging.LogLevel.Error)) + if (logger != null && logger.IsEnabled(LogLevel.Error)) logger?.LogError("Unable to connect to Elasticsearch {Server} after attempting for {Duration:g}", nodes, DateTime.UtcNow.Subtract(startTime)); return false; @@ -213,7 +213,6 @@ private static void Write(JsonElement element, Utf8JsonWriter writer) default: throw new NotImplementedException($"Kind: {element.ValueKind}"); - } } } diff --git a/src/Foundatio.Parsers.ElasticQueries/Extensions/ElasticMappingExtensions.cs b/src/Foundatio.Parsers.ElasticQueries/Extensions/ElasticMappingExtensions.cs index d40836c4..c01507fb 100644 --- a/src/Foundatio.Parsers.ElasticQueries/Extensions/ElasticMappingExtensions.cs +++ b/src/Foundatio.Parsers.ElasticQueries/Extensions/ElasticMappingExtensions.cs @@ -1,4 +1,7 @@ -namespace Nest; +using Elastic.Clients.Elasticsearch.IndexManagement; +using Elastic.Clients.Elasticsearch.Mapping; + +namespace Foundatio.Parsers.ElasticQueries.Extensions; public static class ElasticMapping { @@ -15,7 +18,7 @@ public static TextPropertyDescriptor AddKeywordField(this TextPropertyDesc /// public static TextPropertyDescriptor AddKeywordField(this TextPropertyDescriptor descriptor, string normalizer) where T : class { - return descriptor.Fields(f => f.Keyword(s => s.Name(KeywordFieldName).Normalizer(normalizer).IgnoreAbove(256))); + return descriptor.Fields(f => f.Keyword(KeywordFieldName, s => s.Normalizer(normalizer).IgnoreAbove(256))); } /// @@ -31,14 +34,14 @@ public static TextPropertyDescriptor AddKeywordField(this TextPropertyDesc /// public static TextPropertyDescriptor AddSortField(this TextPropertyDescriptor descriptor, string normalizer = "sort") where T : class { - return descriptor.Fields(f => f.Keyword(s => s.Name(SortFieldName).Normalizer(normalizer).IgnoreAbove(256))); + return descriptor.Fields(f => f.Keyword(SortFieldName, s => s.Normalizer(normalizer).IgnoreAbove(256))); } public static TextPropertyDescriptor AddKeywordAndSortFields(this TextPropertyDescriptor descriptor, string sortNormalizer = "sort", string keywordNormalizer = null) where T : class { return descriptor.Fields(f => f - .Keyword(s => s.Name(KeywordFieldName).Normalizer(keywordNormalizer).IgnoreAbove(256)) - .Keyword(s => s.Name(SortFieldName).Normalizer(sortNormalizer).IgnoreAbove(256))); + .Keyword(KeywordFieldName, s => s.Normalizer(keywordNormalizer).IgnoreAbove(256)) + .Keyword(SortFieldName, s => s.Normalizer(sortNormalizer).IgnoreAbove(256))); } public static TextPropertyDescriptor AddKeywordAndSortFields(this TextPropertyDescriptor descriptor, bool keywordLowercase) where T : class @@ -46,9 +49,9 @@ public static TextPropertyDescriptor AddKeywordAndSortFields(this TextProp return descriptor.AddKeywordAndSortFields(keywordNormalizer: keywordLowercase ? "lowercase" : null); } - public static AnalysisDescriptor AddSortNormalizer(this AnalysisDescriptor descriptor) + public static IndexSettingsAnalysisDescriptor AddSortNormalizer(this IndexSettingsAnalysisDescriptor descriptor) { - return descriptor.Normalizers(d => d.Custom("sort", n => n.Filters("lowercase", "asciifolding"))); + return descriptor.Normalizers(d => d.Custom("sort", n => n.Filter(["lowercase", "asciifolding"]))); } public static string KeywordFieldName = "keyword"; diff --git a/src/Foundatio.Parsers.ElasticQueries/Extensions/QueryNodeExtensions.cs b/src/Foundatio.Parsers.ElasticQueries/Extensions/QueryNodeExtensions.cs index 274108ae..1fe70e5b 100644 --- a/src/Foundatio.Parsers.ElasticQueries/Extensions/QueryNodeExtensions.cs +++ b/src/Foundatio.Parsers.ElasticQueries/Extensions/QueryNodeExtensions.cs @@ -1,27 +1,30 @@ -using System; +using System; using System.Threading.Tasks; +using Elastic.Clients.Elasticsearch; +using Elastic.Clients.Elasticsearch.Aggregations; +using Elastic.Clients.Elasticsearch.Core.Search; +using Elastic.Clients.Elasticsearch.QueryDsl; using Foundatio.Parsers.LuceneQueries.Nodes; -using Nest; namespace Foundatio.Parsers.ElasticQueries.Extensions; public static class QueryNodeExtensions { private const string QueryKey = "@Query"; - public static Task GetQueryAsync(this IQueryNode node, Func> getDefaultValue = null) + public static Task GetQueryAsync(this IQueryNode node, Func> getDefaultValue = null) { if (!node.Data.TryGetValue(QueryKey, out object value)) { if (getDefaultValue == null) - return Task.FromResult(null); + return Task.FromResult(null); return getDefaultValue?.Invoke(); } - return Task.FromResult(value as QueryBase); + return Task.FromResult(value as Query); } - public static void SetQuery(this IQueryNode node, QueryBase container) + public static void SetQuery(this IQueryNode node, Query container) { node.Data[QueryKey] = container; } @@ -42,20 +45,20 @@ public static SourceFilter GetSourceFilter(this IQueryNode node, Func GetAggregationAsync(this IQueryNode node, Func> getDefaultValue = null) + public static Task GetAggregationAsync(this IQueryNode node, Func> getDefaultValue = null) { if (!node.Data.TryGetValue(AggregationKey, out object value)) { if (getDefaultValue == null) - return Task.FromResult(null); + return Task.FromResult(null); return getDefaultValue?.Invoke(); } - return Task.FromResult(value as AggregationBase); + return Task.FromResult(value as Aggregation); } - public static void SetAggregation(this IQueryNode node, AggregationBase aggregation) + public static void SetAggregation(this IQueryNode node, Aggregation aggregation) { node.Data[AggregationKey] = aggregation; } @@ -66,15 +69,15 @@ public static void RemoveAggregation(this IQueryNode node) } private const string SortKey = "@Sort"; - public static IFieldSort GetSort(this IQueryNode node, Func getDefaultValue = null) + public static SortOptions GetSort(this IQueryNode node, Func getDefaultValue = null) { if (!node.Data.TryGetValue(SortKey, out object value)) return getDefaultValue?.Invoke(); - return value as IFieldSort; + return value as SortOptions; } - public static void SetSort(this IQueryNode node, IFieldSort sort) + public static void SetSort(this IQueryNode node, SortOptions sort) { node.Data[SortKey] = sort; } diff --git a/src/Foundatio.Parsers.ElasticQueries/Extensions/QueryVisitorContextExtensions.cs b/src/Foundatio.Parsers.ElasticQueries/Extensions/QueryVisitorContextExtensions.cs index d7c65606..64310b2b 100644 --- a/src/Foundatio.Parsers.ElasticQueries/Extensions/QueryVisitorContextExtensions.cs +++ b/src/Foundatio.Parsers.ElasticQueries/Extensions/QueryVisitorContextExtensions.cs @@ -1,10 +1,10 @@ using System; using System.Threading.Tasks; +using Elastic.Clients.Elasticsearch.QueryDsl; using Foundatio.Parsers.ElasticQueries.Visitors; using Foundatio.Parsers.LuceneQueries.Extensions; using Foundatio.Parsers.LuceneQueries.Nodes; using Foundatio.Parsers.LuceneQueries.Visitors; -using Nest; namespace Foundatio.Parsers.ElasticQueries.Extensions; diff --git a/src/Foundatio.Parsers.ElasticQueries/Extensions/SearchDescriptorExtensions.cs b/src/Foundatio.Parsers.ElasticQueries/Extensions/SearchDescriptorExtensions.cs deleted file mode 100644 index f462b798..00000000 --- a/src/Foundatio.Parsers.ElasticQueries/Extensions/SearchDescriptorExtensions.cs +++ /dev/null @@ -1,33 +0,0 @@ -using System.Collections.Generic; -using Nest; - -namespace Foundatio.Parsers.ElasticQueries.Extensions; - -public static class SearchDescriptorExtensions -{ - public static SearchDescriptor Aggregations(this SearchDescriptor descriptor, AggregationContainer aggregations) where T : class - { - descriptor.Aggregations(f => - { - ((IAggregationContainer)f).Aggregations = aggregations.Aggregations; - return f; - }); - - return descriptor; - } - - public static SearchDescriptor Sort(this SearchDescriptor descriptor, IEnumerable sorts) where T : class - { - var searchRequest = descriptor as ISearchRequest; - - foreach (var sort in sorts) - { - if (searchRequest.Sort == null) - searchRequest.Sort = new List(); - - searchRequest.Sort.Add(sort); - } - - return descriptor; - } -} diff --git a/src/Foundatio.Parsers.ElasticQueries/Foundatio.Parsers.ElasticQueries.csproj b/src/Foundatio.Parsers.ElasticQueries/Foundatio.Parsers.ElasticQueries.csproj index 61260e8d..f09c4c55 100644 --- a/src/Foundatio.Parsers.ElasticQueries/Foundatio.Parsers.ElasticQueries.csproj +++ b/src/Foundatio.Parsers.ElasticQueries/Foundatio.Parsers.ElasticQueries.csproj @@ -1,8 +1,8 @@  + - diff --git a/src/Foundatio.Parsers.ElasticQueries/Visitors/CombineAggregationsVisitor.cs b/src/Foundatio.Parsers.ElasticQueries/Visitors/CombineAggregationsVisitor.cs index ba7985d4..1e5e5001 100644 --- a/src/Foundatio.Parsers.ElasticQueries/Visitors/CombineAggregationsVisitor.cs +++ b/src/Foundatio.Parsers.ElasticQueries/Visitors/CombineAggregationsVisitor.cs @@ -2,11 +2,13 @@ using System.Collections.Generic; using System.Linq; using System.Threading.Tasks; +using Elastic.Clients.Elasticsearch; +using Elastic.Clients.Elasticsearch.Aggregations; +using Elastic.Clients.Elasticsearch.Core.Search; using Foundatio.Parsers.ElasticQueries.Extensions; using Foundatio.Parsers.LuceneQueries.Extensions; using Foundatio.Parsers.LuceneQueries.Nodes; using Foundatio.Parsers.LuceneQueries.Visitors; -using Nest; namespace Foundatio.Parsers.ElasticQueries.Visitors; @@ -20,7 +22,7 @@ public override async Task VisitAsync(GroupNode node, IQueryVisitorContext conte throw new ArgumentException("Context must be of type IElasticQueryVisitorContext", nameof(context)); var container = await GetParentContainerAsync(node, context); - var termsAggregation = container as ITermsAggregation; + var termsAggregation = container as TermsAggregation; foreach (var child in node.Children.OfType()) { @@ -49,11 +51,11 @@ public override async Task VisitAsync(GroupNode node, IQueryVisitorContext conte if (!String.IsNullOrEmpty(termNode.Term) && Int32.TryParse(termNode.UnescapedTerm, out int parsedMinCount)) minCount = parsedMinCount; - termsAggregation.MinimumDocumentCount = minCount; + termsAggregation.MinDocCount = minCount; } } - if (termNode != null && container is ITopHitsAggregation topHitsAggregation) + if (termNode != null && container is TopHitsAggregation topHitsAggregation) { var filter = node.GetSourceFilter(() => new SourceFilter()); if (termNode.Field == "@exclude") @@ -70,10 +72,10 @@ public override async Task VisitAsync(GroupNode node, IQueryVisitorContext conte else filter.Includes.And(termNode.UnescapedTerm); } - topHitsAggregation.Source = filter; + topHitsAggregation.Source = new SourceConfig(filter); } - if (termNode != null && container is IDateHistogramAggregation dateHistogramAggregation) + if (termNode != null && container is DateHistogramAggregation dateHistogramAggregation) { if (termNode.Field == "@missing") { @@ -97,7 +99,7 @@ public override async Task VisitAsync(GroupNode node, IQueryVisitorContext conte if (bucketContainer.Aggregations == null) bucketContainer.Aggregations = new AggregationDictionary(); - bucketContainer.Aggregations[((IAggregation)aggregation).Name] = (AggregationContainer)aggregation; + bucketContainer.Aggregations[((Aggregation)aggregation).Name] = (Aggregation)aggregation; } if (termsAggregation != null && (child.Prefix == "-" || child.Prefix == "+")) @@ -107,8 +109,8 @@ public override async Task VisitAsync(GroupNode node, IQueryVisitorContext conte termsAggregation.Order.Add(new TermsOrder { - Key = ((IAggregation)aggregation).Name, - Order = child.Prefix == "-" ? SortOrder.Descending : SortOrder.Ascending + Key = ((Aggregation)aggregation).Name, + Order = child.Prefix == "-" ? SortOrder.Desc : SortOrder.Asc }); } } @@ -117,9 +119,9 @@ public override async Task VisitAsync(GroupNode node, IQueryVisitorContext conte node.SetAggregation(container); } - private async Task GetParentContainerAsync(IQueryNode node, IQueryVisitorContext context) + private async Task GetParentContainerAsync(IQueryNode node, IQueryVisitorContext context) { - AggregationBase container = null; + Aggregation container = null; var currentNode = node; while (container == null && currentNode != null) { diff --git a/src/Foundatio.Parsers.ElasticQueries/Visitors/CombineQueriesVisitor.cs b/src/Foundatio.Parsers.ElasticQueries/Visitors/CombineQueriesVisitor.cs index 390b5609..7d0c953a 100644 --- a/src/Foundatio.Parsers.ElasticQueries/Visitors/CombineQueriesVisitor.cs +++ b/src/Foundatio.Parsers.ElasticQueries/Visitors/CombineQueriesVisitor.cs @@ -1,11 +1,11 @@ using System; using System.Linq; using System.Threading.Tasks; +using Elastic.Clients.Elasticsearch.QueryDsl; using Foundatio.Parsers.ElasticQueries.Extensions; using Foundatio.Parsers.LuceneQueries.Extensions; using Foundatio.Parsers.LuceneQueries.Nodes; using Foundatio.Parsers.LuceneQueries.Visitors; -using Nest; namespace Foundatio.Parsers.ElasticQueries.Visitors; @@ -24,8 +24,8 @@ public override async Task VisitAsync(GroupNode node, IQueryVisitorContext conte if (context is not IElasticQueryVisitorContext elasticContext) throw new ArgumentException("Context must be of type IElasticQueryVisitorContext", nameof(context)); - QueryBase query = await node.GetQueryAsync(() => node.GetDefaultQueryAsync(context)).ConfigureAwait(false); - QueryBase container = query; + Query query = await node.GetQueryAsync(() => node.GetDefaultQueryAsync(context)).ConfigureAwait(false); + Query container = query; var nested = query as NestedQuery; if (nested != null && node.Parent != null) container = null; diff --git a/src/Foundatio.Parsers.ElasticQueries/Visitors/GeoVisitor.cs b/src/Foundatio.Parsers.ElasticQueries/Visitors/GeoVisitor.cs index e754808b..576529af 100644 --- a/src/Foundatio.Parsers.ElasticQueries/Visitors/GeoVisitor.cs +++ b/src/Foundatio.Parsers.ElasticQueries/Visitors/GeoVisitor.cs @@ -1,10 +1,11 @@ using System; using System.Threading.Tasks; +using Elastic.Clients.Elasticsearch; +using Elastic.Clients.Elasticsearch.QueryDsl; using Foundatio.Parsers.ElasticQueries.Extensions; using Foundatio.Parsers.LuceneQueries; using Foundatio.Parsers.LuceneQueries.Nodes; using Foundatio.Parsers.LuceneQueries.Visitors; -using Nest; namespace Foundatio.Parsers.ElasticQueries.Visitors; @@ -23,7 +24,7 @@ public override async Task VisitAsync(TermNode node, IQueryVisitorContext contex return; string location = _resolveGeoLocation != null ? await _resolveGeoLocation(node.Term).ConfigureAwait(false) ?? node.Term : node.Term; - var query = new GeoDistanceQuery { Field = node.Field, Location = location, Distance = node.Proximity ?? Distance.Miles(10) }; + var query = new GeoDistanceQuery { Field = node.Field, Location = location, Distance = node.Proximity ?? "10mi" }; node.SetQuery(query); } @@ -32,7 +33,7 @@ public override void Visit(TermRangeNode node, IQueryVisitorContext context) if (context is not IElasticQueryVisitorContext elasticContext || !elasticContext.MappingResolver.IsGeoPropertyType(node.Field)) return; - var box = new BoundingBox { TopLeft = node.Min, BottomRight = node.Max }; + var box = GeoBounds.TopLeftBottomRight(new TopLeftBottomRightGeoBounds { TopLeft = node.Min, BottomRight = node.Max }); var query = new GeoBoundingBoxQuery { BoundingBox = box, Field = node.Field }; node.SetQuery(query); } diff --git a/src/Foundatio.Parsers.ElasticQueries/Visitors/GetSortFieldsVisitor.cs b/src/Foundatio.Parsers.ElasticQueries/Visitors/GetSortFieldsVisitor.cs index 6fb2f28b..71b7c47f 100644 --- a/src/Foundatio.Parsers.ElasticQueries/Visitors/GetSortFieldsVisitor.cs +++ b/src/Foundatio.Parsers.ElasticQueries/Visitors/GetSortFieldsVisitor.cs @@ -1,16 +1,16 @@ using System; using System.Collections.Generic; using System.Threading.Tasks; +using Elastic.Clients.Elasticsearch; using Foundatio.Parsers.ElasticQueries.Extensions; using Foundatio.Parsers.LuceneQueries.Nodes; using Foundatio.Parsers.LuceneQueries.Visitors; -using Nest; namespace Foundatio.Parsers.ElasticQueries.Visitors; -public class GetSortFieldsVisitor : QueryNodeVisitorWithResultBase> +public class GetSortFieldsVisitor : QueryNodeVisitorWithResultBase> { - private readonly List _fields = new(); + private readonly List _fields = new(); public override void Visit(TermNode node, IQueryVisitorContext context) { @@ -18,24 +18,24 @@ public override void Visit(TermNode node, IQueryVisitorContext context) return; var sort = node.GetSort(() => node.GetDefaultSort(context)); - if (sort.SortKey == null) + if (sort.AdditionalPropertyName == null) return; _fields.Add(sort); } - public override async Task> AcceptAsync(IQueryNode node, IQueryVisitorContext context) + public override async Task> AcceptAsync(IQueryNode node, IQueryVisitorContext context) { await node.AcceptAsync(this, context).ConfigureAwait(false); return _fields; } - public static Task> RunAsync(IQueryNode node, IQueryVisitorContext context = null) + public static Task> RunAsync(IQueryNode node, IQueryVisitorContext context = null) { return new GetSortFieldsVisitor().AcceptAsync(node, context); } - public static IEnumerable Run(IQueryNode node, IQueryVisitorContext context = null) + public static IEnumerable Run(IQueryNode node, IQueryVisitorContext context = null) { return RunAsync(node, context).GetAwaiter().GetResult(); } diff --git a/src/Foundatio.Parsers.ElasticQueries/Visitors/NestedVisitor.cs b/src/Foundatio.Parsers.ElasticQueries/Visitors/NestedVisitor.cs index 95feaf04..037414ad 100644 --- a/src/Foundatio.Parsers.ElasticQueries/Visitors/NestedVisitor.cs +++ b/src/Foundatio.Parsers.ElasticQueries/Visitors/NestedVisitor.cs @@ -1,10 +1,10 @@ using System; using System.Linq; using System.Threading.Tasks; +using Elastic.Clients.Elasticsearch.QueryDsl; using Foundatio.Parsers.ElasticQueries.Extensions; using Foundatio.Parsers.LuceneQueries.Nodes; using Foundatio.Parsers.LuceneQueries.Visitors; -using Nest; namespace Foundatio.Parsers.ElasticQueries.Visitors; diff --git a/tests/Foundatio.Parsers.ElasticQueries.Tests/AggregationParserTests.cs b/tests/Foundatio.Parsers.ElasticQueries.Tests/AggregationParserTests.cs index d8d9ac89..90997adf 100644 --- a/tests/Foundatio.Parsers.ElasticQueries.Tests/AggregationParserTests.cs +++ b/tests/Foundatio.Parsers.ElasticQueries.Tests/AggregationParserTests.cs @@ -2,13 +2,13 @@ using System.Collections.Generic; using System.Linq; using System.Threading.Tasks; +using Elastic.Clients.Elasticsearch; using Foundatio.Parsers.ElasticQueries.Extensions; using Foundatio.Parsers.ElasticQueries.Visitors; using Foundatio.Parsers.LuceneQueries; using Foundatio.Parsers.LuceneQueries.Extensions; using Foundatio.Parsers.LuceneQueries.Visitors; using Microsoft.Extensions.Logging; -using Nest; using Xunit; using Xunit.Abstractions; @@ -138,7 +138,7 @@ await Client.IndexManyAsync([ _logger.LogInformation("Actual: {Request}", actualRequest); var expectedResponse = Client.Search(d => d.Index(index).Aggregations(a => a - .GeoHash("geogrid_field3", h => h.Field("field3").GeoHashPrecision(GeoHashPrecision.Precision1) + .GeoHash("geogrid_field3", h => h.Field("field3").GeoHashPrecision(1) .Aggregations(a1 => a1.Average("avg_lat", s => s.Script(ss => ss.Source("doc['field3'].lat"))).Average("avg_lon", s => s.Script(ss => ss.Source("doc['field3'].lon"))))) .Terms("terms_field1", t => t.Field("field1.keyword").Meta(m => m.Add("@field_type", "text"))) .Histogram("histogram_field4", h => h.Field("field4").Interval(50).MinimumDocumentCount(0)) @@ -245,7 +245,7 @@ await Client.IndexManyAsync([ _logger.LogInformation("Actual: {Request}", actualRequest); var expectedResponse = Client.Search(d => d.Index(index).Aggregations(a => a - .GeoHash("geogrid_alias3", h => h.Field("field3").GeoHashPrecision(GeoHashPrecision.Precision1) + .GeoHash("geogrid_alias3", h => h.Field("field3").GeoHashPrecision(1) .Aggregations(a1 => a1.Average("avg_lat", s => s.Script(ss => ss.Source("doc['field3'].lat"))).Average("avg_lon", s => s.Script(ss => ss.Source("doc['field3'].lon"))))) .Terms("terms_alias1", t => t.Field("field1.keyword").Meta(m => m.Add("@field_type", "text"))) .Histogram("histogram_alias4", h => h.Field("field4").Interval(50).MinimumDocumentCount(0)) diff --git a/tests/Foundatio.Parsers.ElasticQueries.Tests/CustomVisitorTests.cs b/tests/Foundatio.Parsers.ElasticQueries.Tests/CustomVisitorTests.cs index 4852ff02..171d5b7e 100644 --- a/tests/Foundatio.Parsers.ElasticQueries.Tests/CustomVisitorTests.cs +++ b/tests/Foundatio.Parsers.ElasticQueries.Tests/CustomVisitorTests.cs @@ -6,7 +6,6 @@ using Foundatio.Parsers.LuceneQueries.Nodes; using Foundatio.Parsers.LuceneQueries.Visitors; using Microsoft.Extensions.Logging; -using Nest; using Xunit; using Xunit.Abstractions; diff --git a/tests/Foundatio.Parsers.ElasticQueries.Tests/ElasticMappingResolverTests.cs b/tests/Foundatio.Parsers.ElasticQueries.Tests/ElasticMappingResolverTests.cs index 99812a6b..59ea0f6c 100644 --- a/tests/Foundatio.Parsers.ElasticQueries.Tests/ElasticMappingResolverTests.cs +++ b/tests/Foundatio.Parsers.ElasticQueries.Tests/ElasticMappingResolverTests.cs @@ -1,6 +1,5 @@ using System; using System.Linq.Expressions; -using Nest; using Xunit; using Xunit.Abstractions; @@ -13,7 +12,7 @@ public ElasticMappingResolverTests(ITestOutputHelper output, ElasticsearchFixtur Log.DefaultMinimumLevel = Microsoft.Extensions.Logging.LogLevel.Trace; } - private ITypeMapping MapMyNestedType(TypeMappingDescriptor m) + private TypeMapping MapMyNestedType(TypeMappingDescriptor m) { return m .AutoMap() diff --git a/tests/Foundatio.Parsers.ElasticQueries.Tests/ElasticQueryParserTests.cs b/tests/Foundatio.Parsers.ElasticQueries.Tests/ElasticQueryParserTests.cs index d1967c5e..92b58a8b 100644 --- a/tests/Foundatio.Parsers.ElasticQueries.Tests/ElasticQueryParserTests.cs +++ b/tests/Foundatio.Parsers.ElasticQueries.Tests/ElasticQueryParserTests.cs @@ -2,6 +2,9 @@ using System.Collections.Generic; using System.Linq; using System.Threading.Tasks; +using Elastic.Clients.Elasticsearch; +using Elastic.Clients.Elasticsearch.Mapping; +using Elastic.Clients.Elasticsearch.QueryDsl; using Foundatio.Parsers.ElasticQueries.Extensions; using Foundatio.Parsers.ElasticQueries.Visitors; using Foundatio.Parsers.LuceneQueries; @@ -9,7 +12,6 @@ using Foundatio.Parsers.LuceneQueries.Nodes; using Foundatio.Parsers.LuceneQueries.Visitors; using Microsoft.Extensions.Logging; -using Nest; using Xunit; using Xunit.Abstractions; @@ -1475,7 +1477,7 @@ public async Task GeoRangeQueryProcessor() [Fact] public async Task CanExpandElasticIncludesAsync() { - var client = new ElasticClient(new ConnectionSettings().DisableDirectStreaming().PrettyJson()); + var client = new ElasticsearchClient(new ElasticsearchClientSettings().DisableDirectStreaming().PrettyJson()); var aliases = new FieldMap { { "field", "aliased" }, { "included", "aliasedincluded" } }; var processor = new ElasticQueryParser(c => c.SetLoggerFactory(Log).UseIncludes(GetIncludeAsync).UseFieldMap(aliases)); diff --git a/tests/Foundatio.Parsers.ElasticQueries.Tests/InvertQueryTests.cs b/tests/Foundatio.Parsers.ElasticQueries.Tests/InvertQueryTests.cs index 6037ef46..3447fc49 100644 --- a/tests/Foundatio.Parsers.ElasticQueries.Tests/InvertQueryTests.cs +++ b/tests/Foundatio.Parsers.ElasticQueries.Tests/InvertQueryTests.cs @@ -6,7 +6,6 @@ using Foundatio.Parsers.LuceneQueries.Nodes; using Foundatio.Parsers.LuceneQueries.Visitors; using Microsoft.Extensions.Logging; -using Nest; using Xunit; using Xunit.Abstractions; diff --git a/tests/Foundatio.Parsers.ElasticQueries.Tests/Utility/ElasticsearchTestBase.cs b/tests/Foundatio.Parsers.ElasticQueries.Tests/Utility/ElasticsearchTestBase.cs index 9792421a..00589aa8 100644 --- a/tests/Foundatio.Parsers.ElasticQueries.Tests/Utility/ElasticsearchTestBase.cs +++ b/tests/Foundatio.Parsers.ElasticQueries.Tests/Utility/ElasticsearchTestBase.cs @@ -2,10 +2,12 @@ using System.Collections.Generic; using System.Threading; using System.Threading.Tasks; +using Elastic.Clients.Elasticsearch; +using Elastic.Clients.Elasticsearch.IndexManagement; +using Elastic.Clients.Elasticsearch.Mapping; using Foundatio.Parsers.ElasticQueries.Extensions; using Foundatio.Xunit; using Microsoft.Extensions.Logging; -using Nest; using Xunit; using Xunit.Abstractions; @@ -26,14 +28,14 @@ public ElasticsearchTestBase(ITestOutputHelper output, T fixture) : base(output) _fixture.Log = Log; } - protected IElasticClient Client => _fixture.Client; + protected ElasticsearchClient Client => _fixture.Client; - protected void CreateNamedIndex(string index, Func, ITypeMapping> configureMappings = null, Func> configureIndex = null) where TModel : class + protected void CreateNamedIndex(string index, Func, TypeMapping> configureMappings = null, Func> configureIndex = null) where TModel : class { _fixture.CreateNamedIndex(index, configureMappings, configureIndex); } - protected string CreateRandomIndex(Func, ITypeMapping> configureMappings = null, Func> configureIndex = null) where TModel : class + protected string CreateRandomIndex(Func, TypeMapping> configureMappings = null, Func> configureIndex = null) where TModel : class { return _fixture.CreateRandomIndex(configureMappings, configureIndex); } @@ -65,23 +67,23 @@ public class ElasticsearchFixture : IAsyncLifetime private readonly List _createdIndexes = new(); private static bool _elaticsearchReady; protected readonly ILogger _logger; - private readonly Lazy _client; + private readonly Lazy _client; public ElasticsearchFixture() { - _client = new Lazy(() => GetClient(ConfigureConnectionSettings)); + _client = new Lazy(() => GetClient(ConfigureConnectionSettings)); } public TestLogger Log { get; set; } - public IElasticClient Client => _client.Value; + public ElasticsearchClient Client => _client.Value; - protected IElasticClient GetClient(Action configure = null) + protected ElasticsearchClient GetClient(Action configure = null) { string elasticsearchUrl = Environment.GetEnvironmentVariable("ELASTICSEARCH_URL") ?? "http://localhost:9200"; - var settings = new ConnectionSettings(new Uri(elasticsearchUrl)); + var settings = new ElasticsearchClientSettings(new Uri(elasticsearchUrl)); configure?.Invoke(settings); - var client = new ElasticClient(settings.DisableDirectStreaming().PrettyJson()); + var client = new ElasticsearchClient(settings.DisableDirectStreaming().PrettyJson()); if (!_elaticsearchReady) { @@ -94,9 +96,9 @@ protected IElasticClient GetClient(Action configure = null) return client; } - protected virtual void ConfigureConnectionSettings(ConnectionSettings settings) { } + protected virtual void ConfigureConnectionSettings(ElasticsearchClientSettings settings) { } - public void CreateNamedIndex(string index, Func, ITypeMapping> configureMappings = null, Func> configureIndex = null) where T : class + public void CreateNamedIndex(string index, Func, TypeMapping> configureMappings = null, Func> configureIndex = null) where T : class { if (configureMappings == null) configureMappings = m => m.AutoMap().Dynamic(); @@ -104,10 +106,10 @@ public void CreateNamedIndex(string index, Func, ITy configureIndex = i => i.NumberOfReplicas(0).Analysis(a => a.AddSortNormalizer()); CreateIndex(index, i => i.Settings(configureIndex).Map(configureMappings)); - Client.ConnectionSettings.DefaultIndices[typeof(T)] = index; + Client.ElasticsearchClientSettings.DefaultIndices[typeof(T)] = index; } - public string CreateRandomIndex(Func, ITypeMapping> configureMappings = null, Func> configureIndex = null) where T : class + public string CreateRandomIndex(Func, TypeMapping> configureMappings = null, Func> configureIndex = null) where T : class { string index = "test_" + Guid.NewGuid().ToString("N"); if (configureMappings == null) @@ -116,7 +118,7 @@ public string CreateRandomIndex(Func, ITypeMapping> configureIndex = i => i.NumberOfReplicas(0).Analysis(a => a.AddSortNormalizer()); CreateIndex(index, i => i.Settings(configureIndex).Map(configureMappings)); - Client.ConnectionSettings.DefaultIndices[typeof(T)] = index; + Client.ElasticsearchClientSettings.DefaultIndices[typeof(T)] = index; return index; } @@ -135,7 +137,7 @@ public CreateIndexResponse CreateIndex(IndexName index, Func 0) client.Indices.Delete(Indices.Index(_createdIndexes)); diff --git a/tests/Foundatio.Parsers.LuceneQueries.Tests/Utility/LoggingTracer.cs b/tests/Foundatio.Parsers.LuceneQueries.Tests/Utility/LoggingTracer.cs index 259e28b9..579d15a6 100644 --- a/tests/Foundatio.Parsers.LuceneQueries.Tests/Utility/LoggingTracer.cs +++ b/tests/Foundatio.Parsers.LuceneQueries.Tests/Utility/LoggingTracer.cs @@ -63,7 +63,7 @@ public void TraceRuleEnter(string ruleName, Cursor cursor) ruleStats.Locations.TryGetValue(key, out int count); ruleStats.Locations[key] = count + 1; - _logger.LogInformation($"{GetIndent()}Start '{ruleName}' at ({cursor.Line},{cursor.Column}) with state key {cursor.StateKey}"); + _logger.LogInformation("{Indent}Start \'{RuleName}\' at ({CursorLine},{CursorColumn}) with state key {CursorStateKey}", GetIndent(), ruleName, cursor.Line, cursor.Column, cursor.StateKey); _indentLevel++; } From aa8e564dd949249ff4c5fc1032c8e98376ac7fef Mon Sep 17 00:00:00 2001 From: Blake Niemyjski Date: Thu, 12 Sep 2024 14:30:10 -0500 Subject: [PATCH 03/19] Updated elastic --- docker-compose.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 3434dfab..43325b83 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,6 +1,6 @@ services: elasticsearch: - image: docker.elastic.co/elasticsearch/elasticsearch:8.15.0 + image: docker.elastic.co/elasticsearch/elasticsearch:8.15.1 environment: discovery.type: single-node xpack.security.enabled: 'false' @@ -14,7 +14,7 @@ services: kibana: depends_on: - elasticsearch - image: docker.elastic.co/kibana/kibana:8.15.0 + image: docker.elastic.co/kibana/kibana:8.15.1 ports: - 5601:5601 networks: From 3eec87f5f7278563f20a39924918af90cd6a3f6a Mon Sep 17 00:00:00 2001 From: Blake Niemyjski Date: Thu, 12 Sep 2024 14:32:27 -0500 Subject: [PATCH 04/19] WIP - Added polyfill for what will be added in next client. --- .../ElasticMappingResolver.cs | 39 +++++++++++-------- .../Extensions/ElasticExtensions.cs | 7 ++++ .../Visitors/CombineQueriesVisitor.cs | 3 +- .../Visitors/GetSortFieldsVisitor.cs | 2 +- 4 files changed, 31 insertions(+), 20 deletions(-) diff --git a/src/Foundatio.Parsers.ElasticQueries/ElasticMappingResolver.cs b/src/Foundatio.Parsers.ElasticQueries/ElasticMappingResolver.cs index 5bdf85d2..d3a9998b 100644 --- a/src/Foundatio.Parsers.ElasticQueries/ElasticMappingResolver.cs +++ b/src/Foundatio.Parsers.ElasticQueries/ElasticMappingResolver.cs @@ -92,7 +92,7 @@ public FieldMapping GetMapping(string field, bool followAlias = false) if (currentProperties != null) fieldMapping = ((IDictionary)currentProperties).Values.FirstOrDefault(m => { - string propertyName = _inferrer.PropertyName(m?.Name); + string propertyName = _inferrer.PropertyName(m?.TryGetName()); return propertyName != null && propertyName.Equals(fieldPart, StringComparison.OrdinalIgnoreCase); }); @@ -125,13 +125,14 @@ public FieldMapping GetMapping(string field, bool followAlias = false) } // coded properties sometimes have null Name properties - if (fieldMapping.Name == null && fieldMapping is IPropertyWithClrOrigin clrOrigin && clrOrigin.ClrOrigin != null) - fieldMapping.Name = new PropertyName(clrOrigin.ClrOrigin); + string name = fieldMapping.TryGetName(); + if (name == null && fieldMapping is IPropertyWithClrOrigin clrOrigin && clrOrigin.ClrOrigin != null) + name = new PropertyName(clrOrigin.ClrOrigin); if (depth == 0) - resolvedFieldName += _inferrer.PropertyName(fieldMapping.Name); + resolvedFieldName += _inferrer.PropertyName(name); else - resolvedFieldName += "." + _inferrer.PropertyName(fieldMapping.Name); + resolvedFieldName += "." + _inferrer.PropertyName(name); if (depth == fieldParts.Length - 1) { @@ -415,7 +416,7 @@ private Properties MergeProperties(Properties codeProperties, Properties serverP mergedCodeProperties[kvp.Key] = new FieldAliasProperty { - Meta = aliasProperty.Meta, + //LocalMetadata = aliasProperty.LocalMetadata, Path = _inferrer?.Field(aliasProperty.Path) ?? aliasProperty.Path, Name = aliasProperty.Name }; @@ -431,19 +432,23 @@ private Properties MergeProperties(Properties codeProperties, Properties serverP foreach (var serverProperty in serverProperties) { var merged = serverProperty.Value; - if (mergedCodeProperties.TryGetProperty(serverProperty.Key, out var codeProperty)) - merged.LocalMetadata = codeProperty.LocalMetadata; + // if (mergedCodeProperties.TryGetProperty(serverProperty.Key, out var codeProperty)) + // merged.LocalMetadata = codeProperty.LocalMetadata; - switch (merged) + if (mergedCodeProperties.TryGetProperty(serverProperty.Key, out var codeProperty)) { - case ObjectProperty objectProperty: - var codeObjectProperty = codeProperty as ObjectProperty; - objectProperty.Properties = MergeProperties(codeObjectProperty?.Properties, objectProperty.Properties); - break; - case TextProperty textProperty: - var codeTextProperty = codeProperty as TextProperty; - textProperty.Fields = MergeProperties(codeTextProperty?.Fields, textProperty.Fields); - break; + switch (merged) + { + case ObjectProperty objectProperty: + var codeObjectProperty = codeProperty as ObjectProperty; + objectProperty.Properties = + MergeProperties(codeObjectProperty?.Properties, objectProperty.Properties); + break; + case TextProperty textProperty: + var codeTextProperty = codeProperty as TextProperty; + textProperty.Fields = MergeProperties(codeTextProperty?.Fields, textProperty.Fields); + break; + } } properties.Add(serverProperty.Key, merged); diff --git a/src/Foundatio.Parsers.ElasticQueries/Extensions/ElasticExtensions.cs b/src/Foundatio.Parsers.ElasticQueries/Extensions/ElasticExtensions.cs index aa442120..643d4281 100644 --- a/src/Foundatio.Parsers.ElasticQueries/Extensions/ElasticExtensions.cs +++ b/src/Foundatio.Parsers.ElasticQueries/Extensions/ElasticExtensions.cs @@ -8,6 +8,7 @@ using System.Threading.Tasks; using Elastic.Clients.Elasticsearch; using Elastic.Clients.Elasticsearch.Aggregations; +using Elastic.Clients.Elasticsearch.Mapping; using Elastic.Transport.Products.Elasticsearch; using Microsoft.Extensions.Logging; @@ -15,6 +16,12 @@ namespace Foundatio.Parsers.ElasticQueries.Extensions; public static class ElasticExtensions { + public static string TryGetName(this IProperty property) + { + // TODO until: https://github.com/elastic/elasticsearch-net/issues/8336 + return null; + } + public static TermsInclude AddValue(this TermsInclude include, string value) { if (include?.Values == null) diff --git a/src/Foundatio.Parsers.ElasticQueries/Visitors/CombineQueriesVisitor.cs b/src/Foundatio.Parsers.ElasticQueries/Visitors/CombineQueriesVisitor.cs index 7d0c953a..12de4c9c 100644 --- a/src/Foundatio.Parsers.ElasticQueries/Visitors/CombineQueriesVisitor.cs +++ b/src/Foundatio.Parsers.ElasticQueries/Visitors/CombineQueriesVisitor.cs @@ -26,8 +26,7 @@ public override async Task VisitAsync(GroupNode node, IQueryVisitorContext conte Query query = await node.GetQueryAsync(() => node.GetDefaultQueryAsync(context)).ConfigureAwait(false); Query container = query; - var nested = query as NestedQuery; - if (nested != null && node.Parent != null) + if (query.TryGet(out var nested) && node.Parent != null) container = null; foreach (var child in node.Children.OfType()) diff --git a/src/Foundatio.Parsers.ElasticQueries/Visitors/GetSortFieldsVisitor.cs b/src/Foundatio.Parsers.ElasticQueries/Visitors/GetSortFieldsVisitor.cs index 71b7c47f..4bebe0aa 100644 --- a/src/Foundatio.Parsers.ElasticQueries/Visitors/GetSortFieldsVisitor.cs +++ b/src/Foundatio.Parsers.ElasticQueries/Visitors/GetSortFieldsVisitor.cs @@ -18,7 +18,7 @@ public override void Visit(TermNode node, IQueryVisitorContext context) return; var sort = node.GetSort(() => node.GetDefaultSort(context)); - if (sort.AdditionalPropertyName == null) + if (sort.SortKey == null) return; _fields.Add(sort); From 5dd521ebe5318a744bdb050a8d9759422c719b29 Mon Sep 17 00:00:00 2001 From: Blake Niemyjski Date: Thu, 12 Sep 2024 14:33:13 -0500 Subject: [PATCH 05/19] Added aggregation container that can be used to map to a descriptor --- .../AggregationMap.cs | 11 ++ .../ElasticQueryParser.cs | 4 +- .../DefaultAggregationNodeExtensions.cs | 144 +++++++++++------- .../Extensions/QueryNodeExtensions.cs | 8 +- .../Visitors/CombineAggregationsVisitor.cs | 16 +- 5 files changed, 110 insertions(+), 73 deletions(-) create mode 100644 src/Foundatio.Parsers.ElasticQueries/AggregationMap.cs diff --git a/src/Foundatio.Parsers.ElasticQueries/AggregationMap.cs b/src/Foundatio.Parsers.ElasticQueries/AggregationMap.cs new file mode 100644 index 00000000..e619bf98 --- /dev/null +++ b/src/Foundatio.Parsers.ElasticQueries/AggregationMap.cs @@ -0,0 +1,11 @@ +using System.Collections.Generic; + +namespace Foundatio.Parsers.ElasticQueries; + +public record AggregationMap(string Name, object Value) +{ + public string Name { get; set; } = Name; + public object Value { get; set; } = Value; + public List Aggregations { get; } = new(); + public Dictionary Meta { get; } = new(); +} diff --git a/src/Foundatio.Parsers.ElasticQueries/ElasticQueryParser.cs b/src/Foundatio.Parsers.ElasticQueries/ElasticQueryParser.cs index 4c410f4a..17cf4071 100644 --- a/src/Foundatio.Parsers.ElasticQueries/ElasticQueryParser.cs +++ b/src/Foundatio.Parsers.ElasticQueries/ElasticQueryParser.cs @@ -197,7 +197,7 @@ public async Task ValidateAggregationsAsync(string query, return context.GetValidationResult(); } - public async Task BuildAggregationsAsync(string aggregations, IElasticQueryVisitorContext context = null) + public async Task BuildAggregationsAsync(string aggregations, IElasticQueryVisitorContext context = null) { context ??= new ElasticQueryVisitorContext(); context.QueryType = QueryTypes.Aggregation; @@ -209,7 +209,7 @@ public async Task BuildAggregationsAsync(string aggregations, IElas } #pragma warning disable IDE0060 // Remove unused parameter - public async Task BuildAggregationsAsync(IQueryNode aggregations, IElasticQueryVisitorContext context = null) + public async Task BuildAggregationsAsync(IQueryNode aggregations, IElasticQueryVisitorContext context = null) { if (aggregations == null) return null; diff --git a/src/Foundatio.Parsers.ElasticQueries/Extensions/DefaultAggregationNodeExtensions.cs b/src/Foundatio.Parsers.ElasticQueries/Extensions/DefaultAggregationNodeExtensions.cs index 6450cb66..0b268c3c 100644 --- a/src/Foundatio.Parsers.ElasticQueries/Extensions/DefaultAggregationNodeExtensions.cs +++ b/src/Foundatio.Parsers.ElasticQueries/Extensions/DefaultAggregationNodeExtensions.cs @@ -16,7 +16,7 @@ public static class DefaultAggregationNodeExtensions // NOTE: We may want to read this dynamically from server settings. public const int MAX_BUCKET_SIZE = 10000; - public static Task GetDefaultAggregationAsync(this IQueryNode node, IQueryVisitorContext context) + public static Task GetDefaultAggregationAsync(this IQueryNode node, IQueryVisitorContext context) { if (node is GroupNode groupNode) return groupNode.GetDefaultAggregationAsync(context); @@ -27,7 +27,7 @@ public static Task GetDefaultAggregationAsync(this IQueryNode node, return null; } - public static async Task GetDefaultAggregationAsync(this GroupNode node, IQueryVisitorContext context) + public static async Task GetDefaultAggregationAsync(this GroupNode node, IQueryVisitorContext context) { if (context is not IElasticQueryVisitorContext elasticContext) throw new ArgumentException("Context must be of type IElasticQueryVisitorContext", nameof(context)); @@ -42,10 +42,10 @@ public static async Task GetDefaultAggregationAsync(this GroupNode switch (node.GetOperationType()) { case AggregationType.DateHistogram: - return GetDateHistogramAggregation("date_" + originalField, field, node.UnescapedProximity, node.UnescapedBoost ?? node.GetTimeZone(await elasticContext.GetTimeZoneAsync()), context); + return GetDateHistogramAggregation($"date_{originalField}", field, node.UnescapedProximity, node.UnescapedBoost ?? node.GetTimeZone(await elasticContext.GetTimeZoneAsync()), context); case AggregationType.Histogram: - return GetHistogramAggregation("histogram_" + originalField, field, node.UnescapedProximity, node.UnescapedBoost, context); + return GetHistogramAggregation($"histogram_{originalField}", field, node.UnescapedProximity, node.UnescapedBoost, context); case AggregationType.GeoHashGrid: var precision = new GeohashPrecision(1); @@ -57,41 +57,39 @@ public static async Task GetDefaultAggregationAsync(this GroupNode precision = new GeohashPrecision(parsedPrecision); } - return new Aggregation("geogrid_" + originalField, new GeohashGridAggregation + return new AggregationMap($"geogrid_{originalField}", new GeohashGridAggregation { Field = field, Precision = precision }) { - Field = field, - Precision = precision, - Aggregations = new AverageAggregation("avg_lat", null) - { - Script = new Script { Source = $"doc['{node.Field}'].lat" } - } && new AverageAggregation("avg_lon", null) + Aggregations = { - Script = new Script { Source = $"doc['{node.Field}'].lon" } + new AggregationMap("avg_lat", new AverageAggregation { Script = new Script { Source = $"doc['{node.Field}'].lat" } }), + new AggregationMap("avg_lon", new AverageAggregation { Script = new Script { Source = $"doc['{node.Field}'].lon" } }) } - }); + }; case AggregationType.Terms: - var agg = new TermsAggregation("terms_" + originalField) + var termsAggregation = new TermsAggregation { Field = field, Size = node.GetProximityAsInt32(), - MinDocCount = node.GetBoostAsInt32(), - Meta = new Dictionary { { "@field_type", property?.Type } } + MinDocCount = node.GetBoostAsInt32() }; - if (agg.Size.HasValue && (agg.Size * 1.5 + 10) > MAX_BUCKET_SIZE) - agg.ShardSize = Math.Max((int)agg.Size, MAX_BUCKET_SIZE); + if (termsAggregation.Size.HasValue && (termsAggregation.Size * 1.5 + 10) > MAX_BUCKET_SIZE) + termsAggregation.ShardSize = Math.Max((int)termsAggregation.Size, MAX_BUCKET_SIZE); - return agg; + return new AggregationMap($"terms_{originalField}", termsAggregation) + { + Meta = { { "@field_type", property?.Type } } + }; case AggregationType.TopHits: - return new TopHitsAggregation("tophits") { Size = node.GetProximityAsInt32() }; + return new AggregationMap("tophits", new TopHitsAggregation { Size = node.GetProximityAsInt32() }); } return null; } - public static async Task GetDefaultAggregationAsync(this TermNode node, IQueryVisitorContext context) + public static async Task GetDefaultAggregationAsync(this TermNode node, IQueryVisitorContext context) { if (context is not IElasticQueryVisitorContext elasticContext) throw new ArgumentException("Context must be of type IElasticQueryVisitorContext", nameof(context)); @@ -104,40 +102,63 @@ public static async Task GetDefaultAggregationAsync(this TermNode n switch (node.GetOperationType()) { case AggregationType.Min: - return new MinAggregation("min_" + originalField, aggField) { Missing = node.GetProximityAsDouble(), Meta = new Dictionary { { "@field_type", property?.Type }, { "@timezone", timezone } } }; + return new AggregationMap($"min_{originalField}", new MinAggregation { Field = aggField, Missing = node.GetProximityAsDouble() }) + { + Meta = { { "@field_type", property?.Type }, { "@timezone", timezone } } + }; case AggregationType.Max: - return new MaxAggregation("max_" + originalField, aggField) { Missing = node.GetProximityAsDouble(), Meta = new Dictionary { { "@field_type", property?.Type }, { "@timezone", timezone } } }; + return new AggregationMap($"max_{originalField}", new MaxAggregation { Field = aggField, Missing = node.GetProximityAsDouble() }) + { + Meta = { { "@field_type", property?.Type }, { "@timezone", timezone } } + }; case AggregationType.Avg: - return new AverageAggregation("avg_" + originalField, aggField) { Missing = node.GetProximityAsDouble(), Meta = new Dictionary { { "@field_type", property?.Type } } }; + return new AggregationMap($"avg_{originalField}", new AverageAggregation { Field = aggField, Missing = node.GetProximityAsDouble() }) + { + Meta = { { "@field_type", property?.Type } } + }; case AggregationType.Sum: - return new SumAggregation("sum_" + originalField, aggField) { Missing = node.GetProximityAsDouble(), Meta = new Dictionary { { "@field_type", property?.Type } } }; + return new AggregationMap($"sum_{originalField}", new SumAggregation { Field = aggField, Missing = node.GetProximityAsDouble() }) + { + Meta = { { "@field_type", property?.Type } } + }; case AggregationType.Stats: - return new StatsAggregation("stats_" + originalField, aggField) { Missing = node.GetProximityAsDouble(), Meta = new Dictionary { { "@field_type", property?.Type } } }; + return new AggregationMap($"stats_{originalField}", new StatsAggregation { Field = aggField, Missing = node.GetProximityAsDouble() }) + { + Meta = { { "@field_type", property?.Type } } + }; case AggregationType.ExtendedStats: - return new ExtendedStatsAggregation("exstats_" + originalField, aggField) { Missing = node.GetProximityAsDouble(), Meta = new Dictionary { { "@field_type", property?.Type } } }; + return new AggregationMap($"exstats_{originalField}", new ExtendedStatsAggregation { Field = aggField, Missing = node.GetProximityAsDouble() }) + { + Meta = { { "@field_type", property?.Type } } + }; case AggregationType.Cardinality: - return new CardinalityAggregation("cardinality_" + originalField, aggField) { Missing = node.GetProximityAsDouble(), PrecisionThreshold = node.GetBoostAsInt32() }; + return new AggregationMap($"cardinality_{originalField}", new CardinalityAggregation + { + Field = aggField, + Missing = node.GetProximityAsDouble(), + PrecisionThreshold = node.GetBoostAsInt32() + }); case AggregationType.TopHits: - return new TopHitsAggregation("tophits") { Size = node.GetProximityAsInt32() }; + return new AggregationMap("tophits", new TopHitsAggregation { Size = node.GetProximityAsInt32() }); case AggregationType.Missing: - return new MissingAggregation("missing_" + originalField) { Field = aggField }; + return new AggregationMap($"missing_{originalField}", new MissingAggregation { Field = aggField }); case AggregationType.DateHistogram: - return GetDateHistogramAggregation("date_" + originalField, aggField, node.UnescapedProximity, node.UnescapedBoost, context); + return GetDateHistogramAggregation($"date_{originalField}", aggField, node.UnescapedProximity, node.UnescapedBoost, context); case AggregationType.Histogram: - return GetHistogramAggregation("histogram_" + originalField, aggField, node.UnescapedProximity, node.UnescapedBoost, context); + return GetHistogramAggregation($"histogram_{originalField}", aggField, node.UnescapedProximity, node.UnescapedBoost, context); case AggregationType.Percentiles: - return GetPercentilesAggregation("percentiles_" + originalField, aggField, node.UnescapedProximity, node.UnescapedBoost, context); + return GetPercentilesAggregation($"percentiles_{originalField}", aggField, node.UnescapedProximity, node.UnescapedBoost, context); case AggregationType.GeoHashGrid: var precision = new GeohashPrecision(1); @@ -149,38 +170,36 @@ public static async Task GetDefaultAggregationAsync(this TermNode n precision = new GeohashPrecision(parsedPrecision); } - return new GeohashGridAggregation("geogrid_" + originalField) + return new AggregationMap($"geogrid_{originalField}", new GeohashGridAggregation { Field = aggField, Precision = precision }) { - Field = aggField, - Precision = precision, - Aggregations = new AverageAggregation("avg_lat", null) + Aggregations = { - Script = new Script { Source = $"doc['{node.Field}'].lat" } - } && new AverageAggregation("avg_lon", null) - { - Script = new Script { Source = $"doc['{node.Field}'].lon" } + new AggregationMap("avg_lat", new AverageAggregation { Script = new Script { Source = $"doc['{node.Field}'].lat" } }), + new AggregationMap("avg_lon", new AverageAggregation { Script = new Script { Source = $"doc['{node.Field}'].lon" } }) } }; case AggregationType.Terms: - var agg = new TermsAggregation("terms_" + originalField) + var termsAggregation = new TermsAggregation { Field = aggField, Size = node.GetProximityAsInt32(), - MinDocCount = node.GetBoostAsInt32(), - Meta = new Dictionary { { "@field_type", property?.Type } } + MinDocCount = node.GetBoostAsInt32() }; - if (agg.Size.HasValue && (agg.Size * 1.5 + 10) > MAX_BUCKET_SIZE) - agg.ShardSize = Math.Max((int)agg.Size, MAX_BUCKET_SIZE); + if (termsAggregation.Size.HasValue && (termsAggregation.Size * 1.5 + 10) > MAX_BUCKET_SIZE) + termsAggregation.ShardSize = Math.Max((int)termsAggregation.Size, MAX_BUCKET_SIZE); - return agg; + return new AggregationMap($"terms_{originalField}", termsAggregation) + { + Meta = { { "@field_type", property?.Type } } + }; } return null; } - private static Aggregation GetPercentilesAggregation(string originalField, string field, string proximity, string boost, IQueryVisitorContext context) + private static AggregationMap GetPercentilesAggregation(string originalField, string field, string proximity, string boost, IQueryVisitorContext context) { List percents = null; if (!String.IsNullOrWhiteSpace(proximity)) @@ -194,27 +213,28 @@ private static Aggregation GetPercentilesAggregation(string originalField, strin } } - return new PercentilesAggregation(originalField, field) + return new AggregationMap(originalField, new PercentilesAggregation { + Field = field, Percents = percents - }; + }); } - private static Aggregation GetHistogramAggregation(string originalField, string field, string proximity, string boost, IQueryVisitorContext context) + private static AggregationMap GetHistogramAggregation(string originalField, string field, string proximity, string boost, IQueryVisitorContext context) { double interval = 50; if (Double.TryParse(proximity, out double prox)) interval = prox; - return new HistogramAggregation(originalField) + return new AggregationMap(originalField, new HistogramAggregation { Field = field, MinDocCount = 0, Interval = interval - }; + }); } - private static Aggregation GetDateHistogramAggregation(string originalField, string field, string proximity, string boost, IQueryVisitorContext context) + private static AggregationMap GetDateHistogramAggregation(string originalField, string field, string proximity, string boost, IQueryVisitorContext context) { // NOTE: StartDate and EndDate are set in the Repositories QueryBuilderContext. var start = context.GetDate("StartDate"); @@ -224,18 +244,24 @@ private static Aggregation GetDateHistogramAggregation(string originalField, str var interval = GetInterval(proximity, start, end); string timezone = TryConvertTimeUnitToUtcOffset(boost); - var agg = new DateHistogramAggregation(originalField) + var agg = new DateHistogramAggregation { Field = field, MinDocCount = 0, Format = "date_optional_time", TimeZone = timezone, - Meta = !String.IsNullOrEmpty(boost) ? new Dictionary { { "@timezone", boost } } : null, ExtendedBounds = bounds }; interval.Match(d => agg.CalendarInterval = d, f => agg.FixedInterval = f); - return agg; + + var aggregationMap = new AggregationMap(originalField, agg); + if (!String.IsNullOrEmpty(boost)) + { + aggregationMap.Meta.Add("@timezone", boost); + } + + return aggregationMap; } private static string TryConvertTimeUnitToUtcOffset(string boost) @@ -253,9 +279,9 @@ private static string TryConvertTimeUnitToUtcOffset(string boost) return null; if (timezoneOffset.Value < TimeSpan.Zero) - return "-" + timezoneOffset.Value.ToString("hh\\:mm"); + return $"-{timezoneOffset.Value:hh\\:mm}"; - return "+" + timezoneOffset.Value.ToString("hh\\:mm"); + return $"+{timezoneOffset.Value:hh\\:mm}"; } private static Union GetInterval(string proximity, DateTime? start, DateTime? end) diff --git a/src/Foundatio.Parsers.ElasticQueries/Extensions/QueryNodeExtensions.cs b/src/Foundatio.Parsers.ElasticQueries/Extensions/QueryNodeExtensions.cs index 1fe70e5b..e043af71 100644 --- a/src/Foundatio.Parsers.ElasticQueries/Extensions/QueryNodeExtensions.cs +++ b/src/Foundatio.Parsers.ElasticQueries/Extensions/QueryNodeExtensions.cs @@ -45,20 +45,20 @@ public static SourceFilter GetSourceFilter(this IQueryNode node, Func GetAggregationAsync(this IQueryNode node, Func> getDefaultValue = null) + public static Task GetAggregationAsync(this IQueryNode node, Func> getDefaultValue = null) { if (!node.Data.TryGetValue(AggregationKey, out object value)) { if (getDefaultValue == null) - return Task.FromResult(null); + return Task.FromResult(null); return getDefaultValue?.Invoke(); } - return Task.FromResult(value as Aggregation); + return Task.FromResult(value as AggregationMap); } - public static void SetAggregation(this IQueryNode node, Aggregation aggregation) + public static void SetAggregation(this IQueryNode node, AggregationMap aggregation) { node.Data[AggregationKey] = aggregation; } diff --git a/src/Foundatio.Parsers.ElasticQueries/Visitors/CombineAggregationsVisitor.cs b/src/Foundatio.Parsers.ElasticQueries/Visitors/CombineAggregationsVisitor.cs index 1e5e5001..2eabb8e1 100644 --- a/src/Foundatio.Parsers.ElasticQueries/Visitors/CombineAggregationsVisitor.cs +++ b/src/Foundatio.Parsers.ElasticQueries/Visitors/CombineAggregationsVisitor.cs @@ -22,7 +22,7 @@ public override async Task VisitAsync(GroupNode node, IQueryVisitorContext conte throw new ArgumentException("Context must be of type IElasticQueryVisitorContext", nameof(context)); var container = await GetParentContainerAsync(node, context); - var termsAggregation = container as TermsAggregation; + var termsAggregation = container.Value as TermsAggregation; foreach (var child in node.Children.OfType()) { @@ -55,7 +55,7 @@ public override async Task VisitAsync(GroupNode node, IQueryVisitorContext conte } } - if (termNode != null && container is TopHitsAggregation topHitsAggregation) + if (termNode != null && container.Value is TopHitsAggregation topHitsAggregation) { var filter = node.GetSourceFilter(() => new SourceFilter()); if (termNode.Field == "@exclude") @@ -75,7 +75,7 @@ public override async Task VisitAsync(GroupNode node, IQueryVisitorContext conte topHitsAggregation.Source = new SourceConfig(filter); } - if (termNode != null && container is DateHistogramAggregation dateHistogramAggregation) + if (termNode != null && container.Value is DateHistogramAggregation dateHistogramAggregation) { if (termNode.Field == "@missing") { @@ -94,12 +94,12 @@ public override async Task VisitAsync(GroupNode node, IQueryVisitorContext conte continue; } - if (container is BucketAggregationBase bucketContainer) + if (container.Value is BucketAggregationBase bucketContainer) { if (bucketContainer.Aggregations == null) bucketContainer.Aggregations = new AggregationDictionary(); - bucketContainer.Aggregations[((Aggregation)aggregation).Name] = (Aggregation)aggregation; + bucketContainer.Aggregations[aggregation.Name] = aggregation.Value; } if (termsAggregation != null && (child.Prefix == "-" || child.Prefix == "+")) @@ -119,9 +119,9 @@ public override async Task VisitAsync(GroupNode node, IQueryVisitorContext conte node.SetAggregation(container); } - private async Task GetParentContainerAsync(IQueryNode node, IQueryVisitorContext context) + private async Task GetParentContainerAsync(IQueryNode node, IQueryVisitorContext context) { - Aggregation container = null; + AggregationMap container = null; var currentNode = node; while (container == null && currentNode != null) { @@ -143,7 +143,7 @@ private async Task GetParentContainerAsync(IQueryNode node, IQueryV if (container == null) { - container = new ChildrenAggregation(null, null); + container = new AggregationMap(null, null); currentNode.SetAggregation(container); } From 50c0906cdd4d6ad87c8fc063e84ea47805c35de1 Mon Sep 17 00:00:00 2001 From: Blake Niemyjski Date: Thu, 12 Sep 2024 15:04:50 -0500 Subject: [PATCH 06/19] More fixes --- .../ElasticMappingResolver.cs | 5 +- .../ElasticQueryParserConfiguration.cs | 4 +- .../Extensions/ElasticExtensions.cs | 56 +++++++++++++++++++ .../Visitors/CombineAggregationsVisitor.cs | 12 +--- 4 files changed, 64 insertions(+), 13 deletions(-) diff --git a/src/Foundatio.Parsers.ElasticQueries/ElasticMappingResolver.cs b/src/Foundatio.Parsers.ElasticQueries/ElasticMappingResolver.cs index d3a9998b..83354a8c 100644 --- a/src/Foundatio.Parsers.ElasticQueries/ElasticMappingResolver.cs +++ b/src/Foundatio.Parsers.ElasticQueries/ElasticMappingResolver.cs @@ -234,10 +234,11 @@ public string GetNonAnalyzedFieldName(string field, string preferredSubField = n return field; var multiFieldProperty = mapping.Property; - if (multiFieldProperty?.Fields == null) + var fields = multiFieldProperty.GetFields(); + if ((IDictionary)fields is not { Count: > 0 }) return mapping.FullPath; - var nonAnalyzedProperty = multiFieldProperty.Fields.OrderByDescending(kvp => kvp.Key.Name == preferredSubField).FirstOrDefault(kvp => + var nonAnalyzedProperty = fields.OrderByDescending(kvp => kvp.Key.Name == preferredSubField).FirstOrDefault(kvp => { if (kvp.Value is KeywordProperty) return true; diff --git a/src/Foundatio.Parsers.ElasticQueries/ElasticQueryParserConfiguration.cs b/src/Foundatio.Parsers.ElasticQueries/ElasticQueryParserConfiguration.cs index f4c69b2e..99b5f946 100644 --- a/src/Foundatio.Parsers.ElasticQueries/ElasticQueryParserConfiguration.cs +++ b/src/Foundatio.Parsers.ElasticQueries/ElasticQueryParserConfiguration.cs @@ -286,14 +286,14 @@ public ElasticQueryParserConfiguration AddAggregationVisitorAfter(IChainableQ #endregion - public ElasticQueryParserConfiguration UseMappings(Func, TypeMappingDescriptor> mappingBuilder, ElasticsearchClient client, string index) where T : class + public ElasticQueryParserConfiguration UseMappings(Func, TypeMapping> mappingBuilder, ElasticsearchClient client, string index) where T : class { MappingResolver = ElasticMappingResolver.Create(mappingBuilder, client, index, logger: _logger); return this; } - public ElasticQueryParserConfiguration UseMappings(Func, TypeMappingDescriptor> mappingBuilder, Inferrer inferrer, Func getMapping) where T : class + public ElasticQueryParserConfiguration UseMappings(Func, TypeMapping> mappingBuilder, Inferrer inferrer, Func getMapping) where T : class { MappingResolver = ElasticMappingResolver.Create(mappingBuilder, inferrer, getMapping, logger: _logger); diff --git a/src/Foundatio.Parsers.ElasticQueries/Extensions/ElasticExtensions.cs b/src/Foundatio.Parsers.ElasticQueries/Extensions/ElasticExtensions.cs index 643d4281..29d3b62c 100644 --- a/src/Foundatio.Parsers.ElasticQueries/Extensions/ElasticExtensions.cs +++ b/src/Foundatio.Parsers.ElasticQueries/Extensions/ElasticExtensions.cs @@ -22,6 +22,62 @@ public static string TryGetName(this IProperty property) return null; } + public static Properties GetFields(this IProperty property) + { + return property switch + { + AggregateMetricDoubleProperty p => p.Fields, + BinaryProperty p => p.Fields, + BooleanProperty p => p.Fields, + ByteNumberProperty p => p.Fields, + CompletionProperty p => p.Fields, + ConstantKeywordProperty p => p.Fields, + DateNanosProperty p => p.Fields, + DateProperty p => p.Fields, + DateRangeProperty p => p.Fields, + DenseVectorProperty p => p.Fields, + DoubleNumberProperty p => p.Fields, + DoubleRangeProperty p => p.Fields, + DynamicProperty p => p.Fields, + FieldAliasProperty p => p.Fields, + FlattenedProperty p => p.Fields, + FloatNumberProperty p => p.Fields, + FloatRangeProperty p => p.Fields, + GeoPointProperty p => p.Fields, + GeoShapeProperty p => p.Fields, + HalfFloatNumberProperty p => p.Fields, + HistogramProperty p => p.Fields, + IcuCollationProperty p => p.Fields, + IntegerNumberProperty p => p.Fields, + IntegerRangeProperty p => p.Fields, + IpProperty p => p.Fields, + IpRangeProperty p => p.Fields, + JoinProperty p => p.Fields, + KeywordProperty p => p.Fields, + LongNumberProperty p => p.Fields, + LongRangeProperty p => p.Fields, + MatchOnlyTextProperty p => p.Fields, + Murmur3HashProperty p => p.Fields, + NestedProperty p => p.Fields, + ObjectProperty p => p.Fields, + PercolatorProperty p => p.Fields, + PointProperty p => p.Fields, + RankFeatureProperty p => p.Fields, + RankFeaturesProperty p => p.Fields, + ScaledFloatNumberProperty p => p.Fields, + SearchAsYouTypeProperty p => p.Fields, + ShapeProperty p => p.Fields, + ShortNumberProperty p => p.Fields, + SparseVectorProperty p => p.Fields, + TextProperty p => p.Fields, + TokenCountProperty p => p.Fields, + UnsignedLongNumberProperty p => p.Fields, + VersionProperty p => p.Fields, + WildcardProperty p => p.Fields, + _ => null + }; + } + public static TermsInclude AddValue(this TermsInclude include, string value) { if (include?.Values == null) diff --git a/src/Foundatio.Parsers.ElasticQueries/Visitors/CombineAggregationsVisitor.cs b/src/Foundatio.Parsers.ElasticQueries/Visitors/CombineAggregationsVisitor.cs index 2eabb8e1..ee02d950 100644 --- a/src/Foundatio.Parsers.ElasticQueries/Visitors/CombineAggregationsVisitor.cs +++ b/src/Foundatio.Parsers.ElasticQueries/Visitors/CombineAggregationsVisitor.cs @@ -102,16 +102,10 @@ public override async Task VisitAsync(GroupNode node, IQueryVisitorContext conte bucketContainer.Aggregations[aggregation.Name] = aggregation.Value; } - if (termsAggregation != null && (child.Prefix == "-" || child.Prefix == "+")) + if (termsAggregation != null && child.Prefix is "-" or "+") { - if (termsAggregation.Order == null) - termsAggregation.Order = new List(); - - termsAggregation.Order.Add(new TermsOrder - { - Key = ((Aggregation)aggregation).Name, - Order = child.Prefix == "-" ? SortOrder.Desc : SortOrder.Asc - }); + termsAggregation.Order ??= new List>(); + termsAggregation.Order.Add(new KeyValuePair(aggregation.Name, child.Prefix == "-" ? SortOrder.Desc : SortOrder.Asc)); } } From 3707d56ccb1b8e4854e21459a3ac0c6735e354fa Mon Sep 17 00:00:00 2001 From: Blake Niemyjski Date: Thu, 12 Sep 2024 15:11:26 -0500 Subject: [PATCH 07/19] More work to detect bucket aggs. --- .../Extensions/ElasticExtensions.cs | 12 ++++++++++++ .../Visitors/CombineAggregationsVisitor.cs | 7 ++----- 2 files changed, 14 insertions(+), 5 deletions(-) diff --git a/src/Foundatio.Parsers.ElasticQueries/Extensions/ElasticExtensions.cs b/src/Foundatio.Parsers.ElasticQueries/Extensions/ElasticExtensions.cs index 29d3b62c..de7d893d 100644 --- a/src/Foundatio.Parsers.ElasticQueries/Extensions/ElasticExtensions.cs +++ b/src/Foundatio.Parsers.ElasticQueries/Extensions/ElasticExtensions.cs @@ -22,6 +22,18 @@ public static string TryGetName(this IProperty property) return null; } + public static bool IsBucketAggregation(this object aggregation) + { + // NOTE FilterAggregate was called FilterAggregation in the past. + return aggregation is AdjacencyMatrixAggregation or AutoDateHistogramAggregation or ChildrenAggregation + or CompositeAggregation or DateHistogramAggregation or DateRangeAggregation or DiversifiedSamplerAggregation + or FilterAggregate or FiltersAggregation or GeoDistanceAggregation or GeohashGridAggregation + or GeotileGridAggregation or GlobalAggregation or HistogramAggregation or IpRangeAggregation + or MissingAggregation or MultiTermsAggregation or NestedAggregation or ParentAggregation or RangeAggregation + or RareTermsAggregation or ReverseNestedAggregation or SamplerAggregation or SignificantTermsAggregation + or SignificantTextAggregation or TermsAggregation or VariableWidthHistogramAggregation; + } + public static Properties GetFields(this IProperty property) { return property switch diff --git a/src/Foundatio.Parsers.ElasticQueries/Visitors/CombineAggregationsVisitor.cs b/src/Foundatio.Parsers.ElasticQueries/Visitors/CombineAggregationsVisitor.cs index ee02d950..fd0ee869 100644 --- a/src/Foundatio.Parsers.ElasticQueries/Visitors/CombineAggregationsVisitor.cs +++ b/src/Foundatio.Parsers.ElasticQueries/Visitors/CombineAggregationsVisitor.cs @@ -94,12 +94,9 @@ public override async Task VisitAsync(GroupNode node, IQueryVisitorContext conte continue; } - if (container.Value is BucketAggregationBase bucketContainer) + if (container.Value.IsBucketAggregation()) { - if (bucketContainer.Aggregations == null) - bucketContainer.Aggregations = new AggregationDictionary(); - - bucketContainer.Aggregations[aggregation.Name] = aggregation.Value; + container.Aggregations.Add(aggregation); } if (termsAggregation != null && child.Prefix is "-" or "+") From 1175b506e65e51b9073e8ba22fe73f3e4a80dbbf Mon Sep 17 00:00:00 2001 From: Blake Niemyjski Date: Thu, 12 Sep 2024 15:55:56 -0500 Subject: [PATCH 08/19] Commented out code / hacks to get it to compile --- .../ElasticMappingResolver.cs | 6 +++--- .../Extensions/DefaultAggregationNodeExtensions.cs | 4 +++- .../Visitors/GetSortFieldsVisitor.cs | 7 ++++++- 3 files changed, 12 insertions(+), 5 deletions(-) diff --git a/src/Foundatio.Parsers.ElasticQueries/ElasticMappingResolver.cs b/src/Foundatio.Parsers.ElasticQueries/ElasticMappingResolver.cs index 83354a8c..697e15e8 100644 --- a/src/Foundatio.Parsers.ElasticQueries/ElasticMappingResolver.cs +++ b/src/Foundatio.Parsers.ElasticQueries/ElasticMappingResolver.cs @@ -126,8 +126,8 @@ public FieldMapping GetMapping(string field, bool followAlias = false) // coded properties sometimes have null Name properties string name = fieldMapping.TryGetName(); - if (name == null && fieldMapping is IPropertyWithClrOrigin clrOrigin && clrOrigin.ClrOrigin != null) - name = new PropertyName(clrOrigin.ClrOrigin); + // if (name == null && fieldMapping is IPropertyWithClrOrigin clrOrigin && clrOrigin.ClrOrigin != null) + // name = new PropertyName(clrOrigin.ClrOrigin); if (depth == 0) resolvedFieldName += _inferrer.PropertyName(name); @@ -419,7 +419,7 @@ private Properties MergeProperties(Properties codeProperties, Properties serverP { //LocalMetadata = aliasProperty.LocalMetadata, Path = _inferrer?.Field(aliasProperty.Path) ?? aliasProperty.Path, - Name = aliasProperty.Name + // Name = aliasProperty.Name }; } } diff --git a/src/Foundatio.Parsers.ElasticQueries/Extensions/DefaultAggregationNodeExtensions.cs b/src/Foundatio.Parsers.ElasticQueries/Extensions/DefaultAggregationNodeExtensions.cs index 0b268c3c..84067be7 100644 --- a/src/Foundatio.Parsers.ElasticQueries/Extensions/DefaultAggregationNodeExtensions.cs +++ b/src/Foundatio.Parsers.ElasticQueries/Extensions/DefaultAggregationNodeExtensions.cs @@ -240,7 +240,9 @@ private static AggregationMap GetDateHistogramAggregation(string originalField, var start = context.GetDate("StartDate"); var end = context.GetDate("EndDate"); bool isValidRange = start.HasValue && start.Value > DateTime.MinValue && end.HasValue && end.Value < DateTime.MaxValue && start.Value <= end.Value; - var bounds = isValidRange ? new ExtendedBoundsDate { Min = start.Value, Max = end.Value } : null; + // TODO: https://github.com/elastic/elasticsearch-net/issues/8338 + //var bounds = isValidRange ? new ExtendedBoundsDate { Min = start.Value, Max = end.Value } : null; + var bounds = isValidRange ? new ExtendedBoundsDate { Min = new FieldDateMath(DateMath.Anchored(start.Value).ToString()), Max = new FieldDateMath(DateMath.Anchored(end.Value).ToString()) } : null; var interval = GetInterval(proximity, start, end); string timezone = TryConvertTimeUnitToUtcOffset(boost); diff --git a/src/Foundatio.Parsers.ElasticQueries/Visitors/GetSortFieldsVisitor.cs b/src/Foundatio.Parsers.ElasticQueries/Visitors/GetSortFieldsVisitor.cs index 4bebe0aa..94872fb5 100644 --- a/src/Foundatio.Parsers.ElasticQueries/Visitors/GetSortFieldsVisitor.cs +++ b/src/Foundatio.Parsers.ElasticQueries/Visitors/GetSortFieldsVisitor.cs @@ -18,7 +18,12 @@ public override void Visit(TermNode node, IQueryVisitorContext context) return; var sort = node.GetSort(() => node.GetDefaultSort(context)); - if (sort.SortKey == null) + // TODO: SortKey https://github.com/elastic/elasticsearch-net/issues/8335 + // if (sort.SortKey == null) + // return; + + object additionalPropertyName = sort.GetType().GetProperty("AdditionalPropertyName", System.Reflection.BindingFlags.NonPublic | System.Reflection.BindingFlags.Instance)?.GetValue(sort); + if (additionalPropertyName is null) return; _fields.Add(sort); From 020968cf310ef46c3a7cd7795f19997aab9df8a8 Mon Sep 17 00:00:00 2001 From: Blake Niemyjski Date: Thu, 12 Sep 2024 16:43:34 -0500 Subject: [PATCH 09/19] WIP: Fix test compiler errors. --- .../AggregationParserTests.cs | 106 +++---- .../CustomVisitorTests.cs | 29 +- .../ElasticMappingResolverTests.cs | 16 +- .../ElasticQueryParserTests.cs | 259 +++++++++--------- .../InvertQueryTests.cs | 2 +- .../Utility/ElasticsearchTestBase.cs | 38 ++- 6 files changed, 225 insertions(+), 225 deletions(-) diff --git a/tests/Foundatio.Parsers.ElasticQueries.Tests/AggregationParserTests.cs b/tests/Foundatio.Parsers.ElasticQueries.Tests/AggregationParserTests.cs index 90997adf..27f56abb 100644 --- a/tests/Foundatio.Parsers.ElasticQueries.Tests/AggregationParserTests.cs +++ b/tests/Foundatio.Parsers.ElasticQueries.Tests/AggregationParserTests.cs @@ -34,18 +34,18 @@ await Client.IndexManyAsync([ var processor = new ElasticQueryParser(c => c.SetLoggerFactory(Log).UseMappings(Client, index).UseGeo(_ => "51.5032520,-0.1278990")); var aggregations = await processor.BuildAggregationsAsync("min:field4"); - var actualResponse = Client.Search(d => d.Index(index).Aggregations(aggregations)); + var actualResponse = await Client.SearchAsync(d => d.Index(index).Aggregations(aggregations)); string actualRequest = actualResponse.GetRequest(); _logger.LogInformation("Actual: {Request}", actualRequest); - var expectedResponse = Client.Search(d => d.Index(index).Aggregations(a => a + var expectedResponse = await Client.SearchAsync(d => d.Index(index).Aggregations(a => a .Min("min_field4", c => c.Field("field4").Meta(m => m.Add("@field_type", "long"))))); string expectedRequest = expectedResponse.GetRequest(); _logger.LogInformation("Expected: {Request}", expectedRequest); Assert.Equal(expectedRequest, actualRequest); - Assert.True(actualResponse.IsValid); - Assert.True(expectedResponse.IsValid); + Assert.True(actualResponse.IsValidResponse); + Assert.True(expectedResponse.IsValidResponse); Assert.Equal(expectedResponse.Total, actualResponse.Total); } @@ -66,18 +66,18 @@ await Client.IndexManyAsync([ var processor = new ElasticQueryParser(c => c.SetLoggerFactory(Log).UseMappings(Client, index).UseFieldMap(fieldMap).UseGeo(_ => "51.5032520,-0.1278990")); var aggregations = await processor.BuildAggregationsAsync("min:heynow"); - var actualResponse = Client.Search(d => d.Index(index).Aggregations(aggregations)); + var actualResponse = await Client.SearchAsync(d => d.Index(index).Aggregations(aggregations)); string actualRequest = actualResponse.GetRequest(); _logger.LogInformation("Actual: {Request}", actualRequest); - var expectedResponse = Client.Search(d => d.Index(index).Aggregations(a => a + var expectedResponse = await Client.SearchAsync(d => d.Index(index).Aggregations(a => a .Min("min_heynow", c => c.Field("field4").Meta(m => m.Add("@field_type", "long"))))); string expectedRequest = expectedResponse.GetRequest(); _logger.LogInformation("Expected: {Request}", expectedRequest); Assert.Equal(expectedRequest, actualRequest); - Assert.True(actualResponse.IsValid); - Assert.True(expectedResponse.IsValid); + Assert.True(actualResponse.IsValidResponse); + Assert.True(expectedResponse.IsValidResponse); Assert.Equal(expectedResponse.Total, actualResponse.Total); } @@ -102,18 +102,18 @@ await Client.IndexManyAsync([ var processor = new ElasticQueryParser(c => c.SetLoggerFactory(Log).UseMappings(Client, index).UseFieldMap(fieldMap)); var aggregations = await processor.BuildAggregationsAsync("terms:heynow"); - var actualResponse = Client.Search(d => d.Index(index).Aggregations(aggregations)); + var actualResponse = await Client.SearchAsync(d => d.Index(index).Aggregations(aggregations)); string actualRequest = actualResponse.GetRequest(); _logger.LogInformation("Actual: {Request}", actualRequest); - var expectedResponse = Client.Search(d => d.Index(index).Aggregations(a => a + var expectedResponse = await Client.SearchAsync(d => d.Index(index).Aggregations(a => a .Terms("terms_heynow", c => c.Field("field1.keyword").Meta(m => m.Add("@field_type", "text"))))); string expectedRequest = expectedResponse.GetRequest(); _logger.LogInformation("Expected: {Request}", expectedRequest); Assert.Equal(expectedRequest, actualRequest); - Assert.True(actualResponse.IsValid); - Assert.True(expectedResponse.IsValid); + Assert.True(actualResponse.IsValidResponse); + Assert.True(expectedResponse.IsValidResponse); Assert.Equal(expectedResponse.Total, actualResponse.Total); } @@ -133,11 +133,11 @@ await Client.IndexManyAsync([ var processor = new ElasticQueryParser(c => c.SetLoggerFactory(Log).UseMappings(Client, index).UseGeo(_ => "51.5032520,-0.1278990")); var aggregations = await processor.BuildAggregationsAsync("min:field4 max:field4 avg:field4 sum:field4 percentiles:field4~50,100 cardinality:field4 missing:field2 date:field5 histogram:field4 geogrid:field3 terms:field1"); - var actualResponse = Client.Search(d => d.Index(index).Aggregations(aggregations)); + var actualResponse = await Client.SearchAsync(d => d.Index(index).Aggregations(aggregations)); string actualRequest = actualResponse.GetRequest(); _logger.LogInformation("Actual: {Request}", actualRequest); - var expectedResponse = Client.Search(d => d.Index(index).Aggregations(a => a + var expectedResponse = await Client.SearchAsync(d => d.Index(index).Aggregations(a => a .GeoHash("geogrid_field3", h => h.Field("field3").GeoHashPrecision(1) .Aggregations(a1 => a1.Average("avg_lat", s => s.Script(ss => ss.Source("doc['field3'].lat"))).Average("avg_lon", s => s.Script(ss => ss.Source("doc['field3'].lon"))))) .Terms("terms_field1", t => t.Field("field1.keyword").Meta(m => m.Add("@field_type", "text"))) @@ -154,8 +154,8 @@ await Client.IndexManyAsync([ _logger.LogInformation("Expected: {Request}", expectedRequest); Assert.Equal(expectedRequest, actualRequest); - Assert.True(actualResponse.IsValid); - Assert.True(expectedResponse.IsValid); + Assert.True(actualResponse.IsValidResponse); + Assert.True(expectedResponse.IsValidResponse); Assert.Equal(expectedResponse.Total, actualResponse.Total); } @@ -176,19 +176,19 @@ public async Task ProcessNestedAggregationsWithAliasesAsync() var processor = new ElasticQueryParser(c => c.SetLoggerFactory(Log).UseMappings(Client, index).UseFieldMap(aliasMap)); var aggregations = await processor.BuildAggregationsAsync("terms:(alias1 cardinality:user)"); - var actualResponse = Client.Search(d => d.Index(index).Aggregations(aggregations)); + var actualResponse = await Client.SearchAsync(d => d.Index(index).Aggregations(aggregations)); string actualRequest = actualResponse.GetRequest(); _logger.LogInformation("Actual: {Request}", actualRequest); - var expectedResponse = Client.Search(d => d.Index(index).Aggregations(a => a + var expectedResponse = await Client.SearchAsync(d => d.Index(index).Aggregations(a => a .Terms("terms_alias1", t => t.Field("field1.keyword").Meta(m => m.Add("@field_type", "keyword")) .Aggregations(a1 => a1.Cardinality("cardinality_user", c => c.Field("data.@user.identity.keyword")))))); string expectedRequest = expectedResponse.GetRequest(); _logger.LogInformation("Expected: {Request}", expectedRequest); Assert.Equal(expectedRequest, actualRequest); - Assert.True(actualResponse.IsValid); - Assert.True(expectedResponse.IsValid); + Assert.True(actualResponse.IsValidResponse); + Assert.True(expectedResponse.IsValidResponse); Assert.Equal(expectedResponse.Total, actualResponse.Total); } @@ -206,19 +206,19 @@ await Client.IndexManyAsync([ var processor = new ElasticQueryParser(c => c.SetLoggerFactory(Log).UseMappings(Client, index).UseFieldMap(aliasMap)); var aggregations = await processor.BuildAggregationsAsync("missing:alias2"); - var actualResponse = Client.Search(d => d.Index(index).Aggregations(aggregations)); + var actualResponse = await Client.SearchAsync(d => d.Index(index).Aggregations(aggregations)); string actualRequest = actualResponse.GetRequest(); _logger.LogInformation("Actual: {Request}", actualRequest); - var expectedResponse = Client.Search(d => d.Index(index).Aggregations(a => a + var expectedResponse = await Client.SearchAsync(d => d.Index(index).Aggregations(a => a .Missing("missing_alias2", t => t.Field("field2.keyword")))); string expectedRequest = expectedResponse.GetRequest(); _logger.LogInformation("Expected: {Request}", expectedRequest); Assert.Equal(expectedRequest, actualRequest); - Assert.True(actualResponse.IsValid); - Assert.True(expectedResponse.IsValid); + Assert.True(actualResponse.IsValidResponse); + Assert.True(expectedResponse.IsValidResponse); Assert.Equal(expectedResponse.Total, actualResponse.Total); } @@ -240,11 +240,11 @@ await Client.IndexManyAsync([ var processor = new ElasticQueryParser(c => c.SetLoggerFactory(Log).UseMappings(Client, index).UseGeo(_ => "51.5032520,-0.1278990").UseFieldMap(aliasMap)); var aggregations = await processor.BuildAggregationsAsync("min:alias4 max:alias4 avg:alias4 sum:alias4 percentiles:alias4 cardinality:user missing:alias2 date:alias5 histogram:alias4 geogrid:alias3 terms:alias1"); - var actualResponse = Client.Search(d => d.Index(index).Aggregations(aggregations)); + var actualResponse = await Client.SearchAsync(d => d.Index(index).Aggregations(aggregations)); string actualRequest = actualResponse.GetRequest(); _logger.LogInformation("Actual: {Request}", actualRequest); - var expectedResponse = Client.Search(d => d.Index(index).Aggregations(a => a + var expectedResponse = await Client.SearchAsync(d => d.Index(index).Aggregations(a => a .GeoHash("geogrid_alias3", h => h.Field("field3").GeoHashPrecision(1) .Aggregations(a1 => a1.Average("avg_lat", s => s.Script(ss => ss.Source("doc['field3'].lat"))).Average("avg_lon", s => s.Script(ss => ss.Source("doc['field3'].lon"))))) .Terms("terms_alias1", t => t.Field("field1.keyword").Meta(m => m.Add("@field_type", "text"))) @@ -261,8 +261,8 @@ await Client.IndexManyAsync([ _logger.LogInformation("Expected: {Request}", expectedRequest); Assert.Equal(expectedRequest, actualRequest); - Assert.True(actualResponse.IsValid); - Assert.True(expectedResponse.IsValid); + Assert.True(actualResponse.IsValidResponse); + Assert.True(expectedResponse.IsValidResponse); Assert.Equal(expectedResponse.Total, actualResponse.Total); } @@ -276,11 +276,11 @@ public async Task ProcessTermAggregations() var processor = new ElasticQueryParser(c => c.SetLoggerFactory(Log).UseMappings(Client, index)); var aggregations = await processor.BuildAggregationsAsync("terms:(field1 @exclude:myexclude @include:myinclude @include:otherinclude @missing:mymissing @exclude:otherexclude @min:1)"); - var actualResponse = Client.Search(d => d.Index(index).Aggregations(aggregations)); + var actualResponse = await Client.SearchAsync(d => d.Index(index).Aggregations(aggregations)); string actualRequest = actualResponse.GetRequest(); _logger.LogInformation("Actual: {Request}", actualRequest); - var expectedResponse = Client.Search(d => d.Index(index).Aggregations(a => a + var expectedResponse = await Client.SearchAsync(d => d.Index(index).Aggregations(a => a .Terms("terms_field1", t => t .Field("field1.keyword") .MinimumDocumentCount(1) @@ -305,11 +305,11 @@ public async Task ProcessHistogramIntervalAggregations() var processor = new ElasticQueryParser(c => c.SetLoggerFactory(Log).UseMappings(Client, index)); var aggregations = await processor.BuildAggregationsAsync("histogram:(field1~0.1)"); - var actualResponse = Client.Search(d => d.Index(index).Aggregations(aggregations)); + var actualResponse = await Client.SearchAsync(d => d.Index(index).Aggregations(aggregations)); string actualRequest = actualResponse.GetRequest(); _logger.LogInformation("Actual: {Request}", actualRequest); - var expectedResponse = Client.Search(d => d.Index(index).Aggregations(a => a + var expectedResponse = await Client.SearchAsync(d => d.Index(index).Aggregations(a => a .Histogram("histogram_field1", t => t .Field("field1.keyword") .Interval(0.1) @@ -332,11 +332,11 @@ public async Task ProcessTermTopHitsAggregations() var processor = new ElasticQueryParser(c => c.SetLoggerFactory(Log).UseMappings(Client, index)); var aggregations = await processor.BuildAggregationsAsync("terms:(field1~1000^2 tophits:(_~1000 @include:myinclude))"); - var actualResponse = Client.Search(d => d.Index(index).Aggregations(aggregations)); + var actualResponse = await Client.SearchAsync(d => d.Index(index).Aggregations(aggregations)); string actualRequest = actualResponse.GetRequest(); _logger.LogInformation("Actual: {Request}", actualRequest); - var expectedResponse = Client.Search(d => d.Index(index).Aggregations(a => a + var expectedResponse = await Client.SearchAsync(d => d.Index(index).Aggregations(a => a .Terms("terms_field1", t => t .Field("field1.keyword") .Size(1000) @@ -360,11 +360,11 @@ public async Task ProcessSortedTermAggregations() var processor = new ElasticQueryParser(c => c.SetLoggerFactory(Log).UseMappings(Client, index)); var aggregations = await processor.BuildAggregationsAsync("terms:(field1 -cardinality:field4)"); - var actualResponse = Client.Search(d => d.Index(index).Aggregations(aggregations)); + var actualResponse = await Client.SearchAsync(d => d.Index(index).Aggregations(aggregations)); string actualRequest = actualResponse.GetRequest(); _logger.LogInformation("Actual: {Request}", actualRequest); - var expectedResponse = Client.Search(d => d.Index(index).Aggregations(a => a + var expectedResponse = await Client.SearchAsync(d => d.Index(index).Aggregations(a => a .Terms("terms_field1", t => t .Field("field1.keyword") .Order(o => o.Descending("cardinality_field4")) @@ -375,8 +375,8 @@ public async Task ProcessSortedTermAggregations() _logger.LogInformation("Expected: {Request}", expectedRequest); Assert.Equal(expectedRequest, actualRequest); - Assert.True(actualResponse.IsValid); - Assert.True(expectedResponse.IsValid); + Assert.True(actualResponse.IsValidResponse); + Assert.True(expectedResponse.IsValidResponse); Assert.Equal(expectedResponse.Total, actualResponse.Total); } @@ -390,11 +390,11 @@ public async Task ProcessDateHistogramAggregations() var processor = new ElasticQueryParser(c => c.SetLoggerFactory(Log).UseMappings(Client, index)); var aggregations = await processor.BuildAggregationsAsync("date:(field5^1h @missing:\"0001-01-01T00:00:00\" min:field5^1h max:field5^1h)"); - var actualResponse = Client.Search(d => d.Index(index).Aggregations(aggregations)); + var actualResponse = await Client.SearchAsync(d => d.Index(index).Aggregations(aggregations)); string actualRequest = actualResponse.GetRequest(); _logger.LogInformation("Actual: {Request}", actualRequest); - var expectedResponse = Client.Search(d => d.Index(index).Aggregations(a => a + var expectedResponse = await Client.SearchAsync(d => d.Index(index).Aggregations(a => a .DateHistogram("date_field5", d1 => d1 .Field("field5").Meta(m => m.Add("@timezone", "1h")) .CalendarInterval(DateInterval.Day) @@ -409,8 +409,8 @@ public async Task ProcessDateHistogramAggregations() _logger.LogInformation("Expected: {Request}", expectedRequest); Assert.Equal(expectedRequest, actualRequest); - Assert.True(actualResponse.IsValid, actualResponse.DebugInformation); - Assert.True(expectedResponse.IsValid); + Assert.True(actualResponse.IsValidResponse, actualResponse.DebugInformation); + Assert.True(expectedResponse.IsValidResponse); Assert.Equal(expectedResponse.Total, actualResponse.Total); } @@ -424,11 +424,11 @@ public async Task CanSpecifyDefaultValuesAggregations() var processor = new ElasticQueryParser(c => c.SetLoggerFactory(Log).UseMappings(Client, index)); var aggregations = await processor.BuildAggregationsAsync("min:field4~0 max:field4~0 avg:field4~0 sum:field4~0 cardinality:field4~0"); - var actualResponse = Client.Search(d => d.Index(index).Aggregations(aggregations)); + var actualResponse = await Client.SearchAsync(d => d.Index(index).Aggregations(aggregations)); string actualRequest = actualResponse.GetRequest(); _logger.LogInformation("Actual: {Request}", actualRequest); - var expectedResponse = Client.Search(d => d.Index(index).Aggregations(a => a + var expectedResponse = await Client.SearchAsync(d => d.Index(index).Aggregations(a => a .Sum("sum_field4", c => c.Field("field4").Missing(0).Meta(m => m.Add("@field_type", "integer"))) .Cardinality("cardinality_field4", c => c.Field("field4").Missing(0)) .Average("avg_field4", c => c.Field("field4").Missing(0).Meta(m => m.Add("@field_type", "integer"))) @@ -438,8 +438,8 @@ public async Task CanSpecifyDefaultValuesAggregations() _logger.LogInformation("Expected: {Request}", expectedRequest); Assert.Equal(expectedRequest, actualRequest); - Assert.True(actualResponse.IsValid); - Assert.True(expectedResponse.IsValid); + Assert.True(actualResponse.IsValidResponse); + Assert.True(expectedResponse.IsValidResponse); Assert.Equal(expectedResponse.Total, actualResponse.Total); } @@ -463,10 +463,10 @@ public Task GeoGridDoesNotResolveLocationForAggregation() [InlineData("avg:value", true)] [InlineData(" avg : value", true)] [InlineData("avg:value cardinality:value sum:value min:value max:value", true)] - public Task CanParseAggregations(string query, bool isValid) + public Task CanParseAggregations(string query, bool IsValidResponse) { var parser = new ElasticQueryParser(c => c.SetLoggerFactory(Log)); - return GetAggregationQueryInfoAsync(parser, query, isValid); + return GetAggregationQueryInfoAsync(parser, query, IsValidResponse); } public static IEnumerable AggregationTestCases => @@ -525,23 +525,23 @@ public Task CanParseAggregations(string query, bool isValid) [Theory] [MemberData(nameof(AggregationTestCases))] - public Task GetElasticAggregationQueryInfoAsync(string query, bool isValid, int maxNodeDepth, HashSet fields, Dictionary> operations) + public Task GetElasticAggregationQueryInfoAsync(string query, bool IsValidResponse, int maxNodeDepth, HashSet fields, Dictionary> operations) { var parser = new ElasticQueryParser(c => c.SetLoggerFactory(Log)); - return GetAggregationQueryInfoAsync(parser, query, isValid, maxNodeDepth, fields, operations); + return GetAggregationQueryInfoAsync(parser, query, IsValidResponse, maxNodeDepth, fields, operations); } - private async Task GetAggregationQueryInfoAsync(IQueryParser parser, string query, bool isValid, int maxNodeDepth = -1, HashSet fields = null, Dictionary> operations = null) + private async Task GetAggregationQueryInfoAsync(IQueryParser parser, string query, bool IsValidResponse, int maxNodeDepth = -1, HashSet fields = null, Dictionary> operations = null) { var context = new ElasticQueryVisitorContext { QueryType = QueryTypes.Aggregation }; var queryNode = await parser.ParseAsync(query, context); var result = context.GetValidationResult(); Assert.Equal(QueryTypes.Aggregation, result.QueryType); - if (!result.IsValid) + if (!result.IsValidResponse) _logger.LogInformation(result.Message); - Assert.Equal(isValid, result.IsValid); + Assert.Equal(IsValidResponse, result.IsValidResponse); if (maxNodeDepth >= 0) Assert.Equal(maxNodeDepth, result.MaxNodeDepth); if (fields != null) diff --git a/tests/Foundatio.Parsers.ElasticQueries.Tests/CustomVisitorTests.cs b/tests/Foundatio.Parsers.ElasticQueries.Tests/CustomVisitorTests.cs index 171d5b7e..e1844557 100644 --- a/tests/Foundatio.Parsers.ElasticQueries.Tests/CustomVisitorTests.cs +++ b/tests/Foundatio.Parsers.ElasticQueries.Tests/CustomVisitorTests.cs @@ -2,6 +2,7 @@ using System.Collections.Generic; using System.Text; using System.Threading.Tasks; +using Elastic.Clients.Elasticsearch.QueryDsl; using Foundatio.Parsers.ElasticQueries.Extensions; using Foundatio.Parsers.LuceneQueries.Nodes; using Foundatio.Parsers.LuceneQueries.Visitors; @@ -15,14 +16,14 @@ public class CustomVisitorTests : ElasticsearchTestBase { public CustomVisitorTests(ITestOutputHelper output, ElasticsearchFixture fixture) : base(output, fixture) { - Log.DefaultMinimumLevel = Microsoft.Extensions.Logging.LogLevel.Trace; + Log.DefaultMinimumLevel = LogLevel.Trace; } [Fact] public async Task CanResolveSimpleCustomFilter() { string index = CreateRandomIndex(); - Client.Index(new MyType { Id = "1" }, i => i.Index(index)); + await Client.IndexAsync(new MyType { Id = "1" }, i => i.Index(index)); var processor = new ElasticQueryParser(c => c .SetLoggerFactory(Log) @@ -30,11 +31,11 @@ public async Task CanResolveSimpleCustomFilter() .AddVisitor(new CustomFilterVisitor())); var result = await processor.BuildQueryAsync("@custom:(one)"); - var actualResponse = Client.Search(d => d.Index(index).Query(_ => result)); + var actualResponse = await Client.SearchAsync(d => d.Index(index).Query(_ => result)); string actualRequest = actualResponse.GetRequest(); _logger.LogInformation("Actual: {Request}", actualRequest); - var expectedResponse = Client.Search(d => d + var expectedResponse = await Client.SearchAsync(d => d .Index(index).Query(f => f.Bool(b => b.Filter(filter => filter.Terms(m => m.Field("id").Terms("1")))))); string expectedRequest = expectedResponse.GetRequest(); _logger.LogInformation("Expected: {Request}", expectedRequest); @@ -47,7 +48,7 @@ public async Task CanResolveSimpleCustomFilter() public async Task CanResolveCustomFilterContainingIncludes() { string index = CreateRandomIndex(); - Client.Index(new MyType { Id = "1" }, i => i.Index(index)); + await Client.IndexAsync(new MyType { Id = "1" }, i => i.Index(index)); var processor = new ElasticQueryParser(c => c .SetLoggerFactory(Log) @@ -55,11 +56,11 @@ public async Task CanResolveCustomFilterContainingIncludes() .AddVisitor(new CustomFilterVisitor())); var result = await processor.BuildQueryAsync("@custom:(one @include:3)"); - var actualResponse = Client.Search(d => d.Index(index).Query(_ => result)); + var actualResponse = await Client.SearchAsync(d => d.Index(index).Query(_ => result)); string actualRequest = actualResponse.GetRequest(); _logger.LogInformation("Actual: {Request}", actualRequest); - var expectedResponse = Client.Search(d => d + var expectedResponse = await Client.SearchAsync(d => d .Index(index).Query(f => f.Bool(b => b.Filter(filter => filter.Terms(m => m.Field("id").Terms("1", "3")))))); string expectedRequest = expectedResponse.GetRequest(); _logger.LogInformation("Expected: {Request}", expectedRequest); @@ -72,7 +73,7 @@ public async Task CanResolveCustomFilterContainingIncludes() public async Task CanResolveIncludeToCustomFilterContainingIgnoredInclude() { string index = CreateRandomIndex(); - Client.Index(new MyType { Id = "1" }, i => i.Index(index)); + await Client.IndexAsync(new MyType { Id = "1" }, i => i.Index(index)); var processor = new ElasticQueryParser(c => c .SetLoggerFactory(Log) @@ -80,11 +81,11 @@ public async Task CanResolveIncludeToCustomFilterContainingIgnoredInclude() .AddVisitor(new CustomFilterVisitor(), 1)); var result = await processor.BuildQueryAsync("@include:test"); - var actualResponse = Client.Search(d => d.Index(index).Query(_ => result)); + var actualResponse = await Client.SearchAsync(d => d.Index(index).Query(_ => result)); string actualRequest = actualResponse.GetRequest(); _logger.LogInformation("Actual: {Request}", actualRequest); - var expectedResponse = Client.Search(d => d + var expectedResponse = await Client.SearchAsync(d => d .Index(index).Query(f => f.Bool(b => b.Filter(filter => filter.Terms(m => m.Field("id").Terms("1", "3")))))); string expectedRequest = expectedResponse.GetRequest(); _logger.LogInformation("Expected: {Request}", expectedRequest); @@ -119,7 +120,7 @@ private static Task ResolveIncludeAsync(string expected, string actual, public async Task CanResolveMultipleCustomFilters() { string index = CreateRandomIndex(); - Client.Index(new MyType { Id = "1" }, i => i.Index(index)); + await Client.IndexAsync(new MyType { Id = "1" }, i => i.Index(index)); var processor = new ElasticQueryParser(c => c .SetLoggerFactory(Log) @@ -127,11 +128,11 @@ public async Task CanResolveMultipleCustomFilters() .AddVisitor(new CustomFilterVisitor())); var result = await processor.BuildQueryAsync("@custom:(one) OR (field1:Test @custom:(two))"); - var actualResponse = Client.Search(d => d.Index(index).Query(_ => result)); + var actualResponse = await Client.SearchAsync(d => d.Index(index).Query(_ => result)); string actualRequest = actualResponse.GetRequest(); _logger.LogInformation("Actual: {Request}", actualRequest); - var expectedResponse = Client.Search(d => d.Index(index) + var expectedResponse = await Client.SearchAsync(d => d.Index(index) .Query(f => f .Bool(b => b .Filter(filter => filter @@ -170,7 +171,7 @@ public override async Task VisitAsync(GroupNode node, IQueryVisitorContext conte { string term = ToTerm(node); var ids = await GetIdsAsync(term); - if (ids != null && ids.Count > 0) + if (ids is { Count: > 0 }) node.Parent.SetQuery(new TermsQuery { Field = "id", Terms = ids }); else node.Parent.SetQuery(new TermQuery { Field = "id", Value = "none" }); diff --git a/tests/Foundatio.Parsers.ElasticQueries.Tests/ElasticMappingResolverTests.cs b/tests/Foundatio.Parsers.ElasticQueries.Tests/ElasticMappingResolverTests.cs index 59ea0f6c..1e1577b1 100644 --- a/tests/Foundatio.Parsers.ElasticQueries.Tests/ElasticMappingResolverTests.cs +++ b/tests/Foundatio.Parsers.ElasticQueries.Tests/ElasticMappingResolverTests.cs @@ -1,5 +1,8 @@ using System; using System.Linq.Expressions; +using System.Threading.Tasks; +using Elastic.Clients.Elasticsearch; +using Elastic.Clients.Elasticsearch.Mapping; using Xunit; using Xunit.Abstractions; @@ -27,11 +30,11 @@ private TypeMapping MapMyNestedType(TypeMappingDescriptor m) } [Fact] - public void CanResolveCodedProperty() + public async Task CanResolveCodedProperty() { string index = CreateRandomIndex(MapMyNestedType); - Client.IndexMany([ + await Client.IndexManyAsync([ new MyNestedType { Field1 = "value1", @@ -52,7 +55,8 @@ public void CanResolveCodedProperty() new MyNestedType { Field1 = "value2", Field2 = "value2" }, new MyNestedType { Field1 = "value1", Field2 = "value4" } ], index); - Client.Indices.Refresh(index); + + await Client.Indices.RefreshAsync(index); var resolver = ElasticMappingResolver.Create(MapMyNestedType, Client, index, _logger); @@ -62,11 +66,11 @@ public void CanResolveCodedProperty() } [Fact] - public void CanResolveProperties() + public async Task CanResolveProperties() { string index = CreateRandomIndex(MapMyNestedType); - Client.IndexMany([ + await Client.IndexManyAsync([ new MyNestedType { Field1 = "value1", @@ -87,7 +91,7 @@ public void CanResolveProperties() new MyNestedType { Field1 = "value2", Field2 = "value2" }, new MyNestedType { Field1 = "value1", Field2 = "value4" } ], index); - Client.Indices.Refresh(index); + await Client.Indices.RefreshAsync(index); var resolver = ElasticMappingResolver.Create(MapMyNestedType, Client, index, _logger); diff --git a/tests/Foundatio.Parsers.ElasticQueries.Tests/ElasticQueryParserTests.cs b/tests/Foundatio.Parsers.ElasticQueries.Tests/ElasticQueryParserTests.cs index 92b58a8b..104e9eee 100644 --- a/tests/Foundatio.Parsers.ElasticQueries.Tests/ElasticQueryParserTests.cs +++ b/tests/Foundatio.Parsers.ElasticQueries.Tests/ElasticQueryParserTests.cs @@ -3,6 +3,7 @@ using System.Linq; using System.Threading.Tasks; using Elastic.Clients.Elasticsearch; +using Elastic.Clients.Elasticsearch.Aggregations; using Elastic.Clients.Elasticsearch.Mapping; using Elastic.Clients.Elasticsearch.QueryDsl; using Foundatio.Parsers.ElasticQueries.Extensions; @@ -88,11 +89,11 @@ await Client.IndexManyAsync([ var processor = new ElasticQueryParser(c => c.SetLoggerFactory(Log)); var result = await processor.BuildQueryAsync("field1:value1"); - var actualResponse = Client.Search(d => d.Index(index).Query(_ => result)); + var actualResponse = await Client.SearchAsync(d => d.Index(index).Query(_ => result)); string actualRequest = actualResponse.GetRequest(); _logger.LogInformation("Actual: {Request}", actualRequest); - var expectedResponse = Client.Search(d => d.Index(index).Query(q => q.Bool(b => b.Filter(f => f.Term(m => m.Field1, "value1"))))); + var expectedResponse = await Client.SearchAsync(d => d.Index(index).Query(q => q.Bool(b => b.Filter(f => f.Term(m => m.Field1, "value1"))))); string expectedRequest = expectedResponse.GetRequest(); _logger.LogInformation("Expected: {Request}", expectedRequest); @@ -117,11 +118,11 @@ await Client.IndexManyAsync([ var processor = new ElasticQueryParser(c => c.UseIncludes(includes).SetLoggerFactory(Log)); var result = await processor.BuildQueryAsync("field1:value1 @include:stuff", new ElasticQueryVisitorContext { UseScoring = true }); - var actualResponse = Client.Search(d => d.Index(index).Query(_ => result)); + var actualResponse = await Client.SearchAsync(d => d.Index(index).Query(_ => result)); string actualRequest = actualResponse.GetRequest(); _logger.LogInformation("Actual: {Request}", actualRequest); - var expectedResponse = Client.Search(d => d.Index(index).Query(f => + var expectedResponse = await Client.SearchAsync(d => d.Index(index).Query(f => f.Term(m => m.Field1, "value1") && f.Term(m => m.Field2, "value2"))); @@ -142,11 +143,11 @@ public async Task ShouldGenerateORedTermsQuery() var processor = new ElasticQueryParser(c => c.SetLoggerFactory(Log)); var result = await processor.BuildQueryAsync("field1:value1 field2:value2 field3:value3", new ElasticQueryVisitorContext().UseSearchMode()); - var actualResponse = Client.Search(d => d.Index(index).Query(_ => result)); + var actualResponse = await Client.SearchAsync(d => d.Index(index).Query(_ => result)); string actualRequest = actualResponse.GetRequest(); _logger.LogInformation("Actual: {Request}", actualRequest); - var expectedResponse = Client.Search(d => d.Index(index).Query(f => + var expectedResponse = await Client.SearchAsync(d => d.Index(index).Query(f => f.Term(m => m.Field1, "value1") || f.Term(m => m.Field2, "value2") || f.Term(m => m.Field3, "value3"))); string expectedRequest = expectedResponse.GetRequest(); @@ -171,11 +172,11 @@ public async Task ShouldHandleMultipleTermsForAnalyzedFields() var result = await processor.BuildQueryAsync("field1:(value1 abc def ghi) field2:(value2 jhk)", new ElasticQueryVisitorContext().UseSearchMode()); - var actualResponse = Client.Search(d => d.Index(index).Query(_ => result)); + var actualResponse = await Client.SearchAsync(d => d.Index(index).Query(_ => result)); string actualRequest = actualResponse.GetRequest(); _logger.LogInformation("Actual: {Request}", actualRequest); - var expectedResponse = Client.Search(d => d.Index(index).Query(f => + var expectedResponse = await Client.SearchAsync(d => d.Index(index).Query(f => f.Match(m => m.Field(mf => mf.Field1).Query("value1")) || f.Match(m => m.Field(mf => mf.Field1).Query("abc")) || f.Match(m => m.Field(mf => mf.Field1).Query("def")) @@ -190,11 +191,11 @@ public async Task ShouldHandleMultipleTermsForAnalyzedFields() processor = new ElasticQueryParser(c => c.SetLoggerFactory(Log).SetDefaultFields(["field1"]).UseMappings(Client, index)); result = await processor.BuildQueryAsync("value1 abc def ghi", new ElasticQueryVisitorContext().UseSearchMode()); - actualResponse = Client.Search(d => d.Index(index).Query(_ => result)); + actualResponse = await Client.SearchAsync(d => d.Index(index).Query(_ => result)); actualRequest = actualResponse.GetRequest(); _logger.LogInformation("Actual: {Request}", actualRequest); - expectedResponse = Client.Search(d => d.Index(index).Query(f => + expectedResponse = await Client.SearchAsync(d => d.Index(index).Query(f => f.Match(m => m.Field(mf => mf.Field1).Query("value1")) || f.Match(m => m.Field(mf => mf.Field1).Query("abc")) || f.Match(m => m.Field(mf => mf.Field1).Query("def")) @@ -209,11 +210,11 @@ public async Task ShouldHandleMultipleTermsForAnalyzedFields() // multi-match on multiple default fields processor = new ElasticQueryParser(c => c.SetLoggerFactory(Log).SetDefaultFields(["field1", "field2"]).UseMappings(Client, index)); result = await processor.BuildQueryAsync("value1 abc def ghi", new ElasticQueryVisitorContext().UseSearchMode()); - actualResponse = Client.Search(d => d.Index(index).Query(_ => result)); + actualResponse = await Client.SearchAsync(d => d.Index(index).Query(_ => result)); actualRequest = actualResponse.GetRequest(); _logger.LogInformation("Actual: {Request}", actualRequest); - expectedResponse = Client.Search(d => d.Index(index).Query(f => + expectedResponse = await Client.SearchAsync(d => d.Index(index).Query(f => f.MultiMatch(m => m.Fields(mf => mf.Fields("field1", "field2")).Query("value1")) || f.MultiMatch(m => m.Fields(mf => mf.Fields("field1", "field2")).Query("abc")) || f.MultiMatch(m => m.Fields(mf => mf.Fields("field1", "field2")).Query("def")) @@ -241,7 +242,7 @@ TypeMappingDescriptor GetCodeMappings(TypeMappingDescriptor d) = .Keyword(e => e.Name(m => m.Field2)))); var res = Client.Index(new MyType { Field1 = "value1", Field2 = "value2", Field4 = 1, Field5 = DateTime.Now }, i => i.Index(index)); - Client.Indices.Refresh(index); + Client.Indices.RefreshAsync(index); var parser = new ElasticQueryParser(c => c.SetDefaultFields(["field1"]).UseMappings(GetCodeMappings, Client, index)); @@ -269,11 +270,11 @@ await Client.IndexManyAsync([ var processor = new ElasticQueryParser(c => c.SetLoggerFactory(Log).UseMappings(Client, index)); var result = await processor.BuildQueryAsync("\"\\\"now there\""); - var actualResponse = Client.Search(d => d.Index(index).Query(_ => result)); + var actualResponse = await Client.SearchAsync(d => d.Index(index).Query(_ => result)); string actualRequest = actualResponse.GetRequest(true); _logger.LogInformation("Actual: {Request}", actualRequest); - var expectedResponse = Client.Search(d => d.Index(index) + var expectedResponse = await Client.SearchAsync(d => d.Index(index) .Query(q => q .Bool(b => b .Filter(f => f @@ -297,11 +298,11 @@ await Client.IndexManyAsync([ var processor = new ElasticQueryParser(c => c.SetLoggerFactory(Log).UseMappings(Client, index)); var result = await processor.BuildQueryAsync(@"field1:one\\/two*"); - var actualResponse = Client.Search(d => d.Index(index).Query(_ => result)); + var actualResponse = await Client.SearchAsync(d => d.Index(index).Query(_ => result)); string actualRequest = actualResponse.GetRequest(true); _logger.LogInformation("Actual: {Request}", actualRequest); - var expectedResponse = Client.Search(d => d.Index(index) + var expectedResponse = await Client.SearchAsync(d => d.Index(index) .Query(q => q .Bool(b => b .Filter(f => f @@ -325,11 +326,11 @@ await Client.IndexManyAsync([ var processor = new ElasticQueryParser(c => c.SetLoggerFactory(Log).UseMappings(Client, index)); var result = await processor.BuildQueryAsync(@"field1:one\\/two"); - var actualResponse = Client.Search(d => d.Index(index).Query(_ => result)); + var actualResponse = await Client.SearchAsync(d => d.Index(index).Query(_ => result)); string actualRequest = actualResponse.GetRequest(true); _logger.LogInformation("Actual: {Request}", actualRequest); - var expectedResponse = Client.Search(d => d.Index(index) + var expectedResponse = await Client.SearchAsync(d => d.Index(index) .Query(q => q .Bool(b => b .Filter(f => f @@ -355,11 +356,11 @@ await Client.IndexManyAsync([ var processor = new ElasticQueryParser(c => c.SetLoggerFactory(Log)); var result = await processor.BuildQueryAsync($"_exists_:{nameof(MyType.Field2)}"); - var actualResponse = Client.Search(d => d.Index(index).Query(_ => result)); + var actualResponse = await Client.SearchAsync(d => d.Index(index).Query(_ => result)); string actualRequest = actualResponse.GetRequest(); _logger.LogInformation("Actual: {Request}", actualRequest); - var expectedResponse = Client.Search(d => d + var expectedResponse = await Client.SearchAsync(d => d .Index(index) .Query(q => q.Bool(b => b.Filter(f => f.Exists(e => e.Field(nameof(MyType.Field2))))))); string expectedRequest = expectedResponse.GetRequest(); @@ -383,11 +384,11 @@ await Client.IndexManyAsync([ var processor = new ElasticQueryParser(c => c.SetLoggerFactory(Log)); var result = await processor.BuildQueryAsync($"_missing_:{nameof(MyType.Field2)}", new ElasticQueryVisitorContext { UseScoring = true }); - var actualResponse = Client.Search(d => d.Index(index).Query(_ => result)); + var actualResponse = await Client.SearchAsync(d => d.Index(index).Query(_ => result)); string actualRequest = actualResponse.GetRequest(); _logger.LogInformation("Actual: {Request}", actualRequest); - var expectedResponse = Client.Search(d => d + var expectedResponse = await Client.SearchAsync(d => d .Index(index) .Query(q => q.Bool(b => b.MustNot(f => f.Exists(e => e.Field(nameof(MyType.Field2))))))); string expectedRequest = expectedResponse.GetRequest(); @@ -410,16 +411,16 @@ await Client.IndexManyAsync([ var processor = new ElasticQueryParser(c => c.SetLoggerFactory(Log).UseMappings(Client, index)); var result = await processor.BuildAggregationsAsync("min:field2 max:field2 date:(field5~1d^\"America/Chicago\" min:field2 max:field2 min:field1 @offset:-6h)"); - var actualResponse = Client.Search(d => d.Index(index).Aggregations(result)); + var actualResponse = await Client.SearchAsync(d => d.Index(index).Aggregations(result)); string actualRequest = actualResponse.GetRequest(); _logger.LogInformation("Actual: {Request}", actualRequest); - var expectedResponse = Client.Search(i => i.Index(index).Aggregations(f => f + var expectedResponse = await Client.SearchAsync(i => i.Index(index).Aggregations(f => f .Max("max_field2", m => m.Field("field2.keyword").Meta(m2 => m2.Add("@field_type", "text"))) .DateHistogram("date_field5", d => d.Field(d2 => d2.Field5) - .CalendarInterval(DateInterval.Day) + .CalendarInterval(CalendarInterval.Day) .Format("date_optional_time") .MinimumDocumentCount(0) .TimeZone("America/Chicago") @@ -442,32 +443,32 @@ await Client.IndexManyAsync([ /// /// https://www.elastic.co/guide/en/elasticsearch/reference/current/search-aggregations-bucket-datehistogram-aggregation.html /// - [InlineData("1s", DateInterval.Second)] - [InlineData("second", DateInterval.Second)] - [InlineData("m", DateInterval.Minute)] - [InlineData("1m", DateInterval.Minute)] - [InlineData("minute", DateInterval.Minute)] + [InlineData("1s", CalendarInterval.Second)] + [InlineData("second", CalendarInterval.Second)] + [InlineData("m", CalendarInterval.Minute)] + [InlineData("1m", CalendarInterval.Minute)] + [InlineData("minute", CalendarInterval.Minute)] [InlineData("23m")] - [InlineData("h", DateInterval.Hour)] - [InlineData("1h", DateInterval.Hour)] - [InlineData("hour", DateInterval.Hour)] + [InlineData("h", CalendarInterval.Hour)] + [InlineData("1h", CalendarInterval.Hour)] + [InlineData("hour", CalendarInterval.Hour)] [InlineData("1.5h")] - [InlineData("d", DateInterval.Day)] - [InlineData("1d", DateInterval.Day)] - [InlineData("day", DateInterval.Day)] + [InlineData("d", CalendarInterval.Day)] + [InlineData("1d", CalendarInterval.Day)] + [InlineData("day", CalendarInterval.Day)] [InlineData("2d")] - [InlineData("w", DateInterval.Week)] - [InlineData("1w", DateInterval.Week)] - [InlineData("week", DateInterval.Week)] - [InlineData("M", DateInterval.Month)] - [InlineData("1M", DateInterval.Month)] - [InlineData("month", DateInterval.Month)] - [InlineData("q", DateInterval.Quarter)] - [InlineData("1q", DateInterval.Quarter)] - [InlineData("quarter", DateInterval.Quarter)] - [InlineData("y", DateInterval.Year)] - [InlineData("1y", DateInterval.Year)] - [InlineData("year", DateInterval.Year)] + [InlineData("w", CalendarInterval.Week)] + [InlineData("1w", CalendarInterval.Week)] + [InlineData("week", CalendarInterval.Week)] + [InlineData("M", CalendarInterval.Month)] + [InlineData("1M", CalendarInterval.Month)] + [InlineData("month", CalendarInterval.Month)] + [InlineData("q", CalendarInterval.Quarter)] + [InlineData("1q", CalendarInterval.Quarter)] + [InlineData("quarter", CalendarInterval.Quarter)] + [InlineData("y", CalendarInterval.Year)] + [InlineData("1y", CalendarInterval.Year)] + [InlineData("year", CalendarInterval.Year)] [Theory] public async Task CanUseDateHistogramAggregationInterval(string interval, object expectedInterval = null) { @@ -478,17 +479,17 @@ public async Task CanUseDateHistogramAggregationInterval(string interval, object var processor = new ElasticQueryParser(c => c.SetLoggerFactory(Log).UseMappings(Client, index)); var result = await processor.BuildAggregationsAsync($"date:(field5~{interval})"); - var actualResponse = Client.Search(d => d.Index(index).Aggregations(result)); + var actualResponse = await Client.SearchAsync(d => d.Index(index).Aggregations(result)); string actualRequest = actualResponse.GetRequest(); _logger.LogInformation("Actual: {Request}", actualRequest); - var expectedResponse = Client.Search(i => i.Index(index).Aggregations(f => f + var expectedResponse = await Client.SearchAsync(i => i.Index(index).Aggregations(f => f .DateHistogram("date_field5", d => { d.Field(d2 => d2.Field5); - if (expectedInterval is DateInterval dateInterval) - d.CalendarInterval(dateInterval); + if (expectedInterval is CalendarInterval CalendarInterval) + d.CalendarInterval(CalendarInterval); else if (expectedInterval is string stringInterval) d.FixedInterval(stringInterval); else @@ -511,14 +512,14 @@ public async Task CanUseDateHistogramAggregationInterval(string interval, object public void CanDoNestDateHistogram() { string index = CreateRandomIndex(); - Client.IndexMany([new MyType { Field5 = DateTime.Now }], index); - Client.Indices.Refresh(index); + await Client.IndexManyAsync([new MyType { Field5 = DateTime.Now }], index); + Client.Indices.RefreshAsync(index); - var response = Client.Search(i => i.Index(index).Aggregations(f => f - .DateHistogram("myagg", d => d.Field(d2 => d2.Field5).CalendarInterval(DateInterval.Day)) + var response = await Client.SearchAsync(i => i.Index(index).Aggregations(f => f + .DateHistogram("myagg", d => d.Field(d2 => d2.Field5).CalendarInterval(CalendarInterval.Day)) )); - Assert.True(response.IsValid); + Assert.True(response.IsValidResponse); } [Fact] @@ -534,12 +535,12 @@ await Client.IndexManyAsync([ var processor = new ElasticQueryParser(c => c.SetLoggerFactory(Log)); var result = await processor.BuildAggregationsAsync("date:field5"); - var actualResponse = Client.Search(d => d.Index(index).Aggregations(result)); + var actualResponse = await Client.SearchAsync(d => d.Index(index).Aggregations(result)); string actualRequest = actualResponse.GetRequest(); _logger.LogInformation("Actual: {Request}", actualRequest); - var expectedResponse = Client.Search(i => i.Index(index).Aggregations(f => f - .DateHistogram("date_field5", d => d.Field(d2 => d2.Field5).CalendarInterval(DateInterval.Day).Format("date_optional_time").MinimumDocumentCount(0)) + var expectedResponse = await Client.SearchAsync(i => i.Index(index).Aggregations(f => f + .DateHistogram("date_field5", d => d.Field(d2 => d2.Field5).CalendarInterval(CalendarInterval.Day).Format("date_optional_time").MinimumDocumentCount(0)) )); string expectedRequest = expectedResponse.GetRequest(); _logger.LogInformation("Expected: {Request}", expectedRequest); @@ -564,11 +565,11 @@ await Client.IndexManyAsync([ var processor = new ElasticQueryParser(c => c.SetLoggerFactory(Log).UseMappings(Client, index)); var result = await processor.BuildQueryAsync("field1:value1", new ElasticQueryVisitorContext().UseSearchMode()); - var actualResponse = Client.Search(d => d.Index(index).Query(_ => result)); + var actualResponse = await Client.SearchAsync(d => d.Index(index).Query(_ => result)); string actualRequest = actualResponse.GetRequest(); _logger.LogInformation("Actual: {Request}", actualRequest); - var expectedResponse = Client.Search(d => d.Index(index).Query(q => q.Match(e => e.Field(m => m.Field1).Query("value1")))); + var expectedResponse = await Client.SearchAsync(d => d.Index(index).Query(q => q.Match(e => e.Field(m => m.Field1).Query("value1")))); string expectedRequest = expectedResponse.GetRequest(); _logger.LogInformation("Expected: {Request}", expectedRequest); @@ -576,10 +577,10 @@ await Client.IndexManyAsync([ Assert.Equal(expectedResponse.Total, actualResponse.Total); result = await processor.BuildQueryAsync("field3:hey", new ElasticQueryVisitorContext().UseSearchMode()); - actualResponse = Client.Search(d => d.Index(index).Query(_ => result)); + actualResponse = await Client.SearchAsync(d => d.Index(index).Query(_ => result)); actualRequest = actualResponse.GetRequest(); _logger.LogInformation("Actual: {Request}", actualRequest); - expectedResponse = Client.Search(d => d.Index(index).Query(q => q.Match(m => m + expectedResponse = await Client.SearchAsync(d => d.Index(index).Query(q => q.Match(m => m .Field(f => f.Field3) .Query("hey") ))); @@ -590,10 +591,10 @@ await Client.IndexManyAsync([ Assert.Equal(expectedResponse.Total, actualResponse.Total); result = await processor.BuildQueryAsync("field3:\"hey now\"", new ElasticQueryVisitorContext().UseSearchMode()); - actualResponse = Client.Search(d => d.Index(index).Query(_ => result)); + actualResponse = await Client.SearchAsync(d => d.Index(index).Query(_ => result)); actualRequest = actualResponse.GetRequest(); _logger.LogInformation("Actual: {Request}", actualRequest); - expectedResponse = Client.Search(d => d.Index(index).Query(q => q.MatchPhrase(m => m + expectedResponse = await Client.SearchAsync(d => d.Index(index).Query(q => q.MatchPhrase(m => m .Field(f => f.Field3) .Query("hey now") ))); @@ -604,10 +605,10 @@ await Client.IndexManyAsync([ Assert.Equal(expectedResponse.Total, actualResponse.Total); result = await processor.BuildQueryAsync("field3:hey*", new ElasticQueryVisitorContext().UseSearchMode()); - actualResponse = Client.Search(d => d.Index(index).Query(_ => result)); + actualResponse = await Client.SearchAsync(d => d.Index(index).Query(_ => result)); actualRequest = actualResponse.GetRequest(); _logger.LogInformation("Actual: {Request}", actualRequest); - expectedResponse = Client.Search(d => d.Index(index).Query(q => q.QueryString(m => m + expectedResponse = await Client.SearchAsync(d => d.Index(index).Query(q => q.QueryString(m => m .AllowLeadingWildcard(false) .AnalyzeWildcard(true) .Fields(f => f.Field(f1 => f1.Field3)) @@ -634,11 +635,11 @@ await Client.IndexManyAsync([ var processor = new ElasticQueryParser(c => c.SetLoggerFactory(Log)); var result = await processor.BuildQueryAsync("field1:value1 AND -field2:value2", new ElasticQueryVisitorContext().UseSearchMode()); - var actualResponse = Client.Search(d => d.Index(index).Query(_ => result)); + var actualResponse = await Client.SearchAsync(d => d.Index(index).Query(_ => result)); string actualRequest = actualResponse.GetRequest(); _logger.LogInformation("Actual: {Request}", actualRequest); - var expectedResponse = Client.Search(d => d + var expectedResponse = await Client.SearchAsync(d => d .Index(index).Query(f => f.Term(m => m.Field1, "value1") && !f.Term(m => m.Field2, "value2"))); string expectedRequest = expectedResponse.GetRequest(); _logger.LogInformation("Expected: {Request}", expectedRequest); @@ -649,11 +650,11 @@ await Client.IndexManyAsync([ processor = new ElasticQueryParser(c => c.SetLoggerFactory(Log)); result = await processor.BuildQueryAsync("field1:value1 AND NOT field2:value2", new ElasticQueryVisitorContext().UseSearchMode()); - actualResponse = Client.Search(d => d.Index(index).Query(_ => result)); + actualResponse = await Client.SearchAsync(d => d.Index(index).Query(_ => result)); actualRequest = actualResponse.GetRequest(); _logger.LogInformation("Actual: {Request}", actualRequest); - expectedResponse = Client.Search(d => d + expectedResponse = await Client.SearchAsync(d => d .Index(index).Query(f => f.Term(m => m.Field1, "value1") && !f.Term(m => m.Field2, "value2"))); expectedRequest = expectedResponse.GetRequest(); _logger.LogInformation("Expected: {Request}", expectedRequest); @@ -664,11 +665,11 @@ await Client.IndexManyAsync([ processor = new ElasticQueryParser(c => c.SetLoggerFactory(Log)); result = await processor.BuildQueryAsync("field1:value1 OR NOT field2:value2", new ElasticQueryVisitorContext().UseSearchMode()); - actualResponse = Client.Search(d => d.Index(index).Query(_ => result)); + actualResponse = await Client.SearchAsync(d => d.Index(index).Query(_ => result)); actualRequest = actualResponse.GetRequest(); _logger.LogInformation("Actual: {Request}", actualRequest); - expectedResponse = Client.Search(d => d + expectedResponse = await Client.SearchAsync(d => d .Index(index).Query(f => f.Term(m => m.Field1, "value1") || !f.Term(m => m.Field2, "value2"))); expectedRequest = expectedResponse.GetRequest(); _logger.LogInformation("Expected: {Request}", expectedRequest); @@ -679,11 +680,11 @@ await Client.IndexManyAsync([ processor = new ElasticQueryParser(c => c.SetLoggerFactory(Log)); result = await processor.BuildQueryAsync("field1:value1 OR -field2:value2", new ElasticQueryVisitorContext().UseSearchMode()); - actualResponse = Client.Search(d => d.Index(index).Query(_ => result)); + actualResponse = await Client.SearchAsync(d => d.Index(index).Query(_ => result)); actualRequest = actualResponse.GetRequest(); _logger.LogInformation("Actual: {Request}", actualRequest); - expectedResponse = Client.Search(d => d + expectedResponse = await Client.SearchAsync(d => d .Index(index).Query(f => f.Term(m => m.Field1, "value1") || !f.Term(m => m.Field2, "value2"))); expectedRequest = expectedResponse.GetRequest(); _logger.LogInformation("Expected: {Request}", expectedRequest); @@ -707,11 +708,11 @@ await Client.IndexManyAsync([ var result = await processor.BuildQueryAsync("field1:value1 (field2:value2 OR field3:value3)", new ElasticQueryVisitorContext().UseSearchMode()); - var actualResponse = Client.Search(d => d.Index(index).Query(_ => result)); + var actualResponse = await Client.SearchAsync(d => d.Index(index).Query(_ => result)); string actualRequest = actualResponse.GetRequest(); _logger.LogInformation("Actual: {Request}", actualRequest); - var expectedResponse = Client.Search(d => d.Index(index) + var expectedResponse = await Client.SearchAsync(d => d.Index(index) .Query(f => f.Term(m => m.Field1, "value1") || (f.Term(m => m.Field2, "value2") || f.Term(m => m.Field3, "value3")))); string expectedRequest = expectedResponse.GetRequest(); @@ -735,12 +736,12 @@ await Client.IndexManyAsync([ var processor = new ElasticQueryParser(c => c.SetLoggerFactory(Log)); var result = await processor.BuildQueryAsync("field1:value1 (field2:value2 OR field3:value3)"); - var actualResponse = Client.Search(d => d.Index(index).Query(_ => result)); + var actualResponse = await Client.SearchAsync(d => d.Index(index).Query(_ => result)); string actualRequest = actualResponse.GetRequest(); _logger.LogInformation("Actual: {Request}", actualRequest); var expectedResponse = - Client.Search(d => d.Index(index) + await Client.SearchAsync(d => d.Index(index) .Query(q => q.Bool(b => b.Filter(f => f .Term(m => m.Field1, "value1") && (f.Term(m => m.Field2, "value2") || f.Term(m => m.Field3, "value3")))))); @@ -759,11 +760,11 @@ public async Task MixedCaseTermFilterQueryProcessor() var processor = new ElasticQueryParser(c => c.SetLoggerFactory(Log)); var result = await processor.BuildQueryAsync("field1:Testing.Casing", new ElasticQueryVisitorContext { UseScoring = true }); - var actualResponse = Client.Search(d => d.Index(index).Query(_ => result)); + var actualResponse = await Client.SearchAsync(d => d.Index(index).Query(_ => result)); string actualRequest = actualResponse.GetRequest(); _logger.LogInformation("Actual: {Request}", actualRequest); - var expectedResponse = Client.Search(d => d.Index(index).Query(f => f.Term(m => m.Field1, "Testing.Casing"))); + var expectedResponse = await Client.SearchAsync(d => d.Index(index).Query(f => f.Term(m => m.Field1, "Testing.Casing"))); string expectedRequest = expectedResponse.GetRequest(); _logger.LogInformation("Expected: {Request}", expectedRequest); @@ -779,11 +780,11 @@ public async Task MultipleWordsTermFilterQueryProcessor() var processor = new ElasticQueryParser(c => c.SetLoggerFactory(Log)); var result = await processor.BuildQueryAsync("field1:\"Blake Niemyjski\"", new ElasticQueryVisitorContext { UseScoring = true }); - var actualResponse = Client.Search(d => d.Index(index).Query(_ => result)); + var actualResponse = await Client.SearchAsync(d => d.Index(index).Query(_ => result)); string actualRequest = actualResponse.GetRequest(); _logger.LogInformation("Actual: {Request}", actualRequest); - var expectedResponse = Client.Search(d => d.Index(index).Query(f => f.Term(p => p.Field1, "Blake Niemyjski"))); + var expectedResponse = await Client.SearchAsync(d => d.Index(index).Query(f => f.Term(p => p.Field1, "Blake Niemyjski"))); string expectedRequest = expectedResponse.GetRequest(); _logger.LogInformation("Expected: {Request}", expectedRequest); @@ -799,11 +800,11 @@ public async Task CanTranslateTermQueryProcessor() var processor = new ElasticQueryParser(c => c.SetLoggerFactory(Log).AddVisitor(new UpdateFixedTermFieldToDateFixedExistsQueryVisitor())); var result = await processor.BuildQueryAsync("fixed:true"); - var actualResponse = Client.Search(d => d.Index(index).Query(_ => result)); + var actualResponse = await Client.SearchAsync(d => d.Index(index).Query(_ => result)); string actualRequest = actualResponse.GetRequest(); _logger.LogInformation("Actual: {Request}", actualRequest); - var expectedResponse = Client.Search(d => d + var expectedResponse = await Client.SearchAsync(d => d .Index(index).Query(f => f.Bool(b => b.Filter(filter => filter.Exists(m => m.Field("date_fixed")))))); string expectedRequest = expectedResponse.GetRequest(); _logger.LogInformation("Expected: {Request}", expectedRequest); @@ -827,11 +828,11 @@ await Client.IndexManyAsync([ var result = await processor.BuildQueryAsync("field1:value1 (field2:value2 OR field3:value3)", new ElasticQueryVisitorContext().SetDefaultOperator(Operator.And).UseScoring()); - var actualResponse = Client.Search(d => d.Index(index).Query(_ => result)); + var actualResponse = await Client.SearchAsync(d => d.Index(index).Query(_ => result)); string actualRequest = actualResponse.GetRequest(); _logger.LogInformation("Actual: {Request}", actualRequest); - var expectedResponse = Client.Search(d => d.Index(index) + var expectedResponse = await Client.SearchAsync(d => d.Index(index) .Query(f => f.Term(m => m.Field1, "value1") && (f.Term(m => m.Field2, "value2") || f.Term(m => m.Field3, "value3")))); string expectedRequest = expectedResponse.GetRequest(); @@ -871,11 +872,11 @@ await Client.IndexManyAsync([ var processor = new ElasticQueryParser(c => c.SetLoggerFactory(Log).UseFieldMap(new FieldMap { { "blah", "nested" } }).UseMappings(Client).UseNested()); var result = await processor.BuildQueryAsync("field1:value1 blah:(blah.field1:value1)", new ElasticQueryVisitorContext().UseScoring()); - var actualResponse = Client.Search(d => d.Query(_ => result)); + var actualResponse = await Client.SearchAsync(d => d.Query(_ => result)); string actualRequest = actualResponse.GetRequest(); _logger.LogInformation("Actual: {Request}", actualRequest); - var expectedResponse = Client.Search(d => d + var expectedResponse = await Client.SearchAsync(d => d .Query(q => q.Match(m => m.Field(e => e.Field1).Query("value1")) && q.Nested(n => n .Path(p => p.Nested) @@ -892,11 +893,11 @@ await Client.IndexManyAsync([ result = await processor.BuildQueryAsync("field1:value1 blah:(blah.field1:value1 blah.field4:4)", new ElasticQueryVisitorContext().UseScoring()); - actualResponse = Client.Search(d => d.Query(_ => result)); + actualResponse = await Client.SearchAsync(d => d.Query(_ => result)); actualRequest = actualResponse.GetRequest(); _logger.LogInformation("Actual: {Request}", actualRequest); - expectedResponse = Client.Search(d => d + expectedResponse = await Client.SearchAsync(d => d .Query(q => q.Match(m => m.Field(e => e.Field1).Query("value1")) && q.Nested(n => n .Path(p => p.Nested) @@ -945,11 +946,11 @@ await Client.IndexManyAsync([ var result = await processor.BuildQueryAsync("field1:value1 nested:(nested.field1:value1 nested.field4:4 nested.field3:value3)", new ElasticQueryVisitorContext { UseScoring = true }); - var actualResponse = Client.Search(d => d.Query(_ => result)); + var actualResponse = await Client.SearchAsync(d => d.Query(_ => result)); string actualRequest = actualResponse.GetRequest(); _logger.LogInformation("Actual: {Request}", actualRequest); - var expectedResponse = Client.Search(d => d.Query(q => q.Match(m => m.Field(e => e.Field1).Query("value1")) + var expectedResponse = await Client.SearchAsync(d => d.Query(q => q.Match(m => m.Field(e => e.Field1).Query("value1")) && q.Nested(n => n.Path(p => p.Nested).Query(q2 => q2.Match(m => m.Field("nested.field1").Query("value1")) && q2.Term("nested.field4", "4") @@ -973,11 +974,11 @@ public async Task CanGenerateMatchQuery() var processor = new ElasticQueryParser(c => c.SetLoggerFactory(Log).UseMappings(Client, index)); var result = await processor.BuildQueryAsync("field1:test", new ElasticQueryVisitorContext().UseScoring()); - var actualResponse = Client.Search(d => d.Index(index).Query(_ => result)); + var actualResponse = await Client.SearchAsync(d => d.Index(index).Query(_ => result)); string actualRequest = actualResponse.GetRequest(); _logger.LogInformation("Actual: {Request}", actualRequest); - var expectedResponse = Client.Search(d => d.Index(index).Query(q => q.Match(m => m.Field(e => e.Field1).Query("test")))); + var expectedResponse = await Client.SearchAsync(d => d.Index(index).Query(q => q.Match(m => m.Field(e => e.Field1).Query("test")))); string expectedRequest = expectedResponse.GetRequest(); _logger.LogInformation("Expected: {Request}", expectedRequest); @@ -996,11 +997,11 @@ public async Task CanBuildAliasQueryProcessor() var processor = new ElasticQueryParser(c => c.SetLoggerFactory(Log)); var result = await processor.BuildQueryAsync("browser.version:1", new ElasticQueryVisitorContext().UseScoring()); - var actualResponse = Client.Search(d => d.Index(index).Query(_ => result)); + var actualResponse = await Client.SearchAsync(d => d.Index(index).Query(_ => result)); string actualRequest = actualResponse.GetRequest(); _logger.LogInformation("Actual: {Request}", actualRequest); - var expectedResponse = Client.Search(d => d.Index(index).Query(q => q.Term(m => m.Field("browser.version").Value("1")))); + var expectedResponse = await Client.SearchAsync(d => d.Index(index).Query(q => q.Term(m => m.Field("browser.version").Value("1")))); string expectedRequest = expectedResponse.GetRequest(); _logger.LogInformation("Expected: {Request}", expectedRequest); @@ -1018,11 +1019,11 @@ public async Task NonAnalyzedPrefixQuery() var processor = new ElasticQueryParser(c => c.UseMappings(Client, index)); var result = await processor.BuildQueryAsync("field1:value*", new ElasticQueryVisitorContext().UseSearchMode()); - var actualResponse = Client.Search(d => d.Index(index).Query(_ => result)); + var actualResponse = await Client.SearchAsync(d => d.Index(index).Query(_ => result)); string actualRequest = actualResponse.GetRequest(); _logger.LogInformation("Actual: {Request}", actualResponse); - var expectedResponse = Client.Search(d => d + var expectedResponse = await Client.SearchAsync(d => d .Index(index) .Query(f => f.Prefix(m => m.Field(f2 => f2.Field1).Value("value")))); @@ -1047,12 +1048,12 @@ public async Task RangeQueryProcessor() await processor.BuildQueryAsync("field4:[1 TO 2} OR field1:value1", new ElasticQueryVisitorContext { UseScoring = true }); - var actualResponse = Client.Search(d => d.Index(index).Query(_ => result)); + var actualResponse = await Client.SearchAsync(d => d.Index(index).Query(_ => result)); string actualRequest = actualResponse.GetRequest(); _logger.LogInformation("Actual: {Request}", actualRequest); var expectedResponse = - Client.Search( + await Client.SearchAsync( d => d.Index(index) .Query( @@ -1081,11 +1082,11 @@ public async Task DateRangeWithWildcardMinQueryProcessor() var result = await processor.BuildQueryAsync("field5:[* TO 2017-01-31} OR field1:value1", ctx); - var actualResponse = Client.Search(d => d.Index(index).Query(_ => result)); + var actualResponse = await Client.SearchAsync(d => d.Index(index).Query(_ => result)); string actualRequest = actualResponse.GetRequest(); _logger.LogInformation("Actual: {Request}", actualResponse); - var expectedResponse = Client.Search(d => d + var expectedResponse = await Client.SearchAsync(d => d .Index(index) .Query(f => f .DateRange(m => m.Field(f2 => f2.Field5).LessThan("2017-01-31").TimeZone("America/Chicago")) @@ -1113,11 +1114,11 @@ public async Task DateRangeWithDateMathQueryProcessor() var result = await processor.BuildQueryAsync("field5:[now-1d/d TO now/d]", ctx); - var actualResponse = Client.Search(d => d.Index(index).Query(_ => result)); + var actualResponse = await Client.SearchAsync(d => d.Index(index).Query(_ => result)); string actualRequest = actualResponse.GetRequest(); _logger.LogInformation("Actual: {Request}", actualResponse); - var expectedResponse = Client.Search(d => d + var expectedResponse = await Client.SearchAsync(d => d .Index(index) .Query(f => f.DateRange(m => m.Field(f2 => f2.Field5).GreaterThanOrEquals("now-1d/d").LessThanOrEquals("now/d").TimeZone("America/Chicago")))); @@ -1143,11 +1144,11 @@ public async Task DateRangeWithWildcardMaxQueryProcessor() var result = await processor.BuildQueryAsync("field5:[2017-01-31 TO * } OR field1:value1", ctx); - var actualResponse = Client.Search(d => d.Index(index).Query(_ => result)); + var actualResponse = await Client.SearchAsync(d => d.Index(index).Query(_ => result)); string actualRequest = actualResponse.GetRequest(); _logger.LogInformation("Actual: {Request}", actualResponse); - var expectedResponse = await Client.SearchAsync(d => d + var expectedResponse = await Client.SearchAsyncAsync(d => d .Index(index) .Query(f => f .DateRange(m => m.Field(f2 => f2.Field5).GreaterThanOrEquals("2017-01-31").TimeZone("America/Chicago")) @@ -1174,11 +1175,11 @@ public async Task DateRangeWithTimeZone() var result = await processor.BuildQueryAsync("field5:[2017-01-31 TO * }^\"America/Chicago\" OR field1:value1", ctx); - var actualResponse = Client.Search(d => d.Index(index).Query(_ => result)); + var actualResponse = await Client.SearchAsync(d => d.Index(index).Query(_ => result)); string actualRequest = actualResponse.GetRequest(); _logger.LogInformation("Actual: {Request}", actualResponse); - var expectedResponse = Client.Search(d => d + var expectedResponse = await Client.SearchAsync(d => d .Index(index) .Query(f => f .DateRange(m => m.Field(f2 => f2.Field5).GreaterThanOrEquals("2017-01-31").TimeZone("America/Chicago")) @@ -1204,12 +1205,12 @@ public async Task DateRangeQueryProcessor() var processor = new ElasticQueryParser(c => c.UseMappings(Client, index).SetLoggerFactory(Log)); var result = await processor.BuildQueryAsync("field5:[2017-01-01T00\\:00\\:00Z TO 2017-01-31} OR field1:value1", ctx); - var actualResponse = Client.Search(d => d.Index(index).Query(_ => result)); + var actualResponse = await Client.SearchAsync(d => d.Index(index).Query(_ => result)); string actualRequest = actualResponse.GetRequest(); _logger.LogInformation("Actual: {Request}", actualRequest); var expectedResponse = - Client.Search(d => d + await Client.SearchAsync(d => d .Index(index) .Query(f => f .DateRange(m => m @@ -1239,11 +1240,11 @@ public async Task SimpleGeoRangeQuery() await processor.BuildQueryAsync("field3:[51.5032520,-0.1278990 TO 51.5032520,-0.1278990]", new ElasticQueryVisitorContext { UseScoring = true }); - var actualResponse = Client.Search(d => d.Index(index).Query(_ => result)); + var actualResponse = await Client.SearchAsync(d => d.Index(index).Query(_ => result)); string actualRequest = actualResponse.GetRequest(); _logger.LogInformation("Actual: {Request}", actualRequest); - var expectedResponse = Client.Search(d => d.Index(index).Query(q => + var expectedResponse = await Client.SearchAsync(d => d.Index(index).Query(q => q.GeoBoundingBox( m => m.Field(p => p.Field3).BoundingBox("51.5032520,-0.1278990", "51.5032520,-0.1278990")))); string expectedRequest = expectedResponse.GetRequest(); @@ -1302,13 +1303,13 @@ public async Task CanSortByUnmappedField() var processor = new ElasticQueryParser(c => c.UseMappings(Client, index)); var sort = await processor.BuildSortAsync("-field1"); - var actualResponse = Client.Search(d => d.Index(index).Sort(sort)); + var actualResponse = await Client.SearchAsync(d => d.Index(index).Sort(sort)); - Assert.True(actualResponse.IsValid); + Assert.True(actualResponse.IsValidResponse); string actualRequest = actualResponse.GetRequest(true); _logger.LogInformation("Actual: {Request}", actualResponse); - var expectedResponse = Client.Search(d => d.Index(index) + var expectedResponse = await Client.SearchAsync(d => d.Index(index) .Sort( s => s.Field(f => f.Field(new Field("field1")).Descending().UnmappedType(FieldType.Keyword)) )); @@ -1338,11 +1339,11 @@ public async Task CanParseSort() .UseFieldMap(aliasMap)); var sort = await processor.BuildSortAsync("geo -field1 -(field2 field3 +field4) (field5 field3)"); - var actualResponse = Client.Search(d => d.Index(index).Sort(sort)); + var actualResponse = await Client.SearchAsync(d => d.Index(index).Sort(sort)); string actualRequest = actualResponse.GetRequest(true); _logger.LogInformation("Actual: {Request}", actualResponse); - var expectedResponse = Client.Search(d => d.Index(index).Sort(s => s + var expectedResponse = await Client.SearchAsync(d => d.Index(index).Sort(s => s .Field(f => f.Field(new Field("field3")).Ascending().UnmappedType(FieldType.GeoPoint)) .Field(f => f.Field(new Field("field1.keyword")).Descending().UnmappedType(FieldType.Keyword)) .Field(f => f.Field(new Field("field2.sort")).Descending().UnmappedType(FieldType.Keyword)) @@ -1390,10 +1391,10 @@ await Client.IndexManyAsync([ var sort = await processor.BuildSortAsync("nested.data.spaced\\ field"); var query = await processor.BuildQueryAsync("nested.data.spaced\\ field:hey"); var aggs = await processor.BuildAggregationsAsync("terms:nested.data.spaced\\ field"); - var actualResponse = Client.Search(d => d.Index(index).Sort(sort).Query(_ => query).Aggregations(_ => aggs)); + var actualResponse = await Client.SearchAsync(d => d.Index(index).Sort(sort).Query(_ => query).Aggregations(_ => aggs)); string actualRequest = actualResponse.GetRequest(); _logger.LogInformation("Actual: {Request}", actualRequest); - var expectedResponse = Client.Search(d => d.Index(index) + var expectedResponse = await Client.SearchAsync(d => d.Index(index) .Sort(s => s .Field(f => f.Field("nested.data.spaced field.keyword").UnmappedType(FieldType.Keyword).Ascending())) .Query(q => q.Bool(b => b.Filter(f => f @@ -1417,10 +1418,10 @@ public async Task CanParseMixedCaseSort() await Client.Indices.RefreshAsync(index); var processor = new ElasticQueryParser(c => c.SetLoggerFactory(Log).UseMappings(Client, index)); var sort = await processor.BuildSortAsync("multiWord -multiword"); - var actualResponse = Client.Search(d => d.Index(index).Sort(sort)); + var actualResponse = await Client.SearchAsync(d => d.Index(index).Sort(sort)); string actualRequest = actualResponse.GetRequest(); _logger.LogInformation("Actual: {Request}", actualRequest); - var expectedResponse = Client.Search(d => d.Index(index) + var expectedResponse = await Client.SearchAsync(d => d.Index(index) .Sort(s => s .Field(f => f.Field("multiWord.keyword").UnmappedType(FieldType.Keyword).Ascending()) .Field(f => f.Field("multiWord.keyword").UnmappedType(FieldType.Keyword).Descending()) @@ -1453,11 +1454,11 @@ public async Task GeoRangeQueryProcessor() var result = await processor.BuildQueryAsync("geo:[51.5032520,-0.1278990 TO 51.5032520,-0.1278990] OR field1:value1 OR field2:[1 TO 4] OR -geo:\"Dallas, TX\"~75mi", new ElasticQueryVisitorContext { UseScoring = true }); var sort = await processor.BuildSortAsync("geo -field1"); - var actualResponse = Client.Search(d => d.Index(index).Query(_ => result).Sort(sort)); + var actualResponse = await Client.SearchAsync(d => d.Index(index).Query(_ => result).Sort(sort)); string actualRequest = actualResponse.GetRequest(true); _logger.LogInformation("Actual: {Request}", actualResponse); - var expectedResponse = Client.Search(d => d.Index(index) + var expectedResponse = await Client.SearchAsync(d => d.Index(index) .Sort(s => s .Field(f => f.Field(new Field("field3")).Ascending().UnmappedType(FieldType.GeoPoint)) .Field(f => f.Field(new Field("field1.keyword")).Descending().UnmappedType(FieldType.Keyword)) @@ -1482,11 +1483,11 @@ public async Task CanExpandElasticIncludesAsync() var processor = new ElasticQueryParser(c => c.SetLoggerFactory(Log).UseIncludes(GetIncludeAsync).UseFieldMap(aliases)); var result = await processor.BuildQueryAsync("@include:other"); - var actualResponse = Client.Search(d => d.Index("stuff").Query(_ => result)); + var actualResponse = await Client.SearchAsync(d => d.Index("stuff").Query(_ => result)); string actualRequest = actualResponse.GetRequest(); _logger.LogInformation("Actual: {Request}", actualRequest); - var expectedResponse = Client.Search(d => d.Index("stuff").Query(f => f.Bool(b => b.Filter(f1 => f1.Term("aliasedincluded", "value"))))); + var expectedResponse = await Client.SearchAsync(d => d.Index("stuff").Query(f => f.Bool(b => b.Filter(f1 => f1.Term("aliasedincluded", "value"))))); string expectedRequest = expectedResponse.GetRequest(); _logger.LogInformation("Expected: {Request}", expectedRequest); @@ -1494,7 +1495,7 @@ public async Task CanExpandElasticIncludesAsync() Assert.Equal(expectedResponse.Total, actualResponse.Total); result = await processor.BuildQueryAsync("@include:other"); - actualResponse = Client.Search(d => d.Index("stuff").Query(_ => result)); + actualResponse = await Client.SearchAsync(d => d.Index("stuff").Query(_ => result)); actualRequest = actualResponse.GetRequest(); _logger.LogInformation("Actual: {Request}", actualRequest); _logger.LogInformation("Expected: {Request}", expectedRequest); diff --git a/tests/Foundatio.Parsers.ElasticQueries.Tests/InvertQueryTests.cs b/tests/Foundatio.Parsers.ElasticQueries.Tests/InvertQueryTests.cs index 3447fc49..24f4cfbd 100644 --- a/tests/Foundatio.Parsers.ElasticQueries.Tests/InvertQueryTests.cs +++ b/tests/Foundatio.Parsers.ElasticQueries.Tests/InvertQueryTests.cs @@ -1,6 +1,7 @@ using System; using System.Collections.Generic; using System.Threading.Tasks; +using Elastic.Clients.Elasticsearch; using Foundatio.Parsers.LuceneQueries; using Foundatio.Parsers.LuceneQueries.Extensions; using Foundatio.Parsers.LuceneQueries.Nodes; @@ -188,7 +189,6 @@ public override async Task InitializeAsync() } await Client.IndexManyAsync(records, indexName); - await Client.Indices.RefreshAsync(indexName); } } diff --git a/tests/Foundatio.Parsers.ElasticQueries.Tests/Utility/ElasticsearchTestBase.cs b/tests/Foundatio.Parsers.ElasticQueries.Tests/Utility/ElasticsearchTestBase.cs index 00589aa8..06805b00 100644 --- a/tests/Foundatio.Parsers.ElasticQueries.Tests/Utility/ElasticsearchTestBase.cs +++ b/tests/Foundatio.Parsers.ElasticQueries.Tests/Utility/ElasticsearchTestBase.cs @@ -30,17 +30,17 @@ public ElasticsearchTestBase(ITestOutputHelper output, T fixture) : base(output) protected ElasticsearchClient Client => _fixture.Client; - protected void CreateNamedIndex(string index, Func, TypeMapping> configureMappings = null, Func> configureIndex = null) where TModel : class + protected void CreateNamedIndex(string index, Func, TypeMapping> configureMappings = null, Func configureIndex = null) where TModel : class { _fixture.CreateNamedIndex(index, configureMappings, configureIndex); } - protected string CreateRandomIndex(Func, TypeMapping> configureMappings = null, Func> configureIndex = null) where TModel : class + protected string CreateRandomIndex(Func, TypeMapping> configureMappings = null, Func configureIndex = null) where TModel : class { return _fixture.CreateRandomIndex(configureMappings, configureIndex); } - protected CreateIndexResponse CreateIndex(IndexName index, Func configureIndex = null) + protected CreateIndexResponse CreateIndex(IndexName index, Func configureIndex = null) { return _fixture.CreateIndex(index, configureIndex); } @@ -98,49 +98,44 @@ protected ElasticsearchClient GetClient(Action conf protected virtual void ConfigureConnectionSettings(ElasticsearchClientSettings settings) { } - public void CreateNamedIndex(string index, Func, TypeMapping> configureMappings = null, Func> configureIndex = null) where T : class + public void CreateNamedIndex(string index, Func, TypeMapping> configureMappings = null, Func configureIndex = null) where T : class { - if (configureMappings == null) - configureMappings = m => m.AutoMap().Dynamic(); + configureMappings ??= m => m.AutoMap().Dynamic(); if (configureIndex == null) configureIndex = i => i.NumberOfReplicas(0).Analysis(a => a.AddSortNormalizer()); - CreateIndex(index, i => i.Settings(configureIndex).Map(configureMappings)); + CreateIndex(index, i => i.Settings(configureIndex(new IndexSettingsDescriptor())).Map(configureMappings)); Client.ElasticsearchClientSettings.DefaultIndices[typeof(T)] = index; } - public string CreateRandomIndex(Func, TypeMapping> configureMappings = null, Func> configureIndex = null) where T : class + public string CreateRandomIndex(Func, TypeMapping> configureMappings = null, Func configureIndex = null) where T : class { string index = "test_" + Guid.NewGuid().ToString("N"); - if (configureMappings == null) - configureMappings = m => m.AutoMap().Dynamic(); - if (configureIndex == null) - configureIndex = i => i.NumberOfReplicas(0).Analysis(a => a.AddSortNormalizer()); + configureMappings ??= m => m.AutoMap().Dynamic(); + configureIndex ??= i => i.NumberOfReplicas(0).Analysis(a => a.AddSortNormalizer()); - CreateIndex(index, i => i.Settings(configureIndex).Map(configureMappings)); + CreateIndex(index, i => i.Settings(configureIndex(new IndexSettingsDescriptor())).Map(configureMappings)); Client.ElasticsearchClientSettings.DefaultIndices[typeof(T)] = index; return index; } - public CreateIndexResponse CreateIndex(IndexName index, Func configureIndex = null) + public CreateIndexResponse CreateIndex(IndexName index, Func configureIndex = null) { _createdIndexes.Add(index); - if (configureIndex == null) - configureIndex = d => d.Settings(s => s.NumberOfReplicas(0)); - + configureIndex ??= d => d.Settings(s => s.NumberOfReplicas(0)); var result = Client.Indices.Create(index, configureIndex); - if (!result.IsValid) + if (!result.IsValidResponse) throw new ApplicationException($"Unable to create index {index}: " + result.DebugInformation); return result; } - protected virtual void CleanupTestIndexes(ElasticsearchClient client) + protected virtual async Task CleanupTestIndexesAsync(ElasticsearchClient client) { if (_createdIndexes.Count > 0) - client.Indices.Delete(Indices.Index(_createdIndexes)); + await client.Indices.DeleteAsync(Indices.Index(_createdIndexes)); } public virtual Task InitializeAsync() @@ -150,7 +145,6 @@ public virtual Task InitializeAsync() public virtual Task DisposeAsync() { - CleanupTestIndexes(Client); - return Task.CompletedTask; + return CleanupTestIndexesAsync(Client); } } From 104d5e45ab11b8395ba4bc8b36b97a67007bd2e3 Mon Sep 17 00:00:00 2001 From: Blake Niemyjski Date: Thu, 12 Sep 2024 16:53:34 -0500 Subject: [PATCH 10/19] WIP - TESTS --- .../ElasticMappingResolverTests.cs | 13 +++++++------ .../ElasticQueryParserTests.cs | 7 +++---- .../InvertQueryTests.cs | 15 ++++++++------- .../Utility/ElasticsearchTestBase.cs | 4 ++-- 4 files changed, 20 insertions(+), 19 deletions(-) diff --git a/tests/Foundatio.Parsers.ElasticQueries.Tests/ElasticMappingResolverTests.cs b/tests/Foundatio.Parsers.ElasticQueries.Tests/ElasticMappingResolverTests.cs index 1e1577b1..a9bc67b4 100644 --- a/tests/Foundatio.Parsers.ElasticQueries.Tests/ElasticMappingResolverTests.cs +++ b/tests/Foundatio.Parsers.ElasticQueries.Tests/ElasticMappingResolverTests.cs @@ -3,6 +3,7 @@ using System.Threading.Tasks; using Elastic.Clients.Elasticsearch; using Elastic.Clients.Elasticsearch.Mapping; +using Foundatio.Parsers.ElasticQueries.Extensions; using Xunit; using Xunit.Abstractions; @@ -62,7 +63,7 @@ await Client.IndexManyAsync([ var payloadProperty = resolver.GetMappingProperty("payload"); Assert.IsType(payloadProperty); - Assert.NotNull(payloadProperty.Name); + Assert.NotNull(payloadProperty.TryGetName()); } [Fact] @@ -144,11 +145,11 @@ await Client.IndexManyAsync([ Assert.IsType(field4AliasMapping.Property); Assert.Same(field4Property, field4AliasMapping.Property); - string field4sort = resolver.GetSortFieldName("Field4Alias"); - Assert.Equal("field4.sort", field4sort); + string field4Sort = resolver.GetSortFieldName("Field4Alias"); + Assert.Equal("field4.sort", field4Sort); - string field4aggs = resolver.GetAggregationsFieldName("Field4Alias"); - Assert.Equal("field4.keyword", field4aggs); + string field4Aggs = resolver.GetAggregationsFieldName("Field4Alias"); + Assert.Equal("field4.keyword", field4Aggs); var nestedIdProperty = resolver.GetMappingProperty("Nested.Id"); Assert.IsType(nestedIdProperty); @@ -166,7 +167,7 @@ await Client.IndexManyAsync([ Assert.IsType(nestedField1Property); var nestedField2Property = resolver.GetMappingProperty("Nested.Field4"); - Assert.IsType(nestedField2Property); + Assert.IsType(nestedField2Property); var nestedField5Property = resolver.GetMappingProperty("Nested.Field5"); Assert.IsType(nestedField5Property); diff --git a/tests/Foundatio.Parsers.ElasticQueries.Tests/ElasticQueryParserTests.cs b/tests/Foundatio.Parsers.ElasticQueries.Tests/ElasticQueryParserTests.cs index 104e9eee..e6364e8c 100644 --- a/tests/Foundatio.Parsers.ElasticQueries.Tests/ElasticQueryParserTests.cs +++ b/tests/Foundatio.Parsers.ElasticQueries.Tests/ElasticQueryParserTests.cs @@ -22,7 +22,7 @@ public class ElasticQueryParserTests : ElasticsearchTestBase { public ElasticQueryParserTests(ITestOutputHelper output, ElasticsearchFixture fixture) : base(output, fixture) { - Log.DefaultMinimumLevel = Microsoft.Extensions.Logging.LogLevel.Trace; + Log.DefaultMinimumLevel = LogLevel.Trace; } [Fact] @@ -53,12 +53,11 @@ public async Task CanHandleEmptyAndNullString() public void CanUseElasticQueryParser() { var sut = new ElasticQueryParser(); - var result = sut.Parse("NOT (dog parrot)"); Assert.IsType(result.Left); - Assert.True((result.Left as GroupNode).HasParens); - Assert.True((result.Left as GroupNode).IsNegated); + Assert.True(((GroupNode)result.Left).HasParens); + Assert.True(((GroupNode)result.Left).IsNegated); } [Fact] diff --git a/tests/Foundatio.Parsers.ElasticQueries.Tests/InvertQueryTests.cs b/tests/Foundatio.Parsers.ElasticQueries.Tests/InvertQueryTests.cs index 24f4cfbd..676b2ceb 100644 --- a/tests/Foundatio.Parsers.ElasticQueries.Tests/InvertQueryTests.cs +++ b/tests/Foundatio.Parsers.ElasticQueries.Tests/InvertQueryTests.cs @@ -2,6 +2,7 @@ using System.Collections.Generic; using System.Threading.Tasks; using Elastic.Clients.Elasticsearch; +using Elastic.Clients.Elasticsearch.Core.Search; using Foundatio.Parsers.LuceneQueries; using Foundatio.Parsers.LuceneQueries.Extensions; using Foundatio.Parsers.LuceneQueries.Nodes; @@ -116,8 +117,8 @@ private async Task InvertAndValidateQuery(string query, string expected, string Assert.Equal(expected, invertedQuery); var total = await Client.CountAsync(); - var results = await Client.SearchAsync(s => s.QueryOnQueryString(query).TrackTotalHits(true)); - var invertedResults = await Client.SearchAsync(s => s.QueryOnQueryString(invertedQuery).TrackTotalHits(true)); + var results = await Client.SearchAsync(s => s.QueryLuceneSyntax(query).TrackTotalHits(new TrackHits(true))); + var invertedResults = await Client.SearchAsync(s => s.QueryLuceneSyntax(invertedQuery).TrackTotalHits(new TrackHits(true))); Assert.Equal(total.Count, results.Total + invertedResults.Total); } @@ -144,11 +145,11 @@ public override async Task InitializeAsync() const string indexName = "test_invert"; CreateNamedIndex(indexName, m => m .Properties(p => p - .Keyword(p1 => p1.Name(n => n.Id)) - .Keyword(p1 => p1.Name(n => n.OrganizationId)) - .Text(p1 => p1.Name(n => n.Description)) - .Keyword(p1 => p1.Name(n => n.Status)) - .Boolean(p1 => p1.Name(n => n.IsDeleted)) + .Keyword(p1 => p1.Id) + .Keyword(p1 => p1.OrganizationId) + .Text(p1 => p1.Description) + .Keyword(p1 => p1.Status) + .Boolean(p1 => p1.IsDeleted) )); var records = new List(); diff --git a/tests/Foundatio.Parsers.ElasticQueries.Tests/Utility/ElasticsearchTestBase.cs b/tests/Foundatio.Parsers.ElasticQueries.Tests/Utility/ElasticsearchTestBase.cs index 06805b00..7ea23b3b 100644 --- a/tests/Foundatio.Parsers.ElasticQueries.Tests/Utility/ElasticsearchTestBase.cs +++ b/tests/Foundatio.Parsers.ElasticQueries.Tests/Utility/ElasticsearchTestBase.cs @@ -98,7 +98,7 @@ protected ElasticsearchClient GetClient(Action conf protected virtual void ConfigureConnectionSettings(ElasticsearchClientSettings settings) { } - public void CreateNamedIndex(string index, Func, TypeMapping> configureMappings = null, Func configureIndex = null) where T : class + public void CreateNamedIndex(string index, Func, TypeMappingDescriptor> configureMappings = null, Func configureIndex = null) where T : class { configureMappings ??= m => m.AutoMap().Dynamic(); if (configureIndex == null) @@ -108,7 +108,7 @@ public void CreateNamedIndex(string index, Func, Typ Client.ElasticsearchClientSettings.DefaultIndices[typeof(T)] = index; } - public string CreateRandomIndex(Func, TypeMapping> configureMappings = null, Func configureIndex = null) where T : class + public string CreateRandomIndex(Func, TypeMappingDescriptor> configureMappings = null, Func configureIndex = null) where T : class { string index = "test_" + Guid.NewGuid().ToString("N"); configureMappings ??= m => m.AutoMap().Dynamic(); From af1f553c6ea8e0a4904ca6f8e9d05d8573580bbd Mon Sep 17 00:00:00 2001 From: Blake Niemyjski Date: Fri, 13 Sep 2024 13:15:31 -0500 Subject: [PATCH 11/19] WIP - Test updates --- .../AggregationParserTests.cs | 44 +++--- .../CustomVisitorTests.cs | 6 +- .../ElasticMappingResolverTests.cs | 4 +- .../ElasticQueryParserTests.cs | 138 +++++++++--------- .../Utility/ElasticsearchTestBase.cs | 8 +- 5 files changed, 103 insertions(+), 97 deletions(-) diff --git a/tests/Foundatio.Parsers.ElasticQueries.Tests/AggregationParserTests.cs b/tests/Foundatio.Parsers.ElasticQueries.Tests/AggregationParserTests.cs index 27f56abb..e534bf5f 100644 --- a/tests/Foundatio.Parsers.ElasticQueries.Tests/AggregationParserTests.cs +++ b/tests/Foundatio.Parsers.ElasticQueries.Tests/AggregationParserTests.cs @@ -3,6 +3,7 @@ using System.Linq; using System.Threading.Tasks; using Elastic.Clients.Elasticsearch; +using Elastic.Clients.Elasticsearch.Mapping; using Foundatio.Parsers.ElasticQueries.Extensions; using Foundatio.Parsers.ElasticQueries.Visitors; using Foundatio.Parsers.LuceneQueries; @@ -21,7 +22,7 @@ public AggregationParserTests(ITestOutputHelper output, ElasticsearchFixture fix [Fact] public async Task ProcessSingleAggregationAsync() { - string index = CreateRandomIndex(d => d.Dynamic().Properties(p => p.GeoPoint(g => g.Name(f => f.Field3)))); + string index = CreateRandomIndex(d => d.Dynamic(DynamicMapping.True).Properties(p => p.GeoPoint(g => g.Field3))); await Client.IndexManyAsync([ new MyType { Field1 = "value1", Field4 = 1, Field3 = "51.5032520,-0.1278990", Field5 = DateTime.UtcNow.Subtract(TimeSpan.FromMinutes(5)), Field2 = "field2" }, new MyType { Field1 = "value2", Field4 = 2, Field3 = "51.5032520,-0.1278990", Field5 = DateTime.UtcNow.Subtract(TimeSpan.FromMinutes(4)) }, @@ -52,7 +53,7 @@ await Client.IndexManyAsync([ [Fact] public async Task ProcessSingleAggregationWithAliasAsync() { - string index = CreateRandomIndex(d => d.Dynamic().Properties(p => p.GeoPoint(g => g.Name(f => f.Field3)))); + string index = CreateRandomIndex(d => d.Dynamic(DynamicMapping.True).Properties(p => p.GeoPoint(g => g.Field3))); await Client.IndexManyAsync([ new MyType { Field1 = "value1", Field4 = 1, Field3 = "51.5032520,-0.1278990", Field5 = DateTime.UtcNow.Subtract(TimeSpan.FromMinutes(5)), Field2 = "field2" }, new MyType { Field1 = "value2", Field4 = 2, Field3 = "51.5032520,-0.1278990", Field5 = DateTime.UtcNow.Subtract(TimeSpan.FromMinutes(4)) }, @@ -84,11 +85,11 @@ await Client.IndexManyAsync([ [Fact] public async Task ProcessAnalyzedAggregationWithAliasAsync() { - string index = CreateRandomIndex(d => d.Dynamic().Properties(p => p - .Text(f => f.Name(n => n.Field1) - .Fields(k => k.Keyword(m => m.Name("keyword")))) + string index = CreateRandomIndex(d => d.Dynamic(DynamicMapping.True).Properties(p => p + .Text(f => f.Field1, o => o + .Fields(k => k.Keyword("keyword"))) .FieldAlias(f => f.Name("heynow").Path(k => k.Field1)) - .GeoPoint(g => g.Name(f => f.Field3)))); + .GeoPoint(g => g.Field3))); await Client.IndexManyAsync([ new MyType { Field1 = "value1", Field4 = 1, Field3 = "51.5032520,-0.1278990", Field5 = DateTime.UtcNow.Subtract(TimeSpan.FromMinutes(5)), Field2 = "field2" }, new MyType { Field1 = "value2", Field4 = 2, Field3 = "51.5032520,-0.1278990", Field5 = DateTime.UtcNow.Subtract(TimeSpan.FromMinutes(4)) }, @@ -120,7 +121,7 @@ await Client.IndexManyAsync([ [Fact] public async Task ProcessAggregationsAsync() { - string index = CreateRandomIndex(d => d.Dynamic().Properties(p => p.GeoPoint(g => g.Name(f => f.Field3)))); + string index = CreateRandomIndex(d => d.Dynamic(DynamicMapping.True).Properties(p => p.GeoPoint(g => g.Field3))); await Client.IndexManyAsync([ new MyType { Field1 = "value1", Field4 = 1, Field3 = "51.5032520,-0.1278990", Field5 = DateTime.UtcNow.Subtract(TimeSpan.FromMinutes(5)), Field2 = "field2" }, new MyType { Field1 = "value2", Field4 = 2, Field3 = "51.5032520,-0.1278990", Field5 = DateTime.UtcNow.Subtract(TimeSpan.FromMinutes(4)) }, @@ -138,7 +139,7 @@ await Client.IndexManyAsync([ _logger.LogInformation("Actual: {Request}", actualRequest); var expectedResponse = await Client.SearchAsync(d => d.Index(index).Aggregations(a => a - .GeoHash("geogrid_field3", h => h.Field("field3").GeoHashPrecision(1) + .Geohash("geogrid_field3", h => h.Field("field3").GeoHashPrecision(1) .Aggregations(a1 => a1.Average("avg_lat", s => s.Script(ss => ss.Source("doc['field3'].lat"))).Average("avg_lon", s => s.Script(ss => ss.Source("doc['field3'].lon"))))) .Terms("terms_field1", t => t.Field("field1.keyword").Meta(m => m.Add("@field_type", "text"))) .Histogram("histogram_field4", h => h.Field("field4").Interval(50).MinimumDocumentCount(0)) @@ -162,8 +163,8 @@ await Client.IndexManyAsync([ [Fact] public async Task ProcessNestedAggregationsWithAliasesAsync() { - string index = CreateRandomIndex(d => d.Dynamic().Properties(p => p - .GeoPoint(g => g.Name(f => f.Field3)) + string index = CreateRandomIndex(d => d.Dynamic(DynamicMapping.True).Properties(p => p + .GeoPoint(g => g.Field3) .Object>(o1 => o1.Name(f1 => f1.Data).Properties(p1 => p1 .Object(o2 => o2.Name("@user").Properties(p2 => p2 .Text(f3 => f3.Name("identity") @@ -225,8 +226,8 @@ await Client.IndexManyAsync([ [Fact] public async Task ProcessAggregationsWithAliasesAsync() { - string index = CreateRandomIndex(d => d.Dynamic().Properties(p => p - .GeoPoint(g => g.Name(f => f.Field3)) + string index = CreateRandomIndex(d => d.Dynamic(DynamicMapping.True).Properties(p => p + .GeoPoint(g => g.Field3) .Object>(o1 => o1.Name(f1 => f1.Data).Properties(p1 => p1 .Object(o2 => o2.Name("@user").Properties(p2 => p2 .Text(f3 => f3.Name("identity").Fields(f => f.Keyword(k => k.Name("keyword").IgnoreAbove(256)))))))))); @@ -447,7 +448,7 @@ public async Task CanSpecifyDefaultValuesAggregations() public Task GeoGridDoesNotResolveLocationForAggregation() { string index = CreateRandomIndex(d => d.Properties(p => p - .GeoPoint(g => g.Name(f => f.Field1)) + .GeoPoint(g => g.Field1) .FieldAlias(a => a.Name("geo").Path(f => f.Field1)))); var processor = new ElasticQueryParser(c => c @@ -463,10 +464,10 @@ public Task GeoGridDoesNotResolveLocationForAggregation() [InlineData("avg:value", true)] [InlineData(" avg : value", true)] [InlineData("avg:value cardinality:value sum:value min:value max:value", true)] - public Task CanParseAggregations(string query, bool IsValidResponse) + public Task CanParseAggregations(string query, bool isValid) { var parser = new ElasticQueryParser(c => c.SetLoggerFactory(Log)); - return GetAggregationQueryInfoAsync(parser, query, IsValidResponse); + return GetAggregationQueryInfoAsync(parser, query, isValid); } public static IEnumerable AggregationTestCases => @@ -525,23 +526,24 @@ public Task CanParseAggregations(string query, bool IsValidResponse) [Theory] [MemberData(nameof(AggregationTestCases))] - public Task GetElasticAggregationQueryInfoAsync(string query, bool IsValidResponse, int maxNodeDepth, HashSet fields, Dictionary> operations) + public Task GetElasticAggregationQueryInfoAsync(string query, bool isValid, int maxNodeDepth, HashSet fields, Dictionary> operations) { var parser = new ElasticQueryParser(c => c.SetLoggerFactory(Log)); - return GetAggregationQueryInfoAsync(parser, query, IsValidResponse, maxNodeDepth, fields, operations); + return GetAggregationQueryInfoAsync(parser, query, isValid, maxNodeDepth, fields, operations); } - private async Task GetAggregationQueryInfoAsync(IQueryParser parser, string query, bool IsValidResponse, int maxNodeDepth = -1, HashSet fields = null, Dictionary> operations = null) + private async Task GetAggregationQueryInfoAsync(IQueryParser parser, string query, bool isValid, int maxNodeDepth = -1, HashSet fields = null, Dictionary> operations = null) { var context = new ElasticQueryVisitorContext { QueryType = QueryTypes.Aggregation }; var queryNode = await parser.ParseAsync(query, context); + Assert.NotNull(queryNode); var result = context.GetValidationResult(); Assert.Equal(QueryTypes.Aggregation, result.QueryType); - if (!result.IsValidResponse) - _logger.LogInformation(result.Message); + if (!result.IsValid) + _logger.LogInformation("Result {Request}", result.Message); - Assert.Equal(IsValidResponse, result.IsValidResponse); + Assert.Equal(isValid, result.IsValid); if (maxNodeDepth >= 0) Assert.Equal(maxNodeDepth, result.MaxNodeDepth); if (fields != null) diff --git a/tests/Foundatio.Parsers.ElasticQueries.Tests/CustomVisitorTests.cs b/tests/Foundatio.Parsers.ElasticQueries.Tests/CustomVisitorTests.cs index e1844557..5041fd77 100644 --- a/tests/Foundatio.Parsers.ElasticQueries.Tests/CustomVisitorTests.cs +++ b/tests/Foundatio.Parsers.ElasticQueries.Tests/CustomVisitorTests.cs @@ -1,7 +1,9 @@ using System; using System.Collections.Generic; +using System.Linq; using System.Text; using System.Threading.Tasks; +using Elastic.Clients.Elasticsearch; using Elastic.Clients.Elasticsearch.QueryDsl; using Foundatio.Parsers.ElasticQueries.Extensions; using Foundatio.Parsers.LuceneQueries.Nodes; @@ -172,9 +174,9 @@ public override async Task VisitAsync(GroupNode node, IQueryVisitorContext conte string term = ToTerm(node); var ids = await GetIdsAsync(term); if (ids is { Count: > 0 }) - node.Parent.SetQuery(new TermsQuery { Field = "id", Terms = ids }); + node.Parent.SetQuery(new TermsQuery { Field = "id", Term = new TermsQueryField(ids.Select(FieldValue.String).ToArray()) }); else - node.Parent.SetQuery(new TermQuery { Field = "id", Value = "none" }); + node.Parent.SetQuery(new TermQuery("id") { Value = "none" }); node.Left = null; node.Right = null; diff --git a/tests/Foundatio.Parsers.ElasticQueries.Tests/ElasticMappingResolverTests.cs b/tests/Foundatio.Parsers.ElasticQueries.Tests/ElasticMappingResolverTests.cs index a9bc67b4..7a51b88a 100644 --- a/tests/Foundatio.Parsers.ElasticQueries.Tests/ElasticMappingResolverTests.cs +++ b/tests/Foundatio.Parsers.ElasticQueries.Tests/ElasticMappingResolverTests.cs @@ -16,11 +16,11 @@ public ElasticMappingResolverTests(ITestOutputHelper output, ElasticsearchFixtur Log.DefaultMinimumLevel = Microsoft.Extensions.Logging.LogLevel.Trace; } - private TypeMapping MapMyNestedType(TypeMappingDescriptor m) + private TypeMappingDescriptor MapMyNestedType(TypeMappingDescriptor m) { return m .AutoMap() - .Dynamic() + .Dynamic(DynamicMapping.True) .DynamicTemplates(t => t.DynamicTemplate("idx_text", t => t.Match("text*").Mapping(m => m.Text(mp => mp.AddKeywordAndSortFields())))) .Properties(p => p .Text(p1 => p1.Name(n => n.Field1).AddKeywordAndSortFields()) diff --git a/tests/Foundatio.Parsers.ElasticQueries.Tests/ElasticQueryParserTests.cs b/tests/Foundatio.Parsers.ElasticQueries.Tests/ElasticQueryParserTests.cs index e6364e8c..0790fc80 100644 --- a/tests/Foundatio.Parsers.ElasticQueries.Tests/ElasticQueryParserTests.cs +++ b/tests/Foundatio.Parsers.ElasticQueries.Tests/ElasticQueryParserTests.cs @@ -55,6 +55,7 @@ public void CanUseElasticQueryParser() var sut = new ElasticQueryParser(); var result = sut.Parse("NOT (dog parrot)"); + Assert.NotNull(result); Assert.IsType(result.Left); Assert.True(((GroupNode)result.Left).HasParens); Assert.True(((GroupNode)result.Left).IsNegated); @@ -70,9 +71,10 @@ public void CanUseElasticQueryParserWithVisitor() var result = sut.Parse("NOT (dog parrot)", context) as GroupNode; Assert.Equal(2, testQueryVisitor.GroupNodeCount); + Assert.NotNull(result); Assert.IsType(result.Left); - Assert.True((result.Left as GroupNode).HasParens); - Assert.True((result.Left as GroupNode).IsNegated); + Assert.True(((GroupNode)result.Left).HasParens); + Assert.True((result.Left as GroupNode)?.IsNegated); } [Fact] @@ -92,7 +94,7 @@ await Client.IndexManyAsync([ string actualRequest = actualResponse.GetRequest(); _logger.LogInformation("Actual: {Request}", actualRequest); - var expectedResponse = await Client.SearchAsync(d => d.Index(index).Query(q => q.Bool(b => b.Filter(f => f.Term(m => m.Field1, "value1"))))); + var expectedResponse = await Client.SearchAsync(d => d.Index(index).Query(q => q.Bool(b => b.Filter(f => f.Term(m => m.Field(tf => tf.Field1).Value("value1")))))); string expectedRequest = expectedResponse.GetRequest(); _logger.LogInformation("Expected: {Request}", expectedRequest); @@ -122,8 +124,8 @@ await Client.IndexManyAsync([ _logger.LogInformation("Actual: {Request}", actualRequest); var expectedResponse = await Client.SearchAsync(d => d.Index(index).Query(f => - f.Term(m => m.Field1, "value1") - && f.Term(m => m.Field2, "value2"))); + f.Term(m => m.Field(tf => tf.Field1).Value("value1")) + && f.Term(m => m.Field(tf => tf.Field2).Value("value2")))); string expectedRequest = expectedResponse.GetRequest(); _logger.LogInformation("Expected: {Request}", expectedRequest); @@ -147,7 +149,7 @@ public async Task ShouldGenerateORedTermsQuery() _logger.LogInformation("Actual: {Request}", actualRequest); var expectedResponse = await Client.SearchAsync(d => d.Index(index).Query(f => - f.Term(m => m.Field1, "value1") || f.Term(m => m.Field2, "value2") || f.Term(m => m.Field3, "value3"))); + f.Term(m => m.Field(tf => tf.Field1).Value("value1")) || f.Term(m => m.Field(tf => tf.Field2).Value("value2")) || f.Term(m => m.Field(tf => tf.Field3).Value("value3")))); string expectedRequest = expectedResponse.GetRequest(); _logger.LogInformation("Expected: {Request}", expectedRequest); @@ -160,9 +162,9 @@ public async Task ShouldGenerateORedTermsQuery() public async Task ShouldHandleMultipleTermsForAnalyzedFields() { string index = CreateRandomIndex(d => d - .Dynamic().Properties(p => p.GeoPoint(g => g.Name(f => f.Field3)) - .Text(e => e.Name(m => m.Field1).Fields(f1 => f1.Keyword(e1 => e1.Name("keyword")))) - .Keyword(e => e.Name(m => m.Field2)) + .Dynamic(DynamicMapping.True).Properties(p => p.GeoPoint(g => g.Field3) + .Text(e => e.Field1, o => o.Fields(f1 => f1.Keyword(e1 => e1.Name("keyword")))) + .Keyword(e => e.Field2) )); await Client.IndexAsync(new MyType { Field1 = "value1", Field2 = "value2", Field3 = "value3" }, i => i.Index(index)); await Client.Indices.RefreshAsync(index); @@ -180,7 +182,7 @@ public async Task ShouldHandleMultipleTermsForAnalyzedFields() || f.Match(m => m.Field(mf => mf.Field1).Query("abc")) || f.Match(m => m.Field(mf => mf.Field1).Query("def")) || f.Match(m => m.Field(mf => mf.Field1).Query("ghi")) - || f.Term(m => m.Field2, "value2") || f.Term(m => m.Field2, "jhk"))); + || f.Term(m => m.Field(tf => tf.Field2).Value("value2")) || f.Term(m => m.Field2, "jhk"))); string expectedRequest = expectedResponse.GetRequest(); _logger.LogInformation("Expected: {Request}", expectedRequest); @@ -214,10 +216,10 @@ public async Task ShouldHandleMultipleTermsForAnalyzedFields() _logger.LogInformation("Actual: {Request}", actualRequest); expectedResponse = await Client.SearchAsync(d => d.Index(index).Query(f => - f.MultiMatch(m => m.Fields(mf => mf.Fields("field1", "field2")).Query("value1")) - || f.MultiMatch(m => m.Fields(mf => mf.Fields("field1", "field2")).Query("abc")) - || f.MultiMatch(m => m.Fields(mf => mf.Fields("field1", "field2")).Query("def")) - || f.MultiMatch(m => m.Fields(mf => mf.Fields("field1", "field2")).Query("ghi")))); + f.MultiMatch(m => m.Fields(Fields.FromStrings(["field1", "field2"])).Query("value1")) + || f.MultiMatch(m => m.Fields(Fields.FromStrings(["field1", "field2"])).Query("abc")) + || f.MultiMatch(m => m.Fields(Fields.FromStrings(["field1", "field2"])).Query("def")) + || f.MultiMatch(m => m.Fields(Fields.FromStrings(["field1", "field2"])).Query("ghi")))); expectedRequest = expectedResponse.GetRequest(); _logger.LogInformation("Expected: {Request}", expectedRequest); @@ -227,21 +229,21 @@ public async Task ShouldHandleMultipleTermsForAnalyzedFields() } [Fact] - public void CanGetMappingsFromCode() + public async Task CanGetMappingsFromCode() { TypeMappingDescriptor GetCodeMappings(TypeMappingDescriptor d) => - d.Dynamic() + d.Dynamic(DynamicMapping.True) .Properties(p => p - .GeoPoint(g => g.Name(f => f.Field3)) - .Text(e => e.Name(m => m.Field1))); + .GeoPoint(g => g.Field3) + .Text(e => e.Field1)); - string index = CreateRandomIndex(d => d.Dynamic() + string index = CreateRandomIndex(d => d.Dynamic(DynamicMapping.True) .Properties(p => p - .GeoPoint(g => g.Name(f => f.Field3)) - .Keyword(e => e.Name(m => m.Field2)))); + .GeoPoint(g => g.Field3) + .Keyword(e => e.Field2))); - var res = Client.Index(new MyType { Field1 = "value1", Field2 = "value2", Field4 = 1, Field5 = DateTime.Now }, i => i.Index(index)); - Client.Indices.RefreshAsync(index); + var res = await Client.IndexAsync(new MyType { Field1 = "value1", Field2 = "value2", Field4 = 1, Field5 = DateTime.Now }, i => i.Index(index)); + await Client.Indices.RefreshAsync(index); var parser = new ElasticQueryParser(c => c.SetDefaultFields(["field1"]).UseMappings(GetCodeMappings, Client, index)); @@ -487,8 +489,8 @@ public async Task CanUseDateHistogramAggregationInterval(string interval, object { d.Field(d2 => d2.Field5); - if (expectedInterval is CalendarInterval CalendarInterval) - d.CalendarInterval(CalendarInterval); + if (expectedInterval is CalendarInterval calendarInterval) + d.CalendarInterval(calendarInterval); else if (expectedInterval is string stringInterval) d.FixedInterval(stringInterval); else @@ -508,11 +510,11 @@ public async Task CanUseDateHistogramAggregationInterval(string interval, object } [Fact] - public void CanDoNestDateHistogram() + public async Task CanDoNestDateHistogram() { string index = CreateRandomIndex(); await Client.IndexManyAsync([new MyType { Field5 = DateTime.Now }], index); - Client.Indices.RefreshAsync(index); + await Client.Indices.RefreshAsync(index); var response = await Client.SearchAsync(i => i.Index(index).Aggregations(f => f .DateHistogram("myagg", d => d.Field(d2 => d2.Field5).CalendarInterval(CalendarInterval.Day)) @@ -639,7 +641,7 @@ await Client.IndexManyAsync([ _logger.LogInformation("Actual: {Request}", actualRequest); var expectedResponse = await Client.SearchAsync(d => d - .Index(index).Query(f => f.Term(m => m.Field1, "value1") && !f.Term(m => m.Field2, "value2"))); + .Index(index).Query(f => f.Term(m => m.Field(tf => tf.Field1).Value("value1")) && !f.Term(m => m.Field(tf => tf.Field2).Value("value2")))); string expectedRequest = expectedResponse.GetRequest(); _logger.LogInformation("Expected: {Request}", expectedRequest); @@ -654,7 +656,7 @@ await Client.IndexManyAsync([ _logger.LogInformation("Actual: {Request}", actualRequest); expectedResponse = await Client.SearchAsync(d => d - .Index(index).Query(f => f.Term(m => m.Field1, "value1") && !f.Term(m => m.Field2, "value2"))); + .Index(index).Query(f => f.Term(m => m.Field(tf => tf.Field1).Value("value1")) && !f.Term(m => m.Field(tf => tf.Field2).Value("value2")))); expectedRequest = expectedResponse.GetRequest(); _logger.LogInformation("Expected: {Request}", expectedRequest); @@ -669,7 +671,7 @@ await Client.IndexManyAsync([ _logger.LogInformation("Actual: {Request}", actualRequest); expectedResponse = await Client.SearchAsync(d => d - .Index(index).Query(f => f.Term(m => m.Field1, "value1") || !f.Term(m => m.Field2, "value2"))); + .Index(index).Query(f => f.Term(m => m.Field(tf => tf.Field1).Value("value1")) || !f.Term(m => m.Field(tf => tf.Field2).Value("value2")))); expectedRequest = expectedResponse.GetRequest(); _logger.LogInformation("Expected: {Request}", expectedRequest); @@ -684,7 +686,7 @@ await Client.IndexManyAsync([ _logger.LogInformation("Actual: {Request}", actualRequest); expectedResponse = await Client.SearchAsync(d => d - .Index(index).Query(f => f.Term(m => m.Field1, "value1") || !f.Term(m => m.Field2, "value2"))); + .Index(index).Query(f => f.Term(m => m.Field(tf => tf.Field1).Value("value1")) || !f.Term(m => m.Field(tf => tf.Field2).Value("value2")))); expectedRequest = expectedResponse.GetRequest(); _logger.LogInformation("Expected: {Request}", expectedRequest); @@ -712,8 +714,8 @@ await Client.IndexManyAsync([ _logger.LogInformation("Actual: {Request}", actualRequest); var expectedResponse = await Client.SearchAsync(d => d.Index(index) - .Query(f => f.Term(m => m.Field1, "value1") || - (f.Term(m => m.Field2, "value2") || f.Term(m => m.Field3, "value3")))); + .Query(f => f.Term(m => m.Field(tf => tf.Field1).Value("value1")) || + (f.Term(m => m.Field(tf => tf.Field2).Value("value2")) || f.Term(m => m.Field(tf => tf.Field3).Value("value3"))))); string expectedRequest = expectedResponse.GetRequest(); _logger.LogInformation("Expected: {Request}", expectedRequest); @@ -742,8 +744,8 @@ await Client.IndexManyAsync([ var expectedResponse = await Client.SearchAsync(d => d.Index(index) .Query(q => q.Bool(b => b.Filter(f => f - .Term(m => m.Field1, "value1") && - (f.Term(m => m.Field2, "value2") || f.Term(m => m.Field3, "value3")))))); + .Term(m => m.Field(tf => tf.Field1).Value("value1")) && + (f.Term(m => m.Field(tf => tf.Field2).Value("value2")) || f.Term(m => m.Field(tf => tf.Field3).Value("value3"))))))); string expectedRequest = expectedResponse.GetRequest(); _logger.LogInformation("Expected: {Request}", expectedRequest); @@ -832,8 +834,8 @@ await Client.IndexManyAsync([ _logger.LogInformation("Actual: {Request}", actualRequest); var expectedResponse = await Client.SearchAsync(d => d.Index(index) - .Query(f => f.Term(m => m.Field1, "value1") && - (f.Term(m => m.Field2, "value2") || f.Term(m => m.Field3, "value3")))); + .Query(f => f.Term(m => m.Field(tf => tf.Field1).Value("value1")) && + (f.Term(m => m.Field(tf => tf.Field2).Value("value2")) || f.Term(m => m.Field(tf => tf.Field3).Value("value3"))))); string expectedRequest = expectedResponse.GetRequest(); _logger.LogInformation("Expected: {Request}", expectedRequest); @@ -845,15 +847,15 @@ await Client.IndexManyAsync([ public async Task NestedFilterProcessor() { string index = CreateRandomIndex(d => d.Properties(p => p - .Text(e => e.Name(n => n.Field1).Index()) - .Text(e => e.Name(n => n.Field2).Index()) - .Text(e => e.Name(n => n.Field3).Index()) - .Number(e => e.Name(n => n.Field4).Type(NumberType.Integer)) + .Text(e => e.Field1, o => o.Index()) + .Text(e => e.Field2, o => o.Index()) + .Text(e => e.Field3, o => o.Index()) + .IntegerNumber(e => e.Field4) .Nested(r => r.Name(n => n.Nested.First()).Properties(p1 => p1 - .Text(e => e.Name(n => n.Field1).Index()) - .Text(e => e.Name(n => n.Field2).Index()) - .Text(e => e.Name(n => n.Field3).Index()) - .Number(e => e.Name(n => n.Field4).Type(NumberType.Integer)) + .Text(e => e.Field1, o => o.Index()) + .Text(e => e.Field2, o => o.Index()) + .Text(e => e.Field3, o => o.Index()) + .IntegerNumber(e => e.Field4) )) )); await Client.IndexManyAsync([ @@ -917,15 +919,15 @@ await Client.IndexManyAsync([ public async Task NestedFilterProcessor2() { string index = CreateRandomIndex(d => d.Properties(p => p - .Text(e => e.Name(n => n.Field1).Index()) - .Text(e => e.Name(n => n.Field2).Index()) - .Text(e => e.Name(n => n.Field3).Index()) - .Number(e => e.Name(n => n.Field4).Type(NumberType.Integer)) + .Text(e => e.Field1, o => o.Index()) + .Text(e => e.Field2, o => o.Index()) + .Text(e => e.Field3, o => o.Index()) + .IntegerNumber(e => e.Field4) .Nested(r => r.Name(n => n.Nested.First()).Properties(p1 => p1 - .Text(e => e.Name(n => n.Field1).Index()) - .Text(e => e.Name(n => n.Field2).Index()) - .Text(e => e.Name(n => n.Field3).Index()) - .Number(e => e.Name(n => n.Field4).Type(NumberType.Integer)) + .Text(e => e.Field1, o => o.Index()) + .Text(e => e.Field2, o => o.Index()) + .Text(e => e.Field3, o => o.Index()) + .IntegerNumber(e => e.Field4) )) )); @@ -966,9 +968,9 @@ await Client.IndexManyAsync([ public async Task CanGenerateMatchQuery() { string index = CreateRandomIndex(m => m.Properties(p => p - .Text(f => f.Name(e => e.Field1) + .Text(e => e.Field1) .Fields(f1 => f1 - .Keyword(k => k.Name("keyword").IgnoreAbove(256)))))); + .Keyword(k => k.Name("keyword").IgnoreAbove(256))))); var processor = new ElasticQueryParser(c => c.SetLoggerFactory(Log).UseMappings(Client, index)); var result = await processor.BuildQueryAsync("field1:test", new ElasticQueryVisitorContext().UseScoring()); @@ -1011,7 +1013,7 @@ public async Task CanBuildAliasQueryProcessor() [Fact] public async Task NonAnalyzedPrefixQuery() { - string index = CreateRandomIndex(d => d.Properties(p => p.Keyword(e => e.Name(m => m.Field1)))); + string index = CreateRandomIndex(d => d.Properties(p => p.Keyword(e => e.Field1))); await Client.IndexAsync(new MyType { Field1 = "value123" }, i => i.Index(index)); await Client.Indices.RefreshAsync(index); @@ -1058,7 +1060,7 @@ await Client.SearchAsync( .Query( f => f.TermRange(m => m.Field(f2 => f2.Field4).GreaterThanOrEquals("1").LessThan("2")) || - f.Term(m => m.Field1, "value1"))); + f.Term(m => m.Field(tf => tf.Field1).Value("value1")))); string expectedRequest = expectedResponse.GetRequest(); _logger.LogInformation("Expected: {Request}", expectedRequest); @@ -1147,7 +1149,7 @@ public async Task DateRangeWithWildcardMaxQueryProcessor() string actualRequest = actualResponse.GetRequest(); _logger.LogInformation("Actual: {Request}", actualResponse); - var expectedResponse = await Client.SearchAsyncAsync(d => d + var expectedResponse = await Client.SearchAsync(d => d .Index(index) .Query(f => f .DateRange(m => m.Field(f2 => f2.Field5).GreaterThanOrEquals("2017-01-31").TimeZone("America/Chicago")) @@ -1225,7 +1227,7 @@ await Client.SearchAsync(d => d [Fact] public async Task SimpleGeoRangeQuery() { - string index = CreateRandomIndex(m => m.Properties(p => p.GeoPoint(g => g.Name(f => f.Field3)))); + string index = CreateRandomIndex(m => m.Properties(p => p.GeoPoint(g => g.Field3))); var res = await Client.IndexAsync(new MyType { Field1 = "value1", Field4 = 1, Field3 = "51.5032520,-0.1278990" }, i => i.Index(index)); await Client.IndexAsync(new MyType { Field4 = 2 }, i => i.Index(index)); @@ -1256,7 +1258,7 @@ await processor.BuildQueryAsync("field3:[51.5032520,-0.1278990 TO 51.5032520,-0. [Fact] public async Task CanUseValidationToGetUnresolvedFields() { - string index = CreateRandomIndex(d => d.Properties(p => p.Keyword(e => e.Name(m => m.Field1)))); + string index = CreateRandomIndex(d => d.Properties(p => p.Keyword(e => e.Field1))); await Client.IndexAsync(new MyType { Field1 = "value123" }, i => i.Index(index)); await Client.Indices.RefreshAsync(index); @@ -1298,7 +1300,7 @@ public async Task CanUseValidationToGetUnresolvedFields() [Fact] public async Task CanSortByUnmappedField() { - string index = CreateRandomIndex(m => m.Dynamic()); + string index = CreateRandomIndex(m => m.Dynamic(DynamicMapping.True)); var processor = new ElasticQueryParser(c => c.UseMappings(Client, index)); var sort = await processor.BuildSortAsync("-field1"); @@ -1322,9 +1324,9 @@ public async Task CanSortByUnmappedField() [Fact] public async Task CanParseSort() { - string index = CreateRandomIndex(d => d.Properties(p => p.GeoPoint(g => g.Name(f => f.Field3)) - .Text(e => e.Name(m => m.Field1).Fields(f1 => f1.Keyword(e1 => e1.Name("keyword")))) - .Text(e => e.Name(m => m.Field2).Fields(f2 => f2.Keyword(e1 => e1.Name("keyword")).Keyword(e2 => e2.Name("sort")))) + string index = CreateRandomIndex(d => d.Properties(p => p.GeoPoint(g => g.Field3) + .Text(e => e.Field1, o => o.Fields(f1 => f1.Keyword("keyword"))) + .Text(e => e.Field2, o => o.Fields(f2 => f2.Keyword("keyword").Keyword(e2 => e2.Name("sort")))) )); var res = await Client.IndexAsync(new MyType { Field1 = "value1", Field4 = 1, Field3 = "51.5032520,-0.1278990" }, i => i.Index(index)); @@ -1410,7 +1412,7 @@ await Client.IndexManyAsync([ public async Task CanParseMixedCaseSort() { string index = CreateRandomIndex(d => d.Properties(p => p - .Text(e => e.Name(m => m.MultiWord).Fields(f1 => f1.Keyword(e1 => e1.Name("keyword")))) + .Text(e => e.Name(m => m.MultiWord).Fields(f1 => f1.Keyword("keyword"))) )); var res = await Client.IndexAsync(new MyType { MultiWord = "value1" }, i => i.Index(index)); @@ -1435,8 +1437,8 @@ public async Task CanParseMixedCaseSort() public async Task GeoRangeQueryProcessor() { string index = CreateRandomIndex(m => m.Properties(p => p - .GeoPoint(g => g.Name(f => f.Field3)) - .Text(e => e.Name(m => m.Field1).Fields(f1 => f1.Keyword(e1 => e1.Name("keyword")))))); + .GeoPoint(g => g.Field3) + .Text(e => e.Field1, o => o.Fields(f1 => f1.Keyword("keyword"))))); await Client.IndexAsync(new MyType { Field1 = "value1", Field4 = 1, Field3 = "51.5032520,-0.1278990" }, i => i.Index(index)); await Client.IndexAsync(new MyType { Field4 = 2 }, i => i.Index(index)); @@ -1515,7 +1517,7 @@ private async Task GetIncludeAsync(string name) [InlineData("terms:(field1~100 (@missing:__missing__))")] public async Task CanValidateAggregation(string aggregation) { - string index = CreateRandomIndex(d => d.Properties(p => p.Keyword(e => e.Name(m => m.Field1)))); + string index = CreateRandomIndex(d => d.Properties(p => p.Keyword(e => e.Field1))); var context = new ElasticQueryVisitorContext { QueryType = QueryTypes.Aggregation }; var parser = new ElasticQueryParser(c => c.UseMappings(Client, index).SetLoggerFactory(Log)); var node = await parser.ParseAsync(aggregation, context); diff --git a/tests/Foundatio.Parsers.ElasticQueries.Tests/Utility/ElasticsearchTestBase.cs b/tests/Foundatio.Parsers.ElasticQueries.Tests/Utility/ElasticsearchTestBase.cs index 7ea23b3b..7107e294 100644 --- a/tests/Foundatio.Parsers.ElasticQueries.Tests/Utility/ElasticsearchTestBase.cs +++ b/tests/Foundatio.Parsers.ElasticQueries.Tests/Utility/ElasticsearchTestBase.cs @@ -30,12 +30,12 @@ public ElasticsearchTestBase(ITestOutputHelper output, T fixture) : base(output) protected ElasticsearchClient Client => _fixture.Client; - protected void CreateNamedIndex(string index, Func, TypeMapping> configureMappings = null, Func configureIndex = null) where TModel : class + protected void CreateNamedIndex(string index, Func, TypeMappingDescriptor> configureMappings = null, Func configureIndex = null) where TModel : class { _fixture.CreateNamedIndex(index, configureMappings, configureIndex); } - protected string CreateRandomIndex(Func, TypeMapping> configureMappings = null, Func configureIndex = null) where TModel : class + protected string CreateRandomIndex(Func, TypeMappingDescriptor> configureMappings = null, Func configureIndex = null) where TModel : class { return _fixture.CreateRandomIndex(configureMappings, configureIndex); } @@ -100,7 +100,7 @@ protected virtual void ConfigureConnectionSettings(ElasticsearchClientSettings s public void CreateNamedIndex(string index, Func, TypeMappingDescriptor> configureMappings = null, Func configureIndex = null) where T : class { - configureMappings ??= m => m.AutoMap().Dynamic(); + configureMappings ??= m => m.AutoMap().Dynamic(DynamicMapping.True); if (configureIndex == null) configureIndex = i => i.NumberOfReplicas(0).Analysis(a => a.AddSortNormalizer()); @@ -111,7 +111,7 @@ public void CreateNamedIndex(string index, Func, Typ public string CreateRandomIndex(Func, TypeMappingDescriptor> configureMappings = null, Func configureIndex = null) where T : class { string index = "test_" + Guid.NewGuid().ToString("N"); - configureMappings ??= m => m.AutoMap().Dynamic(); + configureMappings ??= m => m.AutoMap().Dynamic(DynamicMapping.True); configureIndex ??= i => i.NumberOfReplicas(0).Analysis(a => a.AddSortNormalizer()); CreateIndex(index, i => i.Settings(configureIndex(new IndexSettingsDescriptor())).Map(configureMappings)); From 001bce4a59dcd592d895aca6200c8e526829d75e Mon Sep 17 00:00:00 2001 From: Blake Niemyjski Date: Fri, 13 Sep 2024 13:22:50 -0500 Subject: [PATCH 12/19] Upgraded deprecated sync test method. --- .../AggregationParserTests.cs | 32 ++++---- .../CustomVisitorTests.cs | 8 +- .../ElasticMappingResolverTests.cs | 4 +- .../ElasticQueryParserTests.cs | 82 +++++++++---------- .../InvertQueryTests.cs | 2 +- .../Utility/ElasticsearchTestBase.cs | 29 ++++--- 6 files changed, 78 insertions(+), 79 deletions(-) diff --git a/tests/Foundatio.Parsers.ElasticQueries.Tests/AggregationParserTests.cs b/tests/Foundatio.Parsers.ElasticQueries.Tests/AggregationParserTests.cs index e534bf5f..2b01d71f 100644 --- a/tests/Foundatio.Parsers.ElasticQueries.Tests/AggregationParserTests.cs +++ b/tests/Foundatio.Parsers.ElasticQueries.Tests/AggregationParserTests.cs @@ -22,7 +22,7 @@ public AggregationParserTests(ITestOutputHelper output, ElasticsearchFixture fix [Fact] public async Task ProcessSingleAggregationAsync() { - string index = CreateRandomIndex(d => d.Dynamic(DynamicMapping.True).Properties(p => p.GeoPoint(g => g.Field3))); + string index = await CreateRandomIndexAsync(d => d.Dynamic(DynamicMapping.True).Properties(p => p.GeoPoint(g => g.Field3))); await Client.IndexManyAsync([ new MyType { Field1 = "value1", Field4 = 1, Field3 = "51.5032520,-0.1278990", Field5 = DateTime.UtcNow.Subtract(TimeSpan.FromMinutes(5)), Field2 = "field2" }, new MyType { Field1 = "value2", Field4 = 2, Field3 = "51.5032520,-0.1278990", Field5 = DateTime.UtcNow.Subtract(TimeSpan.FromMinutes(4)) }, @@ -53,7 +53,7 @@ await Client.IndexManyAsync([ [Fact] public async Task ProcessSingleAggregationWithAliasAsync() { - string index = CreateRandomIndex(d => d.Dynamic(DynamicMapping.True).Properties(p => p.GeoPoint(g => g.Field3))); + string index = await CreateRandomIndexAsync(d => d.Dynamic(DynamicMapping.True).Properties(p => p.GeoPoint(g => g.Field3))); await Client.IndexManyAsync([ new MyType { Field1 = "value1", Field4 = 1, Field3 = "51.5032520,-0.1278990", Field5 = DateTime.UtcNow.Subtract(TimeSpan.FromMinutes(5)), Field2 = "field2" }, new MyType { Field1 = "value2", Field4 = 2, Field3 = "51.5032520,-0.1278990", Field5 = DateTime.UtcNow.Subtract(TimeSpan.FromMinutes(4)) }, @@ -85,7 +85,7 @@ await Client.IndexManyAsync([ [Fact] public async Task ProcessAnalyzedAggregationWithAliasAsync() { - string index = CreateRandomIndex(d => d.Dynamic(DynamicMapping.True).Properties(p => p + string index = await CreateRandomIndexAsync(d => d.Dynamic(DynamicMapping.True).Properties(p => p .Text(f => f.Field1, o => o .Fields(k => k.Keyword("keyword"))) .FieldAlias(f => f.Name("heynow").Path(k => k.Field1)) @@ -121,7 +121,7 @@ await Client.IndexManyAsync([ [Fact] public async Task ProcessAggregationsAsync() { - string index = CreateRandomIndex(d => d.Dynamic(DynamicMapping.True).Properties(p => p.GeoPoint(g => g.Field3))); + string index = await CreateRandomIndexAsync(d => d.Dynamic(DynamicMapping.True).Properties(p => p.GeoPoint(g => g.Field3))); await Client.IndexManyAsync([ new MyType { Field1 = "value1", Field4 = 1, Field3 = "51.5032520,-0.1278990", Field5 = DateTime.UtcNow.Subtract(TimeSpan.FromMinutes(5)), Field2 = "field2" }, new MyType { Field1 = "value2", Field4 = 2, Field3 = "51.5032520,-0.1278990", Field5 = DateTime.UtcNow.Subtract(TimeSpan.FromMinutes(4)) }, @@ -163,7 +163,7 @@ await Client.IndexManyAsync([ [Fact] public async Task ProcessNestedAggregationsWithAliasesAsync() { - string index = CreateRandomIndex(d => d.Dynamic(DynamicMapping.True).Properties(p => p + string index = await CreateRandomIndexAsync(d => d.Dynamic(DynamicMapping.True).Properties(p => p .GeoPoint(g => g.Field3) .Object>(o1 => o1.Name(f1 => f1.Data).Properties(p1 => p1 .Object(o2 => o2.Name("@user").Properties(p2 => p2 @@ -196,7 +196,7 @@ public async Task ProcessNestedAggregationsWithAliasesAsync() [Fact] public async Task ProcessSingleAggregationWithAlias() { - string index = CreateRandomIndex(); + string index = await CreateRandomIndexAsync(); await Client.IndexManyAsync([ new MyType { Field2 = "field2" } @@ -226,7 +226,7 @@ await Client.IndexManyAsync([ [Fact] public async Task ProcessAggregationsWithAliasesAsync() { - string index = CreateRandomIndex(d => d.Dynamic(DynamicMapping.True).Properties(p => p + string index = await CreateRandomIndexAsync(d => d.Dynamic(DynamicMapping.True).Properties(p => p .GeoPoint(g => g.Field3) .Object>(o1 => o1.Name(f1 => f1.Data).Properties(p1 => p1 .Object(o2 => o2.Name("@user").Properties(p2 => p2 @@ -270,7 +270,7 @@ await Client.IndexManyAsync([ [Fact] public async Task ProcessTermAggregations() { - string index = CreateRandomIndex(); + string index = await CreateRandomIndexAsync(); await Client.IndexManyAsync([new MyType { Field1 = "value1" }], index); await Client.Indices.RefreshAsync(index); @@ -299,7 +299,7 @@ public async Task ProcessTermAggregations() [Fact] public async Task ProcessHistogramIntervalAggregations() { - string index = CreateRandomIndex(); + string index = await CreateRandomIndexAsync(); await Client.IndexManyAsync([new MyType { Field1 = "value1" }], index); await Client.Indices.RefreshAsync(index); @@ -326,7 +326,7 @@ public async Task ProcessHistogramIntervalAggregations() [Fact] public async Task ProcessTermTopHitsAggregations() { - string index = CreateRandomIndex(); + string index = await CreateRandomIndexAsync(); await Client.IndexManyAsync([new MyType { Field1 = "value1" }], index); await Client.Indices.RefreshAsync(index); @@ -354,7 +354,7 @@ public async Task ProcessTermTopHitsAggregations() [Fact] public async Task ProcessSortedTermAggregations() { - string index = CreateRandomIndex(); + string index = await CreateRandomIndexAsync(); await Client.IndexManyAsync([new MyType { Field1 = "value1" }], index); await Client.Indices.RefreshAsync(index); @@ -384,7 +384,7 @@ public async Task ProcessSortedTermAggregations() [Fact] public async Task ProcessDateHistogramAggregations() { - string index = CreateRandomIndex(); + string index = await CreateRandomIndexAsync(); await Client.IndexManyAsync([new MyType { Field5 = DateTime.UtcNow }], index); await Client.Indices.RefreshAsync(index); @@ -418,7 +418,7 @@ public async Task ProcessDateHistogramAggregations() [Fact] public async Task CanSpecifyDefaultValuesAggregations() { - string index = CreateRandomIndex(); + string index = await CreateRandomIndexAsync(); await Client.IndexManyAsync([new MyType { Field1 = "test" }, new MyType { Field4 = 1 }], index); await Client.Indices.RefreshAsync(index); @@ -445,9 +445,9 @@ public async Task CanSpecifyDefaultValuesAggregations() } [Fact] - public Task GeoGridDoesNotResolveLocationForAggregation() + public async Task GeoGridDoesNotResolveLocationForAggregation() { - string index = CreateRandomIndex(d => d.Properties(p => p + string index = await CreateRandomIndexAsync(d => d.Properties(p => p .GeoPoint(g => g.Field1) .FieldAlias(a => a.Name("geo").Path(f => f.Field1)))); @@ -455,7 +455,7 @@ public Task GeoGridDoesNotResolveLocationForAggregation() .UseGeo(_ => "someinvalidvaluehere") .UseMappings(Client, index)); - return processor.BuildAggregationsAsync("geogrid:geo~3"); + await processor.BuildAggregationsAsync("geogrid:geo~3"); } [Theory] diff --git a/tests/Foundatio.Parsers.ElasticQueries.Tests/CustomVisitorTests.cs b/tests/Foundatio.Parsers.ElasticQueries.Tests/CustomVisitorTests.cs index 5041fd77..632fa6dc 100644 --- a/tests/Foundatio.Parsers.ElasticQueries.Tests/CustomVisitorTests.cs +++ b/tests/Foundatio.Parsers.ElasticQueries.Tests/CustomVisitorTests.cs @@ -24,7 +24,7 @@ public CustomVisitorTests(ITestOutputHelper output, ElasticsearchFixture fixture [Fact] public async Task CanResolveSimpleCustomFilter() { - string index = CreateRandomIndex(); + string index = await CreateRandomIndexAsync(); await Client.IndexAsync(new MyType { Id = "1" }, i => i.Index(index)); var processor = new ElasticQueryParser(c => c @@ -49,7 +49,7 @@ public async Task CanResolveSimpleCustomFilter() [Fact] public async Task CanResolveCustomFilterContainingIncludes() { - string index = CreateRandomIndex(); + string index = await CreateRandomIndexAsync(); await Client.IndexAsync(new MyType { Id = "1" }, i => i.Index(index)); var processor = new ElasticQueryParser(c => c @@ -74,7 +74,7 @@ public async Task CanResolveCustomFilterContainingIncludes() [Fact] public async Task CanResolveIncludeToCustomFilterContainingIgnoredInclude() { - string index = CreateRandomIndex(); + string index = await CreateRandomIndexAsync(); await Client.IndexAsync(new MyType { Id = "1" }, i => i.Index(index)); var processor = new ElasticQueryParser(c => c @@ -121,7 +121,7 @@ private static Task ResolveIncludeAsync(string expected, string actual, [Fact] public async Task CanResolveMultipleCustomFilters() { - string index = CreateRandomIndex(); + string index = await CreateRandomIndexAsync(); await Client.IndexAsync(new MyType { Id = "1" }, i => i.Index(index)); var processor = new ElasticQueryParser(c => c diff --git a/tests/Foundatio.Parsers.ElasticQueries.Tests/ElasticMappingResolverTests.cs b/tests/Foundatio.Parsers.ElasticQueries.Tests/ElasticMappingResolverTests.cs index 7a51b88a..8c1fad95 100644 --- a/tests/Foundatio.Parsers.ElasticQueries.Tests/ElasticMappingResolverTests.cs +++ b/tests/Foundatio.Parsers.ElasticQueries.Tests/ElasticMappingResolverTests.cs @@ -33,7 +33,7 @@ private TypeMappingDescriptor MapMyNestedType(TypeMappingDescripto [Fact] public async Task CanResolveCodedProperty() { - string index = CreateRandomIndex(MapMyNestedType); + string index = await CreateRandomIndexAsync(MapMyNestedType); await Client.IndexManyAsync([ new MyNestedType @@ -69,7 +69,7 @@ await Client.IndexManyAsync([ [Fact] public async Task CanResolveProperties() { - string index = CreateRandomIndex(MapMyNestedType); + string index = await CreateRandomIndexAsync(MapMyNestedType); await Client.IndexManyAsync([ new MyNestedType diff --git a/tests/Foundatio.Parsers.ElasticQueries.Tests/ElasticQueryParserTests.cs b/tests/Foundatio.Parsers.ElasticQueries.Tests/ElasticQueryParserTests.cs index 0790fc80..5d9ee832 100644 --- a/tests/Foundatio.Parsers.ElasticQueries.Tests/ElasticQueryParserTests.cs +++ b/tests/Foundatio.Parsers.ElasticQueries.Tests/ElasticQueryParserTests.cs @@ -80,7 +80,7 @@ public void CanUseElasticQueryParserWithVisitor() [Fact] public async Task SimpleFilterProcessor() { - string index = CreateRandomIndex(); + string index = await CreateRandomIndexAsync(); await Client.IndexManyAsync([ new MyType { Field1 = "value1", Field2 = "value2" }, new MyType { Field1 = "value2", Field2 = "value2" }, @@ -105,7 +105,7 @@ await Client.IndexManyAsync([ [Fact] public async Task IncludeProcessor() { - string index = CreateRandomIndex(); + string index = await CreateRandomIndexAsync(); await Client.IndexManyAsync([ new MyType { Field1 = "value1", Field2 = "value2" }, new MyType { Field1 = "value2", Field2 = "value2" }, @@ -137,7 +137,7 @@ await Client.IndexManyAsync([ [Fact] public async Task ShouldGenerateORedTermsQuery() { - string index = CreateRandomIndex(); + string index = await CreateRandomIndexAsync(); await Client.IndexAsync(new MyType { Field1 = "value1", Field2 = "value2", Field3 = "value3" }, i => i.Index(index)); await Client.Indices.RefreshAsync(index); @@ -161,7 +161,7 @@ public async Task ShouldGenerateORedTermsQuery() [Fact] public async Task ShouldHandleMultipleTermsForAnalyzedFields() { - string index = CreateRandomIndex(d => d + string index = await CreateRandomIndexAsync(d => d .Dynamic(DynamicMapping.True).Properties(p => p.GeoPoint(g => g.Field3) .Text(e => e.Field1, o => o.Fields(f1 => f1.Keyword(e1 => e1.Name("keyword")))) .Keyword(e => e.Field2) @@ -237,7 +237,7 @@ TypeMappingDescriptor GetCodeMappings(TypeMappingDescriptor d) = .GeoPoint(g => g.Field3) .Text(e => e.Field1)); - string index = CreateRandomIndex(d => d.Dynamic(DynamicMapping.True) + string index = await CreateRandomIndexAsync(d => d.Dynamic(DynamicMapping.True) .Properties(p => p .GeoPoint(g => g.Field3) .Keyword(e => e.Field2))); @@ -261,7 +261,7 @@ TypeMappingDescriptor GetCodeMappings(TypeMappingDescriptor d) = [Fact] public async Task EscapeFilterProcessor() { - string index = CreateRandomIndex(); + string index = await CreateRandomIndexAsync(); await Client.IndexManyAsync([ new MyType { Field1 = "\"now there\"", Field2 = "value2" }, new MyType { Field1 = "value2", Field2 = "value2" }, @@ -291,7 +291,7 @@ await Client.IndexManyAsync([ [Fact] public async Task CanHandleEscapedQueryWithWildcards() { - string index = CreateRandomIndex(); + string index = await CreateRandomIndexAsync(); await Client.IndexManyAsync([ new MyType { Field1 = "one/two/three" } ], index); @@ -319,7 +319,7 @@ await Client.IndexManyAsync([ [Fact] public async Task CanHandleEscapedQuery() { - string index = CreateRandomIndex(); + string index = await CreateRandomIndexAsync(); await Client.IndexManyAsync([ new MyType { Field1 = "one/two/three" } ], index); @@ -347,7 +347,7 @@ await Client.IndexManyAsync([ [Fact] public async Task ExistsFilterProcessor() { - string index = CreateRandomIndex(); + string index = await CreateRandomIndexAsync(); await Client.IndexManyAsync([ new MyType { Field1 = "value1", Field2 = "value2" }, new MyType { Field1 = "value2", Field2 = "value2" }, @@ -374,7 +374,7 @@ await Client.IndexManyAsync([ [Fact] public async Task MissingFilterProcessor() { - string index = CreateRandomIndex(); + string index = await CreateRandomIndexAsync(); await Client.IndexManyAsync([ new MyType { Field1 = "value1", Field2 = "value2" }, new MyType { Field1 = "value2", Field2 = "value2" }, @@ -402,7 +402,7 @@ await Client.IndexManyAsync([ [Fact] public async Task MinMaxWithDateHistogramAggregation() { - string index = CreateRandomIndex(); + string index = await CreateRandomIndexAsync(); await Client.IndexManyAsync([ new MyType { Field1 = "value1", Field2 = "value2", Field5 = DateTime.Now }, new MyType { Field1 = "value2", Field2 = "value2", Field5 = DateTime.Now }, @@ -473,7 +473,7 @@ await Client.IndexManyAsync([ [Theory] public async Task CanUseDateHistogramAggregationInterval(string interval, object expectedInterval = null) { - string index = CreateRandomIndex(); + string index = await CreateRandomIndexAsync(); await Client.IndexManyAsync([new MyType { Field5 = DateTime.Now }], index); await Client.Indices.RefreshAsync(index); @@ -512,7 +512,7 @@ public async Task CanUseDateHistogramAggregationInterval(string interval, object [Fact] public async Task CanDoNestDateHistogram() { - string index = CreateRandomIndex(); + string index = await CreateRandomIndexAsync(); await Client.IndexManyAsync([new MyType { Field5 = DateTime.Now }], index); await Client.Indices.RefreshAsync(index); @@ -526,7 +526,7 @@ public async Task CanDoNestDateHistogram() [Fact] public async Task DateAggregation() { - string index = CreateRandomIndex(); + string index = await CreateRandomIndexAsync(); await Client.IndexManyAsync([ new MyType { Field1 = "value1", Field2 = "value2", Field5 = DateTime.Now }, new MyType { Field1 = "value2", Field2 = "value2", Field5 = DateTime.Now }, @@ -553,7 +553,7 @@ await Client.IndexManyAsync([ [Fact] public async Task SimpleQueryProcessor() { - string index = CreateRandomIndex(t => t + string index = await CreateRandomIndexAsync(t => t .Properties(p => p .Text(e => e.Name(n => n.Field3).Fields(f => f.Keyword(k => k.Name("keyword").IgnoreAbove(256)))))); @@ -625,7 +625,7 @@ await Client.IndexManyAsync([ [Fact] public async Task NegativeQueryProcessor() { - string index = CreateRandomIndex(); + string index = await CreateRandomIndexAsync(); await Client.IndexManyAsync([ new MyType { Field1 = "value1", Field2 = "value2" }, new MyType { Field1 = "value2", Field2 = "value3" }, @@ -697,7 +697,7 @@ await Client.IndexManyAsync([ [Fact] public async Task NestedQueryProcessor() { - string index = CreateRandomIndex(); + string index = await CreateRandomIndexAsync(); await Client.IndexManyAsync([ new MyType { Field1 = "value1", Field2 = "value2" }, new MyType { Field1 = "value2", Field2 = "value2" }, @@ -726,7 +726,7 @@ await Client.IndexManyAsync([ [Fact] public async Task NestedQuery() { - string index = CreateRandomIndex(); + string index = await CreateRandomIndexAsync(); await Client.IndexManyAsync([ new MyType { Field1 = "value1", Field2 = "value2" }, new MyType { Field1 = "value2", Field2 = "value2" }, @@ -756,7 +756,7 @@ await Client.SearchAsync(d => d.Index(index) [Fact] public async Task MixedCaseTermFilterQueryProcessor() { - string index = CreateRandomIndex(); + string index = await CreateRandomIndexAsync(); await Client.IndexAsync(new MyType { Field1 = "Testing.Casing" }, i => i.Index(index)); var processor = new ElasticQueryParser(c => c.SetLoggerFactory(Log)); @@ -776,7 +776,7 @@ public async Task MixedCaseTermFilterQueryProcessor() [Fact] public async Task MultipleWordsTermFilterQueryProcessor() { - string index = CreateRandomIndex(); + string index = await CreateRandomIndexAsync(); await Client.IndexAsync(new MyType { Field1 = "Blake Niemyjski" }, i => i.Index(index)); var processor = new ElasticQueryParser(c => c.SetLoggerFactory(Log)); @@ -796,7 +796,7 @@ public async Task MultipleWordsTermFilterQueryProcessor() [Fact] public async Task CanTranslateTermQueryProcessor() { - string index = CreateRandomIndex(); + string index = await CreateRandomIndexAsync(); await Client.IndexAsync(new MyType { Field1 = "Testing.Casing" }, i => i.Index(index)); var processor = new ElasticQueryParser(c => c.SetLoggerFactory(Log).AddVisitor(new UpdateFixedTermFieldToDateFixedExistsQueryVisitor())); @@ -817,7 +817,7 @@ public async Task CanTranslateTermQueryProcessor() [Fact] public async Task GroupedOrFilterProcessor() { - string index = CreateRandomIndex(); + string index = await CreateRandomIndexAsync(); await Client.IndexManyAsync([ new MyType { Field1 = "value1", Field2 = "value2" }, new MyType { Field1 = "value2", Field2 = "value2" }, @@ -846,7 +846,7 @@ await Client.IndexManyAsync([ [Fact] public async Task NestedFilterProcessor() { - string index = CreateRandomIndex(d => d.Properties(p => p + string index = await CreateRandomIndexAsync(d => d.Properties(p => p .Text(e => e.Field1, o => o.Index()) .Text(e => e.Field2, o => o.Index()) .Text(e => e.Field3, o => o.Index()) @@ -918,7 +918,7 @@ await Client.IndexManyAsync([ [Fact] public async Task NestedFilterProcessor2() { - string index = CreateRandomIndex(d => d.Properties(p => p + string index = await CreateRandomIndexAsync(d => d.Properties(p => p .Text(e => e.Field1, o => o.Index()) .Text(e => e.Field2, o => o.Index()) .Text(e => e.Field3, o => o.Index()) @@ -967,7 +967,7 @@ await Client.IndexManyAsync([ [Fact] public async Task CanGenerateMatchQuery() { - string index = CreateRandomIndex(m => m.Properties(p => p + string index = await CreateRandomIndexAsync(m => m.Properties(p => p .Text(e => e.Field1) .Fields(f1 => f1 .Keyword(k => k.Name("keyword").IgnoreAbove(256))))); @@ -990,7 +990,7 @@ public async Task CanGenerateMatchQuery() [Fact] public async Task CanBuildAliasQueryProcessor() { - string index = CreateRandomIndex(m => m.Properties(p => p + string index = await CreateRandomIndexAsync(m => m.Properties(p => p .Object>(f => f.Name(e => e.Data).Properties(p2 => p2 .Text(e => e.Name("@browser_version")) .FieldAlias(a => a.Name("browser.version").Path("data.@browser_version")))))); @@ -1013,7 +1013,7 @@ public async Task CanBuildAliasQueryProcessor() [Fact] public async Task NonAnalyzedPrefixQuery() { - string index = CreateRandomIndex(d => d.Properties(p => p.Keyword(e => e.Field1))); + string index = await CreateRandomIndexAsync(d => d.Properties(p => p.Keyword(e => e.Field1))); await Client.IndexAsync(new MyType { Field1 = "value123" }, i => i.Index(index)); await Client.Indices.RefreshAsync(index); @@ -1038,7 +1038,7 @@ public async Task NonAnalyzedPrefixQuery() [Fact] public async Task RangeQueryProcessor() { - string index = CreateRandomIndex(); + string index = await CreateRandomIndexAsync(); var res = await Client.IndexAsync(new MyType { Field1 = "value1", Field4 = 1 }, i => i.Index(index)); await Client.IndexAsync(new MyType { Field4 = 2 }, i => i.Index(index)); await Client.IndexAsync(new MyType { Field1 = "value1", Field4 = 3 }, i => i.Index(index)); @@ -1071,7 +1071,7 @@ await Client.SearchAsync( [Fact] public async Task DateRangeWithWildcardMinQueryProcessor() { - string index = CreateRandomIndex(); + string index = await CreateRandomIndexAsync(); var res = await Client.IndexAsync(new MyType { Field1 = "value1", Field4 = 1, Field5 = DateTime.UtcNow }, i => i.Index(index)); await Client.IndexAsync(new MyType { Field4 = 2 }, i => i.Index(index)); await Client.IndexAsync(new MyType { Field1 = "value1", Field4 = 3, Field5 = DateTime.UtcNow }, i => i.Index(index)); @@ -1103,7 +1103,7 @@ public async Task DateRangeWithWildcardMinQueryProcessor() [Fact] public async Task DateRangeWithDateMathQueryProcessor() { - string index = CreateRandomIndex(); + string index = await CreateRandomIndexAsync(); var res = await Client.IndexAsync(new MyType { Field1 = "value1", Field4 = 1, Field5 = DateTime.UtcNow }, i => i.Index(index)); await Client.IndexAsync(new MyType { Field4 = 2 }, i => i.Index(index)); await Client.IndexAsync(new MyType { Field1 = "value1", Field4 = 3, Field5 = DateTime.UtcNow }, i => i.Index(index)); @@ -1133,7 +1133,7 @@ public async Task DateRangeWithDateMathQueryProcessor() [Fact] public async Task DateRangeWithWildcardMaxQueryProcessor() { - string index = CreateRandomIndex(); + string index = await CreateRandomIndexAsync(); var res = await Client.IndexAsync(new MyType { Field1 = "value1", Field4 = 1, Field5 = DateTime.UtcNow }, i => i.Index(index)); await Client.IndexAsync(new MyType { Field4 = 2 }, i => i.Index(index)); await Client.IndexAsync(new MyType { Field1 = "value1", Field4 = 3, Field5 = DateTime.UtcNow }, i => i.Index(index)); @@ -1164,7 +1164,7 @@ public async Task DateRangeWithWildcardMaxQueryProcessor() [Fact] public async Task DateRangeWithTimeZone() { - string index = CreateRandomIndex(); + string index = await CreateRandomIndexAsync(); var res = await Client.IndexAsync(new MyType { Field1 = "value1", Field4 = 1, Field5 = DateTime.UtcNow }, i => i.Index(index)); await Client.IndexAsync(new MyType { Field4 = 2 }, i => i.Index(index)); await Client.IndexAsync(new MyType { Field1 = "value1", Field4 = 3, Field5 = DateTime.UtcNow }, i => i.Index(index)); @@ -1195,7 +1195,7 @@ public async Task DateRangeWithTimeZone() [Fact] public async Task DateRangeQueryProcessor() { - string index = CreateRandomIndex(); + string index = await CreateRandomIndexAsync(); var res = await Client.IndexAsync(new MyType { Field1 = "value1", Field4 = 1, Field5 = DateTime.UtcNow }, i => i.Index(index)); await Client.IndexAsync(new MyType { Field4 = 2 }, i => i.Index(index)); await Client.IndexAsync(new MyType { Field1 = "value1", Field4 = 3, Field5 = DateTime.UtcNow }, i => i.Index(index)); @@ -1227,7 +1227,7 @@ await Client.SearchAsync(d => d [Fact] public async Task SimpleGeoRangeQuery() { - string index = CreateRandomIndex(m => m.Properties(p => p.GeoPoint(g => g.Field3))); + string index = await CreateRandomIndexAsync(m => m.Properties(p => p.GeoPoint(g => g.Field3))); var res = await Client.IndexAsync(new MyType { Field1 = "value1", Field4 = 1, Field3 = "51.5032520,-0.1278990" }, i => i.Index(index)); await Client.IndexAsync(new MyType { Field4 = 2 }, i => i.Index(index)); @@ -1258,7 +1258,7 @@ await processor.BuildQueryAsync("field3:[51.5032520,-0.1278990 TO 51.5032520,-0. [Fact] public async Task CanUseValidationToGetUnresolvedFields() { - string index = CreateRandomIndex(d => d.Properties(p => p.Keyword(e => e.Field1))); + string index = await CreateRandomIndexAsync(d => d.Properties(p => p.Keyword(e => e.Field1))); await Client.IndexAsync(new MyType { Field1 = "value123" }, i => i.Index(index)); await Client.Indices.RefreshAsync(index); @@ -1300,7 +1300,7 @@ public async Task CanUseValidationToGetUnresolvedFields() [Fact] public async Task CanSortByUnmappedField() { - string index = CreateRandomIndex(m => m.Dynamic(DynamicMapping.True)); + string index = await CreateRandomIndexAsync(m => m.Dynamic(DynamicMapping.True)); var processor = new ElasticQueryParser(c => c.UseMappings(Client, index)); var sort = await processor.BuildSortAsync("-field1"); @@ -1324,7 +1324,7 @@ public async Task CanSortByUnmappedField() [Fact] public async Task CanParseSort() { - string index = CreateRandomIndex(d => d.Properties(p => p.GeoPoint(g => g.Field3) + string index = await CreateRandomIndexAsync(d => d.Properties(p => p.GeoPoint(g => g.Field3) .Text(e => e.Field1, o => o.Fields(f1 => f1.Keyword("keyword"))) .Text(e => e.Field2, o => o.Fields(f2 => f2.Keyword("keyword").Keyword(e2 => e2.Name("sort")))) )); @@ -1363,7 +1363,7 @@ public async Task CanParseSort() [Fact] public async Task CanHandleSpacedFields() { - string index = CreateRandomIndex(); + string index = await CreateRandomIndexAsync(); await Client.IndexManyAsync([ new MyNestedType @@ -1411,7 +1411,7 @@ await Client.IndexManyAsync([ [Fact] public async Task CanParseMixedCaseSort() { - string index = CreateRandomIndex(d => d.Properties(p => p + string index = await CreateRandomIndexAsync(d => d.Properties(p => p .Text(e => e.Name(m => m.MultiWord).Fields(f1 => f1.Keyword("keyword"))) )); @@ -1436,7 +1436,7 @@ public async Task CanParseMixedCaseSort() [Fact] public async Task GeoRangeQueryProcessor() { - string index = CreateRandomIndex(m => m.Properties(p => p + string index = await CreateRandomIndexAsync(m => m.Properties(p => p .GeoPoint(g => g.Field3) .Text(e => e.Field1, o => o.Fields(f1 => f1.Keyword("keyword"))))); @@ -1517,7 +1517,7 @@ private async Task GetIncludeAsync(string name) [InlineData("terms:(field1~100 (@missing:__missing__))")] public async Task CanValidateAggregation(string aggregation) { - string index = CreateRandomIndex(d => d.Properties(p => p.Keyword(e => e.Field1))); + string index = await CreateRandomIndexAsync(d => d.Properties(p => p.Keyword(e => e.Field1))); var context = new ElasticQueryVisitorContext { QueryType = QueryTypes.Aggregation }; var parser = new ElasticQueryParser(c => c.UseMappings(Client, index).SetLoggerFactory(Log)); var node = await parser.ParseAsync(aggregation, context); diff --git a/tests/Foundatio.Parsers.ElasticQueries.Tests/InvertQueryTests.cs b/tests/Foundatio.Parsers.ElasticQueries.Tests/InvertQueryTests.cs index 676b2ceb..fd0caaec 100644 --- a/tests/Foundatio.Parsers.ElasticQueries.Tests/InvertQueryTests.cs +++ b/tests/Foundatio.Parsers.ElasticQueries.Tests/InvertQueryTests.cs @@ -143,7 +143,7 @@ public override async Task InitializeAsync() await base.InitializeAsync(); const string indexName = "test_invert"; - CreateNamedIndex(indexName, m => m + await CreateNamedIndexAsync(indexName, m => m .Properties(p => p .Keyword(p1 => p1.Id) .Keyword(p1 => p1.OrganizationId) diff --git a/tests/Foundatio.Parsers.ElasticQueries.Tests/Utility/ElasticsearchTestBase.cs b/tests/Foundatio.Parsers.ElasticQueries.Tests/Utility/ElasticsearchTestBase.cs index 7107e294..900c023a 100644 --- a/tests/Foundatio.Parsers.ElasticQueries.Tests/Utility/ElasticsearchTestBase.cs +++ b/tests/Foundatio.Parsers.ElasticQueries.Tests/Utility/ElasticsearchTestBase.cs @@ -30,19 +30,19 @@ public ElasticsearchTestBase(ITestOutputHelper output, T fixture) : base(output) protected ElasticsearchClient Client => _fixture.Client; - protected void CreateNamedIndex(string index, Func, TypeMappingDescriptor> configureMappings = null, Func configureIndex = null) where TModel : class + protected Task CreateNamedIndexAsync(string index, Func, TypeMappingDescriptor> configureMappings = null, Func configureIndex = null) where TModel : class { - _fixture.CreateNamedIndex(index, configureMappings, configureIndex); + return _fixture.CreateNamedIndexAsync(index, configureMappings, configureIndex); } - protected string CreateRandomIndex(Func, TypeMappingDescriptor> configureMappings = null, Func configureIndex = null) where TModel : class + protected Task CreateRandomIndexAsync(Func, TypeMappingDescriptor> configureMappings = null, Func configureIndex = null) where TModel : class { - return _fixture.CreateRandomIndex(configureMappings, configureIndex); + return _fixture.CreateRandomIndexAsync(configureMappings, configureIndex); } - protected CreateIndexResponse CreateIndex(IndexName index, Func configureIndex = null) + protected Task CreateIndexAsync(IndexName index, Action configureIndex = null) { - return _fixture.CreateIndex(index, configureIndex); + return _fixture.CreateIndexAsync(index, configureIndex); } /// @@ -98,34 +98,33 @@ protected ElasticsearchClient GetClient(Action conf protected virtual void ConfigureConnectionSettings(ElasticsearchClientSettings settings) { } - public void CreateNamedIndex(string index, Func, TypeMappingDescriptor> configureMappings = null, Func configureIndex = null) where T : class + public async Task CreateNamedIndexAsync(string index, Func, TypeMappingDescriptor> configureMappings = null, Func configureIndex = null) where T : class { configureMappings ??= m => m.AutoMap().Dynamic(DynamicMapping.True); - if (configureIndex == null) - configureIndex = i => i.NumberOfReplicas(0).Analysis(a => a.AddSortNormalizer()); + configureIndex ??= i => i.NumberOfReplicas(0).Analysis(a => a.AddSortNormalizer()); - CreateIndex(index, i => i.Settings(configureIndex(new IndexSettingsDescriptor())).Map(configureMappings)); + await CreateIndexAsync(index, i => i.Settings(configureIndex(new IndexSettingsDescriptor())).Map(configureMappings)); Client.ElasticsearchClientSettings.DefaultIndices[typeof(T)] = index; } - public string CreateRandomIndex(Func, TypeMappingDescriptor> configureMappings = null, Func configureIndex = null) where T : class + public async Task CreateRandomIndexAsync(Func, TypeMappingDescriptor> configureMappings = null, Func configureIndex = null) where T : class { - string index = "test_" + Guid.NewGuid().ToString("N"); + string index = $"test_{Guid.NewGuid():N}"; configureMappings ??= m => m.AutoMap().Dynamic(DynamicMapping.True); configureIndex ??= i => i.NumberOfReplicas(0).Analysis(a => a.AddSortNormalizer()); - CreateIndex(index, i => i.Settings(configureIndex(new IndexSettingsDescriptor())).Map(configureMappings)); + await CreateIndexAsync(index, i => i.Settings(configureIndex(new IndexSettingsDescriptor())).Map(configureMappings)); Client.ElasticsearchClientSettings.DefaultIndices[typeof(T)] = index; return index; } - public CreateIndexResponse CreateIndex(IndexName index, Func configureIndex = null) + public async Task CreateIndexAsync(IndexName index, Action configureIndex = null) { _createdIndexes.Add(index); configureIndex ??= d => d.Settings(s => s.NumberOfReplicas(0)); - var result = Client.Indices.Create(index, configureIndex); + var result = await Client.Indices.CreateAsync(index, configureIndex); if (!result.IsValidResponse) throw new ApplicationException($"Unable to create index {index}: " + result.DebugInformation); From 799f7f37b031539cef1bb08991141979327bb174 Mon Sep 17 00:00:00 2001 From: Blake Niemyjski Date: Fri, 13 Sep 2024 13:27:49 -0500 Subject: [PATCH 13/19] Small updates --- .../ElasticQueryParserTests.cs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/Foundatio.Parsers.ElasticQueries.Tests/ElasticQueryParserTests.cs b/tests/Foundatio.Parsers.ElasticQueries.Tests/ElasticQueryParserTests.cs index 5d9ee832..48e96eb4 100644 --- a/tests/Foundatio.Parsers.ElasticQueries.Tests/ElasticQueryParserTests.cs +++ b/tests/Foundatio.Parsers.ElasticQueries.Tests/ElasticQueryParserTests.cs @@ -1412,7 +1412,7 @@ await Client.IndexManyAsync([ public async Task CanParseMixedCaseSort() { string index = await CreateRandomIndexAsync(d => d.Properties(p => p - .Text(e => e.Name(m => m.MultiWord).Fields(f1 => f1.Keyword("keyword"))) + .Text(e => e.MultiWord, o => o.Fields(f1 => f1.Keyword("keyword"))) )); var res = await Client.IndexAsync(new MyType { MultiWord = "value1" }, i => i.Index(index)); @@ -1529,7 +1529,7 @@ public async Task CanValidateAggregation(string aggregation) } } -public class MyType +public record MyType { public string Id { get; set; } public string Field1 { get; set; } @@ -1541,7 +1541,7 @@ public class MyType public Dictionary Data { get; set; } = new Dictionary(); } -public class MyNestedType +public record MyNestedType { public string Field1 { get; set; } public string Field2 { get; set; } From afc472fca70e1630fad3608f359c3e61ad86e3ad Mon Sep 17 00:00:00 2001 From: Blake Niemyjski Date: Mon, 16 Sep 2024 15:38:04 -0500 Subject: [PATCH 14/19] More test updates --- .../AggregationParserTests.cs | 150 ++++++++++-------- .../CustomVisitorTests.cs | 12 +- .../ElasticMappingResolverTests.cs | 9 +- .../ElasticQueryParserTests.cs | 7 +- 4 files changed, 94 insertions(+), 84 deletions(-) diff --git a/tests/Foundatio.Parsers.ElasticQueries.Tests/AggregationParserTests.cs b/tests/Foundatio.Parsers.ElasticQueries.Tests/AggregationParserTests.cs index 2b01d71f..b24d8dea 100644 --- a/tests/Foundatio.Parsers.ElasticQueries.Tests/AggregationParserTests.cs +++ b/tests/Foundatio.Parsers.ElasticQueries.Tests/AggregationParserTests.cs @@ -3,6 +3,8 @@ using System.Linq; using System.Threading.Tasks; using Elastic.Clients.Elasticsearch; +using Elastic.Clients.Elasticsearch.Aggregations; +using Elastic.Clients.Elasticsearch.Core.Search; using Elastic.Clients.Elasticsearch.Mapping; using Foundatio.Parsers.ElasticQueries.Extensions; using Foundatio.Parsers.ElasticQueries.Visitors; @@ -40,7 +42,7 @@ await Client.IndexManyAsync([ _logger.LogInformation("Actual: {Request}", actualRequest); var expectedResponse = await Client.SearchAsync(d => d.Index(index).Aggregations(a => a - .Min("min_field4", c => c.Field("field4").Meta(m => m.Add("@field_type", "long"))))); + .Add("min_field4", a1 => a1.Min(c => c.Field("field4").Meta(m => m.Add("@field_type", "long")))))); string expectedRequest = expectedResponse.GetRequest(); _logger.LogInformation("Expected: {Request}", expectedRequest); @@ -88,7 +90,7 @@ public async Task ProcessAnalyzedAggregationWithAliasAsync() string index = await CreateRandomIndexAsync(d => d.Dynamic(DynamicMapping.True).Properties(p => p .Text(f => f.Field1, o => o .Fields(k => k.Keyword("keyword"))) - .FieldAlias(f => f.Name("heynow").Path(k => k.Field1)) + .FieldAlias("heynow", o => o.Path(k => k.Field1)) .GeoPoint(g => g.Field3))); await Client.IndexManyAsync([ new MyType { Field1 = "value1", Field4 = 1, Field3 = "51.5032520,-0.1278990", Field5 = DateTime.UtcNow.Subtract(TimeSpan.FromMinutes(5)), Field2 = "field2" }, @@ -108,7 +110,7 @@ await Client.IndexManyAsync([ _logger.LogInformation("Actual: {Request}", actualRequest); var expectedResponse = await Client.SearchAsync(d => d.Index(index).Aggregations(a => a - .Terms("terms_heynow", c => c.Field("field1.keyword").Meta(m => m.Add("@field_type", "text"))))); + .Add("terms_heynow", a1 => a1.Terms(c => c.Field("field1.keyword").Meta(m => m.Add("@field_type", "text")))))); string expectedRequest = expectedResponse.GetRequest(); _logger.LogInformation("Expected: {Request}", expectedRequest); @@ -139,18 +141,21 @@ await Client.IndexManyAsync([ _logger.LogInformation("Actual: {Request}", actualRequest); var expectedResponse = await Client.SearchAsync(d => d.Index(index).Aggregations(a => a - .Geohash("geogrid_field3", h => h.Field("field3").GeoHashPrecision(1) - .Aggregations(a1 => a1.Average("avg_lat", s => s.Script(ss => ss.Source("doc['field3'].lat"))).Average("avg_lon", s => s.Script(ss => ss.Source("doc['field3'].lon"))))) - .Terms("terms_field1", t => t.Field("field1.keyword").Meta(m => m.Add("@field_type", "text"))) - .Histogram("histogram_field4", h => h.Field("field4").Interval(50).MinimumDocumentCount(0)) - .DateHistogram("date_field5", d1 => d1.Field("field5").CalendarInterval(DateInterval.Day).Format("date_optional_time").MinimumDocumentCount(0)) - .Missing("missing_field2", t => t.Field("field2.keyword")) - .Cardinality("cardinality_field4", c => c.Field("field4")) - .Percentiles("percentiles_field4", c => c.Field("field4").Percents(50, 100)) - .Sum("sum_field4", c => c.Field("field4").Meta(m => m.Add("@field_type", "long"))) - .Average("avg_field4", c => c.Field("field4").Meta(m => m.Add("@field_type", "long"))) - .Max("max_field4", c => c.Field("field4").Meta(m => m.Add("@field_type", "long"))) - .Min("min_field4", c => c.Field("field4").Meta(m => m.Add("@field_type", "long"))))); + .Add("geogrid_field3", a1 => a1.GeohashGrid(h => h.Field("field3").Precision(new GeohashPrecision(1))) + .Aggregations(a2 => a2 + .Add("avg_lat", a3 => a3.Avg(s => s.Script(ss => ss.Source("doc['field3'].lat")))) + .Add("avg_lon", a3 => a3.Avg(s => s.Script(ss => ss.Source("doc['field3'].lon")))) + )) + .Add("terms_field1", a1 => a1.Terms(t => t.Field("field1.keyword").Meta(m => m.Add("@field_type", "text")))) + .Add("histogram_field4", a1 => a1.Histogram(h => h.Field("field4").Interval(50).MinDocCount(0))) + .Add("date_field5", a1 => a1.DateHistogram(d1 => d1.Field("field5").CalendarInterval(CalendarInterval.Day).Format("date_optional_time").MinDocCount(0))) + .Add("missing_field2", a1 => a1.Missing(t => t.Field("field2.keyword"))) + .Add("cardinality_field4", a1 => a1.Cardinality(c => c.Field("field4"))) + .Add("percentiles_field4", a1 => a1.Percentiles(c => c.Field("field4").Percents([50, 100]))) + .Add("sum_field4", a1 => a1.Sum(c => c.Field("field4").Meta(m => m.Add("@field_type", "long")))) + .Add("avg_field4", a1 => a1.Avg(c => c.Field("field4").Meta(m => m.Add("@field_type", "long")))) + .Add("max_field4", a1 => a1.Max(c => c.Field("field4").Meta(m => m.Add("@field_type", "long")))) + .Add("min_field4", a1 => a1.Min(c => c.Field("field4").Meta(m => m.Add("@field_type", "long")))))); string expectedRequest = expectedResponse.GetRequest(); _logger.LogInformation("Expected: {Request}", expectedRequest); @@ -165,10 +170,10 @@ public async Task ProcessNestedAggregationsWithAliasesAsync() { string index = await CreateRandomIndexAsync(d => d.Dynamic(DynamicMapping.True).Properties(p => p .GeoPoint(g => g.Field3) - .Object>(o1 => o1.Name(f1 => f1.Data).Properties(p1 => p1 - .Object(o2 => o2.Name("@user").Properties(p2 => p2 - .Text(f3 => f3.Name("identity") - .Fields(f => f.Keyword(k => k.Name("keyword").IgnoreAbove(256)))))))))); + .Object(o1 => o1.Data, o => o.Properties(p1 => p1 + .Object("@user", o1 => o1.Properties(p2 => p2 + .Text("identity", o2 => o2 + .Fields(f => f.Keyword("keyword", o3 => o3.IgnoreAbove(256)))))))))); await Client.IndexManyAsync([new MyType { Field1 = "value1" }], index); await Client.Indices.RefreshAsync(index); @@ -182,8 +187,8 @@ public async Task ProcessNestedAggregationsWithAliasesAsync() _logger.LogInformation("Actual: {Request}", actualRequest); var expectedResponse = await Client.SearchAsync(d => d.Index(index).Aggregations(a => a - .Terms("terms_alias1", t => t.Field("field1.keyword").Meta(m => m.Add("@field_type", "keyword")) - .Aggregations(a1 => a1.Cardinality("cardinality_user", c => c.Field("data.@user.identity.keyword")))))); + .Add("terms_alias1", a1 => a1.Terms(t => t.Field("field1.keyword").Meta(m => m.Add("@field_type", "keyword"))) + .Aggregations(a2 => a2.Add("cardinality_user", a3 => a3.Cardinality(c => c.Field("data.@user.identity.keyword"))))))); string expectedRequest = expectedResponse.GetRequest(); _logger.LogInformation("Expected: {Request}", expectedRequest); @@ -212,7 +217,7 @@ await Client.IndexManyAsync([ _logger.LogInformation("Actual: {Request}", actualRequest); var expectedResponse = await Client.SearchAsync(d => d.Index(index).Aggregations(a => a - .Missing("missing_alias2", t => t.Field("field2.keyword")))); + .Add("missing_alias2", a1 => a1.Missing(t => t.Field("field2.keyword"))))); string expectedRequest = expectedResponse.GetRequest(); _logger.LogInformation("Expected: {Request}", expectedRequest); @@ -228,9 +233,9 @@ public async Task ProcessAggregationsWithAliasesAsync() { string index = await CreateRandomIndexAsync(d => d.Dynamic(DynamicMapping.True).Properties(p => p .GeoPoint(g => g.Field3) - .Object>(o1 => o1.Name(f1 => f1.Data).Properties(p1 => p1 - .Object(o2 => o2.Name("@user").Properties(p2 => p2 - .Text(f3 => f3.Name("identity").Fields(f => f.Keyword(k => k.Name("keyword").IgnoreAbove(256)))))))))); + .Object(o1 => o1.Data, o2 => o2.Properties(p1 => p1 + .Object("@user", o3 => o3.Properties(p2 => p2 + .Text("identity", o4 => o4.Fields(f => f.Keyword("keyword", o5 => o5.IgnoreAbove(256)))))))))); await Client.IndexManyAsync([ new MyType { Field1 = "value1", Field4 = 1, Field3 = "51.5032520,-0.1278990", Field5 = DateTime.UtcNow.Subtract(TimeSpan.FromMinutes(5)), Field2 = "field2" } @@ -246,18 +251,22 @@ await Client.IndexManyAsync([ _logger.LogInformation("Actual: {Request}", actualRequest); var expectedResponse = await Client.SearchAsync(d => d.Index(index).Aggregations(a => a - .GeoHash("geogrid_alias3", h => h.Field("field3").GeoHashPrecision(1) - .Aggregations(a1 => a1.Average("avg_lat", s => s.Script(ss => ss.Source("doc['field3'].lat"))).Average("avg_lon", s => s.Script(ss => ss.Source("doc['field3'].lon"))))) - .Terms("terms_alias1", t => t.Field("field1.keyword").Meta(m => m.Add("@field_type", "text"))) - .Histogram("histogram_alias4", h => h.Field("field4").Interval(50).MinimumDocumentCount(0)) - .DateHistogram("date_alias5", d1 => d1.Field("field5").CalendarInterval(DateInterval.Day).Format("date_optional_time").MinimumDocumentCount(0)) - .Missing("missing_alias2", t => t.Field("field2.keyword")) - .Cardinality("cardinality_user", c => c.Field("data.@user.identity.keyword")) - .Percentiles("percentiles_alias4", c => c.Field("field4")) - .Sum("sum_alias4", c => c.Field("field4").Meta(m => m.Add("@field_type", "long"))) - .Average("avg_alias4", c => c.Field("field4").Meta(m => m.Add("@field_type", "long"))) - .Max("max_alias4", c => c.Field("field4").Meta(m => m.Add("@field_type", "long"))) - .Min("min_alias4", c => c.Field("field4").Meta(m => m.Add("@field_type", "long"))))); + .Add("geogrid_alias3", a1 => a1.GeohashGrid(h => h.Field("field3").Precision(new GeohashPrecision(1))) + .Aggregations(a2 => a2 + .Add("avg_lat", a3 => a3.Avg(s => s.Script(ss => ss.Source("doc['field3'].lat")))) + .Add("avg_lon", a3 => a3.Avg(s => s.Script(ss => ss.Source("doc['field3'].lon")))) + )) + .Add("terms_alias1", a1 => a1.Terms(t => t.Field("field1.keyword").Meta(m => m.Add("@field_type", "text")))) + .Add("histogram_alias4", a1 => a1.Histogram(h => h.Field("field4").Interval(50).MinDocCount(0))) + .Add("date_alias5", a1 => a1.DateHistogram(d1 => d1.Field("field5").CalendarInterval(CalendarInterval.Day).Format("date_optional_time").MinDocCount(0))) + .Add("missing_alias2", a1 => a1.Missing(t => t.Field("field2.keyword"))) + .Add("cardinality_user", a1 => a1.Cardinality(c => c.Field("data.@user.identity.keyword"))) + .Add("percentiles_alias4", a1 => a1.Percentiles(c => c.Field("field4"))) + .Add("sum_alias4", a1 => a1.Sum(c => c.Field("field4").Meta(m => m.Add("@field_type", "long")))) + .Add("avg_alias4", a1 => a1.Avg(c => c.Field("field4").Meta(m => m.Add("@field_type", "long")))) + .Add("max_alias4", a1 => a1.Max(c => c.Field("field4").Meta(m => m.Add("@field_type", "long")))) + .Add("min_alias4", a1 => a1.Min(c => c.Field("field4").Meta(m => m.Add("@field_type", "long")))))); + string expectedRequest = expectedResponse.GetRequest(); _logger.LogInformation("Expected: {Request}", expectedRequest); @@ -282,13 +291,14 @@ public async Task ProcessTermAggregations() _logger.LogInformation("Actual: {Request}", actualRequest); var expectedResponse = await Client.SearchAsync(d => d.Index(index).Aggregations(a => a - .Terms("terms_field1", t => t + .Add("terms_field1", a1 => a1.Terms(t => t .Field("field1.keyword") - .MinimumDocumentCount(1) - .Include(["otherinclude", "myinclude"]) - .Exclude(["otherexclude", "myexclude"]) + .MinDocCount(1) + .Include(new TermsInclude(["otherinclude", "myinclude"])) + .Exclude(new TermsExclude(["otherexclude", "myexclude"])) .Missing("mymissing") - .Meta(m => m.Add("@field_type", "keyword"))))); + .Meta(m => m.Add("@field_type", "keyword")))))); + string expectedRequest = expectedResponse.GetRequest(); _logger.LogInformation("Expected: {Request}", expectedRequest); @@ -311,11 +321,12 @@ public async Task ProcessHistogramIntervalAggregations() _logger.LogInformation("Actual: {Request}", actualRequest); var expectedResponse = await Client.SearchAsync(d => d.Index(index).Aggregations(a => a - .Histogram("histogram_field1", t => t + .Add("histogram_field1", a1 => a1.Histogram(t => t .Field("field1.keyword") .Interval(0.1) - .MinimumDocumentCount(0) - ))); + .MinDocCount(0) + )))); + string expectedRequest = expectedResponse.GetRequest(); _logger.LogInformation("Expected: {Request}", expectedRequest); @@ -338,11 +349,9 @@ public async Task ProcessTermTopHitsAggregations() _logger.LogInformation("Actual: {Request}", actualRequest); var expectedResponse = await Client.SearchAsync(d => d.Index(index).Aggregations(a => a - .Terms("terms_field1", t => t - .Field("field1.keyword") - .Size(1000) - .MinimumDocumentCount(2) - .Aggregations(a1 => a1.TopHits("tophits", t2 => t2.Size(1000).Source(s => s.Includes(i => i.Field("myinclude"))))) + .Add("terms_field1", a1 => a1 + .Terms(t => t.Field("field1.keyword").Size(1000).MinDocCount(2)) + .Aggregations(a2 => a2.Add("tophits", a3 => a3.TopHits(t2 => t2.Size(1000).Source(new SourceConfig(new SourceFilter { Includes = Fields.FromString("myinclude")}))))) .Meta(m => m.Add("@field_type", "keyword"))))); string expectedRequest = expectedResponse.GetRequest(); _logger.LogInformation("Expected: {Request}", expectedRequest); @@ -366,12 +375,10 @@ public async Task ProcessSortedTermAggregations() _logger.LogInformation("Actual: {Request}", actualRequest); var expectedResponse = await Client.SearchAsync(d => d.Index(index).Aggregations(a => a - .Terms("terms_field1", t => t - .Field("field1.keyword") - .Order(o => o.Descending("cardinality_field4")) - .Aggregations(a2 => a2 - .Cardinality("cardinality_field4", c => c.Field("field4"))) - .Meta(m => m.Add("@field_type", "keyword"))))); + .Add("terms_field1", a1 => a1 + .Terms(t => t.Field("field1.keyword").Order(new List> { new ("cardinality_field4", SortOrder.Desc) })) + .Aggregations(a2 => a2.Add("cardinality_field4", a3 => a3.Cardinality(c => c.Field("field4")))) + .Meta(m => m.Add("@field_type", "keyword"))))); string expectedRequest = expectedResponse.GetRequest(); _logger.LogInformation("Expected: {Request}", expectedRequest); @@ -396,16 +403,18 @@ public async Task ProcessDateHistogramAggregations() _logger.LogInformation("Actual: {Request}", actualRequest); var expectedResponse = await Client.SearchAsync(d => d.Index(index).Aggregations(a => a - .DateHistogram("date_field5", d1 => d1 - .Field("field5").Meta(m => m.Add("@timezone", "1h")) - .CalendarInterval(DateInterval.Day) + .Add("date_field5", a1 => a1.DateHistogram(d1 => d1 + .Field("field5") + .Meta(m => m.Add("@timezone", "1h")) + .CalendarInterval(CalendarInterval.Day) .Format("date_optional_time") - .MinimumDocumentCount(0) + .MinDocCount(0) .TimeZone("+01:00") - .Missing(DateTime.MinValue) - .Aggregations(a1 => a1 - .Min("min_field5", s => s.Field(f => f.Field5).Meta(m => m.Add("@field_type", "date").Add("@timezone", "1h"))) - .Max("max_field5", s => s.Field(f => f.Field5).Meta(m => m.Add("@field_type", "date").Add("@timezone", "1h"))))))); + .Missing(DateTime.MinValue)) + .Aggregations(a2 => a2 + .Add("min_field5", a3 => a3.Min(c => c.Field("field5").Meta(m => m.Add("@field_type", "date").Add("@timezone", "1h")))) + .Add("max_field5", a3 => a3.Max(c => c.Field("field5").Meta(m => m.Add("@field_type", "date").Add("@timezone", "1h")))))))); + string expectedRequest = expectedResponse.GetRequest(); _logger.LogInformation("Expected: {Request}", expectedRequest); @@ -430,11 +439,12 @@ public async Task CanSpecifyDefaultValuesAggregations() _logger.LogInformation("Actual: {Request}", actualRequest); var expectedResponse = await Client.SearchAsync(d => d.Index(index).Aggregations(a => a - .Sum("sum_field4", c => c.Field("field4").Missing(0).Meta(m => m.Add("@field_type", "integer"))) - .Cardinality("cardinality_field4", c => c.Field("field4").Missing(0)) - .Average("avg_field4", c => c.Field("field4").Missing(0).Meta(m => m.Add("@field_type", "integer"))) - .Max("max_field4", c => c.Field("field4").Missing(0).Meta(m => m.Add("@field_type", "integer"))) - .Min("min_field4", c => c.Field("field4").Missing(0).Meta(m => m.Add("@field_type", "integer"))))); + .Add("sum_field4", a1 => a1.Sum(c => c.Field("field4").Missing(0).Meta(m => m.Add("@field_type", "integer")))) + .Add("cardinality_field4", a1 => a1.Cardinality(c => c.Field("field4").Missing(0))) + .Add("avg_field4", a1 => a1.Avg(c => c.Field("field4").Missing(0).Meta(m => m.Add("@field_type", "integer")))) + .Add("max_field4", a1 => a1.Max(c => c.Field("field4").Missing(0).Meta(m => m.Add("@field_type", "integer")))) + .Add("min_field4", a1 => a1.Min(c => c.Field("field4").Missing(0).Meta(m => m.Add("@field_type", "integer")))))); + string expectedRequest = expectedResponse.GetRequest(); _logger.LogInformation("Expected: {Request}", expectedRequest); @@ -449,7 +459,7 @@ public async Task GeoGridDoesNotResolveLocationForAggregation() { string index = await CreateRandomIndexAsync(d => d.Properties(p => p .GeoPoint(g => g.Field1) - .FieldAlias(a => a.Name("geo").Path(f => f.Field1)))); + .FieldAlias("geo", o => o.Path(f => f.Field1)))); var processor = new ElasticQueryParser(c => c .UseGeo(_ => "someinvalidvaluehere") diff --git a/tests/Foundatio.Parsers.ElasticQueries.Tests/CustomVisitorTests.cs b/tests/Foundatio.Parsers.ElasticQueries.Tests/CustomVisitorTests.cs index 632fa6dc..846d9039 100644 --- a/tests/Foundatio.Parsers.ElasticQueries.Tests/CustomVisitorTests.cs +++ b/tests/Foundatio.Parsers.ElasticQueries.Tests/CustomVisitorTests.cs @@ -38,7 +38,7 @@ public async Task CanResolveSimpleCustomFilter() _logger.LogInformation("Actual: {Request}", actualRequest); var expectedResponse = await Client.SearchAsync(d => d - .Index(index).Query(f => f.Bool(b => b.Filter(filter => filter.Terms(m => m.Field("id").Terms("1")))))); + .Index(index).Query(f => f.Bool(b => b.Filter(filter => filter.Terms(m => m.Field("id").Term(new TermsQueryField(["1"]))))))); string expectedRequest = expectedResponse.GetRequest(); _logger.LogInformation("Expected: {Request}", expectedRequest); @@ -63,7 +63,7 @@ public async Task CanResolveCustomFilterContainingIncludes() _logger.LogInformation("Actual: {Request}", actualRequest); var expectedResponse = await Client.SearchAsync(d => d - .Index(index).Query(f => f.Bool(b => b.Filter(filter => filter.Terms(m => m.Field("id").Terms("1", "3")))))); + .Index(index).Query(f => f.Bool(b => b.Filter(filter => filter.Terms(m => m.Field("id").Term(new TermsQueryField(["1", "3"]))))))); string expectedRequest = expectedResponse.GetRequest(); _logger.LogInformation("Expected: {Request}", expectedRequest); @@ -88,7 +88,7 @@ public async Task CanResolveIncludeToCustomFilterContainingIgnoredInclude() _logger.LogInformation("Actual: {Request}", actualRequest); var expectedResponse = await Client.SearchAsync(d => d - .Index(index).Query(f => f.Bool(b => b.Filter(filter => filter.Terms(m => m.Field("id").Terms("1", "3")))))); + .Index(index).Query(f => f.Bool(b => b.Filter(filter => filter.Terms(m => m.Field("id").Term(new TermsQueryField(["1", "3"]))))))); string expectedRequest = expectedResponse.GetRequest(); _logger.LogInformation("Expected: {Request}", expectedRequest); @@ -101,7 +101,7 @@ private bool ShouldSkipInclude(TermNode node, IQueryVisitorContext context) var current = node.Parent; while (current != null) { - if (current is GroupNode groupNode && groupNode.Field == "@custom") + if (current is GroupNode { Field: "@custom" }) return true; current = current.Parent; @@ -140,10 +140,10 @@ public async Task CanResolveMultipleCustomFilters() .Filter(filter => filter .Bool(b1 => b1 .Should( - s1 => s1.Terms(m => m.Field("id").Terms("1")), + s1 => s1.Terms(m => m.Field("id").Term(new TermsQueryField(["1"]))), s2 => s2.Bool(b2 => b2 .Must( - m2 => m2.Terms(t1 => t1.Field("id").Terms("2")), + m2 => m2.Terms(t1 => t1.Field("id").Term(new TermsQueryField(["2"]))), m2 => m2.Term(t1 => t1.Field("field1").Value("Test")) ) ) diff --git a/tests/Foundatio.Parsers.ElasticQueries.Tests/ElasticMappingResolverTests.cs b/tests/Foundatio.Parsers.ElasticQueries.Tests/ElasticMappingResolverTests.cs index 8c1fad95..047a1a5d 100644 --- a/tests/Foundatio.Parsers.ElasticQueries.Tests/ElasticMappingResolverTests.cs +++ b/tests/Foundatio.Parsers.ElasticQueries.Tests/ElasticMappingResolverTests.cs @@ -1,4 +1,5 @@ using System; +using System.Collections.Generic; using System.Linq.Expressions; using System.Threading.Tasks; using Elastic.Clients.Elasticsearch; @@ -23,10 +24,10 @@ private TypeMappingDescriptor MapMyNestedType(TypeMappingDescripto .Dynamic(DynamicMapping.True) .DynamicTemplates(t => t.DynamicTemplate("idx_text", t => t.Match("text*").Mapping(m => m.Text(mp => mp.AddKeywordAndSortFields())))) .Properties(p => p - .Text(p1 => p1.Name(n => n.Field1).AddKeywordAndSortFields()) - .Text(p1 => p1.Name(n => n.Field4).AddKeywordAndSortFields()) - .FieldAlias(a => a.Path(n => n.Field4).Name("field4alias")) - .Text(p1 => p1.Name(n => n.Field5).AddKeywordAndSortFields(true)) + .Text(p1 => p1.Field1, o => o.AddKeywordAndSortFields()) + .Text(p1 => p1.Field4, o => o.AddKeywordAndSortFields()) + .FieldAlias(a => a.Field4, o => o.Path("field4alias")) + .Text(p1 => p1.Field5, o => o.AddKeywordAndSortFields(true)) ); } diff --git a/tests/Foundatio.Parsers.ElasticQueries.Tests/ElasticQueryParserTests.cs b/tests/Foundatio.Parsers.ElasticQueries.Tests/ElasticQueryParserTests.cs index 48e96eb4..f788dfa3 100644 --- a/tests/Foundatio.Parsers.ElasticQueries.Tests/ElasticQueryParserTests.cs +++ b/tests/Foundatio.Parsers.ElasticQueries.Tests/ElasticQueryParserTests.cs @@ -163,7 +163,7 @@ public async Task ShouldHandleMultipleTermsForAnalyzedFields() { string index = await CreateRandomIndexAsync(d => d .Dynamic(DynamicMapping.True).Properties(p => p.GeoPoint(g => g.Field3) - .Text(e => e.Field1, o => o.Fields(f1 => f1.Keyword(e1 => e1.Name("keyword")))) + .Text(e => e.Field1, o => o.Fields(f1 => f1.Keyword("keyword"))) .Keyword(e => e.Field2) )); await Client.IndexAsync(new MyType { Field1 = "value1", Field2 = "value2", Field3 = "value3" }, i => i.Index(index)); @@ -1326,10 +1326,9 @@ public async Task CanParseSort() { string index = await CreateRandomIndexAsync(d => d.Properties(p => p.GeoPoint(g => g.Field3) .Text(e => e.Field1, o => o.Fields(f1 => f1.Keyword("keyword"))) - .Text(e => e.Field2, o => o.Fields(f2 => f2.Keyword("keyword").Keyword(e2 => e2.Name("sort")))) + .Text(e => e.Field2, o => o.Fields(f2 => f2.Keyword("keyword").Keyword("sort"))) )); - var res = await Client.IndexAsync(new MyType { Field1 = "value1", Field4 = 1, Field3 = "51.5032520,-0.1278990" }, - i => i.Index(index)); + await Client.IndexAsync(new MyType { Field1 = "value1", Field4 = 1, Field3 = "51.5032520,-0.1278990" }, i => i.Index(index)); await Client.IndexAsync(new MyType { Field4 = 2 }, i => i.Index(index)); await Client.IndexAsync(new MyType { Field1 = "value1", Field4 = 3 }, i => i.Index(index)); await Client.Indices.RefreshAsync(index); From 4b7849241ccb0b4c053ff60b39b23dd4d37874c2 Mon Sep 17 00:00:00 2001 From: Blake Niemyjski Date: Mon, 16 Sep 2024 16:01:41 -0500 Subject: [PATCH 15/19] Fixed a query bug --- .idea/.idea.Foundatio.Parsers/.idea/projectSettingsUpdater.xml | 2 +- .../ElasticQueryParserTests.cs | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/.idea/.idea.Foundatio.Parsers/.idea/projectSettingsUpdater.xml b/.idea/.idea.Foundatio.Parsers/.idea/projectSettingsUpdater.xml index 4bb9f4d2..86cc6c63 100644 --- a/.idea/.idea.Foundatio.Parsers/.idea/projectSettingsUpdater.xml +++ b/.idea/.idea.Foundatio.Parsers/.idea/projectSettingsUpdater.xml @@ -1,6 +1,6 @@ - \ No newline at end of file diff --git a/tests/Foundatio.Parsers.ElasticQueries.Tests/ElasticQueryParserTests.cs b/tests/Foundatio.Parsers.ElasticQueries.Tests/ElasticQueryParserTests.cs index f788dfa3..3819c04e 100644 --- a/tests/Foundatio.Parsers.ElasticQueries.Tests/ElasticQueryParserTests.cs +++ b/tests/Foundatio.Parsers.ElasticQueries.Tests/ElasticQueryParserTests.cs @@ -1553,7 +1553,6 @@ public record MyNestedType public class UpdateFixedTermFieldToDateFixedExistsQueryVisitor : ChainableQueryVisitor { - public override void Visit(TermNode node, IQueryVisitorContext context) { if (!String.Equals(node.Field, "fixed", StringComparison.OrdinalIgnoreCase)) @@ -1562,7 +1561,7 @@ public override void Visit(TermNode node, IQueryVisitorContext context) if (!Boolean.TryParse(node.Term, out bool isFixed)) return; - var query = new ExistsQuery { Field = "date_fixed" }; + var query = Query.Exists(new ExistsQuery { Field = "date_fixed" }); node.SetQuery(isFixed ? query : !query); } } From 9859e81a2a8a9bda8dd0a04cebba30f4ea3757b6 Mon Sep 17 00:00:00 2001 From: Blake Niemyjski Date: Fri, 22 Nov 2024 13:51:41 -0600 Subject: [PATCH 16/19] Updated to latest client --- docker-compose.yml | 4 ++-- .../Foundatio.Parsers.ElasticQueries.csproj | 2 +- .../CustomVisitorTests.cs | 12 ++++++------ 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 4db30583..73c1cfe3 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,6 +1,6 @@ services: elasticsearch: - image: docker.elastic.co/elasticsearch/elasticsearch:8.15.1 + image: docker.elastic.co/elasticsearch/elasticsearch:8.16.1 environment: discovery.type: single-node xpack.security.enabled: 'false' @@ -19,7 +19,7 @@ services: depends_on: elasticsearch: condition: service_healthy - image: docker.elastic.co/kibana/kibana:8.15.1 + image: docker.elastic.co/kibana/kibana:8.16.1 ports: - 5601:5601 networks: diff --git a/src/Foundatio.Parsers.ElasticQueries/Foundatio.Parsers.ElasticQueries.csproj b/src/Foundatio.Parsers.ElasticQueries/Foundatio.Parsers.ElasticQueries.csproj index cee9dd8d..c2569418 100644 --- a/src/Foundatio.Parsers.ElasticQueries/Foundatio.Parsers.ElasticQueries.csproj +++ b/src/Foundatio.Parsers.ElasticQueries/Foundatio.Parsers.ElasticQueries.csproj @@ -1,7 +1,7 @@ - + diff --git a/tests/Foundatio.Parsers.ElasticQueries.Tests/CustomVisitorTests.cs b/tests/Foundatio.Parsers.ElasticQueries.Tests/CustomVisitorTests.cs index 846d9039..ccdf8bc4 100644 --- a/tests/Foundatio.Parsers.ElasticQueries.Tests/CustomVisitorTests.cs +++ b/tests/Foundatio.Parsers.ElasticQueries.Tests/CustomVisitorTests.cs @@ -38,7 +38,7 @@ public async Task CanResolveSimpleCustomFilter() _logger.LogInformation("Actual: {Request}", actualRequest); var expectedResponse = await Client.SearchAsync(d => d - .Index(index).Query(f => f.Bool(b => b.Filter(filter => filter.Terms(m => m.Field("id").Term(new TermsQueryField(["1"]))))))); + .Index(index).Query(f => f.Bool(b => b.Filter(filter => filter.Terms(m => m.Field("id").Terms(new TermsQueryField(["1"]))))))); string expectedRequest = expectedResponse.GetRequest(); _logger.LogInformation("Expected: {Request}", expectedRequest); @@ -63,7 +63,7 @@ public async Task CanResolveCustomFilterContainingIncludes() _logger.LogInformation("Actual: {Request}", actualRequest); var expectedResponse = await Client.SearchAsync(d => d - .Index(index).Query(f => f.Bool(b => b.Filter(filter => filter.Terms(m => m.Field("id").Term(new TermsQueryField(["1", "3"]))))))); + .Index(index).Query(f => f.Bool(b => b.Filter(filter => filter.Terms(m => m.Field("id").Terms(new TermsQueryField(["1", "3"]))))))); string expectedRequest = expectedResponse.GetRequest(); _logger.LogInformation("Expected: {Request}", expectedRequest); @@ -88,7 +88,7 @@ public async Task CanResolveIncludeToCustomFilterContainingIgnoredInclude() _logger.LogInformation("Actual: {Request}", actualRequest); var expectedResponse = await Client.SearchAsync(d => d - .Index(index).Query(f => f.Bool(b => b.Filter(filter => filter.Terms(m => m.Field("id").Term(new TermsQueryField(["1", "3"]))))))); + .Index(index).Query(f => f.Bool(b => b.Filter(filter => filter.Terms(m => m.Field("id").Terms(new TermsQueryField(["1", "3"]))))))); string expectedRequest = expectedResponse.GetRequest(); _logger.LogInformation("Expected: {Request}", expectedRequest); @@ -140,10 +140,10 @@ public async Task CanResolveMultipleCustomFilters() .Filter(filter => filter .Bool(b1 => b1 .Should( - s1 => s1.Terms(m => m.Field("id").Term(new TermsQueryField(["1"]))), + s1 => s1.Terms(m => m.Field("id").Terms(new TermsQueryField(["1"]))), s2 => s2.Bool(b2 => b2 .Must( - m2 => m2.Terms(t1 => t1.Field("id").Term(new TermsQueryField(["2"]))), + m2 => m2.Terms(t1 => t1.Field("id").Terms(new TermsQueryField(["2"]))), m2 => m2.Term(t1 => t1.Field("field1").Value("Test")) ) ) @@ -174,7 +174,7 @@ public override async Task VisitAsync(GroupNode node, IQueryVisitorContext conte string term = ToTerm(node); var ids = await GetIdsAsync(term); if (ids is { Count: > 0 }) - node.Parent.SetQuery(new TermsQuery { Field = "id", Term = new TermsQueryField(ids.Select(FieldValue.String).ToArray()) }); + node.Parent.SetQuery(new TermsQuery { Field = "id", Terms = new TermsQueryField(ids.Select(FieldValue.String).ToArray()) }); else node.Parent.SetQuery(new TermQuery("id") { Value = "none" }); From b679378f40dd7c8a17b3903436aea2a891fd6ea3 Mon Sep 17 00:00:00 2001 From: Blake Niemyjski Date: Fri, 24 Jan 2025 15:40:17 -0600 Subject: [PATCH 17/19] Updated elastic version --- .../Foundatio.Parsers.ElasticQueries.csproj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Foundatio.Parsers.ElasticQueries/Foundatio.Parsers.ElasticQueries.csproj b/src/Foundatio.Parsers.ElasticQueries/Foundatio.Parsers.ElasticQueries.csproj index c2569418..4d1bc5b4 100644 --- a/src/Foundatio.Parsers.ElasticQueries/Foundatio.Parsers.ElasticQueries.csproj +++ b/src/Foundatio.Parsers.ElasticQueries/Foundatio.Parsers.ElasticQueries.csproj @@ -1,7 +1,7 @@ - + From 7d9a25e6ba0d91603c0262bded3d2559d6bffb4a Mon Sep 17 00:00:00 2001 From: Blake Niemyjski Date: Fri, 28 Feb 2025 16:15:11 -0600 Subject: [PATCH 18/19] WIP: Fixed more compiler errors. --- .../.idea/projectSettingsUpdater.xml | 1 + .vscode/settings.json | 5 +- .../ElasticQueryParser.cs | 4 +- .../Visitors/GetSortFieldsVisitor.cs | 8 +- .../AggregationParserTests.cs | 204 ++++++++++++++---- .../ElasticMappingResolverTests.cs | 3 +- .../ElasticQueryParserTests.cs | 197 +++++++++-------- .../Utility/ElasticsearchTestBase.cs | 14 +- 8 files changed, 279 insertions(+), 157 deletions(-) diff --git a/.idea/.idea.Foundatio.Parsers/.idea/projectSettingsUpdater.xml b/.idea/.idea.Foundatio.Parsers/.idea/projectSettingsUpdater.xml index 86cc6c63..64af657f 100644 --- a/.idea/.idea.Foundatio.Parsers/.idea/projectSettingsUpdater.xml +++ b/.idea/.idea.Foundatio.Parsers/.idea/projectSettingsUpdater.xml @@ -1,6 +1,7 @@ + \ No newline at end of file diff --git a/.vscode/settings.json b/.vscode/settings.json index feef7153..f72de2fa 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -1,9 +1,10 @@ { "cSpell.words": [ + "aggs", + "Foundatio", "Lucene", "Niemyjski", - "Xunit", - "aggs" + "Xunit" ], "msbuildProjectTools.nuget.includePreRelease": true } \ No newline at end of file diff --git a/src/Foundatio.Parsers.ElasticQueries/ElasticQueryParser.cs b/src/Foundatio.Parsers.ElasticQueries/ElasticQueryParser.cs index 17cf4071..c1ec84db 100644 --- a/src/Foundatio.Parsers.ElasticQueries/ElasticQueryParser.cs +++ b/src/Foundatio.Parsers.ElasticQueries/ElasticQueryParser.cs @@ -229,7 +229,7 @@ public async Task ValidateSortAsync(string query, QueryVa return context.GetValidationResult(); } - public async Task> BuildSortAsync(string sort, IElasticQueryVisitorContext context = null) + public async Task> BuildSortAsync(string sort, IElasticQueryVisitorContext context = null) { context ??= new ElasticQueryVisitorContext(); context.QueryType = QueryTypes.Sort; @@ -240,7 +240,7 @@ public async Task> BuildSortAsync(string sort, IElastic return await BuildSortAsync(result, context).ConfigureAwait(false); } - public Task> BuildSortAsync(IQueryNode sort, IElasticQueryVisitorContext context = null) + public Task> BuildSortAsync(IQueryNode sort, IElasticQueryVisitorContext context = null) { context ??= new ElasticQueryVisitorContext(); return GetSortFieldsVisitor.RunAsync(sort, context); diff --git a/src/Foundatio.Parsers.ElasticQueries/Visitors/GetSortFieldsVisitor.cs b/src/Foundatio.Parsers.ElasticQueries/Visitors/GetSortFieldsVisitor.cs index 94872fb5..e10a54c9 100644 --- a/src/Foundatio.Parsers.ElasticQueries/Visitors/GetSortFieldsVisitor.cs +++ b/src/Foundatio.Parsers.ElasticQueries/Visitors/GetSortFieldsVisitor.cs @@ -8,7 +8,7 @@ namespace Foundatio.Parsers.ElasticQueries.Visitors; -public class GetSortFieldsVisitor : QueryNodeVisitorWithResultBase> +public class GetSortFieldsVisitor : QueryNodeVisitorWithResultBase> { private readonly List _fields = new(); @@ -29,18 +29,18 @@ public override void Visit(TermNode node, IQueryVisitorContext context) _fields.Add(sort); } - public override async Task> AcceptAsync(IQueryNode node, IQueryVisitorContext context) + public override async Task> AcceptAsync(IQueryNode node, IQueryVisitorContext context) { await node.AcceptAsync(this, context).ConfigureAwait(false); return _fields; } - public static Task> RunAsync(IQueryNode node, IQueryVisitorContext context = null) + public static Task> RunAsync(IQueryNode node, IQueryVisitorContext context = null) { return new GetSortFieldsVisitor().AcceptAsync(node, context); } - public static IEnumerable Run(IQueryNode node, IQueryVisitorContext context = null) + public static ICollection Run(IQueryNode node, IQueryVisitorContext context = null) { return RunAsync(node, context).GetAwaiter().GetResult(); } diff --git a/tests/Foundatio.Parsers.ElasticQueries.Tests/AggregationParserTests.cs b/tests/Foundatio.Parsers.ElasticQueries.Tests/AggregationParserTests.cs index b24d8dea..4ea72272 100644 --- a/tests/Foundatio.Parsers.ElasticQueries.Tests/AggregationParserTests.cs +++ b/tests/Foundatio.Parsers.ElasticQueries.Tests/AggregationParserTests.cs @@ -26,11 +26,42 @@ public async Task ProcessSingleAggregationAsync() { string index = await CreateRandomIndexAsync(d => d.Dynamic(DynamicMapping.True).Properties(p => p.GeoPoint(g => g.Field3))); await Client.IndexManyAsync([ - new MyType { Field1 = "value1", Field4 = 1, Field3 = "51.5032520,-0.1278990", Field5 = DateTime.UtcNow.Subtract(TimeSpan.FromMinutes(5)), Field2 = "field2" }, - new MyType { Field1 = "value2", Field4 = 2, Field3 = "51.5032520,-0.1278990", Field5 = DateTime.UtcNow.Subtract(TimeSpan.FromMinutes(4)) }, - new MyType { Field1 = "value3", Field4 = 3, Field3 = "51.5032520,-0.1278990", Field5 = DateTime.UtcNow.Subtract(TimeSpan.FromMinutes(3)) }, - new MyType { Field1 = "value4", Field4 = 4, Field3 = "51.5032520,-0.1278990", Field5 = DateTime.UtcNow.Subtract(TimeSpan.FromMinutes(2)) }, - new MyType { Field1 = "value5", Field4 = 5, Field3 = "51.5032520,-0.1278990", Field5 = DateTime.UtcNow.Subtract(TimeSpan.FromMinutes(1)) } + new MyType + { + Field1 = "value1", + Field4 = 1, + Field3 = "51.5032520,-0.1278990", + Field5 = DateTime.UtcNow.Subtract(TimeSpan.FromMinutes(5)), + Field2 = "field2" + }, + new MyType + { + Field1 = "value2", + Field4 = 2, + Field3 = "51.5032520,-0.1278990", + Field5 = DateTime.UtcNow.Subtract(TimeSpan.FromMinutes(4)) + }, + new MyType + { + Field1 = "value3", + Field4 = 3, + Field3 = "51.5032520,-0.1278990", + Field5 = DateTime.UtcNow.Subtract(TimeSpan.FromMinutes(3)) + }, + new MyType + { + Field1 = "value4", + Field4 = 4, + Field3 = "51.5032520,-0.1278990", + Field5 = DateTime.UtcNow.Subtract(TimeSpan.FromMinutes(2)) + }, + new MyType + { + Field1 = "value5", + Field4 = 5, + Field3 = "51.5032520,-0.1278990", + Field5 = DateTime.UtcNow.Subtract(TimeSpan.FromMinutes(1)) + } ], index); await Client.Indices.RefreshAsync(index); @@ -42,7 +73,7 @@ await Client.IndexManyAsync([ _logger.LogInformation("Actual: {Request}", actualRequest); var expectedResponse = await Client.SearchAsync(d => d.Index(index).Aggregations(a => a - .Add("min_field4", a1 => a1.Min(c => c.Field("field4").Meta(m => m.Add("@field_type", "long")))))); + .Add("min_field4", a1 => a1.Min(c => c.Field("field4")).Meta(m => m.Add("@field_type", "long"))))); string expectedRequest = expectedResponse.GetRequest(); _logger.LogInformation("Expected: {Request}", expectedRequest); @@ -57,11 +88,42 @@ public async Task ProcessSingleAggregationWithAliasAsync() { string index = await CreateRandomIndexAsync(d => d.Dynamic(DynamicMapping.True).Properties(p => p.GeoPoint(g => g.Field3))); await Client.IndexManyAsync([ - new MyType { Field1 = "value1", Field4 = 1, Field3 = "51.5032520,-0.1278990", Field5 = DateTime.UtcNow.Subtract(TimeSpan.FromMinutes(5)), Field2 = "field2" }, - new MyType { Field1 = "value2", Field4 = 2, Field3 = "51.5032520,-0.1278990", Field5 = DateTime.UtcNow.Subtract(TimeSpan.FromMinutes(4)) }, - new MyType { Field1 = "value3", Field4 = 3, Field3 = "51.5032520,-0.1278990", Field5 = DateTime.UtcNow.Subtract(TimeSpan.FromMinutes(3)) }, - new MyType { Field1 = "value4", Field4 = 4, Field3 = "51.5032520,-0.1278990", Field5 = DateTime.UtcNow.Subtract(TimeSpan.FromMinutes(2)) }, - new MyType { Field1 = "value5", Field4 = 5, Field3 = "51.5032520,-0.1278990", Field5 = DateTime.UtcNow.Subtract(TimeSpan.FromMinutes(1)) } + new MyType + { + Field1 = "value1", + Field4 = 1, + Field3 = "51.5032520,-0.1278990", + Field5 = DateTime.UtcNow.Subtract(TimeSpan.FromMinutes(5)), + Field2 = "field2" + }, + new MyType + { + Field1 = "value2", + Field4 = 2, + Field3 = "51.5032520,-0.1278990", + Field5 = DateTime.UtcNow.Subtract(TimeSpan.FromMinutes(4)) + }, + new MyType + { + Field1 = "value3", + Field4 = 3, + Field3 = "51.5032520,-0.1278990", + Field5 = DateTime.UtcNow.Subtract(TimeSpan.FromMinutes(3)) + }, + new MyType + { + Field1 = "value4", + Field4 = 4, + Field3 = "51.5032520,-0.1278990", + Field5 = DateTime.UtcNow.Subtract(TimeSpan.FromMinutes(2)) + }, + new MyType + { + Field1 = "value5", + Field4 = 5, + Field3 = "51.5032520,-0.1278990", + Field5 = DateTime.UtcNow.Subtract(TimeSpan.FromMinutes(1)) + } ], index); await Client.Indices.RefreshAsync(index); @@ -93,11 +155,42 @@ public async Task ProcessAnalyzedAggregationWithAliasAsync() .FieldAlias("heynow", o => o.Path(k => k.Field1)) .GeoPoint(g => g.Field3))); await Client.IndexManyAsync([ - new MyType { Field1 = "value1", Field4 = 1, Field3 = "51.5032520,-0.1278990", Field5 = DateTime.UtcNow.Subtract(TimeSpan.FromMinutes(5)), Field2 = "field2" }, - new MyType { Field1 = "value2", Field4 = 2, Field3 = "51.5032520,-0.1278990", Field5 = DateTime.UtcNow.Subtract(TimeSpan.FromMinutes(4)) }, - new MyType { Field1 = "value3", Field4 = 3, Field3 = "51.5032520,-0.1278990", Field5 = DateTime.UtcNow.Subtract(TimeSpan.FromMinutes(3)) }, - new MyType { Field1 = "value4", Field4 = 4, Field3 = "51.5032520,-0.1278990", Field5 = DateTime.UtcNow.Subtract(TimeSpan.FromMinutes(2)) }, - new MyType { Field1 = "value5", Field4 = 5, Field3 = "51.5032520,-0.1278990", Field5 = DateTime.UtcNow.Subtract(TimeSpan.FromMinutes(1)) } + new MyType + { + Field1 = "value1", + Field4 = 1, + Field3 = "51.5032520,-0.1278990", + Field5 = DateTime.UtcNow.Subtract(TimeSpan.FromMinutes(5)), + Field2 = "field2" + }, + new MyType + { + Field1 = "value2", + Field4 = 2, + Field3 = "51.5032520,-0.1278990", + Field5 = DateTime.UtcNow.Subtract(TimeSpan.FromMinutes(4)) + }, + new MyType + { + Field1 = "value3", + Field4 = 3, + Field3 = "51.5032520,-0.1278990", + Field5 = DateTime.UtcNow.Subtract(TimeSpan.FromMinutes(3)) + }, + new MyType + { + Field1 = "value4", + Field4 = 4, + Field3 = "51.5032520,-0.1278990", + Field5 = DateTime.UtcNow.Subtract(TimeSpan.FromMinutes(2)) + }, + new MyType + { + Field1 = "value5", + Field4 = 5, + Field3 = "51.5032520,-0.1278990", + Field5 = DateTime.UtcNow.Subtract(TimeSpan.FromMinutes(1)) + } ], index); await Client.Indices.RefreshAsync(index); @@ -110,7 +203,7 @@ await Client.IndexManyAsync([ _logger.LogInformation("Actual: {Request}", actualRequest); var expectedResponse = await Client.SearchAsync(d => d.Index(index).Aggregations(a => a - .Add("terms_heynow", a1 => a1.Terms(c => c.Field("field1.keyword").Meta(m => m.Add("@field_type", "text")))))); + .Add("terms_heynow", a1 => a1.Terms(c => c.Field("field1.keyword")).Meta(m => m.Add("@field_type", "text"))))); string expectedRequest = expectedResponse.GetRequest(); _logger.LogInformation("Expected: {Request}", expectedRequest); @@ -125,11 +218,42 @@ public async Task ProcessAggregationsAsync() { string index = await CreateRandomIndexAsync(d => d.Dynamic(DynamicMapping.True).Properties(p => p.GeoPoint(g => g.Field3))); await Client.IndexManyAsync([ - new MyType { Field1 = "value1", Field4 = 1, Field3 = "51.5032520,-0.1278990", Field5 = DateTime.UtcNow.Subtract(TimeSpan.FromMinutes(5)), Field2 = "field2" }, - new MyType { Field1 = "value2", Field4 = 2, Field3 = "51.5032520,-0.1278990", Field5 = DateTime.UtcNow.Subtract(TimeSpan.FromMinutes(4)) }, - new MyType { Field1 = "value3", Field4 = 3, Field3 = "51.5032520,-0.1278990", Field5 = DateTime.UtcNow.Subtract(TimeSpan.FromMinutes(3)) }, - new MyType { Field1 = "value4", Field4 = 4, Field3 = "51.5032520,-0.1278990", Field5 = DateTime.UtcNow.Subtract(TimeSpan.FromMinutes(2)) }, - new MyType { Field1 = "value5", Field4 = 5, Field3 = "51.5032520,-0.1278990", Field5 = DateTime.UtcNow.Subtract(TimeSpan.FromMinutes(1)) } + new MyType + { + Field1 = "value1", + Field4 = 1, + Field3 = "51.5032520,-0.1278990", + Field5 = DateTime.UtcNow.Subtract(TimeSpan.FromMinutes(5)), + Field2 = "field2" + }, + new MyType + { + Field1 = "value2", + Field4 = 2, + Field3 = "51.5032520,-0.1278990", + Field5 = DateTime.UtcNow.Subtract(TimeSpan.FromMinutes(4)) + }, + new MyType + { + Field1 = "value3", + Field4 = 3, + Field3 = "51.5032520,-0.1278990", + Field5 = DateTime.UtcNow.Subtract(TimeSpan.FromMinutes(3)) + }, + new MyType + { + Field1 = "value4", + Field4 = 4, + Field3 = "51.5032520,-0.1278990", + Field5 = DateTime.UtcNow.Subtract(TimeSpan.FromMinutes(2)) + }, + new MyType + { + Field1 = "value5", + Field4 = 5, + Field3 = "51.5032520,-0.1278990", + Field5 = DateTime.UtcNow.Subtract(TimeSpan.FromMinutes(1)) + } ], index); await Client.Indices.RefreshAsync(index); @@ -146,16 +270,16 @@ await Client.IndexManyAsync([ .Add("avg_lat", a3 => a3.Avg(s => s.Script(ss => ss.Source("doc['field3'].lat")))) .Add("avg_lon", a3 => a3.Avg(s => s.Script(ss => ss.Source("doc['field3'].lon")))) )) - .Add("terms_field1", a1 => a1.Terms(t => t.Field("field1.keyword").Meta(m => m.Add("@field_type", "text")))) + .Add("terms_field1", a1 => a1.Terms(t => t.Field("field1.keyword")).Meta(m => m.Add("@field_type", "text"))) .Add("histogram_field4", a1 => a1.Histogram(h => h.Field("field4").Interval(50).MinDocCount(0))) .Add("date_field5", a1 => a1.DateHistogram(d1 => d1.Field("field5").CalendarInterval(CalendarInterval.Day).Format("date_optional_time").MinDocCount(0))) .Add("missing_field2", a1 => a1.Missing(t => t.Field("field2.keyword"))) .Add("cardinality_field4", a1 => a1.Cardinality(c => c.Field("field4"))) .Add("percentiles_field4", a1 => a1.Percentiles(c => c.Field("field4").Percents([50, 100]))) - .Add("sum_field4", a1 => a1.Sum(c => c.Field("field4").Meta(m => m.Add("@field_type", "long")))) - .Add("avg_field4", a1 => a1.Avg(c => c.Field("field4").Meta(m => m.Add("@field_type", "long")))) - .Add("max_field4", a1 => a1.Max(c => c.Field("field4").Meta(m => m.Add("@field_type", "long")))) - .Add("min_field4", a1 => a1.Min(c => c.Field("field4").Meta(m => m.Add("@field_type", "long")))))); + .Add("sum_field4", a1 => a1.Sum(c => c.Field("field4")).Meta(m => m.Add("@field_type", "long"))) + .Add("avg_field4", a1 => a1.Avg(c => c.Field("field4")).Meta(m => m.Add("@field_type", "long"))) + .Add("max_field4", a1 => a1.Max(c => c.Field("field4")).Meta(m => m.Add("@field_type", "long"))) + .Add("min_field4", a1 => a1.Min(c => c.Field("field4")).Meta(m => m.Add("@field_type", "long"))))); string expectedRequest = expectedResponse.GetRequest(); _logger.LogInformation("Expected: {Request}", expectedRequest); @@ -187,7 +311,7 @@ public async Task ProcessNestedAggregationsWithAliasesAsync() _logger.LogInformation("Actual: {Request}", actualRequest); var expectedResponse = await Client.SearchAsync(d => d.Index(index).Aggregations(a => a - .Add("terms_alias1", a1 => a1.Terms(t => t.Field("field1.keyword").Meta(m => m.Add("@field_type", "keyword"))) + .Add("terms_alias1", a1 => a1.Terms(t => t.Field("field1.keyword")).Meta(m => m.Add("@field_type", "keyword")) .Aggregations(a2 => a2.Add("cardinality_user", a3 => a3.Cardinality(c => c.Field("data.@user.identity.keyword"))))))); string expectedRequest = expectedResponse.GetRequest(); _logger.LogInformation("Expected: {Request}", expectedRequest); @@ -256,16 +380,16 @@ await Client.IndexManyAsync([ .Add("avg_lat", a3 => a3.Avg(s => s.Script(ss => ss.Source("doc['field3'].lat")))) .Add("avg_lon", a3 => a3.Avg(s => s.Script(ss => ss.Source("doc['field3'].lon")))) )) - .Add("terms_alias1", a1 => a1.Terms(t => t.Field("field1.keyword").Meta(m => m.Add("@field_type", "text")))) + .Add("terms_alias1", a1 => a1.Terms(t => t.Field("field1.keyword")).Meta(m => m.Add("@field_type", "text"))) .Add("histogram_alias4", a1 => a1.Histogram(h => h.Field("field4").Interval(50).MinDocCount(0))) .Add("date_alias5", a1 => a1.DateHistogram(d1 => d1.Field("field5").CalendarInterval(CalendarInterval.Day).Format("date_optional_time").MinDocCount(0))) .Add("missing_alias2", a1 => a1.Missing(t => t.Field("field2.keyword"))) .Add("cardinality_user", a1 => a1.Cardinality(c => c.Field("data.@user.identity.keyword"))) .Add("percentiles_alias4", a1 => a1.Percentiles(c => c.Field("field4"))) - .Add("sum_alias4", a1 => a1.Sum(c => c.Field("field4").Meta(m => m.Add("@field_type", "long")))) - .Add("avg_alias4", a1 => a1.Avg(c => c.Field("field4").Meta(m => m.Add("@field_type", "long")))) - .Add("max_alias4", a1 => a1.Max(c => c.Field("field4").Meta(m => m.Add("@field_type", "long")))) - .Add("min_alias4", a1 => a1.Min(c => c.Field("field4").Meta(m => m.Add("@field_type", "long")))))); + .Add("sum_alias4", a1 => a1.Sum(c => c.Field("field4")).Meta(m => m.Add("@field_type", "long"))) + .Add("avg_alias4", a1 => a1.Avg(c => c.Field("field4")).Meta(m => m.Add("@field_type", "long"))) + .Add("max_alias4", a1 => a1.Max(c => c.Field("field4")).Meta(m => m.Add("@field_type", "long"))) + .Add("min_alias4", a1 => a1.Min(c => c.Field("field4")).Meta(m => m.Add("@field_type", "long"))))); string expectedRequest = expectedResponse.GetRequest(); _logger.LogInformation("Expected: {Request}", expectedRequest); @@ -405,15 +529,15 @@ public async Task ProcessDateHistogramAggregations() var expectedResponse = await Client.SearchAsync(d => d.Index(index).Aggregations(a => a .Add("date_field5", a1 => a1.DateHistogram(d1 => d1 .Field("field5") - .Meta(m => m.Add("@timezone", "1h")) .CalendarInterval(CalendarInterval.Day) .Format("date_optional_time") .MinDocCount(0) .TimeZone("+01:00") .Missing(DateTime.MinValue)) + .Meta(m => m.Add("@timezone", "1h")) .Aggregations(a2 => a2 - .Add("min_field5", a3 => a3.Min(c => c.Field("field5").Meta(m => m.Add("@field_type", "date").Add("@timezone", "1h")))) - .Add("max_field5", a3 => a3.Max(c => c.Field("field5").Meta(m => m.Add("@field_type", "date").Add("@timezone", "1h")))))))); + .Add("min_field5", a3 => a3.Min(c => c.Field("field5")).Meta(m => m.Add("@field_type", "date").Add("@timezone", "1h"))) + .Add("max_field5", a3 => a3.Max(c => c.Field("field5")).Meta(m => m.Add("@field_type", "date").Add("@timezone", "1h"))))))); string expectedRequest = expectedResponse.GetRequest(); _logger.LogInformation("Expected: {Request}", expectedRequest); @@ -439,11 +563,11 @@ public async Task CanSpecifyDefaultValuesAggregations() _logger.LogInformation("Actual: {Request}", actualRequest); var expectedResponse = await Client.SearchAsync(d => d.Index(index).Aggregations(a => a - .Add("sum_field4", a1 => a1.Sum(c => c.Field("field4").Missing(0).Meta(m => m.Add("@field_type", "integer")))) + .Add("sum_field4", a1 => a1.Sum(c => c.Field("field4").Missing(0)).Meta(m => m.Add("@field_type", "integer"))) .Add("cardinality_field4", a1 => a1.Cardinality(c => c.Field("field4").Missing(0))) - .Add("avg_field4", a1 => a1.Avg(c => c.Field("field4").Missing(0).Meta(m => m.Add("@field_type", "integer")))) - .Add("max_field4", a1 => a1.Max(c => c.Field("field4").Missing(0).Meta(m => m.Add("@field_type", "integer")))) - .Add("min_field4", a1 => a1.Min(c => c.Field("field4").Missing(0).Meta(m => m.Add("@field_type", "integer")))))); + .Add("avg_field4", a1 => a1.Avg(c => c.Field("field4").Missing(0)).Meta(m => m.Add("@field_type", "integer"))) + .Add("max_field4", a1 => a1.Max(c => c.Field("field4").Missing(0)).Meta(m => m.Add("@field_type", "integer"))) + .Add("min_field4", a1 => a1.Min(c => c.Field("field4").Missing(0)).Meta(m => m.Add("@field_type", "integer"))))); string expectedRequest = expectedResponse.GetRequest(); _logger.LogInformation("Expected: {Request}", expectedRequest); diff --git a/tests/Foundatio.Parsers.ElasticQueries.Tests/ElasticMappingResolverTests.cs b/tests/Foundatio.Parsers.ElasticQueries.Tests/ElasticMappingResolverTests.cs index 047a1a5d..45029ba5 100644 --- a/tests/Foundatio.Parsers.ElasticQueries.Tests/ElasticMappingResolverTests.cs +++ b/tests/Foundatio.Parsers.ElasticQueries.Tests/ElasticMappingResolverTests.cs @@ -20,9 +20,8 @@ public ElasticMappingResolverTests(ITestOutputHelper output, ElasticsearchFixtur private TypeMappingDescriptor MapMyNestedType(TypeMappingDescriptor m) { return m - .AutoMap() .Dynamic(DynamicMapping.True) - .DynamicTemplates(t => t.DynamicTemplate("idx_text", t => t.Match("text*").Mapping(m => m.Text(mp => mp.AddKeywordAndSortFields())))) + .DynamicTemplates(t => t.DynamicTemplate("idx_text", t1 => t1.Match("text*").Mapping(m1 => m1.Text(mp => mp.AddKeywordAndSortFields())))) .Properties(p => p .Text(p1 => p1.Field1, o => o.AddKeywordAndSortFields()) .Text(p1 => p1.Field4, o => o.AddKeywordAndSortFields()) diff --git a/tests/Foundatio.Parsers.ElasticQueries.Tests/ElasticQueryParserTests.cs b/tests/Foundatio.Parsers.ElasticQueries.Tests/ElasticQueryParserTests.cs index 3819c04e..94e9dcb9 100644 --- a/tests/Foundatio.Parsers.ElasticQueries.Tests/ElasticQueryParserTests.cs +++ b/tests/Foundatio.Parsers.ElasticQueries.Tests/ElasticQueryParserTests.cs @@ -90,7 +90,7 @@ await Client.IndexManyAsync([ var processor = new ElasticQueryParser(c => c.SetLoggerFactory(Log)); var result = await processor.BuildQueryAsync("field1:value1"); - var actualResponse = await Client.SearchAsync(d => d.Index(index).Query(_ => result)); + var actualResponse = await Client.SearchAsync(d => d.Index(index).Query(result)); string actualRequest = actualResponse.GetRequest(); _logger.LogInformation("Actual: {Request}", actualRequest); @@ -119,7 +119,7 @@ await Client.IndexManyAsync([ var processor = new ElasticQueryParser(c => c.UseIncludes(includes).SetLoggerFactory(Log)); var result = await processor.BuildQueryAsync("field1:value1 @include:stuff", new ElasticQueryVisitorContext { UseScoring = true }); - var actualResponse = await Client.SearchAsync(d => d.Index(index).Query(_ => result)); + var actualResponse = await Client.SearchAsync(d => d.Index(index).Query(result)); string actualRequest = actualResponse.GetRequest(); _logger.LogInformation("Actual: {Request}", actualRequest); @@ -144,7 +144,7 @@ public async Task ShouldGenerateORedTermsQuery() var processor = new ElasticQueryParser(c => c.SetLoggerFactory(Log)); var result = await processor.BuildQueryAsync("field1:value1 field2:value2 field3:value3", new ElasticQueryVisitorContext().UseSearchMode()); - var actualResponse = await Client.SearchAsync(d => d.Index(index).Query(_ => result)); + var actualResponse = await Client.SearchAsync(d => d.Index(index).Query(result)); string actualRequest = actualResponse.GetRequest(); _logger.LogInformation("Actual: {Request}", actualRequest); @@ -173,7 +173,7 @@ public async Task ShouldHandleMultipleTermsForAnalyzedFields() var result = await processor.BuildQueryAsync("field1:(value1 abc def ghi) field2:(value2 jhk)", new ElasticQueryVisitorContext().UseSearchMode()); - var actualResponse = await Client.SearchAsync(d => d.Index(index).Query(_ => result)); + var actualResponse = await Client.SearchAsync(d => d.Index(index).Query(result)); string actualRequest = actualResponse.GetRequest(); _logger.LogInformation("Actual: {Request}", actualRequest); @@ -182,7 +182,7 @@ public async Task ShouldHandleMultipleTermsForAnalyzedFields() || f.Match(m => m.Field(mf => mf.Field1).Query("abc")) || f.Match(m => m.Field(mf => mf.Field1).Query("def")) || f.Match(m => m.Field(mf => mf.Field1).Query("ghi")) - || f.Term(m => m.Field(tf => tf.Field2).Value("value2")) || f.Term(m => m.Field2, "jhk"))); + || f.Term(m => m.Field(tf => tf.Field2).Value("value2")) || f.Term(m => m.Field(mf => mf.Field2), "jhk"))); string expectedRequest = expectedResponse.GetRequest(); _logger.LogInformation("Expected: {Request}", expectedRequest); @@ -192,7 +192,7 @@ public async Task ShouldHandleMultipleTermsForAnalyzedFields() processor = new ElasticQueryParser(c => c.SetLoggerFactory(Log).SetDefaultFields(["field1"]).UseMappings(Client, index)); result = await processor.BuildQueryAsync("value1 abc def ghi", new ElasticQueryVisitorContext().UseSearchMode()); - actualResponse = await Client.SearchAsync(d => d.Index(index).Query(_ => result)); + actualResponse = await Client.SearchAsync(d => d.Index(index).Query(result)); actualRequest = actualResponse.GetRequest(); _logger.LogInformation("Actual: {Request}", actualRequest); @@ -211,7 +211,7 @@ public async Task ShouldHandleMultipleTermsForAnalyzedFields() // multi-match on multiple default fields processor = new ElasticQueryParser(c => c.SetLoggerFactory(Log).SetDefaultFields(["field1", "field2"]).UseMappings(Client, index)); result = await processor.BuildQueryAsync("value1 abc def ghi", new ElasticQueryVisitorContext().UseSearchMode()); - actualResponse = await Client.SearchAsync(d => d.Index(index).Query(_ => result)); + actualResponse = await Client.SearchAsync(d => d.Index(index).Query(result)); actualRequest = actualResponse.GetRequest(); _logger.LogInformation("Actual: {Request}", actualRequest); @@ -242,9 +242,8 @@ TypeMappingDescriptor GetCodeMappings(TypeMappingDescriptor d) = .GeoPoint(g => g.Field3) .Keyword(e => e.Field2))); - var res = await Client.IndexAsync(new MyType { Field1 = "value1", Field2 = "value2", Field4 = 1, Field5 = DateTime.Now }, i => i.Index(index)); + await Client.IndexAsync(new MyType { Field1 = "value1", Field2 = "value2", Field4 = 1, Field5 = DateTime.Now }, i => i.Index(index)); await Client.Indices.RefreshAsync(index); - var parser = new ElasticQueryParser(c => c.SetDefaultFields(["field1"]).UseMappings(GetCodeMappings, Client, index)); var dynamicServerMappingProperty = parser.Configuration.MappingResolver.GetMapping("field5").Property; @@ -271,7 +270,7 @@ await Client.IndexManyAsync([ var processor = new ElasticQueryParser(c => c.SetLoggerFactory(Log).UseMappings(Client, index)); var result = await processor.BuildQueryAsync("\"\\\"now there\""); - var actualResponse = await Client.SearchAsync(d => d.Index(index).Query(_ => result)); + var actualResponse = await Client.SearchAsync(d => d.Index(index).Query(result)); string actualRequest = actualResponse.GetRequest(true); _logger.LogInformation("Actual: {Request}", actualRequest); @@ -299,7 +298,7 @@ await Client.IndexManyAsync([ var processor = new ElasticQueryParser(c => c.SetLoggerFactory(Log).UseMappings(Client, index)); var result = await processor.BuildQueryAsync(@"field1:one\\/two*"); - var actualResponse = await Client.SearchAsync(d => d.Index(index).Query(_ => result)); + var actualResponse = await Client.SearchAsync(d => d.Index(index).Query(result)); string actualRequest = actualResponse.GetRequest(true); _logger.LogInformation("Actual: {Request}", actualRequest); @@ -307,7 +306,7 @@ await Client.IndexManyAsync([ .Query(q => q .Bool(b => b .Filter(f => f - .QueryString(m => m.Query("one\\/two*").Fields(f => f.Field(f2 => f2.Field1)).AllowLeadingWildcard(false).AnalyzeWildcard()))))); + .QueryString(m => m.Query("one\\/two*").Fields(f1 => f1.Field(f2 => f2.Field1)).AllowLeadingWildcard(false).AnalyzeWildcard()))))); string expectedRequest = expectedResponse.GetRequest(true); _logger.LogInformation("Expected: {Request}", expectedRequest); @@ -327,7 +326,7 @@ await Client.IndexManyAsync([ var processor = new ElasticQueryParser(c => c.SetLoggerFactory(Log).UseMappings(Client, index)); var result = await processor.BuildQueryAsync(@"field1:one\\/two"); - var actualResponse = await Client.SearchAsync(d => d.Index(index).Query(_ => result)); + var actualResponse = await Client.SearchAsync(d => d.Index(index).Query(result)); string actualRequest = actualResponse.GetRequest(true); _logger.LogInformation("Actual: {Request}", actualRequest); @@ -357,7 +356,7 @@ await Client.IndexManyAsync([ var processor = new ElasticQueryParser(c => c.SetLoggerFactory(Log)); var result = await processor.BuildQueryAsync($"_exists_:{nameof(MyType.Field2)}"); - var actualResponse = await Client.SearchAsync(d => d.Index(index).Query(_ => result)); + var actualResponse = await Client.SearchAsync(d => d.Index(index).Query(result)); string actualRequest = actualResponse.GetRequest(); _logger.LogInformation("Actual: {Request}", actualRequest); @@ -385,7 +384,7 @@ await Client.IndexManyAsync([ var processor = new ElasticQueryParser(c => c.SetLoggerFactory(Log)); var result = await processor.BuildQueryAsync($"_missing_:{nameof(MyType.Field2)}", new ElasticQueryVisitorContext { UseScoring = true }); - var actualResponse = await Client.SearchAsync(d => d.Index(index).Query(_ => result)); + var actualResponse = await Client.SearchAsync(d => d.Index(index).Query(result)); string actualRequest = actualResponse.GetRequest(); _logger.LogInformation("Actual: {Request}", actualRequest); @@ -417,22 +416,22 @@ await Client.IndexManyAsync([ _logger.LogInformation("Actual: {Request}", actualRequest); var expectedResponse = await Client.SearchAsync(i => i.Index(index).Aggregations(f => f - .Max("max_field2", m => m.Field("field2.keyword").Meta(m2 => m2.Add("@field_type", "text"))) - .DateHistogram("date_field5", + .Add("min_field2", m => m.Max(d => d.Field(df => df.Field2)).Meta(m2 => m2.Add("@field_type", "text"))) + .Add("max_field2", m => m.Max(d => d.Field(df => df.Field2)).Meta(m2 => m2.Add("@field_type", "text"))) + .Add("date_field5", m => m.DateHistogram( d => d.Field(d2 => d2.Field5) .CalendarInterval(CalendarInterval.Day) .Format("date_optional_time") - .MinimumDocumentCount(0) + .MinDocCount(0) .TimeZone("America/Chicago") - .Offset("-6h") + .Offset("-6h")) .Meta(m2 => m2.Add("@timezone", "America/Chicago")) .Aggregations(l => l - .Min("min_field1", m => m.Field("field1.keyword").Meta(m2 => m2.Add("@field_type", "text"))) - .Max("max_field2", m => m.Field("field2.keyword").Meta(m2 => m2.Add("@field_type", "text"))) - .Min("min_field2", m => m.Field("field2.keyword").Meta(m2 => m2.Add("@field_type", "text"))) + .Add("min_field1", m3 => m3.Min(d => d.Field(df => df.Field1)).Meta(m2 => m2.Add("@field_type", "text"))) + .Add("max_field2", m3 => m3.Max(d => d.Field(df => df.Field2)).Meta(m2 => m2.Add("@field_type", "text"))) + .Add("min_field2", m3 => m3.Min(d => d.Field(df => df.Field2)).Meta(m2 => m2.Add("@field_type", "text"))) )) - .Min("min_field2", m => m.Field("field2.keyword").Meta(m2 => m2.Add("@field_type", "text"))) )); string expectedRequest = expectedResponse.GetRequest(); _logger.LogInformation("Expected: {Request}", expectedRequest); @@ -485,7 +484,7 @@ public async Task CanUseDateHistogramAggregationInterval(string interval, object _logger.LogInformation("Actual: {Request}", actualRequest); var expectedResponse = await Client.SearchAsync(i => i.Index(index).Aggregations(f => f - .DateHistogram("date_field5", d => + .Add("date_field5", m => m.DateHistogram(d => { d.Field(d2 => d2.Field5); @@ -497,10 +496,8 @@ public async Task CanUseDateHistogramAggregationInterval(string interval, object d.FixedInterval(interval); d.Format("date_optional_time"); - d.MinimumDocumentCount(0); - - return d; - }) + d.MinDocCount(0); + })) )); string expectedRequest = expectedResponse.GetRequest(); _logger.LogInformation("Expected: {Request}", expectedRequest); @@ -517,7 +514,7 @@ public async Task CanDoNestDateHistogram() await Client.Indices.RefreshAsync(index); var response = await Client.SearchAsync(i => i.Index(index).Aggregations(f => f - .DateHistogram("myagg", d => d.Field(d2 => d2.Field5).CalendarInterval(CalendarInterval.Day)) + .Add("myagg", m => m.DateHistogram(d => d.Field(d2 => d2.Field5).CalendarInterval(CalendarInterval.Day))) )); Assert.True(response.IsValidResponse); @@ -541,7 +538,7 @@ await Client.IndexManyAsync([ _logger.LogInformation("Actual: {Request}", actualRequest); var expectedResponse = await Client.SearchAsync(i => i.Index(index).Aggregations(f => f - .DateHistogram("date_field5", d => d.Field(d2 => d2.Field5).CalendarInterval(CalendarInterval.Day).Format("date_optional_time").MinimumDocumentCount(0)) + .Add("date_field5", m => m.DateHistogram(d => d.Field(d2 => d2.Field5).CalendarInterval(CalendarInterval.Day).Format("date_optional_time").MinDocCount(0))) )); string expectedRequest = expectedResponse.GetRequest(); _logger.LogInformation("Expected: {Request}", expectedRequest); @@ -555,7 +552,7 @@ public async Task SimpleQueryProcessor() { string index = await CreateRandomIndexAsync(t => t .Properties(p => p - .Text(e => e.Name(n => n.Field3).Fields(f => f.Keyword(k => k.Name("keyword").IgnoreAbove(256)))))); + .Text(e => e.Field3, d => d.Fields(f => f.Keyword(k => k.Field3, o => o.IgnoreAbove(256)))))); await Client.IndexManyAsync([ new MyType { Field1 = "value1", Field2 = "value2" }, @@ -566,7 +563,7 @@ await Client.IndexManyAsync([ var processor = new ElasticQueryParser(c => c.SetLoggerFactory(Log).UseMappings(Client, index)); var result = await processor.BuildQueryAsync("field1:value1", new ElasticQueryVisitorContext().UseSearchMode()); - var actualResponse = await Client.SearchAsync(d => d.Index(index).Query(_ => result)); + var actualResponse = await Client.SearchAsync(d => d.Index(index).Query(result)); string actualRequest = actualResponse.GetRequest(); _logger.LogInformation("Actual: {Request}", actualRequest); @@ -578,7 +575,7 @@ await Client.IndexManyAsync([ Assert.Equal(expectedResponse.Total, actualResponse.Total); result = await processor.BuildQueryAsync("field3:hey", new ElasticQueryVisitorContext().UseSearchMode()); - actualResponse = await Client.SearchAsync(d => d.Index(index).Query(_ => result)); + actualResponse = await Client.SearchAsync(d => d.Index(index).Query(result)); actualRequest = actualResponse.GetRequest(); _logger.LogInformation("Actual: {Request}", actualRequest); expectedResponse = await Client.SearchAsync(d => d.Index(index).Query(q => q.Match(m => m @@ -592,7 +589,7 @@ await Client.IndexManyAsync([ Assert.Equal(expectedResponse.Total, actualResponse.Total); result = await processor.BuildQueryAsync("field3:\"hey now\"", new ElasticQueryVisitorContext().UseSearchMode()); - actualResponse = await Client.SearchAsync(d => d.Index(index).Query(_ => result)); + actualResponse = await Client.SearchAsync(d => d.Index(index).Query(result)); actualRequest = actualResponse.GetRequest(); _logger.LogInformation("Actual: {Request}", actualRequest); expectedResponse = await Client.SearchAsync(d => d.Index(index).Query(q => q.MatchPhrase(m => m @@ -606,7 +603,7 @@ await Client.IndexManyAsync([ Assert.Equal(expectedResponse.Total, actualResponse.Total); result = await processor.BuildQueryAsync("field3:hey*", new ElasticQueryVisitorContext().UseSearchMode()); - actualResponse = await Client.SearchAsync(d => d.Index(index).Query(_ => result)); + actualResponse = await Client.SearchAsync(d => d.Index(index).Query(result)); actualRequest = actualResponse.GetRequest(); _logger.LogInformation("Actual: {Request}", actualRequest); expectedResponse = await Client.SearchAsync(d => d.Index(index).Query(q => q.QueryString(m => m @@ -636,7 +633,7 @@ await Client.IndexManyAsync([ var processor = new ElasticQueryParser(c => c.SetLoggerFactory(Log)); var result = await processor.BuildQueryAsync("field1:value1 AND -field2:value2", new ElasticQueryVisitorContext().UseSearchMode()); - var actualResponse = await Client.SearchAsync(d => d.Index(index).Query(_ => result)); + var actualResponse = await Client.SearchAsync(d => d.Index(index).Query(result)); string actualRequest = actualResponse.GetRequest(); _logger.LogInformation("Actual: {Request}", actualRequest); @@ -651,7 +648,7 @@ await Client.IndexManyAsync([ processor = new ElasticQueryParser(c => c.SetLoggerFactory(Log)); result = await processor.BuildQueryAsync("field1:value1 AND NOT field2:value2", new ElasticQueryVisitorContext().UseSearchMode()); - actualResponse = await Client.SearchAsync(d => d.Index(index).Query(_ => result)); + actualResponse = await Client.SearchAsync(d => d.Index(index).Query(result)); actualRequest = actualResponse.GetRequest(); _logger.LogInformation("Actual: {Request}", actualRequest); @@ -666,7 +663,7 @@ await Client.IndexManyAsync([ processor = new ElasticQueryParser(c => c.SetLoggerFactory(Log)); result = await processor.BuildQueryAsync("field1:value1 OR NOT field2:value2", new ElasticQueryVisitorContext().UseSearchMode()); - actualResponse = await Client.SearchAsync(d => d.Index(index).Query(_ => result)); + actualResponse = await Client.SearchAsync(d => d.Index(index).Query(result)); actualRequest = actualResponse.GetRequest(); _logger.LogInformation("Actual: {Request}", actualRequest); @@ -681,7 +678,7 @@ await Client.IndexManyAsync([ processor = new ElasticQueryParser(c => c.SetLoggerFactory(Log)); result = await processor.BuildQueryAsync("field1:value1 OR -field2:value2", new ElasticQueryVisitorContext().UseSearchMode()); - actualResponse = await Client.SearchAsync(d => d.Index(index).Query(_ => result)); + actualResponse = await Client.SearchAsync(d => d.Index(index).Query(result)); actualRequest = actualResponse.GetRequest(); _logger.LogInformation("Actual: {Request}", actualRequest); @@ -709,7 +706,7 @@ await Client.IndexManyAsync([ var result = await processor.BuildQueryAsync("field1:value1 (field2:value2 OR field3:value3)", new ElasticQueryVisitorContext().UseSearchMode()); - var actualResponse = await Client.SearchAsync(d => d.Index(index).Query(_ => result)); + var actualResponse = await Client.SearchAsync(d => d.Index(index).Query(result)); string actualRequest = actualResponse.GetRequest(); _logger.LogInformation("Actual: {Request}", actualRequest); @@ -737,7 +734,7 @@ await Client.IndexManyAsync([ var processor = new ElasticQueryParser(c => c.SetLoggerFactory(Log)); var result = await processor.BuildQueryAsync("field1:value1 (field2:value2 OR field3:value3)"); - var actualResponse = await Client.SearchAsync(d => d.Index(index).Query(_ => result)); + var actualResponse = await Client.SearchAsync(d => d.Index(index).Query(result)); string actualRequest = actualResponse.GetRequest(); _logger.LogInformation("Actual: {Request}", actualRequest); @@ -761,11 +758,11 @@ public async Task MixedCaseTermFilterQueryProcessor() var processor = new ElasticQueryParser(c => c.SetLoggerFactory(Log)); var result = await processor.BuildQueryAsync("field1:Testing.Casing", new ElasticQueryVisitorContext { UseScoring = true }); - var actualResponse = await Client.SearchAsync(d => d.Index(index).Query(_ => result)); + var actualResponse = await Client.SearchAsync(d => d.Index(index).Query(result)); string actualRequest = actualResponse.GetRequest(); _logger.LogInformation("Actual: {Request}", actualRequest); - var expectedResponse = await Client.SearchAsync(d => d.Index(index).Query(f => f.Term(m => m.Field1, "Testing.Casing"))); + var expectedResponse = await Client.SearchAsync(d => d.Index(index).Query(f => f.Term(m => m.Field(mf => mf.Field1).Value("Testing.Casing")))); string expectedRequest = expectedResponse.GetRequest(); _logger.LogInformation("Expected: {Request}", expectedRequest); @@ -781,11 +778,11 @@ public async Task MultipleWordsTermFilterQueryProcessor() var processor = new ElasticQueryParser(c => c.SetLoggerFactory(Log)); var result = await processor.BuildQueryAsync("field1:\"Blake Niemyjski\"", new ElasticQueryVisitorContext { UseScoring = true }); - var actualResponse = await Client.SearchAsync(d => d.Index(index).Query(_ => result)); + var actualResponse = await Client.SearchAsync(d => d.Index(index).Query(result)); string actualRequest = actualResponse.GetRequest(); _logger.LogInformation("Actual: {Request}", actualRequest); - var expectedResponse = await Client.SearchAsync(d => d.Index(index).Query(f => f.Term(p => p.Field1, "Blake Niemyjski"))); + var expectedResponse = await Client.SearchAsync(d => d.Index(index).Query(f => f.Term(m => m.Field(mf => mf.Field1).Value("Blake Niemyjski")))); string expectedRequest = expectedResponse.GetRequest(); _logger.LogInformation("Expected: {Request}", expectedRequest); @@ -801,7 +798,7 @@ public async Task CanTranslateTermQueryProcessor() var processor = new ElasticQueryParser(c => c.SetLoggerFactory(Log).AddVisitor(new UpdateFixedTermFieldToDateFixedExistsQueryVisitor())); var result = await processor.BuildQueryAsync("fixed:true"); - var actualResponse = await Client.SearchAsync(d => d.Index(index).Query(_ => result)); + var actualResponse = await Client.SearchAsync(d => d.Index(index).Query(result)); string actualRequest = actualResponse.GetRequest(); _logger.LogInformation("Actual: {Request}", actualRequest); @@ -829,7 +826,7 @@ await Client.IndexManyAsync([ var result = await processor.BuildQueryAsync("field1:value1 (field2:value2 OR field3:value3)", new ElasticQueryVisitorContext().SetDefaultOperator(Operator.And).UseScoring()); - var actualResponse = await Client.SearchAsync(d => d.Index(index).Query(_ => result)); + var actualResponse = await Client.SearchAsync(d => d.Index(index).Query(result)); string actualRequest = actualResponse.GetRequest(); _logger.LogInformation("Actual: {Request}", actualRequest); @@ -873,7 +870,7 @@ await Client.IndexManyAsync([ var processor = new ElasticQueryParser(c => c.SetLoggerFactory(Log).UseFieldMap(new FieldMap { { "blah", "nested" } }).UseMappings(Client).UseNested()); var result = await processor.BuildQueryAsync("field1:value1 blah:(blah.field1:value1)", new ElasticQueryVisitorContext().UseScoring()); - var actualResponse = await Client.SearchAsync(d => d.Query(_ => result)); + var actualResponse = await Client.SearchAsync(d => d.Query(result)); string actualRequest = actualResponse.GetRequest(); _logger.LogInformation("Actual: {Request}", actualRequest); @@ -894,7 +891,7 @@ await Client.IndexManyAsync([ result = await processor.BuildQueryAsync("field1:value1 blah:(blah.field1:value1 blah.field4:4)", new ElasticQueryVisitorContext().UseScoring()); - actualResponse = await Client.SearchAsync(d => d.Query(_ => result)); + actualResponse = await Client.SearchAsync(d => d.Query(result)); actualRequest = actualResponse.GetRequest(); _logger.LogInformation("Actual: {Request}", actualRequest); @@ -906,7 +903,7 @@ await Client.IndexManyAsync([ .Match(m => m .Field("nested.field1") .Query("value1")) - && q2.Term("nested.field4", "4"))))); + && q2.Term(mt => mt.Field("nested.field4").Value("4")))))); expectedRequest = expectedResponse.GetRequest(); _logger.LogInformation("Expected: {Request}", expectedRequest); @@ -947,14 +944,14 @@ await Client.IndexManyAsync([ var result = await processor.BuildQueryAsync("field1:value1 nested:(nested.field1:value1 nested.field4:4 nested.field3:value3)", new ElasticQueryVisitorContext { UseScoring = true }); - var actualResponse = await Client.SearchAsync(d => d.Query(_ => result)); + var actualResponse = await Client.SearchAsync(d => d.Query(result)); string actualRequest = actualResponse.GetRequest(); _logger.LogInformation("Actual: {Request}", actualRequest); var expectedResponse = await Client.SearchAsync(d => d.Query(q => q.Match(m => m.Field(e => e.Field1).Query("value1")) && q.Nested(n => n.Path(p => p.Nested).Query(q2 => q2.Match(m => m.Field("nested.field1").Query("value1")) - && q2.Term("nested.field4", "4") + && q2.Term(m => m.Field("nested.field4").Value("4")) && q2.Match(m => m.Field("nested.field3").Query("value3")))))); string expectedRequest = expectedResponse.GetRequest(); @@ -975,7 +972,7 @@ public async Task CanGenerateMatchQuery() var processor = new ElasticQueryParser(c => c.SetLoggerFactory(Log).UseMappings(Client, index)); var result = await processor.BuildQueryAsync("field1:test", new ElasticQueryVisitorContext().UseScoring()); - var actualResponse = await Client.SearchAsync(d => d.Index(index).Query(_ => result)); + var actualResponse = await Client.SearchAsync(d => d.Index(index).Query(result)); string actualRequest = actualResponse.GetRequest(); _logger.LogInformation("Actual: {Request}", actualRequest); @@ -998,7 +995,7 @@ public async Task CanBuildAliasQueryProcessor() var processor = new ElasticQueryParser(c => c.SetLoggerFactory(Log)); var result = await processor.BuildQueryAsync("browser.version:1", new ElasticQueryVisitorContext().UseScoring()); - var actualResponse = await Client.SearchAsync(d => d.Index(index).Query(_ => result)); + var actualResponse = await Client.SearchAsync(d => d.Index(index).Query(result)); string actualRequest = actualResponse.GetRequest(); _logger.LogInformation("Actual: {Request}", actualRequest); @@ -1020,7 +1017,7 @@ public async Task NonAnalyzedPrefixQuery() var processor = new ElasticQueryParser(c => c.UseMappings(Client, index)); var result = await processor.BuildQueryAsync("field1:value*", new ElasticQueryVisitorContext().UseSearchMode()); - var actualResponse = await Client.SearchAsync(d => d.Index(index).Query(_ => result)); + var actualResponse = await Client.SearchAsync(d => d.Index(index).Query(result)); string actualRequest = actualResponse.GetRequest(); _logger.LogInformation("Actual: {Request}", actualResponse); @@ -1039,7 +1036,7 @@ public async Task NonAnalyzedPrefixQuery() public async Task RangeQueryProcessor() { string index = await CreateRandomIndexAsync(); - var res = await Client.IndexAsync(new MyType { Field1 = "value1", Field4 = 1 }, i => i.Index(index)); + await Client.IndexAsync(new MyType { Field1 = "value1", Field4 = 1 }, i => i.Index(index)); await Client.IndexAsync(new MyType { Field4 = 2 }, i => i.Index(index)); await Client.IndexAsync(new MyType { Field1 = "value1", Field4 = 3 }, i => i.Index(index)); await Client.Indices.RefreshAsync(index); @@ -1049,7 +1046,7 @@ public async Task RangeQueryProcessor() await processor.BuildQueryAsync("field4:[1 TO 2} OR field1:value1", new ElasticQueryVisitorContext { UseScoring = true }); - var actualResponse = await Client.SearchAsync(d => d.Index(index).Query(_ => result)); + var actualResponse = await Client.SearchAsync(d => d.Index(index).Query(result)); string actualRequest = actualResponse.GetRequest(); _logger.LogInformation("Actual: {Request}", actualRequest); @@ -1059,7 +1056,7 @@ await Client.SearchAsync( d.Index(index) .Query( f => - f.TermRange(m => m.Field(f2 => f2.Field4).GreaterThanOrEquals("1").LessThan("2")) || + f.Range(r => r.TermRange(rtr => rtr.Field(rtf => rtf.Field4).Gte("1").Lt("2"))) || f.Term(m => m.Field(tf => tf.Field1).Value("value1")))); string expectedRequest = expectedResponse.GetRequest(); _logger.LogInformation("Expected: {Request}", expectedRequest); @@ -1072,7 +1069,7 @@ await Client.SearchAsync( public async Task DateRangeWithWildcardMinQueryProcessor() { string index = await CreateRandomIndexAsync(); - var res = await Client.IndexAsync(new MyType { Field1 = "value1", Field4 = 1, Field5 = DateTime.UtcNow }, i => i.Index(index)); + await Client.IndexAsync(new MyType { Field1 = "value1", Field4 = 1, Field5 = DateTime.UtcNow }, i => i.Index(index)); await Client.IndexAsync(new MyType { Field4 = 2 }, i => i.Index(index)); await Client.IndexAsync(new MyType { Field1 = "value1", Field4 = 3, Field5 = DateTime.UtcNow }, i => i.Index(index)); await Client.Indices.RefreshAsync(index); @@ -1083,14 +1080,14 @@ public async Task DateRangeWithWildcardMinQueryProcessor() var result = await processor.BuildQueryAsync("field5:[* TO 2017-01-31} OR field1:value1", ctx); - var actualResponse = await Client.SearchAsync(d => d.Index(index).Query(_ => result)); + var actualResponse = await Client.SearchAsync(d => d.Index(index).Query(result)); string actualRequest = actualResponse.GetRequest(); _logger.LogInformation("Actual: {Request}", actualResponse); var expectedResponse = await Client.SearchAsync(d => d .Index(index) .Query(f => f - .DateRange(m => m.Field(f2 => f2.Field5).LessThan("2017-01-31").TimeZone("America/Chicago")) + .Range(r => r.DateRange(dr => dr.Field(drf => drf.Field5).Lt("2017-01-31").TimeZone("America/Chicago"))) || f.Match(e => e.Field(m => m.Field1).Query("value1")))); string expectedRequest = expectedResponse.GetRequest(); @@ -1104,7 +1101,7 @@ public async Task DateRangeWithWildcardMinQueryProcessor() public async Task DateRangeWithDateMathQueryProcessor() { string index = await CreateRandomIndexAsync(); - var res = await Client.IndexAsync(new MyType { Field1 = "value1", Field4 = 1, Field5 = DateTime.UtcNow }, i => i.Index(index)); + await Client.IndexAsync(new MyType { Field1 = "value1", Field4 = 1, Field5 = DateTime.UtcNow }, i => i.Index(index)); await Client.IndexAsync(new MyType { Field4 = 2 }, i => i.Index(index)); await Client.IndexAsync(new MyType { Field1 = "value1", Field4 = 3, Field5 = DateTime.UtcNow }, i => i.Index(index)); await Client.Indices.RefreshAsync(index); @@ -1115,13 +1112,13 @@ public async Task DateRangeWithDateMathQueryProcessor() var result = await processor.BuildQueryAsync("field5:[now-1d/d TO now/d]", ctx); - var actualResponse = await Client.SearchAsync(d => d.Index(index).Query(_ => result)); + var actualResponse = await Client.SearchAsync(d => d.Index(index).Query(result)); string actualRequest = actualResponse.GetRequest(); _logger.LogInformation("Actual: {Request}", actualResponse); var expectedResponse = await Client.SearchAsync(d => d .Index(index) - .Query(f => f.DateRange(m => m.Field(f2 => f2.Field5).GreaterThanOrEquals("now-1d/d").LessThanOrEquals("now/d").TimeZone("America/Chicago")))); + .Query(f => f.Range(r => r.DateRange(dr => dr.Field(drf => drf.Field5).Gte("now-1d/d").Lte("now/d").TimeZone("America/Chicago"))))); string expectedRequest = expectedResponse.GetRequest(); _logger.LogInformation("Expected: {Request}", expectedRequest); @@ -1134,7 +1131,7 @@ public async Task DateRangeWithDateMathQueryProcessor() public async Task DateRangeWithWildcardMaxQueryProcessor() { string index = await CreateRandomIndexAsync(); - var res = await Client.IndexAsync(new MyType { Field1 = "value1", Field4 = 1, Field5 = DateTime.UtcNow }, i => i.Index(index)); + await Client.IndexAsync(new MyType { Field1 = "value1", Field4 = 1, Field5 = DateTime.UtcNow }, i => i.Index(index)); await Client.IndexAsync(new MyType { Field4 = 2 }, i => i.Index(index)); await Client.IndexAsync(new MyType { Field1 = "value1", Field4 = 3, Field5 = DateTime.UtcNow }, i => i.Index(index)); await Client.Indices.RefreshAsync(index); @@ -1145,14 +1142,14 @@ public async Task DateRangeWithWildcardMaxQueryProcessor() var result = await processor.BuildQueryAsync("field5:[2017-01-31 TO * } OR field1:value1", ctx); - var actualResponse = await Client.SearchAsync(d => d.Index(index).Query(_ => result)); + var actualResponse = await Client.SearchAsync(d => d.Index(index).Query(result)); string actualRequest = actualResponse.GetRequest(); _logger.LogInformation("Actual: {Request}", actualResponse); var expectedResponse = await Client.SearchAsync(d => d .Index(index) .Query(f => f - .DateRange(m => m.Field(f2 => f2.Field5).GreaterThanOrEquals("2017-01-31").TimeZone("America/Chicago")) + .Range(r => r.DateRange(dr => dr.Field(drf => drf.Field5).Gte("2017-01-31").TimeZone("America/Chicago"))) || f.Match(e => e.Field(m => m.Field1).Query("value1")))); string expectedRequest = expectedResponse.GetRequest(); _logger.LogInformation("Expected: {Request}", expectedRequest); @@ -1165,7 +1162,7 @@ public async Task DateRangeWithWildcardMaxQueryProcessor() public async Task DateRangeWithTimeZone() { string index = await CreateRandomIndexAsync(); - var res = await Client.IndexAsync(new MyType { Field1 = "value1", Field4 = 1, Field5 = DateTime.UtcNow }, i => i.Index(index)); + await Client.IndexAsync(new MyType { Field1 = "value1", Field4 = 1, Field5 = DateTime.UtcNow }, i => i.Index(index)); await Client.IndexAsync(new MyType { Field4 = 2 }, i => i.Index(index)); await Client.IndexAsync(new MyType { Field1 = "value1", Field4 = 3, Field5 = DateTime.UtcNow }, i => i.Index(index)); await Client.Indices.RefreshAsync(index); @@ -1176,14 +1173,14 @@ public async Task DateRangeWithTimeZone() var result = await processor.BuildQueryAsync("field5:[2017-01-31 TO * }^\"America/Chicago\" OR field1:value1", ctx); - var actualResponse = await Client.SearchAsync(d => d.Index(index).Query(_ => result)); + var actualResponse = await Client.SearchAsync(d => d.Index(index).Query(result)); string actualRequest = actualResponse.GetRequest(); _logger.LogInformation("Actual: {Request}", actualResponse); var expectedResponse = await Client.SearchAsync(d => d .Index(index) - .Query(f => f - .DateRange(m => m.Field(f2 => f2.Field5).GreaterThanOrEquals("2017-01-31").TimeZone("America/Chicago")) + .Query(f => + f.Range(r => r.DateRange(dr => dr.Field(drf => drf.Field5).Gte("2017-01-31").TimeZone("America/Chicago"))) || f.Match(e => e.Field(m => m.Field1).Query("value1")))); string expectedRequest = expectedResponse.GetRequest(); _logger.LogInformation("Expected: {Request}", expectedRequest); @@ -1196,7 +1193,7 @@ public async Task DateRangeWithTimeZone() public async Task DateRangeQueryProcessor() { string index = await CreateRandomIndexAsync(); - var res = await Client.IndexAsync(new MyType { Field1 = "value1", Field4 = 1, Field5 = DateTime.UtcNow }, i => i.Index(index)); + await Client.IndexAsync(new MyType { Field1 = "value1", Field4 = 1, Field5 = DateTime.UtcNow }, i => i.Index(index)); await Client.IndexAsync(new MyType { Field4 = 2 }, i => i.Index(index)); await Client.IndexAsync(new MyType { Field1 = "value1", Field4 = 3, Field5 = DateTime.UtcNow }, i => i.Index(index)); await Client.Indices.RefreshAsync(index); @@ -1206,16 +1203,14 @@ public async Task DateRangeQueryProcessor() var processor = new ElasticQueryParser(c => c.UseMappings(Client, index).SetLoggerFactory(Log)); var result = await processor.BuildQueryAsync("field5:[2017-01-01T00\\:00\\:00Z TO 2017-01-31} OR field1:value1", ctx); - var actualResponse = await Client.SearchAsync(d => d.Index(index).Query(_ => result)); + var actualResponse = await Client.SearchAsync(d => d.Index(index).Query(result)); string actualRequest = actualResponse.GetRequest(); _logger.LogInformation("Actual: {Request}", actualRequest); var expectedResponse = await Client.SearchAsync(d => d .Index(index) - .Query(f => f - .DateRange(m => m - .Field(f2 => f2.Field5).GreaterThanOrEquals("2017-01-01T00:00:00Z").LessThan("2017-01-31").TimeZone("America/Chicago")) + .Query(f => f.Range(r => r.DateRange(dr => dr.Field(drf => drf.Field5).Gte("2017-01-01T00:00:00Z").Lt("2017-01-31").TimeZone("America/Chicago"))) || f.Match(e => e.Field(m => m.Field1).Query("value1")))); string expectedRequest = expectedResponse.GetRequest(); _logger.LogInformation("Expected: {Request}", expectedRequest); @@ -1228,7 +1223,7 @@ await Client.SearchAsync(d => d public async Task SimpleGeoRangeQuery() { string index = await CreateRandomIndexAsync(m => m.Properties(p => p.GeoPoint(g => g.Field3))); - var res = await Client.IndexAsync(new MyType { Field1 = "value1", Field4 = 1, Field3 = "51.5032520,-0.1278990" }, + await Client.IndexAsync(new MyType { Field1 = "value1", Field4 = 1, Field3 = "51.5032520,-0.1278990" }, i => i.Index(index)); await Client.IndexAsync(new MyType { Field4 = 2 }, i => i.Index(index)); await Client.IndexAsync(new MyType { Field1 = "value1", Field4 = 3 }, i => i.Index(index)); @@ -1241,7 +1236,7 @@ public async Task SimpleGeoRangeQuery() await processor.BuildQueryAsync("field3:[51.5032520,-0.1278990 TO 51.5032520,-0.1278990]", new ElasticQueryVisitorContext { UseScoring = true }); - var actualResponse = await Client.SearchAsync(d => d.Index(index).Query(_ => result)); + var actualResponse = await Client.SearchAsync(d => d.Index(index).Query(result)); string actualRequest = actualResponse.GetRequest(); _logger.LogInformation("Actual: {Request}", actualRequest); @@ -1312,7 +1307,7 @@ public async Task CanSortByUnmappedField() _logger.LogInformation("Actual: {Request}", actualResponse); var expectedResponse = await Client.SearchAsync(d => d.Index(index) .Sort( - s => s.Field(f => f.Field(new Field("field1")).Descending().UnmappedType(FieldType.Keyword)) + s => s.Field(f => f.Field1, so => so.Order(SortOrder.Desc).UnmappedType(FieldType.Keyword)) )); string expectedRequest = expectedResponse.GetRequest(true); _logger.LogInformation("Expected: {Request}", expectedRequest); @@ -1344,13 +1339,13 @@ public async Task CanParseSort() _logger.LogInformation("Actual: {Request}", actualResponse); var expectedResponse = await Client.SearchAsync(d => d.Index(index).Sort(s => s - .Field(f => f.Field(new Field("field3")).Ascending().UnmappedType(FieldType.GeoPoint)) - .Field(f => f.Field(new Field("field1.keyword")).Descending().UnmappedType(FieldType.Keyword)) - .Field(f => f.Field(new Field("field2.sort")).Descending().UnmappedType(FieldType.Keyword)) - .Field(f => f.Field(new Field("field3")).Descending().UnmappedType(FieldType.GeoPoint)) - .Field(f => f.Field(new Field("field4")).Ascending().UnmappedType(FieldType.Long)) - .Field(f => f.Field(new Field("field5")).Ascending().UnmappedType(FieldType.Date)) - .Field(f => f.Field(new Field("field3")).Ascending().UnmappedType(FieldType.GeoPoint)) + .Field(f => f.Field3, so => so.Order(SortOrder.Asc).UnmappedType(FieldType.GeoPoint)) + .Field("field1.keyword", so => so.Order(SortOrder.Desc).UnmappedType(FieldType.Keyword)) + .Field("field2.sort", so => so.Order(SortOrder.Desc).UnmappedType(FieldType.Keyword)) + .Field(f => f.Field3, so => so.Order(SortOrder.Desc).UnmappedType(FieldType.GeoPoint)) + .Field(f => f.Field4, so => so.Order(SortOrder.Asc).UnmappedType(FieldType.Long)) + .Field(f => f.Field5, so => so.Order(SortOrder.Asc).UnmappedType(FieldType.Date)) + .Field(f => f.Field3, so => so.Order(SortOrder.Asc).UnmappedType(FieldType.GeoPoint)) )); string expectedRequest = expectedResponse.GetRequest(true); _logger.LogInformation("Expected: {Request}", expectedRequest); @@ -1391,16 +1386,17 @@ await Client.IndexManyAsync([ var sort = await processor.BuildSortAsync("nested.data.spaced\\ field"); var query = await processor.BuildQueryAsync("nested.data.spaced\\ field:hey"); var aggs = await processor.BuildAggregationsAsync("terms:nested.data.spaced\\ field"); - var actualResponse = await Client.SearchAsync(d => d.Index(index).Sort(sort).Query(_ => query).Aggregations(_ => aggs)); + var actualResponse = await Client.SearchAsync(d => d.Index(index).Sort(sort).Query(query).Aggregations(aggs)); string actualRequest = actualResponse.GetRequest(); _logger.LogInformation("Actual: {Request}", actualRequest); var expectedResponse = await Client.SearchAsync(d => d.Index(index) .Sort(s => s - .Field(f => f.Field("nested.data.spaced field.keyword").UnmappedType(FieldType.Keyword).Ascending())) + .Field("nested.data.spaced field.keyword", so => so.Order(SortOrder.Asc).UnmappedType(FieldType.Keyword)) + ) .Query(q => q.Bool(b => b.Filter(f => f - .Match(f => f.Field("nested.data.spaced field").Query("hey"))))) + .Match(mf => mf.Field("nested.data.spaced field").Query("hey"))))) .Aggregations(a => a - .Terms("terms_nested.data.spaced field", f => f.Field("nested.data.spaced field.keyword").Meta(m2 => m2.Add("@field_type", "text"))))); + .Add("terms_nested.data.spaced field.keyword", ad => ad.Terms(t => t.Field("nested.data.spaced field.keyword").Meta(m2 => m2.Add("@field_type", "text")))))); string expectedRequest = expectedResponse.GetRequest(); _logger.LogInformation("Expected: {Request}", expectedRequest); Assert.Equal(expectedRequest, actualRequest); @@ -1414,7 +1410,7 @@ public async Task CanParseMixedCaseSort() .Text(e => e.MultiWord, o => o.Fields(f1 => f1.Keyword("keyword"))) )); - var res = await Client.IndexAsync(new MyType { MultiWord = "value1" }, i => i.Index(index)); + await Client.IndexAsync(new MyType { MultiWord = "value1" }, i => i.Index(index)); await Client.Indices.RefreshAsync(index); var processor = new ElasticQueryParser(c => c.SetLoggerFactory(Log).UseMappings(Client, index)); var sort = await processor.BuildSortAsync("multiWord -multiword"); @@ -1423,8 +1419,8 @@ public async Task CanParseMixedCaseSort() _logger.LogInformation("Actual: {Request}", actualRequest); var expectedResponse = await Client.SearchAsync(d => d.Index(index) .Sort(s => s - .Field(f => f.Field("multiWord.keyword").UnmappedType(FieldType.Keyword).Ascending()) - .Field(f => f.Field("multiWord.keyword").UnmappedType(FieldType.Keyword).Descending()) + .Field("multiWord.keyword", so => so.Order(SortOrder.Asc).UnmappedType(FieldType.Keyword)) + .Field("multiWord.keyword", so => so.Order(SortOrder.Desc).UnmappedType(FieldType.Keyword)) )); string expectedRequest = expectedResponse.GetRequest(); _logger.LogInformation("Expected: {Request}", expectedRequest); @@ -1454,19 +1450,19 @@ public async Task GeoRangeQueryProcessor() var result = await processor.BuildQueryAsync("geo:[51.5032520,-0.1278990 TO 51.5032520,-0.1278990] OR field1:value1 OR field2:[1 TO 4] OR -geo:\"Dallas, TX\"~75mi", new ElasticQueryVisitorContext { UseScoring = true }); var sort = await processor.BuildSortAsync("geo -field1"); - var actualResponse = await Client.SearchAsync(d => d.Index(index).Query(_ => result).Sort(sort)); + var actualResponse = await Client.SearchAsync(d => d.Index(index).Query(result).Sort(sort)); string actualRequest = actualResponse.GetRequest(true); _logger.LogInformation("Actual: {Request}", actualResponse); var expectedResponse = await Client.SearchAsync(d => d.Index(index) .Sort(s => s - .Field(f => f.Field(new Field("field3")).Ascending().UnmappedType(FieldType.GeoPoint)) - .Field(f => f.Field(new Field("field1.keyword")).Descending().UnmappedType(FieldType.Keyword)) + .Field(f => f.Field3, so => so.Order(SortOrder.Asc).UnmappedType(FieldType.GeoPoint)) + .Field("field1.keyword", so => so.Order(SortOrder.Desc).UnmappedType(FieldType.Keyword)) ).Query(q => q .GeoBoundingBox(m => m .Field(p => p.Field3).BoundingBox("51.5032520,-0.1278990", "51.5032520,-0.1278990")) || q.Match(y => y.Field(e => e.Field1).Query("value1")) - || q.TermRange(m => m.Field(g => g.Field2).GreaterThanOrEquals("1").LessThanOrEquals("4")) + || q.Range(qr => qr.TermRange(m => m.Field(g => g.Field2).Gte("1").Lte("4"))) || !q.GeoDistance(m => m.Field(p => p.Field3).Location("51.5032520,-0.1278990").Distance("75mi")))); string expectedRequest = expectedResponse.GetRequest(true); _logger.LogInformation("Expected: {Request}", expectedRequest); @@ -1478,16 +1474,15 @@ public async Task GeoRangeQueryProcessor() [Fact] public async Task CanExpandElasticIncludesAsync() { - var client = new ElasticsearchClient(new ElasticsearchClientSettings().DisableDirectStreaming().PrettyJson()); var aliases = new FieldMap { { "field", "aliased" }, { "included", "aliasedincluded" } }; var processor = new ElasticQueryParser(c => c.SetLoggerFactory(Log).UseIncludes(GetIncludeAsync).UseFieldMap(aliases)); var result = await processor.BuildQueryAsync("@include:other"); - var actualResponse = await Client.SearchAsync(d => d.Index("stuff").Query(_ => result)); + var actualResponse = await Client.SearchAsync(d => d.Index("stuff").Query(result)); string actualRequest = actualResponse.GetRequest(); _logger.LogInformation("Actual: {Request}", actualRequest); - var expectedResponse = await Client.SearchAsync(d => d.Index("stuff").Query(f => f.Bool(b => b.Filter(f1 => f1.Term("aliasedincluded", "value"))))); + var expectedResponse = await Client.SearchAsync(d => d.Index("stuff").Query(f => f.Bool(b => b.Filter(f1 => f1.Term(t => t.Field("aliasedincluded").Value("value")))))); string expectedRequest = expectedResponse.GetRequest(); _logger.LogInformation("Expected: {Request}", expectedRequest); @@ -1495,7 +1490,7 @@ public async Task CanExpandElasticIncludesAsync() Assert.Equal(expectedResponse.Total, actualResponse.Total); result = await processor.BuildQueryAsync("@include:other"); - actualResponse = await Client.SearchAsync(d => d.Index("stuff").Query(_ => result)); + actualResponse = await Client.SearchAsync(d => d.Index("stuff").Query(result)); actualRequest = actualResponse.GetRequest(); _logger.LogInformation("Actual: {Request}", actualRequest); _logger.LogInformation("Expected: {Request}", expectedRequest); diff --git a/tests/Foundatio.Parsers.ElasticQueries.Tests/Utility/ElasticsearchTestBase.cs b/tests/Foundatio.Parsers.ElasticQueries.Tests/Utility/ElasticsearchTestBase.cs index 900c023a..4547af98 100644 --- a/tests/Foundatio.Parsers.ElasticQueries.Tests/Utility/ElasticsearchTestBase.cs +++ b/tests/Foundatio.Parsers.ElasticQueries.Tests/Utility/ElasticsearchTestBase.cs @@ -96,24 +96,26 @@ protected ElasticsearchClient GetClient(Action conf return client; } - protected virtual void ConfigureConnectionSettings(ElasticsearchClientSettings settings) { } + protected virtual void ConfigureConnectionSettings(ElasticsearchClientSettings settings) + { + } public async Task CreateNamedIndexAsync(string index, Func, TypeMappingDescriptor> configureMappings = null, Func configureIndex = null) where T : class { - configureMappings ??= m => m.AutoMap().Dynamic(DynamicMapping.True); + configureMappings ??= m => m.Dynamic(DynamicMapping.True); configureIndex ??= i => i.NumberOfReplicas(0).Analysis(a => a.AddSortNormalizer()); - await CreateIndexAsync(index, i => i.Settings(configureIndex(new IndexSettingsDescriptor())).Map(configureMappings)); + await CreateIndexAsync(index, i => i.Settings(configureIndex(new IndexSettingsDescriptor())).Mappings(configureMappings)); Client.ElasticsearchClientSettings.DefaultIndices[typeof(T)] = index; } public async Task CreateRandomIndexAsync(Func, TypeMappingDescriptor> configureMappings = null, Func configureIndex = null) where T : class { string index = $"test_{Guid.NewGuid():N}"; - configureMappings ??= m => m.AutoMap().Dynamic(DynamicMapping.True); + configureMappings ??= m => m.Dynamic(DynamicMapping.True); configureIndex ??= i => i.NumberOfReplicas(0).Analysis(a => a.AddSortNormalizer()); - await CreateIndexAsync(index, i => i.Settings(configureIndex(new IndexSettingsDescriptor())).Map(configureMappings)); + await CreateIndexAsync(index, i => i.Settings(configureIndex(new IndexSettingsDescriptor())).Mappings(configureMappings)); Client.ElasticsearchClientSettings.DefaultIndices[typeof(T)] = index; return index; @@ -134,7 +136,7 @@ public async Task CreateIndexAsync(IndexName index, Action< protected virtual async Task CleanupTestIndexesAsync(ElasticsearchClient client) { if (_createdIndexes.Count > 0) - await client.Indices.DeleteAsync(Indices.Index(_createdIndexes)); + await client.Indices.DeleteAsync(_createdIndexes.ToArray()); } public virtual Task InitializeAsync() From 805cfdc167e8195bb098136c09223133d2f91069 Mon Sep 17 00:00:00 2001 From: Blake Niemyjski Date: Fri, 28 Feb 2025 17:03:39 -0600 Subject: [PATCH 19/19] WIP: Down to 71 compiler errors --- .../AggregationParserTests.cs | 77 +++++++++++-------- .../CustomVisitorTests.cs | 8 +- .../ElasticMappingResolverTests.cs | 8 +- .../ElasticQueryParserTests.cs | 25 +++--- 4 files changed, 72 insertions(+), 46 deletions(-) diff --git a/tests/Foundatio.Parsers.ElasticQueries.Tests/AggregationParserTests.cs b/tests/Foundatio.Parsers.ElasticQueries.Tests/AggregationParserTests.cs index 4ea72272..9c5e2e3e 100644 --- a/tests/Foundatio.Parsers.ElasticQueries.Tests/AggregationParserTests.cs +++ b/tests/Foundatio.Parsers.ElasticQueries.Tests/AggregationParserTests.cs @@ -136,7 +136,7 @@ await Client.IndexManyAsync([ _logger.LogInformation("Actual: {Request}", actualRequest); var expectedResponse = await Client.SearchAsync(d => d.Index(index).Aggregations(a => a - .Min("min_heynow", c => c.Field("field4").Meta(m => m.Add("@field_type", "long"))))); + .Add("min_heynow", a1 => a1.Min(c => c.Field(cf => cf.Field4)).Meta(m => m.Add("@field_type", "long"))))); string expectedRequest = expectedResponse.GetRequest(); _logger.LogInformation("Expected: {Request}", expectedRequest); @@ -265,21 +265,21 @@ await Client.IndexManyAsync([ _logger.LogInformation("Actual: {Request}", actualRequest); var expectedResponse = await Client.SearchAsync(d => d.Index(index).Aggregations(a => a - .Add("geogrid_field3", a1 => a1.GeohashGrid(h => h.Field("field3").Precision(new GeohashPrecision(1))) + .Add("geogrid_field3", a1 => a1.GeohashGrid(h => h.Field(f => f.Field3).Precision(new GeohashPrecision(1))) .Aggregations(a2 => a2 .Add("avg_lat", a3 => a3.Avg(s => s.Script(ss => ss.Source("doc['field3'].lat")))) .Add("avg_lon", a3 => a3.Avg(s => s.Script(ss => ss.Source("doc['field3'].lon")))) )) .Add("terms_field1", a1 => a1.Terms(t => t.Field("field1.keyword")).Meta(m => m.Add("@field_type", "text"))) - .Add("histogram_field4", a1 => a1.Histogram(h => h.Field("field4").Interval(50).MinDocCount(0))) - .Add("date_field5", a1 => a1.DateHistogram(d1 => d1.Field("field5").CalendarInterval(CalendarInterval.Day).Format("date_optional_time").MinDocCount(0))) + .Add("histogram_field4", a1 => a1.Histogram(h => h.Field(f => f.Field4).Interval(50).MinDocCount(0))) + .Add("date_field5", a1 => a1.DateHistogram(d1 => d1.Field(f => f.Field5).CalendarInterval(CalendarInterval.Day).Format("date_optional_time").MinDocCount(0))) .Add("missing_field2", a1 => a1.Missing(t => t.Field("field2.keyword"))) - .Add("cardinality_field4", a1 => a1.Cardinality(c => c.Field("field4"))) - .Add("percentiles_field4", a1 => a1.Percentiles(c => c.Field("field4").Percents([50, 100]))) - .Add("sum_field4", a1 => a1.Sum(c => c.Field("field4")).Meta(m => m.Add("@field_type", "long"))) - .Add("avg_field4", a1 => a1.Avg(c => c.Field("field4")).Meta(m => m.Add("@field_type", "long"))) - .Add("max_field4", a1 => a1.Max(c => c.Field("field4")).Meta(m => m.Add("@field_type", "long"))) - .Add("min_field4", a1 => a1.Min(c => c.Field("field4")).Meta(m => m.Add("@field_type", "long"))))); + .Add("cardinality_field4", a1 => a1.Cardinality(c => c.Field(f => f.Field4))) + .Add("percentiles_field4", a1 => a1.Percentiles(c => c.Field(f => f.Field4).Percents([50, 100]))) + .Add("sum_field4", a1 => a1.Sum(c => c.Field(f => f.Field4)).Meta(m => m.Add("@field_type", "long"))) + .Add("avg_field4", a1 => a1.Avg(c => c.Field(f => f.Field4)).Meta(m => m.Add("@field_type", "long"))) + .Add("max_field4", a1 => a1.Max(c => c.Field(f => f.Field4)).Meta(m => m.Add("@field_type", "long"))) + .Add("min_field4", a1 => a1.Min(c => c.Field(f => f.Field4)).Meta(m => m.Add("@field_type", "long"))))); string expectedRequest = expectedResponse.GetRequest(); _logger.LogInformation("Expected: {Request}", expectedRequest); @@ -362,11 +362,26 @@ public async Task ProcessAggregationsWithAliasesAsync() .Text("identity", o4 => o4.Fields(f => f.Keyword("keyword", o5 => o5.IgnoreAbove(256)))))))))); await Client.IndexManyAsync([ - new MyType { Field1 = "value1", Field4 = 1, Field3 = "51.5032520,-0.1278990", Field5 = DateTime.UtcNow.Subtract(TimeSpan.FromMinutes(5)), Field2 = "field2" } + new MyType + { + Field1 = "value1", + Field4 = 1, + Field3 = "51.5032520,-0.1278990", + Field5 = DateTime.UtcNow.Subtract(TimeSpan.FromMinutes(5)), + Field2 = "field2" + } ], index); await Client.Indices.RefreshAsync(index); - var aliasMap = new FieldMap { { "user", "data.@user.identity" }, { "alias1", "field1" }, { "alias2", "field2" }, { "alias3", "field3" }, { "alias4", "field4" }, { "alias5", "field5" } }; + var aliasMap = new FieldMap + { + { "user", "data.@user.identity" }, + { "alias1", "field1" }, + { "alias2", "field2" }, + { "alias3", "field3" }, + { "alias4", "field4" }, + { "alias5", "field5" } + }; var processor = new ElasticQueryParser(c => c.SetLoggerFactory(Log).UseMappings(Client, index).UseGeo(_ => "51.5032520,-0.1278990").UseFieldMap(aliasMap)); var aggregations = await processor.BuildAggregationsAsync("min:alias4 max:alias4 avg:alias4 sum:alias4 percentiles:alias4 cardinality:user missing:alias2 date:alias5 histogram:alias4 geogrid:alias3 terms:alias1"); @@ -375,21 +390,21 @@ await Client.IndexManyAsync([ _logger.LogInformation("Actual: {Request}", actualRequest); var expectedResponse = await Client.SearchAsync(d => d.Index(index).Aggregations(a => a - .Add("geogrid_alias3", a1 => a1.GeohashGrid(h => h.Field("field3").Precision(new GeohashPrecision(1))) + .Add("geogrid_alias3", a1 => a1.GeohashGrid(h => h.Field(f => f.Field3).Precision(new GeohashPrecision(1))) .Aggregations(a2 => a2 .Add("avg_lat", a3 => a3.Avg(s => s.Script(ss => ss.Source("doc['field3'].lat")))) .Add("avg_lon", a3 => a3.Avg(s => s.Script(ss => ss.Source("doc['field3'].lon")))) )) .Add("terms_alias1", a1 => a1.Terms(t => t.Field("field1.keyword")).Meta(m => m.Add("@field_type", "text"))) - .Add("histogram_alias4", a1 => a1.Histogram(h => h.Field("field4").Interval(50).MinDocCount(0))) - .Add("date_alias5", a1 => a1.DateHistogram(d1 => d1.Field("field5").CalendarInterval(CalendarInterval.Day).Format("date_optional_time").MinDocCount(0))) + .Add("histogram_alias4", a1 => a1.Histogram(h => h.Field(f => f.Field4).Interval(50).MinDocCount(0))) + .Add("date_alias5", a1 => a1.DateHistogram(d1 => d1.Field(f => f.Field5).CalendarInterval(CalendarInterval.Day).Format("date_optional_time").MinDocCount(0))) .Add("missing_alias2", a1 => a1.Missing(t => t.Field("field2.keyword"))) .Add("cardinality_user", a1 => a1.Cardinality(c => c.Field("data.@user.identity.keyword"))) - .Add("percentiles_alias4", a1 => a1.Percentiles(c => c.Field("field4"))) - .Add("sum_alias4", a1 => a1.Sum(c => c.Field("field4")).Meta(m => m.Add("@field_type", "long"))) - .Add("avg_alias4", a1 => a1.Avg(c => c.Field("field4")).Meta(m => m.Add("@field_type", "long"))) - .Add("max_alias4", a1 => a1.Max(c => c.Field("field4")).Meta(m => m.Add("@field_type", "long"))) - .Add("min_alias4", a1 => a1.Min(c => c.Field("field4")).Meta(m => m.Add("@field_type", "long"))))); + .Add("percentiles_alias4", a1 => a1.Percentiles(c => c.Field(f => f.Field4))) + .Add("sum_alias4", a1 => a1.Sum(c => c.Field(f => f.Field4)).Meta(m => m.Add("@field_type", "long"))) + .Add("avg_alias4", a1 => a1.Avg(c => c.Field(f => f.Field4)).Meta(m => m.Add("@field_type", "long"))) + .Add("max_alias4", a1 => a1.Max(c => c.Field(f => f.Field4)).Meta(m => m.Add("@field_type", "long"))) + .Add("min_alias4", a1 => a1.Min(c => c.Field(f => f.Field4)).Meta(m => m.Add("@field_type", "long"))))); string expectedRequest = expectedResponse.GetRequest(); _logger.LogInformation("Expected: {Request}", expectedRequest); @@ -420,8 +435,8 @@ public async Task ProcessTermAggregations() .MinDocCount(1) .Include(new TermsInclude(["otherinclude", "myinclude"])) .Exclude(new TermsExclude(["otherexclude", "myexclude"])) - .Missing("mymissing") - .Meta(m => m.Add("@field_type", "keyword")))))); + .Missing("mymissing")) + .Meta(m => m.Add("@field_type", "keyword"))))); string expectedRequest = expectedResponse.GetRequest(); _logger.LogInformation("Expected: {Request}", expectedRequest); @@ -501,7 +516,7 @@ public async Task ProcessSortedTermAggregations() var expectedResponse = await Client.SearchAsync(d => d.Index(index).Aggregations(a => a .Add("terms_field1", a1 => a1 .Terms(t => t.Field("field1.keyword").Order(new List> { new ("cardinality_field4", SortOrder.Desc) })) - .Aggregations(a2 => a2.Add("cardinality_field4", a3 => a3.Cardinality(c => c.Field("field4")))) + .Aggregations(a2 => a2.Add("cardinality_field4", a3 => a3.Cardinality(c => c.Field(f => f.Field4)))) .Meta(m => m.Add("@field_type", "keyword"))))); string expectedRequest = expectedResponse.GetRequest(); _logger.LogInformation("Expected: {Request}", expectedRequest); @@ -528,7 +543,7 @@ public async Task ProcessDateHistogramAggregations() var expectedResponse = await Client.SearchAsync(d => d.Index(index).Aggregations(a => a .Add("date_field5", a1 => a1.DateHistogram(d1 => d1 - .Field("field5") + .Field(f => f.Field5) .CalendarInterval(CalendarInterval.Day) .Format("date_optional_time") .MinDocCount(0) @@ -536,8 +551,8 @@ public async Task ProcessDateHistogramAggregations() .Missing(DateTime.MinValue)) .Meta(m => m.Add("@timezone", "1h")) .Aggregations(a2 => a2 - .Add("min_field5", a3 => a3.Min(c => c.Field("field5")).Meta(m => m.Add("@field_type", "date").Add("@timezone", "1h"))) - .Add("max_field5", a3 => a3.Max(c => c.Field("field5")).Meta(m => m.Add("@field_type", "date").Add("@timezone", "1h"))))))); + .Add("min_field5", a3 => a3.Min(c => c.Field(f => f.Field5)).Meta(m => m.Add("@field_type", "date").Add("@timezone", "1h"))) + .Add("max_field5", a3 => a3.Max(c => c.Field(f => f.Field5)).Meta(m => m.Add("@field_type", "date").Add("@timezone", "1h"))))))); string expectedRequest = expectedResponse.GetRequest(); _logger.LogInformation("Expected: {Request}", expectedRequest); @@ -563,11 +578,11 @@ public async Task CanSpecifyDefaultValuesAggregations() _logger.LogInformation("Actual: {Request}", actualRequest); var expectedResponse = await Client.SearchAsync(d => d.Index(index).Aggregations(a => a - .Add("sum_field4", a1 => a1.Sum(c => c.Field("field4").Missing(0)).Meta(m => m.Add("@field_type", "integer"))) - .Add("cardinality_field4", a1 => a1.Cardinality(c => c.Field("field4").Missing(0))) - .Add("avg_field4", a1 => a1.Avg(c => c.Field("field4").Missing(0)).Meta(m => m.Add("@field_type", "integer"))) - .Add("max_field4", a1 => a1.Max(c => c.Field("field4").Missing(0)).Meta(m => m.Add("@field_type", "integer"))) - .Add("min_field4", a1 => a1.Min(c => c.Field("field4").Missing(0)).Meta(m => m.Add("@field_type", "integer"))))); + .Add("sum_field4", a1 => a1.Sum(c => c.Field(f => f.Field4).Missing(0)).Meta(m => m.Add("@field_type", "integer"))) + .Add("cardinality_field4", a1 => a1.Cardinality(c => c.Field(f => f.Field4).Missing(0))) + .Add("avg_field4", a1 => a1.Avg(c => c.Field(f => f.Field4).Missing(0)).Meta(m => m.Add("@field_type", "integer"))) + .Add("max_field4", a1 => a1.Max(c => c.Field(f => f.Field4).Missing(0)).Meta(m => m.Add("@field_type", "integer"))) + .Add("min_field4", a1 => a1.Min(c => c.Field(f => f.Field4).Missing(0)).Meta(m => m.Add("@field_type", "integer"))))); string expectedRequest = expectedResponse.GetRequest(); _logger.LogInformation("Expected: {Request}", expectedRequest); diff --git a/tests/Foundatio.Parsers.ElasticQueries.Tests/CustomVisitorTests.cs b/tests/Foundatio.Parsers.ElasticQueries.Tests/CustomVisitorTests.cs index ccdf8bc4..f1bd2bed 100644 --- a/tests/Foundatio.Parsers.ElasticQueries.Tests/CustomVisitorTests.cs +++ b/tests/Foundatio.Parsers.ElasticQueries.Tests/CustomVisitorTests.cs @@ -33,7 +33,7 @@ public async Task CanResolveSimpleCustomFilter() .AddVisitor(new CustomFilterVisitor())); var result = await processor.BuildQueryAsync("@custom:(one)"); - var actualResponse = await Client.SearchAsync(d => d.Index(index).Query(_ => result)); + var actualResponse = await Client.SearchAsync(d => d.Index(index).Query(result)); string actualRequest = actualResponse.GetRequest(); _logger.LogInformation("Actual: {Request}", actualRequest); @@ -58,7 +58,7 @@ public async Task CanResolveCustomFilterContainingIncludes() .AddVisitor(new CustomFilterVisitor())); var result = await processor.BuildQueryAsync("@custom:(one @include:3)"); - var actualResponse = await Client.SearchAsync(d => d.Index(index).Query(_ => result)); + var actualResponse = await Client.SearchAsync(d => d.Index(index).Query(result)); string actualRequest = actualResponse.GetRequest(); _logger.LogInformation("Actual: {Request}", actualRequest); @@ -83,7 +83,7 @@ public async Task CanResolveIncludeToCustomFilterContainingIgnoredInclude() .AddVisitor(new CustomFilterVisitor(), 1)); var result = await processor.BuildQueryAsync("@include:test"); - var actualResponse = await Client.SearchAsync(d => d.Index(index).Query(_ => result)); + var actualResponse = await Client.SearchAsync(d => d.Index(index).Query(result)); string actualRequest = actualResponse.GetRequest(); _logger.LogInformation("Actual: {Request}", actualRequest); @@ -130,7 +130,7 @@ public async Task CanResolveMultipleCustomFilters() .AddVisitor(new CustomFilterVisitor())); var result = await processor.BuildQueryAsync("@custom:(one) OR (field1:Test @custom:(two))"); - var actualResponse = await Client.SearchAsync(d => d.Index(index).Query(_ => result)); + var actualResponse = await Client.SearchAsync(d => d.Index(index).Query(result)); string actualRequest = actualResponse.GetRequest(); _logger.LogInformation("Actual: {Request}", actualRequest); diff --git a/tests/Foundatio.Parsers.ElasticQueries.Tests/ElasticMappingResolverTests.cs b/tests/Foundatio.Parsers.ElasticQueries.Tests/ElasticMappingResolverTests.cs index 45029ba5..5f28b3b0 100644 --- a/tests/Foundatio.Parsers.ElasticQueries.Tests/ElasticMappingResolverTests.cs +++ b/tests/Foundatio.Parsers.ElasticQueries.Tests/ElasticMappingResolverTests.cs @@ -21,7 +21,13 @@ private TypeMappingDescriptor MapMyNestedType(TypeMappingDescripto { return m .Dynamic(DynamicMapping.True) - .DynamicTemplates(t => t.DynamicTemplate("idx_text", t1 => t1.Match("text*").Mapping(m1 => m1.Text(mp => mp.AddKeywordAndSortFields())))) + .DynamicTemplates(new List> + { + new Dictionary + { + { "idx_text", DynamicTemplate.Mapping(new TextProperty().AddKeywordAndSortFields()).Match("text*") } + } + }) .Properties(p => p .Text(p1 => p1.Field1, o => o.AddKeywordAndSortFields()) .Text(p1 => p1.Field4, o => o.AddKeywordAndSortFields()) diff --git a/tests/Foundatio.Parsers.ElasticQueries.Tests/ElasticQueryParserTests.cs b/tests/Foundatio.Parsers.ElasticQueries.Tests/ElasticQueryParserTests.cs index 94e9dcb9..9f37c68b 100644 --- a/tests/Foundatio.Parsers.ElasticQueries.Tests/ElasticQueryParserTests.cs +++ b/tests/Foundatio.Parsers.ElasticQueries.Tests/ElasticQueryParserTests.cs @@ -182,7 +182,7 @@ public async Task ShouldHandleMultipleTermsForAnalyzedFields() || f.Match(m => m.Field(mf => mf.Field1).Query("abc")) || f.Match(m => m.Field(mf => mf.Field1).Query("def")) || f.Match(m => m.Field(mf => mf.Field1).Query("ghi")) - || f.Term(m => m.Field(tf => tf.Field2).Value("value2")) || f.Term(m => m.Field(mf => mf.Field2), "jhk"))); + || f.Term(m => m.Field(tf => tf.Field2).Value("value2")) || f.Term(m => m.Field(mf => mf.Field2).Value("jhk")))); string expectedRequest = expectedResponse.GetRequest(); _logger.LogInformation("Expected: {Request}", expectedRequest); @@ -305,8 +305,7 @@ await Client.IndexManyAsync([ var expectedResponse = await Client.SearchAsync(d => d.Index(index) .Query(q => q .Bool(b => b - .Filter(f => f - .QueryString(m => m.Query("one\\/two*").Fields(f1 => f1.Field(f2 => f2.Field1)).AllowLeadingWildcard(false).AnalyzeWildcard()))))); + .Filter(f => f.QueryString(m => m.Query("one\\/two*").Fields(Fields.FromExpression((MyType f1) => f1.Field1)).AllowLeadingWildcard(false).AnalyzeWildcard()))))); string expectedRequest = expectedResponse.GetRequest(true); _logger.LogInformation("Expected: {Request}", expectedRequest); @@ -609,7 +608,7 @@ await Client.IndexManyAsync([ expectedResponse = await Client.SearchAsync(d => d.Index(index).Query(q => q.QueryString(m => m .AllowLeadingWildcard(false) .AnalyzeWildcard(true) - .Fields(f => f.Field(f1 => f1.Field3)) + .Fields(Fields.FromExpression((MyType f1) => f1.Field3)) .Query("hey*") ))); expectedRequest = expectedResponse.GetRequest(); @@ -965,9 +964,7 @@ await Client.IndexManyAsync([ public async Task CanGenerateMatchQuery() { string index = await CreateRandomIndexAsync(m => m.Properties(p => p - .Text(e => e.Field1) - .Fields(f1 => f1 - .Keyword(k => k.Name("keyword").IgnoreAbove(256))))); + .Text(e => e.Field1, o => o.Fields(f1 => f1.Keyword("keyword")).IgnoreAbove(256)))); var processor = new ElasticQueryParser(c => c.SetLoggerFactory(Log).UseMappings(Client, index)); var result = await processor.BuildQueryAsync("field1:test", new ElasticQueryVisitorContext().UseScoring()); @@ -1242,7 +1239,11 @@ await processor.BuildQueryAsync("field3:[51.5032520,-0.1278990 TO 51.5032520,-0. var expectedResponse = await Client.SearchAsync(d => d.Index(index).Query(q => q.GeoBoundingBox( - m => m.Field(p => p.Field3).BoundingBox("51.5032520,-0.1278990", "51.5032520,-0.1278990")))); + m => m.Field(p => p.Field3).BoundingBox(new TopLeftBottomRightGeoBounds + { + TopLeft = "51.5032520,-0.1278990", + BottomRight = "51.5032520,-0.1278990" + })))); string expectedRequest = expectedResponse.GetRequest(); _logger.LogInformation("Expected: {Request}", expectedRequest); @@ -1396,7 +1397,7 @@ await Client.IndexManyAsync([ .Query(q => q.Bool(b => b.Filter(f => f .Match(mf => mf.Field("nested.data.spaced field").Query("hey"))))) .Aggregations(a => a - .Add("terms_nested.data.spaced field.keyword", ad => ad.Terms(t => t.Field("nested.data.spaced field.keyword").Meta(m2 => m2.Add("@field_type", "text")))))); + .Add("terms_nested.data.spaced field.keyword", ad => ad.Terms(t => t.Field("nested.data.spaced field.keyword")).Meta(m2 => m2.Add("@field_type", "text"))))); string expectedRequest = expectedResponse.GetRequest(); _logger.LogInformation("Expected: {Request}", expectedRequest); Assert.Equal(expectedRequest, actualRequest); @@ -1460,7 +1461,11 @@ public async Task GeoRangeQueryProcessor() .Field("field1.keyword", so => so.Order(SortOrder.Desc).UnmappedType(FieldType.Keyword)) ).Query(q => q .GeoBoundingBox(m => m - .Field(p => p.Field3).BoundingBox("51.5032520,-0.1278990", "51.5032520,-0.1278990")) + .Field(p => p.Field3).BoundingBox(new TopLeftBottomRightGeoBounds + { + TopLeft = "51.5032520,-0.1278990", + BottomRight = "51.5032520,-0.1278990" + })) || q.Match(y => y.Field(e => e.Field1).Query("value1")) || q.Range(qr => qr.TermRange(m => m.Field(g => g.Field2).Gte("1").Lte("4"))) || !q.GeoDistance(m => m.Field(p => p.Field3).Location("51.5032520,-0.1278990").Distance("75mi"))));