-
Notifications
You must be signed in to change notification settings - Fork 2.7k
feat: add tag-based filtering API endpoints #7505
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
base: main
Are you sure you want to change the base?
Changes from all commits
df6b491
9407158
164fa70
0a59c2e
0c01cff
5245e5e
5a7781f
37144b2
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
Original file line number | Diff line number | Diff line change | ||||||||||||||||||||||||||||
---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|
|
@@ -5,6 +5,7 @@ package elasticsearch | |||||||||||||||||||||||||||||
|
||||||||||||||||||||||||||||||
import ( | ||||||||||||||||||||||||||||||
"context" | ||||||||||||||||||||||||||||||
"fmt" | ||||||||||||||||||||||||||||||
"strconv" | ||||||||||||||||||||||||||||||
"strings" | ||||||||||||||||||||||||||||||
"time" | ||||||||||||||||||||||||||||||
|
@@ -114,6 +115,40 @@ func (*QueryBuilder) buildTimeSeriesAggQuery(params metricstore.BaseQueryParamet | |||||||||||||||||||||||||||||
return dateHistAgg | ||||||||||||||||||||||||||||||
} | ||||||||||||||||||||||||||||||
|
||||||||||||||||||||||||||||||
// This function builds an Elasticsearch query to fetch unique values for a specified attribute (tag), optionally filtered by a service name. | ||||||||||||||||||||||||||||||
func (*QueryBuilder) BuildAttributeValuesQuery(params *metricstore.AttributeValuesQueryParameters) (elastic.Query, elastic.Aggregation) { | ||||||||||||||||||||||||||||||
// Create a bool query to filter by service name if provided | ||||||||||||||||||||||||||||||
boolQuery := elastic.NewBoolQuery() | ||||||||||||||||||||||||||||||
if params.ServiceName != "" { | ||||||||||||||||||||||||||||||
boolQuery.Filter(elastic.NewTermQuery("process.serviceName", params.ServiceName)) | ||||||||||||||||||||||||||||||
} | ||||||||||||||||||||||||||||||
|
||||||||||||||||||||||||||||||
// Create a global aggregation that will contain all nested aggregations | ||||||||||||||||||||||||||||||
globalAgg := elastic.NewGlobalAggregation() | ||||||||||||||||||||||||||||||
|
||||||||||||||||||||||||||||||
for i, path := range spanstore.NestedTagFieldList { | ||||||||||||||||||||||||||||||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Why do you loop through all these? There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. So I ditched the ResourceTarget query parameter in favour of searching 3-4 places where a tag could be present in ES. I took inspiration from the already working code in the search page where we filter by tag jaeger/internal/storage/v1/elasticsearch/spanstore/reader.go Lines 656 to 668 in fa9a5e7
In span fitlter by tag, We only seem to search in 3 places, and its been working fine I guess?
We are doing something similar in the new api to get tag values. So instead of providing the exact path to search at, which you know requires the user to dig into ES and figure out the nested path, we always search in some predefined places. |
||||||||||||||||||||||||||||||
// Create nested aggregation for each path | ||||||||||||||||||||||||||||||
nestedAgg := elastic.NewNestedAggregation().Path(path) | ||||||||||||||||||||||||||||||
|
||||||||||||||||||||||||||||||
filterAgg := elastic.NewFilterAggregation(). | ||||||||||||||||||||||||||||||
Filter(elastic.NewTermQuery(path+".key", params.AttributeKey)) | ||||||||||||||||||||||||||||||
|
||||||||||||||||||||||||||||||
// Get unique values | ||||||||||||||||||||||||||||||
valuesAgg := elastic.NewTermsAggregation(). | ||||||||||||||||||||||||||||||
Field(path + ".value"). | ||||||||||||||||||||||||||||||
Size(100) | ||||||||||||||||||||||||||||||
|
||||||||||||||||||||||||||||||
// Chain aggregations | ||||||||||||||||||||||||||||||
filterAgg.SubAggregation("values", valuesAgg) | ||||||||||||||||||||||||||||||
nestedAgg.SubAggregation("filtered_by_key", filterAgg) | ||||||||||||||||||||||||||||||
|
||||||||||||||||||||||||||||||
// Add to global aggregation with unique name | ||||||||||||||||||||||||||||||
globalAgg.SubAggregation(fmt.Sprintf("path_%d", i), nestedAgg) | ||||||||||||||||||||||||||||||
} | ||||||||||||||||||||||||||||||
|
||||||||||||||||||||||||||||||
return boolQuery, globalAgg | ||||||||||||||||||||||||||||||
} | ||||||||||||||||||||||||||||||
|
||||||||||||||||||||||||||||||
// Execute runs the Elasticsearch search with the provided bool and aggregation queries. | ||||||||||||||||||||||||||||||
func (q *QueryBuilder) Execute(ctx context.Context, boolQuery elastic.BoolQuery, aggQuery elastic.Aggregation, timeRange TimeRange) (*elastic.SearchResult, error) { | ||||||||||||||||||||||||||||||
indexName := q.cfg.Indices.IndexPrefix.Apply("jaeger-span-") | ||||||||||||||||||||||||||||||
|
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -8,6 +8,7 @@ import ( | |
"errors" | ||
"fmt" | ||
"math" | ||
"strings" | ||
"time" | ||
|
||
"github.com/olivere/elastic/v7" | ||
|
@@ -166,6 +167,89 @@ func (r MetricsReader) GetErrorRates(ctx context.Context, params *metricstore.Er | |
return CalculateErrorRates(rawErrorsMetrics, callRateMetrics, params.BaseQueryParameters, timeRange), nil | ||
} | ||
|
||
// GetAttributeValues implements metricstore.Reader. | ||
func (r MetricsReader) GetAttributeValues(ctx context.Context, params *metricstore.AttributeValuesQueryParameters) ([]string, error) { | ||
boolQuery, aggQuery := r.queryBuilder.BuildAttributeValuesQuery(params) | ||
|
||
searchResult, err := r.executeSearchWithAggregation(ctx, boolQuery, aggQuery) | ||
if err != nil { | ||
return nil, fmt.Errorf("failed to execute attribute values query: %w", err) | ||
} | ||
|
||
// Collect values from all paths (path_0, path_1, path_2, etc.) | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. what do you mean by path_0 ... ? There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. This is simply the alias/unique identifier for each nested aggregation. The path_, idx is simply the index of path in There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Its defined here
|
||
allValues := make(map[string]bool) // Use map to deduplicate | ||
|
||
// The aggregation is wrapped in aggName ("results_buckets") | ||
if resultsAgg, found := searchResult.Aggregations.Global(aggName); found { | ||
// Look for path aggregations directly in results_buckets | ||
for name := range resultsAgg.Aggregations { | ||
if !strings.HasPrefix(name, "path_") { | ||
continue | ||
} | ||
nestedAgg, _ := resultsAgg.Aggregations.Nested(name) | ||
filterAgg, found := nestedAgg.Aggregations.Filter("filtered_by_key") | ||
if !found { | ||
continue | ||
} | ||
valuesAgg, found := filterAgg.Aggregations.Terms("values") | ||
if !found { | ||
continue | ||
} | ||
for _, bucket := range valuesAgg.Buckets { | ||
if bucket.Key == nil { | ||
continue | ||
} | ||
keyStr, ok := bucket.Key.(string) | ||
if !ok { | ||
keyStr = fmt.Sprintf("%v", bucket.Key) | ||
} | ||
allValues[keyStr] = true | ||
} | ||
} | ||
} | ||
|
||
// Convert map keys to slice | ||
values := make([]string, 0, len(allValues)) | ||
for value := range allValues { | ||
values = append(values, value) | ||
} | ||
|
||
return values, nil | ||
} | ||
|
||
// executeSearchWithAggregation is a helper method to execute a search with an aggregation | ||
func (r MetricsReader) executeSearchWithAggregation( | ||
ctx context.Context, | ||
query elastic.Query, | ||
aggQuery elastic.Aggregation, | ||
) (*elastic.SearchResult, error) { | ||
// Calculate a default time range for the last hour, keeping this low to reduce data volume | ||
timeRange := TimeRange{ | ||
startTimeMillis: time.Now().Add(-1 * time.Hour).UnixMilli(), | ||
endTimeMillis: time.Now().UnixMilli(), | ||
extendedStartTimeMillis: time.Now().Add(-1 * time.Hour).UnixMilli(), | ||
} | ||
|
||
// Here we'll execute using a method similar to the QueryBuilder's Execute | ||
// but using our own custom aggregation | ||
searchRequest := elastic.NewSearchRequest() | ||
searchRequest.Query(query) | ||
searchRequest.Size(0) // Only interested in aggregations | ||
searchRequest.Aggregation(aggName, aggQuery) | ||
|
||
// Directly cast the query to BoolQuery | ||
boolQuery, _ := query.(*elastic.BoolQuery) | ||
|
||
metricsParams := MetricsQueryParams{ | ||
metricName: "attribute_values", | ||
metricDesc: "Search for attribute values", | ||
boolQuery: *boolQuery, | ||
aggQuery: aggQuery, | ||
} | ||
|
||
return r.executeSearch(ctx, metricsParams, timeRange) | ||
} | ||
|
||
// GetMinStepDuration returns the minimum step duration. | ||
func (MetricsReader) GetMinStepDuration(_ context.Context, _ *metricstore.MinStepDurationQueryParameters) (time.Duration, error) { | ||
return minStep, nil | ||
|
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
why is this an aggregation query? Seems like you're doing something like
GROUP BY
, but isn't there a simplerSELECT DISTINCT
flavor?There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Since we search in multiple nested documents (three currently), defined here
jaeger/internal/storage/v1/elasticsearch/spanstore/reader.go
Line 78 in fa9a5e7
a simpler appraoch would work in case we just search in a given location. Like
process.tags
, but I feel predefining the location of the attributes is not needed.