Skip to content

Commit

Permalink
Fix querying the index on a entity tag (#579)
Browse files Browse the repository at this point in the history
* Fix querying the index on a entity tag

Signed-off-by: Gao Hongtao <[email protected]>

* Remove e2e
  • Loading branch information
hanahmily authored Dec 23, 2024
1 parent 3658400 commit 6e99ab7
Show file tree
Hide file tree
Showing 37 changed files with 142 additions and 504 deletions.
2 changes: 0 additions & 2 deletions .github/workflows/e2e.storage.yml
Original file line number Diff line number Diff line change
Expand Up @@ -43,8 +43,6 @@ jobs:
config: test/e2e-v2/cases/storage/banyandb/e2e.yaml
- name: Trace Profiling BanyanDB
config: test/e2e-v2/cases/profiling/trace/banyandb/e2e.yaml
- name: eBPF Profiling BanyanDB
config: test/e2e-v2/cases/profiling/ebpf/oncpu/banyandb/e2e.yaml
- name: Event BanyanDB
config: test/e2e-v2/cases/event/banyandb/e2e.yaml
- name: BanyanDB Cluster Mode
Expand Down
1 change: 1 addition & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -54,6 +54,7 @@ target
# mock files
*mock.go
*mock_test.go
gomock_reflect*

# snky cache
.dccache
Expand Down
1 change: 1 addition & 0 deletions CHANGES.md
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,7 @@ Release Notes.
### Chores

- Fix metrics system typo.
- Bump up OAP in CI to 6d262cce62e156bd197177abb3640ea65bb2d38e.

### CVEs

Expand Down
7 changes: 6 additions & 1 deletion api/proto/banyandb/measure/v1/rpc.proto
Original file line number Diff line number Diff line change
Expand Up @@ -38,5 +38,10 @@ service MeasureService {
}

rpc Write(stream WriteRequest) returns (stream WriteResponse);
rpc TopN(TopNRequest) returns (TopNResponse);
rpc TopN(TopNRequest) returns (TopNResponse) {
option (google.api.http) = {
post: "/v1/measure/topn"
body: "*"
};
}
}
12 changes: 9 additions & 3 deletions banyand/measure/measure.go
Original file line number Diff line number Diff line change
Expand Up @@ -51,14 +51,15 @@ type option struct {

type measure struct {
databaseSupplier schema.Supplier
indexTagMap map[string]struct{}
l *logger.Logger
schema *databasev1.Measure
processorManager *topNProcessorManager
fieldIndexLocation partition.FieldIndexLocation
name string
group string
indexRules []*databasev1.IndexRule
indexRuleLocators partition.IndexRuleLocator
fieldIndexLocation partition.FieldIndexLocation
indexRules []*databasev1.IndexRule
topNAggregations []*databasev1.TopNAggregation
interval time.Duration
shardNum uint32
Expand Down Expand Up @@ -104,7 +105,12 @@ func (s *measure) parseSpec() (err error) {
s.interval, err = timestamp.ParseDuration(s.schema.Interval)
}
s.indexRuleLocators, s.fieldIndexLocation = partition.ParseIndexRuleLocators(s.schema.GetEntity(), s.schema.GetTagFamilies(), s.indexRules, s.schema.IndexMode)

s.indexTagMap = make(map[string]struct{})
for j := range s.indexRules {
for k := range s.indexRules[j].Tags {
s.indexTagMap[s.indexRules[j].Tags[k]] = struct{}{}
}
}
return err
}

Expand Down
3 changes: 3 additions & 0 deletions banyand/measure/write.go
Original file line number Diff line number Diff line change
Expand Up @@ -284,6 +284,9 @@ func (w *writeCallback) appendEntityTagsToIndexFields(fields []index.Field, stm
f.NoSort = true
fields = append(fields, f)
for i := range stm.schema.Entity.TagNames {
if _, exists := stm.indexTagMap[stm.schema.Entity.TagNames[i]]; exists {
continue
}
tagName := stm.schema.Entity.TagNames[i]
var t *databasev1.TagSpec
for j := range stm.schema.TagFamilies {
Expand Down
11 changes: 7 additions & 4 deletions pkg/index/inverted/query.go
Original file line number Diff line number Diff line change
Expand Up @@ -197,14 +197,14 @@ func buildIndexModeCriteria(criteria *modelv1.Criteria, schema logical.Schema, e
if err != nil {
return nil, err
}
if _, ok := entityDict[cond.Name]; ok {
fk := index.FieldKey{TagName: index.IndexModeEntityTagPrefix + cond.Name}
return parseConditionToQuery(cond, nil, expr, fk.Marshal())
}
if ok, indexRule := schema.IndexDefined(cond.Name); ok {
fk := index.FieldKey{IndexRuleID: indexRule.Metadata.Id}
return parseConditionToQuery(cond, indexRule, expr, fk.Marshal())
}
if _, ok := entityDict[cond.Name]; ok {
fk := index.FieldKey{TagName: index.IndexModeEntityTagPrefix + cond.Name}
return parseConditionToQuery(cond, nil, expr, fk.Marshal())
}
return nil, errors.Wrapf(logical.ErrUnsupportedConditionOp, "mandatory index rule conf:%s", cond)
case *modelv1.Criteria_Le:
le := criteria.GetLe()
Expand Down Expand Up @@ -301,6 +301,9 @@ func parseConditionToQuery(cond *modelv1.Condition, indexRule *databasev1.IndexR
node := newTermNode(str, indexRule)
return &queryNode{query, node}, nil
case modelv1.Condition_BINARY_OP_MATCH:
if indexRule == nil {
return nil, errors.WithMessagef(logical.ErrUnsupportedConditionOp, "index rule is mandatory for match operation: %s", cond)
}
bb := expr.Bytes()
if len(bb) != 1 {
return nil, errors.WithMessagef(logical.ErrUnsupportedConditionOp, "don't support multiple or null value: %s", cond)
Expand Down
5 changes: 2 additions & 3 deletions test/docker/base-compose.yml
Original file line number Diff line number Diff line change
Expand Up @@ -67,9 +67,6 @@ services:
- 5005
environment:
SW_STORAGE_BANYANDB_TARGETS: banyandb:17912
SW_CONFIG_ETCD_PERIOD: 1
SW_CONFIG_ETCD_ENDPOINTS: http://etcd:2379
SW_CLUSTER_ETCD_ENDPOINTS: http://etcd:2379
healthcheck:
test: ["CMD", "bash", "-c", "cat < /dev/null > /dev/tcp/127.0.0.1/11800"]
interval: 5s
Expand Down Expand Up @@ -98,6 +95,7 @@ services:
SW_AGENT_INSTANCE_NAME: provider1
SW_AGENT_COLLECTOR_GET_PROFILE_TASK_INTERVAL: 1
SW_AGENT_COLLECTOR_GET_AGENT_DYNAMIC_CONFIG_INTERVAL: 1
SW_METER_ACTIVE: 'false'
healthcheck:
test: ["CMD", "bash", "-c", "cat < /dev/null > /dev/tcp/127.0.0.1/9090"]
interval: 5s
Expand All @@ -120,6 +118,7 @@ services:
SW_AGENT_INSTANCE_NAME: consumer1
SW_AGENT_COLLECTOR_GET_PROFILE_TASK_INTERVAL: 1
SW_AGENT_COLLECTOR_GET_AGENT_DYNAMIC_CONFIG_INTERVAL: 1
SW_METER_ACTIVE: 'false'
healthcheck:
test: ["CMD", "bash", "-c", "cat < /dev/null > /dev/tcp/127.0.0.1/9092"]
interval: 5s
Expand Down
16 changes: 16 additions & 0 deletions test/e2e-v2/cases/cluster/storage-cases.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -160,3 +160,19 @@ cases:
| yq e '.traces[0].traceids[0]' - \
)
expected: ../storage/expected/trace-users-detail.yml
# topN-OP-service Global
- query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql metrics exec --expression="top_n(service_sla,3,des)/100"
expected: ../storage/expected/topN-OP-service.yml
# topN-OP-service Global with attrs
- query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql metrics exec --expression="top_n(service_sla,3,des,attr0='GENERAL')/100"
expected: ../storage/expected/topN-OP-service.yml
- query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql metrics exec --expression="top_n(service_sla,3,des,attr0!='Not_GENERAL')/100"
expected: ../storage/expected/topN-OP-service.yml
# topN-OP-endpoint Global
- query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql metrics exec --expression="top_n(endpoint_sla,3,des)/100"
expected: ../storage/expected/topN-OP-endpoint.yml
# topN-OP-endpoint Global with attrs
- query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql metrics exec --expression="top_n(endpoint_sla,3,des,attr0='GENERAL')/100"
expected: ../storage/expected/topN-OP-endpoint.yml
- query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql metrics exec --expression="top_n(endpoint_sla,3,des,attr0!='Not_GENERAL')/100"
expected: ../storage/expected/topN-OP-endpoint.yml
23 changes: 0 additions & 23 deletions test/e2e-v2/cases/profiling/ebpf/oncpu/Dockerfile.sqrt

This file was deleted.

58 changes: 0 additions & 58 deletions test/e2e-v2/cases/profiling/ebpf/oncpu/banyandb/docker-compose.yml

This file was deleted.

37 changes: 0 additions & 37 deletions test/e2e-v2/cases/profiling/ebpf/oncpu/banyandb/e2e.yaml

This file was deleted.

62 changes: 0 additions & 62 deletions test/e2e-v2/cases/profiling/ebpf/oncpu/docker-compose.yml

This file was deleted.

This file was deleted.

Loading

0 comments on commit 6e99ab7

Please sign in to comment.