Fix querying the index on a entity tag (#579)

* Fix querying the index on a entity tag

Signed-off-by: Gao Hongtao <[email protected]>

* Remove e2e

Author: hanahmily

.github/workflows/e2e.storage.yml

@@ -43,8 +43,6 @@ jobs:
             config: test/e2e-v2/cases/storage/banyandb/e2e.yaml
           - name: Trace Profiling BanyanDB
             config: test/e2e-v2/cases/profiling/trace/banyandb/e2e.yaml
-          - name: eBPF Profiling BanyanDB
-            config: test/e2e-v2/cases/profiling/ebpf/oncpu/banyandb/e2e.yaml
           - name: Event BanyanDB
             config: test/e2e-v2/cases/event/banyandb/e2e.yaml
           - name: BanyanDB Cluster Mode

.gitignore

@@ -54,6 +54,7 @@ target
 # mock files
 *mock.go
 *mock_test.go
+gomock_reflect*
 
 # snky cache
 .dccache

CHANGES.md

@@ -41,6 +41,7 @@ Release Notes.
 ### Chores
 
 - Fix metrics system typo.
+- Bump up OAP in CI to 6d262cce62e156bd197177abb3640ea65bb2d38e.
 
 ### CVEs
 

api/proto/banyandb/measure/v1/rpc.proto

@@ -38,5 +38,10 @@ service MeasureService {
   }
 
   rpc Write(stream WriteRequest) returns (stream WriteResponse);
-  rpc TopN(TopNRequest) returns (TopNResponse);
+  rpc TopN(TopNRequest) returns (TopNResponse) {
+    option (google.api.http) = {
+      post: "/v1/measure/topn"
+      body: "*"
+    };
+  }
 }

banyand/measure/measure.go

@@ -51,14 +51,15 @@ type option struct {
 
 type measure struct {
 	databaseSupplier   schema.Supplier
+	indexTagMap        map[string]struct{}
 	l                  *logger.Logger
 	schema             *databasev1.Measure
 	processorManager   *topNProcessorManager
+	fieldIndexLocation partition.FieldIndexLocation
 	name               string
 	group              string
-	indexRules         []*databasev1.IndexRule
 	indexRuleLocators  partition.IndexRuleLocator
-	fieldIndexLocation partition.FieldIndexLocation
+	indexRules         []*databasev1.IndexRule
 	topNAggregations   []*databasev1.TopNAggregation
 	interval           time.Duration
 	shardNum           uint32
@@ -104,7 +105,12 @@ func (s *measure) parseSpec() (err error) {
 		s.interval, err = timestamp.ParseDuration(s.schema.Interval)
 	}
 	s.indexRuleLocators, s.fieldIndexLocation = partition.ParseIndexRuleLocators(s.schema.GetEntity(), s.schema.GetTagFamilies(), s.indexRules, s.schema.IndexMode)
-
+	s.indexTagMap = make(map[string]struct{})
+	for j := range s.indexRules {
+		for k := range s.indexRules[j].Tags {
+			s.indexTagMap[s.indexRules[j].Tags[k]] = struct{}{}
+		}
+	}
 	return err
 }
 

banyand/measure/write.go

@@ -284,6 +284,9 @@ func (w *writeCallback) appendEntityTagsToIndexFields(fields []index.Field, stm
 	f.NoSort = true
 	fields = append(fields, f)
 	for i := range stm.schema.Entity.TagNames {
+		if _, exists := stm.indexTagMap[stm.schema.Entity.TagNames[i]]; exists {
+			continue
+		}
 		tagName := stm.schema.Entity.TagNames[i]
 		var t *databasev1.TagSpec
 		for j := range stm.schema.TagFamilies {

pkg/index/inverted/query.go

@@ -197,14 +197,14 @@ func buildIndexModeCriteria(criteria *modelv1.Criteria, schema logical.Schema, e
 		if err != nil {
 			return nil, err
 		}
-		if _, ok := entityDict[cond.Name]; ok {
-			fk := index.FieldKey{TagName: index.IndexModeEntityTagPrefix + cond.Name}
-			return parseConditionToQuery(cond, nil, expr, fk.Marshal())
-		}
 		if ok, indexRule := schema.IndexDefined(cond.Name); ok {
 			fk := index.FieldKey{IndexRuleID: indexRule.Metadata.Id}
 			return parseConditionToQuery(cond, indexRule, expr, fk.Marshal())
 		}
+		if _, ok := entityDict[cond.Name]; ok {
+			fk := index.FieldKey{TagName: index.IndexModeEntityTagPrefix + cond.Name}
+			return parseConditionToQuery(cond, nil, expr, fk.Marshal())
+		}
 		return nil, errors.Wrapf(logical.ErrUnsupportedConditionOp, "mandatory index rule conf:%s", cond)
 	case *modelv1.Criteria_Le:
 		le := criteria.GetLe()
@@ -301,6 +301,9 @@ func parseConditionToQuery(cond *modelv1.Condition, indexRule *databasev1.IndexR
 		node := newTermNode(str, indexRule)
 		return &queryNode{query, node}, nil
 	case modelv1.Condition_BINARY_OP_MATCH:
+		if indexRule == nil {
+			return nil, errors.WithMessagef(logical.ErrUnsupportedConditionOp, "index rule is mandatory for match operation: %s", cond)
+		}
 		bb := expr.Bytes()
 		if len(bb) != 1 {
 			return nil, errors.WithMessagef(logical.ErrUnsupportedConditionOp, "don't support multiple or null value: %s", cond)

test/docker/base-compose.yml

@@ -67,9 +67,6 @@ services:
       - 5005
     environment:
       SW_STORAGE_BANYANDB_TARGETS: banyandb:17912
-      SW_CONFIG_ETCD_PERIOD: 1
-      SW_CONFIG_ETCD_ENDPOINTS: http://etcd:2379
-      SW_CLUSTER_ETCD_ENDPOINTS: http://etcd:2379
     healthcheck:
       test: ["CMD", "bash", "-c", "cat < /dev/null > /dev/tcp/127.0.0.1/11800"]
       interval: 5s
@@ -98,6 +95,7 @@ services:
       SW_AGENT_INSTANCE_NAME: provider1
       SW_AGENT_COLLECTOR_GET_PROFILE_TASK_INTERVAL: 1
       SW_AGENT_COLLECTOR_GET_AGENT_DYNAMIC_CONFIG_INTERVAL: 1
+      SW_METER_ACTIVE: 'false'
     healthcheck:
       test: ["CMD", "bash", "-c", "cat < /dev/null > /dev/tcp/127.0.0.1/9090"]
       interval: 5s
@@ -120,6 +118,7 @@ services:
       SW_AGENT_INSTANCE_NAME: consumer1
       SW_AGENT_COLLECTOR_GET_PROFILE_TASK_INTERVAL: 1
       SW_AGENT_COLLECTOR_GET_AGENT_DYNAMIC_CONFIG_INTERVAL: 1
+      SW_METER_ACTIVE: 'false'
     healthcheck:
       test: ["CMD", "bash", "-c", "cat < /dev/null > /dev/tcp/127.0.0.1/9092"]
       interval: 5s

test/e2e-v2/cases/cluster/storage-cases.yaml

@@ -160,3 +160,19 @@ cases:
           | yq e '.traces[0].traceids[0]' - \
       )
     expected: ../storage/expected/trace-users-detail.yml
+  # topN-OP-service Global
+  - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql metrics exec --expression="top_n(service_sla,3,des)/100"
+    expected: ../storage/expected/topN-OP-service.yml
+    # topN-OP-service Global with attrs
+  - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql metrics exec --expression="top_n(service_sla,3,des,attr0='GENERAL')/100"
+    expected: ../storage/expected/topN-OP-service.yml
+  - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql metrics exec --expression="top_n(service_sla,3,des,attr0!='Not_GENERAL')/100"
+    expected: ../storage/expected/topN-OP-service.yml
+  # topN-OP-endpoint Global
+  - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql metrics exec --expression="top_n(endpoint_sla,3,des)/100"
+    expected: ../storage/expected/topN-OP-endpoint.yml
+    # topN-OP-endpoint Global with attrs
+  - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql metrics exec --expression="top_n(endpoint_sla,3,des,attr0='GENERAL')/100"
+    expected: ../storage/expected/topN-OP-endpoint.yml
+  - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql metrics exec --expression="top_n(endpoint_sla,3,des,attr0!='Not_GENERAL')/100"
+    expected: ../storage/expected/topN-OP-endpoint.yml