Merge pull request etcd-io#20604 from AwesomePatrol/add-tracing-to-robustness-tests

Add an option to collect traces from robustness tests

Author: serathius

tests/framework/e2e/cluster.go

@@ -390,6 +390,15 @@ func WithExtensiveMetrics() EPClusterOption {
 	return func(c *EtcdProcessClusterConfig) { c.ServerConfig.Metrics = "extensive" }
 }
 
+func WithEnableDistributedTracing(addr string) EPClusterOption {
+	return func(c *EtcdProcessClusterConfig) {
+		c.ServerConfig.EnableDistributedTracing = true
+		c.ServerConfig.DistributedTracingServiceName = "etcd"
+		c.ServerConfig.DistributedTracingAddress = addr
+		c.ServerConfig.DistributedTracingSamplingRatePerMillion = 1_000_000
+	}
+}
+
 // NewEtcdProcessCluster launches a new cluster from etcd processes, returning
 // a new EtcdProcessCluster once all nodes are ready to accept client requests.
 func NewEtcdProcessCluster(ctx context.Context, tb testing.TB, opts ...EPClusterOption) (*EtcdProcessCluster, error) {
@@ -585,6 +594,14 @@ func (cfg *EtcdProcessClusterConfig) EtcdServerProcessConfig(tb testing.TB, i in
 	if !cfg.ServerConfig.StrictReconfigCheck {
 		args = append(args, "--strict-reconfig-check=false")
 	}
+	if cfg.ServerConfig.EnableDistributedTracing {
+		args = append(args,
+			"--enable-distributed-tracing",
+			fmt.Sprintf("--distributed-tracing-address=%s", cfg.ServerConfig.DistributedTracingAddress),
+			fmt.Sprintf("--distributed-tracing-service-name=%s", cfg.ServerConfig.DistributedTracingServiceName),
+			fmt.Sprintf("--distributed-tracing-sampling-rate=%d", cfg.ServerConfig.DistributedTracingSamplingRatePerMillion),
+		)
+	}
 
 	var murl string
 	if cfg.MetricsURLScheme != "" {

tests/robustness/README.md

@@ -110,6 +110,7 @@ Etcd provides strict serializability for KV operations and eventual consistency
     * `EXPECT_DEBUG=true` - to get logs from the cluster.
     * `RESULTS_DIR` - to change the location where the results report will be saved.
     * `PERSIST_RESULTS` - to persist the results report of the test. By default this will not be persisted in the case of a successful run.
+    * `TRACING_SERVER_ADDR` - to export Open Telemetry traces from test runs to the collector running at given address, for example: `localhost:4317`
 
 ## Re-evaluate existing report
 

tests/robustness/coverage/README.md

@@ -80,39 +80,79 @@ cluster](https://kind.sigs.k8s.io/docs/user/quick-start/#installation):
 1. Exercise Kubernetes API. For example, build and run Conformance tests from
 Kubernetes repository (this usually takes 30-40m or will time out after 1 hour):
 
-```shell
-make WHAT="test/e2e/e2e.test"
-./_output/bin/e2e.test \
-  -context kind-kind-with-external-etcd \
-  -ginkgo.focus="\[sig-apps\].*Conformance" \
-  -num-nodes 2
-build/run.sh make test-cmd
-```
+   ```shell
+   make WHAT="test/e2e/e2e.test"
+   ./_output/bin/e2e.test \
+     -context kind-kind-with-external-etcd \
+     -ginkgo.focus="\[sig-apps\].*Conformance" \
+     -num-nodes 2
+   build/run.sh make test-cmd
+   ```
 
 1. Download traces and put them into `tests/robustness/coverage/testdata`
 directory in Etcd git repository:
 
-```shell
-curl -v --get --retry 10 --retry-connrefused -o testdata/demo_traces.json \
-  -H "Content-Type: application/json" \
-  --data-urlencode "query.start_time_min=$(date --date="5 days ago" -Ins)" \
-  --data-urlencode "query.start_time_max=$(date --date="2 minutes ago" -Ins)" \
-  --data-urlencode "query.service_name=etcd" \
-  "http://192.168.32.1:16686/api/v3/traces"
-```
+   ```shell
+   curl -v --get --retry 10 --retry-connrefused -o testdata/demo_traces.json \
+     -H "Content-Type: application/json" \
+     --data-urlencode "query.start_time_min=$(date --date="5 days ago" -Ins)" \
+     --data-urlencode "query.start_time_max=$(date --date="2 minutes ago" -Ins)" \
+     --data-urlencode "query.service_name=etcd" \
+     "http://192.168.32.1:16686/api/v3/traces"
+   ```
 
 1. Run Go test
 
-```shell
-go test -v -timeout 60s go.etcd.io/etcd/tests/v3/robustness/coverage
-```
+   ```shell
+   go test -v -timeout 60s go.etcd.io/etcd/tests/v3/robustness/coverage
+   ```
 
 1. Clean up the environment
 
-```shell
-kind delete cluster --name kind-with-external-etcd
-docker network rm kind-with-external-etcd
-```
+   ```shell
+   kind delete cluster --name kind-with-external-etcd
+   docker network rm kind-with-external-etcd
+   ```
+
+### Manual trace collection from robustness tests
+
+1. Run [Jaeger](https://www.jaegertracing.io/) container:
+
+   ```shell
+   docker run --rm --name jaeger \
+     -p 16686:16686 \
+     -p 4317:4317 \
+     jaegertracing/jaeger:2.6.0 --set=extensions.jaeger_storage.backends.some_storage.memory.max_traces=20000000
+   ```
+
+1. Run robustness tests. For example:
+
+   ```shell
+   env \
+     TRACING_SERVER_ADDR=localhost:4317 \
+     GO_TEST_FLAGS='--timeout 10m --count=1 -v --run "^TestRobustness.*/Kubernetes.*"' \
+     make test-robustness
+   ```
+
+1. Download traces and put them into `tests/robustness/coverage/testdata`
+directory in Etcd git repository:
+
+   ```shell
+   curl -v --get --retry 10 --retry-connrefused -o testdata/demo_traces.json \
+     -H "Content-Type: application/json" \
+     --data-urlencode "query.start_time_min=$(date --date="5 days ago" -Ins)" \
+     --data-urlencode "query.start_time_max=$(date -Ins)" \
+     --data-urlencode "query.service_name=etcd" \
+     "http://localhost:16686/api/v3/traces"
+   ```
+
+1. Run Go test
+
+   ```shell
+   go test -v -timeout 60s go.etcd.io/etcd/tests/v3/robustness/coverage
+   ```
+
+   It will show the coverage of Kubernetes-Etcd surface by robustness tests.:w
 
 ### Automated test execution
 

tests/robustness/coverage/coverage_test.go

@@ -353,14 +353,17 @@ func spansMap(t *testing.T, traces []*tracev1.ResourceSpans) map[string]*tracev1
 			}
 		}
 	}
+	if len(inApiserver) == 0 {
+		t.Logf("WARN: no records of traces from the apiserver")
+	}
 
 	// Map traces by their span ID.
 	spansByID := make(map[string]*tracev1.Span)
 	skipped := 0
 	for _, trace := range traces {
 		for _, scopeSpan := range trace.GetScopeSpans() {
 			for _, span := range scopeSpan.GetSpans() {
-				if !inApiserver[string(span.GetTraceId())] {
+				if len(inApiserver) > 0 && !inApiserver[string(span.GetTraceId())] {
 					skipped++
 					continue
 				}

tests/robustness/scenarios/scenarios.go

@@ -15,6 +15,7 @@
 package scenarios
 
 import (
+	"os"
 	"path/filepath"
 	"testing"
 	"time"
@@ -101,6 +102,10 @@ func Exploratory(_ *testing.T) []TestScenario {
 		e2e.WithWatchProcessNotifyInterval(100 * time.Millisecond),
 	}
 
+	if addr := os.Getenv("TRACING_SERVER_ADDR"); addr != "" {
+		baseOptions = append(baseOptions, e2e.WithEnableDistributedTracing(addr))
+	}
+
 	if e2e.CouldSetSnapshotCatchupEntries(e2e.BinPath.Etcd) {
 		baseOptions = append(baseOptions, options.WithSnapshotCatchUpEntries(100, etcdserver.DefaultSnapshotCatchUpEntries))
 	}