Load/Chaos codegen (#2218)

* WIP: load/chaos codegen

* remove old havoc

* update go mods

* Remove replace in generated go mod

* remove old chaos templates, switch to wasp

* fix lint and remove havoc linter CI

* more cases, add default params, error when no uniq labels

* correct k8s labelling

* use default wasp test prefix

* do not remove chaos experiments by default

* remove replace

* bump CVE packages

* changesets

* chaos&load codegen

* fix copilot review

Author: skudasov

.github/workflows/linters.yml

@@ -13,7 +13,6 @@ jobs:
           - parrot
           - wasp
           - seth
-          - havoc
           - k8s-test-runner
           - lib
           - tools/workflowresultparser

book/src/SUMMARY.md

@@ -12,6 +12,7 @@
   - [CLI](./framework/cli.md)
   - [Configuration](./framework/configuration.md)
   - [Debugging Tests](framework/components/debug.md)
+  - [Generating Tests](framework/generate.md)
   - [Creating your own components](./developing/developing_components.md)
   - [Exposing Components](framework/components/state.md)
   - [Asserting Logs](./developing/asserting_logs.md)

book/src/framework/generate.md

@@ -0,0 +1,7 @@
+# Generating Tests
+
+Some types of tests may require a lot of boilerplate and configurability, for example load and chaos tests.
+
+We provide code generation tool that help you to start with best practes in such cases.
+
+All generators have `-h` or `--help` flag, please read the docs!

framework/.changeset/v0.11.8.md

@@ -1 +1 @@
-- Usnet Docker Build Ctx and Filepath if `CTF_*_IMAGE` env vars are used
+- Usnet Docker Build Ctx and Filepath if `CTF_*_IMAGE` env vars are used

framework/.changeset/v0.11.9.md

@@ -0,0 +1 @@
+- Chaos and Load codegen

framework/cmd/.gitignore

@@ -2,3 +2,5 @@ blockscout/services/blockscout-db-data
 blockscout/services/logs
 blockscout/services/redis-data
 blockscout/services/stats-db-data
+wasp-self-test/
+wasp-test/

framework/cmd/main.go

@@ -11,6 +11,7 @@ import (
 	"github.com/urfave/cli/v2"
 
 	"github.com/smartcontractkit/chainlink-testing-framework/framework"
+	"github.com/smartcontractkit/chainlink-testing-framework/wasp"
 )
 
 func main() {
@@ -19,6 +20,127 @@ func main() {
 		Usage:     "Chainlink Testing Framework CLI",
 		UsageText: "'ctf' is a useful utility that can:\n- clean up test docker containers\n- modify test files\n- create a local observability stack with Grafana/Loki/Pyroscope",
 		Commands: []*cli.Command{
+			{
+				Name:    "gen",
+				Aliases: []string{"g"},
+				Usage:   "Generates various test templates",
+				Subcommands: []*cli.Command{
+					{
+						Name:    "load",
+						Aliases: []string{"l"},
+						Usage:   "Generates a load/chaos test template for Kubernetes namespace",
+						Description: `Scans a Kubernetes namespace and generates load testing templates for discovered services.
+
+Prerequisites:
+
+	Connect to K8s and don't forget to switch context first:
+		kubectl config use-context <your_ctx>
+	By default test sends data to a local CTF stack, see //TODO comments to change that, spin up the stack:
+		ctf obs up
+
+Usage:
+
+	Generate basic kill/latency tests:
+		ctf gen k8s-load my-namespace
+	With workload:
+		ctf gen k8s-load -w my-namespace
+	With workload and name:
+		ctf gen k8s-load -w -n TestSomething my-namespace
+
+Be aware that any TODO requires your attention before your run the final test!
+`,
+						ArgsUsage: "--workload --name $name --output-dir $dir --module $go_mod_name [NAMESPACE]",
+						Flags: []cli.Flag{
+							&cli.StringFlag{
+								Name:    "name",
+								Aliases: []string{"n"},
+								Value:   "TestGeneratedLoadChaos",
+								Usage:   "Test suite name",
+							},
+							&cli.StringFlag{
+								Name:    "output-dir",
+								Aliases: []string{"o"},
+								Value:   "wasp-test",
+								Usage:   "Output directory for generated files",
+							},
+							&cli.StringFlag{
+								Name:    "module",
+								Aliases: []string{"m"},
+								Value:   "github.com/smartcontractkit/chainlink-testing-framework/wasp-test",
+								Usage:   "Go module name for generated project",
+							},
+							&cli.BoolFlag{
+								Name:    "workload",
+								Aliases: []string{"w"},
+								Value:   false,
+								Usage:   "Include workload generation in tests",
+							},
+							&cli.StringFlag{
+								Name:    "pod-label-key",
+								Aliases: []string{"k"},
+								Value:   "app.kubernetes.io/instance",
+								Usage:   "Default unique pod key, read more here: https://kubernetes.io/docs/concepts/overview/working-with-objects/common-labels/",
+							},
+							&cli.StringFlag{
+								Name:    "latency-ms",
+								Aliases: []string{"l"},
+								Value:   "300",
+								Usage:   "Default latency for delay experiments in milliseconds",
+							},
+							&cli.StringFlag{
+								Name:    "jitter-ms",
+								Aliases: []string{"j"},
+								Value:   "100",
+								Usage:   "Default jitter for delay experiments in milliseconds",
+							},
+						},
+						Action: func(c *cli.Context) error {
+							if c.Args().Len() == 0 {
+								return fmt.Errorf("Kubernetes namespace argument is required")
+							}
+							ns := c.Args().First()
+							testSuiteName := c.String("name")
+							podLabelKey := c.String("pod-label-key")
+							latencyMs := c.Int("latency-ms")
+							jitterMs := c.Int("jitter-ms")
+							outputDir := c.String("output-dir")
+							moduleName := c.String("module")
+							includeWorkload := c.Bool("workload")
+							framework.L.Info().
+								Str("SuiteName", testSuiteName).
+								Str("OutputDir", outputDir).
+								Str("GoModuleName", moduleName).
+								Bool("Workload", includeWorkload).
+								Msg("Generating load&chaos test template")
+
+							k8sClient, err := wasp.NewK8s()
+							if err != nil {
+								return fmt.Errorf("failed to create K8s client")
+							}
+
+							cg, err := wasp.NewLoadTestGenBuilder(k8sClient, ns).
+								TestSuiteName(testSuiteName).
+								UniqPodLabelKey(podLabelKey).
+								Latency(latencyMs).
+								Jitter(jitterMs).
+								Workload(includeWorkload).
+								OutputDir(outputDir).
+								GoModName(moduleName).
+								Build()
+							if err != nil {
+								return fmt.Errorf("failed to create codegen: %w", err)
+							}
+							if err := cg.Read(); err != nil {
+								return fmt.Errorf("failed to scan namespace: %w", err)
+							}
+							if err := cg.Write(); err != nil {
+								return fmt.Errorf("failed to generate module: %w", err)
+							}
+							return nil
+						},
+					},
+				},
+			},
 			{
 				Name:    "config",
 				Aliases: []string{"c"},
@@ -206,7 +328,7 @@ func PrettyPrintTOML(inputFile string, outputFile string) error {
 	}
 
 	//nolint
-	err = os.WriteFile(outputFile, []byte(dumpData), 0644)
+	err = os.WriteFile(outputFile, []byte(dumpData), 0o644)
 	if err != nil {
 		return fmt.Errorf("error writing to output file: %v", err)
 	}

framework/components/dockercompose/go.mod

@@ -15,7 +15,7 @@ require (
 	github.com/smartcontractkit/freeport v0.1.2
 	github.com/testcontainers/testcontainers-go v0.37.0
 	github.com/testcontainers/testcontainers-go/modules/compose v0.37.0
-	golang.org/x/oauth2 v0.25.0
+	golang.org/x/oauth2 v0.32.0
 )
 
 require (
@@ -80,17 +80,17 @@ require (
 	github.com/go-logr/logr v1.4.2 // indirect
 	github.com/go-logr/stdr v1.2.2 // indirect
 	github.com/go-ole/go-ole v1.3.0 // indirect
-	github.com/go-openapi/jsonpointer v0.19.6 // indirect
-	github.com/go-openapi/jsonreference v0.20.2 // indirect
-	github.com/go-openapi/swag v0.22.4 // indirect
+	github.com/go-openapi/jsonpointer v0.21.0 // indirect
+	github.com/go-openapi/jsonreference v0.21.0 // indirect
+	github.com/go-openapi/swag v0.23.0 // indirect
 	github.com/go-playground/locales v0.14.1 // indirect
 	github.com/go-playground/universal-translator v0.18.1 // indirect
 	github.com/go-playground/validator/v10 v10.22.1 // indirect
-	github.com/go-resty/resty/v2 v2.15.3 // indirect
+	github.com/go-resty/resty/v2 v2.16.3 // indirect
 	github.com/go-viper/mapstructure/v2 v2.0.0 // indirect
 	github.com/gofrs/flock v0.12.1 // indirect
 	github.com/gogo/protobuf v1.3.2 // indirect
-	github.com/golang-jwt/jwt/v5 v5.2.2 // indirect
+	github.com/golang-jwt/jwt/v5 v5.3.0 // indirect
 	github.com/golang/protobuf v1.5.4 // indirect
 	github.com/google/gnostic-models v0.6.8 // indirect
 	github.com/google/go-cmp v0.7.0 // indirect
@@ -99,13 +99,12 @@ require (
 	github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 // indirect
 	github.com/google/uuid v1.6.0 // indirect
 	github.com/gorilla/mux v1.8.1 // indirect
-	github.com/gorilla/websocket v1.5.1 // indirect
+	github.com/gorilla/websocket v1.5.3 // indirect
 	github.com/grpc-ecosystem/grpc-gateway/v2 v2.22.0 // indirect
 	github.com/hashicorp/errwrap v1.1.0 // indirect
 	github.com/hashicorp/go-cleanhttp v0.5.2 // indirect
 	github.com/hashicorp/go-multierror v1.1.1 // indirect
 	github.com/hashicorp/go-version v1.7.0 // indirect
-	github.com/imdario/mergo v0.3.16 // indirect
 	github.com/in-toto/in-toto-golang v0.5.0 // indirect
 	github.com/inconshreveable/mousetrap v1.1.0 // indirect
 	github.com/inhies/go-bytesize v0.0.0-20220417184213-4913239db9cf // indirect
@@ -115,7 +114,7 @@ require (
 	github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 // indirect
 	github.com/klauspost/compress v1.18.0 // indirect
 	github.com/leodido/go-urn v1.4.0 // indirect
-	github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 // indirect
+	github.com/lufia/plan9stats v0.0.0-20240226150601-1dcf7310316a // indirect
 	github.com/magiconair/properties v1.8.10 // indirect
 	github.com/mailru/easyjson v0.7.7 // indirect
 	github.com/mattn/go-colorable v0.1.13 // indirect
@@ -125,13 +124,13 @@ require (
 	github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b // indirect
 	github.com/miekg/pkcs11 v1.1.1 // indirect
 	github.com/mitchellh/hashstructure/v2 v2.0.2 // indirect
-	github.com/mitchellh/mapstructure v1.5.0 // indirect
+	github.com/mitchellh/mapstructure v1.5.1-0.20220423185008-bf980b35cac4 // indirect
 	github.com/moby/buildkit v0.20.1 // indirect
 	github.com/moby/docker-image-spec v1.3.1 // indirect
 	github.com/moby/go-archive v0.1.0 // indirect
 	github.com/moby/locker v1.0.1 // indirect
 	github.com/moby/patternmatcher v0.6.0 // indirect
-	github.com/moby/spdystream v0.4.0 // indirect
+	github.com/moby/spdystream v0.5.0 // indirect
 	github.com/moby/sys/atomicwriter v0.1.0 // indirect
 	github.com/moby/sys/capability v0.4.0 // indirect
 	github.com/moby/sys/mountinfo v0.7.2 // indirect
@@ -152,10 +151,10 @@ require (
 	github.com/pelletier/go-toml/v2 v2.2.3 // indirect
 	github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 // indirect
 	github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
-	github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c // indirect
-	github.com/prometheus/client_golang v1.20.5 // indirect
+	github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55 // indirect
+	github.com/prometheus/client_golang v1.21.0-rc.0 // indirect
 	github.com/prometheus/client_model v0.6.1 // indirect
-	github.com/prometheus/common v0.55.0 // indirect
+	github.com/prometheus/common v0.62.0 // indirect
 	github.com/prometheus/procfs v0.15.1 // indirect
 	github.com/r3labs/sse v0.0.0-20210224172625-26fe804710bc // indirect
 	github.com/rivo/uniseg v0.4.7 // indirect
@@ -171,8 +170,8 @@ require (
 	github.com/stretchr/testify v1.10.0 // indirect
 	github.com/theupdateframework/notary v0.7.0 // indirect
 	github.com/tilt-dev/fsnotify v1.4.8-0.20220602155310-fff9c274a375 // indirect
-	github.com/tklauser/go-sysconf v0.3.12 // indirect
-	github.com/tklauser/numcpus v0.6.1 // indirect
+	github.com/tklauser/go-sysconf v0.3.13 // indirect
+	github.com/tklauser/numcpus v0.7.0 // indirect
 	github.com/tonistiigi/dchapes-mode v0.0.0-20241001053921-ca0759fec205 // indirect
 	github.com/tonistiigi/fsutil v0.0.0-20250113203817-b14e27f4135a // indirect
 	github.com/tonistiigi/go-csvvalue v0.0.0-20240710180619-ddb21b71c0b4 // indirect
@@ -187,8 +186,8 @@ require (
 	github.com/zclconf/go-cty v1.16.0 // indirect
 	go.opentelemetry.io/auto/sdk v1.1.0 // indirect
 	go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.56.0 // indirect
-	go.opentelemetry.io/contrib/instrumentation/net/http/httptrace/otelhttptrace v0.56.0 // indirect
-	go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.56.0 // indirect
+	go.opentelemetry.io/contrib/instrumentation/net/http/httptrace/otelhttptrace v0.59.0 // indirect
+	go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.59.0 // indirect
 	go.opentelemetry.io/otel v1.35.0 // indirect
 	go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.31.0 // indirect
 	go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v1.31.0 // indirect
@@ -203,30 +202,30 @@ require (
 	go.uber.org/mock v0.5.0 // indirect
 	go.uber.org/multierr v1.11.0 // indirect
 	golang.org/x/crypto v0.39.0 // indirect
-	golang.org/x/exp v0.0.0-20241108190413-2d47ceb2692f // indirect
+	golang.org/x/exp v0.0.0-20250106191152-7588d65b2ba8 // indirect
 	golang.org/x/net v0.40.0 // indirect
 	golang.org/x/sync v0.15.0 // indirect
 	golang.org/x/sys v0.33.0 // indirect
 	golang.org/x/term v0.32.0 // indirect
 	golang.org/x/text v0.26.0 // indirect
-	golang.org/x/time v0.6.0 // indirect
-	google.golang.org/genproto/googleapis/api v0.0.0-20250106144421-5f5ef82da422 // indirect
-	google.golang.org/genproto/googleapis/rpc v0.0.0-20250115164207-1a7da9e5054f // indirect
+	golang.org/x/time v0.10.0 // indirect
+	google.golang.org/genproto/googleapis/api v0.0.0-20250124145028-65684f501c47 // indirect
+	google.golang.org/genproto/googleapis/rpc v0.0.0-20250207221924-e9438ea467c6 // indirect
 	google.golang.org/grpc v1.71.0 // indirect
-	google.golang.org/protobuf v1.36.4 // indirect
+	google.golang.org/protobuf v1.36.5 // indirect
 	gopkg.in/cenkalti/backoff.v1 v1.1.0 // indirect
+	gopkg.in/evanphx/json-patch.v4 v4.12.0 // indirect
 	gopkg.in/inf.v0 v0.9.1 // indirect
 	gopkg.in/ini.v1 v1.67.0 // indirect
-	gopkg.in/yaml.v2 v2.4.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
-	k8s.io/api v0.31.2 // indirect
-	k8s.io/apimachinery v0.31.2 // indirect
-	k8s.io/client-go v0.31.2 // indirect
+	k8s.io/api v0.32.2 // indirect
+	k8s.io/apimachinery v0.32.2 // indirect
+	k8s.io/client-go v0.32.2 // indirect
 	k8s.io/klog/v2 v2.130.1 // indirect
-	k8s.io/kube-openapi v0.0.0-20240228011516-70dd3763d340 // indirect
-	k8s.io/utils v0.0.0-20240711033017-18e509b52bc8 // indirect
-	sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect
-	sigs.k8s.io/structured-merge-diff/v4 v4.4.1 // indirect
+	k8s.io/kube-openapi v0.0.0-20241105132330-32ad38e42d3f // indirect
+	k8s.io/utils v0.0.0-20241104163129-6fe5fd82f078 // indirect
+	sigs.k8s.io/json v0.0.0-20241010143419-9aa6b5e7a4b3 // indirect
+	sigs.k8s.io/structured-merge-diff/v4 v4.4.2 // indirect
 	sigs.k8s.io/yaml v1.4.0 // indirect
 	tags.cncf.io/container-device-interface v1.0.1 // indirect
 )