Merge pull request #426 from buildkite/te-5134-add-tag-filters-option-to-bktec

[TE-5134] Add --tag-filters option to filter tests by markers

Author: gchan

.buildkite/Dockerfile

@@ -12,7 +12,7 @@ RUN gem install rspec cucumber base64
 RUN gem install bigdecimal -v 3.2.0
 RUN yarn global add jest
 RUN pip install pytest
-RUN pip install buildkite-test-collector==0.2.0
+RUN pip install buildkite-test-collector>=1.3.0
 RUN curl --proto '=https' --tlsv1.2 -fsSL https://static.pantsbuild.org/setup/get-pants.sh | bash -s -- --bin-dir /usr/local/bin
 
 # Install curl, download bktec binary, make it executable, place it, and cleanup

.tool-versions

@@ -1,3 +1,4 @@
 golang 1.25.2
 nodejs 24.9.0
 python 3.13.2
+uv     0.9.26

bin/setup

@@ -9,14 +9,14 @@ go install github.com/pact-foundation/pact-go/v2
 go install gotest.tools/[email protected]
 
 # Check if asdf is installed and being used for Go
-if command -v asdf &> /dev/null && asdf current golang &> /dev/null; then
+if command -v asdf &>/dev/null && asdf current golang &>/dev/null; then
   echo "🔄 Reshimming asdf golang..."
   asdf reshim golang
 fi
 
 # download and install the required libraries.
 # TODO if pact-go check return non- zero then install it
-if ! pact-go check &> /dev/null; then
+if ! pact-go check &>/dev/null; then
   echo "🔄 Installing pact-go dependencies..."
   sudo pact-go -l DEBUG install
 else
@@ -28,8 +28,7 @@ echo "🛠️ Installing dependencies for sample projects..."
 pushd ./internal/runner/testdata || exit 1
 # if yarn is available, use it to install dependencies
 # otherwise, use npm
-if command -v yarn &> /dev/null
-then
+if command -v yarn &>/dev/null; then
   yarn install
 else
   npm install
@@ -68,7 +67,7 @@ else
   python -m venv .venv && source .venv/bin/activate
 fi
 pip install pytest
-pip install buildkite-test-collector==0.2.0
+pip install "buildkite-test-collector>=1.3.0"
 curl --proto '=https' --tlsv1.2 -fsSL https://static.pantsbuild.org/setup/get-pants.sh | bash
 
 echo "💖 Everything is fantastic!"

cli.go

@@ -115,12 +115,19 @@ var filesFlag = &cli.StringFlag{
 	Sources:  cli.EnvVars("BUILDKITE_TEST_ENGINE_FILES"),
 }
 
-var testCommandFlag = &cli.StringFlag{
-	Name:        "test-command",
+var tagFiltersFlag = &cli.StringFlag{
+	Name:        "tag-filters",
 	Category:    "TEST RUNNER",
-	Usage:       "Test command",
-	Sources:     cli.EnvVars("BUILDKITE_TEST_ENGINE_TEST_CMD"),
-	Destination: &cfg.TestCommand,
+	Usage:       "Tag filters to apply when selecting tests to run (currently only Pytest is supported)",
+	Sources:     cli.EnvVars("BUILDKITE_TEST_ENGINE_TAG_FILTERS"),
+	Destination: &cfg.TagFilters,
+}
+
+var testCommandFlag = &cli.StringFlag{
+	Name:     "test-command",
+	Category: "TEST RUNNER",
+	Usage:    "Test command",
+	Sources:  cli.EnvVars("BUILDKITE_TEST_ENGINE_TEST_CMD"),
 }
 
 var testFilePatternFlag = &cli.StringFlag{
@@ -267,6 +274,7 @@ var cliCommand = &cli.Command{
 			Action: run,
 			Flags: []cli.Flag{
 				filesFlag,
+				tagFiltersFlag,
 				planIdentifierFlag,
 				// Build Environment Flags
 				organizationSlugFlag,
@@ -304,6 +312,7 @@ var cliCommand = &cli.Command{
 				// we will remove these in future iterations.
 
 				filesFlag,
+				tagFiltersFlag,
 				// Dynamic Parallelism Flags
 				maxParallelismFlag,
 				targetTimeFlag,

go.mod

@@ -1,6 +1,6 @@
 module github.com/buildkite/test-engine-client
 
-go 1.24
+go 1.24.0
 
 toolchain go1.24.1
 
@@ -16,6 +16,7 @@ require (
 	github.com/pact-foundation/pact-go/v2 v2.4.2
 	github.com/stretchr/testify v1.11.1
 	github.com/urfave/cli/v3 v3.6.2
+	golang.org/x/mod v0.32.0
 )
 
 require (

go.sum

@@ -61,6 +61,8 @@ go.opentelemetry.io/otel/sdk/metric v1.35.0 h1:1RriWBmCKgkeHEhM7a2uMjMUfP7MsOF5J
 go.opentelemetry.io/otel/sdk/metric v1.35.0/go.mod h1:is6XYCUMpcKi+ZsOvfluY5YstFnhW0BidkR+gL+qN+w=
 go.opentelemetry.io/otel/trace v1.35.0 h1:dPpEfJu1sDIqruz7BHFG3c7528f6ddfSWfFDVt/xgMs=
 go.opentelemetry.io/otel/trace v1.35.0/go.mod h1:WUk7DtFp1Aw2MkvqGdwiXYDZZNvA/1J8o6xRXLrIkyc=
+golang.org/x/mod v0.32.0 h1:9F4d3PHLljb6x//jOyokMv3eX+YDeepZSEo3mFJy93c=
+golang.org/x/mod v0.32.0/go.mod h1:SgipZ/3h2Ci89DlEtEXWUk/HteuRin+HHhN+WbNhguU=
 golang.org/x/net v0.38.0 h1:vRMAPTMaeGqVhG5QyLJHqNDwecKTomGeqbnfZyKlBI8=
 golang.org/x/net v0.38.0/go.mod h1:ivrbrMbzFq5J41QOQh0siUuly180yBYtLp+CKbEaFx8=
 golang.org/x/sys v0.33.0 h1:q3i8TbbEz+JRD9ywIRlyRAQbM0qF7hu24q3teo2hbuw=

internal/api/filter_tests.go

@@ -22,8 +22,15 @@ type FilteredTestResponse struct {
 	Tests []FilteredTest `json:"tests"`
 }
 
-// FilterTests filters tests from the server. It returns a list of tests that need to be split by example.
-// Currently, it only filters tests that are slow.
+// FilterTests fetches test files from the server. It returns a list of test files that
+// need to be split by example.
+//
+// Currently, it only fetches tests file that are slow and test files that have tests
+// marked for skipping.
+//
+// The splitByExample flag is passed through to the server, which is false will only
+// return test files that contain skipped tests, while true will also return slow test
+// files.
 func (c Client) FilterTests(ctx context.Context, suiteSlug string, params FilterTestsParams) ([]FilteredTest, error) {
 	url := fmt.Sprintf("%s/v2/analytics/organizations/%s/suites/%s/test_plan/filter_tests", c.ServerBaseUrl, c.OrganizationSlug, suiteSlug)
 
@@ -33,7 +40,6 @@ func (c Client) FilterTests(ctx context.Context, suiteSlug string, params Filter
 		URL:    url,
 		Body:   params,
 	}, &response)
-
 	if err != nil {
 		return []FilteredTest{}, err
 	}

internal/command/request_param.go

@@ -11,8 +11,13 @@ import (
 )
 
 // createRequestParam generates the parameters needed for a test plan request.
-// For runners other than "rspec", it constructs the test plan parameters with all test files.
-// For the "rspec" runner, it filters the test files through the Test Engine API and splits the filtered files into examples.
+//
+// For the Rspec, Cucumber and Pytest runner, it fetches test files through the Test Engine API
+// that are slow or contain skipped tests. These files are then split into examples
+// The remaining files are sent as is.
+//
+// If tag filtering is enabled, all files are split into examples to support filtering.
+// Currently only the Pytest runner supports tag filtering.
 func createRequestParam(ctx context.Context, cfg *config.Config, files []string, client api.Client, runner TestRunner) (api.TestPlanParams, error) {
 	testFiles := []plan.TestCase{}
 	for _, file := range files {
@@ -50,10 +55,21 @@ func createRequestParam(ctx context.Context, cfg *config.Config, files []string,
 		debug.Println("Splitting by example")
 	}
 
-	// The SplitByExample flag indicates whether to filter slow files for splitting by example.
-	// Regardless of the flag's state, the API will still filter other files that need to be split by example, such as those containing skipped tests.
-	// Therefore, we must filter and split files even when SplitByExample is disabled.
-	testParams, err := filterAndSplitFiles(ctx, cfg, client, testFiles, runner)
+	var testParams api.TestPlanParamsTest
+	var err error
+
+	// If tag filtering is enabled, we must split all files to allow to enable filtering.
+	// Tag filtering is currently only supported for pytest.
+	if cfg.TagFilters != "" && runner.Name() == "pytest" {
+		testParams, err = splitAllFiles(testFiles, runner)
+	} else {
+		// The SplitByExample flag indicates whether to split slow files into examples.
+		// Regardless of the flag's state, the API will still return other test files that need to
+		// be split by example, such as those containing skipped tests.
+		// Therefore, we must fetch and split files even when SplitByExample is disabled.
+		testParams, err = filterAndSplitFiles(ctx, cfg, client, testFiles, runner)
+	}
+
 	if err != nil {
 		return api.TestPlanParams{}, err
 	}
@@ -69,6 +85,26 @@ func createRequestParam(ctx context.Context, cfg *config.Config, files []string,
 	}, nil
 }
 
+// Splits all the test files into examples to support tag filtering.
+func splitAllFiles(files []plan.TestCase, runner TestRunner) (api.TestPlanParamsTest, error) {
+	debug.Printf("Splitting all %d files", len(files))
+	filePaths := make([]string, 0, len(files))
+	for _, file := range files {
+		filePaths = append(filePaths, file.Path)
+	}
+
+	examples, err := runner.GetExamples(filePaths)
+	if err != nil {
+		return api.TestPlanParamsTest{}, fmt.Errorf("get examples: %w", err)
+	}
+
+	debug.Printf("Got %d examples from all files", len(examples))
+
+	return api.TestPlanParamsTest{
+		Examples: examples,
+	}, nil
+}
+
 // filterAndSplitFiles filters the test files through the Test Engine API and splits the filtered files into examples.
 // It returns the test plan parameters with the examples from the filtered files and the remaining files.
 // An error is returned if there is a failure in any of the process.

internal/command/run_test.go

@@ -693,7 +693,6 @@ func TestCreateRequestParams(t *testing.T) {
 			TestCommand: "rspec",
 		},
 	})
-
 	if err != nil {
 		t.Errorf("createRequestParam() error = %v", err)
 	}
@@ -778,7 +777,6 @@ func TestCreateRequestParams_NonRSpec(t *testing.T) {
 			}
 
 			got, err := createRequestParam(context.Background(), &cfg, files, *client, r)
-
 			if err != nil {
 				t.Errorf("createRequestParam() error = %v", err)
 			}
@@ -838,7 +836,6 @@ func TestCreateRequestParams_PytestPants(t *testing.T) {
 		}
 
 		got, err := createRequestParam(context.Background(), &cfg, files, *client, runner)
-
 		if err != nil {
 			t.Errorf("createRequestParam() error = %v", err)
 		}
@@ -935,7 +932,6 @@ func TestCreateRequestParams_NoFilteredFiles(t *testing.T) {
 			TestCommand: "rspec",
 		},
 	})
-
 	if err != nil {
 		t.Errorf("createRequestParam() error = %v", err)
 	}
@@ -962,6 +958,122 @@ func TestCreateRequestParams_NoFilteredFiles(t *testing.T) {
 	}
 }
 
+func TestCreateRequestParams_WithTagFilters(t *testing.T) {
+	cfg := config.Config{
+		OrganizationSlug: "my-org",
+		SuiteSlug:        "my-suite",
+		Identifier:       "identifier",
+		Parallelism:      2,
+		Branch:           "main",
+		TestRunner:       "pytest",
+		TagFilters:       "team:frontend",
+	}
+
+	client := api.NewClient(api.ClientConfig{
+		ServerBaseUrl: "example.com",
+	})
+
+	files := []string{
+		"../runner/testdata/pytest/failed_test.py",
+		"../runner/testdata/pytest/test_sample.py",
+		"../runner/testdata/pytest/spells/test_expelliarmus.py",
+	}
+
+	got, err := createRequestParam(context.Background(), &cfg, files, *client, runner.Pytest{
+		RunnerConfig: runner.RunnerConfig{
+			TestCommand: "pytest",
+			TagFilters:  "team:frontend",
+		},
+	})
+	if err != nil {
+		t.Errorf("createRequestParam() error = %v", err)
+	}
+
+	want := api.TestPlanParams{
+		Identifier:  "identifier",
+		Parallelism: 2,
+		Branch:      "main",
+		Runner:      "pytest",
+		Tests: api.TestPlanParamsTest{
+			Examples: []plan.TestCase{
+				{
+					Format:     "example",
+					Identifier: "runner/testdata/pytest/test_sample.py::test_happy",
+					Name:       "test_happy",
+					Path:       "runner/testdata/pytest/test_sample.py::test_happy",
+					Scope:      "runner/testdata/pytest/test_sample.py",
+				},
+				{
+					Format:     "example",
+					Identifier: "runner/testdata/pytest/spells/test_expelliarmus.py::TestExpelliarmus::test_knocks_wand_out",
+					Name:       "test_knocks_wand_out",
+					Path:       "runner/testdata/pytest/spells/test_expelliarmus.py::TestExpelliarmus::test_knocks_wand_out",
+					Scope:      "runner/testdata/pytest/spells/test_expelliarmus.py::TestExpelliarmus",
+				},
+			},
+		},
+	}
+
+	if diff := cmp.Diff(got, want); diff != "" {
+		t.Errorf("createRequestParam() diff (-got +want):\n%s", diff)
+	}
+}
+
+func TestCreateRequestParams_WithTagFilters_NonPytest(t *testing.T) {
+	svr := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		fmt.Fprint(w, `
+{
+	"tests": []
+}`)
+	}))
+	defer svr.Close()
+
+	cfg := config.Config{
+		OrganizationSlug: "my-org",
+		SuiteSlug:        "my-suite",
+		Identifier:       "identifier",
+		Parallelism:      2,
+		Branch:           "main",
+		TestRunner:       "rspec",
+		TagFilters:       "team:frontend",
+	}
+
+	client := api.NewClient(api.ClientConfig{
+		ServerBaseUrl: svr.URL,
+	})
+
+	files := []string{
+		"testdata/rspec/spec/fruits/apple_spec.rb",
+		"testdata/rspec/spec/fruits/banana_spec.rb",
+	}
+
+	got, err := createRequestParam(context.Background(), &cfg, files, *client, runner.Rspec{
+		RunnerConfig: runner.RunnerConfig{
+			TestCommand: "rspec",
+		},
+	})
+	if err != nil {
+		t.Errorf("createRequestParam() error = %v", err)
+	}
+
+	want := api.TestPlanParams{
+		Identifier:  "identifier",
+		Parallelism: 2,
+		Branch:      "main",
+		Runner:      "rspec",
+		Tests: api.TestPlanParamsTest{
+			Files: []plan.TestCase{
+				{Path: "testdata/rspec/spec/fruits/apple_spec.rb"},
+				{Path: "testdata/rspec/spec/fruits/banana_spec.rb"},
+			},
+		},
+	}
+
+	if diff := cmp.Diff(got, want); diff != "" {
+		t.Errorf("createRequestParam() diff (-got +want):\n%s", diff)
+	}
+}
+
 func TestSendMetadata(t *testing.T) {
 	originalVersion := version.Version
 	version.Version = "0.1.0"
@@ -1019,7 +1131,6 @@ func TestSendMetadata(t *testing.T) {
 		} else {
 			w.WriteHeader(http.StatusOK)
 		}
-
 	}))
 	defer svr.Close()
 
@@ -1070,7 +1181,6 @@ func TestRunTestsWithRetry_NoTestCases_Success(t *testing.T) {
 	failOnNoTests := false
 
 	testResult, err := runTestsWithRetry(testRunner, &testCases, maxRetries, []plan.TestCase{}, &timeline, true, failOnNoTests)
-
 	if err != nil {
 		t.Errorf("runTestsWithRetry(...) error = %v, want nil", err)
 	}

internal/config/config.go

@@ -52,6 +52,8 @@ type Config struct {
 	DebugEnabled bool `json:"BUILDKITE_TEST_ENGINE_DEBUG_ENABLED"`
 	// FailOnNoTests causes the client to exit with an error if no tests are assigned to the node
 	FailOnNoTests bool `json:"BUILDKITE_TEST_ENGINE_FAIL_ON_NO_TESTS"`
+	// TagFilters filters test examples by execution tags.
+	TagFilters string `json:"BUILDKITE_TEST_ENGINE_TAG_FILTERS"`
 	// errs is a map of environment variables name and the validation errors associated with them.
 	errs InvalidConfigError
 }