Merge pull request #17785 from siyuanfoundation/flaky

testgrid: print out all failed tests for visibility.

scripts/measure-testgrid-flakiness.sh

@@ -10,6 +10,12 @@ then
 fi
 
 pushd ./tools/testgrid-analysis
-go run main.go flaky --create-issue --dashboard=sig-etcd-periodics --tab=ci-etcd-e2e-amd64
-go run main.go flaky --create-issue --dashboard=sig-etcd-periodics --tab=ci-etcd-unit-test-amd64
+# ci-etcd-e2e-amd64 and ci-etcd-unit-test-amd64 runs 6 times a day. Keeping a rolling window of 14 days.
+go run main.go flaky --create-issue --dashboard=sig-etcd-periodics --tab=ci-etcd-e2e-amd64 --max-days=14
+go run main.go flaky --create-issue --dashboard=sig-etcd-periodics --tab=ci-etcd-unit-test-amd64 --max-days=14
+
+# do not create issues for presubmit tests
+go run main.go flaky --dashboard=sig-etcd-presubmits --tab=pull-etcd-e2e-amd64
+go run main.go flaky --dashboard=sig-etcd-presubmits --tab=pull-etcd-unit-test
+
 popd

tools/testgrid-analysis/cmd/data.go

@@ -20,6 +20,7 @@ import (
 	"net/http"
 	"os"
 	"strings"
+	"time"
 
 	apipb "github.com/GoogleCloudPlatform/testgrid/pb/api/v1"
 	statuspb "github.com/GoogleCloudPlatform/testgrid/pb/test_status"
@@ -77,6 +78,7 @@ func processRow(dashboard, tab string, row *apipb.ListRowsResponse_Row, allTests
 	if !strings.HasPrefix(row.Name, "go.etcd.io") {
 		return &t
 	}
+	earliestTimeToConsider := time.Now().AddDate(0, 0, -1*maxDays)
 	total := 0
 	failed := 0
 	logs := []string{}
@@ -89,13 +91,19 @@ func processRow(dashboard, tab string, row *apipb.ListRowsResponse_Row, allTests
 			}
 			continue
 		}
+		header := headers[i]
+		if maxDays > 0 && header.Started.AsTime().Before(earliestTimeToConsider) {
+			continue
+		}
 		total += 1
 		if _, ok := failureTestStatusesInt[cell.Result]; ok {
 			failed += 1
-			header := headers[i]
 			// markdown table format of | commit | log |
 			logs = append(logs, fmt.Sprintf("| %s | %s | https://prow.k8s.io/view/gs/kubernetes-jenkins/logs/%s/%s |", strings.Join(header.Extra, ","), header.Started.AsTime().String(), tab, header.Build))
 		}
+		if maxRuns > 0 && total >= maxRuns {
+			break
+		}
 	}
 	t.FailedRuns = failed
 	t.TotalRuns = total
@@ -106,6 +114,7 @@ func processRow(dashboard, tab string, row *apipb.ListRowsResponse_Row, allTests
 		t.IssueBody = fmt.Sprintf("## %s Test: %s \nTest failed %.1f%% (%d/%d) of the time\n\nfailure logs are:\n| commit | started | log |\n| --- | --- | --- |\n%s\n",
 			dashboardUrl, t.FullName, t.FailureRate*100, t.FailedRuns, t.TotalRuns, strings.Join(t.FailureLogs, "\n"))
 		t.IssueBody += "\nPlease follow the [instructions in the contributing guide](https://github.com/etcd-io/etcd/blob/main/CONTRIBUTING.md#check-for-flaky-tests) to reproduce the issue.\n"
+		fmt.Printf("%s failed %.1f%% (%d/%d) of the time\n", t.FullName, t.FailureRate*100, t.FailedRuns, t.TotalRuns)
 	}
 	return &t
 }

tools/testgrid-analysis/cmd/flaky.go

@@ -31,6 +31,8 @@ var flakyCmd = &cobra.Command{
 var (
 	flakyThreshold    float32
 	minRuns           int
+	maxRuns           int
+	maxDays           int
 	createGithubIssue bool
 	githubOwner       string
 	githubRepo        string
@@ -44,6 +46,8 @@ func init() {
 	flakyCmd.Flags().BoolVar(&createGithubIssue, "create-issue", false, "create Github issue for each flaky test")
 	flakyCmd.Flags().Float32Var(&flakyThreshold, "flaky-threshold", 0.1, "fraction threshold of test failures for a test to be considered flaky")
 	flakyCmd.Flags().IntVar(&minRuns, "min-runs", 20, "minimum test runs for a test to be included in flaky analysis")
+	flakyCmd.Flags().IntVar(&maxRuns, "max-runs", 0, "maximum test runs for a test to be included in flaky analysis, 0 to include all")
+	flakyCmd.Flags().IntVar(&maxDays, "max-days", 0, "maximum days of results before today to be included in flaky analysis, 0 to include all")
 	flakyCmd.Flags().StringVar(&githubOwner, "github-owner", "etcd-io", "the github organization to create the issue for")
 	flakyCmd.Flags().StringVar(&githubRepo, "github-repo", "etcd", "the github repo to create the issue for")
 }
@@ -59,7 +63,7 @@ func flakyFunc(cmd *cobra.Command, args []string) {
 		}
 	}
 	fmt.Println(lineSep)
-	fmt.Printf("Detected total %d flaky tests for %s#%s\n", len(flakyTests), dashboard, tab)
+	fmt.Printf("Detected total %d flaky tests above the %.0f%% threshold for %s#%s\n", len(flakyTests), flakyThreshold*100, dashboard, tab)
 	fmt.Println(lineSep)
 	if len(flakyTests) == 0 {
 		return