Merge branch 'main' into migrate-experiminal-enable-distributed-tracing

Signed-off-by: Omer Aplatony <[email protected]>

server/embed/config.go

@@ -145,6 +145,7 @@ var (
 		"experimental-compact-hash-check-time":           "compact-hash-check-time",
 		"experimental-corrupt-check-time":                "corrupt-check-time",
 		"experimental-compaction-batch-limit":            "compaction-batch-limit",
+		"experimental-watch-progress-notify-interval":    "watch-progress-notify-interval",
 	}
 )
 
@@ -403,8 +404,12 @@ type Config struct {
 	ExperimentalCompactionBatchLimit int `json:"experimental-compaction-batch-limit"`
 	CompactionBatchLimit             int `json:"compaction-batch-limit"`
 	// ExperimentalCompactionSleepInterval is the sleep interval between every etcd compaction loop.
-	ExperimentalCompactionSleepInterval     time.Duration `json:"experimental-compaction-sleep-interval"`
+	ExperimentalCompactionSleepInterval time.Duration `json:"experimental-compaction-sleep-interval"`
+	// ExperimentalWatchProgressNotifyInterval is the time duration of periodic watch progress notifications.
+	// Deprecated in v3.6 and will be decommissioned in v3.7.
+	// TODO: Delete in v3.7
 	ExperimentalWatchProgressNotifyInterval time.Duration `json:"experimental-watch-progress-notify-interval"`
+	WatchProgressNotifyInterval             time.Duration `json:"watch-progress-notify-interval"`
 	// ExperimentalWarningApplyDuration is the time duration after which a warning is generated if applying request
 	// takes more time than this value.
 	ExperimentalWarningApplyDuration time.Duration `json:"experimental-warning-apply-duration"`
@@ -821,7 +826,9 @@ func (cfg *Config) AddFlags(fs *flag.FlagSet) {
 	fs.IntVar(&cfg.ExperimentalCompactionBatchLimit, "experimental-compaction-batch-limit", cfg.ExperimentalCompactionBatchLimit, "Sets the maximum revisions deleted in each compaction batch. Deprecated in v3.6 and will be decommissioned in v3.7. Use --compaction-batch-limit instead.")
 	fs.IntVar(&cfg.CompactionBatchLimit, "compaction-batch-limit", cfg.CompactionBatchLimit, "Sets the maximum revisions deleted in each compaction batch.")
 	fs.DurationVar(&cfg.ExperimentalCompactionSleepInterval, "experimental-compaction-sleep-interval", cfg.ExperimentalCompactionSleepInterval, "Sets the sleep interval between each compaction batch.")
-	fs.DurationVar(&cfg.ExperimentalWatchProgressNotifyInterval, "experimental-watch-progress-notify-interval", cfg.ExperimentalWatchProgressNotifyInterval, "Duration of periodic watch progress notifications.")
+	// TODO: delete in v3.7
+	fs.DurationVar(&cfg.ExperimentalWatchProgressNotifyInterval, "experimental-watch-progress-notify-interval", cfg.ExperimentalWatchProgressNotifyInterval, "Duration of periodic watch progress notifications. Deprecated in v3.6 and will be decommissioned in v3.7. Use --watch-progress-notify-interval instead.")
+	fs.DurationVar(&cfg.WatchProgressNotifyInterval, "watch-progress-notify-interval", cfg.WatchProgressNotifyInterval, "Duration of periodic watch progress notifications.")
 	fs.DurationVar(&cfg.ExperimentalDowngradeCheckTime, "experimental-downgrade-check-time", cfg.ExperimentalDowngradeCheckTime, "Duration of time between two downgrade status checks.")
 	fs.DurationVar(&cfg.ExperimentalWarningApplyDuration, "experimental-warning-apply-duration", cfg.ExperimentalWarningApplyDuration, "Time duration after which a warning is generated if request takes more time.")
 	fs.DurationVar(&cfg.WarningUnaryRequestDuration, "warning-unary-request-duration", cfg.WarningUnaryRequestDuration, "Time duration after which a warning is generated if a unary request takes more time.")

server/etcdmain/config.go

@@ -72,6 +72,7 @@ var (
 		"experimental-distributed-tracing-sampling-rate": "--experimental-distributed-tracing-sampling-rate is deprecated in 3.6 and will be decommissioned in 3.7. Use --distributed-tracing-sampling-rate instead.",
 		"experimental-corrupt-check-time":                "--experimental-corrupt-check-time is deprecated in v3.6 and will be decommissioned in v3.7. Use '--corrupt-check-time' instead.",
 		"experimental-compaction-batch-limit":            "--experimental-compaction-batch-limit is deprecated in v3.6 and will be decommissioned in v3.7. Use '--compaction-batch-limit' instead.",
+		"experimental-watch-progress-notify-interval":    "--experimental-watch-progress-notify-interval is deprecated in v3.6 and will be decommissioned in v3.7. Use '--watch-progress-notify-interval' instead.",
 	}
 )
 
@@ -189,6 +190,10 @@ func (cfg *config) parse(arguments []string) error {
 		cfg.ec.CompactionBatchLimit = cfg.ec.ExperimentalCompactionBatchLimit
 	}
 
+	if cfg.ec.FlagsExplicitlySet["experimental-watch-progress-notify-interval"] {
+		cfg.ec.WatchProgressNotifyInterval = cfg.ec.ExperimentalWatchProgressNotifyInterval
+	}
+
 	// `V2Deprecation` (--v2-deprecation) is deprecated and scheduled for removal in v3.8. The default value is enforced, ignoring user input.
 	cfg.ec.V2Deprecation = cconfig.V2DeprDefault
 

server/etcdmain/config_test.go

@@ -26,6 +26,7 @@ import (
 	"time"
 
 	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
 	"sigs.k8s.io/yaml"
 
 	"go.etcd.io/etcd/pkg/v3/featuregate"
@@ -520,18 +521,14 @@ func TestCompactHashCheckTimeFlagMigration(t *testing.T) {
 			if tc.compactHashCheckTime != "" {
 				cmdLineArgs = append(cmdLineArgs, fmt.Sprintf("--compact-hash-check-time=%s", tc.compactHashCheckTime))
 				compactHashCheckTime, err := time.ParseDuration(tc.compactHashCheckTime)
-				if err != nil {
-					t.Fatal(err)
-				}
+				require.NoError(t, err)
 				yc.CompactHashCheckTime = compactHashCheckTime
 			}
 
 			if tc.experimentalCompactHashCheckTime != "" {
 				cmdLineArgs = append(cmdLineArgs, fmt.Sprintf("--experimental-compact-hash-check-time=%s", tc.experimentalCompactHashCheckTime))
 				experimentalCompactHashCheckTime, err := time.ParseDuration(tc.experimentalCompactHashCheckTime)
-				if err != nil {
-					t.Fatal(err)
-				}
+				require.NoError(t, err)
 				yc.ExperimentalCompactHashCheckTime = experimentalCompactHashCheckTime
 			}
 
@@ -547,12 +544,8 @@ func TestCompactHashCheckTimeFlagMigration(t *testing.T) {
 				t.Fatal("error parsing config")
 			}
 
-			if cfgFromCmdLine.ec.CompactHashCheckTime != tc.expectedCompactHashCheckTime {
-				t.Errorf("expected CompactHashCheckTime=%v, got %v", tc.expectedCompactHashCheckTime, cfgFromCmdLine.ec.CompactHashCheckTime)
-			}
-			if cfgFromFile.ec.CompactHashCheckTime != tc.expectedCompactHashCheckTime {
-				t.Errorf("expected CompactHashCheckTime=%v, got %v", tc.expectedCompactHashCheckTime, cfgFromFile.ec.CompactHashCheckTime)
-			}
+			require.Equal(t, tc.expectedCompactHashCheckTime, cfgFromCmdLine.ec.CompactHashCheckTime)
+			require.Equal(t, tc.expectedCompactHashCheckTime, cfgFromFile.ec.CompactHashCheckTime)
 		})
 	}
 }
@@ -596,18 +589,14 @@ func TestCorruptCheckTimeFlagMigration(t *testing.T) {
 			if tc.corruptCheckTime != "" {
 				cmdLineArgs = append(cmdLineArgs, fmt.Sprintf("--corrupt-check-time=%s", tc.corruptCheckTime))
 				corruptCheckTime, err := time.ParseDuration(tc.corruptCheckTime)
-				if err != nil {
-					t.Fatal(err)
-				}
+				require.NoError(t, err)
 				yc.CorruptCheckTime = corruptCheckTime
 			}
 
 			if tc.experimentalCorruptCheckTime != "" {
 				cmdLineArgs = append(cmdLineArgs, fmt.Sprintf("--experimental-corrupt-check-time=%s", tc.experimentalCorruptCheckTime))
 				experimentalCorruptCheckTime, err := time.ParseDuration(tc.experimentalCorruptCheckTime)
-				if err != nil {
-					t.Fatal(err)
-				}
+				require.NoError(t, err)
 				yc.ExperimentalCorruptCheckTime = experimentalCorruptCheckTime
 			}
 
@@ -623,12 +612,8 @@ func TestCorruptCheckTimeFlagMigration(t *testing.T) {
 				t.Fatal("error parsing config")
 			}
 
-			if cfgFromCmdLine.ec.CorruptCheckTime != tc.expectedCorruptCheckTime {
-				t.Errorf("expected CorruptCheckTime=%v, got %v", tc.expectedCorruptCheckTime, cfgFromCmdLine.ec.CorruptCheckTime)
-			}
-			if cfgFromFile.ec.CorruptCheckTime != tc.expectedCorruptCheckTime {
-				t.Errorf("expected CorruptCheckTime=%v, got %v", tc.expectedCorruptCheckTime, cfgFromFile.ec.CorruptCheckTime)
-			}
+			require.Equal(t, tc.expectedCorruptCheckTime, cfgFromCmdLine.ec.CorruptCheckTime)
+			require.Equal(t, tc.expectedCorruptCheckTime, cfgFromFile.ec.CorruptCheckTime)
 		})
 	}
 }
@@ -691,22 +676,84 @@ func TestCompactionBatchLimitFlagMigration(t *testing.T) {
 				t.Fatal("error parsing config")
 			}
 
-			if cfgFromCmdLine.ec.CompactionBatchLimit != tc.expectedCompactionBatchLimit {
-				t.Errorf("expected CorruptCheckTime=%v, got %v", tc.expectedCompactionBatchLimit, cfgFromCmdLine.ec.CompactionBatchLimit)
+			require.Equal(t, tc.expectedCompactionBatchLimit, cfgFromCmdLine.ec.CompactionBatchLimit)
+			require.Equal(t, tc.expectedCompactionBatchLimit, cfgFromFile.ec.CompactionBatchLimit)
+		})
+	}
+}
+
+// TestWatchProgressNotifyInterval tests the migration from
+// --experimental-watch-progress-notify-interval to --watch-progress-notify-interval
+// TODO: delete in v3.7
+func TestWatchProgressNotifyInterval(t *testing.T) {
+	testCases := []struct {
+		name                                    string
+		watchProgressNotifyInterval             string
+		experimentalWatchProgressNotifyInterval string
+		expectErr                               bool
+		expectedWatchProgressNotifyInterval     time.Duration
+	}{
+		{
+			name:                                    "cannot set both experimental flag and non experimental flag",
+			watchProgressNotifyInterval:             "2m",
+			experimentalWatchProgressNotifyInterval: "3m",
+			expectErr:                               true,
+		},
+		{
+			name:                                    "can set experimental flag",
+			experimentalWatchProgressNotifyInterval: "3m",
+			expectedWatchProgressNotifyInterval:     3 * time.Minute,
+		},
+		{
+			name:                                "can set non experimental flag",
+			watchProgressNotifyInterval:         "2m",
+			expectedWatchProgressNotifyInterval: 2 * time.Minute,
+		},
+	}
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			cmdLineArgs := []string{}
+			yc := struct {
+				ExperimentalWatchProgressNotifyInterval time.Duration `json:"experimental-watch-progress-notify-interval,omitempty"`
+				WatchProgressNotifyInterval             time.Duration `json:"watch-progress-notify-interval,omitempty"`
+			}{}
+
+			if tc.watchProgressNotifyInterval != "" {
+				cmdLineArgs = append(cmdLineArgs, fmt.Sprintf("--watch-progress-notify-interval=%s", tc.watchProgressNotifyInterval))
+				watchProgressNotifyInterval, err := time.ParseDuration(tc.watchProgressNotifyInterval)
+				require.NoError(t, err)
+				yc.WatchProgressNotifyInterval = watchProgressNotifyInterval
+			}
+
+			if tc.experimentalWatchProgressNotifyInterval != "" {
+				cmdLineArgs = append(cmdLineArgs, fmt.Sprintf("--experimental-watch-progress-notify-interval=%s", tc.experimentalWatchProgressNotifyInterval))
+				experimentalWatchProgressNotifyInterval, err := time.ParseDuration(tc.experimentalWatchProgressNotifyInterval)
+				require.NoError(t, err)
+				yc.ExperimentalWatchProgressNotifyInterval = experimentalWatchProgressNotifyInterval
+			}
+
+			cfgFromCmdLine, errFromCmdLine, cfgFromFile, errFromFile := generateCfgsFromFileAndCmdLine(t, yc, cmdLineArgs)
+
+			if tc.expectErr {
+				if errFromCmdLine == nil || errFromFile == nil {
+					t.Fatal("expect parse error")
+				}
+				return
 			}
-			if cfgFromFile.ec.CompactionBatchLimit != tc.expectedCompactionBatchLimit {
-				t.Errorf("expected CorruptCheckTime=%v, got %v", tc.expectedCompactionBatchLimit, cfgFromFile.ec.CompactionBatchLimit)
+			if errFromCmdLine != nil || errFromFile != nil {
+				t.Fatal("error parsing config")
 			}
+
+			require.Equal(t, tc.expectedWatchProgressNotifyInterval, cfgFromCmdLine.ec.WatchProgressNotifyInterval)
+			require.Equal(t, tc.expectedWatchProgressNotifyInterval, cfgFromFile.ec.WatchProgressNotifyInterval)
 		})
 	}
 }
 
 // TODO delete in v3.7
 func generateCfgsFromFileAndCmdLine(t *testing.T, yc any, cmdLineArgs []string) (*config, error, *config, error) {
 	b, err := yaml.Marshal(&yc)
-	if err != nil {
-		t.Fatal(err)
-	}
+	require.NoError(t, err)
 
 	tmpfile := mustCreateCfgFile(t, b)
 	defer os.Remove(tmpfile.Name())
@@ -819,6 +866,7 @@ func TestConfigFileDeprecatedOptions(t *testing.T) {
 		ExperimentalDistributedTracingServiceName  string        `json:"experimental-distributed-tracing-service-name,omitempty"`
 		ExperimentalDistributedTracingInstanceID   string        `json:"experimental-distributed-tracing-instance-id,omitempty"`
 		ExperimentalDistributedTracingSamplingRate int           `json:"experimental-distributed-tracing-sampling-rate,omitempty"`
+		ExperimentalWatchProgressNotifyInterval    time.Duration `json:"experimental-watch-progress-notify-interval,omitempty"`
 	}
 
 	testCases := []struct {
@@ -839,12 +887,14 @@ func TestConfigFileDeprecatedOptions(t *testing.T) {
 				ExperimentalWarningUnaryRequestDuration: time.Second,
 				ExperimentalCorruptCheckTime:            time.Minute,
 				ExperimentalCompactionBatchLimit:        1,
+				ExperimentalWatchProgressNotifyInterval: 3 * time.Minute,
 			},
 			expectedFlags: map[string]struct{}{
-				"experimental-compact-hash-check-enabled": {},
-				"experimental-compact-hash-check-time":    {},
-				"experimental-corrupt-check-time":         {},
-				"experimental-compaction-batch-limit":     {},
+				"experimental-compact-hash-check-enabled":     {},
+				"experimental-compact-hash-check-time":        {},
+				"experimental-corrupt-check-time":             {},
+				"experimental-compaction-batch-limit":         {},
+				"experimental-watch-progress-notify-interval": {},
 			},
 		},
 		{

server/etcdmain/help.go

@@ -306,6 +306,8 @@ Experimental feature:
   --experimental-peer-skip-client-san-verification 'false'
     Skip verification of SAN field in client certificate for peer connections.
   --experimental-watch-progress-notify-interval '10m'
+    Duration of periodical watch progress notification. Deprecated in v3.6 and will be decommissioned in v3.7. Use 'watch-progress-notify-interval' instead.
+  --watch-progress-notify-interval '10m'
     Duration of periodical watch progress notification.
   --experimental-warning-apply-duration '100ms'
     Warning is generated if requests take more than this duration.

tests/e2e/watch_test.go

@@ -91,7 +91,7 @@ func TestWatchDelayForPeriodicProgressNotification(t *testing.T) {
 		tc := tc
 		cfg := e2e.DefaultConfig()
 		cfg.ClusterSize = 1
-		cfg.ServerConfig.ExperimentalWatchProgressNotifyInterval = watchResponsePeriod
+		cfg.ServerConfig.WatchProgressNotifyInterval = watchResponsePeriod
 		cfg.Client = tc.client
 		cfg.ClientHTTPSeparate = tc.clientHTTPSeparate
 		t.Run(tc.name, func(t *testing.T) {

tests/framework/e2e/cluster.go

@@ -385,6 +385,10 @@ func WithCompactionSleepInterval(time time.Duration) EPClusterOption {
 }
 
 func WithWatchProcessNotifyInterval(interval time.Duration) EPClusterOption {
+	return func(c *EtcdProcessClusterConfig) { c.ServerConfig.WatchProgressNotifyInterval = interval }
+}
+
+func WithExperimentalWatchProcessNotifyInterval(interval time.Duration) EPClusterOption {
 	return func(c *EtcdProcessClusterConfig) { c.ServerConfig.ExperimentalWatchProgressNotifyInterval = interval }
 }
 

tests/robustness/scenarios/scenarios.go

@@ -98,7 +98,7 @@ func Exploratory(_ *testing.T) []TestScenario {
 		e2e.WithGoFailEnabled(true),
 		// Set low minimal compaction batch limit to allow for triggering multi batch compaction failpoints.
 		options.WithExperimentalCompactionBatchLimit(10, 100, 1000),
-		e2e.WithWatchProcessNotifyInterval(100 * time.Millisecond),
+		e2e.WithExperimentalWatchProcessNotifyInterval(100 * time.Millisecond),
 	}
 
 	if e2e.CouldSetSnapshotCatchupEntries(e2e.BinPath.Etcd) {