Skip to content

Commit 68f5e3f

Browse files
siyuanfoundationsiyuanfoundation
committed
local test for MemberDowngrade failpoint
Signed-off-by: Siyuan Zhang <[email protected]>
1 parent 0dcdcff commit 68f5e3f

File tree

4 files changed

+50
-46
lines changed

4 files changed

+50
-46
lines changed

‎tests/framework/e2e/cluster.go

+1
Original file line numberDiff line numberDiff line change
@@ -564,6 +564,7 @@ func (cfg *EtcdProcessClusterConfig) EtcdServerProcessConfig(tb testing.TB, i in
564564
"--initial-cluster-token=" + cfg.ServerConfig.InitialClusterToken,
565565
"--data-dir", dataDirPath,
566566
"--snapshot-count=" + fmt.Sprintf("%d", cfg.ServerConfig.SnapshotCount),
567+
"--max-wals=1000", "--max-snapshots=1000",
567568
}
568569
var clientHTTPURL string
569570
if cfg.ClientHTTPSeparate {

‎tests/robustness/failpoint/cluster.go

+13-11
Original file line numberDiff line numberDiff line change
@@ -181,23 +181,23 @@ func (f memberDowngrade) Inject(ctx context.Context, t *testing.T, lg *zap.Logge
181181
member = clus.Procs[memberID]
182182
lg.Info("Downgrading member", zap.String("member", member.Config().Name))
183183
for member.IsRunning() {
184-
err = member.Kill()
184+
err = member.Stop()
185185
if err != nil {
186-
lg.Info("Sending kill signal failed", zap.Error(err))
186+
lg.Info("Stopping server failed", zap.Error(err))
187187
}
188188
err = member.Wait(ctx)
189189
if err != nil && !strings.Contains(err.Error(), "unexpected exit code") {
190-
lg.Info("Failed to kill the process", zap.Error(err))
191-
return nil, fmt.Errorf("failed to kill the process within %s, err: %w", triggerTimeout, err)
192-
}
193-
}
194-
if lazyfs := member.LazyFS(); lazyfs != nil {
195-
lg.Info("Removing data that was not fsynced")
196-
err := lazyfs.ClearCache(ctx)
197-
if err != nil {
198-
return nil, err
190+
lg.Info("Failed to stop the process", zap.Error(err))
191+
return nil, fmt.Errorf("failed to stop the process within %s, err: %w", triggerTimeout, err)
199192
}
200193
}
194+
// if lazyfs := member.LazyFS(); lazyfs != nil {
195+
// lg.Info("Removing data that was not fsynced")
196+
// err := lazyfs.ClearCache(ctx)
197+
// if err != nil {
198+
// return nil, err
199+
// }
200+
// }
201201
member.Config().ExecPath = e2e.BinPath.EtcdLastRelease
202202
err = patchArgs(member.Config().Args, "initial-cluster-state", "existing")
203203
if err != nil {
@@ -208,9 +208,11 @@ func (f memberDowngrade) Inject(ctx context.Context, t *testing.T, lg *zap.Logge
208208
if err != nil {
209209
return nil, err
210210
}
211+
time.Sleep(etcdserver.HealthInterval)
211212
err = verifyVersion(t, clus, member, targetVersion)
212213
}
213214
time.Sleep(etcdserver.HealthInterval)
215+
lg.Info("Finished downgrading members", zap.Any("members", membersToDowngrade))
214216
return nil, err
215217
}
216218

‎tests/robustness/failpoint/failpoint.go

+15-15
Original file line numberDiff line numberDiff line change
@@ -36,22 +36,22 @@ const (
3636
)
3737

3838
var allFailpoints = []Failpoint{
39-
KillFailpoint, BeforeCommitPanic, AfterCommitPanic, RaftBeforeSavePanic, RaftAfterSavePanic,
40-
DefragBeforeCopyPanic, DefragBeforeRenamePanic, BackendBeforePreCommitHookPanic, BackendAfterPreCommitHookPanic,
41-
BackendBeforeStartDBTxnPanic, BackendAfterStartDBTxnPanic, BackendBeforeWritebackBufPanic,
42-
BackendAfterWritebackBufPanic, CompactBeforeCommitScheduledCompactPanic, CompactAfterCommitScheduledCompactPanic,
43-
CompactBeforeSetFinishedCompactPanic, CompactAfterSetFinishedCompactPanic, CompactBeforeCommitBatchPanic,
44-
CompactAfterCommitBatchPanic, RaftBeforeLeaderSendPanic, BlackholePeerNetwork, DelayPeerNetwork,
45-
RaftBeforeFollowerSendPanic, RaftBeforeApplySnapPanic, RaftAfterApplySnapPanic, RaftAfterWALReleasePanic,
46-
RaftBeforeSaveSnapPanic, RaftAfterSaveSnapPanic, BlackholeUntilSnapshot,
47-
BeforeApplyOneConfChangeSleep,
48-
MemberReplace,
39+
// KillFailpoint, BeforeCommitPanic, AfterCommitPanic, RaftBeforeSavePanic, RaftAfterSavePanic,
40+
// DefragBeforeCopyPanic, DefragBeforeRenamePanic, BackendBeforePreCommitHookPanic, BackendAfterPreCommitHookPanic,
41+
// BackendBeforeStartDBTxnPanic, BackendAfterStartDBTxnPanic, BackendBeforeWritebackBufPanic,
42+
// BackendAfterWritebackBufPanic, CompactBeforeCommitScheduledCompactPanic, CompactAfterCommitScheduledCompactPanic,
43+
// CompactBeforeSetFinishedCompactPanic, CompactAfterSetFinishedCompactPanic, CompactBeforeCommitBatchPanic,
44+
// CompactAfterCommitBatchPanic, RaftBeforeLeaderSendPanic, BlackholePeerNetwork, DelayPeerNetwork,
45+
// RaftBeforeFollowerSendPanic, RaftBeforeApplySnapPanic, RaftAfterApplySnapPanic, RaftAfterWALReleasePanic,
46+
// RaftBeforeSaveSnapPanic, RaftAfterSaveSnapPanic, BlackholeUntilSnapshot,
47+
// BeforeApplyOneConfChangeSleep,
48+
// MemberReplace,
4949
MemberDowngrade,
50-
DropPeerNetwork,
51-
RaftBeforeSaveSleep,
52-
RaftAfterSaveSleep,
53-
ApplyBeforeOpenSnapshot,
54-
SleepBeforeSendWatchResponse,
50+
// DropPeerNetwork,
51+
// RaftBeforeSaveSleep,
52+
// RaftAfterSaveSleep,
53+
// ApplyBeforeOpenSnapshot,
54+
// SleepBeforeSendWatchResponse,
5555
}
5656

5757
func PickRandom(clus *e2e.EtcdProcessCluster, profile traffic.Profile) (Failpoint, error) {

‎tests/robustness/scenarios/scenarios.go

+21-20
Original file line numberDiff line numberDiff line change
@@ -81,40 +81,41 @@ func Exploratory(_ *testing.T) []TestScenario {
8181
// 60% with all members of current version
8282
{Choice: options.ClusterOptions{options.WithVersion(e2e.CurrentVersion)}, Weight: 60},
8383
// 10% with 2 members of current version, 1 member last version, leader is current version
84-
{Choice: options.ClusterOptions{options.WithVersion(e2e.MinorityLastVersion), options.WithInitialLeaderIndex(0)}, Weight: 10},
85-
// 10% with 2 members of current version, 1 member last version, leader is last version
86-
{Choice: options.ClusterOptions{options.WithVersion(e2e.MinorityLastVersion), options.WithInitialLeaderIndex(2)}, Weight: 10},
87-
// 10% with 2 members of last version, 1 member current version, leader is last version
88-
{Choice: options.ClusterOptions{options.WithVersion(e2e.QuorumLastVersion), options.WithInitialLeaderIndex(0)}, Weight: 10},
89-
// 10% with 2 members of last version, 1 member current version, leader is current version
90-
{Choice: options.ClusterOptions{options.WithVersion(e2e.QuorumLastVersion), options.WithInitialLeaderIndex(2)}, Weight: 10},
84+
// {Choice: options.ClusterOptions{options.WithVersion(e2e.MinorityLastVersion), options.WithInitialLeaderIndex(0)}, Weight: 10},
85+
// // 10% with 2 members of current version, 1 member last version, leader is last version
86+
// {Choice: options.ClusterOptions{options.WithVersion(e2e.MinorityLastVersion), options.WithInitialLeaderIndex(2)}, Weight: 10},
87+
// // 10% with 2 members of last version, 1 member current version, leader is last version
88+
// {Choice: options.ClusterOptions{options.WithVersion(e2e.QuorumLastVersion), options.WithInitialLeaderIndex(0)}, Weight: 10},
89+
// // 10% with 2 members of last version, 1 member current version, leader is current version
90+
// {Choice: options.ClusterOptions{options.WithVersion(e2e.QuorumLastVersion), options.WithInitialLeaderIndex(2)}, Weight: 10},
9191
}
9292
mixedVersionOption := options.WithClusterOptionGroups(random.PickRandom[options.ClusterOptions](mixedVersionOptionChoices))
9393

9494
baseOptions := []e2e.EPClusterOption{
95-
options.WithSnapshotCount(50, 100, 1000),
95+
options.WithSnapshotCount(100000),
9696
options.WithSubsetOptions(randomizableOptions...),
9797
e2e.WithGoFailEnabled(true),
98+
e2e.WithKeepDataDir(true),
9899
// Set low minimal compaction batch limit to allow for triggering multi batch compaction failpoints.
99-
options.WithCompactionBatchLimit(10, 100, 1000),
100+
options.WithCompactionBatchLimit(100000),
100101
e2e.WithWatchProcessNotifyInterval(100 * time.Millisecond),
101102
}
102103

103104
if e2e.CouldSetSnapshotCatchupEntries(e2e.BinPath.Etcd) {
104105
baseOptions = append(baseOptions, e2e.WithSnapshotCatchUpEntries(100))
105106
}
106107
scenarios := []TestScenario{}
107-
for _, tp := range trafficProfiles {
108-
name := filepath.Join(tp.Name, "ClusterOfSize1")
109-
clusterOfSize1Options := baseOptions
110-
clusterOfSize1Options = append(clusterOfSize1Options, e2e.WithClusterSize(1))
111-
scenarios = append(scenarios, TestScenario{
112-
Name: name,
113-
Traffic: tp.Traffic,
114-
Profile: tp.Profile,
115-
Cluster: *e2e.NewConfig(clusterOfSize1Options...),
116-
})
117-
}
108+
// for _, tp := range trafficProfiles {
109+
// name := filepath.Join(tp.Name, "ClusterOfSize1")
110+
// clusterOfSize1Options := baseOptions
111+
// clusterOfSize1Options = append(clusterOfSize1Options, e2e.WithClusterSize(1))
112+
// scenarios = append(scenarios, TestScenario{
113+
// Name: name,
114+
// Traffic: tp.Traffic,
115+
// Profile: tp.Profile,
116+
// Cluster: *e2e.NewConfig(clusterOfSize1Options...),
117+
// })
118+
// }
118119

119120
for _, tp := range trafficProfiles {
120121
name := filepath.Join(tp.Name, "ClusterOfSize3")

0 commit comments

Comments
 (0)