Skip to content

Commit 0e45288

Browse files
ahrtrahrtr
authored
Merge pull request #19128 from ahrtr/etcdutl_migrate_20250106
Update etcdutl migrate command: load wal records from the latest snapshot
2 parents 6277dbe + ec52e35 commit 0e45288

File tree

3 files changed

+173
-10
lines changed

3 files changed

+173
-10
lines changed

‎etcdutl/etcdutl/migrate_command.go

+27-1
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@
1515
package etcdutl
1616

1717
import (
18+
"errors"
1819
"fmt"
1920
"strings"
2021

@@ -24,6 +25,7 @@ import (
2425

2526
"go.etcd.io/etcd/api/v3/version"
2627
"go.etcd.io/etcd/pkg/v3/cobrautl"
28+
"go.etcd.io/etcd/server/v3/etcdserver/api/snap"
2729
"go.etcd.io/etcd/server/v3/storage/backend"
2830
"go.etcd.io/etcd/server/v3/storage/datadir"
2931
"go.etcd.io/etcd/server/v3/storage/schema"
@@ -95,7 +97,11 @@ func (o *migrateOptions) Config() (*migrateConfig, error) {
9597
c.be = backend.NewDefaultBackend(GetLogger(), dbPath)
9698

9799
walPath := datadir.ToWALDir(o.dataDir)
98-
w, err := wal.OpenForRead(c.lg, walPath, walpb.Snapshot{})
100+
walSnap, err := getLatestWALSnap(c.lg, o.dataDir)
101+
if err != nil {
102+
return nil, fmt.Errorf("failed to get the lastest snapshot: %w", err)
103+
}
104+
w, err := wal.OpenForRead(c.lg, walPath, walSnap)
99105
if err != nil {
100106
return nil, fmt.Errorf(`failed to open wal: %w`, err)
101107
}
@@ -108,6 +114,26 @@ func (o *migrateOptions) Config() (*migrateConfig, error) {
108114
return c, nil
109115
}
110116

117+
func getLatestWALSnap(lg *zap.Logger, dataDir string) (walpb.Snapshot, error) {
118+
walPath := datadir.ToWALDir(dataDir)
119+
walSnaps, err := wal.ValidSnapshotEntries(lg, walPath)
120+
if err != nil {
121+
return walpb.Snapshot{}, err
122+
}
123+
124+
ss := snap.New(lg, datadir.ToSnapDir(dataDir))
125+
snapshot, err := ss.LoadNewestAvailable(walSnaps)
126+
if err != nil && !errors.Is(err, snap.ErrNoSnapshot) {
127+
return walpb.Snapshot{}, err
128+
}
129+
130+
var walsnap walpb.Snapshot
131+
if snapshot != nil {
132+
walsnap.Index, walsnap.Term = snapshot.Metadata.Index, snapshot.Metadata.Term
133+
}
134+
return walsnap, nil
135+
}
136+
111137
type migrateConfig struct {
112138
lg *zap.Logger
113139
be backend.Backend

‎etcdutl/etcdutl/migrate_command_test.go

+143
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,143 @@
1+
// Copyright 2025 The etcd Authors
2+
//
3+
// Licensed under the Apache License, Version 2.0 (the "License");
4+
// you may not use this file except in compliance with the License.
5+
// You may obtain a copy of the License at
6+
//
7+
// http://www.apache.org/licenses/LICENSE-2.0
8+
//
9+
// Unless required by applicable law or agreed to in writing, software
10+
// distributed under the License is distributed on an "AS IS" BASIS,
11+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
// See the License for the specific language governing permissions and
13+
// limitations under the License.
14+
15+
package etcdutl
16+
17+
import (
18+
"testing"
19+
20+
"github.com/stretchr/testify/require"
21+
"go.uber.org/zap"
22+
23+
"go.etcd.io/etcd/api/v3/etcdserverpb"
24+
"go.etcd.io/etcd/client/pkg/v3/fileutil"
25+
"go.etcd.io/etcd/pkg/v3/pbutil"
26+
"go.etcd.io/etcd/server/v3/etcdserver/api/snap"
27+
"go.etcd.io/etcd/server/v3/storage/datadir"
28+
"go.etcd.io/etcd/server/v3/storage/wal"
29+
"go.etcd.io/etcd/server/v3/storage/wal/walpb"
30+
"go.etcd.io/raft/v3/raftpb"
31+
)
32+
33+
func TestGetLatestWalSnap(t *testing.T) {
34+
testCases := []struct {
35+
name string
36+
walSnaps []walpb.Snapshot
37+
snapshots []raftpb.Snapshot
38+
expectedLatestWALSnap walpb.Snapshot
39+
}{
40+
{
41+
name: "wal snapshot records match the snapshot files",
42+
walSnaps: []walpb.Snapshot{
43+
{Index: 10, Term: 2},
44+
{Index: 20, Term: 3},
45+
{Index: 30, Term: 5},
46+
},
47+
snapshots: []raftpb.Snapshot{
48+
{Metadata: raftpb.SnapshotMetadata{Index: 10, Term: 2}},
49+
{Metadata: raftpb.SnapshotMetadata{Index: 20, Term: 3}},
50+
{Metadata: raftpb.SnapshotMetadata{Index: 30, Term: 5}},
51+
},
52+
expectedLatestWALSnap: walpb.Snapshot{Index: 30, Term: 5},
53+
},
54+
{
55+
name: "there are orphan snapshot files",
56+
walSnaps: []walpb.Snapshot{
57+
{Index: 10, Term: 2},
58+
{Index: 20, Term: 3},
59+
{Index: 35, Term: 5},
60+
},
61+
snapshots: []raftpb.Snapshot{
62+
{Metadata: raftpb.SnapshotMetadata{Index: 10, Term: 2}},
63+
{Metadata: raftpb.SnapshotMetadata{Index: 20, Term: 3}},
64+
{Metadata: raftpb.SnapshotMetadata{Index: 35, Term: 5}},
65+
{Metadata: raftpb.SnapshotMetadata{Index: 40, Term: 6}},
66+
{Metadata: raftpb.SnapshotMetadata{Index: 50, Term: 7}},
67+
},
68+
expectedLatestWALSnap: walpb.Snapshot{Index: 35, Term: 5},
69+
},
70+
{
71+
name: "there are orphan snapshot records in wal file",
72+
walSnaps: []walpb.Snapshot{
73+
{Index: 10, Term: 2},
74+
{Index: 20, Term: 3},
75+
{Index: 30, Term: 4},
76+
{Index: 45, Term: 5},
77+
{Index: 55, Term: 6},
78+
},
79+
snapshots: []raftpb.Snapshot{
80+
{Metadata: raftpb.SnapshotMetadata{Index: 10, Term: 2}},
81+
{Metadata: raftpb.SnapshotMetadata{Index: 20, Term: 3}},
82+
{Metadata: raftpb.SnapshotMetadata{Index: 30, Term: 4}},
83+
},
84+
expectedLatestWALSnap: walpb.Snapshot{Index: 30, Term: 4},
85+
},
86+
{
87+
name: "wal snapshot records do not match the snapshot files at all",
88+
walSnaps: []walpb.Snapshot{
89+
{Index: 10, Term: 2},
90+
{Index: 20, Term: 3},
91+
{Index: 30, Term: 4},
92+
},
93+
snapshots: []raftpb.Snapshot{
94+
{Metadata: raftpb.SnapshotMetadata{Index: 40, Term: 5}},
95+
{Metadata: raftpb.SnapshotMetadata{Index: 50, Term: 6}},
96+
},
97+
expectedLatestWALSnap: walpb.Snapshot{},
98+
},
99+
}
100+
101+
for _, tc := range testCases {
102+
t.Run(tc.name, func(t *testing.T) {
103+
dataDir := t.TempDir()
104+
lg := zap.NewNop()
105+
106+
require.NoError(t, fileutil.TouchDirAll(lg, datadir.ToMemberDir(dataDir)))
107+
require.NoError(t, fileutil.TouchDirAll(lg, datadir.ToWALDir(dataDir)))
108+
require.NoError(t, fileutil.TouchDirAll(lg, datadir.ToSnapDir(dataDir)))
109+
110+
// populate wal file
111+
w, err := wal.Create(lg, datadir.ToWALDir(dataDir), pbutil.MustMarshal(
112+
&etcdserverpb.Metadata{
113+
NodeID: 1,
114+
ClusterID: 2,
115+
},
116+
))
117+
require.NoError(t, err)
118+
119+
for _, walSnap := range tc.walSnaps {
120+
walSnap.ConfState = &raftpb.ConfState{Voters: []uint64{1}}
121+
walErr := w.SaveSnapshot(walSnap)
122+
require.NoError(t, walErr)
123+
walErr = w.Save(raftpb.HardState{Term: walSnap.Term, Commit: walSnap.Index, Vote: 1}, nil)
124+
require.NoError(t, walErr)
125+
}
126+
err = w.Close()
127+
require.NoError(t, err)
128+
129+
// generate snapshot files
130+
ss := snap.New(lg, datadir.ToSnapDir(dataDir))
131+
for _, snap := range tc.snapshots {
132+
snap.Metadata.ConfState = raftpb.ConfState{Voters: []uint64{1}}
133+
snapErr := ss.SaveSnap(snap)
134+
require.NoError(t, snapErr)
135+
}
136+
137+
walSnap, err := getLatestWALSnap(lg, dataDir)
138+
require.NoError(t, err)
139+
140+
require.Equal(t, tc.expectedLatestWALSnap, walSnap)
141+
})
142+
}
143+
}

‎tests/e2e/utl_migrate_test.go

+3-9
Original file line numberDiff line numberDiff line change
@@ -90,16 +90,10 @@ func TestEtctlutlMigrate(t *testing.T) {
9090
expectStorageVersion: &version.V3_6,
9191
},
9292
{
93-
name: "Downgrade v3.6 to v3.5 should fail until it's implemented",
93+
name: "Downgrade v3.6 to v3.5 should work",
9494
targetVersion: "3.5",
95-
expectLogsSubString: "cannot downgrade storage, WAL contains newer entries",
96-
expectStorageVersion: &version.V3_6,
97-
},
98-
{
99-
name: "Downgrade v3.6 to v3.5 with force should work",
100-
targetVersion: "3.5",
101-
force: true,
102-
expectLogsSubString: "forcefully cleared storage version",
95+
expectLogsSubString: "updated storage version",
96+
expectStorageVersion: nil, // 3.5 doesn't have the field `storageVersion`, so it returns nil.
10397
},
10498
{
10599
name: "Upgrade v3.6 to v3.7 with force should work",

0 commit comments

Comments
 (0)