Skip to content

Commit 8097a71

Browse files
committed
support migarte etcd mds and snapshot
Signed-off-by: caoxianfei1 <[email protected]>
1 parent b1a3a4e commit 8097a71

File tree

14 files changed

+466
-10
lines changed

14 files changed

+466
-10
lines changed

cli/command/migrate.go

+24-5
Original file line numberDiff line numberDiff line change
@@ -37,15 +37,24 @@ import (
3737
"github.com/spf13/cobra"
3838
)
3939

40+
const (
41+
ETCD_INIT_CLUSTER_STATE = "initial-cluster-state"
42+
ETCD_STATE_EXSITING = "existing"
43+
)
44+
4045
var (
4146
MIGRATE_ETCD_STEPS = []int{
42-
playbook.STOP_SERVICE,
43-
playbook.CLEAN_SERVICE, // only container
47+
playbook.ADD_ETCD_MEMBER,
4448
playbook.PULL_IMAGE,
4549
playbook.CREATE_CONTAINER,
4650
playbook.SYNC_CONFIG,
51+
playbook.AMEND_ETCD_CONFIG,
4752
playbook.START_ETCD,
48-
playbook.UPDATE_TOPOLOGY,
53+
playbook.AMEND_MDS_CONFIG,
54+
playbook.RESTART_SERVICE, // restart all mds
55+
// playbook.STOP_SERVICE,
56+
// playbook.CLEAN_SERVICE, // only container
57+
// playbook.UPDATE_TOPOLOGY,
4958
}
5059

5160
// mds
@@ -157,7 +166,7 @@ func checkMigrateTopology(curveadm *cli.CurveAdm, data string) error {
157166
} else if len(dcs2add) < len(dcs2del) {
158167
return errno.ERR_DELETE_SERVICE_WHILE_MIGRATING_IS_DENIED
159168
}
160-
// len(dcs2add) == len(dcs2del)
169+
161170
if len(dcs2add) == 0 {
162171
return errno.ERR_NO_SERVICES_FOR_MIGRATING
163172
}
@@ -199,6 +208,7 @@ func genMigratePlaybook(curveadm *cli.CurveAdm,
199208
migrates := getMigrates(curveadm, data)
200209
role := migrates[0].From.GetRole()
201210
steps := MIGRATE_ROLE_STEPS[role]
211+
etcdDCs := curveadm.FilterDeployConfigByRole(dcs, topology.ROLE_ETCD)
202212

203213
// post clean
204214
if options.clean {
@@ -221,10 +231,14 @@ func genMigratePlaybook(curveadm *cli.CurveAdm,
221231
config := dcs2add
222232
switch step {
223233
case playbook.STOP_SERVICE,
224-
playbook.CLEAN_SERVICE:
234+
playbook.CLEAN_SERVICE,
235+
playbook.ADD_ETCD_MEMBER:
225236
config = dcs2del
226237
case playbook.BACKUP_ETCD_DATA:
227238
config = curveadm.FilterDeployConfigByRole(dcs, topology.ROLE_ETCD)
239+
case playbook.AMEND_MDS_CONFIG,
240+
playbook.RESTART_SERVICE:
241+
config = curveadm.FilterDeployConfigByRole(dcs, topology.ROLE_MDS)
228242
case
229243
playbook.CREATE_PHYSICAL_POOL,
230244
playbook.CREATE_LOGICAL_POOL,
@@ -251,6 +265,11 @@ func genMigratePlaybook(curveadm *cli.CurveAdm,
251265
optionsKV[comm.KEY_POOLSET] = poolset
252266
case playbook.UPDATE_TOPOLOGY:
253267
optionsKV[comm.KEY_NEW_TOPOLOGY_DATA] = data
268+
case playbook.ADD_ETCD_MEMBER,
269+
playbook.AMEND_ETCD_CONFIG,
270+
playbook.AMEND_MDS_CONFIG:
271+
optionsKV[comm.KEY_MIGRATE_SERVERS] = migrates
272+
optionsKV[comm.KEY_CLUSTER_DCS] = etcdDCs
254273
}
255274

256275
pb.AddStep(&playbook.PlaybookStep{

curve

16.4 MB
Binary file not shown.

internal/common/common.go

+1
Original file line numberDiff line numberDiff line change
@@ -57,6 +57,7 @@ const (
5757
// migrate
5858
KEY_MIGRATE_STATUS = "MIGRATE_STATUS"
5959
KEY_MIGRATE_COMMON_STATUS = "MIGRATE_COMMON_STATUS"
60+
KEY_CLUSTER_DCS = "CLUSTER_DCS"
6061

6162
// check
6263
KEY_CHECK_WITH_WEAK = "CHECK_WITH_WEAK"

internal/configure/topology/dc_get.go

+5-1
Original file line numberDiff line numberDiff line change
@@ -121,7 +121,11 @@ func (dc *DeployConfig) GetInstances() int { return dc.instanc
121121
func (dc *DeployConfig) GetHostSequence() int { return dc.hostSequence }
122122
func (dc *DeployConfig) GetInstancesSequence() int { return dc.instancesSequence }
123123
func (dc *DeployConfig) GetServiceConfig() map[string]string { return dc.serviceConfig }
124-
func (dc *DeployConfig) GetVariables() *variable.Variables { return dc.variables }
124+
func (dc *DeployConfig) SetServiceConfig(key, value string) {
125+
dc.serviceConfig[key] = value
126+
}
127+
128+
func (dc *DeployConfig) GetVariables() *variable.Variables { return dc.variables }
125129

126130
// (2): config item
127131
func (dc *DeployConfig) GetPrefix() string { return dc.getString(CONFIG_PREFIX) }

internal/configure/topology/variables.go

+2-2
Original file line numberDiff line numberDiff line change
@@ -118,9 +118,9 @@ var (
118118
{name: "cluster_mds_dummy_addr"},
119119
{name: "cluster_mds_dummy_port"},
120120
{name: "cluster_chunkserver_addr", kind: []string{KIND_CURVEBS}},
121-
{name: "cluster_snapshotclone_addr", kind: []string{KIND_CURVEBS}},
121+
{name: "cluster_snapshotclone_addr"},
122122
{name: "cluster_snapshotclone_proxy_addr", kind: []string{KIND_CURVEBS}},
123-
{name: "cluster_snapshotclone_dummy_port", kind: []string{KIND_CURVEBS}},
123+
{name: "cluster_snapshotclone_dummy_port"},
124124
{name: "cluster_snapshotclone_nginx_upstream", kind: []string{KIND_CURVEBS}},
125125
{name: "cluster_snapshot_addr"}, // tools-v2: compatible with some old version image
126126
{name: "cluster_snapshot_dummy_addr"}, // tools-v2

internal/errno/errno.go

+2
Original file line numberDiff line numberDiff line change
@@ -404,6 +404,8 @@ var (
404404
ERR_GET_CHUNKSERVER_COPYSET = EC(410026, "failed to get chunkserver copyset")
405405
ERR_GET_MIGRATE_COPYSET = EC(410027, "migrate chunkserver copyset info must be 2")
406406
ERR_CONTAINER_NOT_REMOVED = EC(410027, "container not removed")
407+
ERR_GET_CLUSTER_ETCD_ADDR = EC(410028, "failed to get cluster_etcd_addr variable")
408+
ERR_ADD_ETCD_MEMEBER = EC(410029, "failed to add member to existing etcd cluster")
407409
// 420: common (curvebs client)
408410
ERR_VOLUME_ALREADY_MAPPED = EC(420000, "volume already mapped")
409411
ERR_VOLUME_CONTAINER_LOSED = EC(420001, "volume container is losed")

internal/playbook/factory.go

+9
Original file line numberDiff line numberDiff line change
@@ -84,6 +84,9 @@ const (
8484
INSTALL_CLIENT
8585
UNINSTALL_CLIENT
8686
ATTACH_LEADER_OR_RANDOM_CONTAINER
87+
ADD_ETCD_MEMBER
88+
AMEND_ETCD_CONFIG
89+
AMEND_MDS_CONFIG
8790

8891
// bs
8992
FORMAT_CHUNKFILE_POOL
@@ -251,6 +254,12 @@ func (p *Playbook) createTasks(step *PlaybookStep) (*tasks.Tasks, error) {
251254
t, err = comm.NewInstallClientTask(curveadm, config.GetCC(i))
252255
case UNINSTALL_CLIENT:
253256
t, err = comm.NewUninstallClientTask(curveadm, nil)
257+
case ADD_ETCD_MEMBER:
258+
t, err = comm.NewAddEtcdMemberTask(curveadm, config.GetDC(i))
259+
case AMEND_ETCD_CONFIG:
260+
t, err = comm.NewAmendEtcdConfigTask(curveadm, config.GetDC(i))
261+
case AMEND_MDS_CONFIG:
262+
t, err = comm.NewAmendMdsConfigTask(curveadm, config.GetDC(i))
254263
// bs
255264
case FORMAT_CHUNKFILE_POOL:
256265
t, err = bs.NewFormatChunkfilePoolTask(curveadm, config.GetFC(i))

internal/task/scripts/enable_etcd_auth.go

+2-2
Original file line numberDiff line numberDiff line change
@@ -12,13 +12,13 @@
1212
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
1313
* See the License for the specific language governing permissions and
1414
* limitations under the License.
15-
*/
15+
*/
1616

1717
/*
1818
* Project: Curveadm
1919
* Created Date: 2023-08-02
2020
* Author: wanghai (SeanHai)
21-
*/
21+
*/
2222

2323
package scripts
2424

internal/task/scripts/script.go

+2
Original file line numberDiff line numberDiff line change
@@ -39,6 +39,8 @@ var (
3939
WAIT string
4040
//go:embed shell/report.sh
4141
REPORT string
42+
//go:embed shell/add_etcd.sh
43+
ADD_ETCD string
4244

4345
// CurveBS
4446

+43
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,43 @@
1+
#!/usr/bin/env bash
2+
3+
# Usage:
4+
# Example:
5+
# Created Date: 2023-12-15
6+
# Author: Caoxianfei
7+
8+
etcdctl=$1
9+
endpoints=$2
10+
old_name=$3
11+
new_name=$4
12+
new_peer_url=$5
13+
14+
${etcdctl} --endpoints=${endpoints} member add ${new_name} --peer-urls ${new_peer_url} > add_etcd.log 2>&1
15+
if [ $? -ne 0 ]; then
16+
if cat add_etcd.log | grep -q "Peer URLs already exists"; then
17+
exit 0
18+
else
19+
exit 1
20+
fi
21+
fi
22+
23+
# output=$(${etcdctl} --endpoints=${endpoints} member list)
24+
# if [ $? -ne 0 ]; then
25+
# echo "failed to list all etcd members"
26+
# exit 1
27+
# fi
28+
29+
# id=$(echo "$output" | awk -v name="$old_name" -F ', ' '$3 == name {print $1}')
30+
# if [ -z "${id}" ]; then
31+
# echo "failed to get id of member ${old_name}"
32+
# exit 1
33+
# fi
34+
35+
# ${etcdctl} --endpoints=${endpoints} member remove ${id}
36+
# if [ $? -ne 0 ]; then
37+
# echo "failed to remove member ${old_name}"
38+
# exit 1
39+
# fi
40+
41+
42+
43+
+116
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,116 @@
1+
/*
2+
* Copyright (c) 2023 NetEase Inc.
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
17+
/*
18+
* Project: CurveAdm
19+
* Created Date: 2023-12-20
20+
* Author: Caoxianfei
21+
*/
22+
23+
package common
24+
25+
import (
26+
"fmt"
27+
"strconv"
28+
29+
"github.com/opencurve/curveadm/cli/cli"
30+
comm "github.com/opencurve/curveadm/internal/common"
31+
"github.com/opencurve/curveadm/internal/configure"
32+
"github.com/opencurve/curveadm/internal/configure/topology"
33+
"github.com/opencurve/curveadm/internal/errno"
34+
"github.com/opencurve/curveadm/internal/task/context"
35+
"github.com/opencurve/curveadm/internal/task/scripts"
36+
"github.com/opencurve/curveadm/internal/task/step"
37+
"github.com/opencurve/curveadm/internal/task/task"
38+
tui "github.com/opencurve/curveadm/internal/tui/common"
39+
)
40+
41+
func checkAddEtcdMemberStatus(success *bool, out *string) step.LambdaType {
42+
return func(ctx *context.Context) error {
43+
if !*success {
44+
return errno.ERR_ADD_ETCD_MEMEBER.S(*out)
45+
}
46+
return nil
47+
}
48+
}
49+
50+
func NewAddEtcdMemberTask(curveadm *cli.CurveAdm, dc *topology.DeployConfig) (*task.Task, error) {
51+
serviceId := curveadm.GetServiceId(dc.GetId())
52+
containerId, err := curveadm.GetContainerId(serviceId)
53+
if curveadm.IsSkip(dc) {
54+
return nil, nil
55+
} else if err != nil {
56+
return nil, err
57+
}
58+
hc, err := curveadm.GetHost(dc.GetHost())
59+
if err != nil {
60+
return nil, err
61+
}
62+
63+
subname := fmt.Sprintf("host=%s role=%s containerId=%s",
64+
dc.GetHost(), dc.GetRole(), tui.TrimContainerId(containerId))
65+
t := task.NewTask("Backup Etcd Data", subname, hc.GetSSHConfig())
66+
67+
host, role := dc.GetHost(), dc.GetRole()
68+
script := scripts.ADD_ETCD
69+
layout := dc.GetProjectLayout()
70+
scriptPath := fmt.Sprintf("%s/add_etcd.sh", layout.ServiceBinDir)
71+
etcdctlPath := layout.ServiceBinDir + "/etcdctl"
72+
endpoints, err := dc.GetVariables().Get("cluster_etcd_addr")
73+
if err != nil {
74+
return nil, errno.ERR_GET_CLUSTER_ETCD_ADDR
75+
}
76+
oldName := fmt.Sprint("etcd", strconv.Itoa(dc.GetHostSequence()), strconv.Itoa(dc.GetInstancesSequence()))
77+
newName := fmt.Sprint("etcd", strconv.Itoa(dc.GetHostSequence()+3), strconv.Itoa(dc.GetInstancesSequence()))
78+
migrates := []*configure.MigrateServer{}
79+
if curveadm.MemStorage().Get(comm.KEY_MIGRATE_SERVERS) != nil {
80+
migrates = curveadm.MemStorage().Get(comm.KEY_MIGRATE_SERVERS).([]*configure.MigrateServer)
81+
}
82+
toService := migrates[0].To
83+
peerUrl := fmt.Sprint("http://", toService.GetListenIp(), ":", strconv.Itoa(toService.GetListenPort()))
84+
amendEtcdConfCmd := fmt.Sprintf("/bin/bash %s %s %s %s %s %s", scriptPath, etcdctlPath, endpoints, oldName, newName, peerUrl)
85+
86+
var success bool
87+
var out string
88+
t.AddStep(&step.ListContainers{
89+
ShowAll: true,
90+
Format: `"{{.ID}}"`,
91+
Filter: fmt.Sprintf("id=%s", containerId),
92+
Out: &out,
93+
ExecOptions: curveadm.ExecOptions(),
94+
})
95+
t.AddStep(&step.Lambda{
96+
Lambda: CheckContainerExist(host, role, containerId, &out),
97+
})
98+
t.AddStep(&step.InstallFile{
99+
ContainerId: &containerId,
100+
ContainerDestPath: scriptPath,
101+
Content: &script,
102+
ExecOptions: curveadm.ExecOptions(),
103+
})
104+
t.AddStep(&step.ContainerExec{
105+
ContainerId: &containerId,
106+
Success: &success,
107+
Out: &out,
108+
Command: amendEtcdConfCmd,
109+
ExecOptions: curveadm.ExecOptions(),
110+
})
111+
t.AddStep(&step.Lambda{
112+
Lambda: checkAddEtcdMemberStatus(&success, &out),
113+
})
114+
115+
return t, nil
116+
}

0 commit comments

Comments
 (0)