From ab13de3486bf55812b1a218f4f0df423923483f3 Mon Sep 17 00:00:00 2001 From: zyb521 Date: Mon, 12 Jun 2023 11:38:39 +0800 Subject: [PATCH 1/2] [feat] curvebs: The formatting and deployment phases support separate partitioning of WAL and data Signed-off-by: zyb521 --- internal/configure/format.go | 23 ++++++++++++++----- internal/configure/topology/dc_get.go | 10 +++++++- internal/configure/topology/dc_item.go | 7 ++++++ internal/errno/errno.go | 1 + internal/task/task/bs/format.go | 23 ++++++++++++++----- internal/task/task/common/create_container.go | 15 +++++++++--- 6 files changed, 63 insertions(+), 16 deletions(-) diff --git a/internal/configure/format.go b/internal/configure/format.go index 7dc7f9140..cd6cb09e9 100644 --- a/internal/configure/format.go +++ b/internal/configure/format.go @@ -32,6 +32,8 @@ import ( const ( DEFAULT_CONTAINER_IMAGE = "opencurvedocker/curvebs:v1.2" + FORMAT_TYPE_DATA = "data" + FORMAT_TYPE_WAL = "wal" ) /* @@ -40,14 +42,18 @@ const ( * - machine2 * - machine3 * disk: - * - /dev/sda:/data/chunkserver0:10 # device:mount_path:format_percent - * - /dev/sdb:/data/chunkserver1:10 - * - /dev/sdc:/data/chunkserver2:10 + * - data:/dev/sda:/data/chunkserver0:10 # fortmat_type:device:mount_path:format_percent + * - data:/dev/sdb:/data/chunkserver1:10 + * - data:/dev/sdc:/data/chunkserver2:10 + * - wal:/dev/nvme0n1p1:/data/wal/chunkserver0:10 + * - wal:/dev/nvme0n1p2:/data/wal/chunkserver1:10 + * - wal:/dev/nvme0n1p3:/data/wal/chunkserver2:10 */ type ( FormatConfig struct { ContainerIamge string Host string + Type string Device string MountPoint string FormtPercent int @@ -64,12 +70,15 @@ type ( func NewFormatConfig(containerImage, host, disk string) (*FormatConfig, error) { items := strings.Split(disk, ":") - if len(items) != 3 { + if len(items) != 4 { return nil, errno.ERR_INVALID_DISK_FORMAT.S(disk) } - device, mountPoint, percent := items[0], items[1], items[2] - if !strings.HasPrefix(device, "/") { + formatType, device, mountPoint, percent := items[0], items[1], items[2], items[3] + if formatType != FORMAT_TYPE_DATA && formatType != FORMAT_TYPE_WAL { + return nil, errno.ERR_INVALID_FORMAT_TYPE. + F("formatType: %s", formatType) + } else if !strings.HasPrefix(device, "/") { return nil, errno.ERR_INVALID_DEVICE. F("device: %s", device) } else if !strings.HasPrefix(mountPoint, "/") { @@ -89,6 +98,7 @@ func NewFormatConfig(containerImage, host, disk string) (*FormatConfig, error) { return &FormatConfig{ ContainerIamge: containerImage, Host: host, + Type: formatType, Device: device, MountPoint: mountPoint, FormtPercent: formatPercent, @@ -136,6 +146,7 @@ func ParseFormat(filename string) ([]*FormatConfig, error) { func (fc *FormatConfig) GetContainerImage() string { return fc.ContainerIamge } func (fc *FormatConfig) GetHost() string { return fc.Host } +func (fc *FormatConfig) GetFormatType() string { return fc.Type } func (fc *FormatConfig) GetDevice() string { return fc.Device } func (fc *FormatConfig) GetMountPoint() string { return fc.MountPoint } func (fc *FormatConfig) GetFormatPercent() int { return fc.FormtPercent } diff --git a/internal/configure/topology/dc_get.go b/internal/configure/topology/dc_get.go index 98a9b8f7c..948b16bf1 100644 --- a/internal/configure/topology/dc_get.go +++ b/internal/configure/topology/dc_get.go @@ -42,6 +42,7 @@ const ( LAYOUT_SERVICE_CONF_DIR = "/conf" LAYOUT_SERVICE_LOG_DIR = "/logs" LAYOUT_SERVICE_DATA_DIR = "/data" + LAYOUT_SERVICE_WAL_DIR = "/wal" LAYOUT_TOOLS_DIR = "/tools" LAYOUT_TOOLS_V2_DIR = "/tools-v2" LAYOUT_CURVEBS_CHUNKFILE_POOL_DIR = "chunkfilepool" @@ -131,6 +132,7 @@ func (dc *DeployConfig) GetContainerImage() string { } func (dc *DeployConfig) GetLogDir() string { return dc.getString(CONFIG_LOG_DIR) } func (dc *DeployConfig) GetDataDir() string { return dc.getString(CONFIG_DATA_DIR) } +func (dc *DeployConfig) GetWalDir() string { return dc.getString(CONFIG_WAL_DIR) } func (dc *DeployConfig) GetCoreDir() string { return dc.getString(CONFIG_CORE_DIR) } func (dc *DeployConfig) GetListenIp() string { return dc.getString(CONFIG_LISTEN_IP) } func (dc *DeployConfig) GetListenPort() int { return dc.getInt(CONFIG_LISTEN_PORT) } @@ -181,7 +183,8 @@ func (dc *DeployConfig) GetListenExternalPort() int { * │ ├── conf * │ ├── data * │ ├── log - * │ └── sbin + * │ ├── sbin + * │ └── wal * ├── snapshotclone * │ ├── conf * │ ├── data @@ -212,6 +215,7 @@ type ( ServiceConfDir string // /curvebs/mds/conf ServiceLogDir string // /curvebs/mds/logs ServiceDataDir string // /curvebs/mds/data + ServiceWalDir string // /curvebs/chunkserver/wal ServiceConfPath string // /curvebs/mds/conf/mds.conf ServiceConfSrcPath string // /curvebs/conf/mds.conf ServiceConfFiles []ConfFile @@ -236,6 +240,9 @@ type ( ChunkfilePoolRootDir string // /curvebs/chunkserver/data ChunkfilePoolDir string // /curvebs/chunkserver/data/chunkfilepool ChunkfilePoolMetaPath string // /curvebs/chunkserver/data/chunkfilepool.meta + WalfilePoolRootDir string // /curvebs/chunkserver/wal + WalfilePoolDir string // /curvebs/chunkserver/wal/walfilepool + WalfilePoolMetaPath string // /curvebs/chunkserver/wal/walfilepool.meta // core CoreSystemDir string @@ -292,6 +299,7 @@ func (dc *DeployConfig) GetProjectLayout() Layout { ServiceConfDir: serviceRootDir + LAYOUT_SERVICE_CONF_DIR, ServiceLogDir: serviceRootDir + LAYOUT_SERVICE_LOG_DIR, ServiceDataDir: serviceRootDir + LAYOUT_SERVICE_DATA_DIR, + ServiceWalDir: serviceRootDir + LAYOUT_SERVICE_WAL_DIR, ServiceConfPath: fmt.Sprintf("%s/%s.conf", serviceConfDir, role), ServiceConfSrcPath: fmt.Sprintf("%s/%s.conf", confSrcDir, role), ServiceConfFiles: serviceConfFiles, diff --git a/internal/configure/topology/dc_item.go b/internal/configure/topology/dc_item.go index 1b6c39740..bdf4158f3 100644 --- a/internal/configure/topology/dc_item.go +++ b/internal/configure/topology/dc_item.go @@ -124,6 +124,13 @@ var ( nil, ) + CONFIG_WAL_DIR = itemset.insert( + "wal_dir", + REQUIRE_STRING, + true, + nil, + ) + CONFIG_CORE_DIR = itemset.insert( "core_dir", REQUIRE_STRING, diff --git a/internal/errno/errno.go b/internal/errno/errno.go index 07de4561c..1e433e2b7 100644 --- a/internal/errno/errno.go +++ b/internal/errno/errno.go @@ -355,6 +355,7 @@ var ( ERR_MOUNT_POINT_REQUIRE_ABSOLUTE_PATH = EC(341002, "mount point must be an absolute path") ERR_FORMAT_PERCENT_REQUIRES_INTERGET = EC(341003, "format percentage requires an integer") ERR_FORMAT_PERCENT_MUST_BE_BETWEEN_1_AND_100 = EC(341004, "format percentage must be between 1 and 100") + ERR_INVALID_FORMAT_TYPE = EC(341005, "format type must be (data) or (wal)") // 350: configure (client.yaml: parse failed) ERR_PARSE_CLIENT_CONFIGURE_FAILED = EC(350000, "parse client configure failed") diff --git a/internal/task/task/bs/format.go b/internal/task/task/bs/format.go index e9a04aafc..b3e36eab9 100644 --- a/internal/task/task/bs/format.go +++ b/internal/task/task/bs/format.go @@ -231,22 +231,33 @@ func NewFormatChunkfilePoolTask(curveadm *cli.CurveAdm, fc *configure.FormatConf } // new task + formatTpye := fc.GetFormatType() device := fc.GetDevice() mountPoint := fc.GetMountPoint() usagePercent := fc.GetFormatPercent() - subname := fmt.Sprintf("host=%s device=%s mountPoint=%s usage=%d%%", - fc.GetHost(), device, mountPoint, usagePercent) - t := task.NewTask("Start Format Chunkfile Pool", subname, hc.GetSSHConfig()) + subname := fmt.Sprintf("host=%s formatTpye=%s device=%s mountPoint=%s usage=%d%%", + fc.GetHost(), formatTpye, device, mountPoint, usagePercent) + t := task.NewTask("Start Format filePool", subname, hc.GetSSHConfig()) // add step to task var oldContainerId, containerId, oldUUID string + var filePoolRootDir, filePoolDir, filePoolMetaPath string containerName := device2ContainerName(device) layout := topology.GetCurveBSProjectLayout() - chunkfilePoolRootDir := layout.ChunkfilePoolRootDir + + if formatTpye == configure.FORMAT_TYPE_WAL { + filePoolRootDir = layout.WalfilePoolRootDir + filePoolDir = layout.WalfilePoolDir + filePoolMetaPath = layout.WalfilePoolMetaPath + } else { + filePoolRootDir = layout.ChunkfilePoolRootDir + filePoolDir = layout.ChunkfilePoolDir + filePoolMetaPath = layout.ChunkfilePoolMetaPath + } formatScript := scripts.SCRIPT_FORMAT formatScriptPath := fmt.Sprintf("%s/format.sh", layout.ToolsBinDir) formatCommand := fmt.Sprintf("%s %s %d %d %s %s", formatScriptPath, layout.FormatBinaryPath, - usagePercent, DEFAULT_CHUNKFILE_SIZE, layout.ChunkfilePoolDir, layout.ChunkfilePoolMetaPath) + usagePercent, DEFAULT_CHUNKFILE_SIZE, filePoolDir, filePoolMetaPath) // 1: skip if formating container exist t.AddStep(&step.ListContainers{ @@ -320,7 +331,7 @@ func NewFormatChunkfilePoolTask(curveadm *cli.CurveAdm, fc *configure.FormatConf Entrypoint: "/bin/bash", Name: containerName, Remove: true, - Volumes: []step.Volume{{HostPath: mountPoint, ContainerPath: chunkfilePoolRootDir}}, + Volumes: []step.Volume{{HostPath: mountPoint, ContainerPath: filePoolRootDir}}, Out: &containerId, ExecOptions: curveadm.ExecOptions(), }) diff --git a/internal/task/task/common/create_container.go b/internal/task/task/common/create_container.go index 7de49259c..30b0d6afe 100644 --- a/internal/task/task/common/create_container.go +++ b/internal/task/task/common/create_container.go @@ -106,6 +106,7 @@ func getArguments(dc *topology.DeployConfig) string { // only chunkserver need so many arguments, but who cares layout := dc.GetProjectLayout() dataDir := layout.ServiceDataDir + walDir := layout.ServiceWalDir chunkserverArguments := map[string]interface{}{ // chunkserver "conf": layout.ServiceConfPath, @@ -115,11 +116,11 @@ func getArguments(dc *topology.DeployConfig) string { "chunkServerPort": dc.GetListenPort(), "chunkFilePoolDir": dataDir, "chunkFilePoolMetaPath": fmt.Sprintf("%s/chunkfilepool.meta", dataDir), - "walFilePoolDir": dataDir, - "walFilePoolMetaPath": fmt.Sprintf("%s/walfilepool.meta", dataDir), + "walFilePoolDir": walDir, + "walFilePoolMetaPath": fmt.Sprintf("%s/walfilepool.meta", walDir), "copySetUri": fmt.Sprintf("local://%s/copysets", dataDir), "recycleUri": fmt.Sprintf("local://%s/recycler", dataDir), - "raftLogUri": fmt.Sprintf("curve://%s/copysets", dataDir), + "raftLogUri": fmt.Sprintf("curve://%s/copysets", walDir), "raftSnapshotUri": fmt.Sprintf("curve://%s/copysets", dataDir), "chunkServerStoreUri": fmt.Sprintf("local://%s", dataDir), "chunkServerMetaUri": fmt.Sprintf("local://%s/chunkserver.dat", dataDir), @@ -158,6 +159,7 @@ func getMountVolumes(dc *topology.DeployConfig, serviceMountDevice bool) []step. layout := dc.GetProjectLayout() logDir := dc.GetLogDir() dataDir := dc.GetDataDir() + walDir := dc.GetWalDir() coreDir := dc.GetCoreDir() if len(logDir) > 0 { @@ -175,6 +177,13 @@ func getMountVolumes(dc *topology.DeployConfig, serviceMountDevice bool) []step. }) } + if len(walDir) > 0 { + volumes = append(volumes, step.Volume{ + HostPath: walDir, + ContainerPath: layout.ServiceWalDir, + }) + } + if len(coreDir) > 0 { volumes = append(volumes, step.Volume{ HostPath: coreDir, From e2eea99fc315d3dbd439abc12ab851401273c27e Mon Sep 17 00:00:00 2001 From: zyb521 Date: Wed, 19 Jul 2023 16:33:30 +0800 Subject: [PATCH 2/2] [fix] some bug fix, and support clean walfiles Signed-off-by: zyb521 --- internal/configure/topology/dc_get.go | 6 ++++++ internal/task/task/common/clean_service.go | 17 +++++++++++++++++ internal/task/task/common/create_container.go | 3 +++ 3 files changed, 26 insertions(+) diff --git a/internal/configure/topology/dc_get.go b/internal/configure/topology/dc_get.go index 948b16bf1..9b464a6d7 100644 --- a/internal/configure/topology/dc_get.go +++ b/internal/configure/topology/dc_get.go @@ -46,6 +46,7 @@ const ( LAYOUT_TOOLS_DIR = "/tools" LAYOUT_TOOLS_V2_DIR = "/tools-v2" LAYOUT_CURVEBS_CHUNKFILE_POOL_DIR = "chunkfilepool" + LAYOUT_CURVEBS_WALFILE_POOL_DIR = "walfilepool" LAYOUT_CURVEBS_COPYSETS_DIR = "copysets" LAYOUT_CURVEBS_RECYCLER_DIR = "recycler" LAYOUT_CURVEBS_TOOLS_CONFIG_SYSTEM_PATH = "/etc/curve/tools.conf" @@ -58,6 +59,7 @@ const ( BINARY_CURVEFS_TOOL = "curvefs_tool" BINARY_CURVE_TOOL_V2 = "curve" METAFILE_CHUNKFILE_POOL = "chunkfilepool.meta" + METAFILE_WALFILE_POOL = "walfilepool.meta" METAFILE_CHUNKSERVER_ID = "chunkserver.dat" ) @@ -285,6 +287,7 @@ func (dc *DeployConfig) GetProjectLayout() Layout { // format chunkserverDataDir := fmt.Sprintf("%s/%s%s", root, ROLE_CHUNKSERVER, LAYOUT_SERVICE_DATA_DIR) + chunkserverWalDir := fmt.Sprintf("%s/%s%s", root, ROLE_CHUNKSERVER, LAYOUT_SERVICE_WAL_DIR) return Layout{ // project @@ -324,6 +327,9 @@ func (dc *DeployConfig) GetProjectLayout() Layout { ChunkfilePoolRootDir: chunkserverDataDir, ChunkfilePoolDir: fmt.Sprintf("%s/%s", chunkserverDataDir, LAYOUT_CURVEBS_CHUNKFILE_POOL_DIR), ChunkfilePoolMetaPath: fmt.Sprintf("%s/%s", chunkserverDataDir, METAFILE_CHUNKFILE_POOL), + WalfilePoolRootDir: chunkserverWalDir, + WalfilePoolDir: fmt.Sprintf("%s/%s", chunkserverWalDir, LAYOUT_CURVEBS_WALFILE_POOL_DIR), + WalfilePoolMetaPath: fmt.Sprintf("%s/%s", chunkserverWalDir, METAFILE_WALFILE_POOL), // core CoreSystemDir: LAYOUT_CORE_SYSTEM_DIR, diff --git a/internal/task/task/common/clean_service.go b/internal/task/task/common/clean_service.go index e800e70f1..ec3b4c693 100644 --- a/internal/task/task/common/clean_service.go +++ b/internal/task/task/common/clean_service.go @@ -46,6 +46,7 @@ import ( const ( LAYOUT_CURVEBS_CHUNKFILE_POOL_DIR = topology.LAYOUT_CURVEBS_CHUNKFILE_POOL_DIR + LAYOUT_CURVEBS_WALFILE_POOL_DIR = topology.LAYOUT_CURVEBS_WALFILE_POOL_DIR LAYOUT_CURVEBS_COPYSETS_DIR = topology.LAYOUT_CURVEBS_COPYSETS_DIR LAYOUT_CURVEBS_RECYCLER_DIR = topology.LAYOUT_CURVEBS_RECYCLER_DIR METAFILE_CHUNKSERVER_ID = topology.METAFILE_CHUNKSERVER_ID @@ -97,6 +98,17 @@ func (s *step2RecycleChunk) Execute(ctx *context.Context) error { if err != nil { errno.ERR_RUN_SCRIPT_FAILED.E(err) } + + walDir := dc.GetWalDir() + if walDir != "" { + source = fmt.Sprintf("%s/%s", walDir, LAYOUT_CURVEBS_COPYSETS_DIR) + dest = fmt.Sprintf("%s/%s", walDir, LAYOUT_CURVEBS_WALFILE_POOL_DIR) + cmd = ctx.Module().Shell().BashScript(s.recycleScriptPath, source, dest, chunkSize) + _, err = cmd.Execute(s.execOptions) + if err != nil { + errno.ERR_RUN_SCRIPT_FAILED.E(err) + } + } return nil } @@ -133,9 +145,14 @@ func getCleanFiles(clean map[string]bool, dc *topology.DeployConfig, recycle boo files = append(files, dc.GetDataDir()) } else { dataDir := dc.GetDataDir() + walDir := dc.GetWalDir() copysetsDir := fmt.Sprintf("%s/%s", dataDir, LAYOUT_CURVEBS_COPYSETS_DIR) chunkserverIdMetafile := fmt.Sprintf("%s/%s", dataDir, METAFILE_CHUNKSERVER_ID) files = append(files, copysetsDir, chunkserverIdMetafile) + if walDir != "" { + walCopysetsDir := fmt.Sprintf("%s/%s", walDir, LAYOUT_CURVEBS_COPYSETS_DIR) + files = append(files, walCopysetsDir) + } } } } diff --git a/internal/task/task/common/create_container.go b/internal/task/task/common/create_container.go index 30b0d6afe..1afabbf6a 100644 --- a/internal/task/task/common/create_container.go +++ b/internal/task/task/common/create_container.go @@ -107,6 +107,9 @@ func getArguments(dc *topology.DeployConfig) string { layout := dc.GetProjectLayout() dataDir := layout.ServiceDataDir walDir := layout.ServiceWalDir + if dc.GetWalDir() == "" { + walDir = dataDir + } chunkserverArguments := map[string]interface{}{ // chunkserver "conf": layout.ServiceConfPath,