Skip to content

Commit

Permalink
[feat]curvefs/curvebs: auto restart&mount
Browse files Browse the repository at this point in the history
Signed-off-by: Cyber-SiKu <[email protected]>

Set the restart policy of the client container to unless-stopped,
so that it will automatically remount when the machine restarts

Signed-off-by: Cyber-SiKu <[email protected]>
  • Loading branch information
Cyber-SiKu committed Apr 18, 2023
1 parent 78c415a commit d7db7c6
Show file tree
Hide file tree
Showing 21 changed files with 627 additions and 35 deletions.
5 changes: 3 additions & 2 deletions cli/command/target/list.go
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@ import (
"github.com/opencurve/curveadm/cli/cli"
comm "github.com/opencurve/curveadm/internal/common"
"github.com/opencurve/curveadm/internal/playbook"
"github.com/opencurve/curveadm/internal/task/step"
"github.com/opencurve/curveadm/internal/task/task/bs"
"github.com/opencurve/curveadm/internal/tui"
cliutil "github.com/opencurve/curveadm/internal/utils"
Expand Down Expand Up @@ -80,10 +81,10 @@ func genListPlaybook(curveadm *cli.CurveAdm, options listOptions) (*playbook.Pla
}

func displayTargets(curveadm *cli.CurveAdm) {
targets := []bs.Target{}
targets := []step.Target{}
value := curveadm.MemStorage().Get(comm.KEY_ALL_TARGETS)
if value != nil {
m := value.(map[string]*bs.Target)
m := value.(map[string]*step.Target)
for _, target := range m {
targets = append(targets, *target)
}
Expand Down
7 changes: 7 additions & 0 deletions internal/common/common.go
Original file line number Diff line number Diff line change
Expand Up @@ -130,3 +130,10 @@ const (
AUDIT_STATUS_FAIL
AUDIT_STATUS_CANCEL
)

// container restart policy
const (
POLICY_ALWAYS_RESTART = "always"
POLICY_NEVER_RESTART = "no"
POLICY_UNLESS_STOPPED = "unless-stopped"
)
1 change: 1 addition & 0 deletions internal/errno/errno.go
Original file line number Diff line number Diff line change
Expand Up @@ -539,6 +539,7 @@ var (
ERR_COPY_INTO_CONTAINER_FAILED = EC(630011, "copy file into container failed (docker cp SRC_PATH CONTAINER:DEST_PATH)")
ERR_INSPECT_CONTAINER_FAILED = EC(630012, "get container low-level information failed (docker inspect ID)")
ERR_GET_CONTAINER_LOGS_FAILED = EC(630013, "get container logs failed (docker logs ID)")
ERR_UPDATE_CONTAINER_FAILED = EC(630014, "update container failed (docker update ID)")

// 690: execuetr task (others)
ERR_START_CRONTAB_IN_CONTAINER_FAILED = EC(690000, "start crontab in container failed")
Expand Down
17 changes: 17 additions & 0 deletions internal/task/step/container.go
Original file line number Diff line number Diff line change
Expand Up @@ -152,6 +152,14 @@ type (
Success *bool
module.ExecOptions
}

UpdateContainer struct {
ContainerId *string
Restart string
Out *string
Success *bool
module.ExecOptions
}
)

func (s *DockerInfo) Execute(ctx *context.Context) error {
Expand Down Expand Up @@ -312,3 +320,12 @@ func (s *ContainerLogs) Execute(ctx *context.Context) error {
out, err := cli.Execute(s.ExecOptions)
return PostHandle(s.Success, s.Out, out, err, errno.ERR_GET_CONTAINER_LOGS_FAILED)
}

func (s *UpdateContainer) Execute(ctx *context.Context) error {
cli := ctx.Module().DockerCli().UpdateContainer(*s.ContainerId)
if len(s.Restart) > 0 {
cli.AddOption("--restart %s", s.Restart)
}
out, err := cli.Execute(s.ExecOptions)
return PostHandle(s.Success, s.Out, out, err, errno.ERR_UPDATE_CONTAINER_FAILED)
}
161 changes: 161 additions & 0 deletions internal/task/step/daemon.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,161 @@
/*
* Copyright (c) 2023 NetEase Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

/*
* Project: CurveAdm
* Created Date: 2023-03-16
* Author: Cyber-SiKu
*/

package step

import (
"encoding/json"
"fmt"
"strconv"
"strings"

comm "github.com/opencurve/curveadm/internal/common"
"github.com/opencurve/curveadm/internal/task/context"
"github.com/opencurve/curveadm/internal/utils"
"github.com/opencurve/curveadm/pkg/module"
)

const (
AFTER_TASK_DIR = "/curve/init.d/"
)

var ALLOC_ID int = 0

type afterRunTask struct {
ID int `json:"ID"`
Path string `json:"Path"`
Args []string `json:"Args"`
Env []string `json:"Env"`
Dir string `json:"Dir"`
OutputPath string `json:"OutputPath"`
InputPath string `json:"InputPath"`
}

func (task afterRunTask) ToString() string {
b, err := json.Marshal(task)
if err != nil {
return ""
}
return string(b)
}

func newDaemonTask(path string, args ...string) *afterRunTask {
task := afterRunTask{
ID: ALLOC_ID,
Path: path,
Args: args,
}
return &task
}

type AddDaemonTask struct {
ContainerId *string
Cmd string
Args []string
TaskName string
module.ExecOptions
}

type DelDaemonTask struct {
ContainerId *string
Tid string
MemStorage *utils.SafeMap
module.ExecOptions
}

func (s *AddDaemonTask) getAllocId(ctx *context.Context) {
if ALLOC_ID != 0 {
ALLOC_ID++
return
} else {
// first add daemon task
// create dir AFTER_TASK_DIR
step := ContainerExec{
ContainerId: s.ContainerId,
Command: fmt.Sprintf("mkdir -p %s", AFTER_TASK_DIR),
ExecOptions: s.ExecOptions,
}
err := step.Execute(ctx)
if err != nil {
return
}
}
var count string
// get max id
// The contents of the file are as follows:
// {"ID":1,"Path":"tgtd","Args":null,"Env":null,"Dir":"","OutputPath":"","InputPath":""}
step := ContainerExec{
ContainerId: s.ContainerId,
Command: fmt.Sprintf("grep -r '\"ID\":[0-9]*,' %s | awk -F \":\" '{print $3}' | awk -F \",\" '{print $1}' | sort -n | tail -1", AFTER_TASK_DIR),
Out: &count,
ExecOptions: s.ExecOptions,
}
err := step.Execute(ctx)
if err != nil {
ALLOC_ID = 1
}
id, err := strconv.Atoi(count)
if err != nil {
ALLOC_ID = 1
}
ALLOC_ID = id + 1
}

func (s *AddDaemonTask) Execute(ctx *context.Context) error {
s.getAllocId(ctx)
content := newDaemonTask(s.Cmd, s.Args...).ToString()
step := InstallFile{
Content: &content,
ContainerId: s.ContainerId,
ContainerDestPath: AFTER_TASK_DIR + s.TaskName + ".task",
ExecOptions: s.ExecOptions,
}
return step.Execute(ctx)
}

type Target struct {
Host string
Tid string
Name string
Store string
Portal string
}

func (s *DelDaemonTask) Execute(ctx *context.Context) error {
v := s.MemStorage.Get(comm.KEY_ALL_TARGETS)
target := v.(map[string]*Target)[s.Tid]
if target == nil {
return nil
}
stores := strings.Split(target.Store, "//")
if len(stores) < 2 {
// unable to recognize cbd:pool
return nil
}
path := AFTER_TASK_DIR + "addTarget_" + stores[1] + ".task"
step := ContainerExec{
ContainerId: s.ContainerId,
Command: "rm -f " + path,
ExecOptions: s.ExecOptions,
}
return step.Execute(ctx)
}
27 changes: 21 additions & 6 deletions internal/task/task/bs/add_target.go
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,8 @@ package bs

import (
"fmt"
"regexp"
"strconv"

"github.com/opencurve/curveadm/cli/cli"
comm "github.com/opencurve/curveadm/internal/common"
Expand All @@ -34,12 +36,12 @@ import (
)

type TargetOption struct {
Host string
User string
Volume string
Create bool
Size int
Tid string
Host string
User string
Volume string
Create bool
Size int
Tid string
Blocksize uint64
}

Expand Down Expand Up @@ -91,5 +93,18 @@ func NewAddTargetTask(curveadm *cli.CurveAdm, cc *configure.ClientConfig) (*task
ExecOptions: curveadm.ExecOptions(),
})

t.AddStep(&step.AddDaemonTask{ // install addTarget.task
ContainerId: &containerId,
Cmd: "/bin/bash",
Args: []string{targetScriptPath, user, volume, strconv.FormatBool(options.Create), strconv.Itoa(options.Size), strconv.FormatUint(options.Blocksize, 10)},
TaskName: "addTarget"+TranslateVolumeName(volume, user),
ExecOptions: curveadm.ExecOptions(),
})

return t, nil
}

func TranslateVolumeName(volume, user string) string {
reg, _ := regexp.Compile("[^a-zA-Z0-9]+")
return reg.ReplaceAllString(volume, "_") + "_" + user + "_"
}
1 change: 1 addition & 0 deletions internal/task/task/bs/create_volume.go
Original file line number Diff line number Diff line change
Expand Up @@ -84,6 +84,7 @@ func NewCreateVolumeTask(curveadm *cli.CurveAdm, cc *configure.ClientConfig) (*t
script := scripts.CREATE_VOLUME
scriptPath := "/curvebs/nebd/sbin/create.sh"
command := fmt.Sprintf("/bin/bash %s %s %s %d", scriptPath, options.User, options.Volume, options.Size)

t.AddStep(&step.ListContainers{
ShowAll: true,
Format: "'{{.Status}}'",
Expand Down
18 changes: 18 additions & 0 deletions internal/task/task/bs/delete_target.go
Original file line number Diff line number Diff line change
Expand Up @@ -75,6 +75,24 @@ func NewDeleteTargetTask(curveadm *cli.CurveAdm, cc *client.ClientConfig) (*task
t.AddStep(&step2CheckTgtdStatus{
output: &output,
})
t.AddStep(&step.ContainerExec{
ContainerId: &containerId,
Command: fmt.Sprintf("tgtadm --lld iscsi --mode target --op show"),
Out: &output,
ExecOptions: curveadm.ExecOptions(),
})
t.AddStep(&step2FormatTarget{
host: options.Host,
hostname: hc.GetHostname(),
output: &output,
memStorage: curveadm.MemStorage(),
})
t.AddStep(&step.DelDaemonTask{
ContainerId: &containerId,
Tid: tid,
MemStorage: curveadm.MemStorage(),
ExecOptions: curveadm.ExecOptions(),
})
t.AddStep(&step.ContainerExec{
ContainerId: &containerId,
Command: fmt.Sprintf("tgtadm --lld iscsi --mode target --op delete --tid %s", tid),
Expand Down
18 changes: 5 additions & 13 deletions internal/task/task/bs/list_targets.go
Original file line number Diff line number Diff line change
Expand Up @@ -46,22 +46,14 @@ type (
output *string
memStorage *utils.SafeMap
}

Target struct {
Host string
Tid string
Name string
Store string
Portal string
}
)

func addTarget(memStorage *utils.SafeMap, id string, target *Target) {
func addTarget(memStorage *utils.SafeMap, id string, target *step.Target) {
memStorage.TX(func(kv *utils.SafeMap) error {
m := map[string]*Target{}
m := map[string]*step.Target{}
v := kv.Get(comm.KEY_ALL_TARGETS)
if v != nil {
m = v.(map[string]*Target)
m = v.(map[string]*step.Target)
}
m[id] = target
kv.Set(comm.KEY_ALL_TARGETS, m)
Expand All @@ -84,13 +76,13 @@ func (s *step2FormatTarget) Execute(ctx *context.Context) error {
output := *s.output
lines := strings.Split(output, "\n")

var target *Target
var target *step.Target
titlePattern := regexp.MustCompile("^Target ([0-9]+): (.+)$")
storePattern := regexp.MustCompile("Backing store path: (cbd:pool//.+)$")
for _, line := range lines {
mu := titlePattern.FindStringSubmatch(line)
if len(mu) > 0 {
target = &Target{
target = &step.Target{
Host: s.host,
Tid: mu[1],
Name: mu[2],
Expand Down
14 changes: 14 additions & 0 deletions internal/task/task/bs/map.go
Original file line number Diff line number Diff line change
Expand Up @@ -106,6 +106,13 @@ func NewMapTask(curveadm *cli.CurveAdm, cc *configure.ClientConfig) (*task.Task,
Args: []string{"nbds_max=64"},
ExecOptions: curveadm.ExecOptions(),
})
t.AddStep(&step.AddDaemonTask{ // install modprobe.task
ContainerId: &containerId,
Cmd: "modprobe",
Args: []string{comm.KERNERL_MODULE_NBD, "nbds_max=64"},
TaskName: "modProbe",
ExecOptions: curveadm.ExecOptions(),
})
t.AddStep(&step.SyncFile{ // sync nebd-client config
ContainerSrcId: &containerId,
ContainerSrcPath: "/curvebs/conf/nebd-client.conf",
Expand All @@ -130,6 +137,13 @@ func NewMapTask(curveadm *cli.CurveAdm, cc *configure.ClientConfig) (*task.Task,
Mutate: newToolsV2Mutate(cc, TOOLSV2_CONFIG_DELIMITER),
ExecOptions: curveadm.ExecOptions(),
})
t.AddStep(&step.AddDaemonTask{ // install map.task
ContainerId: &containerId,
Cmd: "/bin/bash",
Args: []string{scriptPath, options.User, options.Volume, mapOptions},
TaskName: "map",
ExecOptions: curveadm.ExecOptions(),
})
t.AddStep(&step.ContainerExec{
ContainerId: &containerId,
Command: command,
Expand Down
1 change: 1 addition & 0 deletions internal/task/task/bs/start_nebd.go
Original file line number Diff line number Diff line change
Expand Up @@ -194,6 +194,7 @@ func NewStartNEBDServiceTask(curveadm *cli.CurveAdm, cc *configure.ClientConfig)
Privileged: true,
Volumes: getVolumes(cc),
Out: &containerId,
Restart: comm.POLICY_UNLESS_STOPPED,
ExecOptions: curveadm.ExecOptions(),
})
t.AddStep(&step2InsertClient{
Expand Down
Loading

0 comments on commit d7db7c6

Please sign in to comment.