Skip to content

Commit

Permalink
support spdk tgt
Browse files Browse the repository at this point in the history
Signed-off-by: YLShiJustFly <[email protected]>
  • Loading branch information
asignmisspast committed Aug 31, 2023
1 parent e04c5fb commit 7466b1e
Show file tree
Hide file tree
Showing 14 changed files with 720 additions and 11 deletions.
1 change: 0 additions & 1 deletion Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -73,4 +73,3 @@ upload:
lint:
go install github.com/golangci/golangci-lint/cmd/golangci-lint@$(GOLANGCILINT_VERSION)
$(GOBIN_GOLANGCILINT) run -v

2 changes: 1 addition & 1 deletion go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@ require (
github.com/google/uuid v1.3.0
github.com/jpillora/longestcommon v0.0.0-20161227235612-adb9d91ee629
github.com/kpango/glg v1.6.14
github.com/mattn/go-sqlite3 v1.14.16
github.com/mattn/go-sqlite3 v1.14.17
github.com/mcuadros/go-defaults v1.2.0
github.com/melbahja/goph v1.3.0
github.com/mitchellh/hashstructure/v2 v2.0.2
Expand Down
2 changes: 2 additions & 0 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -331,6 +331,8 @@ github.com/mattn/go-runewidth v0.0.14/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh
github.com/mattn/go-sqlite3 v1.6.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc=
github.com/mattn/go-sqlite3 v1.14.16 h1:yOQRA0RpS5PFz/oikGwBEqvAWhWg5ufRz4ETLjwpU1Y=
github.com/mattn/go-sqlite3 v1.14.16/go.mod h1:2eHXhiwb8IkHr+BDWZGa96P6+rkvnG63S2DGjv9HUNg=
github.com/mattn/go-sqlite3 v1.14.17 h1:mCRHCLDUBXgpKAqIKsaAaAsrAlbkeomtRFKXh2L6YIM=
github.com/mattn/go-sqlite3 v1.14.17/go.mod h1:2eHXhiwb8IkHr+BDWZGa96P6+rkvnG63S2DGjv9HUNg=
github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zkfA9PSy5pEvNWRP0ET0TIVo=
github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4=
Expand Down
4 changes: 4 additions & 0 deletions internal/common/common.go
Original file line number Diff line number Diff line change
Expand Up @@ -120,6 +120,10 @@ const (
KEY_TARGET_OPTIONS = "TARGET_OPTIONS"
KEY_ALL_TARGETS = "ALL_TARGETS"

// spdk_target
KEY_SPDK_TARGET_OPTIONS = "SPDK_TARGET_OPTIONS"
KEY_ALL_SPDK_TARGETS = "ALL_SPDK_TARGETS"

// playground
KEY_ALL_PLAYGROUNDS_STATUS = "ALL_PLAYGROUNDS_STATUS"
PLAYGROUDN_STATUS_LOSED = "Losed"
Expand Down
20 changes: 11 additions & 9 deletions internal/errno/errno.go
Original file line number Diff line number Diff line change
Expand Up @@ -419,15 +419,17 @@ var (
ERR_CLIENT_ID_NOT_FOUND = EC(410022, "client id not found")

// 420: common (curvebs client)
ERR_VOLUME_ALREADY_MAPPED = EC(420000, "volume already mapped")
ERR_VOLUME_CONTAINER_LOSED = EC(420001, "volume container is losed")
ERR_VOLUME_CONTAINER_ABNORMAL = EC(420002, "volume container is abnormal")
ERR_CREATE_VOLUME_FAILED = EC(420003, "create volume failed")
ERR_MAP_VOLUME_FAILED = EC(420004, "map volume to NBD device failed")
ERR_ENCODE_VOLUME_INFO_TO_JSON_FAILED = EC(420005, "encode volume info to json failed")
ERR_UNMAP_VOLUME_FAILED = EC(420006, "unmap volume failed")
ERR_OLD_TARGET_DAEMON_IS_ABNORMAL = EC(420007, "old target daemon is abnormal")
ERR_TARGET_DAEMON_IS_ABNORMAL = EC(420008, "target daemon is abnormal")
ERR_VOLUME_ALREADY_MAPPED = EC(420000, "volume already mapped")
ERR_VOLUME_CONTAINER_LOSED = EC(420001, "volume container is losed")
ERR_VOLUME_CONTAINER_ABNORMAL = EC(420002, "volume container is abnormal")
ERR_CREATE_VOLUME_FAILED = EC(420003, "create volume failed")
ERR_MAP_VOLUME_FAILED = EC(420004, "map volume to NBD device failed")
ERR_ENCODE_VOLUME_INFO_TO_JSON_FAILED = EC(420005, "encode volume info to json failed")
ERR_UNMAP_VOLUME_FAILED = EC(420006, "unmap volume failed")
ERR_OLD_TARGET_DAEMON_IS_ABNORMAL = EC(420007, "old target daemon is abnormal")
ERR_TARGET_DAEMON_IS_ABNORMAL = EC(420008, "target daemon is abnormal")
ERR_OLD_SPDK_TARGET_DAEMON_IS_ABNORMAL = EC(420008, "old spdk target daemon is abnormal")
ERR_SPDK_TARGET_DAEMON_IS_ABNORMAL = EC(420009, "spdk target daemon is abnormal")

// 430: common (curvefs client)
ERR_FS_PATH_ALREADY_MOUNTED = EC(430000, "path already mounted")
Expand Down
21 changes: 21 additions & 0 deletions internal/playbook/factory.go
Original file line number Diff line number Diff line change
Expand Up @@ -121,6 +121,13 @@ const (
DELETE_TARGET
LIST_TARGETS

// bs/target
START_SPDK_TARGET_DAEMON
STOP_SPDK_TARGET_DAEMON
ADD_SPDK_TARGET
DELETE_SPDK_TARGET
LIST_SPDK_TARGETS

// fs
CHECK_CLIENT_S3
MOUNT_FILESYSTEM
Expand Down Expand Up @@ -275,6 +282,7 @@ func (p *Playbook) createTasks(step *PlaybookStep) (*tasks.Tasks, error) {
t, err = bs.NewMapTask(curveadm, config.GetCC(i))
case UNMAP_IMAGE:
t, err = bs.NewUnmapTask(curveadm, nil)

// bs/target
case START_TARGET_DAEMON:
t, err = bs.NewStartTargetDaemonTask(curveadm, config.GetCC(i))
Expand All @@ -286,6 +294,19 @@ func (p *Playbook) createTasks(step *PlaybookStep) (*tasks.Tasks, error) {
t, err = bs.NewDeleteTargetTask(curveadm, nil)
case LIST_TARGETS:
t, err = bs.NewListTargetsTask(curveadm, nil)

// bs/target
case START_SPDK_TARGET_DAEMON:
t, err = bs.NewStartSpdkTargetDaemonTask(curveadm, config.GetCC(i))
case STOP_SPDK_TARGET_DAEMON:
t, err = bs.NewStopSpdkTargetDaemonTask(curveadm, nil)
case ADD_SPDK_TARGET:
t, err = bs.NewAddSpdkTargetTask(curveadm, config.GetCC(i))
case DELETE_SPDK_TARGET:
t, err = bs.NewDeleteSpdkTargetTask(curveadm, nil)
case LIST_SPDK_TARGETS:
t, err = bs.NewListSpdkTargetsTask(curveadm, nil)

// fs
case CHECK_CLIENT_S3:
t, err = checker.NewClientS3ConfigureTask(curveadm, config.GetCC(i))
Expand Down
79 changes: 79 additions & 0 deletions internal/task/scripts/add_spdk_target.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,79 @@
/*
* Copyright (c) 2021 NetEase Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

/*
* Project: CurveAdm
* Created Date: 2022-02-08
* Author: Jingli Chen (Wine93)
*/

package scripts

/*
* Usage: target USER VOLUME CREATE SIZE
* Example: target curve test true 10
* See Also: https://linux.die.net/man/8/tgtadm
*/

var ADD__SPDK_TARGET = `
#!/usr/bin/env bash
g_sockname=$1
g_volume=$2
g_originbdev=$3
g_curvebdev=$4
g_rpcpath=$5
g_blocksize=$6
g_cdbpath=$7
g_isexclusive=$8
g_targetip=$9
g_targetport=$10
g_initiatorip=$11
g_initiatormask=$12
g_user=$13
g_create=$14
g_size=%15
g_volumename=%16
mkdir -p /curvebs/nebd/data/lock
touch /etc/curve/curvetab
mkdir -p /curvebs/nebd/data/lock
touch /etc/curve/curvetab
if [ $g_create == "true" ]; then
output=$(curve_ops_tool create -userName=$g_user -fileName=$g_volume -fileLength=$g_size)
if [ $? -ne 0 ]; then
if [ "$output" != "CreateFile fail with errCode: 101" ]; then
exit 1
fi
fi
fi
if ${g_rpcpath} -s ${g_sockname} iscsi_get_target_nodes >/dev/null | grep "$g_curvebdev" | grep name >/dev/null; then
echo "spdk target is already created, please try anther name"
exit 1
fi
${g_rpcpath} -s ${g_sockname} bdev_aio_create ${g_volume} ${g_originbdev} 512 >/dev/null
${g_rpcpath} -s ${g_sockname} bdev_cdb_create -b ${g_curvebdev} --cbd ${g_cbdpath} --exclusive=${g_isexclusive} --blocksize=${g_blocksize} >/dev/null
${g_rpcpath} -s ${g_sockname} bdev_ocf_create bdev_ocf_create myocf wb ${g_originbdev} > /dev/null
${g_rpcpath} -s ${g_sockname} iscsi_create_portal_group 1 ${g_targetip}:${g_targetport} > /dev/null
${g_rpcpath} -s ${g_sockname} iscsi_create_initiator_group 2 ANY ${g_initiatorip}/${g_initiatormask} > /dev/null
${g_rpcpath} -s ${g_sockname} iscsi_create_target_node ${g_volumename} "Data Disk1" "myocf:0 ${g_originbdev}:1" 1:2 64 -d > /dev/null
exit 0
`
76 changes: 76 additions & 0 deletions internal/task/scripts/spdk_target.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,76 @@
/*
* Copyright (c) 2021 NetEase Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

/*
* Project: CurveAdm
* Created Date: 2022-02-08
* Author: Jingli Chen (Wine93)
*/

package scripts

/*
* Usage: target USER VOLUME CREATE SIZE
* Example: target curve test true 10
* See Also: https://linux.die.net/man/8/tgtadm
*/

var SPDK_TARGET = `
#!/usr/bin/env bash
g_binarypath=$1
g_cpumask=$2
g_sockname=$3
g_iscsi_log=$4
mkdir -p /curvebs/nebd/data/lock
touch /etc/curve/curvetab
process_name="iscsi_tgt"
if ps aux | grep -v grep | grep "$process_name" >/dev/null; then
echo "spdk iscsi_tgt has already been started, now exit!"
exit 1
fi
nohup ${g_binarypath} -m ${g_cpumask} -r ${g_sockname} > ${g_iscsi_log} 2>&1 &
i=1
while true
do
result=$(cat $VSDK_LOG_FILE | grep "please run rpc")
if [[ "$result" != "" ]]; then
i=0
break
else
sleep 0.1
i++
fi
if [ "$i" -ge 50 ]; then
break
fi
done
if [ "$i" -eq 0]; then
echo "spdk iscsi_tgt started success!"
else
echo "wait for 5s, spdk iscsi_tgt has not launched, please try again!"
exit 1
fi
exit 0
`
3 changes: 3 additions & 0 deletions internal/task/step/daemon.go
Original file line number Diff line number Diff line change
Expand Up @@ -77,6 +77,9 @@ type AddDaemonTask struct {

type DelDaemonTask struct {
ContainerId *string
Cmd string
Args []string
TaskName string
Tid string
MemStorage *utils.SafeMap
module.ExecOptions
Expand Down
Loading

0 comments on commit 7466b1e

Please sign in to comment.