bcache+mdraid1

Signed-off-by: Liao PengFei <[email protected]>

playbook/bcache/hosts.yaml

@@ -0,0 +1,42 @@
+global:
+  user: curve
+  ssh_port: 22
+  private_key_file: /home/curve/.ssh/id_rsa
+
+hosts:
+  - host: server-host1
+    hostname: 10.0.1.1
+    labels:
+      - bcache
+    envs:
+      - SUDO_ALIAS=sudo
+      - BACKING_DEV="/dev/sdc /dev/sdd /dev/sde"
+      - CACHE_DEV_0="/dev/nvme0n1p1 /dev/nvme0n1p2 /dev/nvme0n1p3"
+      - CACHE_DEV_1="/dev/nvme1n1p1 /dev/nvme1n1p2 /dev/nvme1n1p3"
+      - CACHE_MODE=writeback
+      - PERF_TUNE=true
+      - CLEAN_DATA=true
+  - host: server-host2
+    hostname: 10.0.1.2
+    labels:
+      - bcache
+    envs:
+      - SUDO_ALIAS=sudo
+      - BACKING_DEV="/dev/sdb /dev/sdc /dev/sdd"
+      - CACHE_DEV_0="/dev/nvme0n1p1 /dev/nvme0n1p2 /dev/nvme0n1p3"
+      - CACHE_DEV_1="/dev/nvme1n1p1 /dev/nvme1n1p2 /dev/nvme1n1p3"
+      - CACHE_MODE=writeback
+      - PERF_TUNE=true
+      - CLEAN_DATA=true
+  - host: server-host3
+    hostname: 10.0.1.3
+    labels:
+      - bcache
+    envs:
+      - SUDO_ALIAS=sudo
+      - BACKING_DEV="/dev/sdb /dev/sdc /dev/sdd"
+      - CACHE_DEV_0="/dev/nvme0n1p1 /dev/nvme0n1p2 /dev/nvme0n1p3"
+      - CACHE_DEV_1="/dev/nvme1n1p1 /dev/nvme1n1p2 /dev/nvme1n1p3"
+      - CACHE_MODE=writeback
+      - PERF_TUNE=true
+      - CLEAN_DATA=true

playbook/bcache/scripts/clean.sh

@@ -0,0 +1,125 @@
+#!/bin/bash
+
+g_ls="${SUDO_ALIAS} ls"
+g_ps="${SUDO_ALIAS} ps"
+g_cat="${SUDO_ALIAS} cat"
+g_tee="${SUDO_ALIAS} tee"
+g_umount="${SUDO_ALIAS} umount"
+g_wipefs="${SUDO_ALIAS} wipefs"
+g_bcache_super_show="${SUDO_ALIAS} bcache-super-show"
+g_mdadm="${SUDO_ALIAS} mdadm"
+
+
+set_value()
+{
+    local value=$1
+    local path=$2
+    echo ${value} | ${g_tee} ${path} &> /dev/null
+}
+
+pre_check()
+{
+    #check chunkserver is running
+    pid=$(${g_ps} -ef | grep chunkserver | grep -v grep | awk '{print $2}')
+    if [ -n "${pid}" ]; then
+        echo "chunkserver is running, please stop it first"
+        exit 1
+    fi
+
+    #check bcache dirty data
+    for bcache in $(${g_ls} /sys/block | grep bcache)
+    do
+        if [ "$(${g_cat} /sys/block/${bcache}/bcache/dirty_data)" != "0.0k" ]; then
+            echo "${bcache} has dirty data, please stop chunkserver and wait it cleaned"
+            exit 1
+        fi
+    done
+
+    echo "pre_check success"
+}
+
+
+stop_bcache()
+{
+    ${g_umount} /data/chunkserver* &> /dev/null
+    ${g_umount} /data/wal/chunkserver* &> /dev/null
+
+    bcache_devs=$(${g_ls} /sys/block | grep bcache)
+    for bcache in ${bcache_devs}
+    do
+        backdev=/dev/$(${g_cat} /sys/block/${bcache}/bcache/backing_dev_name)
+        uuid=$(${g_bcache_super_show} ${backdev} |grep cset |awk '{print $NF}')
+
+        set_value 1 /sys/block/${bcache}/bcache/detach
+        set_value 1 /sys/fs/bcache/${uuid}/unregister
+        set_value 1 /sys/block/${bcache}/bcache/stop
+    done
+
+    set_value 1 /sys/fs/bcache/pendings_cleanup
+
+    sleep 1
+
+    bcache_devs=$(${g_ls} /sys/block | grep bcache)
+    cache_sets=$(${g_ls} /sys/fs/bcache | grep "-")
+    if [ -n "${bcache_devs}" ] || [ -n "${cache_sets}" ]; then
+        # need retry to wait bcache stop
+        echo "stop bcache failed"
+        exit 1
+    fi
+    echo "stop bcache success"
+}
+
+stop_mdraid1()
+{
+    for cache in $(${g_ls} /dev | grep md)
+    do
+        ${g_mdadm} --stop ${cache}
+        if [ $? != 0 ]; then
+            echo "stop mdraid1 device md${i} failed"
+            exit 1
+        fi
+    done
+}
+
+clean_bcache_data()
+{
+    if [ x"${CLEAN_DATA}" != x"true" ]; then
+        echo "no need to clean data"
+        exit 0
+    fi
+
+    for hdd in ${BACKING_DEV}
+    do
+        ${g_wipefs} -a --force ${hdd} &> /dev/null
+        if [ $? != 0 ]; then
+            echo "wipefs backing device ${hdd} failed"
+            exit 1
+        fi
+    done
+
+    for cache in ${CACHE_DEV_0}
+    do
+        ${g_wipefs} -a --force ${cache} &> /dev/null
+        if [ $? != 0 ]; then
+            echo "wipefs cache device ${cache} failed"
+            exit 1
+        fi
+    done
+
+    for cache in ${CACHE_DEV_1}
+    do
+        ${g_wipefs} -a --force ${cache} &> /dev/null
+        if [ $?!= 0 ]; then
+            echo "wipefs cache device ${cache} failed"
+            exit 1
+        fi
+    done
+
+    echo "clean backing and cache devices data success"
+}
+
+
+pre_check
+stop_bcache
+clean_bcache_data
+

playbook/bcache/scripts/deploy.sh

@@ -0,0 +1,135 @@
+#!/bin/bash
+
+g_ls="${SUDO_ALIAS} ls"
+g_lsmod="${SUDO_ALIAS} lsmod"
+g_modinfo="${SUDO_ALIAS} modinfo"
+g_which="${SUDO_ALIAS} which"
+g_tee="${SUDO_ALIAS} tee"
+g_make_bcache="${SUDO_ALIAS} make-bcache"
+g_bcache_super_show="${SUDO_ALIAS} bcache-super-show"
+g_mdadm="${SUDO_ALIAS} mdadm"
+g_yes="${SUDO_ALIAS} yes"
+
+defalut_cache_mode=none
+
+
+set_value()
+{
+    local value=$1
+    local path=$2
+    echo ${value} | ${g_tee} ${path} &> /dev/null
+}
+
+pre_check()
+{
+    # check bcache-tools is installed
+    if [ -z "$(${g_which} make-bcache)" ]; then
+        echo "make-bcache could not be found"
+        exit 1
+    fi
+
+    if [ -z "$(${g_which} bcache-super-show)" ]; then
+        echo "bcache-super-show could not be found"
+        exit 1
+    fi
+
+    # check mdadm is installed
+    if [ -z "$(${g_which} mdadm)" ]; then
+        echo "mdadm could not be found"
+        exit 1
+    fi
+
+    # check bcache module is exist
+    ${g_modinfo} bcache &> /dev/null
+    if [ $? != 0 ]; then
+        echo "bcache module not be found"
+        exit 1
+    fi
+
+    # check bcache device is exist
+    if [ -n "$(${g_ls} /sys/block | grep bcache)" ];then
+        echo "bcache device is exist, clean it first"
+        exit 1
+    fi
+
+    # check mdraid device is exist
+    if [ -n "$(${g_ls} /dev | grep md)" ];then
+        echo "mdraid device is exist, clean it first"
+        exit 1
+    fi
+
+    # check backend and cache device number
+    if [ $(echo ${BACKING_DEV} |wc -l) != $(echo ${CACHE_DEV_0} |wc -l) ];then
+        echo "only support one cache device with one backing device now!"
+        exit 1 
+    fi
+    if [ $(echo ${BACKING_DEV} |wc -l) != $(echo ${CACHE_DEV_1} |wc -l) ];then
+        echo "only support one cache device with one backing device now!"
+        exit 1 
+    fi
+
+    echo "pre_check success"
+}
+
+deploy_mdraid1()
+{
+    CACHE_ARRAY_0=($CACHE_DEV_0)
+    CACHE_ARRAY_1=($CACHE_DEV_1)
+
+    for ((i=0; i<${#CACHE_ARRAY_0[@]}; i++)); do
+        cache_0=${CACHE_ARRAY_0[i]}
+        cache_1=${CACHE_ARRAY_1[i]}
+        ${g_yes} | ${g_mdadm} --create --verbose /dev/md${i} --level=1 --raid-devices=2 ${cache_0} ${cache_1}
+        if [ $? != 0 ]; then
+            echo "make mdraid1 device md${i} with ${cache_0} and ${cache_1} failed"
+            exit 1
+        fi
+    done
+}
+
+deploy_bcache()
+{
+    for hdd in ${BACKING_DEV}
+    do
+        ${g_make_bcache} -B --wipe-bcache ${hdd} &> /dev/null
+        if [ $? = 0 ]; then
+            set_value ${hdd} /sys/fs/bcache/register
+        else
+            echo "make bcache device ${hdd} failed"
+            exit 1
+        fi
+    done
+
+    for cache in $(${g_ls} /dev | grep md)
+    do
+        ${g_make_bcache} -C --wipe-bcache -b 262144 ${cache} &> /dev/null
+        if [ $? = 0 ]; then
+            set_value ${cache} /sys/fs/bcache/register
+        else
+            echo "make bcache device ${cache} failed"
+            exit 1
+        fi
+    done
+
+    idx=0
+    for cache in $(${g_ls} /dev | grep md)
+    do
+        uuid=$(${g_bcache_super_show} ${cache} | grep cset.uuid | awk '{print $2}')
+        set_value ${uuid} /sys/block/bcache${idx}/bcache/attach
+        idx=$((idx+1))
+    done
+
+    echo "now set cache mode to ${defalut_cache_mode}"
+    # using none mode before chunkfilepool formated
+    for bcache in $(${g_ls} /sys/block | grep bcache)
+    do
+        set_value ${defalut_cache_mode} /sys/block/${bcache}/bcache/cache_mode
+    done
+
+    echo "bcache deploy success, please format chunkfilepool and walfilepool manually"
+}
+
+pre_check
+deploy_mdraid1
+deploy_bcache
+

playbook/bcache/scripts/perf.sh

@@ -0,0 +1,43 @@
+#!/bin/bash
+
+g_ls="${SUDO_ALIAS} ls"
+g_ps="${SUDO_ALIAS} ps"
+g_cat="${SUDO_ALIAS} cat"
+g_tee="${SUDO_ALIAS} tee"
+
+if [ ${PERF_TUNE} != "true" ]; then
+    echo 'PERF_TUNE is not true, exit'
+    exit
+fi
+
+set_value()
+{
+    local value=$1
+    local path=$2
+    echo ${value} | ${g_tee} ${path} &> /dev/null
+}
+
+for bcache in $(${g_ls} /sys/block | grep bcache)
+do
+    backing_dev=$(${g_cat} /sys/block/${bcache}/bcache/backing_dev_name)
+    backing_sectors=$(${g_cat} /sys/block/${backing_dev}/queue/max_sectors_kb)
+    backing_ahead=$(${g_cat} /sys/block/${backing_dev}/queue/read_ahead_kb)
+
+    set_value ${backing_sectors} /sys/block/${bcache}/queue/max_sectors_kb 
+    set_value ${backing_ahead} /sys/block/${bcache}/queue/read_ahead_kb
+    set_value ${CACHE_MODE} /sys/block/${bcache}/bcache/cache_mode
+    set_value 1 /sys/block/${bcache}/bcache/clear_stats
+    set_value 0 /sys/block/${bcache}/bcache/readahead
+    set_value 40 /sys/block/${bcache}/bcache/writeback_percent
+    set_value 10 /sys/block/${bcache}/bcache/writeback_delay
+    set_value 1 /sys/block/${bcache}/bcache/writeback_rate_minimum 
+    set_value 0 /sys/block/${bcache}/bcache/cache/congested_read_threshold_us
+    set_value 0 /sys/block/${bcache}/bcache/cache/congested_write_threshold_us
+    set_value 0 /sys/block/${bcache}/bcache/sequential_cutoff
+    set_value lru /sys/block/${bcache}/bcache/cache/cache0/cache_replacement_policy
+    set_value 1 /sys/block/${bcache}/bcache/cache/internal/gc_after_writeback
+
+done
+
+echo "bcache perf tune success, cache mode is ${CACHE_MODE}"
+

playbook/bcache/scripts/show.sh

@@ -0,0 +1,29 @@
+#!/bin/bash
+
+g_ls="${SUDO_ALIAS} ls"
+g_cat="${SUDO_ALIAS} cat"
+g_which="${SUDO_ALIAS} which"
+g_readlink="${SUDO_ALIAS} readlink"
+
+show_bcache()
+{
+    if [ -n "$(${g_which} bcache-status)" ]; then
+        ${SUDO_ALIAS} bcache-status -s
+    elif [ -n "$(${g_which} bcache)" ]; then
+        ${SUDO_ALIAS} bcache show
+    else
+        for bcache in $(${g_ls} /sys/block | grep bcache)
+        do
+            echo "${bcache} info:"
+            echo "----------------------------"
+            echo "backing device: /dev/$(${g_cat} /sys/block/${bcache}/bcache/backing_dev_name)"
+            echo "cache device: /dev/$(${g_readlink} /sys/block/${bcache}/bcache/cache/cache0 |awk -F'/' '{print $(NF-1)}')"
+            echo "cache mode: $(${g_cat} /sys/block/${bcache}/bcache/cache_mode | grep -oP "(?<=\[)[^\]]*(?=\])")"
+            echo "cache state: $(${g_cat} /sys/block/${bcache}/bcache/state)"
+            echo
+        done
+    fi
+}
+
+show_bcache
+

playbook/bcache/scripts/show_sync.sh

@@ -0,0 +1,17 @@
+#!/bin/bash
+
+g_ls="${SUDO_ALIAS} ls"
+g_mdadm="${SUDO_ALIAS} mdadm"
+
+show_sync()
+{
+    for cache in $(${g_ls} /dev | grep md)
+    do
+        echo "${cache} info:"
+        echo "----------------------------"
+        ${g_mdadm} --detail ${cache}
+        echo
+    done
+}
+
+show_sync