virtual.subr

if [ ! "$_CBSD_VIRTUAL_SUBR" ]; then
_CBSD_VIRTUAL_SUBR=1
###

check_iso_sha256sum()
{
	local _sha256sum_new _sha256sum
	local _file _sha256sum_passed

	while getopts "f:m:" opt; do
		case "$opt" in
			f) _file="${OPTARG}" ;;
			m) _sha256sum="${OPTARG}" ;;
		esac
		shift $(($OPTIND - 1))
	done

	if [ -n "${_sha256sum}" -a "${_sha256sum}" != "0" ]; then
		printf "${N1_COLOR}Checking CRC sum: ${N2_COLOR}${_sha256sum}${N1_COLOR}...${N0_COLOR}"
		[ ! -r ${_file} ] && return 1
		_sha256sum_new=$( /usr/bin/nice -n 20 ${SHA256_CMD} -q ${_file} )
		if [ "${_sha256sum}" = "${_sha256sum_new}" ]; then
			${ECHO} "${N2_COLOR}Passed${N0_COLOR}"
			_sha256sum_passed=1
		else
			${ECHO} "${N1_COLOR}Invalid CRC: ${N2_COLOR}${_sha256sum_new}${N0_COLOR}"
			${ECHO} "${N1_COLOR}See more details about CRC: ${N2_COLOR}https://www.bsdstore.ru/en/cbsd_fetch.html${N0_COLOR}"
			_sha256sum_passed=0
		fi
	else
		${ECHO} "${W1_COLOR}Check CRC sum skipped due to settings. ${N1_COLOR}See more: ${N2_COLOR}https://www.bsdstore.ru/en/cbsd_fetch.html${N0_COLOR}"
		_sha256sum_passed=1
	fi

	return ${_sha256sum_passed}
}


# generic/shared func for custom virtuals
# $sha256sum - optional - check for sha256sum
init_iso()
{
	local sha256sum_new
	local sha256sum_passed
	local _sorted_mirror_list _sorted_mirror_list2
	local _sorted_mirror_list_rate
	local _fetch_attempt=0
	local _mirror_num=0 _mirror_cur=0
	local _iso_img_type=
	local _ret _is_link

	readconf fetch.conf

	if [ -z "${iso_img}" ]; then
		${ECHO} "no cd: iso_img and register_iso_as is empty"
		return 1
	fi

	# test for media database
	local media_db

	if [ "${CBSD_ISO_SKIP_CHECKSUM}" = "yes" -o "${sha256sum}" = "0" ]; then
		sha256sum_passed=1
		sha256sum=0
	else
		sha256sum_passed=0
	fi

	case "${iso_img_type}" in
		cloud)
			_iso_img_type="cloud"
			;;
		*)
			_iso_img_type="iso"
			;;
	esac

	if [ -n "${vm_iso_path}" ]; then
		${ECHO} "${N1_COLOR}vm_iso_path: ${N2_COLOR}${vm_iso_path}${N0_COLOR}"

		media_db=$( cbsdsqlro storage_media SELECT path FROM media WHERE name=\"${vm_iso_path}\" AND type=\"${_iso_img_type}\" LIMIT 1 )

		if [ -n "${media_db}" -a -r "${media_db}" ]; then
			${ECHO} "${N1_COLOR}media found: ${N2_COLOR}${vm_iso_path}${N1_COLOR} --> ${N2_COLOR}${media_db}${N0_COLOR}"
			iso_img="${media_db}"
			return 0
		fi
	fi

	local _mydir="${srcdir}/iso"
	local _myfile="${_mydir}/${iso_img}"
	local i

	[ ! -d "${_mydir}" -o ! -h "${_mydir}" ] && ${MKDIR_CMD} -p ${_mydir}

	[ -z "${iso_img_dist}" ] && iso_img_dist="${iso_img}"

	# Unessesary when we boot not from cd
	[ "${vm_boot}" != "cd" ] && iso_img="${_myfile}" && return 0

	if [ -z "${register_iso_name}" ]; then
		_myfile="${_mydir}/${iso_img}"
		${ECHO} "${N1_COLOR}Warning: empty register_iso_name. Use iso_img: ${_myfile}"
	fi

	_myfile=$( media mode=get name="${register_iso_name}" path='' )
	[ -z "${_myfile}" ] && _myfile="${_mydir}/${register_iso_name}"

	if [ -r "${_myfile}" ]; then
		iso_img="${_myfile}"
		# convert CLOUD images into ZVOL when ZFS feature is on

		if [ "${_iso_img_type}" = "cloud" -a "${zfsfeat}" = "1" ]; then
			. ${zfstool}
			DATA=$( ${ZFS_CMD} get -Ho value name ${jaildatadir} )
			_ret=$( ${BASENAME_CMD} ${_myfile} )
			if [ ! -r "/dev/zvol/${DATA}/${_ret}" ]; then
				# create ZVOL from file
				zfs_create_zvol_from_file -f ${_myfile} -n ${_ret} -p ${DATA}
				if [ -r "/dev/zvol/${DATA}/${_ret}" ]; then
					# recreate symlink
					${RM_CMD} -f ${_myfile}
					${LN_CMD} -s /dev/zvol/${DATA}/${_ret} ${_myfile}
				fi
			else
				# vol already exist ?
				# recreate symlink
				is_link=$( ${READLINK_CMD} ${_myfile} )
				if [ "${is_link}" != "/dev/zvol/${DATA}/${_ret}" ]; then
					${RM_CMD} -f ${_myfile}
					${LN_CMD} -s /dev/zvol/${DATA}/${_ret} ${_myfile}
				fi
			fi

			return 0
		fi
	fi

	_mirror_num=$( echo "${iso_site} ${cbsd_iso_mirrors}" | ${WC_CMD} -w | ${AWK_CMD} {'printf $1'} )

	# check for broken link
	if [ -h "${_myfile}" ]; then
		_ret=$( ${READLINK_CMD} ${_myfile} 2>&1 )
		if [ ! -e "${_ret}" ]; then
			${ECHO} "${N2_COLOR}init_iso: remove stale broken symlink: ${N1_COLOR}${_myfile}${N0_COLOR}"
			${RM_CMD} -f ${_myfile}
		fi
	fi

	if [ ! -f "${_myfile}" -a ! -h "${_myfile}" -a "${fetch}" = "1" ]; then
		[ -z "${iso_site}" -a -z "${cbsd_iso_mirrors}" ] && ${ECHO} "${N1_COLOR}No such iso: ${N2_COLOR}${_myfile}${N1_COLOR} and ${N2_COLOR}iso_site/cbsd_iso_mirrors${N1_COLOR} variable not found" && return 1
		${ECHO} "${N1_COLOR}No such media: ${N2_COLOR}${_myfile}${N1_COLOR} in ${N2_COLOR}${_mydir}${N0_COLOR}"
		ALWAYS_YES=1

		if [ "${iso_auto_fetch}" = "1" ]; then
			ointer="${inter}"
			inter=0
		fi

		if getyesno "Shall i download it from: ${iso_site}?"; then
			${ECHO} "${N1_COLOR}Download to: ${N2_COLOR}${_myfile}${N0_COLOR}"

			if [ -n "${iso_extract}" ]; then
				outfile="${_mydir}/${iso_img_dist}"
			else
				outfile="${_myfile}"
			fi

			local win_mirror=
			local win_rate=0
			local t_size=0
			local url=

			local max_url_len=0
			local cur_url_len=0
			local free_len=0

			if [ ${_mirror_num} -gt 1 ]; then
				${ECHO} "${N1_COLOR}Scanning for fastest mirror...${N0_COLOR}"
				_sorted_mirror_list=$( for i in ${iso_site} ${cbsd_iso_mirrors}; do echo ${i}; done | ${SORT_CMD} -u )

				# determine longest url
				for i in ${iso_site} ${cbsd_iso_mirrors}; do
					cur_url_len=$( strlen ${i} )
					[ ${cur_url_len} -gt ${max_url_len} ] && max_url_len=${cur_url_len}
				done

				max_url_len=$(( max_url_len + 1 ))
				max_url_len_header=$(( max_url_len - 15 ))

				printf "%-12s ${UNDERLINE}${H1_COLOR}Mirror source:${N0_COLOR} %-${max_url_len_header}s ${UNDERLINE}${H1_COLOR}Bytes per 3sec:${N0_COLOR}\n"

				_sorted_mirror_list_rate=
				# for broken_crc_fetch_order="cbsd_mirror"
				_sorted_mirror_list2=$( for i in ${cbsd_iso_mirrors} ${iso_site}; do echo ${i}; done | ${SORT_CMD} -u )

				for i in ${_sorted_mirror_list}; do
					_mirror_cur=$(( _mirror_cur + 1 ))
					url="${i}${iso_img_dist}"

					cur_url_len=$( strlen ${i} )
					free_len=$(( max_url_len - cur_url_len ))

					printf " ${H1_COLOR}* ${H2_COLOR}[ %-5s ] ${N2_COLOR}${i}${N0_COLOR}:%-${free_len}s" ${_mirror_cur}/${_mirror_num}
					t_size=$( ${TIMEOUT_CMD} -s HUP 3 ${bindir}/cfetch -s 1 -o /dev/null -u ${url} 2>/dev/null )
					if [ -z "${t_size}" ]; then
						${ECHO} "${W1_COLOR}failed${N0_COLOR}"
						t_size=0
					else
						if [ "${t_size}" = "0" ]; then
							${ECHO} "${N4_COLOR}${t_size}${N0_COLOR}"
						else
							${ECHO} "${H5_COLOR}${t_size}${N0_COLOR}"
						
						fi
						if ! is_number ${t_size}; then
							if [ ${t_size} -gt ${win_rate} ]; then
								win_rate=${t_size}
								win_mirror="${i}"
							fi
						else
							t_size=0
							${ECHO} "${W1_COLOR}failed, no size${N0_COLOR}"
						fi
					fi
					_sorted_mirror_list_rate="${_sorted_mirror_list_rate}${t_size} ${i}\n"
				done

				_sorted_mirror_list_rate=$( printf "${_sorted_mirror_list_rate}" | ${SORT_CMD} -k1 -n -r )
				_sorted_mirror_list=$( printf "${_sorted_mirror_list_rate}" | ${AWK_CMD} {'printf $2" "'} )
				# sorted list "<rate> <mirror>":
				#echo "${_sorted_mirror_list_rate}"

				[ -n "${win_mirror}" ] && ${ECHO} "${H3_COLOR} Winner: ${N2_COLOR}${win_mirror}${N0_COLOR}"
				iso_site="${_sorted_mirror_list}"
			else
				# single site without mirrors
				_sorted_mirror_list2=
			fi
			for i in ${iso_site}; do
				sha256sum_passed=0
				${ECHO} "${N1_COLOR}Processing: ${N2_COLOR}${i}${H2_COLOR}${iso_img_dist}${N0_COLOR}"
				[ -s "${_myfile}" ] && continue # skip all other mirror when file exist
				jswmode jname=${jname} mode=maintenance comment='Fetching_ISO'
				fetchme -o ${outfile} -u ${i}${iso_img_dist}
				jswmode jname=${jname} mode=master comment='Starting'
				check_iso_sha256sum -f ${outfile} -m ${sha256sum}
				sha256sum_passed=$?
				[ -r ${outfile} -a ${sha256sum_passed} -eq 1 ] && break
				[ -r ${outfile} ] && ${RM_CMD} -f ${outfile}
				_fetch_attempt=$(( _fetch_attempt + 1 ))
				if [ "${broken_crc_fetch_order}" = "cbsd_mirror" -a ${_fetch_attempt} -eq 1 ]; then
					# reset mirror list
					for x in ${_sorted_mirror_list2}; do
						${ECHO} "${N1_COLOR}Processing from CBSD Mirrors: ${N2_COLOR}${x}${N0_COLOR}"
						fetchme -o ${outfile} -u ${x}${iso_img_dist}
						check_iso_sha256sum -f ${outfile} -m ${sha256sum}
						sha256sum_passed=$?
						[ -r ${outfile} -a ${sha256sum_passed} -eq 1 ] && break
					done
					break
				fi
			done

			if [ ${sha256sum_passed} -ne 1 ]; then
				${ECHO} "${N1_COLOR}SHA256 sum wrong. Please update profiles or use ${N2_COLOR}CBSD_ISO_SKIP_CHECKSUM=yes${N1_COLOR} variable"
				${ECHO} "via env(1) or ${emulator}-default-default.conf config file to disable sha256sum validating${N0_COLOR}"
			
			fi
		fi
		ALWAYS_YES=0

		# restore inter value
		if [ "{iso_auto_fetch}" = "1" ]; then
			inter="${ointer}"
		fi
	fi

	if [ -n "${iso_extract}" -a -f "${outfile}" ]; then
		${ECHO} "${N1_COLOR}Extracting...${N0_COLOR}"
		set -o xtrace
		cd ${_mydir}
		${iso_extract}
		#${outfile}
		set +o xtrace
		[ -f "${outfile}" ] && ${RM_CMD} -f ${outfile}
		[ -f "${_mydir}/${iso_img}" ] && ${MV_CMD} ${_mydir}/${iso_img} ${_myfile}
	fi

	if [ ! -s "${_myfile}" ]; then
		${ECHO} "${N1_COLOR}No such ISO image. Try to download it manually and put into ${N2_COLOR}${_myfile}${N0_COLOR}"
		err 1 "${N1_COLOR}Or register (via cbsd media mode=register) it with vm_iso_path name: ${N2_COLOR}${vm_iso_path}${N0_COLOR}"
	fi

	if [ -r "${_myfile}" ]; then
		${ECHO} "${N1_COLOR}Automatically register iso as: ${N2_COLOR}${register_iso_name}${N0_COLOR}"
		local reg_name
		if [ -n "${register_iso_as}" ]; then
			reg_name="${register_iso_as}"
		else
			reg_name="${register_iso_name}"
		fi
		media mode=register name="${reg_name}" path="${_myfile}" type="${_iso_img_type}" jname="${jname}"
	fi

	# Final check for ISO existance
	# if symlink - store realpath from symlink
	[ -h ${_myfile} ] && _myfile=$( ${READLINK_CMD} ${_myfile} )

	if [ -r "${_myfile}" ]; then
		iso_img="${_myfile}"
	else
		iso_img=
	fi

	# convert CLOUD images into ZVOL when ZFS feature is on
	if [ "${_iso_img_type}" = "cloud" ]; then
		. ${zfstool}
		convert_source_image_to_zvol -s "${_myfile}"
	fi

	return 0
}

manage_boot_by_empty_hdd()
{
	local _err
	local dsk_size=0
	local tmp_zvol=

	[ "${vm_boot}" != "hdd" ] && return 0
	# check for empty disk
	local checkres

	if [ -z "${MDFILE}" ]; then
		 ${ECHO} "${H5_COLOR}Notice: ${N1_COLOR}no hard disk, boot from CD...${N0_COLOR}"
		vm_boot="cd";
		return 0
	fi

	check_for_empty_hdd path="${data}/${MDFILE}"
	_ret=$?

	[ ${_ret} -eq 2 ] && return 0		# unknown error

	if [ ${_ret} -eq 0 ]; then
		${ECHO} "${N1_COLOR}Looks like ${N2_COLOR}${data}/${MDFILE}${N1_COLOR} is empty.${N0_COLOR}"
		ALWAYS_YES=1
		if getyesno "May be you want to boot from CD? "; then
			${ECHO} "${N1_COLOR}Temporary boot device: ${N2_COLOR}cd${N0_COLOR}"
			cd_boot_once=1
			vm_boot="cd"
		else
			vm_boot="hdd"
		fi
		ALWAYS_YES=0
	else
		# Eject CD/ISO ?
		media_auto_eject=$( cbsdsqlro ${jailsysdir}/${jname}/local.sqlite 'SELECT media_auto_eject FROM settings ORDER BY (created) DESC LIMIT 1;' 2>/dev/null )
		if [ "${media_auto_eject}" = "1" ]; then
			media mode=eject jname=${jname}
			cbsdsqlrw ${jailsysdir}/${jname}/local.sqlite "UPDATE settings SET media_auto_eject='0'"
		fi
	fi
}


# print next free tcp port on specified IP
# do a tcp port scan through nc(1)
# we use mainly for searching free port for VNC/Spice or port for live migration
# -a (optional) address to scan (e.g: -a 127.0.0.1 [ or :: for IPv6 ] which is default)
# -e (mandatory) end port range (e.g: -e 6000)
# -s (mandatory) start port range (e.g: -s 5900)
# -w (optional) nc(1) timeout (e.g: -w 2 which is default)
# -x (optional) skip for "X B C D" values
# return 0 and print found port to stdout when free port available
# return 1 and print '0' to stdout when no free port in current range
get_next_tcp_port()
{
	local _start_port _end_port _nc_timeout _address _ipver _ret
	local _skip i _skipme

	while getopts "a:e:s:w:x:" opt; do
		case "${opt}" in
			a) _address="${OPTARG}" ;;
			e) _end_port="${OPTARG}" ;;
			s) _start_port="${OPTARG}" ;;
			w) _nc_timeout="${OPTARG}" ;;
			x) _skip="${OPTARG}" ;;
		esac
		shift $(($OPTIND - 1))
	done

	# some defaults
	[ -z "${_address}" ] && _address="127.0.0.1"
	[ -z "${_nc_timeout}" ] && _nc_timeout=2

	if [ -z "${_start_port}" -o -z "${_end_port}" ]; then
		${ECHO} "${W1_COLOR}Warning: ${N1_COLOR}${CBSD_APP}: get_next_tcp_port: empty _start_port or _end_port params${N0_COLOR}"
		cbsdlogger WARNING ${CBSD_APP}: get_next_tcp_port: empty _start_port or _end_port params
		return 1
	fi

	if [ ${_start_port} -gt ${_end_port} ]; then
		${ECHO} "${W1_COLOR}Warning: ${N1_COLOR}${CBSD_APP}: _start_port ${_start_port} greater then _end_port ${_end_port}${N0_COLOR}"
		cbsdlogger WARNING ${CBSD_APP}: _start_port ${_start_port} greater then _end_port ${_end_port}
		return 1
	fi

	iptype ${_address}
	_ret=$?
	case ${_ret} in
		2)
			_ipver="-6"
			;;
		*)
			_ipver="-4"
			;;
	esac

	for _tcp_port in $( ${SEQ_CMD} ${_start_port} ${_end_port} ); do
		if [ -n "${_skip}" ]; then
			_skipme=0
			for i in ${_skip}; do
				if ! is_number "${i}"; then
					if [ ${i} -eq ${_tcp_port} ]; then
						_skipme=1
						break
					fi
				fi
			done
			[ ${_skipme} -eq 1 ] && continue
		fi
		${NC_CMD} ${_ipver} -w ${_nc_timeout} -z ${_address} ${_tcp_port} > /dev/null 2>&1 || break
	done

	if [ ${_tcp_port} -eq ${_end_port} ]; then
		cbsdlogger WARNING ${CBSD_APP}: no free port available from ${_start_port} to ${_end_port} range at ${_address}
		echo "0"
		return 1
	fi

	echo ${_tcp_port}
	return 0
}

# check 1 when DMAR detected
# check 0 if not
check_dmar()
{
	local _dmar_test=$( ${ACPIDUMP_CMD} -t | ${GREP_CMD} DMAR )
	[ -z "${_dmar_test}" ] && return 0

	return 1
}

# Init 'system' persist tap interface in bridge
# work around in order to exclude loosing networking upon stop/start first vm
# https://svnweb.freebsd.org/base/head/share/man/man4/bridge.4?r1=324659&r2=324658&pathrev=324659
# https://bugs.freebsd.org/bugzilla/show_bug.cgi?id=221122
# https://bugs.freebsd.org/bugzilla/show_bug.cgi?id=214359
init_systap()
{
	local _i _iface _desc _mytap _sys_uplink _ret _parent_mtu
	local count netwait_if_timeout output _is_bridge _test_ifaces

	_iface=$( ${IFCONFIG_CMD} -l )
	_mytap=

	for _i in ${_iface}; do
		_desc=$( ${IFCONFIG_CMD} ${_i} 2>/dev/null | ${AWK_CMD} '/description:/{print $2}' )
		[ "${_desc}" = "CBSDSYSTEM0" ] && return 0
	done

	. ${distdir}/initenv.subr
	update_netinfo
	if [ "${node_ip6_active}" = "1" ]; then
		_sys_uplink="${CBSD_UPLINK_IFACE6}"
	else
		_sys_uplink="${CBSD_UPLINK_IFACE4}"
	fi

	# no uplink interface detected, skip system tap
	if [ -z "${_sys_uplink}" ]; then
		cbsdlogger NOTICE ${CBSD_APP}: no uplink interface detected. skip init system tap
		return 0
	fi
	_is_phys=$( ${toolsdir}/nic_info --nic=${_sys_uplink} --phys --quiet 2>/dev/null )

	# if interface not physical, skipp init systap
	if [ "${_is_phys}" != "1" ]; then
		cbsdlogger NOTICE ${CBSD_APP}: uplink interface ${_sys_uplink} not physical, skip init system tap
		return 0
	fi

	. ${vimageconf}

	if ! get_vm_uplink_interface -p ${_sys_uplink}; then
		${ECHO} "${N1_COLOR}bstart init_systap warning: error get_vm_uplink_interface ${_sys_uplink}${N0_COLOR}"
		return 1
	fi

	_test_ifaces=$( get_nic_by_descr ${interface} )
	cbsdlogger NOTICE ${CBSD_APP}: init_systap: interfaces found with ${interface} desc: ${_test_ifaces}.
	mybridge=
	for _i in ${_test_ifaces}; do
		_is_bridge=$( ${IFCONFIG_CMD} ${_i} | ${AWK_CMD} '/groups:/{print $2}' | ${GREP_CMD} -o bridge )
		if [ "${_is_bridge}" = "bridge" ]; then
			cbsdlogger NOTICE ${CBSD_APP}: init_systap: the ${_i} interface is in the bridge group. bridge found
			mybridge="${_i}"
			break
		fi
	done

	if [ -z "${mybridge}" ]; then
		if ! mybridge=$( get_my_device bridge ${interface} ); then
			${ECHO} "${N1_COLOR}bstart init_systap warning: errmsg=\"Error:get_my_device_bridge_for_${interface}: ${mybridge}\""
			return 1
		fi
	fi

	_parent_mtu=$( ${toolsdir}/nic_info --nic=${mybridge} --mtu --quiet 2>/dev/null )
	_ret=$?

	_mytap=$( get_my_tap ${mybridge} )
	_ret=$?
	if [ ${_ret} -eq 1 ]; then
		cbsdlogger NOTICE ${CBSD_APP}: bstart init_systap warning: errmsg="Error:get_my_tap_for_${mybridge}: ${mybridge}"
		${ECHO} "${N1_COLOR}bstart init_systap warning: errmsg=\"Error:get_my_tap_for_${mybridge}: ${mybridge}\""
		return 1
	fi

	${IFCONFIG_CMD} ${_mytap} description CBSDSYSTEM0

	# netwait

	link=
	count=1
	# set timeout to 5 seconds
	netwait_if_timeout=5

	${ECHO} "${N1_COLOR}init_systap: waiting for link: ${N2_COLOR}${_sys_uplink}${N1_COLOR}"

	while [ ${count} -le ${netwait_if_timeout} ]; do
		if output=`${IFCONFIG_CMD} ${_sys_uplink} 2>/dev/null`; then
			link=`expr "${output}" : '.*[[:blank:]]status: \(no carrier\)'`
			if [ -z "${link}" ]; then
				break
			fi
		fi
		sleep 1
		count=$((count+1))
	done
}

# export ${dsk_size},${dsk_bsize},${dsk_realsize} variable
# $dsk_size can in form: "100M/200G"
# $dsk_bsize - size in bytes
# require: dsk_path as "dsk1.vhd" or $1 as full path
populate_dsk_size()
{
	local dsk_fullpath
	local tmp_zvol _zvol_pref _is_zvol

	if [ -n "${1}" ]; then
		dsk_fullpath="${1}"
	else
		dsk_fullpath="${jaildatadir}/${jname}-${jaildatapref}/${dsk_path}"
	fi

	dsk_size=0
	dsk_bsize=0
	dsk_realsize=

	if [ -f "${dsk_fullpath}" ]; then
		dsk_realsize=$( ${DU_CMD} -sh ${dsk_fullpath} | ${AWK_CMD} '{printf $1}' 2>/dev/null )
		dsk_bsize=$( ${STAT_CMD} -f "%z" ${dsk_fullpath} 2>/dev/null )
	elif [ -h "${dsk_fullpath}" ]; then
		tmp_zvol=$( ${READLINK_CMD} ${dsk_fullpath} 2>/dev/null )
		_zvol_pref=$( substr --pos=0 --len=10 --str=${tmp_zvol} )
		# not started with /dev/zvol ?
		[ "${_zvol_pref}" != "/dev/zvol/" ] && return 0
		_is_zvol=$( echo ${tmp_zvol} | ${SED_CMD} s:/dev/zvol/::g )
		dsk_bsize=$( ${ZFS_CMD} get -Hp -o value volsize ${_is_zvol} )
	else
		# no such file or symlink
		return 0
	fi

	if conv2human "${dsk_bsize}"; then
		dsk_size=${convval}
	fi

	if [ -z "${dsk_size}" ]; then
		dsk_size=0
		return 0
	fi

	[ -n "${dsk_realsize}" ] && dsk_size="${dsk_realsize}/${dsk_size}"
}

# require: dsk_path as "dsk1.vhd" or -f<arg> as full filename path
# return 0 when ok
# return 1 when error
modify_dsk_size()
{
	local _dsk_fullpath _res
	local _old_dsk_bsize _zvol_rec_size
	local _new_dsk_bsize _zvol_rec_bsize
	local _new_dsk_bdiff _new_dsk_diff
	local tmp_zvol _zvol_pref _is_zvol

	while getopts "f:s:" opt; do
		case "${opt}" in
			f) _dsk_fullpath="${OPTARG}" ;;
			s) _new_size="${OPTARG}" ;;
		esac
		shift $(($OPTIND - 1))
	done

	# md/vnode ?
	if [ -f "${_dsk_fullpath}" ]; then
		_old_dsk_bsize=$( ${STAT_CMD} -f "%z" ${_dsk_fullpath} 2>/dev/null )
		_is_zvol=
	elif [ -h "${_dsk_fullpath}" ]; then
		tmp_zvol=$( ${READLINK_CMD} ${_dsk_fullpath} 2>/dev/null )
		_zvol_pref=$( substr --pos=0 --len=10 --str=${tmp_zvol} )
		# not started with /dev/zvol ?
		if [ "${_zvol_pref}" != "/dev/zvol/" ]; then
			# not zvol
			_dsk_fullpath="${tmp_zvol}"
			_old_dsk_bsize=$( ${STAT_CMD} -f "%z" ${_dsk_fullpath} 2>/dev/null )
			_is_zvol=
		else
			_is_zvol=$( echo ${tmp_zvol} | ${SED_CMD} s:/dev/zvol/::g )
			_old_dsk_bsize=$( ${ZFS_CMD} get -Hp -o value volsize ${_is_zvol} )
		fi
	else
		# no such file or symlink
		${ECHO} "${N1_COLOR}not a file or symlink: ${N2_COLOR}${_dsk_fullpath}${N0_COLOR}" 1>&2
		return 1
	fi

	if [ -z "${_old_dsk_bsize}" -o "${_old_dsk_bsize}" = "1" ]; then
		${ECHO} "${N1_COLOR}unable to calculate current disk size in bytes for ${_dsk_fullpath}: ${N2_COLOR}${_old_dsk_bsize}${N0_COLOR}" 1>&2
		return 1
	fi

	if is_number "${_new_size}"; then
		if conv2bytes ${_new_size}; then
			_new_dsk_bsize="${convval}"
		else
			${ECHO} "${N1_COLOR}unable to convert into bytes: ${N2_COLOR}${_new_size}${N0_COLOR}" 1>&2
			return 1
		fi
	else
		# already on bytes?
		_new_dsk_bsize="${_new_size}"
	fi

	if [ -z "${_new_dsk_bsize}" -o "${_new_dsk_bsize}" = "1" ]; then
		${ECHO} "${N1_COLOR}unable to calculate new disk size in bytes for ${_dsk_fullpath}: ${N2_COLOR}${_new_dsk_bsize}${N0_COLOR}" 1>&2
		return 1
	fi

	if [ ${_old_dsk_bsize} -gt ${_new_dsk_bsize} ]; then
		${ECHO} "${N1_COLOR}${CBSD_APP}: disk reduction is not yet supported (${_old_dsk_bsize} > ${_new_dsk_bsize})${N0_COLOR}" 1>&2
		return 1
	fi

	if [ -n "${_is_zvol}" ]; then
		_zvol_rec_size=$( ${ZFS_CMD} get -Hp -o value volblocksize ${_is_zvol} )

		if is_number "${_zvol_rec_size}"; then
			if conv2bytes ${_zvol_rec_size}; then
				_zvol_rec_bsize="${convval}"
			else
				${ECHO} "${N1_COLOR}unable to convert into bytes: ${N2_COLOR}${_zvol_rec_size}${N0_COLOR}" 1>&2
				return 1
			fi
		else
			# already on bytes?
			_zvol_rec_bsize="${_zvol_rec_size}"
		fi

		# volume size must be a multiple of volume block size
		_new_dsk_bsize=$( roundup --num=${_new_dsk_bsize} --multiple=${_zvol_rec_bsize} 2>/dev/null )
		_new_dsk_bdiff=$(( _new_dsk_bsize - _old_dsk_bsize ))

		cbsdlogger NOTICE ${CBSD_APP}: modify_dsk_size: roundup by recsize ${_zvol_rec_size}: [${_new_dsk_bsize}]
		${ECHO} "${N1_COLOR}resize ${N2_COLOR}${_is_zvol}${N1_COLOR} up to ${N2_COLOR}${_new_dsk_bsize}${N0_COLOR}"
		_res=$( ${ZFS_CMD} set volsize=${_new_dsk_bsize} ${_is_zvol} )
		_ret=$?
		if [ ${_ret} -ne 0 ]; then
			${ECHO} "${N1_COLOR}${CBSD_APP}: modify_dsk_size error: ${_res}${N0_COLOR}" 1>&2
			cbsdlogger NOTICE ${CBSD_APP}: modify_dsk_size error: ${_res}
			return 1
		fi
	else
		_new_dsk_bdiff=$(( _new_dsk_bsize - _old_dsk_bsize ))
		_res=$( ${TRUNCATE_CMD} -s +${_new_dsk_bdiff} ${_dsk_fullpath} )
		_ret=$?
		if [ ${_ret} -ne 0 ]; then
			${ECHO} "${N1_COLOR}${CBSD_APP}: modify_dsk_size error: ${_res}${N0_COLOR}" 1>&2
			cbsdlogger NOTICE ${CBSD_APP}: modify_dsk_size error: ${_res}
			return 1
		fi
	fi

	if conv2human "${_new_dsk_bdiff}"; then
		_new_dsk_diff="${convval}"
	else
		_new_dsk_diff="${_new_dsk_bdiff}"
	fi

	${ECHO} "${N1_COLOR}modify_dsk_size: volume size increased by: ${N2_COLOR}${_new_dsk_diff}${N0_COLOR}" 1>&2
	return 0
}


# print to stdout core count by topology
# $1 - ${_sockets}
# $2 - ${_cores}
# $3 ${_threads}
get_vm_cores_by_topology()
{
	local _sockets _cores _threads _vm_cpus
	_sockets="${1}"
	_cores="${2}"
	_threads="${3}"

	[ -z "${_sockets}" -o -z "${_cores}" -o -z "${_threads}" ] && return 1

	_vm_cpus=$(( _sockets * _cores * _threads ))
	printf "${_vm_cpus}"
	return 0
}

# create new disk image for bhyve/XEN VMs
# -b volblocksize ( must be power of 2 from 512B to 128KB, e.g: 128K)
# -f fsfeat ( 1 or 0 ) - use of skip FS features
# -p full path
# -o (opt) zvol_options (e.g: -o "-o readonly=yes -o dedup=yes")
# -s size (eg: 1g)
# -t type (zvol, md)
#
# if ! virtual_create_dsk -p /tmp/olala.vhd -s 10g -f 0; then
#    echo ";=("
# fi
virtual_create_dsk()
{
	local newdsk imgsize fsfeat imgtype _msg volblocksize= sectorsize=
	local _zvol_rec_size _res zfs_custom_opt
	local _zvol_opts _zvol_opts_merged=

	while getopts "b:f:o:p:s:t:z:" opt; do
		case "${opt}" in
			b) volblocksize="${OPTARG}" ;;
			f) feat="${OPTARG}" ;;
			o) zfs_custom_opt="${OPTARG}" ;;
			p) newdsk="${OPTARG}" ;;
			s) imgsize="${OPTARG}" ;;
			t) imgtype="${OPTARG}" ;;
			z) sectorsize="${OPTARG}" ;;
		esac
		shift $(($OPTIND - 1))
	done

	if [ "${fsfeat}" = "0" ]; then
		zfsfeat=0
		hammerfeat=0
		zfs_custom_opt=
	fi

	local lunname=$( ${BASENAME_CMD} ${newdsk} ) # for ZFS vol
	local datadir=$( ${DIRNAME_CMD} ${newdsk} )  # for ZFS vol

	local convval imgbytes blockcount

	if [ -f "${newdsk}" ]; then
		echo "disk ${newdsk} already exist"
		return 1
	fi

	# test for human
	if is_number ${imgsize}; then
		conv2bytes ${imgsize} || err 1 "${N1_COLOR}conv2bytes from virtual_create_dsk error for: ${N2_COLOR}${imgsize}${N0_COLOR}"
		imgbytes="${convval}"
	else
		imgbytes=${imgsize}
	fi

	[ ! -d "${datadir}" ] && ${MKDIR_CMD} -p ${datadir}
	if [ -z "${imgtype}" ]; then
		case ${zfsfeat} in
			1)
				imgtype="zvol"
				;;
			*)
				imgtype="md"
				;;
		esac
	fi

	if [ "${zfsfeat}" = "1" -a "${imgtype}" = "zvol" ]; then
		# default_vm_zvol_volblocksize etc...
		readconf zfs.conf
		. ${zfstool}

		_zvol_opts=
		_zvol_opts_merged=

		# TODO! check and remove dup with zfs_custom_opt (which is high prio)
		# lookup for "default_vm_zvol_*" defaults
		for i in ${default_vm_zvol_managed_feature}; do
			eval _def_val="\$default_vm_zvol_$i"
			if [ -n "${_def_val}" ]; then
				if [ -z "${_zvol_opts}" ]; then
					_zvol_opts="-o ${i}=${_def_val}"
				else
					_zvol_opts="${_zvol_opts} -o ${i}=${_def_val}"
				fi
			fi
		done

		if [ -z "${zfs_custom_opt}" ]; then
			# no merge with custom zvol_opts=
			_zvol_opts_merged="${_zvol_opts}"
		else
			_zvol_opts_merged=$( zfs_merge_opts -d "${zfs_custom_opt}" -s "${_zvol_opts}" 2>/dev/null )
		fi

		DATA=$( ${ZFS_CMD} get -Ho value name ${datadir} )

		_zvol_rec_size=$( ${ZFS_CMD} get -Hp -o value recordsize ${DATA} )
		# volume size must be a multiple of volume block size
		imgbytes=$( roundup --num=${imgbytes} --multiple=${_zvol_rec_size} 2>/dev/null )
		cbsdlogger NOTICE ${CBSD_APP}: virtual_create_dsk: roundup by recsize ${_zvol_rec_size}: [${imgbytes}]

		# zfs_create_sparse defined via zfs.conf
		if [ "${zfs_create_sparse}" = "0" ]; then
			local _sparse_args=
		else
			local _sparse_args="-s"
		fi

		# check for SPECNAMELEN
		if [ ${freebsdhostversion} -gt 1300030 ]; then
			local _specnamelen=255
		else
			local _specnamelen=63
		fi

		local _zvol_len=$( strlen "zvol/${DATA}/${lunname}" )

		if [ ${_zvol_len} -gt ${_specnamelen} ]; then
			if [ ${_specnamelen} -eq 63 ]; then
				# FreeBSD < 13
				${ECHO} "${N1_COLOR}Error: zvol length too long, greater than ${_specnamelen}: ${N2_COLOR}zvol/${DATA}/${lunname} (${_zvol_len})${N0_COLOR}"
				${ECHO} "${N1_COLOR}See: grep SPECNAMELEN /usr/src/sys/sys/param.h${N0_COLOR}"
				${ECHO} "${N1_COLOR}Please upgrade to FreeBSD 13 or use a shorter name or change SPECNAMELEN${N0_COLOR}"
			else
				${ECHO} "${N1_COLOR}Error: zvol length too long, greater than ${_specnamelen}: ${N2_COLOR}zvol/${DATA}/${lunname} (${_zvol_len})${N0_COLOR}"
				${ECHO} "${N1_COLOR}See: grep SPECNAMELEN /usr/src/sys/sys/param.h. Please use a shorter name or change SPECNAMELEN${N0_COLOR}"
			fi
			return 1
		fi

		_msg=$( ${ZFS_CMD} create -V ${imgbytes} ${_sparse_args} ${_zvol_opts_merged} ${DATA}/${lunname} 2>&1 )
		_res=$?

		if [ ${_res} -ne 0 ]; then
			echo "${_msg}"
			return ${_res}
		fi

		${LN_CMD} -sf /dev/zvol/${DATA}/${lunname} ${newdsk}

	elif [ "${hammerfeat}" = "1" ]; then
		err 1 "Hammer not supported yet"
	else
		blockcount=$(( imgbytes  / 1048576 ))
		[ ! -f "${newdsk}" -a ! -h "${newdsk}" ] && ${TOUCH_CMD} "${newdsk}"
		_msg=$( ${DD_CMD} if=/dev/zero of="${newdsk}" bs=1m count=0 seek=${blockcount} 2>&1 )
		_res=$?
		[ ${_res} -ne 0 ] && echo "${_msg}"
		return ${_res}
	fi

	if [ -z "${sectorsize}" ]; then
		[ -n "${default_sectorsize}" ] && sectorsize="${default_sectorsize}"
		[ -z "${sectorsize}" ] && sectorsize="512"
	fi

	media mode=register name="hdd-${lunname}" path="${newdsk}" type="hdd" jname="${jname}" sectorsize="${sectorsize}"
	return 0
}

# return 0 when free disk name available + print dsk name
# return 1 when no free disk name available
# -j specify jname ( otherwise $jname variable will used )
# -e specify emulator engine ( otherwise $emulator variable will used)
get_next_free_dsk()
{
	local _num=0
	local _jname= _emulator=
	local _freename= _ret=

	[ -n "${jname}" ] && _jname="${jname}"
	[ -n "${emulator}" ] && _emulator="${emulator}"

	while getopts "e:j:" opt; do
		case "${opt}" in
			e) _emulator="${OPTARG}" ;;
			j) _jail="${OPTARG}" ;;
		esac
		shift $(($OPTIND - 1))
	done

	[ -z "${_jname}" -o -z "${_emulator}" ] && return 1

	for _num in $( ${SEQ_CMD} 1 16 ); do
		_freename="dsk${_num}.vhd"
		_ret=$( cbsdsqlro ${jailsysdir}/${jname}/local.sqlite SELECT dsk_path FROM ${emulator}dsk WHERE dsk_path=\"${_freename}\" 2>/dev/null )
		if [ -z "${_ret}" ]; then
			printf "${_freename}"
			return 0
		fi
	done

	# loop/dsk num exhausted
	return 1
}

###
fi