diff --git a/build-tests/x86/tumbleweed/test-image-disk/appliance.kiwi b/build-tests/x86/tumbleweed/test-image-disk/appliance.kiwi index 5bf73f473e0..c00882d6560 100644 --- a/build-tests/x86/tumbleweed/test-image-disk/appliance.kiwi +++ b/build-tests/x86/tumbleweed/test-image-disk/appliance.kiwi @@ -1,11 +1,15 @@ - + Marcus Schäfer ms@suse.com Disk test build + + + + 1.42.1 zypper @@ -16,10 +20,13 @@ false breeze openSUSE + + true + true 1024 false @@ -28,8 +35,28 @@ + + + + + true + true + 512 + + 2048 + false + true + + + + + + 3 + + + diff --git a/build-tests/x86/tumbleweed/test-image-disk/config.sh b/build-tests/x86/tumbleweed/test-image-disk/config.sh index cda81d9c4b6..3551376a6d8 100644 --- a/build-tests/x86/tumbleweed/test-image-disk/config.sh +++ b/build-tests/x86/tumbleweed/test-image-disk/config.sh @@ -1,9 +1,22 @@ #!/bin/bash set -ex +declare kiwi_profiles=${kiwi_profiles} + #====================================== # Activate services #-------------------------------------- systemctl enable sshd systemctl enable grub_config systemctl enable dracut_hostonly + +# Just in case the kiwi resizer is disabled in the system and +# gets dropped from the initrd by the customer for some reason +for profile in ${kiwi_profiles//,/ }; do + if [ "${profile}" = "Retain" ]; then + cat > /etc/fstab.script <<-EOF + sed -ie "s@/home ext4 defaults@/home ext4 x-systemd.growfs,defaults@" /etc/fstab + rm /etc/fstabe + EOF + fi +done diff --git a/doc/source/concept_and_workflow/customize_the_boot_process.rst b/doc/source/concept_and_workflow/customize_the_boot_process.rst index 379b2d26478..6a0f661069d 100644 --- a/doc/source/concept_and_workflow/customize_the_boot_process.rst +++ b/doc/source/concept_and_workflow/customize_the_boot_process.rst @@ -222,6 +222,17 @@ the available kernel boot parameters for these modules: Note that options starting with `rd.kiwi` are not passed to avoid side effects. +``rd.kiwi.install.retain_last`` + Instructs an OEM installation to retain the contents of the + last partition on the target disk. This setting is only useful + if the last partition does not belong to the main OS e.g. an + extra data partition added via the `spare_part` attribute in + the type setup of the image description. The implementation + also checks if the start address of the last partition on the + target disk matches with the start adress of the image to be + deployed. Only if they match the data on the last partition + can be retained. + ``rd.kiwi.oem.luks.reencrypt`` For OEM LUKS2 encrypted disk images. If set, reencrypts the disk prior an eventual resize and therefore creates a new key pool and diff --git a/dracut/modules.d/55kiwi-dump/kiwi-dump-image.sh b/dracut/modules.d/55kiwi-dump/kiwi-dump-image.sh index c9d453d14a6..c60e43b0a18 100755 --- a/dracut/modules.d/55kiwi-dump/kiwi-dump-image.sh +++ b/dracut/modules.d/55kiwi-dump/kiwi-dump-image.sh @@ -330,15 +330,50 @@ function dump_image { local load_text="Loading ${image_basename}..." local title_text="Installation..." local dump + local parttable + local count_32k + local image_size + local count=0 + local block_size=32k + + # can we dump this + check_image_fits_target "${image_target}" + # select dump method if [ -n "${image_from_remote}" ];then dump=dump_remote_image else dump=dump_local_image fi - check_image_fits_target "${image_target}" + # setup blocks and blocksize to retain last + if getargbool 0 rd.kiwi.install.retain_last; then + if [ -n "${image_from_remote}" ];then + image_size=$((blocks * blocksize)) + parttable=$( + fetch_remote_partition_table "${image_source}" "${image_size}" + ) + else + parttable=$( + fetch_local_partition_table "${image_source}" + ) + fi + if compatible_to_retain "${parttable}" "${image_target}"; then + count=$(get_disk_offset_retain_last_partition "${parttable}") + fi + if [ "${count}" -gt 0 ];then + block_size=$(get_sector_size_from_table_dump "${parttable}") + count_32k=$( + optimize_count_for_32k_blocksize "${block_size}" "${count}" + ) + if [ ! "${count}" = "${count_32k}" ];then + count="${count_32k}" + block_size=32k + fi + fi + fi + # last chance to stop us if [ "${kiwi_oemunattended}" = "false" ];then local ack_dump_text="Destroying ALL data on ${image_target}, continue ?" if ! run_dialog --yesno "\"${ack_dump_text}\"" 7 80; then @@ -347,48 +382,157 @@ function dump_image { fi fi + # deploy echo "${load_text} [${image_target}]..." if command -v pv &>/dev/null && [ "${kiwi_oemsilentinstall}" = "false" ] then # dump with dialog based progress information setup_progress_fifo ${progress} - eval "${dump}" "${image_source}" "${image_target}" "${progress}" & + eval \ + "${dump}" \ + "${image_source}" \ + "${image_target}" \ + "${count}" \ + "${block_size}" \ + "${progress}" \ + & run_progress_dialog "${load_text}" "${title_text}" else # dump with silently blocked console - if ! eval "${dump}" "${image_source}" "${image_target}"; then + if ! eval \ + "${dump}" \ + "${image_source}" \ + "${image_target}" \ + "${count}" \ + "${block_size}" + then report_and_quit "Failed to install image" fi fi } +function fetch_local_partition_table { + local image_source=$1 + local parttable=/tmp/parttable + sfdisk -d "${image_source}" > "${parttable}" 2>/dev/null + echo "${parttable}" +} + +function fetch_remote_partition_table { + local image_source=$1 + local image_size=$2 + local parttable=/tmp/parttable + dd if=/dev/zero of=/tmp/table bs=1 count=0 seek="${image_size}" &>/dev/null + fetch_file "${image_source}" 2>/dev/null | dd of=/tmp/table bs=512 count=1 conv=notrunc &>/dev/null + sfdisk -d /tmp/table > "${parttable}" 2>/dev/null + echo "${parttable}" +} + +function get_sector_size_from_table_dump { + local parttable=$1 + sector_size=$(grep sector-size: "${parttable}" | cut -f2 -d:) + echo "${sector_size}" +} + +function compatible_to_retain { + local parttable=$1 + local image_target=$2 + local source_start + local target_start + source_start=$( + tail -n 1 "${parttable}" | cut -f2 -d= | cut -f1 -d, + ) + target_start=$( + sfdisk -d "${image_target}" 2>/dev/null |\ + tail -n 1 | cut -f2 -d= | cut -f1 -d, + ) + if [ -z "${source_start}" ] || [ -z "${target_start}" ];then + # no partition information for either source or target + # broken or net new deployment on empty disk + touch /tmp/retain_not_applicable + return 1 + fi + if [ ! "${source_start}" = "${target_start}" ];then + report_and_quit "Cannot retain partition, start address mismatch" + fi + return 0 +} + +function get_disk_offset_retain_last_partition { + local parttable=$1 + local next_to_last + local start + local size + local offset + next_to_last=$(tail -n 2 "${parttable}" | head -n 1) + if [ -n "${next_to_last}" ];then + start=$(echo "${next_to_last}" | cut -f2 -d= | cut -f1 -d,) + size=$(echo "${next_to_last}" | cut -f3 -d= | cut -f1 -d,) + offset=$((start + size)) + echo "${offset}" + else + echo 0 + fi +} + +function optimize_count_for_32k_blocksize { + local block_size=$1 + local count=$2 + local dump_bytes + dump_bytes=$((block_size * count)) + if [ $((dump_bytes % 32768)) -eq 0 ];then + # dump_bytes is a multiple of 32k, use it for better I/O performance + count=$((dump_bytes / 32768)) + fi + echo "${count}" +} + function dump_local_image { local image_source=$1 local image_target=$2 - local progress=$3 + local count=$3 + local block_size=$4 + local progress=$5 + if [ "${count}" -gt 0 ];then + count="count=${count}" + else + unset count + fi + # shellcheck disable=SC2086 if [ -e "${progress}" ];then ( - pv -n "${image_source}" | dd bs=32k of="${image_target}" &>/dev/null + pv -n "${image_source}" |\ + dd bs="${block_size}" ${count} of="${image_target}" &>/dev/null ) 2>"${progress}" else - dd if="${image_source}" bs=32k of="${image_target}" &>/dev/null + dd \ + if="${image_source}" bs="${block_size}" ${count} \ + of="${image_target}" &>/dev/null fi } function dump_remote_image { local image_source=$1 local image_target=$2 - local progress=$3 + local count=$3 + local block_size=$4 + local progress=$5 local image_size image_size=$((blocks * blocksize)) + if [ "${count}" -gt 0 ];then + count="count=${count}" + else + unset count + fi + # shellcheck disable=SC2086 if [ -e "${progress}" ];then ( fetch_file "${image_source}" "${image_size}" |\ - dd bs=32k of="${image_target}" &>/dev/null + dd bs="${block_size}" ${count} of="${image_target}" &>/dev/null ) 2>"${progress}" else fetch_file "${image_source}" |\ - dd bs=32k of="${image_target}" &>/dev/null + dd bs="${block_size}" ${count} of="${image_target}" &>/dev/null fi } @@ -406,6 +550,13 @@ function check_image_integrity { # no verification wanted return fi + if getargbool 0 rd.kiwi.install.retain_last; then + if [ ! -e /tmp/retain_not_applicable ];then + # no verification possible as only a portion of + # the image got deployed intentionally + return + fi + fi if command -v pv &>/dev/null && [ "${kiwi_oemsilentverify}" = "false" ] then # verify with dialog based progress information diff --git a/dracut/modules.d/55kiwi-repart/kiwi-repart-disk.sh b/dracut/modules.d/55kiwi-repart/kiwi-repart-disk.sh index dc03fe0ac98..082bf3edc2e 100755 --- a/dracut/modules.d/55kiwi-repart/kiwi-repart-disk.sh +++ b/dracut/modules.d/55kiwi-repart/kiwi-repart-disk.sh @@ -47,8 +47,8 @@ function initialize { export disk fi - root_device=${root#block:} - export root_device + last_device=$(get_last_partition_device "${disk}") + export last_device disk_free_mbytes=$(( $(get_free_disk_bytes "${disk}") / 1048576 @@ -56,11 +56,33 @@ function initialize { export disk_free_mbytes disk_root_mbytes=$(( - $(get_block_device_kbsize "${root_device}") / 1024 + $(get_block_device_kbsize "${last_device}") / 1024 )) export disk_root_mbytes } +function get_last_partition_id_from_config_file { + # """ + # Read the partition id with the biggest value from + # config.partids. This method will be replaced by + # get_last_partition_id() in the future. As of today + # there is still code in 59kiwi-lib which reads + # the imported variables from config.partids. This + # code needs to be refactored first before we can + # get rid of config.partids + # """ + local partition_ids=/config.partids + local partid=1 + local partid_cur + while read -r partname; do + partid_cur=$(echo "${partname}" | cut -f2 -d= | tr -d \") + if [ "${partid_cur}" -gt "${partid}" ];then + partid="${partid_cur}" + fi + done < "${partition_ids}" + echo "${partid}" +} + function deactivate_device_mappings { if lvm_system;then deactivate_volume_group @@ -74,10 +96,11 @@ function deactivate_device_mappings { } function finalize_disk_repart { - declare kiwi_RootPart=${kiwi_RootPart} + local kiwi_ResizePart + kiwi_ResizePart=$(get_last_partition_id_from_config_file) finalize_partition_table "${disk}" set_root_map \ - "$(get_partition_node_name "${disk}" "${kiwi_RootPart}")" + "$(get_partition_node_name "${disk}" "${kiwi_ResizePart}")" } function get_target_rootpart_size { @@ -103,9 +126,14 @@ function repart_standard_disk { # pX+1: ( root ) [+luks +raid] # ------------------------------------- # """ - declare kiwi_RootPart=${kiwi_RootPart} local kiwi_oemrootMB + local kiwi_ResizePart + local command_query + local root_part_size + local part_name kiwi_oemrootMB=$(get_target_rootpart_size) + kiwi_ResizePart=$(get_last_partition_id_from_config_file) + part_name=$(get_partition_name "${disk}" "${kiwi_ResizePart}") if [ -z "${kiwi_oemrootMB}" ];then local disk_have_root_system_mbytes=$(( disk_root_mbytes + disk_free_mbytes @@ -129,21 +157,20 @@ function repart_standard_disk { # deactivate all active device mappings deactivate_device_mappings # repart root partition - local command_query - local root_part_size=+${disk_have_root_system_mbytes}M + root_part_size=+${disk_have_root_system_mbytes}M if [ -z "${kiwi_oemrootMB}" ];then # no new parts and no rootsize limit, use rest disk space root_part_size=. fi command_query=" - d ${kiwi_RootPart} - n p:lxroot ${kiwi_RootPart} . ${root_part_size} + d ${kiwi_ResizePart} + n ${part_name} ${kiwi_ResizePart} . ${root_part_size} " if mdraid_system; then command_query=" - d ${kiwi_RootPart} - n p:lxraid ${kiwi_RootPart} . ${root_part_size} - t ${kiwi_RootPart} fd + d ${kiwi_ResizePart} + n ${part_name} ${kiwi_ResizePart} . ${root_part_size} + t ${kiwi_ResizePart} fd " fi if ! create_partitions "${disk}" "${command_query}";then @@ -165,9 +192,14 @@ function repart_lvm_disk { # pX+1: ( LVM ) [+luks +raid] # ------------------------------------- # """ - declare kiwi_RootPart=${kiwi_RootPart} local kiwi_oemrootMB + local kiwi_ResizePart + local command_query + local lvm_part_size + local part_name kiwi_oemrootMB=$(get_target_rootpart_size) + kiwi_ResizePart=$(get_last_partition_id_from_config_file) + part_name=$(get_partition_name "${disk}" "${kiwi_ResizePart}") if [ -z "${kiwi_oemrootMB}" ];then local disk_have_root_system_mbytes=$(( disk_root_mbytes + disk_free_mbytes @@ -195,16 +227,15 @@ function repart_lvm_disk { # create lvm.conf appropriate for resize setup_lvm_config # repart lvm partition - local command_query - local lvm_part_size=+${disk_have_root_system_mbytes}M + lvm_part_size=+${disk_have_root_system_mbytes}M if [ -z "${kiwi_oemrootMB}" ];then # no rootsize limit, use rest disk space lvm_part_size=. fi command_query=" - d ${kiwi_RootPart} - n p:lxlvm ${kiwi_RootPart} . ${lvm_part_size} - t ${kiwi_RootPart} 8e + d ${kiwi_ResizePart} + n ${part_name} ${kiwi_ResizePart} . ${lvm_part_size} + t ${kiwi_ResizePart} 8e " if ! create_partitions "${disk}" "${command_query}";then die "Failed to create partition table" @@ -284,10 +315,10 @@ if luks_system "${disk}";then fi # wait for the root device to appear -wait_for_storage_device "${root_device}" +wait_for_storage_device "${last_device}" # check if repart/resize is wanted -if ! resize_wanted "${root_device}" "${disk}"; then +if ! resize_wanted "${last_device}" "${disk}"; then return fi @@ -297,7 +328,7 @@ if [ "$(get_partition_table_type "${disk}")" = 'gpt' ];then fi # wait for the root device to appear -wait_for_storage_device "${root_device}" +wait_for_storage_device "${last_device}" # resize disk partition table if lvm_system;then @@ -328,4 +359,4 @@ else fi # wait for the root device to appear -wait_for_storage_device "${root_device}" +wait_for_storage_device "${last_device}" diff --git a/dracut/modules.d/59kiwi-lib/kiwi-partitions-lib.sh b/dracut/modules.d/59kiwi-lib/kiwi-partitions-lib.sh index a8c8f5d7cd9..9eb8f63ad0b 100644 --- a/dracut/modules.d/59kiwi-lib/kiwi-partitions-lib.sh +++ b/dracut/modules.d/59kiwi-lib/kiwi-partitions-lib.sh @@ -213,6 +213,29 @@ function get_partition_node_name { return 1 } +function get_partition_name { + local disk=$1 + local part_id=$2 + local part_name + local table_type + table_type=$(get_partition_table_type "${disk}") + if [ "${table_type}" = "gpt" ];then + part_name=$(sfdisk --part-label "${disk}" "${part_id}") + else + part_name=unsupported + fi + echo "${part_name}" +} + +function get_last_partition_device { + # """ + # Get unix node of last partition in table + # """ + local disk=$1 + lsblk -p -n -r -o NAME,TYPE "${disk}" |\ + grep -E "part|md$" | tail -n 1 | cut -f1 -d ' ' +} + function get_last_partition_id { # """ # Get index of last partition from the current table diff --git a/kiwi/builder/disk.py b/kiwi/builder/disk.py index 188dae5a765..6cc0a1b084a 100644 --- a/kiwi/builder/disk.py +++ b/kiwi/builder/disk.py @@ -1210,7 +1210,14 @@ def _build_and_map_disk_partitions( if root_clone_count: rootfs_mbsize = int(rootfs_mbsize / (root_clone_count + 1)) else: - if self.oem_systemsize and not self.oem_resize: + if self.oem_systemsize and ( + not self.oem_resize or self.spare_part_is_last + ): + # if resize on boot is switched off, we have to take + # the given size into account at build time. + # if resize is active but the rootfs is not the last + # partition we also have to take the given size into + # account, because only the last can resize rootfs_mbsize = self.oem_systemsize else: rootfs_mbsize = 'all_free' diff --git a/kiwi/schema/kiwi.rnc b/kiwi/schema/kiwi.rnc index aeaf5d39f25..c9400d72046 100644 --- a/kiwi/schema/kiwi.rnc +++ b/kiwi/schema/kiwi.rnc @@ -1452,19 +1452,6 @@ div { ] ] ] - sch:pattern [ - abstract = "true" - id = "image_expandable" - sch:rule [ - context = "type[@$attr='true']" - sch:assert [ - test = "oemconfig/oem-resize[contains(text(), 'false')]" - "$attr attribute also needs the setting: " - "false in the " - " section of the selected " - ] - ] - ] sch:pattern [ abstract = "true" id = "image_verity_requirement" @@ -1632,24 +1619,12 @@ div { ] k.type.spare_part_is_last.attribute = ## Specify if the spare partition should be the last one in - ## the partition table. Can only be configured for the oem - ## type with oem-resize switched off. By default the root - ## partition is the last one and the spare partition lives - ## before it. With this attribute that setup can be toggled. - ## However if the root partition is no longer the last one - ## the oem repart/resize code can no longer work because - ## the spare part would block it. Because of that moving - ## the spare part at the end of the disk is only applied - ## if oem-resize is switched off. There is a runtime - ## check in the kiwi code to check this condition + ## the partition table. attribute spare_part_is_last { xsd:boolean } >> sch:pattern [ id = "spare_part_is_last" is-a = "image_type" sch:param [ name = "attr" value = "spare_part_is_last" ] sch:param [ name = "types" value = "oem" ] ] - >> sch:pattern [ id = "spare_part_is_last_valid" is-a = "image_expandable" - sch:param [ name = "attr" value = "spare_part_is_last" ] - ] k.type.bootpartsize.attribute = ## For images with a separate boot partition this attribute ## specifies the size in MB. If not set the min bootpart diff --git a/kiwi/schema/kiwi.rng b/kiwi/schema/kiwi.rng index 24c551d9f0d..7a2c0b67470 100644 --- a/kiwi/schema/kiwi.rng +++ b/kiwi/schema/kiwi.rng @@ -2196,11 +2196,6 @@ volume management system <vagrantconfig> section is required for the vagrant format - - - $attr attribute also needs the setting: <oem-resize>false</oem-resize> in the <oemconfig> section of the selected <type> - - $attr attribute must be set for embed_verity_metadata @@ -2405,25 +2400,13 @@ Can only be configured for the disk image type oem Specify if the spare partition should be the last one in -the partition table. Can only be configured for the oem -type with oem-resize switched off. By default the root -partition is the last one and the spare partition lives -before it. With this attribute that setup can be toggled. -However if the root partition is no longer the last one -the oem repart/resize code can no longer work because -the spare part would block it. Because of that moving -the spare part at the end of the disk is only applied -if oem-resize is switched off. There is a runtime -check in the kiwi code to check this condition +the partition table. - - -