Make integration tests to build outside of OBS

Makefile

@@ -117,6 +117,7 @@ check: setup
 	# shell code checks
 	bash -c 'shellcheck -e ${sc_disable} dracut/modules.d/*/*.sh -s bash'
 	bash -c 'shellcheck -e ${sc_disable} kiwi/config/functions.sh -s bash'
+	bash -c 'shellcheck build-tests.sh'
 	# python flake tests
 	poetry run flake8 --statistics -j auto --count kiwi
 	poetry run flake8 --statistics -j auto --count test/unit

README.rst

@@ -18,6 +18,7 @@ KIWI - Next Generation
 .. |Doc| replace:: `Documentation <https://osinside.github.io/kiwi/>`__
 .. |Installation| replace:: `Installation <https://osinside.github.io/kiwi/installation.html>`__
 .. |Contributing| replace:: `Contributing <https://osinside.github.io/kiwi/contributing.html>`__
+.. |IntegrationTesting| replace:: `Integration Testing <https://osinside.github.io/kiwi/integration_testing.html>`__
 .. |Donate| image:: https://www.paypalobjects.com/en_US/i/btn/btn_donateCC_LG.gif
    :target: https://www.paypal.com/donate/?hosted_button_id=CYZY57A7Q4TCC
 
@@ -27,10 +28,13 @@ KIWI - Next Generation
 
 * |Installation|
 
+* |IntegrationTesting|
+
 * |Contributing|
 
 * |Doc|
 
-We'll donate it for good luck so's you're sure to come back :)
+KIWI has helped you in your work ? Even the smallest gift is
+a way to help that we don't run out of coffee :)
 
 |Donate|

build-tests.sh

@@ -0,0 +1,155 @@
+#!/bin/bash
+# git clone https://github.com/OSInside/kiwi.git
+# Simple build test script to build the integration test
+# images from a given test directory. The host to run this
+# command requires the following tools:
+#
+# - tree
+# - git
+# - xmllint
+# - podman
+# - pip
+#
+# And requires the installation of the kiwi box plugin
+#
+# $ pip install --upgrade kiwi-boxed-plugin
+#
+set -e
+
+ARGUMENT_LIST=(
+    "test-dir:"
+    "test-name:"
+    "box-name:"
+)
+
+function usage() {
+    echo "usage: build-tests --test-dir <dir>"
+    echo "  --test-dir <dir>"
+    echo "    Some test dir name, e.g. build-tests/x86/tumbleweed/"
+    echo "  --test-name <name>"
+    echo "    some test name, e.g. test-image-disk"
+    echo "  --box-name <name>"
+    echo "    name of the box to use for the build, default: universal"
+}
+
+if ! opts=$(getopt \
+    --longoptions "$(printf "%s," "${ARGUMENT_LIST[@]}")" \
+    --name "$(basename "$0")" \
+    --options "" \
+    -- "$@"
+); then
+    usage
+    exit 0
+fi
+
+eval set --"${opts}"
+
+while [[ $# -gt 0 ]]; do
+    case "$1" in
+        --test-dir)
+            argTestDir=$2
+            shift 2
+            ;;
+
+        --test-name)
+            argTestName=$2
+            shift 2
+            ;;
+
+        --box-name)
+            argBoxName=$2
+            shift 2
+            ;;
+
+        *)
+            break
+            ;;
+    esac
+done
+
+if [ ! "${argTestDir}" ];then
+    usage
+    exit 1
+fi
+
+if [ ! -e "${argTestDir}"/.repos ];then
+    echo "No .repos information for specified test dir"
+    exit 1
+fi
+
+boxname=universal
+if [ "${argBoxName}" ];then
+    boxname="${argBoxName}"
+fi
+
+function create_repo_list() {
+    local build_dir=$1
+    if [ -s "${build_dir}"/.repos ];then
+        local repo_options="--ignore-repos"
+        while read -r repo;do
+            repo_options="${repo_options} --add-repo ${repo}"
+        done < "${build_dir}"/.repos
+        echo "${repo_options}"
+    fi
+}
+
+function create_build_commands() {
+    local build_dir=$1
+    local test_name=$2
+    build_commands=()
+    for image in "${build_dir}"/*;do
+        test -e "${image}/appliance.kiwi" || continue
+        test -e "${image}/.skip_boxbuild_container" && continue
+        base_image=$(basename "${image}")
+        if [ -n "${test_name}" ] && [ ! "${test_name}" = "${base_image}" ];then
+            continue
+        fi
+        build_command="kiwi-ng --debug"
+        has_profiles=false
+        repo_options=$(create_repo_list "${build_dir}")
+        for profile in $(
+            xmllint --xpath "//image/profiles/profile/@name" \
+            "${image}/appliance.kiwi" 2>/dev/null | cut -f2 -d\"
+        );do
+            has_profiles=true
+            target_dir="build_results/${base_image}/${profile}"
+            build_command="${build_command} --profile ${profile}"
+            build_command="${build_command} system boxbuild"
+            build_command="${build_command} --box ${boxname} --container --"
+            build_command="${build_command} --description $image"
+            build_command="${build_command} ${repo_options}"
+            build_command="${build_command} --target-dir ${target_dir}"
+            echo "${build_command}" \
+                > "build_results/${base_image}-${profile}.build"
+            build_commands+=( "${build_command}" )
+            build_command="kiwi-ng --debug"
+        done
+        if [ "${has_profiles}" = "false" ];then
+            target_dir="build_results/${base_image}"
+            build_command="${build_command} system boxbuild"
+            build_command="${build_command} --box ${boxname} --container --"
+            build_command="${build_command} --description $image"
+            build_command="${build_command} ${repo_options}"
+            build_command="${build_command} --target-dir ${target_dir}"
+            echo "${build_command}" \
+                > "build_results/${base_image}.build"
+            build_commands+=( "${build_command}" )
+        fi
+    done
+}
+
+# create results directory
+mkdir -p build_results
+
+# build command list
+create_build_commands "${argTestDir}" "${argTestName}"
+
+# build them in a row
+for build in "${build_commands[@]}";do
+    ${build}
+    sudo rm -rf build_results/*/*/build/
+    sudo rm -rf build_results/*/build/
+done
+
+# show result tree
+test -d build_results && tree -L 3 build_results

build-tests/arm/fedora/.repos

@@ -0,0 +1,2 @@
+obs://Virtualization:Appliances:Staging/Fedora_39
+https://download.fedoraproject.org/pub/fedora/linux/releases/39/Everything/aarch64/os/

build-tests/arm/rawhide/.repos

@@ -0,0 +1,2 @@
+obs://Virtualization:Appliances:Staging/Fedora_Rawhide
+https://download.fedoraproject.org/pub/fedora/development/rawhide/Everything/aarch64/os/

build-tests/arm/tumbleweed/.repos

@@ -0,0 +1,2 @@
+obs://Virtualization:Appliances:Staging/openSUSE_Tumbleweed
+https://download.opensuse.org/ports/aarch64/tumbleweed/repo/oss/

build-tests/arm/ubuntu/.repos

@@ -0,0 +1,9 @@
+obs://Virtualization:Appliances:Staging/xUbuntu_24.04_aarch64,apt-deb,kiwi,,,,,,,false
+https://ports.ubuntu.com/ubuntu-ports,apt-deb,noble_1,,,,,main,noble,false
+https://ports.ubuntu.com/ubuntu-ports,apt-deb,noble_2,,,,,multiverse,noble,false
+https://ports.ubuntu.com/ubuntu-ports,apt-deb,noble_3,,,,,restricted,noble,false
+https://ports.ubuntu.com/ubuntu-ports,apt-deb,noble_4,,,,,universe,noble,false
+https://ports.ubuntu.com/ubuntu-ports,apt-deb,noble-updates_1,,,,,main,noble-updates,false
+https://ports.ubuntu.com/ubuntu-ports,apt-deb,noble-updates_2,,,,,multiverse,noble-updates,false
+https://ports.ubuntu.com/ubuntu-ports,apt-deb,noble-updates_3,,,,,restricted,noble-updates,false
+https://ports.ubuntu.com/ubuntu-ports,apt-deb,noble-updates_4,,,,,universe,noble-updates,false

build-tests/arm/ubuntu/test-image-rpi/appliance.kiwi

@@ -9,9 +9,6 @@
     <preferences>
         <version>1.22.4</version>
         <packagemanager>apt</packagemanager>
-        <locale>en_US</locale>
-        <keytable>us</keytable>
-        <timezone>UTC</timezone>
         <rpm-excludedocs>true</rpm-excludedocs>
         <type image="oem" filesystem="xfs" firmware="efi" bootpartition="false" efipartsize="128" devicepersistency="by-label" editbootinstall="editbootinstall_rpi.sh">
             <bootloader name="custom"/>
@@ -31,13 +28,13 @@
     </users>
 
     <repository type="apt-deb" alias="kiwi-next-generation" priority="1" repository_gpgcheck="false" architectures="arm64">
-        <source path="obs://Virtualization:Appliances:Staging/xUbuntu_23.04"/>
+        <source path="obs://Virtualization:Appliances:Staging/xUbuntu_24.04_aarch64"/>
     </repository>
-    <repository type="apt-deb" alias="Ubuntu-Lunar-Universe" distribution="lunar" components="main multiverse restricted universe" repository_gpgcheck="false">
-        <source path="obs://Ubuntu:23.04/universe"/>
+    <repository type="apt-deb" alias="Ubuntu-Noble-Universe" distribution="noble" components="main multiverse restricted universe" repository_gpgcheck="false">
+        <source path="obs://Ubuntu:24.04/universe"/>
     </repository>
-    <repository type="apt-deb" alias="Ubuntu-Lunar" distribution="lunar" components="main multiverse restricted universe" repository_gpgcheck="false">
-        <source path="obs://Ubuntu:23.04/standard"/>
+    <repository type="apt-deb" alias="Ubuntu-Noble" distribution="noble" components="main multiverse restricted universe" repository_gpgcheck="false">
+        <source path="obs://Ubuntu:24.04/standard"/>
     </repository>
 
     <packages type="image">
@@ -56,7 +53,6 @@
         <package name="git"/>
         <package name="sudo"/>
         <package name="net-tools"/>
-        <package name="tzdata"/>
         <package name="apt-utils"/>
         <package name="systemd"/>
         <package name="systemd-timesyncd"/>
@@ -69,21 +65,19 @@
         <package name="dbus"/>
         <package name="xfsprogs"/>
         <package name="dracut-kiwi-oem-repart"/>
-        <package name="usrmerge"/>
         <package name="mawk"/>
         <package name="openssh-client"/>
         <package name="openssh-server"/>
-        <package name="netcat"/>
         <package name="zstd"/>
         <package name="util-linux"/>
         <package name="less"/>
         <package name="vim"/>
         <package name="fdisk"/>
         <package name="language-pack-en"/>
+        <package name="locales-all"/>
+        <package name="tzdata"/>
     </packages>
     <packages type="bootstrap">
-        <package name="apt-utils"/>
-        <package name="debconf"/>
-        <package name="mawk"/>
+        <package name="ca-certificates"/>
     </packages>
 </image>

build-tests/arm/ubuntu/test-image-rpi/pre_disk_sync.sh

@@ -17,8 +17,8 @@ cp -a /usr/lib/firmware/*-raspi/device-tree/broadcom/* /boot/efi/
 #==========================================
 # copy initrd and kernel
 #------------------------------------------
-cp /boot/initrd-*-raspi /boot/efi/initrd.img
-cp /boot/vmlinuz-*-raspi /boot/efi/vmlinuz
+cp /boot/initrd*-raspi /boot/efi/initrd.img
+cp /boot/vmlinuz*-raspi /boot/efi/vmlinuz
 
 #==========================================
 # copy u-boot