add userspace_btrfs

docs/bootloader_config.md

@@ -24,6 +24,9 @@ The `build_dir`can be embedded into the Linux kernel with:
 CONFIG_INITRAMFS_SOURCE="/tmp/initramfs"
 ```
 
+Take care to set `make_nodes = true` to create `/dev/console` in the build dir (instead of in the cpio when packing it).
+The console device node file is required in builtin initrds.
+
 A CPIO file can be embedded into the Linux kernel with: 
 
 ```

docs/configuration.md

@@ -41,7 +41,7 @@ Modules write to a shared config dict that is accessible by other modules.
 * `build_dir` (initramfs_build) If relative, it will be placed under `tmpdir`, defines the build directory.
 * `random_build_dir` (false) Adds a uuid to the end of the build dir name when true.
 * `build_logging` (false) Enables additional logging during the build process.
-* `make_nodes` (false) Create real device nodes in the build dir. 
+* `make_nodes` (false) Create real device nodes in the build dir. Otherwise they are created in the CPIO archive. 
 * `find_libgcc` (true) Automatically locates libgcc using ldconfig -p and adds it to the initramfs.
 * `musl_libc` (false) Disable ldconfig -p usage for libgcc detection, skip ld.so.cache regeneration.
 * `out_dir` (initramfs_out) If relative, it will be placed under `tmpdir`, defines the output directory.
@@ -151,6 +151,12 @@ Defines /dev/ttyS1 as a `vt100` terminal with a `115200` baud rate.
 
 `primary_console` (tty0) Used to set which console will be initialized with agetty on boot.
 
+#### base.keymap
+
+This module can be used to set keymaps in the initramfs.
+
+The only available parameter is `keymap_file`, which should point to a valid keymap file.
+
 #### base.debug
 
 This module contains debug programs such as `cp`, `mv`, `rm`, `grep`, `dmesg`, `find`, and an editor,
@@ -290,7 +296,6 @@ This module can be enabled by adding `ugrd.fs.fakeudev` to the `modules` list.
 
 This module handles CPIO creation.
 
-* `mknod_cpio` (true) Only create device nodes within the CPIO.
 * `cpio_compression` (xz) Sets the compression method for the CPIO file, passed to PyCPIO.
 * `cpio_rotate` (true) Rotates old CPIO files, keeping `old_count` number of old files.
 

docs/dev_manual.md

@@ -10,6 +10,8 @@ Python functions can be added imported into `init` and `build` runlevels to exec
 
 Within modules, all config values are imported, then processed according to the order of the `custom_parameters` list.
 
+> If config values have validation which may fail if other config is not loaded, those values can be added to the `_late_args' list to be processed last.
+
 `_module_name` can be set within a module for logging purposes, it is verified to be accurate when imported but optional.
 
 ## Imports

examples/example.toml

@@ -1,3 +1,9 @@
+##############################################################################
+##                                                                          ##
+##    there are docs at https://github.com/desultory/ugrd/tree/main/docs    ##
+##                                                                          ##
+##############################################################################
+
 # This config will decrypt the LUKS volume with uuid "fdf442da-0574-4531-98c7-55227a041f1d", mapping it to "/dev/mapper/root"
 # Note: In most cases, the uuid of the LUKS volume will be detected from the root mount. It's set as an example here.
 # It will attempt to mount the btrfs volume with label "rootfs" to /target_rootfs
@@ -21,10 +27,13 @@ modules = [
 #kmod_ignore_sound = false
 #kmod_ignore_network = false
 
-# The initramfs will be built in /tmp/initramfs if "build_dir" is not specified not specified
+# The initramfs will be built in /tmp/initramfs if "build_dir" is not specified
 #out_dir = "/usr/src/initramfs"
 #out_dir = "/boot"
 
+# create device nodes in build dir, requires root
+#make_nodes = true
+
 # By default, xz compression is used. This can be disabled or set to zstd if the zstandard python module is installed
 #cpio_compression = "zstd"  # Use zstd compression
 #cpio_compression = false  # No compression

pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "ugrd"
-version = "2.0.2"
+version = "2.1.0"
 authors = [
   { name="Desultory", email="[email protected]" },
 ]
@@ -19,7 +19,7 @@ classifiers = [
 
 dependencies = [
     "zenlib >= 3.0.2",
-    "pycpio >= 1.5.2"
+    "pycpio >= 1.5.5"
 ]
 
 [project.optional-dependencies]

src/ugrd/base/banner.py

@@ -1,10 +1,10 @@
 __author__ = "desultory"
-__version__ = "0.1.0"
+__version__ = "0.2.0"
 
 
 def print_banner(self) -> list[str]:
     """Prints the banner. Prints the kernel version if set"""
     banner = [self.banner]
     if kver := self.get("kernel_version"):
-        banner.append(f"einfo 'Kernel version: {kver}'")
+        banner.append(f"einfo 'Built for kernel version: {kver}'")
     return banner

src/ugrd/base/core.py

@@ -1,5 +1,5 @@
 __author__ = "desultory"
-__version__ = "4.7.0"
+__version__ = "4.7.1"
 
 from os import environ, makedev, mknod, uname
 from pathlib import Path
@@ -303,9 +303,7 @@ def deploy_nodes(self) -> None:
             self.logger.info("Created device node '%s' at path: %s" % (node, node_path))
         except PermissionError as e:
             self.logger.error("Unable to create device node %s at path: %s" % (node, node_path))
-            self.logger.info(
-                "`mknod_cpio` in `ugrd.base` can be used to generate device nodes within the initramfs archive if they cannot be created on the host system."
-            )
+            self.logger.info("When `make_nodes` is disabled, device nodes are synthetically created in the resulting CPIO archive.")
             raise e
 
 

src/ugrd/base/core.toml

@@ -63,8 +63,8 @@ hostonly = "bool"  # If true, the initramfs will be built specifically for the h
 validate = "bool"  # If true, the configuration of the initramfs will be validated against the host
 timeout = "int"  # The timeout for _run commands, defaults to 15 seconds
 _custom_init_file = "str"  # Add the _custom_init_file propety, used to set where the custom init file is located
-tmpdir = "Path"  # The base directory for builds
-build_dir = "Path"  # The directory where the initramfs is built
+tmpdir = "Path"  # The base directory for builds, defaults to /tmp. the build and output directories are created inside this directory
+build_dir = "Path"  # The directory where the initramfs is built, inside the tmpdir unless an absolute path is given
 random_build_dir = "bool"  # If true, a random build directory will be used
 build_logging = "bool" # If true, additional build information will be logged to the console
 _build_log_level = "int"  # The level of logging to use for the build log, set to 10 by default and incremeted by if build_log is true (min 20)
@@ -78,17 +78,17 @@ zstd_dependencies = "NoDupFlatList"  # ZStandard compressed dependencies propert
 gz_dependencies = "NoDupFlatList"  # GZipped dependencies property, used to define the gzipped dependencies (will be extracted)
 library_paths = "NoDupFlatList"  # library_paths property, used to define the library paths to add to LD_LIBRARY_PATH
 find_libgcc = "bool"  # If true, the initramfs will search for libgcc_s.so.1 and add it to the initramfs
-musl_libc = "bool"  # If true, disables find_libgcc and regen_ld_so_cache
+musl_libc = "bool"  # If true, disables find_libgcc and regen_ld_so_cache (not needed for musl libc based systems)
 libraries = "NoDupFlatList"  # Additional libraries, by name, added to the initramfs
 binaries = "NoDupFlatList"  # Binaries which should be included in the intiramfs, dependencies resolved with lddtree
 binary_search_paths = "NoDupFlatList"  # Binary paths, used to define the paths to search for binaries
 copies = "dict"  # Copies dict, defines the files to be copied to the initramfs
 nodes = "dict"  # Nodes dict, defines the device nodes to be created
 paths = "NoDupFlatList"  # Paths to be created in the initramfs
 masks = "dict"  # Imports to be masked in the initramfs
-make_nodes = "bool"  # If true, actual device nodes will be created in the build dir
-out_dir = "Path"  # The directory where the initramfs is packed/output. If no packer is used, this is the final output directory.
+make_nodes = "bool"  # If true, actual device nodes will be created in the build dir instead of only being created in the cpio archive
+out_dir = "Path"  # The directory where the initramfs is packed/output
 out_file = "str"  # The name of the output file, if absolute, overrides out dir with the path, and sets out_file to the filename
 old_count = "int"  # The number of times to cycle old files before deleting
-clean = "bool"  # Add the clean property, used to define if the mounts should be cleaned up after boot
+clean = "bool"  # Add the clean property, used to define if the build directory should be cleaned before building
 shell = "str"  # Set the shell to use for the init process

src/ugrd/crypto/cryptsetup.py

@@ -1,5 +1,5 @@
 __author__ = "desultory"
-__version__ = "4.1.3"
+__version__ = "4.2.0"
 
 from json import loads
 from pathlib import Path
@@ -28,6 +28,7 @@
     "include_header",
     "validate_key",
     "validate_header",
+    "_dm-integrity",  # Internal parameter for when dm-integrity was detected. mostly used for test automation
 ]
 
 
@@ -147,6 +148,9 @@ def _get_dm_info(self, mapped_name: str) -> dict:
 def _get_dm_slave_info(self, device_info: dict) -> (str, dict):
     """Gets the device mapper slave information for a particular device."""
     slave_source = device_info["slaves"][0]
+    # For integrity backed devices, get the slave's slave
+    if self["_vblk_info"].get(slave_source, {}).get("uuid", "").startswith("CRYPT-SUBDEV"):
+        slave_source = self["_vblk_info"][slave_source]["slaves"][0]
     slave_name = self["_vblk_info"].get(slave_source, {}).get("name")
     search_paths = ["/dev/", "/dev/mapper/"]
 
@@ -207,7 +211,7 @@ def _detect_luks_aes_module(self, luks_cipher_name: str) -> None:
         self["_kmod_auto"] = crypto_config["module"]
 
 
-def _detect_luks_header_aes(self, luks_info: dict) -> dict:
+def _detect_luks_header_aes(self, luks_info: dict) -> None:
     """Checks the cipher type in the LUKS header, reads /proc/crypto to find the
     corresponding driver. If it's not builtin, adds the module to the kernel modules."""
     for keyslot in luks_info.get("keyslots", {}).values():
@@ -218,7 +222,7 @@ def _detect_luks_header_aes(self, luks_info: dict) -> dict:
             _detect_luks_aes_module(self, segment["encryption"])
 
 
-def _detect_luks_header_sha(self, luks_info: dict) -> dict:
+def _detect_luks_header_sha(self, luks_info: dict) -> None:
     """Reads the hash algorithm from the LUKS header,
     enables the corresponding kernel module using _crypto_ciphers"""
     for keyslot in luks_info.get("keyslots", {}).values():
@@ -229,6 +233,21 @@ def _detect_luks_header_sha(self, luks_info: dict) -> dict:
             self["kernel_modules"] = self._crypto_ciphers[digest["hash"]]["driver"]
 
 
+def _detect_luks_header_integrity(self, luks_info: dict, mapped_name: str) -> None:
+    """Reads the integrity algorithm from the LUKS header,
+    Enables the dm-integrity module, and returns the integrity type."""
+    for segment in luks_info.get("segments", {}).values():
+        if integrity_type := segment.get("integrity", {}).get("type"):
+            integrity_kmods = ["dm_integrity", "authenc"]
+            if integrity_type.startswith("hmac"):
+                integrity_kmods.append("hmac")
+            self.logger.info(
+                f"[{c_(mapped_name, 'blue')}]({c_(integrity_type, 'cyan')}) Enabling kernel modules for dm-integrity: {c_(', '.join(integrity_kmods), 'magenta', bright=True)}"
+            )
+            self["cryptsetup"][mapped_name]["_dm-integrity"] = integrity_type
+            return
+
+
 @contains("cryptsetup_header_validation", "Skipping cryptsetup header validation.", log_level=30)
 def _validate_cryptsetup_header(self, mapped_name: str) -> None:
     """Validates configured cryptsetup volumes against the LUKS header."""
@@ -265,6 +284,7 @@ def _validate_cryptsetup_header(self, mapped_name: str) -> None:
 
     _detect_luks_header_aes(self, luks_info)
     _detect_luks_header_sha(self, luks_info)
+    _detect_luks_header_integrity(self, luks_info, mapped_name)
 
     if not self["argon2"]:  # if argon support was not detected, check if the header wants it
         for keyslot in luks_info.get("keyslots", {}).values():

src/ugrd/fs/cpio.py

@@ -1,5 +1,5 @@
 __author__ = "desultory"
-__version__ = "3.7.3"
+__version__ = "3.8.0"
 
 from pathlib import Path
 
@@ -86,13 +86,13 @@ def make_cpio(self) -> None:
     """
     Populates the CPIO archive using the build directory,
     writes it to the output file, and rotates the output file if necessary.
-    Creates device nodes in the CPIO archive if the mknod_cpio option is set.
+    Creates device nodes in the CPIO archive if make_nodes is False. (make_nodes will create actual files instead)
     Raises FileNotFoundError if the output directory does not exist.
     """
     cpio = self._cpio_archive
     cpio.append_recursive(self._get_build_path("/"), relative=True)
 
-    if self.get("mknod_cpio"):
+    if not self.get("make_nodes"):
         for node in self["nodes"].values():
             self.logger.debug("Adding CPIO node: %s" % node)
             cpio.add_chardev(name=node["path"], mode=node["mode"], major=node["major"], minor=node["minor"])

src/ugrd/fs/cpio.toml

@@ -1,4 +1,3 @@
-mknod_cpio = true
 cpio_compression = "xz"
 cpio_rotate = true
 check_cpio = true
@@ -14,7 +13,6 @@ check_cpio = true
 
 [custom_parameters]
 cpio_rotate = "bool"  # makes a .old backup of the cpio file if it already exists.
-mknod_cpio = "bool"  # When enabled, mknod is not used to create device nodes, they are just created in the cpio.
 cpio_compression = "str"  # The compression method to use for the cpio file. XZ and ZSTD are supported.
 _cpio_archive = "PyCPIO"  # The cpio archive object.
 check_cpio = "bool"  # When enabled, the CPIO archive contents are checked for errors.