Merge branch 'desultory:main' into patch-6

Author: redjard

docs/configuration.md

@@ -234,19 +234,27 @@ Additional modules include:
 #### ugrd.fs.mounts
 
 * `autodetect_root` (true) Set the root mount parameter based on the current root label or uuid.
-* `autodetect_root_dm` (true) Attempt to automatically configure virtual block devices such as LUKS/LVM/MDRAID.
-* `autodetect_root_luks` (true) Attempt to automatically configure LUKS mounts for the root device.
-* `autodetect_root_lvm` (true) Attempt to automatically configure LVM mounts for the root device.
-* `autodetect_root_mdraid` (true) Attempt to automatically configure MDRAID mounts for the root device.
+* `autodetect_dm` (true) Attempt to automatically configure virtual block devices such as LUKS/LVM/MDRAID.
+* `autodetect_luks` (true) Attempt to automatically configure LUKS mounts for the root device.
+* `autodetect_lvm` (true) Attempt to automatically configure LVM mounts for the root device.
+* `autodetect_mdraid` (true) Attempt to automatically configure MDRAID mounts for the root device.
 * `autodetect_init_mount'` (true) Automatically detect the mountpoint for the init binary, and add it to `late_mounts`.
 * `run_dirs` A list of directories to create under `/run/` at runtime
 
-> `autodetect_root` is required for `autodetect_root_<type>` to work.
+> `autodetect_root` is required for `autodetect_<type>` to work.
 
 `mounts`: A dictionary containing entries for mounts, with their associated config.
 
+Mounts defined here are mounted before `init_main` is run. This cannot be used for mounts backed by LUKS, LVM, or MDRAID devices, because the backend will not be available when these mounts are attempted.
+
+> `mounts` can be automatically populated by configuring paths as list items in `auto_mounts`.
+
 `mounts.root` is predefined to have a destination of `/target_rootfs` and defines the root filesystem mount, used by `switch_root`.
 
+`late_mounts`: A dictionary containing entries for mounts that should be mounted after `init_main` is run.
+
+>  `late_mounts` can be automatically populated by configuring paths as list items in `auto_late_mounts`.
+
 Each mount has the following available parameters:
 
 * `type` (auto) Mount filesystem type.
@@ -285,6 +293,8 @@ label = "extra"
 
 Paths added to `auto_mounts` will be auto-configured to mount before `init_main` is run.
 
+Paths added to `auto_late_mounts` will be auto-configured to mount after `init_main` is run.  
+
 #### ugrd.fs.fakeudev
 
 This module is used to create fake udev entries for DM devices.
@@ -314,6 +324,7 @@ Importing this module will run `btrfs device scan` and pull btrfs modules.
 * `autodetect_root_subvol` (true) Autodetect the root subvolume, unless `root_subvol` or `subvol_selector` is set. Depends on `hostonly`.
 * `root_subvol` - Set the desired root subvolume.
 * `_base_mount_path` (/root_base) Sets where the subvolume selector mounts the base filesytem to scan for subvolumes.
+* `btrfs_userspace` (true) Add btrfs binary to be able to mount multi-device btrfs partitions.
 
 #### ugrd.fs.resume
 

examples/luks.toml

@@ -7,7 +7,7 @@ modules = [
 ]
 
 # Device mapper autodetection is enabled by default
-# autodetect_root_luks = true
+# autodetect_luks = true
 
 # Information about the LUKS volume can be manually specified
 #[cryptsetup.root]

readme.md

@@ -26,15 +26,19 @@ The original goal of this project was to create an initramfs suitable for decryp
 ### Auto-detection
 
 * Root mount, using `/proc/mounts`. `root=` and `rootflags=` can be used but are not required
-* MDRAID auto-configuration for the root mount
-* LUKS auto-configuration and validation for the root mount
+* MDRAID auto-configuration
+* LVM auto-configuration
+* LUKS auto-configuration and validation
   - LUKS under LVM support
   - LUKS under MDRAID support
-* LVM based root volumes are auto-mounted
-* BTRFS root subvolumes are automatically detected to `root_subvol`
-    - `subvol_selector` can be used to select a subvolume at boot time
-* `/usr` auto-mounting if the init system requires it
-* Auto-detection of kernel modules required by the storage device used by the root filesystem
+  - Detached header support
+  - YubiKey (OpenPGP smartcard) support
+  - Recovery using a passprhase using `try_nokey`
+  - DM-Integrity support
+* BTRFS root subvolumes are automatically detected or can be manually set with `root_subvol`
+    - `subvol_selector` can be used to interactively select a subvolume at boot time
+* `/usr`, `/var`, and `/etc` auto-mounting if the init system requires it
+* Auto-detection of kernel modules required by storage devices and filesystems
 * Init system/target auto-detection
 
 ### Validation

src/ugrd/crypto/cryptsetup.py

@@ -100,7 +100,7 @@ def _validate_cryptsetup_config(self, mapped_name: str) -> None:
         ).exists():  # Make sure the header file exists, it may not be present at build time
             self.logger.warning("[%s] Header file not found: %s" % (mapped_name, c_(config["header_file"], "yellow")))
     elif not any([config.get("partuuid"), config.get("uuid"), config.get("path")]):
-        if not self["autodetect_root_luks"]:
+        if not self["autodetect_luks"]:
             raise ValidationError(
                 "A device uuid, partuuid, or path must be specified for cryptsetup mount: %s" % mapped_name
             )

src/ugrd/fs/btrfs.py

@@ -75,6 +75,7 @@ def _process_subvol_selector(self, subvol_selector: bool) -> None:
         self["paths"] = self["_base_mount_path"]
 
 
+@contains("btrfs_userspace", message="btrfs_userspace is disabled, will not add btrfs_scan to init.")
 def btrfs_scan(self) -> str:
     """scan for new btrfs devices."""
     return 'einfo "$(btrfs device scan)"'
@@ -98,6 +99,7 @@ def autodetect_root_subvol(self):
 
 @contains("subvol_selector", message="subvol_selector is not enabled, skipping.")
 @unset("root_subvol", message="root_subvol is set, skipping.")
+@contains("btrfs_userspace", message="btrfs_userspace is not enabled, skipping.")
 def select_subvol(self) -> str:
     """Returns a POSIX shell script to list subvolumes on the root volume."""
     return f"""
@@ -134,3 +136,8 @@ def set_root_subvol(self) -> str:
     """Adds the root_subvol to the root_mount options."""
     _validate_root_subvol(self)
     return f"""setvar root_extra_options ',subvol={self["root_subvol"]}'"""
+
+@contains("btrfs_userspace", message="btrfs_userspace is disabled, skipping adding btrfs to binaries list.", log_level=30)
+def pull_btrfs_userspace(self):
+    self.logger.debug("Adding btrfs to binaries list.")
+    self["binaries"] = "btrfs"

src/ugrd/fs/btrfs.toml

@@ -1,16 +1,19 @@
-binaries = [ "btrfs" ]
 kmod_init = [ "btrfs", "crc32c" ] # crc32c is a softdep but more or less required
 
 _base_mount_path = "/root_base"
 subvol_selector = false
 autodetect_root_subvol = true
+btrfs_userspace = true
 
 [imports.config_processing]
 "ugrd.fs.btrfs" = [ "_process_root_subvol", "_process_subvol_selector" ]
 
 [imports.build_pre]
 "ugrd.fs.btrfs" = [ "autodetect_root_subvol" ]
 
+[imports.build_late]
+"ugrd.fs.btrfs" = [ "pull_btrfs_userspace" ]
+
 [imports.init_mount]
 "ugrd.fs.btrfs" = [ "btrfs_scan", "set_root_subvol", "select_subvol" ]
 
@@ -28,3 +31,4 @@ _base_mount_path = "Path"  # Set the mount point for the root filesystem when us
 root_subvol = "str"  # Set the default btrfs subvolume for the root filesystem
 subvol_selector = "bool"  # Select a btrfs subvolume for the root partition at runtime
 autodetect_root_subvol = "bool"  # Automatically detect the root subvolume
+btrfs_userspace = "bool"  # Add btrfs binary to be able to mount multi-device btrfs partitions

src/ugrd/fs/mounts.py

@@ -1,5 +1,5 @@
 __author__ = "desultory"
-__version__ = "7.2.1"
+__version__ = "7.3.0"
 
 from pathlib import Path
 from re import search
@@ -436,7 +436,7 @@ def get_virtual_block_info(self):
     sys_block = Path("/sys/devices/virtual/block")
 
     if not sys_block.exists():
-        self["autodetect_root_dm"] = False
+        self["autodetect_dm"] = False
         return self.logger.warning("Virtual block devices unavailable, disabling device mapper autodetection.")
 
     devices = []
@@ -450,7 +450,7 @@ def get_virtual_block_info(self):
                 devices.append(part)
 
     if not devices:
-        self["autodetect_root_dm"] = False
+        self["autodetect_dm"] = False
         return self.logger.warning("No virtual block devices found, disabling device mapper autodetection.")
 
     for virt_dev in devices:
@@ -541,6 +541,17 @@ def _autodetect_dm(self, mountpoint, device=None) -> None:
     major, minor = _get_device_id(source_device)
     self.logger.debug("[%s] Major: %s, Minor: %s" % (source_device, major, minor))
 
+    # If the mountpoint is in auto_mounts, log a big error about it being prone to failure, allow
+    if mountpoint in self["auto_mounts"]:
+        self.logger.error(f"Found a device mapper mount in auto_mounts: {c_(mountpoint, 'yellow', bold=True)}")
+        self.logger.warning(
+            "auto_mounts is used for mounts before LVM/LUKS init (during mount_fstab). Device mapper mounts defined here may fail to activate and stop the boot process!"
+        )
+        if self["validate"]:
+            raise ValidationError(
+                f"Device mapper mount found in auto_mounts, auto_mounts cannot be device mapper based: {c_(mountpoint, 'red', bold=True)}"
+            )
+
     # Get the virtual block device name using the major/minor
     for name, info in self["_vblk_info"].items():
         if info["major"] == str(major) and info["minor"] == str(minor):
@@ -559,7 +570,9 @@ def _autodetect_dm(self, mountpoint, device=None) -> None:
     # If the slave source is a CRYPT-SUBDEV device, use its slave instead
     if self["_vblk_info"].get(slave_source, {}).get("uuid", "").startswith("CRYPT-SUBDEV"):
         slave_source = self["_vblk_info"][slave_source]["slaves"][0]
-        self.logger.info(f"[{c_(dev_name, 'blue')}] Slave is a CRYPT-SUBDEV, using its slave instead: {c_(slave_source, 'cyan')}")
+        self.logger.info(
+            f"[{c_(dev_name, 'blue')}] Slave is a CRYPT-SUBDEV, using its slave instead: {c_(slave_source, 'cyan')}"
+        )
         # Add the kmod for it
         self.logger.info(f"[{c_(dev_name, 'blue')}] Adding kmod for CRYPT-SUBDEV: {c_('dm-crypt', 'magenta')}")
         self["_kmod_auto"] = ["dm_integrity", "authenc"]
@@ -624,7 +637,7 @@ def _autodetect_dm(self, mountpoint, device=None) -> None:
             self.logger.debug("Slave does not appear to be a DM device: %s" % slave)
 
 
-@contains("autodetect_root_raid", "Skipping RAID autodetection, autodetect_root_raid is disabled.", log_level=30)
+@contains("autodetect_raid", "Skipping RAID autodetection, autodetect_raid is disabled.", log_level=30)
 @contains("hostonly", "Skipping RAID autodetection, hostonly mode is disabled.", log_level=30)
 def autodetect_raid(self, source_dev, dm_name, blkid_info) -> None:
     """Autodetects MD RAID mounts and sets the raid config.
@@ -641,7 +654,7 @@ def autodetect_raid(self, source_dev, dm_name, blkid_info) -> None:
         raise AutodetectError("[%s] Failed to autodetect MDRAID level: %s" % (dm_name, blkid_info))
 
 
-@contains("autodetect_root_lvm", "Skipping LVM autodetection, autodetect_root_lvm is disabled.", log_level=20)
+@contains("autodetect_lvm", "Skipping LVM autodetection, autodetect_lvm is disabled.", log_level=20)
 @contains("hostonly", "Skipping LVM autodetection, hostonly mode is disabled.", log_level=30)
 def autodetect_lvm(self, source_dev, dm_num, blkid_info) -> None:
     """Autodetects LVM mounts and sets the lvm config."""
@@ -664,7 +677,7 @@ def autodetect_lvm(self, source_dev, dm_num, blkid_info) -> None:
     self["lvm"] = {source_dev.name: lvm_config}
 
 
-@contains("autodetect_root_luks", "Skipping LUKS autodetection, autodetect_root_luks is disabled.", log_level=30)
+@contains("autodetect_luks", "Skipping LUKS autodetection, autodetect_luks is disabled.", log_level=30)
 @contains("hostonly", "Skipping LUKS autodetection, hostonly mode is disabled.", log_level=30)
 def autodetect_luks(self, source_dev, dm_num, blkid_info) -> None:
     """Autodetects LUKS mounts and sets the cryptsetup config."""
@@ -747,22 +760,10 @@ def autodetect_root(self) -> None:
         raise AutodetectError(
             "Root mount not found in host mounts.\nCurrent mounts: %s" % pretty_print(self["_mounts"])
         )
-    root_dev = _autodetect_mount(self, "/")
-    if self["autodetect_root_dm"]:
-        if self["mounts"]["root"]["type"] == "btrfs":
-            from ugrd.fs.btrfs import _get_btrfs_mount_devices
-
-            # Btrfs volumes may be backed by multiple dm devices
-            for device in _get_btrfs_mount_devices(self, "/", root_dev):
-                _autodetect_dm(self, "/", device)
-        elif self["mounts"]["root"]["type"] == "zfs":
-            for device in get_zpool_info(self, root_dev)["devices"]:
-                _autodetect_dm(self, "/", device)
-        else:
-            _autodetect_dm(self, "/")
+    _autodetect_mount(self, "/")
 
 
-def _autodetect_mount(self, mountpoint, mount_class="mounts", missing_ok=False) -> str:
+def _autodetect_mount(self, mountpoint, mount_class="mounts", missing_ok=False) -> None:
     """Sets mount config for the specified mountpoint, in the specified mount class.
 
     Returns the "real" device path for the mountpoint.
@@ -815,7 +816,7 @@ def _autodetect_mount(self, mountpoint, mount_class="mounts", missing_ok=False)
         # Inherit mount options from the host mount for certain mount types
         if fs_type in MOUNT_INHERIT_OPTIONS:
             mount_options = self["_mounts"][mountpoint].get("options", ["ro"])
-            if 'rw' in mount_options:
+            if "rw" in mount_options:
                 mount_options.pop(mount_options.index("rw"))  # Remove rw option if it exists
         else:  # For standard mounts, default ro
             mount_options = ["ro"]
@@ -840,8 +841,21 @@ def _autodetect_mount(self, mountpoint, mount_class="mounts", missing_ok=False)
     if fs_type == "zfs":
         mount_config[mount_name]["path"] = mount_device
 
+    # Run device mapper autodetection if enabled
+    if self["autodetect_dm"]:
+        if fs_type == "btrfs":
+            from ugrd.fs.btrfs import _get_btrfs_mount_devices
+
+            # Btrfs volumes may be backed by multiple dm devices
+            for device in _get_btrfs_mount_devices(self, mountpoint, mount_device):
+                _autodetect_dm(self, mountpoint, mount_device)
+        elif fs_type == "zfs":
+            for device in get_zpool_info(self, mount_device)["devices"]:
+                _autodetect_dm(self, mountpoint, mount_device)
+        else:
+            _autodetect_dm(self, mountpoint)
+
     self[mount_class] = mount_config
-    return mount_device
 
 
 @contains("auto_mounts", "Skipping auto mounts, auto_mounts is empty.", log_level=10)
@@ -852,6 +866,14 @@ def autodetect_mounts(self) -> None:
         _autodetect_mount(self, mountpoint)
 
 
+@contains("auto_late_mounts", "Skipping auto late mounts, auto_late_mounts is empty.", log_level=10)
+@contains("hostonly", "Skipping late mount autodetection, hostonly mode is disabled.", log_level=30)
+def autodetect_late_mounts(self) -> None:
+    """Configured the late_mounts config for a device based on the host mount config."""
+    for mountpoint in self["auto_late_mounts"]:
+        _autodetect_mount(self, mountpoint, mount_class="late_mounts")
+
+
 def mount_base(self) -> list[str]:
     """Generates mount commands for the base mounts.
     Must be run before variables are used, as it creates the /run/ugrd directory.
@@ -906,7 +928,9 @@ def mount_fstab(self) -> list[str]:
     mount_retries sets the number of times to retry the mount, infinite otherwise.
     """
     if not self._get_build_path("/etc/fstab").exists():
-        return self.logger.info("No initramfs fstab found, skipping mount_fstab. If non-root storage devices are not needed at boot, this is fine.")
+        return self.logger.info(
+            "No initramfs fstab found, skipping mount_fstab. If non-root storage devices are not needed at boot, this is fine."
+        )
 
     out = [
         'einfo "Attempting to mount all filesystems."',
@@ -1039,8 +1063,12 @@ def export_mount_info(self) -> None:
         self.logger.critical(f"Failed to get source info for the root mount: {e}")
         if not self["hostonly"]:
             self.logger.info("Root mount infomrmation can be defined under the '[mounts.root]' section.")
-            raise ValidationError("Root mount source information is not set, when hostonly mode is disabled, it must be manually defined.")
-        raise ValidationError("Root mount source information is not set even though hostonly mode is enabled. Please report a bug.")
+            raise ValidationError(
+                "Root mount source information is not set. When hostonly mode is disabled, it must be manually defined."
+            )
+        raise ValidationError(
+            "Root mount source information is not set even though hostonly mode is enabled. Please report a bug."
+        )
     self["exports"]["MOUNTS_ROOT_TYPE"] = self["mounts"]["root"].get("type", "auto")
     self["exports"]["MOUNTS_ROOT_OPTIONS"] = ",".join(self["mounts"]["root"]["options"])
     self["exports"]["MOUNTS_ROOT_TARGET"] = self["mounts"]["root"]["destination"]

src/ugrd/fs/mounts.toml

@@ -9,10 +9,10 @@ run_dirs = [ "ugrd" ]
 
 mount_timeout = 1
 autodetect_root = true
-autodetect_root_dm = true
-autodetect_root_luks = true
-autodetect_root_lvm = true
-autodetect_root_raid = true
+autodetect_dm = true
+autodetect_luks = true
+autodetect_lvm = true
+autodetect_raid = true
 autodetect_init_mount = true
 
 [imports.config_processing]
@@ -22,7 +22,7 @@ autodetect_init_mount = true
 
 [imports.build_enum]
 "ugrd.fs.mounts" = [ "get_mounts_info", "get_virtual_block_info", "get_blkid_info", "get_zpool_info",
-		     "autodetect_root", "autodetect_mounts", "autodetect_init_mount" ]
+		     "autodetect_root", "autodetect_mounts", "autodetect_late_mounts", "autodetect_init_mount" ]
 
 [imports.build_tasks]
 "ugrd.fs.mounts" = [ "export_mount_info" ]
@@ -58,14 +58,15 @@ mount_devpts = "bool"  # Whether or not to mount devpts
 run_dirs = "NoDupFlatList"  # A list of directories to be created under /run
 late_mounts = "dict"  # Like mounts, but run after the root is mounted
 auto_mounts = "NoDupFlatList"  # A list of mounts to be automatically added to the mounts list
+auto_late_mounts = "NoDupFlatList"  # A list of mounts to be automatically added to the late mounts list
 mount_timeout = "float"  # The time to wait between mount attempts
 mount_retries = "int"  # The number of times to re-attempt mounting the fstab, infinite if not set
 mount_cmd = "str"  # The mount command called by mount_root, can be overridden
 autodetect_root = "bool"  # Add the autodetect_root property, if defined, the root mount will be autodetected
-autodetect_root_dm = "bool"  # Whether or not to try to autodetect device-mapper partitions
-autodetect_root_luks = "bool"  # Whether or not to try to autodetect LUKS partitions
-autodetect_root_lvm = "bool"  # Whether or not to try to autodetect LVM partitions
-autodetect_root_raid = "bool"  # Whether or not to try to autodetect MDRAID partitions
+autodetect_dm = "bool"  # Whether or not to try to autodetect device-mapper partitions
+autodetect_luks = "bool"  # Whether or not to try to autodetect LUKS partitions
+autodetect_lvm = "bool"  # Whether or not to try to autodetect LVM partitions
+autodetect_raid = "bool"  # Whether or not to try to autodetect MDRAID partitions
 autodetect_init_mount = "bool"  # Adds a late_mount for the init target if it exists under a mount on the host
 no_fsck = "bool"  # Whether or not to skip fsck on the root device when applicable
 _mounts = "dict"  # The mounts information