Merge branch 'desultory:main' into no-userspace-btrfs

Author: redjard

docs/configuration.md

@@ -234,19 +234,27 @@ Additional modules include:
 #### ugrd.fs.mounts
 
 * `autodetect_root` (true) Set the root mount parameter based on the current root label or uuid.
-* `autodetect_root_dm` (true) Attempt to automatically configure virtual block devices such as LUKS/LVM/MDRAID.
-* `autodetect_root_luks` (true) Attempt to automatically configure LUKS mounts for the root device.
-* `autodetect_root_lvm` (true) Attempt to automatically configure LVM mounts for the root device.
-* `autodetect_root_mdraid` (true) Attempt to automatically configure MDRAID mounts for the root device.
+* `autodetect_dm` (true) Attempt to automatically configure virtual block devices such as LUKS/LVM/MDRAID.
+* `autodetect_luks` (true) Attempt to automatically configure LUKS mounts for the root device.
+* `autodetect_lvm` (true) Attempt to automatically configure LVM mounts for the root device.
+* `autodetect_mdraid` (true) Attempt to automatically configure MDRAID mounts for the root device.
 * `autodetect_init_mount'` (true) Automatically detect the mountpoint for the init binary, and add it to `late_mounts`.
 * `run_dirs` A list of directories to create under `/run/` at runtime
 
-> `autodetect_root` is required for `autodetect_root_<type>` to work.
+> `autodetect_root` is required for `autodetect_<type>` to work.
 
 `mounts`: A dictionary containing entries for mounts, with their associated config.
 
+Mounts defined here are mounted before `init_main` is run. This cannot be used for mounts backed by LUKS, LVM, or MDRAID devices, because the backend will not be available when these mounts are attempted.
+
+> `mounts` can be automatically populated by configuring paths as list items in `auto_mounts`.
+
 `mounts.root` is predefined to have a destination of `/target_rootfs` and defines the root filesystem mount, used by `switch_root`.
 
+`late_mounts`: A dictionary containing entries for mounts that should be mounted after `init_main` is run.
+
+>  `late_mounts` can be automatically populated by configuring paths as list items in `auto_late_mounts`.
+
 Each mount has the following available parameters:
 
 * `type` (auto) Mount filesystem type.
@@ -285,6 +293,8 @@ label = "extra"
 
 Paths added to `auto_mounts` will be auto-configured to mount before `init_main` is run.
 
+Paths added to `auto_late_mounts` will be auto-configured to mount after `init_main` is run.  
+
 #### ugrd.fs.fakeudev
 
 This module is used to create fake udev entries for DM devices.

examples/luks.toml

@@ -7,7 +7,7 @@ modules = [
 ]
 
 # Device mapper autodetection is enabled by default
-# autodetect_root_luks = true
+# autodetect_luks = true
 
 # Information about the LUKS volume can be manually specified
 #[cryptsetup.root]

readme.md

@@ -26,15 +26,19 @@ The original goal of this project was to create an initramfs suitable for decryp
 ### Auto-detection
 
 * Root mount, using `/proc/mounts`. `root=` and `rootflags=` can be used but are not required
-* MDRAID auto-configuration for the root mount
-* LUKS auto-configuration and validation for the root mount
+* MDRAID auto-configuration
+* LVM auto-configuration
+* LUKS auto-configuration and validation
   - LUKS under LVM support
   - LUKS under MDRAID support
-* LVM based root volumes are auto-mounted
-* BTRFS root subvolumes are automatically detected to `root_subvol`
-    - `subvol_selector` can be used to select a subvolume at boot time
-* `/usr` auto-mounting if the init system requires it
-* Auto-detection of kernel modules required by the storage device used by the root filesystem
+  - Detached header support
+  - YubiKey (OpenPGP smartcard) support
+  - Recovery using a passprhase using `try_nokey`
+  - DM-Integrity support
+* BTRFS root subvolumes are automatically detected or can be manually set with `root_subvol`
+    - `subvol_selector` can be used to interactively select a subvolume at boot time
+* `/usr`, `/var`, and `/etc` auto-mounting if the init system requires it
+* Auto-detection of kernel modules required by storage devices and filesystems
 * Init system/target auto-detection
 
 ### Validation

src/ugrd/crypto/cryptsetup.py

@@ -100,7 +100,7 @@ def _validate_cryptsetup_config(self, mapped_name: str) -> None:
         ).exists():  # Make sure the header file exists, it may not be present at build time
             self.logger.warning("[%s] Header file not found: %s" % (mapped_name, c_(config["header_file"], "yellow")))
     elif not any([config.get("partuuid"), config.get("uuid"), config.get("path")]):
-        if not self["autodetect_root_luks"]:
+        if not self["autodetect_luks"]:
             raise ValidationError(
                 "A device uuid, partuuid, or path must be specified for cryptsetup mount: %s" % mapped_name
             )

src/ugrd/fs/mounts.py

@@ -1,5 +1,5 @@
 __author__ = "desultory"
-__version__ = "7.2.1"
+__version__ = "7.3.0"
 
 from pathlib import Path
 from re import search
@@ -436,7 +436,7 @@ def get_virtual_block_info(self):
     sys_block = Path("/sys/devices/virtual/block")
 
     if not sys_block.exists():
-        self["autodetect_root_dm"] = False
+        self["autodetect_dm"] = False
         return self.logger.warning("Virtual block devices unavailable, disabling device mapper autodetection.")
 
     devices = []
@@ -450,7 +450,7 @@ def get_virtual_block_info(self):
                 devices.append(part)
 
     if not devices:
-        self["autodetect_root_dm"] = False
+        self["autodetect_dm"] = False
         return self.logger.warning("No virtual block devices found, disabling device mapper autodetection.")
 
     for virt_dev in devices:
@@ -541,6 +541,17 @@ def _autodetect_dm(self, mountpoint, device=None) -> None:
     major, minor = _get_device_id(source_device)
     self.logger.debug("[%s] Major: %s, Minor: %s" % (source_device, major, minor))
 
+    # If the mountpoint is in auto_mounts, log a big error about it being prone to failure, allow
+    if mountpoint in self["auto_mounts"]:
+        self.logger.error(f"Found a device mapper mount in auto_mounts: {c_(mountpoint, 'yellow', bold=True)}")
+        self.logger.warning(
+            "auto_mounts is used for mounts before LVM/LUKS init (during mount_fstab). Device mapper mounts defined here may fail to activate and stop the boot process!"
+        )
+        if self["validate"]:
+            raise ValidationError(
+                f"Device mapper mount found in auto_mounts, auto_mounts cannot be device mapper based: {c_(mountpoint, 'red', bold=True)}"
+            )
+
     # Get the virtual block device name using the major/minor
     for name, info in self["_vblk_info"].items():
         if info["major"] == str(major) and info["minor"] == str(minor):
@@ -559,7 +570,9 @@ def _autodetect_dm(self, mountpoint, device=None) -> None:
     # If the slave source is a CRYPT-SUBDEV device, use its slave instead
     if self["_vblk_info"].get(slave_source, {}).get("uuid", "").startswith("CRYPT-SUBDEV"):
         slave_source = self["_vblk_info"][slave_source]["slaves"][0]
-        self.logger.info(f"[{c_(dev_name, 'blue')}] Slave is a CRYPT-SUBDEV, using its slave instead: {c_(slave_source, 'cyan')}")
+        self.logger.info(
+            f"[{c_(dev_name, 'blue')}] Slave is a CRYPT-SUBDEV, using its slave instead: {c_(slave_source, 'cyan')}"
+        )
         # Add the kmod for it
         self.logger.info(f"[{c_(dev_name, 'blue')}] Adding kmod for CRYPT-SUBDEV: {c_('dm-crypt', 'magenta')}")
         self["_kmod_auto"] = ["dm_integrity", "authenc"]
@@ -624,7 +637,7 @@ def _autodetect_dm(self, mountpoint, device=None) -> None:
             self.logger.debug("Slave does not appear to be a DM device: %s" % slave)
 
 
-@contains("autodetect_root_raid", "Skipping RAID autodetection, autodetect_root_raid is disabled.", log_level=30)
+@contains("autodetect_raid", "Skipping RAID autodetection, autodetect_raid is disabled.", log_level=30)
 @contains("hostonly", "Skipping RAID autodetection, hostonly mode is disabled.", log_level=30)
 def autodetect_raid(self, source_dev, dm_name, blkid_info) -> None:
     """Autodetects MD RAID mounts and sets the raid config.
@@ -641,7 +654,7 @@ def autodetect_raid(self, source_dev, dm_name, blkid_info) -> None:
         raise AutodetectError("[%s] Failed to autodetect MDRAID level: %s" % (dm_name, blkid_info))
 
 
-@contains("autodetect_root_lvm", "Skipping LVM autodetection, autodetect_root_lvm is disabled.", log_level=20)
+@contains("autodetect_lvm", "Skipping LVM autodetection, autodetect_lvm is disabled.", log_level=20)
 @contains("hostonly", "Skipping LVM autodetection, hostonly mode is disabled.", log_level=30)
 def autodetect_lvm(self, source_dev, dm_num, blkid_info) -> None:
     """Autodetects LVM mounts and sets the lvm config."""
@@ -664,7 +677,7 @@ def autodetect_lvm(self, source_dev, dm_num, blkid_info) -> None:
     self["lvm"] = {source_dev.name: lvm_config}
 
 
-@contains("autodetect_root_luks", "Skipping LUKS autodetection, autodetect_root_luks is disabled.", log_level=30)
+@contains("autodetect_luks", "Skipping LUKS autodetection, autodetect_luks is disabled.", log_level=30)
 @contains("hostonly", "Skipping LUKS autodetection, hostonly mode is disabled.", log_level=30)
 def autodetect_luks(self, source_dev, dm_num, blkid_info) -> None:
     """Autodetects LUKS mounts and sets the cryptsetup config."""
@@ -747,22 +760,10 @@ def autodetect_root(self) -> None:
         raise AutodetectError(
             "Root mount not found in host mounts.\nCurrent mounts: %s" % pretty_print(self["_mounts"])
         )
-    root_dev = _autodetect_mount(self, "/")
-    if self["autodetect_root_dm"]:
-        if self["mounts"]["root"]["type"] == "btrfs":
-            from ugrd.fs.btrfs import _get_btrfs_mount_devices
-
-            # Btrfs volumes may be backed by multiple dm devices
-            for device in _get_btrfs_mount_devices(self, "/", root_dev):
-                _autodetect_dm(self, "/", device)
-        elif self["mounts"]["root"]["type"] == "zfs":
-            for device in get_zpool_info(self, root_dev)["devices"]:
-                _autodetect_dm(self, "/", device)
-        else:
-            _autodetect_dm(self, "/")
+    _autodetect_mount(self, "/")
 
 
-def _autodetect_mount(self, mountpoint, mount_class="mounts", missing_ok=False) -> str:
+def _autodetect_mount(self, mountpoint, mount_class="mounts", missing_ok=False) -> None:
     """Sets mount config for the specified mountpoint, in the specified mount class.
 
     Returns the "real" device path for the mountpoint.
@@ -815,7 +816,7 @@ def _autodetect_mount(self, mountpoint, mount_class="mounts", missing_ok=False)
         # Inherit mount options from the host mount for certain mount types
         if fs_type in MOUNT_INHERIT_OPTIONS:
             mount_options = self["_mounts"][mountpoint].get("options", ["ro"])
-            if 'rw' in mount_options:
+            if "rw" in mount_options:
                 mount_options.pop(mount_options.index("rw"))  # Remove rw option if it exists
         else:  # For standard mounts, default ro
             mount_options = ["ro"]
@@ -840,8 +841,21 @@ def _autodetect_mount(self, mountpoint, mount_class="mounts", missing_ok=False)
     if fs_type == "zfs":
         mount_config[mount_name]["path"] = mount_device
 
+    # Run device mapper autodetection if enabled
+    if self["autodetect_dm"]:
+        if fs_type == "btrfs":
+            from ugrd.fs.btrfs import _get_btrfs_mount_devices
+
+            # Btrfs volumes may be backed by multiple dm devices
+            for device in _get_btrfs_mount_devices(self, mountpoint, mount_device):
+                _autodetect_dm(self, mountpoint, mount_device)
+        elif fs_type == "zfs":
+            for device in get_zpool_info(self, mount_device)["devices"]:
+                _autodetect_dm(self, mountpoint, mount_device)
+        else:
+            _autodetect_dm(self, mountpoint)
+
     self[mount_class] = mount_config
-    return mount_device
 
 
 @contains("auto_mounts", "Skipping auto mounts, auto_mounts is empty.", log_level=10)
@@ -852,6 +866,14 @@ def autodetect_mounts(self) -> None:
         _autodetect_mount(self, mountpoint)
 
 
+@contains("auto_late_mounts", "Skipping auto late mounts, auto_late_mounts is empty.", log_level=10)
+@contains("hostonly", "Skipping late mount autodetection, hostonly mode is disabled.", log_level=30)
+def autodetect_late_mounts(self) -> None:
+    """Configured the late_mounts config for a device based on the host mount config."""
+    for mountpoint in self["auto_late_mounts"]:
+        _autodetect_mount(self, mountpoint, mount_class="late_mounts")
+
+
 def mount_base(self) -> list[str]:
     """Generates mount commands for the base mounts.
     Must be run before variables are used, as it creates the /run/ugrd directory.
@@ -906,7 +928,9 @@ def mount_fstab(self) -> list[str]:
     mount_retries sets the number of times to retry the mount, infinite otherwise.
     """
     if not self._get_build_path("/etc/fstab").exists():
-        return self.logger.info("No initramfs fstab found, skipping mount_fstab. If non-root storage devices are not needed at boot, this is fine.")
+        return self.logger.info(
+            "No initramfs fstab found, skipping mount_fstab. If non-root storage devices are not needed at boot, this is fine."
+        )
 
     out = [
         'einfo "Attempting to mount all filesystems."',
@@ -1039,8 +1063,12 @@ def export_mount_info(self) -> None:
         self.logger.critical(f"Failed to get source info for the root mount: {e}")
         if not self["hostonly"]:
             self.logger.info("Root mount infomrmation can be defined under the '[mounts.root]' section.")
-            raise ValidationError("Root mount source information is not set, when hostonly mode is disabled, it must be manually defined.")
-        raise ValidationError("Root mount source information is not set even though hostonly mode is enabled. Please report a bug.")
+            raise ValidationError(
+                "Root mount source information is not set. When hostonly mode is disabled, it must be manually defined."
+            )
+        raise ValidationError(
+            "Root mount source information is not set even though hostonly mode is enabled. Please report a bug."
+        )
     self["exports"]["MOUNTS_ROOT_TYPE"] = self["mounts"]["root"].get("type", "auto")
     self["exports"]["MOUNTS_ROOT_OPTIONS"] = ",".join(self["mounts"]["root"]["options"])
     self["exports"]["MOUNTS_ROOT_TARGET"] = self["mounts"]["root"]["destination"]

src/ugrd/fs/mounts.toml

@@ -9,10 +9,10 @@ run_dirs = [ "ugrd" ]
 
 mount_timeout = 1
 autodetect_root = true
-autodetect_root_dm = true
-autodetect_root_luks = true
-autodetect_root_lvm = true
-autodetect_root_raid = true
+autodetect_dm = true
+autodetect_luks = true
+autodetect_lvm = true
+autodetect_raid = true
 autodetect_init_mount = true
 
 [imports.config_processing]
@@ -22,7 +22,7 @@ autodetect_init_mount = true
 
 [imports.build_enum]
 "ugrd.fs.mounts" = [ "get_mounts_info", "get_virtual_block_info", "get_blkid_info", "get_zpool_info",
-		     "autodetect_root", "autodetect_mounts", "autodetect_init_mount" ]
+		     "autodetect_root", "autodetect_mounts", "autodetect_late_mounts", "autodetect_init_mount" ]
 
 [imports.build_tasks]
 "ugrd.fs.mounts" = [ "export_mount_info" ]
@@ -58,14 +58,15 @@ mount_devpts = "bool"  # Whether or not to mount devpts
 run_dirs = "NoDupFlatList"  # A list of directories to be created under /run
 late_mounts = "dict"  # Like mounts, but run after the root is mounted
 auto_mounts = "NoDupFlatList"  # A list of mounts to be automatically added to the mounts list
+auto_late_mounts = "NoDupFlatList"  # A list of mounts to be automatically added to the late mounts list
 mount_timeout = "float"  # The time to wait between mount attempts
 mount_retries = "int"  # The number of times to re-attempt mounting the fstab, infinite if not set
 mount_cmd = "str"  # The mount command called by mount_root, can be overridden
 autodetect_root = "bool"  # Add the autodetect_root property, if defined, the root mount will be autodetected
-autodetect_root_dm = "bool"  # Whether or not to try to autodetect device-mapper partitions
-autodetect_root_luks = "bool"  # Whether or not to try to autodetect LUKS partitions
-autodetect_root_lvm = "bool"  # Whether or not to try to autodetect LVM partitions
-autodetect_root_raid = "bool"  # Whether or not to try to autodetect MDRAID partitions
+autodetect_dm = "bool"  # Whether or not to try to autodetect device-mapper partitions
+autodetect_luks = "bool"  # Whether or not to try to autodetect LUKS partitions
+autodetect_lvm = "bool"  # Whether or not to try to autodetect LVM partitions
+autodetect_raid = "bool"  # Whether or not to try to autodetect MDRAID partitions
 autodetect_init_mount = "bool"  # Adds a late_mount for the init target if it exists under a mount on the host
 no_fsck = "bool"  # Whether or not to skip fsck on the root device when applicable
 _mounts = "dict"  # The mounts information

src/ugrd/main.py

@@ -94,36 +94,36 @@ def main():
             "dest": "autodetect_root",
         },
         {
-            "flags": ["--autodetect-root-luks"],
+            "flags": ["--autodetect-luks"],
             "action": "store_true",
             "help": "autodetect LUKS volumes under the root partition",
         },
         {
-            "flags": ["--no-autodetect-root-luks"],
+            "flags": ["--no-autodetect-luks"],
             "action": "store_false",
-            "help": "do not autodetect root LUKS volumes",
-            "dest": "autodetect_root_luks",
+            "help": "do not autodetect LUKS volumes",
+            "dest": "autodetect_luks",
         },
-        {"flags": ["--autodetect-root-lvm"], "action": "store_true", "help": "autodetect LVM volumes"},
+        {"flags": ["--autodetect-lvm"], "action": "store_true", "help": "autodetect LVM volumes"},
         {
-            "flags": ["--no-autodetect-root-lvm"],
+            "flags": ["--no-autodetect-lvm"],
             "action": "store_false",
             "help": "do not autodetect LVM volumes",
-            "dest": "autodetect_root_lvm",
+            "dest": "autodetect_lvm",
         },
-        {"flags": ["--autodetect-root-raid"], "action": "store_true", "help": "autodetect MRRAID volumes"},
+        {"flags": ["--autodetect-raid"], "action": "store_true", "help": "autodetect MRRAID volumes"},
         {
-            "flags": ["--no-autodetect-root-raid"],
+            "flags": ["--no-autodetect-raid"],
             "action": "store_false",
             "help": "do not autodetect MRRAID volumes",
-            "dest": "autodetect_root_raid",
+            "dest": "autodetect_raid",
         },
-        {"flags": ["--autodetect-root-dm"], "action": "store_true", "help": "autodetect DM (LUKS/LVM) root partitions"},
+        {"flags": ["--autodetect-dm"], "action": "store_true", "help": "autodetect DM (LUKS/LVM) root partitions"},
         {
-            "flags": ["--no-autodetect-root-dm"],
+            "flags": ["--no-autodetect-dm"],
             "action": "store_false",
-            "help": "do not autodetect root DM volumes",
-            "dest": "autodetect_root_dm",
+            "help": "do not autodetect device mapper volumes",
+            "dest": "autodetect_dm",
         },
         {"flags": ["--print-config"], "action": "store_true", "help": "print the final config dict"},
         {"flags": ["--print-init"], "action": "store_true", "help": "print the final init structure"},
@@ -156,7 +156,7 @@ def main():
     if test:
         logger.warning("TEST MODE ENABLED")
         logger.info("Disabling DM autodetection")
-        kwargs["autodetect_root_dm"] = False
+        kwargs["autodetect_dm"] = False
         kwargs["modules"] = kwargs["modules"] + ",ugrd.base.test" if kwargs.get("modules") else "ugrd.base.test"
 
     logger.debug(f"Using the following kwargs: {kwargs}")

tests/fs/overlayfs.toml

@@ -3,4 +3,4 @@ modules = [ "ugrd.base.test", "ugrd.fs.overlayfs"]
 
 out_dir = "initramfs_test"
 cpio_compression = false
-autodetect_root_dm = false
+autodetect_dm = false

tests/fullauto.toml

@@ -12,7 +12,7 @@ out_dir = "initramfs_test"
 #test_kernel = "/boot/vmlinuz-6.6.35-gentoo-dist"
 cpio_compression = false
 
-autodetect_root_dm = false
+autodetect_dm = false
 
 # Optionally supply a kernel version, uses the current kernel version if not specified
 #kernel_version = "6.1.53-gentoo-dist"