Add dedicated ext4 test, improve docs and logging

docs/configuration.md

@@ -26,21 +26,24 @@ Modules write to a shared config dict that is accessible by other modules.
 
 > The main module, mostly pulls basic binaries and pulls the `core`, `mounts`, and `cpio` module.
 
-* `init_target` Set the init target for `switch_root`.
+* `init_target` Sets the init target for `switch_root`.
 * `autodetect_init` (true) Automatically set the init target based `which init`.
+* `loglevel` (5) Sets the kernel log level in the init script.
 * `shebang` (#!/bin/bash) sets the shebang on the init script.
 
 ### base.core
 
+* `hostonly` (true) Builds the initramfs for the current host, if disabled, validation is automatically disabled.
+* `validate` (true) adds additional checks to verify the initramfs will work on the build host.
 * `tmpdir` (/tmp) Sets the temporary directory as the base for the build and out dir.
 * `build_dir` (initramfs_build) If relative, it will be placed under `tmpdir`, defines the build directory.
+* `build_logging` (false) Enables additional logging during the build process.
+* `make_nodes` (false) Create real device nodes in the build dir. 
+* `find_libgcc` (true) Automatically locates libgcc using ldconfig -p and adds it to the initramfs.
 * `out_dir` (initramfs_out) If relative, it will be placed under `tmpdir`, defines the output directory.
-* `out_file` Sets the name of the output file, under `out_dir` unless a path is defined.
+* `out_file` Sets the name of the output file, under `out_dir`.
 * `clean` (true) forces the build dir to be cleaned on each run.
-* `hostonly` (true) Builds the initramfs for the current host, if disabled, validation is automatically disabled.
-* `validate` (true) adds additional checks to verify the initramfs will work on the build host.
 * `old_count` (1) Sets the number of old file to keep when running the `_rotate_old` function.
-* `file_owner` (portage) sets the owner for items pulled into the initramfs on the build system
 * `binaries` - A list used to define programs to be pulled into the initrams. `which` is used to find the path of added entries, and `lddtree` is used to resolve dependendies.
 * `paths` - A list of directores to create in the `build_dir`. They do not need a leading `/`.
 
@@ -52,6 +55,14 @@ Using the `dependencies` list will pull files into the initramfs using the same
 dependencies = [ "/etc/ugrd/testfile" ]
 ```
 
+##### Optional dependencies
+
+`opt_dependencies` attempts to add a file to `dependencies` but will not raise an error if the file is not found.
+
+##### Compressed dependencies
+
+`xz_dependencies` and `gz_dependnencies` can be used to decompress dependencies before adding them to the initramfs.
+
 #### Copying files to a different destination
 
 To copy files to a different path:
@@ -92,8 +103,9 @@ minor = 1
 
 Creates `/dev/console` with permissions `0o644`
 
-> Using `mknod_cpio` from `ugrd.fs.cpio` will not create the device nodes in the build dir, but within the CPIO archive
+By default, they will be processed by `ugrd.fs.cpio` and added to the CPIO archive.
 
+To create device nodes in the build dir, set `make_nodes` to `true`.
 
 ### base.cmdline
 

readme.md

@@ -13,7 +13,7 @@
 
 Generated images are as static and secure as possible, only including components and features required to mount the root and switch to it.
 
-µgRD itself is pure python, and uses the `pycpio` library to generate the CPIO archive.
+µgRD itself is written in pure Python, and uses the `pycpio` library to generate the CPIO archive.
 
 The final build environment is left in the specified `build_dir`, where it can be examined or repacked.
 
@@ -25,11 +25,11 @@ The original goal of this project was to create an initramfs suitable for decryp
 
 ### Auto-detection
 
-* Root mount, using `/proc/mounts`. `root=` and `rootflags=` can be used but are not required.
+* Root mount, using `/proc/mounts`. `root=` and `rootflags=` can be used but are not required
 * LUKS auto-configuration and validation for the root mount
 * Rootfs LVM, including under LUKS, is auto-mounted
-* MDRAID auto-configuration for the root mount.
-* BTRFS root subvolumes are automatically detected, but can be overridden or `subvol_selector` can be used to select a subvolume at boot time.
+* MDRAID auto-configuration for the root mount
+* BTRFS root subvolumes are automatically detected, but can be overridden or `subvol_selector` can be used to select a subvolume at boot time
 * `/usr` auto-mounting if the init system requires it
 * Auto-detection of kernel modules required by the storage device used by the root filesystem
 

src/ugrd/base/core.py

@@ -1,10 +1,10 @@
 __author__ = "desultory"
-__version__ = "3.9.2"
+__version__ = "3.10.0"
 
 from pathlib import Path
 from typing import Union
 
-from zenlib.util import NoDupFlatList, contains, unset
+from zenlib.util import NoDupFlatList, contains
 
 
 def detect_tmpdir(self) -> None:
@@ -139,7 +139,7 @@ def deploy_symlinks(self) -> None:
         self._symlink(symlink_parameters["source"], symlink_parameters["target"])
 
 
-@unset("mknod_cpio", "Skipping real device node creation with mknod, as mknod_cpio is not specified.", log_level=20)
+@contains("make_nodes", "Skipping real device node creation with mknod, as make_nodes is not specified.", log_level=20)
 def deploy_nodes(self) -> None:
     """Generates specified device nodes."""
     from os import makedev, mknod

src/ugrd/base/core.toml

@@ -71,6 +71,7 @@ copies = "dict"  # Copies dict, defines the files to be copied to the initramfs
 nodes = "dict"  # Nodes dict, defines the device nodes to be created
 paths = "NoDupFlatList"  # Paths to be created in the initramfs
 masks = "dict"  # Imports to be masked in the initramfs
+make_nodes = "bool"  # If true, actual device nodes will be created in the build dir
 out_dir = "Path"  # The directory where the initramfs is packed/output. If no packer is used, this is the final output directory.
 out_file = "str"  # The name of the output file, if absolute, overrids out dir with the path, and sets out_file to the filename
 old_count = "int"  # The number of times to cycle old files before deleting

src/ugrd/base/test.py

@@ -81,7 +81,7 @@ def make_test_image(self):
     target_fs = InitramfsGenerator(**kwargs)
     try:
         target_fs.build()
-    except (FileNotFoundError, RuntimeError) as e:
+    except (FileNotFoundError, RuntimeError, PermissionError) as e:
         self.logger.error("Test image configuration:\n%s", target_fs)
         raise RuntimeError("Failed to build test rootfs: %s" % e)
 

src/ugrd/fs/test_image.py

@@ -1,4 +1,4 @@
-__version__ = "0.6.0"
+__version__ = "0.7.1"
 
 from zenlib.util import contains
 
@@ -9,6 +9,20 @@ def init_banner(self):
     self['banner'] = f"echo {self['test_flag']}"
 
 
+def _allocate_image(self, image_path):
+    """ Allocate the test image size """
+    if image_path.exists():
+        if self.clean:
+            self.logger.warning("Removing existing filesystem image file: %s" % image_path)
+            image_path.unlink()
+        else:
+            raise Exception("File already exists and 'clean' is off: %s" % image_path)
+
+    with open(image_path, "wb") as f:
+        self.logger.info("Allocating test image file: %s" % f.name)
+        f.write(b"\0" * self.test_image_size * 2 ** 20)
+
+
 def make_test_image(self):
     """ Creates a test image from the build dir """
     build_dir = self._get_build_path('/').resolve()
@@ -20,9 +34,7 @@ def make_test_image(self):
     image_path = self._get_out_path(self['out_file'])
     if rootfs_type == 'ext4':
         # Create the test image file, flll with 0s
-        with open(image_path, "wb") as f:
-            self.logger.info("Creating test image file: %s" % f.name)
-            f.write(b"\0" * self.test_image_size * 2 ** 20)
+        _allocate_image(self, image_path)
         self._run(['mkfs', '-t', rootfs_type, '-d', build_dir, '-U', rootfs_uuid, '-F', image_path])
     elif rootfs_type == 'btrfs':
         if self['clean'] and image_path.exists():

tests/ext4.toml

@@ -0,0 +1,15 @@
+# EXT4 test
+modules = [ "ugrd.base.test" ]
+
+# The initramfs will be built in /tmp/initramfs if "build_dir" is not specified not specified
+out_dir = "initramfs_test"
+
+cpio_compression = false
+
+autodetect_root = false
+
+hostonly = false
+
+[mounts.root]
+uuid = "aaaabbbb-cccc-dddd-eeee-ffff00000000"
+type = "ext4"

tests/test_ugrd.py

@@ -11,6 +11,10 @@ def test_fullauto(self):
         generator = InitramfsGenerator(logger=self.logger, config='tests/fullauto.toml')
         generator.build()
 
+    def test_ext4(self):
+        generator = InitramfsGenerator(logger=self.logger, config='tests/ext4.toml')
+        generator.build()
+
     def test_xz(self):
         generator = InitramfsGenerator(logger=self.logger, config='tests/fullauto.toml', cpio_compression='xz')
         generator.build()