Merge pull request #311 from desultory/dev

add kmod_init_optional, move some USB modules there

Author: desultory

docs/configuration.md

@@ -171,6 +171,7 @@ The following parameters can be used to change the kernel module pulling and ini
 * `kernel_version` (uname -r) Used to specify the kernel version to pull modules for, should be a directory under `/lib/modules/<kernel_version>`.
 * `kmod_pull_firmware` (true) Adds kernel module firmware to dependencies
 * `kmod_init` - Kernel modules to `modprobe` at boot.
+* `kmod_init_optional` - Modules to attempt to add to `kmod_init`, failing with a warning if not found.
 * `kmod_autodetect_lspci` (false) Finds kernel modules for PCI devices using `/sys/bus/pci/drivers`, formely used `lspci -k`.
 * `kmod_autodetect_lsmod` (false) Pulls kernel modules using `/proc/modules`, formerly used `lsmod`.
 * `kernel_modules` - Kernel modules to pull into the initramfs. These modules will not be `modprobe`'d automatically.

src/ugrd/kmod/kmod.py

@@ -1,5 +1,5 @@
 __author__ = "desultory"
-__version__ = "3.3.7"
+__version__ = "3.4.0"
 
 from pathlib import Path
 from platform import uname
@@ -8,7 +8,13 @@
 from subprocess import run
 
 from ugrd.exceptions import AutodetectError, ValidationError
-from ugrd.kmod import BuiltinModuleError, MissingModuleError, DependencyResolutionError, IgnoredModuleError, _normalize_kmod_name
+from ugrd.kmod import (
+    BuiltinModuleError,
+    DependencyResolutionError,
+    IgnoredModuleError,
+    MissingModuleError,
+    _normalize_kmod_name,
+)
 from zenlib.util import colorize as c_
 from zenlib.util import contains, unset
 
@@ -37,6 +43,20 @@ def _process_kmod_init_multi(self, module: str) -> None:
     self["kernel_modules"] = module
 
 
+def _process_kmod_init_optional_multi(self, module: str) -> None:
+    """Adds an optional kmod init module"""
+    module = _normalize_kmod_name(module)
+    if module in self["kmod_ignore"]:
+        self.logger.warning(f"Optional kmod_init module is in the ignore list: {c_(module, 'yellow', bold=True)}")
+        self["_kmod_removed"] = module
+        return
+    if module in self["kmod_init"]:
+        self.logger.debug(f"Optional kmod_init module is already in kmod_init: {c_(module, 'yellow', bold=True)}")
+        return
+    self.logger.debug(f"Adding optional kmod_init module: {c_(module, 'magenta')}")
+    self["kmod_init_optional"].append(module)
+
+
 def _process__kmod_auto_multi(self, module: str) -> None:
     """Adds autodetected modules to self['kernel_modules']."""
     module = _normalize_kmod_name(module)
@@ -201,7 +221,9 @@ def _process_kernel_version(self, kver: str) -> None:
         if self["no_kmod"]:
             return self.logger.warning("[%s] Kernel module directory does not exist, but no_kmod is set." % kver)
         self.logger.error(f"Available kernel versions: {', '.join([d.name for d in Path('/lib/modules').iterdir()])}")
-        self.logger.info("If kernel modules are not installed, and not required, set `no_kmod = true` to skip this check.")
+        self.logger.info(
+            "If kernel modules are not installed, and not required, set `no_kmod = true` to skip this check."
+        )
         raise ValidationError(f"Kernel module directory does not exist for kernel: {kver}")
 
     self.data["kernel_version"] = kver
@@ -373,7 +395,9 @@ def add_kmod_deps(self):
         if self.get("kernel_version"):
             _add_kmod_firmware(self, kmod)
         else:
-            self.logger.warning(f"Kernel version is not set, skipping firmware detection for kmod: {c_(kmod, 'yellow')}")
+            self.logger.warning(
+                f"Kernel version is not set, skipping firmware detection for kmod: {c_(kmod, 'yellow')}"
+            )
         # if no_kmod is set, continue and check for the firmware of the next module
         if self["no_kmod"]:
             continue
@@ -421,9 +445,28 @@ def process_ignored_modules(self) -> None:
         process_ignored_module(self, module)
 
 
+def _process_optional_modules(self) -> None:
+    """Processes optional kernel modules."""
+    for kmod in self["kmod_init_optional"]:
+        if kmod in self["kmod_init"]:
+            self.logger.debug(f"Optional kmod_init module is already in kmod_init: {c_(kmod, 'yellow', bold=True)}")
+            continue
+        try:
+            _process_kmod_dependencies(self, kmod)
+            self["kmod_init"] = kmod  # add to kmod_init so it will be loaded
+        except IgnoredModuleError as e:
+            self.logger.info(e)
+        except BuiltinModuleError:
+            self.logger.debug(f"Optional kmod_init module is built-in, skipping: {c_(kmod, 'yellow')}")
+            continue
+        except DependencyResolutionError as e:
+            self.logger.warning(f"[{c_(kmod, 'yellow', bold=True)}] Failed to process optional kernel module dependencies: {e}")
+
+
 @unset("no_kmod", "no_kmod is enabled, skipping.", log_level=30)
 def process_modules(self) -> None:
     """Processes all kernel modules, adding dependencies to the initramfs."""
+    _process_optional_modules(self)
     self.logger.debug("Processing kernel modules: %s" % self["kernel_modules"])
     for kmod in self["kernel_modules"].copy():
         self.logger.debug("Processing kernel module: %s" % kmod)
@@ -441,6 +484,7 @@ def process_modules(self) -> None:
             self.logger.info(e)
         except DependencyResolutionError as e:
             if kmod in self["kmod_init"]:
+                # Once optional modules are fully implemented, this should raise an exception instead
                 self.logger.error("[%s] Failed to get modinfo for init kernel module: %s" % (kmod, e))
             self.logger.debug("[%s] Failed to get modinfo for kernel module: %s" % (kmod, e))
         self["kmod_ignore"] = kmod
@@ -457,8 +501,10 @@ def process_modules(self) -> None:
         except BuiltinModuleError:
             continue  # Don't add built-in modules to the ignore list
         except IgnoredModuleError as e:
-            self.logger.debug(e)
-        except DependencyResolutionError as e:
+            self.logger.debug(e)  # when autodetected modules are ignored, only debug log
+        except (
+            DependencyResolutionError
+        ) as e:  # log a warning, not error or exception when autodetected modules have missing deps
             self.logger.warning("[%s] Failed to process autodetected kernel module dependencies: %s" % (kmod, e))
         self["kmod_ignore"] = kmod
 
@@ -471,7 +517,9 @@ def load_modules(self) -> str:
     removed_kmods = ", ".join(self["_kmod_removed"])
     if self["no_kmod"]:
         if included_kmods or init_kmods:
-            self.logger.warning("no_kmod is enabled, but kernel modules are set, ensure the following kernel modules are built into the kernel:")
+            self.logger.warning(
+                "no_kmod is enabled, but kernel modules are set, ensure the following kernel modules are built into the kernel:"
+            )
             self.logger.warning(f"Init kernel modules: {c_(init_kmods, 'red', bold=True)}")
             self.logger.warning(f"Included kernel modules: {c_(included_kmods, 'red', bold=True)}")
         return

src/ugrd/kmod/kmod.toml

@@ -20,6 +20,7 @@ kmod_autodetect_lsmod = "bool"  # Whether or not to automatically pull currently
 kmod_autodetect_lspci = "bool"  # Whether or not to automatically pull kernel modules from lspci -k
 kernel_modules = "NoDupFlatList"  # Kernel modules to pull into the initramfs
 kmod_init = "NoDupFlatList"  # Kernel modules to load at initramfs startup
+kmod_init_optional = "NoDupFlatList"  # Kernel modules to try to add to kmod_init
 no_kmod = "bool" # Disables kernel modules entirely
 
 [imports.config_processing]

src/ugrd/kmod/usb.toml

@@ -1 +1,2 @@
-kmod_init = ['uas', 'usbcore', 'scsi_mod', 'ehci_hcd', 'ohci_hcd', 'uhci_hcd', 'xhci_hcd']
+kmod_init = ['uas', 'usbcore']
+kmod_init_optional = ['ehci_hcd', 'ohci_hcd', 'uhci_hcd', 'xhci_hcd']