Merge pull request #341 from desultory/dev

revamp module alias/resolution system, add mmc platform driver detection

Author: desultory

src/ugrd/fs/mounts.py

@@ -6,6 +6,7 @@
 from typing import Union
 
 from ugrd.exceptions import AutodetectError, ValidationError
+from ugrd.kmod.platform import _get_platform_mmc_drivers
 from zenlib.util import colorize as c_
 from zenlib.util import contains, pretty_print
 
@@ -1075,6 +1076,7 @@ def resolve_blkdev_kmod(self, device) -> list[str]:
             kmods.append("virtio_scsi")
     elif device_name.startswith("mmcblk"):
         kmods.append("mmc_block")
+        kmods.extend(_get_platform_mmc_drivers(self, device_name))
     elif device_name.startswith("sr"):
         kmods.append("sr_mod")
     elif device_name.startswith("md"):

src/ugrd/kmod/kmod.py

@@ -1,25 +1,23 @@
 __author__ = "desultory"
-__version__ = "3.5.0"
+__version__ = "4.1.0"
 
 from pathlib import Path
 from platform import uname
+from re import search
 from struct import error as StructError
 from struct import unpack
 from subprocess import run
 from typing import Union
 
 from ugrd.exceptions import AutodetectError, ValidationError
-from ugrd.kmod import (
-    BuiltinModuleError,
-    DependencyResolutionError,
-    IgnoredModuleError,
-    MissingModuleError,
-)
+from ugrd.kmod import BuiltinModuleError, DependencyResolutionError, IgnoredModuleError, MissingModuleError
 from zenlib.util import colorize as c_
 from zenlib.util import contains, unset
 
+_KMOD_ALIASES = {}
 MODULE_METADATA_FILES = ["modules.order", "modules.builtin", "modules.builtin.modinfo"]
 
+
 def _normalize_kmod_name(self, module: Union[str, list]) -> str:
     """Replaces -'s with _'s in a kernel module name.
     ignores modules defined in kmod_no_normalize.
@@ -34,6 +32,32 @@ def _normalize_kmod_name(self, module: Union[str, list]) -> str:
     return module.replace("-", "_")
 
 
+def _normalize_kmod_alias(self, alias: str) -> str:
+    """Gets the base alias name from kmod alias info
+    gets data after : and , if present.
+    """
+    if not alias:
+        return ""
+    alias = alias.split(":", 1)[-1]  # Strip bus type
+    alias = alias.split(",", 1)[-1]
+    return _normalize_kmod_name(self, alias)
+
+
+def _resolve_kmod_alias(self, module: str) -> str:
+    """Attempts to resolve a kernel module alias to a module name.
+    Uses  /lib/modules/<kernel_version>/modules.alias to find the module name.
+    normalizes - to _ then replaces _ with [_-] to allow for both _ and - in the module name.
+    """
+    module = module.replace("-", "_")
+    module = module.replace("_", "[_-]")  # Allow for both _ and - in the module name
+    for alias, kmod in _KMOD_ALIASES.items():
+        if search(module, alias):
+            self.logger.info(f"Resolved kernel module alias: {c_(alias, 'blue')} -> {c_(kmod, 'cyan')}")
+            return kmod
+
+    raise MissingModuleError(f"Failed to resolve kernel module alias: {module}")
+
+
 def _process_kernel_modules_multi(self, module: str) -> None:
     """Adds kernel modules to self['kernel_modules']."""
     module = _normalize_kmod_name(self, module)
@@ -81,14 +105,22 @@ def _process__kmod_auto_multi(self, module: str) -> None:
     self["_kmod_auto"].append(module)
 
 
-def _get_kmod_info(self, module: str):
+def _get_kmod_info(self, module: str) -> dict:
     """
     Runs modinfo on a kernel module, parses the output and stored the results in self['_kmod_modinfo'].
     !!! Should be run after metadata is processed so the kver is set properly !!!
+
+    Returns the module info as a dictionary with the following keys:
+    - filename: The path to the module file.
+    - depends: A list of module dependencies.
+    - softdep: A list of soft dependencies.
+    - firmware: A list of firmware files required by the module.
+    Raises:
+        DependencyResolutionError: If the modinfo command fails, returns no output, or the module name can't be resolved.
     """
     module = _normalize_kmod_name(self, module)
     if module in self["_kmod_modinfo"]:
-        return self.logger.debug("[%s] Module info already exists." % module)
+        return module, self["_kmod_modinfo"][module]
     args = ["modinfo", module, "--set-version", self["kernel_version"]]
 
     try:
@@ -98,9 +130,15 @@ def _get_kmod_info(self, module: str):
         raise DependencyResolutionError("[%s] Failed to run modinfo command: %s" % (module, " ".join(args))) from e
 
     if not cmd.stdout and cmd.stderr:
-        raise DependencyResolutionError("[%s] Modinfo returned no output." % module)
+        try:
+            resolved_module = _resolve_kmod_alias(self, module)
+            return _get_kmod_info(self, resolved_module)
+        except MissingModuleError:
+            raise DependencyResolutionError(
+                "[%s] Modinfo returned no output and the alias name could no be resolved." % module
+            )
 
-    module_info = {}
+    module_info = {"filename": None, "depends": [], "softdep": [], "firmware": []}
     for line in cmd.stdout.decode().split("\n"):
         line = line.strip()
         if line.startswith("filename:"):
@@ -125,6 +163,51 @@ def _get_kmod_info(self, module: str):
 
     self.logger.debug("[%s] Module info: %s" % (module, module_info))
     self["_kmod_modinfo"][module] = module_info
+    return module, module_info
+
+
+@unset("no_kmod", "no_kmod is enabled, skipping module alias enumeration.", log_level=30)
+def get_module_aliases(self):
+    """Processes the kernel module aliases from /lib/modules/<kernel_version>/modules.alias."""
+    alias_file = Path("/lib/modules") / self["kernel_version"] / "modules.alias"
+    if not alias_file.exists():
+        self.logger.error(f"Kernel module alias file does not exist: {c_(alias_file, 'red', bold=True)}")
+    else:
+        for line in alias_file.read_text().splitlines():
+            _, alias, module = line.strip().split(" ", 2)
+            _KMOD_ALIASES[_normalize_kmod_alias(self, alias)] = _normalize_kmod_name(self, module)
+
+
+@unset("no_kmod", "no_kmod is enabled, skipping builtin module enumeration.", log_level=30)
+def get_builtin_module_info(self) -> None:
+    """Gets the kernel module aliases from /lib/modules/<kernel_version>/modules.builtin.modinfo.
+    puts it in _kmod_modinfo.
+    also populates the _KMOD_ALIASES global variable with the aliases.
+    """
+
+    builtin_modinfo_file = Path("/lib/modules") / self["kernel_version"] / "modules.builtin.modinfo"
+    if not builtin_modinfo_file.exists():
+        self.logger.error(f"Builtin modinfo file does not exist: {c_(builtin_modinfo_file, 'red', bold=True)}")
+    else:
+        for line in builtin_modinfo_file.read_bytes().split(b"\x00"):
+            """ Lines are in the format <name>.<parameter>=<value>"""
+            line = line.decode("utf-8", errors="ignore").strip()
+            if not line or "." not in line or "=" not in line:
+                continue
+            name, parameter = line.split(".", 1)
+            name = _normalize_kmod_name(self, name)
+            parameter, value = parameter.split("=", 1)
+            modinfo = self["_kmod_modinfo"].get(
+                name, {"filename": "(builtin)", "depends": [], "softdep": [], "firmware": []}
+            )
+            if parameter == "firmware":
+                modinfo["firmware"].append(value)
+            elif parameter != "alias":
+                continue
+
+            alias = _normalize_kmod_alias(self, value)
+            self["_kmod_modinfo"][name] = modinfo
+            _KMOD_ALIASES[alias] = name  # Store the alias in the global aliases dict
 
 
 @contains("kmod_autodetect_lspci", "kmod_autodetect_lspci is not enabled, skipping.")
@@ -159,12 +242,14 @@ def _autodetect_modules_lsmod(self) -> None:
         modules = [line.split()[0] for line in f.readlines()]
 
     if len(modules) > 25:
-        self.logger.warning(f"[{len(modules)}] More than 25 kernel modules were autodetected from the running kernel. If lsmod detection is required for your use case, please file a bug report so more appropriate detection methods can be implemented.")
+        self.logger.warning(
+            f"[{len(modules)}] More than 25 kernel modules were autodetected from the running kernel. If lsmod detection is required for your use case, please file a bug report so more appropriate detection methods can be implemented."
+        )
     for module in modules:
         self["_kmod_auto"] = module.split()[0]
 
 
-@unset("no_kmod", "no_kmod is enabled, skipping.", log_level=30)
+@unset("no_kmod", "no_kmod is enabled, skipping module detection.", log_level=30)
 @contains("hostonly", "Skipping kmod autodetection, hostonly is disabled.", log_level=30)
 def autodetect_modules(self) -> None:
     """Autodetects kernel modules from lsmod and/or lspci -k."""
@@ -309,24 +394,24 @@ def _add_kmod_firmware(self, kmod: str) -> None:
 
     Attempts to run even if no_kmod is set; this will not work if there are no kmods/no kernel version set
     """
-    kmod = _normalize_kmod_name(self, kmod)
-
-    if kmod not in self["_kmod_modinfo"]:
+    try:
+        kmod, modinfo = _get_kmod_info(self, kmod)
+    except DependencyResolutionError as e:
         if self["no_kmod"]:
             return self.logger.warning(
                 "[%s] Kernel module info for firmware detection does not exist, but no_kmod is set." % kmod
             )
-        raise DependencyResolutionError("Kernel module info does not exist: %s" % kmod)
+        raise DependencyResolutionError("Kernel module info does not exist: %s" % kmod) from e
 
-    if self["_kmod_modinfo"][kmod].get("firmware") and not self["kmod_pull_firmware"]:
+    if modinfo["firmware"] and not self["kmod_pull_firmware"]:
         # Log a warning if the kernel module has firmware files, but kmod_pull_firmware is not set
         self.logger.warning("[%s] Kernel module has firmware files, but kmod_pull_firmware is not set." % kmod)
 
-    if not self["_kmod_modinfo"][kmod].get("firmware") or not self.get("kmod_pull_firmware"):
+    if not modinfo["firmware"] or not self.get("kmod_pull_firmware"):
         # No firmware files to add, or kmod_pull_firmware is not set
         return
 
-    for firmware in self["_kmod_modinfo"][kmod]["firmware"]:
+    for firmware in modinfo["firmware"]:
         _add_firmware_dep(self, kmod, firmware)
 
 
@@ -356,17 +441,18 @@ def _process_kmod_dependencies(self, kmod: str, mod_tree=None) -> None:
 
     Iterate over dependencies, adding them to kernel_mdules if they (or sub-dependencies) are not in the ignore list.
     If the dependency is already in the module tree, skip it to prevent infinite recursion.
+
+    returns the name of the kernel module (in case it was an alias) and the list of dependencies.
     """
     mod_tree = mod_tree or set()
-    kmod = _normalize_kmod_name(self, kmod)
-    _get_kmod_info(self, kmod)
+    kmod, modinfo = _get_kmod_info(self, kmod)
 
     # Get kernel module dependencies, softedeps if not ignored
     dependencies = []
-    if harddeps := self["_kmod_modinfo"][kmod].get("depends"):
+    if harddeps := modinfo["depends"]:
         dependencies += harddeps
 
-    if sofdeps := self["_kmod_modinfo"][kmod].get("softdep"):
+    if sofdeps := modinfo["softdep"]:
         if self.get("kmod_ignore_softdeps", False):
             self.logger.warning("[%s] Soft dependencies were detected, but are being ignored: %s" % (kmod, sofdeps))
         else:
@@ -378,12 +464,11 @@ def _process_kmod_dependencies(self, kmod: str, mod_tree=None) -> None:
         if dependency in mod_tree:
             self.logger.debug("[%s] Dependency is already in mod_tree: %s" % (kmod, dependency))
             continue
+        dependency, dep_modinfo = _get_kmod_info(self, dependency)  # Get modinfo for the dependency
         if dependency in self["kmod_ignore"]:  # Don't add modules with ignored dependencies
-            _get_kmod_info(self, dependency)  # Make sure modinfo is queried in case it's built-in
-            if modinfo := self["_kmod_modinfo"].get(dependency):
-                if modinfo["filename"] == "(builtin)":  # If it's ignored because builtin, that's fine
-                    self.logger.debug("[%s] Ignored dependency is a built-in module: %s" % (kmod, dependency))
-                    continue
+            if dep_modinfo["filename"] == "(builtin)":
+                self.logger.debug("[%s] Ignored dependency is a built-in module: %s" % (kmod, dependency))
+                continue
             # If modinfo doesn't exist, or it's not builtin, simply raise an ignored module error
             raise IgnoredModuleError("[%s] Kernel module dependency is in ignore list: %s" % (kmod, dependency))
         if dependency in self["kernel_modules"]:
@@ -398,10 +483,12 @@ def _process_kmod_dependencies(self, kmod: str, mod_tree=None) -> None:
             continue
         self["kernel_modules"] = dependency
 
-    if self["_kmod_modinfo"][kmod]["filename"] == "(builtin)":  # for built-in modules, just add firmware and return
+    if modinfo["filename"] == "(builtin)":  # for built-in modules, just add firmware and return
         _add_kmod_firmware(self, kmod)
         raise BuiltinModuleError("Not adding built-in module to dependencies: %s" % kmod)
 
+    return kmod, dependencies
+
 
 def add_kmod_deps(self):
     """Adds all kernel modules to the initramfs dependencies.
@@ -420,7 +507,8 @@ def add_kmod_deps(self):
             continue
 
         # Add the kmod file to the initramfs dependenceis
-        filename = self["_kmod_modinfo"][kmod]["filename"]
+        kmod, modinfo = _get_kmod_info(self, kmod)
+        filename = modinfo["filename"]
         if filename.endswith(".ko"):
             self["dependencies"] = filename
         elif filename.endswith(".ko.xz"):
@@ -440,16 +528,19 @@ def process_ignored_module(self, module: str) -> None:
     for key in ["kmod_init", "kernel_modules", "_kmod_auto"]:
         if module in self[key]:
             if key == "kmod_init":
-                if module in self["_kmod_modinfo"] and self["_kmod_modinfo"][module]["filename"] == "(builtin)":
-                    self.logger.debug("Removing built-in module from kmod_init: %s" % module)
-                elif module == "zfs":
-                    self.logger.critical("ZFS module is required but missing.")
-                    self.logger.critical("Please build/install the required kmods before running this script.")
-                    self.logger.critical("Detected kernel version: %s" % self["kernel_version"])
-                    # https://github.com/projg2/installkernel-gentoo/commit/1c70dda8cd2700e5306d2ed74886b66ad7ccfb42
-                    exit(77)
-                else:
-                    raise MissingModuleError("Required module cannot be imported and is not builtin: %s" % module)
+                try:
+                    module, modinfo = _get_kmod_info(self, module)
+                    if modinfo["filename"] == "(builtin)":
+                        self.logger.debug("Removing built-in module from kmod_init: %s" % module)
+                except DependencyResolutionError:
+                    if module == "zfs":
+                        self.logger.critical("ZFS module is required but missing.")
+                        self.logger.critical("Please build/install the required kmods before running this script.")
+                        self.logger.critical("Detected kernel version: %s" % self["kernel_version"])
+                        # https://github.com/projg2/installkernel-gentoo/commit/1c70dda8cd2700e5306d2ed74886b66ad7ccfb42
+                        exit(77)
+                    else:
+                        raise MissingModuleError("Required module cannot be imported and is not builtin: %s" % module)
             else:
                 self.logger.debug("Removing ignored kernel module from %s: %s" % (key, module))
             self[key].remove(module)
@@ -469,15 +560,17 @@ def _process_optional_modules(self) -> None:
             self.logger.debug(f"Optional kmod_init module is already in kmod_init: {c_(kmod, 'yellow', bold=True)}")
             continue
         try:
-            _process_kmod_dependencies(self, kmod)
+            kmod, dependencies = _process_kmod_dependencies(self, kmod)
             self["kmod_init"] = kmod  # add to kmod_init so it will be loaded
         except IgnoredModuleError as e:
             self.logger.warning(e)
         except BuiltinModuleError:
             self.logger.debug(f"Optional kmod_init module is built-in, skipping: {c_(kmod, 'yellow')}")
             continue
         except DependencyResolutionError as e:
-            self.logger.warning(f"[{c_(kmod, 'yellow', bold=True)}] Failed to process optional kernel module dependencies: {e}")
+            self.logger.warning(
+                f"[{c_(kmod, 'yellow', bold=True)}] Failed to process optional kernel module dependencies: {e}"
+            )
 
 
 @unset("no_kmod", "no_kmod is enabled, skipping.", log_level=30)
@@ -486,6 +579,11 @@ def process_modules(self) -> None:
     _process_optional_modules(self)
     self.logger.debug("Processing kernel modules: %s" % self["kernel_modules"])
     for kmod in self["kernel_modules"].copy():
+        """ Process all kernel modules
+        for kmod_init modules, log an error if info can't be retreived, but continue processing.
+        in successful cases, continue, if he module processing fails, add to the ignore list.
+        Later, when ignored modules are processed, an exception is raised if the module is required.
+        """
         self.logger.debug("Processing kernel module: %s" % kmod)
         try:
             _process_kmod_dependencies(self, kmod)
@@ -507,13 +605,14 @@ def process_modules(self) -> None:
         self["kmod_ignore"] = kmod
 
     for kmod in self["_kmod_auto"]:
+        """ Do similar for automatic modules, but log warnings insead of errors if dependencies are missing. """
         if kmod in self["kernel_modules"]:
             self.logger.debug("Autodetected module is already in kernel_modules: %s" % kmod)
             continue
         self.logger.debug("Processing autodetected kernel module: %s" % kmod)
         try:
-            _process_kmod_dependencies(self, kmod)
-            self["kmod_init"] = kmod
+            kmod_name, dependencies = _process_kmod_dependencies(self, kmod)
+            self["kmod_init"] = kmod_name
             continue
         except BuiltinModuleError:
             continue  # Don't add built-in modules to the ignore list

src/ugrd/kmod/kmod.toml

@@ -31,7 +31,7 @@ kmod_no_normalize = "NoDupFlatList" # Kernel modules to not normalize (i.e. not
 		     "_process__kmod_auto_multi" ]
 
 [imports.build_enum]
-"ugrd.kmod.kmod" = [ "get_kernel_version", "autodetect_modules" ]
+"ugrd.kmod.kmod" = [ "get_kernel_version", "get_module_aliases", "get_builtin_module_info", "autodetect_modules" ]
 
 [imports.build_late]
 "ugrd.kmod.kmod" = [ "process_modules", "process_ignored_modules", "process_module_metadata", "add_kmod_deps" ]