improve module alias system

Signed-off-by: Zen <[email protected]>

Author: desultory

src/ugrd/kmod/kmod.py

@@ -1,5 +1,5 @@
 __author__ = "desultory"
-__version__ = "3.6.0"
+__version__ = "4.0.0"
 
 from pathlib import Path
 from platform import uname
@@ -32,25 +32,15 @@ def _normalize_kmod_name(self, module: Union[str, list]) -> str:
     return module.replace("-", "_")
 
 
-def _get_kmod_aliases(self):
-    """Returns a processed dictionary of kernel module aliases."""
-    if _KMOD_ALIASES:
-        return _KMOD_ALIASES
-
-    if not self.get("kernel_version"):
-        raise AutodetectError("Kernel version is not set, cannot resolve kernel module aliases.")
-
-    alias_file = Path("/lib/modules") / self["kernel_version"] / "modules.alias"
-    if not alias_file.exists():
-        raise FileNotFoundError(f"Kernel module alias file does not exist: {c_(alias_file, 'red', bold=True)}")
-
-    for line in alias_file.read_text().splitlines():
-        _, alias, module = line.strip().split(" ", 2)
-        # Strip bus type
-        alias = alias.split(":", 1)[-1]
-        alias = alias.split(",", 1)[-1]
-        _KMOD_ALIASES[alias] = _normalize_kmod_name(self, module)
-    return _KMOD_ALIASES
+def _normalize_kmod_alias(self, alias: str) -> str:
+    """Gets the base alias name from kmod alias info
+    gets data after : and , if present.
+    """
+    if not alias:
+        return ""
+    alias = alias.split(":", 1)[-1]  # Strip bus type
+    alias = alias.split(",", 1)[-1]
+    return _normalize_kmod_name(self, alias)
 
 
 def _resolve_kmod_alias(self, module: str) -> str:
@@ -60,8 +50,7 @@ def _resolve_kmod_alias(self, module: str) -> str:
     """
     module = module.replace("-", "_")
     module = module.replace("_", "[_-]")  # Allow for both _ and - in the module name
-    aliases = _get_kmod_aliases(self)
-    for alias, kmod in aliases.items():
+    for alias, kmod in _KMOD_ALIASES.items():
         if search(module, alias):
             self.logger.debug(f"Resolved kernel module alias: {c_(alias, 'green')} -> {c_(kmod, 'cyan')}")
             return kmod
@@ -116,14 +105,14 @@ def _process__kmod_auto_multi(self, module: str) -> None:
     self["_kmod_auto"].append(module)
 
 
-def _get_kmod_info(self, module: str):
+def _get_kmod_info(self, module: str) -> dict:
     """
     Runs modinfo on a kernel module, parses the output and stored the results in self['_kmod_modinfo'].
     !!! Should be run after metadata is processed so the kver is set properly !!!
     """
     module = _normalize_kmod_name(self, module)
     if module in self["_kmod_modinfo"]:
-        return self.logger.debug("[%s] Module info already exists." % module)
+        return self["_kmod_modinfo"][module]
     args = ["modinfo", module, "--set-version", self["kernel_version"]]
 
     try:
@@ -138,10 +127,10 @@ def _get_kmod_info(self, module: str):
             return _get_kmod_info(self, resolved_module)
         except MissingModuleError:
             raise DependencyResolutionError(
-                "[%s] Modinfo returned no output and alias name could no be resolved." % module
+                "[%s] Modinfo returned no output and the alias name could no be resolved." % module
             )
 
-    module_info = {}
+    module_info = {"filename": None, "depends": [], "softdep": [], "firmware": []}
     for line in cmd.stdout.decode().split("\n"):
         line = line.strip()
         if line.startswith("filename:"):
@@ -166,6 +155,51 @@ def _get_kmod_info(self, module: str):
 
     self.logger.debug("[%s] Module info: %s" % (module, module_info))
     self["_kmod_modinfo"][module] = module_info
+    return module_info
+
+
+@unset("no_kmod", "no_kmod is enabled, skipping module alias enumeration.", log_level=30)
+def get_module_aliases(self):
+    """Processes the kernel module aliases from /lib/modules/<kernel_version>/modules.alias."""
+    alias_file = Path("/lib/modules") / self["kernel_version"] / "modules.alias"
+    if not alias_file.exists():
+        self.logger.error(f"Kernel module alias file does not exist: {c_(alias_file, 'red', bold=True)}")
+    else:
+        for line in alias_file.read_text().splitlines():
+            _, alias, module = line.strip().split(" ", 2)
+            _KMOD_ALIASES[_normalize_kmod_alias(self, alias)] = _normalize_kmod_name(self, module)
+
+
+@unset("no_kmod", "no_kmod is enabled, skipping builtin module enumeration.", log_level=30)
+def get_builtin_module_info(self) -> None:
+    """Gets the kernel module aliases from /lib/modules/<kernel_version>/modules.builtin.modinfo.
+    puts it in _kmod_modinfo.
+    also populates the _KMOD_ALIASES global variable with the aliases.
+    """
+
+    builtin_modinfo_file = Path("/lib/modules") / self["kernel_version"] / "modules.builtin.modinfo"
+    if not builtin_modinfo_file.exists():
+        self.logger.error(f"Builtin modinfo file does not exist: {c_(builtin_modinfo_file, 'red', bold=True)}")
+    else:
+        for line in builtin_modinfo_file.read_bytes().split(b"\x00"):
+            """ Lines are in the format <name>.<parameter>=<value>"""
+            line = line.decode("utf-8", errors="ignore").strip()
+            if not line or "." not in line or "=" not in line:
+                continue
+            name, parameter = line.split(".", 1)
+            name = _normalize_kmod_name(self, name)
+            parameter, value = parameter.split("=", 1)
+            modinfo = self["_kmod_modinfo"].get(
+                name, {"filename": "(builtin)", "depends": [], "softdep": [], "firmware": []}
+            )
+            if parameter == "firmware":
+                modinfo["firmware"] += value
+            elif parameter != "alias":
+                continue
+
+            alias = _normalize_kmod_alias(self, value)
+            self["_kmod_modinfo"][name] = modinfo
+            _KMOD_ALIASES[alias] = name  # Store the alias in the global aliases dict
 
 
 @contains("kmod_autodetect_lspci", "kmod_autodetect_lspci is not enabled, skipping.")
@@ -207,7 +241,7 @@ def _autodetect_modules_lsmod(self) -> None:
         self["_kmod_auto"] = module.split()[0]
 
 
-@unset("no_kmod", "no_kmod is enabled, skipping.", log_level=30)
+@unset("no_kmod", "no_kmod is enabled, skipping module detection.", log_level=30)
 @contains("hostonly", "Skipping kmod autodetection, hostonly is disabled.", log_level=30)
 def autodetect_modules(self) -> None:
     """Autodetects kernel modules from lsmod and/or lspci -k."""
@@ -352,24 +386,24 @@ def _add_kmod_firmware(self, kmod: str) -> None:
 
     Attempts to run even if no_kmod is set; this will not work if there are no kmods/no kernel version set
     """
-    kmod = _normalize_kmod_name(self, kmod)
-
-    if kmod not in self["_kmod_modinfo"]:
+    try:
+        modinfo = _get_kmod_info(self, kmod)
+    except DependencyResolutionError as e:
         if self["no_kmod"]:
             return self.logger.warning(
                 "[%s] Kernel module info for firmware detection does not exist, but no_kmod is set." % kmod
             )
-        raise DependencyResolutionError("Kernel module info does not exist: %s" % kmod)
+        raise DependencyResolutionError("Kernel module info does not exist: %s" % kmod) from e
 
-    if self["_kmod_modinfo"][kmod].get("firmware") and not self["kmod_pull_firmware"]:
+    if modinfo["firmware"] and not self["kmod_pull_firmware"]:
         # Log a warning if the kernel module has firmware files, but kmod_pull_firmware is not set
         self.logger.warning("[%s] Kernel module has firmware files, but kmod_pull_firmware is not set." % kmod)
 
-    if not self["_kmod_modinfo"][kmod].get("firmware") or not self.get("kmod_pull_firmware"):
+    if not modinfo["firmware"] or not self.get("kmod_pull_firmware"):
         # No firmware files to add, or kmod_pull_firmware is not set
         return
 
-    for firmware in self["_kmod_modinfo"][kmod]["firmware"]:
+    for firmware in modinfo["firmware"]:
         _add_firmware_dep(self, kmod, firmware)
 
 
@@ -401,15 +435,14 @@ def _process_kmod_dependencies(self, kmod: str, mod_tree=None) -> None:
     If the dependency is already in the module tree, skip it to prevent infinite recursion.
     """
     mod_tree = mod_tree or set()
-    kmod = _normalize_kmod_name(self, kmod)
-    _get_kmod_info(self, kmod)
+    modinfo = _get_kmod_info(self, kmod)
 
     # Get kernel module dependencies, softedeps if not ignored
     dependencies = []
-    if harddeps := self["_kmod_modinfo"][kmod].get("depends"):
+    if harddeps := modinfo["depends"]:
         dependencies += harddeps
 
-    if sofdeps := self["_kmod_modinfo"][kmod].get("softdep"):
+    if sofdeps := modinfo["softdep"]:
         if self.get("kmod_ignore_softdeps", False):
             self.logger.warning("[%s] Soft dependencies were detected, but are being ignored: %s" % (kmod, sofdeps))
         else:
@@ -421,12 +454,11 @@ def _process_kmod_dependencies(self, kmod: str, mod_tree=None) -> None:
         if dependency in mod_tree:
             self.logger.debug("[%s] Dependency is already in mod_tree: %s" % (kmod, dependency))
             continue
+        dep_modinfo = _get_kmod_info(self, dependency)  # Get modinfo for the dependency
         if dependency in self["kmod_ignore"]:  # Don't add modules with ignored dependencies
-            _get_kmod_info(self, dependency)  # Make sure modinfo is queried in case it's built-in
-            if modinfo := self["_kmod_modinfo"].get(dependency):
-                if modinfo["filename"] == "(builtin)":  # If it's ignored because builtin, that's fine
-                    self.logger.debug("[%s] Ignored dependency is a built-in module: %s" % (kmod, dependency))
-                    continue
+            if dep_modinfo["filename"] == "(builtin)":
+                self.logger.debug("[%s] Ignored dependency is a built-in module: %s" % (kmod, dependency))
+                continue
             # If modinfo doesn't exist, or it's not builtin, simply raise an ignored module error
             raise IgnoredModuleError("[%s] Kernel module dependency is in ignore list: %s" % (kmod, dependency))
         if dependency in self["kernel_modules"]:
@@ -441,7 +473,7 @@ def _process_kmod_dependencies(self, kmod: str, mod_tree=None) -> None:
             continue
         self["kernel_modules"] = dependency
 
-    if self["_kmod_modinfo"][kmod]["filename"] == "(builtin)":  # for built-in modules, just add firmware and return
+    if modinfo["filename"] == "(builtin)":  # for built-in modules, just add firmware and return
         _add_kmod_firmware(self, kmod)
         raise BuiltinModuleError("Not adding built-in module to dependencies: %s" % kmod)
 
@@ -463,7 +495,8 @@ def add_kmod_deps(self):
             continue
 
         # Add the kmod file to the initramfs dependenceis
-        filename = self["_kmod_modinfo"][kmod]["filename"]
+        modinfo = _get_kmod_info(self, kmod)
+        filename = modinfo["filename"]
         if filename.endswith(".ko"):
             self["dependencies"] = filename
         elif filename.endswith(".ko.xz"):
@@ -483,7 +516,8 @@ def process_ignored_module(self, module: str) -> None:
     for key in ["kmod_init", "kernel_modules", "_kmod_auto"]:
         if module in self[key]:
             if key == "kmod_init":
-                if module in self["_kmod_modinfo"] and self["_kmod_modinfo"][module]["filename"] == "(builtin)":
+                modinfo = _get_kmod_info(self, module)
+                if modinfo["filename"] == "(builtin)":
                     self.logger.debug("Removing built-in module from kmod_init: %s" % module)
                 elif module == "zfs":
                     self.logger.critical("ZFS module is required but missing.")

src/ugrd/kmod/kmod.toml

@@ -31,7 +31,7 @@ kmod_no_normalize = "NoDupFlatList" # Kernel modules to not normalize (i.e. not
 		     "_process__kmod_auto_multi" ]
 
 [imports.build_enum]
-"ugrd.kmod.kmod" = [ "get_kernel_version", "autodetect_modules" ]
+"ugrd.kmod.kmod" = [ "get_kernel_version", "get_module_aliases", "get_builtin_module_info", "autodetect_modules" ]
 
 [imports.build_late]
 "ugrd.kmod.kmod" = [ "process_modules", "process_ignored_modules", "process_module_metadata", "add_kmod_deps" ]