Use product-specific repositories for SLFO/Gitea incidents

This is not configurable via metadata because metadata only seems to come
into play when scheduling products. However, here we need to be aware of
whether/which products to schedule already from the beginning (when syncing
incidents from Gitea).

Hence I made this configurable via two env variables for now:

* `OBS_REPO_TYPE`: specifies the repo typo to look at, e.g. `standard` or
  `product` or an empty string to consider all repo types
* `OBS_PRODUCTS`: specifies a comma-separated list of products to consider
  and if an empty value appears on the list the code stream is considered
  as well

So with `OBS_REPO_TYPE=standard` and `OBS_PRODUCTS=` we get the current
behavior of using the code stream repository. With `OBS_REPO_TYPE=product`
and `OBS_PRODUCTS=SLES` we would use the SLES-specific repository.

Note that the product version (e.g. `15.99` as in PR 166) can be read from
the build results of OBS (from the `scmsync` tag) which we read anyway. So
there's no need to supply a mapping from e.g. `1.1.99` to `15.99`.

To determine the available architectures of product repositories the bot
now reads the `_multibuild` info from OBS. This seems to be required
because the build info returns results like this:

```
<result project="SUSE:SLFO:1.1.99:PullRequest:166:SLES" repository="product" arch="ppc64le" code="published" state="published">
```

However, there is no actual product repository for this repository present.
The absence of `ppc64le` and `local` in the `_multibuild` XML explains why
these product repositories are missing and therefore we need to check this
file for the actually relevant archs. Otherwise the bot would run later
into errors, e.g. when computing the repo hash of these non-existent repos.

Related ticket: https://progress.opensuse.org/issues/180812

Author: Martchus

openqabot/__init__.py

@@ -12,6 +12,8 @@
 OBS_DOWNLOAD_URL = os.environ.get("OBS_DOWNLOAD_URL", "http://download.suse.de/ibs")
 OBS_MAINT_PRJ = "SUSE:Maintenance"
 OBS_GROUP = "qam-openqa"
+OBS_REPO_TYPE = os.environ.get("OBS_REPO_TYPE", "product")
+OBS_PRODUCTS = set(os.environ.get("OBS_PRODUCTS", "SLES").split(","))
 ALLOW_DEVELOPMENT_GROUPS = os.environ.get("QEM_BOT_ALLOW_DEVELOPMENT_GROUPS")
 DEVELOPMENT_PARENT_GROUP_ID = 9
 DOWNLOAD_BASE = os.environ.get(

openqabot/loader/gitea.py

@@ -2,22 +2,21 @@
 # SPDX-License-Identifier: MIT
 import concurrent.futures as CT
 from logging import getLogger
-from typing import Any, List, Set, Dict
+from typing import Any, List, Set, Dict, Tuple
 import re
 import xml.etree.ElementTree as ET
 
 import json
 import urllib3
 import urllib3.exceptions
 
+from osc.core import MultibuildFlavorResolver
 import osc.conf
 import osc.core
 import osc.util.xml
 
 from ..utils import retry10 as requests
-from .. import GITEA, OBS_GROUP, OBS_URL
-
-PROJECT_REGEX = ".*:PullRequest:\\d+:(.*)"
+from .. import GITEA, OBS_GROUP, OBS_URL, OBS_REPO_TYPE, OBS_PRODUCTS
 
 log = getLogger("bot.loader.gitea")
 
@@ -51,6 +50,11 @@ def post_json(
         raise e
 
 
+def read_utf8(name: str) -> Any:
+    with open("responses/%s" % name, "r", encoding="utf8") as utf8:
+        return utf8.read()
+
+
 def read_json(name: str) -> Any:
     with open("responses/%s.json" % name, "r", encoding="utf8") as json_file:
         return json.loads(json_file.read())
@@ -75,6 +79,39 @@ def comments_url(repo_name: str, number: int):
     return "repos/%s/issues/%s/comments" % (repo_name, number)
 
 
+def get_product_name(obs_project: str):
+    product_match = re.search(".*:PullRequest:\\d+:(.*)", obs_project)
+    return product_match.group(1) if product_match else ""
+
+
+def get_product_name_and_version_from_scmsync(scmsync_url: str):
+    m = re.search(".*/products/(.*)#(.*)", scmsync_url)
+    return (m.group(1), m.group(2)) if m else ("", "")
+
+
+def compute_repo_url(
+    base: str,
+    product_name: str,
+    repo: List[Tuple[str, str, str]],
+    arch: str,
+    path: str = "repodata/repomd.xml",
+):
+    # use codestream repo if product name is empty
+    if product_name == "":
+        # assing something like `http://download.suse.de/ibs/SUSE:/SLFO:/1.1.99:/PullRequest:/166/standard/repodata/repomd.xml`
+        return f"{base}/{repo[0].replace(':', ':/')}:/{repo[1].replace(':', ':/')}/{OBS_REPO_TYPE}/{path}"
+    # use product repo
+    # assing something like `https://download.suse.de/ibs/SUSE:/SLFO:/1.1.99:/PullRequest:/166:/SLES/product/repo/SLES-15.99-x86_64/repodata/repomd.xml`
+    return f"{base}/{repo[0].replace(':', ':/')}:/{repo[1].replace(':', ':/')}/{OBS_REPO_TYPE}/repo/{product_name}-{repo[2]}-{arch}/{path}"
+
+
+def compute_repo_url_for_job_setting(
+    base: str, repo: List[Tuple[str, str, str]], arch: str
+):
+    product_name = get_product_name(repo[1])
+    return compute_repo_url(base, product_name, repo, arch, "")
+
+
 def get_open_prs(token: Dict[str, str], repo: str, dry: bool) -> List[Any]:
     if dry:
         return read_json("pulls")
@@ -152,14 +189,18 @@ def add_reviews(incident: Dict[str, Any], reviews: List[Any]) -> int:
 def add_build_result(
     incident: Dict[str, Any],
     res: Any,
+    projects: Set[str],
     successful_packages: Set[str],
     unpublished_repos: Set[str],
     failed_packages: Set[str],
 ):
     state = res.get("state")
     project = res.get("project")
-    project_match = re.search(PROJECT_REGEX, project)
-    scm_info_key = "scminfo_" + project_match.group(1) if project_match else "scminfo"
+    product_name = get_product_name(project)
+    arch = res.get("arch")
+    channel = ":".join([project, arch])
+    # read Git hash from scminfo element
+    scm_info_key = "scminfo_" + product_name if len(product_name) != 0 else "scminfo"
     for scminfo_element in res.findall("scminfo"):
         found_scminfo = scminfo_element.text
         existing_scminfo = incident.get(scm_info_key, None)
@@ -174,11 +215,20 @@ def add_build_result(
                     found_scminfo,
                     existing_scminfo,
                 )
-    # require codestream builds to be successful and published …
-    if project_match:
-        return  # … but skip those checks for project-specific builds/repos as we do not use them anyway
+    # read product version from scmsync element, e.g. 15.99
+    for scmsync_element in res.findall("scmsync"):
+        (_, product_version) = get_product_name_and_version_from_scmsync(
+            scmsync_element.text
+        )
+        if len(product_version) > 0:
+            channel = "#".join([channel, product_version])
+            break
+    projects.add(channel)
+    # require only relevant projects to be built/published
+    if product_name not in OBS_PRODUCTS:
+        return
     if state != "published":
-        unpublished_repos.add("@".join([project, res.get("arch")]))
+        unpublished_repos.add(channel)
         return
     for status in res.findall("status"):
         code = status.get("code")
@@ -190,6 +240,43 @@ def add_build_result(
             failed_packages.add(status.get("package"))
 
 
+def get_multibuild_data(obs_project: str):
+    r = MultibuildFlavorResolver(OBS_URL, obs_project, "000productcompose")
+    return r.get_multibuild_data()
+
+
+def determine_relevant_archs_from_multibuild_info(obs_project: str, dry: bool):
+    # retrieve the _multibuild info like `osc cat SUSE:SLFO:1.1.99:PullRequest:124:SLES 000productcompose _multibuild`
+    product_name = get_product_name(obs_project)
+    if product_name == "":
+        return None
+    product_prefix = product_name.replace(":", "_").lower() + "_"
+    prefix_len = len(product_prefix)
+    if dry:
+        multibuild_data = read_utf8("_multibuild-124-" + obs_project + ".xml")
+    else:
+        try:
+            multibuild_data = get_multibuild_data(obs_project)
+        except Exception as e:
+            log.warning("Unable to determine relevant archs for %s: %s", obs_project, e)
+            return None
+
+    # determine from the flavors we got what architectures are actually expected to be present
+    # note: The build info will contain result elements for archs like `local` and `ppc64le` that and the published
+    #       flag set even though no repos for those products are actually present. Considering these would lead to
+    #       problems later on (e.g. when computing the repohash) so it makes sense to reduce the archs we are considering
+    #       to actually relevant ones.
+    flavors = MultibuildFlavorResolver.parse_multibuild_data(multibuild_data)
+    relevant_archs = set()
+    for flavor in flavors:
+        if flavor.startswith(product_prefix):
+            arch = flavor[prefix_len:]
+            if arch in ("x86_64", "aarch64", "ppc64le", "s390x"):
+                relevant_archs.add(arch)
+    log.debug("Relevant archs for %s: %s", obs_project, str(sorted(relevant_archs)))
+    return relevant_archs
+
+
 def add_build_results(incident: Dict[str, Any], obs_urls: List[str], dry: bool):
     successful_packages = set()
     unpublished_repos = set()
@@ -198,18 +285,26 @@ def add_build_results(incident: Dict[str, Any], obs_urls: List[str], dry: bool):
     for url in obs_urls:
         project_match = re.search(".*/project/show/(.*)", url)
         if project_match:
+            obs_project = project_match.group(1)
+            relevant_archs = determine_relevant_archs_from_multibuild_info(
+                obs_project, dry
+            )
             build_info_url = osc.core.makeurl(
-                OBS_URL, ["build", project_match.group(1), "_result"]
+                OBS_URL, ["build", obs_project, "_result"]
             )
             if dry:
-                build_info = read_xml("build-results-124-" + project_match.group(1))
+                build_info = read_xml("build-results-124-" + obs_project)
             else:
                 build_info = osc.util.xml.xml_parse(osc.core.http_GET(build_info_url))
             for res in build_info.getroot().findall("result"):
-                projects.add(":".join([res.get("project"), res.get("arch")]))
+                if OBS_REPO_TYPE != "" and res.get("repository") != OBS_REPO_TYPE:
+                    continue
+                if relevant_archs is not None and res.get("arch") not in relevant_archs:
+                    continue
                 add_build_result(
                     incident,
                     res,
+                    projects,
                     successful_packages,
                     unpublished_repos,
                     failed_packages,
@@ -229,6 +324,8 @@ def add_build_results(incident: Dict[str, Any], obs_urls: List[str], dry: bool):
     incident["channels"] = [*projects]
     incident["failed_or_unpublished_packages"] = [*failed_packages, *unpublished_repos]
     incident["successful_packages"] = [*successful_packages]
+    if "scminfo" not in incident and len(OBS_PRODUCTS) == 1:
+        incident["scminfo"] = incident.get("scminfo_" + next(iter(OBS_PRODUCTS)), "")
 
 
 def add_comments_and_referenced_build_results(

openqabot/loader/repohash.py

@@ -4,15 +4,14 @@
 from logging import getLogger
 from typing import List, Tuple
 from xml.etree import ElementTree as ET
-import re
 
 from requests import ConnectionError, HTTPError
 from requests.exceptions import RetryError
 
-from .. import OBS_DOWNLOAD_URL
-from .gitea import PROJECT_REGEX
+from .. import OBS_DOWNLOAD_URL, OBS_PRODUCTS
 from ..errors import NoRepoFoundError
 from ..utils import retry5 as requests
+from . import gitea
 
 log = getLogger("bot.loader.repohash")
 
@@ -23,17 +22,21 @@ def get_max_revision(
     project: str,
 ) -> int:
     max_rev = 0
-
     url_base = f"{OBS_DOWNLOAD_URL}/{project.replace(':', ':/')}"
 
     for repo in repos:
         # handle URLs for SLFO specifically
         if project == "SLFO":
-            # assing something like `http://download.suse.de/ibs/SUSE:/SLFO:/1.1.99:/PullRequest:/166/standard/repodata/repomd.xml`
-            url = f"{OBS_DOWNLOAD_URL}/{repo[0].replace(':', ':/')}:/{repo[1].replace(':', ':/')}/standard/repodata/repomd.xml"
-            if re.search(PROJECT_REGEX, repo[1]):
-                log.info("skipping repohash of product-specifc repo '%s'" % url)
-                continue  # skip product repositories here (only consider code stream repositories)
+            product_name = gitea.get_product_name(repo[1])
+            if product_name not in OBS_PRODUCTS:
+                log.info(
+                    "skipping repo '%s' as product '%s' is not considered",
+                    repo[1],
+                    product_name,
+                )
+                continue
+            url = gitea.compute_repo_url(OBS_DOWNLOAD_URL, product_name, repo, arch)
+            log.debug("computing repohash for '%s' via: %s", repo[1], url)
         # openSUSE and SLE incidents have different handling of architecture
         elif repo[0].startswith("openSUSE"):
             url = f"{url_base}/SUSE_Updates_{repo[0]}_{repo[1]}/repodata/repomd.xml"
@@ -58,7 +61,6 @@ def get_max_revision(
         if cs is None:
             log.error("%s's revision is None", url)
             raise NoRepoFoundError
-
         max_rev = max(max_rev, int(str(cs.text)))
 
     return max_rev

openqabot/types/__init__.py

@@ -7,6 +7,7 @@ class Repos(NamedTuple):
     product: str
     version: str
     arch: str
+    product_version: str = ""  # if non-empty, "version" is the codestream version
 
 
 class ProdVer(NamedTuple):

openqabot/types/incident.py

@@ -58,7 +58,7 @@ def __init__(self, incident):
         ]
         # add channels for Gitea-based incidents
         self.channels += [
-            Repos(":".join(val[0:2]), ":".join(val[2:-1]), val[-1])
+            Repos(":".join(val[0:2]), ":".join(val[2:-1]), *(val[-1].split("#")))
             for val in (
                 r.split(":")
                 for r in (i for i in incident["channels"] if i.startswith("SUSE:SLFO"))
@@ -110,12 +110,11 @@ def _rev(channels: List[Repos], project: str) -> Dict[ArchVer, int]:
             if v:
                 version = v.group(0)
 
+            repo_info = (repo.product, repo.version, repo.product_version)
             if ArchVer(repo.arch, version) in tmpdict:
-                tmpdict[ArchVer(repo.arch, version)].append(
-                    (repo.product, repo.version)
-                )
+                tmpdict[ArchVer(repo.arch, version)].append(repo_info)
             else:
-                tmpdict[ArchVer(repo.arch, version)] = [(repo.product, repo.version)]
+                tmpdict[ArchVer(repo.arch, version)] = [repo_info]
 
         if tmpdict:
             for archver, lrepos in tmpdict.items():

openqabot/types/incidents.py

@@ -81,9 +81,9 @@ def _is_scheduled_job(
 
         return False
 
-    def _make_repo_url(self, inc: Incident, chan: Repos):
+    def _make_repo_url(self, inc: Incident, chan: Repos, arch: str):
         return (
-            f"{DOWNLOAD_SLFO}{chan.version}:/PullRequest:/{inc.id}/standard/"
+            gitea.compute_repo_url_for_job_setting(DOWNLOAD_BASE, chan, arch)
             if chan.product == "SLFO"
             else f"{DOWNLOAD_MAINTENANCE}{inc.id}/SUSE_Updates_{'_'.join(self._repo_osuse(chan))}"
         )
@@ -157,6 +157,7 @@ def _handle_incident(  # pylint: disable=too-many-return-statements
         issue_dict = {}
 
         log.debug("Incident channels: %s", inc.channels)
+        # FIXME: populate channels_set from inc.channels for SLFO to get the product version
         for issue, channel in data["issues"].items():
             log.debug(
                 "Meta-data channel: %s, %s, %s", channel.product, channel.version, arch
@@ -217,7 +218,7 @@ def _handle_incident(  # pylint: disable=too-many-return-statements
             full_post["openqa"][key] = str(value.id)
 
         full_post["openqa"]["INCIDENT_REPO"] = ",".join(
-            sorted(self._make_repo_url(inc, chan) for chan in channels_set)
+            sorted(self._make_repo_url(inc, chan, arch) for chan in channels_set)
         )  # sorted for testability
 
         full_post["qem"]["withAggregate"] = True

responses/_multibuild-124-SUSE:SLFO:1.1.99:PullRequest:124:SLES.xml

@@ -0,0 +1,15 @@
+<multibuild>
+ <flavor>sles_aarch64</flavor>
+ <flavor>sles_x86_64</flavor>
+
+ <flavor>sles_offline_aarch64</flavor>
+ <flavor>sles_offline_x86_64</flavor>
+
+ <flavor>sles_sap_x86_64</flavor>
+
+ <flavor>sles_ha_aarch64</flavor>
+ <flavor>sles_ha_x86_64</flavor>
+
+ <flavor>sles_online_aarch64</flavor>
+ <flavor>sles_online_x86_64</flavor>
+</multibuild>

responses/build-results-124-SUSE:SLFO:1.1.99:PullRequest:124:SLES.xml

@@ -21,28 +21,28 @@
     <status package="busybox-image" code="excluded" />
     <status package="gcc-15-image" code="succeeded" />
   </result>
-  <result project="SUSE:SLFO:1.1.99:PullRequest:124:SLES" repository="product" arch="aarch64" code="published" state="published">
+  <result project="SUSE:SLFO:1.1.99:PullRequest:124:SLES" repository="product" arch="aarch64" code="published" state="unpublished">
     <scmsync>https://src.suse.de/products/SLES#15.99</scmsync>
     <scminfo>18bfa2a23fb7985d5d0cc356474a96a19d91d2d8652442badf7f13bc07cd1f3d</scminfo>
     <status package="base-image" code="excluded" />
     <status package="busybox-image" code="excluded" />
     <status package="gcc-15-image" code="succeeded" />
   </result>
-  <result project="SUSE:SLFO:1.1.99:PullRequest:124:SLES" repository="product" arch="local" code="published" state="published">
+  <result project="SUSE:SLFO:1.1.99:PullRequest:124:SLES" repository="product" arch="local" code="published" state="unpublished">
     <scmsync>https://src.suse.de/products/SLES#15.99</scmsync>
     <scminfo>18bfa2a23fb7985d5d0cc356474a96a19d91d2d8652442badf7f13bc07cd1f3d</scminfo>
     <status package="base-image" code="excluded" />
     <status package="busybox-image" code="excluded" />
     <status package="gcc-15-image" code="succeeded" />
   </result>
-  <result project="SUSE:SLFO:1.1.99:PullRequest:124:SLES" repository="product" arch="ppc64le" code="published" state="published">
+  <result project="SUSE:SLFO:1.1.99:PullRequest:124:SLES" repository="product" arch="ppc64le" code="published" state="unpublished">
     <scmsync>https://src.suse.de/products/SLES#15.99</scmsync>
     <scminfo>18bfa2a23fb7985d5d0cc356474a96a19d91d2d8652442badf7f13bc07cd1f3d</scminfo>
     <status package="base-image" code="excluded" />
     <status package="busybox-image" code="excluded" />
     <status package="gcc-15-image" code="succeeded" />
   </result>
-  <result project="SUSE:SLFO:1.1.99:PullRequest:124:SLES" repository="product" arch="x86_64" code="published" state="published">
+  <result project="SUSE:SLFO:1.1.99:PullRequest:124:SLES" repository="product" arch="x86_64" code="published" state="unpublished">
     <scmsync>https://src.suse.de/products/SLES#15.99</scmsync>
     <scminfo>18bfa2a23fb7985d5d0cc356474a96a19d91d2d8652442badf7f13bc07cd1f3d</scminfo>
     <status package="base-image" code="excluded" />

tests/test_aggregate.py

@@ -89,6 +89,7 @@ class Repos(NamedTuple):
         product: str
         version: str
         arch: str
+        product_version: str = ""
 
     class MockIncident:
         def __init__(self, repo, embargoed):