frontend, backend, rpmbuild: extract Exclu*Arch/BuildArch for all tar…

backend/copr_backend/background_worker_build.py

@@ -44,7 +44,7 @@
 
 MAX_HOST_ATTEMPTS = 3
 MAX_SSH_ATTEMPTS = 5
-MIN_BUILDER_VERSION = "1.3.1"
+MIN_BUILDER_VERSION = "1.6.1"
 CANCEL_CHECK_PERIOD = 5
 DATETIME_FORMAT = "%Y-%m-%d %H:%M"
 

backend/copr_backend/frontend.py

@@ -9,7 +9,7 @@
 from copr_backend.exceptions import FrontendClientException
 
 # The frontend counterpart is in `backend_general:send_frontend_version`
-MIN_FE_BE_API = 7
+MIN_FE_BE_API = 8
 
 class FrontendClient:
     """

beaker-tests/Sanity/copr-cli-basic-operations/runtest-coprdir.sh

@@ -49,13 +49,18 @@ rlJournalStart
         # be something like `python-copr` or `python-copr-common`, and the
         # second package would be something like `copr-cli` or `copr-backend`.
 
+        # We need a package with well-structured Source statements so they can
+        # be downloaded easily.  GNU Hello RPM is a great example; it is
+        # intentionally a textbook example of how RPMs should be structured.
+        spec_url=https://src.fedoraproject.org/rpms/hello/raw/f43/f/hello.spec
+
         # This is the dependency (e.g. python-copr)
-        rlRun "curl https://src.fedoraproject.org/rpms/hello/raw/rawhide/f/hello.spec > $tmp/hello-1.spec"
+        rlRun "curl $spec_url > $tmp/hello-1.spec"
         rlRun "sed -i '1s/^/Epoch: 6\n/' $tmp/hello-1.spec"
         rlRun "copr-cli build $PROJECT:custom:foo $tmp/hello-1.spec"
 
         # And this is the package that builds on top of it (e.g. copr-cli)
-        rlRun "curl https://src.fedoraproject.org/rpms/hello/raw/rawhide/f/hello.spec > $tmp/hello-2.spec"
+        rlRun "curl $spec_url > $tmp/hello-2.spec"
         rlRun "sed -i '1s/^/BuildRequires: hello >= 6:\n/' $tmp/hello-2.spec"
         rlRun "copr-cli build $PROJECT:custom:foo $tmp/hello-2.spec"
     rlPhaseEnd

frontend/coprs_frontend/coprs/logic/builds_logic.py

@@ -1144,16 +1144,16 @@ def finish(chroot, status):
             # Skip excluded architectures
             if upd_dict.get("chroot") == "srpm-builds" and upd_dict.get("results"):
                 for chroot in build.build_chroots:
+                    tags = upd_dict["results"]["architecture_specific_tags"]
+                    tags = tags[chroot.mock_chroot.name_release]
                     arch = chroot.mock_chroot.arch
-
-                    exclusivearch = upd_dict["results"].get("exclusivearch")
+                    exclusivearch = tags.get("exclusivearch")
                     if exclusivearch and arch not in exclusivearch:
                         chroot.status_reason = \
                             "This chroot was skipped because of ExclusiveArch"
                         finish(chroot, StatusEnum("skipped"))
-
-                    excludearch = upd_dict["results"].get("excludearch")
-                    if arch in excludearch:
+                    excludearch = tags.get("excludearch")
+                    if excludearch and arch in excludearch:
                         chroot.status_reason = \
                             "This chroot was skipped because of ExcludeArch"
                         finish(chroot, StatusEnum("skipped"))

frontend/coprs_frontend/coprs/views/backend_ns/backend_general.py

@@ -25,7 +25,7 @@ def send_frontend_version(response):
     setup the version according to our needs.
     For the backend counterpart, see the `MIN_FE_BE_API` constant.
     """
-    response.headers['Copr-FE-BE-API-Version'] = '7'
+    response.headers['Copr-FE-BE-API-Version'] = '8'
     return response
 
 
@@ -241,6 +241,10 @@ def get_srpm_build_record(task, for_backend=False):
         "package_name": task.package.name if task.package else None,
         "appstream": bool(task.copr.appstream),
         "repos": BuildConfigLogic.get_additional_repo_views(repos, chroot),
+        "distributions_in_project": sorted(list({x.name_release for x in
+                                                 task.copr.active_chroots})),
+        "distributions_in_build": sorted(list({x.mock_chroot.name_release for x
+                                               in task.build_chroots})),
     })
 
     return build_record

frontend/coprs_frontend/tests/test_logic/test_builds_logic.py

@@ -680,9 +680,12 @@ def test_skipping_chroots(self, exclusivearch, states):
                 "pkg_version": 1,
                 "chroot": "srpm-builds",
                 "results": {
+                    "architecture_specific_tags": {
+                        "fedora-17": {
+                            "exclusivearch": exclusivearch,
+                        }
+                    },
                     "epoch": None,
-                    "excludearch": [],
-                    "exclusivearch": exclusivearch,
                     "name": "biosdevname",
                     "release": "17",
                     "version": "0.7.3"

releng/detect-changed-packages

@@ -2,7 +2,7 @@
 
 info() { echo "INFO: $*" >&2 ; }
 
-: "${PYTHON_PKG_SUFFIX=}"
+: "${PYTHON_PKG_SUFFIX=3}"
 
 # By default we check against `origin/main`.  This is the case also in GH Action
 # for pull-requests that submits builds and detects changed packages (we are

rpmbuild/copr-rpmbuild.spec

@@ -14,7 +14,7 @@ Requires: %1 \
 %{expand: %%global latest_requires_packages %1 %%{?latest_requires_packages}}
 
 Name:    copr-rpmbuild
-Version: 1.6
+Version: 1.6.1
 Summary: Run COPR build tasks
 Release: 1%{?dist}
 URL: https://github.com/fedora-copr/copr
@@ -35,6 +35,7 @@ BuildRequires: %{python}-daemon
 BuildRequires: %{python}-devel
 BuildRequires: %{python}-distro
 BuildRequires: %{python}-httmock
+BuildRequires: python3-norpm
 BuildRequires: %{rpm_python}
 BuildRequires: asciidoc
 BuildRequires: dist-git-client
@@ -70,6 +71,7 @@ Requires: %{python_pfx}-specfile >= 0.21.0
 Requires: python3-backoff >= 1.9.0
 Requires: python3-daemon
 Requires: python3-pyyaml
+Requires: python3-norpm
 
 Requires: mock >= 5.0
 Requires(pre): mock-filesystem

rpmbuild/copr_rpmbuild/automation/srpm_results.py

@@ -6,6 +6,7 @@
 import os
 
 from copr_rpmbuild.automation.base import AutomationTool
+from copr_rpmbuild.extract_specfile_tags import get_architecture_specific_tags
 from copr_rpmbuild.helpers import (
     get_rpm_header,
     macros_for_task,
@@ -38,25 +39,73 @@ def run(self):
         with open(path, "w", encoding="utf-8") as dst:
             dst.write(data_json)
 
+    @property
+    def target_distros(self):
+        """
+        Get the list of distributions we build this package against.
+        """
+        # Handle distributions_in_build first to optimize a bit; the
+        # distributions_in_build is a subset of distributions_in_project
+        # and if available - we can avoid some macro expansion cycles.
+        for field_name in ["distributions_in_build",
+                           "distributions_in_project"]:
+            if self.task[field_name]:
+                self.log.info("Using %s for this build.", field_name)
+                return self.task[field_name]
+        raise RuntimeError("Running against too old copr-frontend")
+
     def get_package_info(self):
         """
         Return ``dict`` with interesting package metadata
         """
-        keys = ["name", "epoch", "version", "release",
-                "exclusivearch", "excludearch"]
+        output_tags = {}
+
+        # While this is highly inconvenient, many packages still use %lua to
+        # define these fields, and Copr must be able to build them.  Unlike
+        # other "rpm parsing" use-cases, this does not pose a security risk; we
+        # run this script on an disposable worker, so there are no serious
+        # consequences if a user "bricks" the machine.
+        #
+        # Although these fields may expand into target-specific values in
+        # theory, we need a single NEVRA for single build.  Consequently,
+        # we do not perform separate expansions for each individual target
+        # distribution.  Note these fields are not critical for the overall
+        # build process, we store some metadata about the build sing these.
+        #
+        # To fully resolve the issue #1315, we have to fix the
+        # backend → distgit protocol, and upload the right srpm.  Or even
+        # better, upload all possible src.rpm variants.
+        rpm_tags = ["name", "epoch", "version", "release"]
+
+        # These are a bit more important, since these are used to decide which
+        # BuildChroots are going to be skipped or not.  And we need to extract
+        # them for each target distribution version separately.
+        norpm_tags = ["exclusivearch", "excludearch", "buildarch"]
+
+        specfile_path = locate_spec(self.resultdir)
+
+        # TODO: host override_database= on backend and configure
+        output_tags["architecture_specific_tags"] = get_architecture_specific_tags(
+            specfile_path,
+            norpm_tags,
+            self.target_distros,
+            log=self.log,
+        )
+
         try:
             macros = macros_for_task(self.task, self.config)
-            path = locate_spec(self.resultdir)
-            spec = Spec(path, macros)
-            return {key: getattr(spec, key) for key in keys}
+            spec = Spec(specfile_path, macros)
+            output_tags.update({key: getattr(spec, key) for key in rpm_tags})
 
         except Exception:  # pylint: disable=broad-exception-caught
             # Specfile library raises too many exception to name the
             # in except block
-            msg = "Exception appeared during handling spec file: {0}".format(path)
+            msg = "Exception appeared during handling spec file: {0}".format(specfile_path)
             self.log.exception(msg)
 
             path = locate_srpm(self.resultdir)
             self.log.warning("Querying NEVRA from SRPM header: %s", path)
             hdr = get_rpm_header(path)
-            return {key: hdr[key] for key in keys}
+            output_tags.update({key: hdr[key] for key in rpm_tags})
+
+        return output_tags

rpmbuild/copr_rpmbuild/extract_specfile_tags.py

@@ -0,0 +1,150 @@
+"""
+Extract macro-expanded tag value (e.g., BuildArch) from given specfile.
+"""
+
+import logging
+import os
+import tempfile
+
+from copr_common.request import SafeRequest
+
+from norpm.macrofile import system_macro_registry
+from norpm.specfile import specfile_expand, ParserHooks
+from norpm.overrides import override_macro_registry
+from norpm.exceptions import NorpmError
+
+DEFAULT_OVERRIDE_URL = "https://raw.githubusercontent.com/praiskup/norpm-macro-overrides/refs/heads/main/distro-arch-specific.json"
+
+DEFAULT_TAG_MAP = {
+    # TODO: extract rhel+epel into the datafile
+    "epel-7": "rhel-7",
+    "epel-8": "rhel-8",
+    "epel-9": "rhel-9",
+    "epel-10": "centos-stream+epel-10",
+    # what do we do about custom-* chroots?
+    "custom-0": "fedora-rawhide",
+    "custom-1": "fedora-rawhide",
+}
+
+class _TagHooks(ParserHooks):
+    """ Gather access to spec tags """
+    def __init__(self, expanded_tags):
+        self.expanded_tags = set(expanded_tags)
+        self.tags = {}
+
+    def tag_found(self, name, value, _tag_raw):
+        """
+        Parser hook that gathers the tags' values, if defined.
+        """
+        if name not in self.expanded_tags:
+            return
+        if name not in self.tags:
+            self.tags[name] = []
+        # tags may be specified multiple times within a single spec file
+        self.tags[name] += [value]
+
+
+def collapse_tag_values_cb(array_of_tag_values):
+    """
+    Process tags that represent sets of strings (e.g., ExcludeArch).
+
+    If a tag is specified multiple times within the specfile, this function
+    performs a union of all encountered values.  The resulting set contains
+    every unique string defined across all instances of the tag.
+
+    Returns:
+        set: A set of all unique strings extracted from the tag definitions.
+    """
+    concat = " ".join(array_of_tag_values)
+    return list(set(concat.split()))
+
+
+def extract_tags_from_specfile(specfile, extract_tags, override_database=None,
+                               target=None, tag_cb=None):
+    """
+    Parse the given SPECFILE against system macros and optionally TARGET macros.
+
+    If TARGET is specified, OVERRIDE_DATABASE (a file path) must also be
+    provided.  If TARGET is omitted, the local system macros are used by
+    default.
+
+    Args:
+        specfile (str): Path to the specfile to be parsed.
+        target (str, optional): Target distribution (e.g., "rhel-7").
+        override_database (str, optional): Database file path required if target
+            is set.
+        tag_cb (callable, optional): A callback function used to transform the
+            list of values for each tag.  If provided, each item in the return
+            dictionary is passed through this function. A common choice is
+            `collapse_tag_values_cb`, which collapses the list into a set of
+            unique strings.
+
+    Returns:
+        dict: A mapping of lowercase tagnames to their processed values.
+            Default format: {'excludearch': ['ppc64 ppc64le', 's390x i386']}
+            With tag_cb: The value type is determined by the callback's return.
+            With tag_cb=collapse_tag_values_cb:
+                {'excludearch': {'ppc64', 'ppc64le', 's390x', 'i386'}}
+    Note:
+        Since RPM tags are case-insensitive, all tagnames are normalized
+        using .lower().
+    """
+    registry = system_macro_registry()
+    if override_database:
+        registry = override_macro_registry(registry, override_database, target)
+
+    # %dist definition contains %lua mess, it's safer to clear it (since we
+    # don't necessarily need it)
+    registry["dist"] = ""
+
+    # norpm maintains a few tricks to ease the spec file parsing
+    registry.known_norpm_hacks()
+
+    tags = _TagHooks(extract_tags)
+    try:
+        with open(specfile, "r", encoding="utf8") as fd:
+            specfile_expand(fd.read(), registry, tags)
+    except NorpmError as err:
+        print("WARNING: Building for all architectures since "
+              f"the spec file parser: failed: {err}")
+
+    if not tag_cb:
+        return tags.tags
+
+    processed = {}
+    for tag, values in tags.tags.items():
+        processed[tag] = tag_cb(values)
+
+    return processed
+
+
+def get_architecture_specific_tags(specfile, extract_tags, targets,
+                                   override_database=DEFAULT_OVERRIDE_URL,
+                                   log=logging):
+    """
+    A high-level tool for Copr, working with temporary files, etc.
+    """
+    architecture_tags = {}
+    request = SafeRequest(log=log)
+    response = request.get(override_database)
+    with tempfile.NamedTemporaryFile(mode='wb', delete=False) as temp:
+        temp_path = temp.name
+        temp.write(response.content)
+
+    try:
+        for distro in targets:
+            ask = DEFAULT_TAG_MAP.get(distro, distro)
+            log.info("Extracting arch-specific tags for %s", distro)
+            architecture_tags[distro] = extract_tags_from_specfile(
+                specfile,
+                extract_tags,
+                override_database=temp_path,
+                target=ask,
+                tag_cb=collapse_tag_values_cb
+            )
+    finally:
+        try:
+            os.unlink(temp_path)
+        except OSError:
+            pass
+    return architecture_tags

testing-farm/prepare/roles/pre-release-packages/tasks/main.yaml

@@ -1,4 +1,20 @@
 ---
+- name: GitHub PR Processing
+  when: lookup('env', 'PACKIT_PR_ID') != ""
+  block:
+    - name: Fetch PR data from GitHub
+      ansible.builtin.command: >
+        curl --location --connect-timeout 60 --retry 5 --retry-delay 20
+        --remote-time --show-error --fail
+        https://api.github.com/repos/fedora-copr/copr/pulls/{{ lookup('env', 'PACKIT_PR_ID') }}
+      register: raw_pr_response
+
+    - name: Set pull request related variables
+      ansible.builtin.set_fact:
+        pull_request_id: "{{ lookup('env', 'PACKIT_PR_ID') }}"
+        pull_request_head_sha: "{{ (raw_pr_response.stdout | from_json).head.sha }}"
+        pull_request_commits: "{{ (raw_pr_response.stdout | from_json).commits }}"
+
 - name: pre-release copr packages
   community.general.copr:
     name: '@copr/copr-dev'