testing-farm: force pre-release copr-rpmbuild onto builder

beaker-tests/Sanity/copr-cli-basic-operations/runtest-coprdir.sh

@@ -49,13 +49,18 @@ rlJournalStart
         # be something like `python-copr` or `python-copr-common`, and the
         # second package would be something like `copr-cli` or `copr-backend`.
 
+        # We need a package with well-structured Source statements so they can
+        # be downloaded easily.  GNU Hello RPM is a great example; it is
+        # intentionally a textbook example of how RPMs should be structured.
+        spec_url=https://src.fedoraproject.org/rpms/hello/raw/f43/f/hello.spec
+
         # This is the dependency (e.g. python-copr)
-        rlRun "curl https://src.fedoraproject.org/rpms/hello/raw/rawhide/f/hello.spec > $tmp/hello-1.spec"
+        rlRun "curl $spec_url > $tmp/hello-1.spec"
         rlRun "sed -i '1s/^/Epoch: 6\n/' $tmp/hello-1.spec"
         rlRun "copr-cli build $PROJECT:custom:foo $tmp/hello-1.spec"
 
         # And this is the package that builds on top of it (e.g. copr-cli)
-        rlRun "curl https://src.fedoraproject.org/rpms/hello/raw/rawhide/f/hello.spec > $tmp/hello-2.spec"
+        rlRun "curl $spec_url > $tmp/hello-2.spec"
         rlRun "sed -i '1s/^/BuildRequires: hello >= 6:\n/' $tmp/hello-2.spec"
         rlRun "copr-cli build $PROJECT:custom:foo $tmp/hello-2.spec"
     rlPhaseEnd

releng/detect-changed-packages

@@ -2,7 +2,7 @@
 
 info() { echo "INFO: $*" >&2 ; }
 
-: "${PYTHON_PKG_SUFFIX=}"
+: "${PYTHON_PKG_SUFFIX=3}"
 
 # By default we check against `origin/main`.  This is the case also in GH Action
 # for pull-requests that submits builds and detects changed packages (we are

testing-farm/README.md

@@ -0,0 +1,40 @@
+Running Sanity Tests Against a PR
+==================================
+
+Simply open a PR and add a comment containing the `/packit test` command.  Our
+CI/CD tooling will automatically build the RPMs, install them into a freshly
+spawned VM in the Testing Farm, and execute the sanity tests.
+
+
+Handling a Test Failure
+-----------------------
+
+Testing Farm does not allow easy SSH access to the machine that failed the
+tests.  However, you can use a Fedora VM and mimic the environment preparation
+using the playbook in this directory, and re-run the failing tests.
+
+Follow these steps:
+
+- **Prepare a Fedora VM** with an accessible IP address (e.g., `1.2.3.4`).  (See
+  the `spawn-testing-machine` subdirectory for instructions on how to spawn one
+  via EC2).
+
+- **Ensure SSH access** to the `root` account: `ssh [email protected]`.
+
+- **Create an inventory file**:
+
+      $ cd ./prepare
+      $ cat inventory
+      [all]
+      1.2.3.4
+
+- **Run the preparation playbook**:
+
+      $ PACKIT_PR_ID=4135 ansible-playbook ./machine-prepare.yml -i inventory
+
+- **Re-run the tests**:
+
+      $ ssh [email protected]
+      # cd /tmp/copr_code/testing-farm
+      # COPR_CLEANUP=false ./all-on-single-host.sh runtest-coprdir.sh  # one test
+      # COPR_CLEANUP=false ./all-on-single-host.sh  # all tests

testing-farm/prepare/roles/pre-release-packages/tasks/main.yaml

@@ -1,4 +1,20 @@
 ---
+- name: GitHub PR Processing
+  when: lookup('env', 'PACKIT_PR_ID') != ""
+  block:
+    - name: Fetch PR data from GitHub
+      ansible.builtin.command: >
+        curl --location --connect-timeout 60 --retry 5 --retry-delay 20
+        --remote-time --show-error --fail
+        https://api.github.com/repos/fedora-copr/copr/pulls/{{ lookup('env', 'PACKIT_PR_ID') }}
+      register: raw_pr_response
+
+    - name: Set pull request related variables
+      ansible.builtin.set_fact:
+        pull_request_id: "{{ lookup('env', 'PACKIT_PR_ID') }}"
+        pull_request_head_sha: "{{ (raw_pr_response.stdout | from_json).head.sha }}"
+        pull_request_commits: "{{ (raw_pr_response.stdout | from_json).commits }}"
+
 - name: pre-release copr packages
   community.general.copr:
     name: '@copr/copr-dev'

testing-farm/prepare/roles/pre-release-packages/templates/install-pre-release-packages.j2

@@ -9,23 +9,46 @@ set -ex
 export {{ var }}={{ lookup('env', var ) }}
 {% endfor %}
 
-# Get us into the git root directory.
-# TODO: git root dir, we need to ask TMT team to export a variable for this
-cd "$TMT_TREE/../discover/default-0/tests"
-cd "$(git rev-parse --show-toplevel)"
-
-curl=(curl --location --connect-timeout 60 --retry 5 --retry-delay 20 --remote-time --show-error --fail)
-pr_data=$("${curl[@]}" "https://api.github.com/repos/fedora-copr/copr/pulls/$PACKIT_PR_ID")
+upstream=https://github.com/fedora-copr/copr
+
+if test -z "$TMT_TREE"; then
+    cd /tmp
+    directory=copr_code
+    test ! -d "$directory" && git clone "$upstream" "$directory"
+    cd "$directory"
+    git checkout main
+    git fetch -f origin pull/$PACKIT_PR_ID/head:pr-$PACKIT_PR_ID
+    git checkout "pr-$PACKIT_PR_ID"
+else
+    # Get us into the git root directory.
+    # TODO: git root dir, we need to ask TMT team to export a variable for this
+    cd "$TMT_TREE/../discover/default-0/tests"
+    cd "$(git rev-parse --show-toplevel)"
+fi
 
 # We are diffing this revision against $commit_sha.  We are not sure where we
 # are, the checkout process is a TF's responsibility.
 git rev-parse HEAD >&2
 
 # We are in a monorepo.  Check package names that need to be installed.
-commits=$(echo "$pr_data" | jq .commits)
-packages=$(COMMIT_SHA="HEAD~$commits" PYTHON_PKG_SUFFIX=3 ./releng/detect-changed-packages)
+packages=$(COMMIT_SHA="HEAD~{{ pull_request_commits }}" ./releng/detect-changed-packages)
+
+# We want to know if a pre-release variant of copr-rpmbuild (from PR) should be
+# installed into the Resalloc's OCI builder image.
+set -- $packages
+
+rpmbuild_check_file=/tmp/install-copr-rpmbuild
+rpmbuild_is_changed=false
+for arg; do
+    test "$arg" = "copr-rpmbuild" && rpmbuild_is_changed=true
+done
+
+if $rpmbuild_is_changed; then
+    touch "$rpmbuild_check_file"
+else
+    rm -f "$rpmbuild_check_file"
+fi
 
 # In Copr, we build packages from the head commits (see
 # .github/workflows/copr-builds.yml).
-commit_sha=$(echo "$pr_data" | jq -r .head.sha)
-./releng/install-copr-packages "@copr/copr-pull-requests:pr:$PACKIT_PR_ID" "$commit_sha" $packages
+./releng/install-copr-packages "@copr/copr-pull-requests:pr:$PACKIT_PR_ID" "{{ pull_request_head_sha }}" "$@"

testing-farm/prepare/roles/resalloc/handlers/main.yaml

@@ -1,7 +1,7 @@
 ---
 - name: build builder image
   ansible.builtin.command:
-    cmd: "podman build -t builder_x86_64 ."
+    cmd: bash ./build-image
   args:
     chdir: "{{ resalloc_image_dir }}"
   become: true

testing-farm/prepare/roles/resalloc/tasks/main.yaml

@@ -63,6 +63,7 @@
   loop:
     - Containerfile
     - main.ini
+    - build-image
 
 - name: Ensure parent directory exists
   ansible.builtin.file:

testing-farm/prepare/roles/resalloc/templates/Containerfile.j2

@@ -1,14 +1,23 @@
-FROM registry.fedoraproject.org/fedora:42
+FROM registry.fedoraproject.org/fedora:43
+
+ARG PULL_REQUEST_ID= \
+    PULL_REQUEST_HEAD_SHA=
 
 RUN set -ex ; \
     dnf -y update && \
     dnf -y copr enable @mock/mock && \
+    dnf -y copr enable @copr/copr-dev && \
     dnf -y install \
         copr-builder \
         openssh-server \
         which \
     && dnf clean all
 
+RUN set -x ; set -e ; test -n "$PULL_REQUEST_ID" || exit 0 ; \
+    curl https://raw.githubusercontent.com/fedora-copr/copr/refs/heads/main/releng/install-copr-packages -o /usr/local/bin/install-copr-packages ; \
+    chmod +x /usr/local/bin/install-copr-packages ; \
+    install-copr-packages "@copr/copr-pull-requests:pr:$PULL_REQUEST_ID" "$PULL_REQUEST_HEAD_SHA" copr-rpmbuild
+
 RUN ssh-keygen -f /etc/ssh/ssh_host_rsa_key -N '' -q
 
 # allow ssh to root@localhost without password

testing-farm/prepare/roles/resalloc/templates/build-image.j2

@@ -0,0 +1,12 @@
+#! /bin/bash -x
+
+args=()
+
+if test -f /tmp/install-copr-rpmbuild; then
+    args=(
+        --build-arg=PULL_REQUEST_ID={{ pull_request_id }}
+        --build-arg=PULL_REQUEST_HEAD_SHA={{ pull_request_head_sha }}
+    )
+fi
+
+exec podman build "${args[@]}" -t builder_x86_64 .

testing-farm/spawn-testing-machine/README.md

@@ -0,0 +1,51 @@
+Starting EC2 machine
+--------------------
+
+1. setup AWS vars like this (change fields appropriately):
+
+       cat > group_vars/all.yaml <<EOF
+       ---
+       aws:
+         profile: fedora-copr
+         # Fedora Cloud 43 -- Paris
+         image: ami-055e6291d0779237f
+         ssh_key: praiskup
+         instance_type: c5a.xlarge
+         security_group: CoprSingleHost
+         root_volume_size: 60
+         infra_subnet: subnet-42f58f2b
+       EOF
+
+2. setup AWS credentials/aws config:
+
+       cat ~/.aws/config
+       [profile fedora-copr]
+       region = us-east-1
+       output = table
+       $ cast ~/.aws/
+
+       cat ~/.aws/credentials
+       [fedora-copr]
+       aws_access_key_id=<the-key-id>
+       aws_secret_access_key=<the-secret-key>
+
+3. start the machine
+
+       ansible-playbook spawn-test-machine-ec2.yaml
+
+       PLAY [Start new machine] *******************
+
+       TASK [create the testing mock vm in ec2] ***
+       changed: [localhost]
+
+       TASK [print ipv4] **************************
+       ok: [localhost] => {
+           "msg": [
+               "Instance ID: i-02f769285490cbb64",
+               "Network ID: eni-0298fa7a391ecc42e",
+               "Unusable Public IP: 107.20.103.13"
+           ]
+       }
+
+       PLAY RECAP ********************************
+       localhost : ok=2 changed=1 unreachable=0 ...

testing-farm/spawn-testing-machine/group_vars/.gitignore

@@ -0,0 +1 @@
+*

testing-farm/spawn-testing-machine/spawn-test-machine-ec2.yaml

@@ -0,0 +1,40 @@
+---
+- name: "Start new machine"
+  hosts: localhost
+  connection: local
+  gather_facts: false
+  tasks:
+    - name: "create the testing mock vm in ec2"
+      amazon.aws.ec2_instance:
+        name: "single-host-copr-instance-testing"
+        state: running
+        profile: "{{ aws.profile }}"
+        region: eu-west-3
+        key_name: "{{ aws.ssh_key }}"
+        count: 1
+        image_id: "{{ aws.image }}"
+        instance_type: "{{ aws.instance_type }}"
+        detailed_monitoring: true
+        network:
+          assign_public_ip: true
+        vpc_subnet_id: "{{ aws.infra_subnet }}"
+        security_group: "{{ aws.security_group }}"
+        termination_protection: false
+        wait: true
+        tags:
+          FedoraGroup: copr
+          CoprPurpose: testing-copr
+        volumes:
+          - ebs:
+              volume_size: "{{ aws.root_volume_size }}"
+              encrypted: true
+              delete_on_termination: true
+            device_name: /dev/sda1
+      register: instances_started
+
+    - name: print ipv4
+      debug:
+        msg:
+          - "Instance ID: {{ instances_started.instances[0].instance_id }}"
+          - "Network ID: {{ instances_started.instances[0].network_interfaces[0].network_interface_id }}"
+          - "Unusable Public IP: {{ instances_started.instances[0].public_ip_address }}"