Merge branch 'master' of github.com:fmidev/himan #130
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| on: # yamllint disable-line rule:truthy | |
| push: | |
| branches: | |
| - master | |
| env: | |
| PYTHONUNBUFFERED: "1" | |
| name: Create RPM Release | |
| jobs: | |
| check-if-himan-bin-changed: | |
| name: check-if-himan-bin-changed | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: dorny/paths-filter@v3 | |
| id: himan-bin-changed | |
| with: | |
| filters: | | |
| src: | |
| - 'himan-bin/**' | |
| outputs: | |
| changed: ${{ steps.himan-bin-changed.outputs.src }} | |
| check-if-himan-lib-changed: | |
| name: check-if-himan-lib-changed | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: dorny/paths-filter@v3 | |
| id: himan-lib-changed | |
| with: | |
| filters: | | |
| src: | |
| - 'himan-lib/**' | |
| outputs: | |
| changed: ${{ steps.himan-lib-changed.outputs.src }} | |
| check-if-himan-plugins-changed: | |
| name: check-if-himan-plugins-changed | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: dorny/paths-filter@v3 | |
| id: himan-plugins-changed | |
| with: | |
| filters: | | |
| src: | |
| - 'himan-plugins/**' | |
| outputs: | |
| changed: ${{ steps.himan-plugins-changed.outputs.src }} | |
| check-if-himan-scripts-changed: | |
| name: check-if-himan-scripts-changed | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: dorny/paths-filter@v3 | |
| id: himan-scripts-changed | |
| with: | |
| filters: | | |
| src: | |
| - 'himan-scripts/**' | |
| outputs: | |
| changed: ${{ steps.himan-scripts-changed.outputs.src }} | |
| build-rpms: | |
| name: build-rpms | |
| runs-on: ubuntu-latest | |
| container: | |
| image: rockylinux/rockylinux:8 | |
| volumes: | |
| - ${{ github.workspace }}:/github/workspace | |
| needs: | |
| - check-if-himan-bin-changed | |
| - check-if-himan-lib-changed | |
| - check-if-himan-scripts-changed | |
| - check-if-himan-plugins-changed | |
| outputs: | |
| trigger: ${{ steps.set-trigger.outputs.trigger }} | |
| steps: | |
| - name: Install build tools | |
| shell: bash | |
| run: | | |
| dnf -y install rpm-build rpmdevtools git yum-utils dnf-plugins-core findutils | |
| git config --global --add safe.directory '*' | |
| - name: Checkout code | |
| uses: actions/checkout@v4 | |
| - name: List workspace | |
| run: ls -lah | |
| - name: Setup repositories | |
| shell: bash | |
| run: | | |
| dnf -y install epel-release https://download.fmi.fi/smartmet-open/rhel/8/x86_64/smartmet-open-release-latest-8.noarch.rpm | |
| dnf config-manager --set-enabled powertools | |
| dnf -y module disable postgresql | |
| dnf config-manager --add-repo https://developer.download.nvidia.com/compute/cuda/repos/rhel8/x86_64/cuda-rhel8.repo | |
| dnf config-manager --setopt='epel.exclude=eccodes*' --save | |
| - name: Setup rpmtree | |
| shell: bash | |
| run: | | |
| rpmdev-setuptree | |
| - name: Build himan-lib rpm | |
| id: build-himan-lib | |
| if: needs.check-if-himan-lib-changed.outputs.changed == 'true' | |
| shell: bash | |
| run: | | |
| cd himan-lib | |
| yum-builddep -y himan-lib.spec --nobest | |
| make rpm | |
| - name: Build himan-bin rpm | |
| id: build-himan-bin | |
| if: needs.check-if-himan-bin-changed.outputs.changed == 'true' | |
| shell: bash | |
| run: | | |
| cd himan-bin | |
| yum-builddep -y himan-bin.spec | |
| make rpm | |
| - name: Build himan-plugins rpm | |
| id: build-himan-plugins | |
| if: needs.check-if-himan-plugins-changed.outputs.changed == 'true' | |
| shell: bash | |
| run: | | |
| cd himan-plugins | |
| yum-builddep -y himan-plugins.spec --nobest | |
| make rpm | |
| - name: Build himan-scripts rpm | |
| id: build-himan-scripts | |
| if: needs.check-if-himan-scripts-changed.outputs.changed == 'true' | |
| shell: bash | |
| run: | | |
| cd himan-scripts | |
| yum-builddep -y himan-scripts.spec | |
| make rpm | |
| - name: Set output | |
| id: set-output | |
| if: | | |
| always() && (steps.build-himan-bin.conclusion == 'success' || steps.build-himan-lib.conclusion == 'success' || steps.build-himan-plugins.conclusion == 'success' || steps.build-himan-scripts.conclusion == 'success') | |
| shell: bash | |
| run: | | |
| mkdir -p /github/workspace/rpmbuild/SRPMS /github/workspace/rpmbuild/RPMS | |
| find /github/home/rpmbuild/SRPMS/ -type f -name "*.src.rpm" -exec cp {} /github/workspace/rpmbuild/SRPMS \; | |
| find /github/home/rpmbuild/RPMS/ -type f -name "*.rpm" -not -name "*debug*" -exec cp {} /github/workspace/rpmbuild/RPMS \; | |
| ls -la /github/workspace/rpmbuild/SRPMS /github/workspace/rpmbuild/RPMS | |
| echo "srpm_dir_path=rpmbuild/SRPMS/" >> "$GITHUB_OUTPUT" | |
| echo "rpm_dir_path=rpmbuild/RPMS/" >> "$GITHUB_OUTPUT" | |
| - name: Save RPM as an artifact | |
| id: save-rpm | |
| if: always() && steps.set-output.conclusion == 'success' | |
| uses: actions/[email protected] | |
| with: | |
| name: rpm-files | |
| path: ${{ steps.set-output.outputs.rpm_dir_path }} | |
| retention-days: 1 | |
| - name: Set trigger | |
| id: set-trigger | |
| if: always() && steps.save-rpm.conclusion == 'success' | |
| shell: bash | |
| run: | | |
| echo "trigger=yes" >> "$GITHUB_OUTPUT" | |
| sync-to-beta-repo: | |
| name: sync-to-beta-repo | |
| needs: build-rpms | |
| runs-on: ubuntu-latest | |
| if: always() && needs.build-rpms.outputs.trigger == 'yes' | |
| steps: | |
| - name: Retrieve saved rpms | |
| uses: actions/[email protected] | |
| with: | |
| name: rpm-files | |
| - name: List rpms | |
| run: find ${{ github.workspace }} | |
| shell: bash | |
| - name: Copy to repo | |
| shell: bash | |
| env: | |
| USER: ${{ secrets.RPM_API_USER }} | |
| PASS: ${{ secrets.RPM_API_PASS }} | |
| HOST: ${{ secrets.RPM_API_HOST_BETA_RHEL8 }} | |
| run: | | |
| find ${{ github.workspace }} -type f -name "*.rpm" -not -name "*debug*" -exec curl --show-error --fail -u $USER:$PASS -F file=@{} $HOST/api/upload \; | |
| run-tests: | |
| name: run-tests | |
| runs-on: ubuntu-latest | |
| needs: sync-to-beta-repo | |
| steps: | |
| - name: Install launch dependencies | |
| shell: bash | |
| run: | | |
| sudo apt -y update && sudo apt -y install python3 python3-boto3 python3-requests | |
| - name: Start tests | |
| id: start-tests | |
| shell: python | |
| run: | | |
| import boto3 | |
| import datetime | |
| import os | |
| batch = boto3.client('batch', | |
| aws_access_key_id="${{ secrets.AWS_ACCESS_KEY_ID }}", | |
| aws_secret_access_key="${{ secrets.AWS_SECRET_ACCESS_KEY }}", | |
| region_name="eu-west-1") | |
| response = batch.submit_job( | |
| jobName='himantests-testgh-{}'.format(datetime.datetime.now().strftime("%Y%m%dT%H%M%S")), | |
| jobQueue='himantests-test-queue', | |
| jobDefinition='himantests-testgh-job' | |
| ) | |
| print(response) | |
| with open(os.environ['GITHUB_OUTPUT'], 'a') as fh: | |
| print(f"job-id={response['jobId']}", file=fh) | |
| - name: Wait until finished | |
| id: wait-until-finished | |
| shell: python | |
| run: | | |
| import boto3 | |
| import time | |
| import datetime | |
| import os | |
| import sys | |
| batch = boto3.client('batch', | |
| aws_access_key_id="${{ secrets.AWS_ACCESS_KEY_ID }}", | |
| aws_secret_access_key="${{ secrets.AWS_SECRET_ACCESS_KEY }}", | |
| region_name="eu-west-1") | |
| jobid = "${{ steps.start-tests.outputs.job-id }}" | |
| status = batch.describe_jobs(jobs=[jobid])['jobs'][0] | |
| print(status) | |
| wait = 30 | |
| while status['status'] not in ['FAILED', 'SUCCEEDED']: | |
| time.sleep(wait) | |
| status = batch.describe_jobs(jobs=[jobid])['jobs'][0] | |
| curtime = datetime.datetime.now() | |
| print(f"{curtime}: job status: {status['status']}") | |
| print(f"Finished with status: {status['status']}") | |
| with open(os.environ['GITHUB_OUTPUT'], 'a') as fh: | |
| print(f"status={status['status']}", file=fh) | |
| print(f"logstream-name={status['attempts'][0]['container']['logStreamName']}", file=fh) | |
| if status['status'] == 'FAILED': | |
| sys.exit(1) | |
| - name: Retrieve logs | |
| if: always() | |
| id: retrieve-logs | |
| shell: python | |
| run: | | |
| import boto3 | |
| import datetime | |
| logstream = "${{ steps.wait-until-finished.outputs.logstream-name }}" | |
| print(f"logs at {logstream}") | |
| logs = boto3.client('logs', | |
| aws_access_key_id="${{ secrets.AWS_ACCESS_KEY_ID }}", | |
| aws_secret_access_key="${{ secrets.AWS_SECRET_ACCESS_KEY }}", | |
| region_name="eu-west-1") | |
| response = logs.get_log_events( | |
| logGroupName="/aws/batch/job", | |
| logStreamName=logstream, | |
| startFromHead=True | |
| ) | |
| for line in response['events']: | |
| time = datetime.datetime.fromtimestamp(int(line['timestamp'])/1000).strftime("%Y-%m-%d %H:%M:%S.%f") | |
| msg = line['message'] | |
| print(f"{time}: {msg}") | |
| sync-to-release-repo: | |
| name: update-rpm-release-repository | |
| needs: run-tests | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Retrieve saved rpms | |
| uses: actions/[email protected] | |
| with: | |
| name: rpm-files | |
| - name: List rpms | |
| run: find ${{ github.workspace }} | |
| shell: bash | |
| - name: Copy to repo | |
| shell: bash | |
| env: | |
| USER: ${{ secrets.RPM_API_USER }} | |
| PASS: ${{ secrets.RPM_API_PASS }} | |
| HOST: ${{ secrets.RPM_API_HOST }} | |
| run: | | |
| find ${{ github.workspace }} -type f -name "*.rpm" -not -name "*debug*" -exec curl --show-error --fail -u $USER:$PASS -F file=@{} $HOST/api/upload \; | |
| build-image: | |
| name: build-image-and-push-to-quay.io | |
| runs-on: ubuntu-latest | |
| needs: sync-to-release-repo | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v4 | |
| - name: Build image | |
| id: build-image | |
| uses: redhat-actions/buildah-build@v2 | |
| with: | |
| image: himan | |
| tags: latest | |
| containerfiles: Containerfile | |
| - name: Test image | |
| uses: fmidev/podman-run-action@v1 | |
| with: | |
| image: ${{ steps.build-image.outputs.image-with-tag }} | |
| run: himan -d 5 -l | |
| - name: Push to quay.io | |
| id: push-to-quay | |
| uses: redhat-actions/push-to-registry@v2 | |
| with: | |
| image: ${{ steps.build-image.outputs.image }} | |
| tags: ${{ steps.build-image.outputs.tags }} | |
| registry: quay.io/fmi | |
| username: ${{ secrets.QUAY_USER }} | |
| password: ${{ secrets.QUAY_TOKEN }} | |
| - name: Print image url | |
| run: echo "Image pushed to ${{ steps.push-to-quay.outputs.registry-paths }}" | |
| build-rpms-for-rhel9: | |
| name: build-rpms-for-rhel9 | |
| runs-on: ubuntu-latest | |
| container: | |
| image: rockylinux/rockylinux:9 | |
| volumes: | |
| - ${{ github.workspace }}:/github/workspace | |
| needs: | |
| - run-tests | |
| - check-if-himan-bin-changed | |
| - check-if-himan-lib-changed | |
| - check-if-himan-scripts-changed | |
| - check-if-himan-plugins-changed | |
| outputs: | |
| trigger: ${{ steps.set-trigger.outputs.trigger }} | |
| steps: | |
| - name: Install build tools | |
| shell: bash | |
| run: | | |
| dnf -y install rpm-build rpmdevtools git yum-utils dnf-plugins-core findutils | |
| git config --global --add safe.directory '*' | |
| - name: Checkout code | |
| uses: actions/checkout@v4 | |
| - name: List workspace | |
| run: ls -lah | |
| - name: Setup repositories | |
| shell: bash | |
| run: | | |
| dnf -y install epel-release https://download.fmi.fi/smartmet-open/rhel/9/x86_64/smartmet-open-release-latest-9.noarch.rpm | |
| dnf config-manager --set-enabled crb | |
| dnf -y module disable postgresql | |
| dnf config-manager --add-repo https://developer.download.nvidia.com/compute/cuda/repos/rhel9/x86_64/cuda-rhel9.repo | |
| dnf config-manager --setopt='epel.exclude=eccodes*' --setopt='epel.exclude=proj' --save | |
| - name: Setup rpmtree | |
| shell: bash | |
| run: | | |
| rpmdev-setuptree | |
| - name: Build himan-lib rpm | |
| id: build-himan-lib | |
| if: needs.check-if-himan-lib-changed.outputs.changed == 'true' | |
| shell: bash | |
| run: | | |
| cd himan-lib | |
| yum-builddep -y himan-lib.spec --nobest | |
| make rpm | |
| - name: Build himan-bin rpm | |
| id: build-himan-bin | |
| if: needs.check-if-himan-bin-changed.outputs.changed == 'true' | |
| shell: bash | |
| run: | | |
| cd himan-bin | |
| yum-builddep -y himan-bin.spec | |
| make rpm | |
| - name: Build himan-plugins rpm | |
| id: build-himan-plugins | |
| if: needs.check-if-himan-plugins-changed.outputs.changed == 'true' | |
| shell: bash | |
| run: | | |
| cd himan-plugins | |
| yum-builddep -y himan-plugins.spec --nobest | |
| make rpm | |
| - name: Build himan-scripts rpm | |
| id: build-himan-scripts | |
| if: needs.check-if-himan-scripts-changed.outputs.changed == 'true' | |
| shell: bash | |
| run: | | |
| cd himan-scripts | |
| yum-builddep -y himan-scripts.spec | |
| make rpm | |
| - name: Set output | |
| id: set-output | |
| if: | | |
| always() && (steps.build-himan-bin.conclusion == 'success' || steps.build-himan-lib.conclusion == 'success' || steps.build-himan-plugins.conclusion == 'success' || steps.build-himan-scripts.conclusion == 'success') | |
| shell: bash | |
| run: | | |
| mkdir -p /github/workspace/rpmbuild/SRPMS /github/workspace/rpmbuild/RPMS | |
| find /github/home/rpmbuild/SRPMS/ -type f -name "*.src.rpm" -exec cp {} /github/workspace/rpmbuild/SRPMS \; | |
| find /github/home/rpmbuild/RPMS/ -type f -name "*.rpm" -not -name "*debug*" -exec cp {} /github/workspace/rpmbuild/RPMS \; | |
| ls -la /github/workspace/rpmbuild/SRPMS /github/workspace/rpmbuild/RPMS | |
| echo "srpm_dir_path=rpmbuild/SRPMS/" >> "$GITHUB_OUTPUT" | |
| echo "rpm_dir_path=rpmbuild/RPMS/" >> "$GITHUB_OUTPUT" | |
| - name: Save RPM as an artifact | |
| id: save-rpm | |
| if: always() && steps.set-output.conclusion == 'success' | |
| uses: actions/[email protected] | |
| with: | |
| name: rpm-files-rhel9 | |
| path: ${{ steps.set-output.outputs.rpm_dir_path }} | |
| retention-days: 1 | |
| - name: Set trigger | |
| id: set-trigger | |
| if: always() && steps.save-rpm.conclusion == 'success' | |
| shell: bash | |
| run: | | |
| echo "trigger=yes" >> "$GITHUB_OUTPUT" | |
| sync-to-release-repo-rhel9: | |
| name: update-rpm-release-repository-rhel9 | |
| needs: build-rpms-for-rhel9 | |
| if: always() && needs.build-rpms-for-rhel9.outputs.trigger == 'yes' | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Retrieve saved rpms | |
| uses: actions/[email protected] | |
| with: | |
| name: rpm-files-rhel9 | |
| - name: List rpms | |
| run: find ${{ github.workspace }} | |
| shell: bash | |
| - name: Copy to repo | |
| shell: bash | |
| env: | |
| USER: ${{ secrets.RPM_API_USER }} | |
| PASS: ${{ secrets.RPM_API_PASS }} | |
| HOST: ${{ secrets.RPM_API_HOST_RHEL9 }} | |
| run: | | |
| find ${{ github.workspace }} -type f -name "*.rpm" -not -name "*debug*" -exec curl --show-error --fail -u $USER:$PASS -F file=@{} $HOST/api/upload \; |