test_pull_requests.yml

name: Build and Test

on:
  pull_request:
    branches: [ develop ]

env:
  GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
  FC: gfortran-13
  Python_REQUIRED_VERSION: 3.12.3  # 3.12.2 not available on Ubuntu 24 GHA

jobs:
  build_and_test:
    name: Testing on ${{ matrix.pretty }}
    runs-on: ${{ matrix.os }}
    permissions:
      pull-requests: write
    strategy:
      fail-fast: false
      matrix:
        include:
        - os: macos-14
          macos_dev_target: 14.0
          arch: arm64
          python-arch: arm64
          run_regressions: true
          pretty: "Mac arm64"
        - os: ubuntu-24.04
          arch: x86_64
          python-arch: x64
          run_regressions: true
          pretty: "Ubuntu 24.04"
        - os: windows-2022
          arch: x86_64
          python-arch: x64
          run_regressions: false
          pretty: "Windows x64"

    steps:

    - uses: actions/checkout@v6

    - name: Setup System
      id: setup-runner
      uses: ./.github/actions/setup-runner
      with:
        python-version: ${{ env.Python_REQUIRED_VERSION }}
        python-arch: ${{ matrix.python-arch }}

    - name: Get Develop and Branch Git SHA
      shell: bash
      run: |
        echo "Figuring out the develop GIT SHA"
        # github.event.pull_request.base.sha points to develop at the time the PR was created, not the current one
        DEVELOP_SHA=$(git ls-remote https://github.com/$GITHUB_REPOSITORY.git develop | cut -f1 | cut -c1-10)
        echo "Develop GIT SHA is '${DEVELOP_SHA}'"
        echo "DEVELOP_SHA=$DEVELOP_SHA" >> $GITHUB_ENV

        CURRENT_HEAD_SHA=${{ github.event.pull_request.head.sha }}
        echo "Current Branch HEAD SHA is '${CURRENT_HEAD_SHA}'"
        echo "CURRENT_HEAD_SHA=$DEVELOP_SHA" >> $GITHUB_ENV

        echo "Current merge commit SHA is '${GITHUB_SHA}'"
        echo "::endgroup::"
        # We default to using PCH, in case we don't find the baseline reg tests,
        # since it speeds up the build significantly and we'll build TWICE
        echo "ENABLE_PCH=ON" >> $GITHUB_ENV

    - name: Baseline Regression Testing caching
      uses: actions/cache@v5
      id: cachebaselineregression
      with:
        path: |
          ./build/testfiles-develop
        key: baselineregression-${{ matrix.os }}-${{ env.DEVELOP_SHA }}

    - name: Did restoring the Baseline Regression Tests work? Yes
      if: steps.cachebaselineregression.outputs.cache-hit == 'true'
      shell: bash
      run: |
        echo "Found the regression tests!"
        # We turn off the PCH in that case, since that's how develop builds it
        echo "ENABLE_PCH=OFF" >> $GITHUB_ENV

    - name: Restore Cache CCACHE
      uses: actions/cache/restore@v5
      id: cacheccache-restore
      with:
        path: |
          ${{ steps.setup-runner.outputs.ccache-dir }}
        key: ccache-${{ matrix.os }}-${{ steps.setup-runner.outputs.compiler-id }}-${{ github.head_ref }}-pch=${{ env.ENABLE_PCH }}
        restore-keys: |
          ccache-${{ matrix.os }}-${{ steps.setup-runner.outputs.compiler-id }}-${{ github.head_ref }}
          ccache-${{ matrix.os }}-${{ steps.setup-runner.outputs.compiler-id }}-${{ env.DEVELOP_SHA }}
          ccache-${{ matrix.os }}-${{ steps.setup-runner.outputs.compiler-id }}-
          ccache-${{ matrix.os }}-

    - name: Did restoring the CCache-cache work?
      shell: bash
      run: |
        begin_group() { echo -e "::group::\033[93m$1\033[0m"; }
        Color_Off='\033[0m'       # Text Reset

        # Regular Colors
        Red='\033[0;31m'
        Green='\033[0;32m'
        BrightYellow='\033[93m'

        if [ "${{ steps.cacheccache-restore.outputs.cache-hit }}" == "true" ]; then
          echo -e "✅ ${Green}CCache primary key was hit${Color_Off}"
        else
          FILES_IN_CACHE=$(ccache --print-stats | grep "^files_in_cache" | awk '{print $2}')
          if [ "$FILES_IN_CACHE" -gt 0 ]; then
            echo -e "⚠️  ${BrightYellow}Secondary cache hit${Color_Off}"
          else
            echo -e "❌ ${Red}No cache hit${Color_Off}"
          fi;
        fi;

        begin_group "Showing existing ccache stats"
        ccache --show-stats -vv || ccache --show-stats
        echo "::endgroup::"

        ccache --zero-stats
        begin_group "Showing CCache config"
        ccache -p
        echo "::endgroup::"

    - name: Create Build Directory
      shell: bash
      run: |
        cmake -E make_directory ./build/
        if [ "$RUNNER_OS" == "macOS" ]; then
          # The MACOSX_DEPLOYMENT_TARGET environment variable sets the default value for the CMAKE_OSX_DEPLOYMENT_TARGET variable.
          echo MACOSX_DEPLOYMENT_TARGET=${{ matrix.macos_dev_target }} >> $GITHUB_ENV
        fi;

    # Checkout baseline, overrides the branch basically
    - name: Baseline Checkout
      if: matrix.run_regressions && steps.cachebaselineregression.outputs.cache-hit != 'true'
      uses: actions/checkout@v6
      with:
        ref: develop
        clean: false # Do NOT wipe the build/ and .ccache folder (`git clean -ffdx && git reset --hard HEAD`)

    - name: Baseline Configure and Build
      if: matrix.run_regressions && steps.cachebaselineregression.outputs.cache-hit != 'true'
      uses: ./.github/actions/configure-and-build
      with:
        enable-pch: ${{ env.ENABLE_PCH }} # this is 'ON' in this case
        minimal-targets: true
        build-directory: ./build
        nproc: ${{ steps.setup-runner.outputs.nproc }}
        python-version: ${{ env.Python_REQUIRED_VERSION }}
        python-root-dir: ${{ steps.setup-runner.outputs.python-root-dir }}

    - name: Baseline Test
      if: matrix.run_regressions && steps.cachebaselineregression.outputs.cache-hit != 'true'
      working-directory: ./build
      shell: bash
      run: |
        begin_group() { echo -e "::group::\033[93m$1\033[0m"; }

        Color_Off='\033[0m'       # Text Reset

        # Regular Colors
        Red='\033[0;31m'
        Green='\033[0;32m'

        begin_group "Running CTests"
        if ctest -j ${{ env.NPROC }}; then
          echo "::endgroup::"
          echo -e "✅ ${Green}All tests passed${Color_Off}"
        else
          echo "::endgroup::"
          echo -e "❌ ${Red}Some Tests Failed${Color_Off}"
          begin_group "Re-running failed tests verbosely... But not failing the workflow"
          ctest --rerun-failed -VV || true
          echo "::endgroup::"
        fi;
        mv testfiles testfiles-develop

    - name: Checkout Branch again
      if: matrix.run_regressions && steps.cachebaselineregression.outputs.cache-hit != 'true'
      uses: actions/checkout@v6
      with:
        clean: false # Do NOT wipe the build/ and .ccache folder (`git clean -ffdx && git reset --hard HEAD`)

    - name: Install problem matcher
      shell: bash
      run: echo "::add-matcher::./.github/workflows/cpp-problem-matcher.json"

    - name: Configure and Build
      id: branch_build
      uses: ./.github/actions/configure-and-build
      with:
        enable-pch: ${{ env.ENABLE_PCH }}
        minimal-targets: false
        build-directory: ./build
        nproc: ${{ steps.setup-runner.outputs.nproc }}
        python-version: ${{ env.Python_REQUIRED_VERSION }}
        python-root-dir: ${{ steps.setup-runner.outputs.python-root-dir }}

    - name: Remove problem matcher
      shell: bash
      run: echo "::remove-matcher owner=gcc-problem-matcher::"

    - name: Save CCache cache
      if: always() && steps.cacheccache-restore.outputs.cache-hit != 'true'
      uses: actions/cache/save@v5
      with:
        path: |
          ${{ steps.setup-runner.outputs.ccache-dir }}
        key: ${{ steps.cacheccache-restore.outputs.cache-primary-key }}

    - name: Run Tests
      working-directory: ./build
      shell: bash
      run: |
        begin_group() { echo -e "::group::\033[93m$1\033[0m"; }

        Color_Off='\033[0m'       # Text Reset

        # Regular Colors
        Red='\033[0;31m'
        Green='\033[0;32m'

        begin_group "Running CTests"
        if ctest -j ${{ env.NPROC }}; then
          echo "::endgroup::"
          echo -e "✅ ${Green}All tests passed${Color_Off}"
        else
          echo "::endgroup::"
          echo -e "❌ ${Red}Some Tests Failed${Color_Off}"
          begin_group "Re-running failed tests verbosely..."
          ctest --rerun-failed -VV
          echo "::endgroup::"
        fi;

    - name: Install Regression Tool
      if: always() && matrix.run_regressions && steps.branch_build.outcome != 'failure'  # always run this step as long as we actually built
      run: pip install energyplus-regressions>=2.1.4

    - name: Run Regressions
      if: always() && matrix.run_regressions && steps.branch_build.outcome != 'failure'  # always run this step as long as we actually built
      id: regressions
      # steps.regressions.conclusion is always "success", but if no regressions, steps.regressions.outcome is "success"
      continue-on-error: true
      run: python ./scripts/dev/gha_regressions.py ./build/testfiles-develop ./build/testfiles ./regressions

    - uses: actions/upload-artifact@v7
      id: upload_regressions
      if: always() && matrix.run_regressions && steps.regressions.outcome == 'failure'  # only run this if regressions were encountered "failed"
      with:
        name: "regressions-${{ matrix.os }}-${{ github.event.pull_request.head.sha }}"
        path: "${{ github.workspace }}/regressions"

    - name: Generate Regression Summary GitHub Script
      if: always() && matrix.run_regressions && steps.regressions.outcome == 'failure'
      run: >
        python ./scripts/dev/build_regression_summary.py
        ${{ github.workspace }}
        ${{ matrix.os }}
        ${{ github.sha }}
        ${{ github.run_id }}
        ${{ steps.upload_regressions.outputs.artifact-url }}

    - uses: actions/github-script@v8
      if: always() && matrix.run_regressions && steps.regressions.outcome == 'failure' && github.event.pull_request.head.repo.full_name == 'NatLabRockies/EnergyPlus'
      with:
        script: |
          const path = require('path')
          const summaryPath = path.join(process.env.GITHUB_WORKSPACE, 'regressions', 'summary.js')
          const script = require(summaryPath)
          console.log(script({ github, context }))

    # - uses: lhotari/action-upterm@v1

    - name: Fail on Regressions from Forked Repository
      if: always() && matrix.run_regressions && steps.regressions.outcome == 'failure' && github.event.pull_request.head.repo.full_name != 'NatLabRockies/EnergyPlus'
      run: |
        echo "::error::Regressions detected in pull request from forked repository, check job summary for details and to download regression results"
        exit 1