environment for sycl development and test #1638
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: CI | |
| # Controls when the action will run. | |
| on: | |
| # Triggers the workflow on push or pull request events but only for the master branch | |
| push: | |
| branches: [master] | |
| pull_request: | |
| branches: [master] | |
| # Allows you to run this workflow manually from the Actions tab | |
| workflow_dispatch: | |
| inputs: | |
| linux_test_debug_enabled: | |
| type: boolean | |
| description: "Run the linux test with VS Code Sever debugging enabled" | |
| required: false | |
| default: false | |
| windows_build_debug_enabled: | |
| type: boolean | |
| description: "Run the windows build with VS Code Sever debugging enabled" | |
| required: false | |
| default: false | |
| env: | |
| VCPKG_VERSION: "2023.10.19" | |
| CTEST_PARALLEL_LEVEL: "1" | |
| # A workflow is made up of one or more jobs that can run sequentially or in parallel | |
| jobs: | |
| ############################################################################### | |
| Linux-sycl: | |
| if: ${{ github.event_name != 'workflow_dispatch' }} | |
| runs-on: ubuntu-22.04 | |
| env: | |
| VCPKG_DEFAULT_TRIPLET: x64-linux | |
| steps: | |
| # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it | |
| - uses: actions/checkout@v3 | |
| - name: Install system dependencies | |
| run: | | |
| sudo apt update | |
| sudo apt install -y \ | |
| apt-utils \ | |
| build-essential \ | |
| curl zip unzip tar `# when starting fresh on a WSL image for bootstrapping vcpkg`\ | |
| pkg-config `# for installing libraries with vcpkg`\ | |
| git \ | |
| cmake \ | |
| ninja-build \ | |
| libfontconfig1-dev `# From here required for vcpkg opencascade`\ | |
| libx11-dev \ | |
| libgl-dev | |
| wget -O- https://apt.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS.PUB | gpg --dearmor | sudo tee /usr/share/keyrings/oneapi-archive-keyring.gpg > /dev/null | |
| echo "deb [signed-by=/usr/share/keyrings/oneapi-archive-keyring.gpg] https://apt.repos.intel.com/oneapi all main" | sudo tee /etc/apt/sources.list.d/oneAPI.list | |
| sudo apt update | |
| sudo apt install intel-basekit \ | |
| intel-oneapi-runtime-opencl | |
| - uses: hendrikmuhs/[email protected] | |
| with: | |
| key: ${{ github.job }} | |
| - uses: friendlyanon/setup-vcpkg@v1 # Setup vcpkg into ${{github.workspace}} | |
| with: | |
| committish: ${{ env.VCPKG_VERSION }} | |
| cache-version: ${{env.VCPKG_VERSION}} | |
| - name: Install dependencies | |
| run: | | |
| ${{github.workspace}}/vcpkg/vcpkg install --clean-after-build openblas[dynamic-arch] --allow-unsupported # last argument to remove after regression introduced by microsoft/vcpkg#30192 is addressed | |
| # Simbody depends on (open)blas implementation, which -march=native by default, conflicting with cache restore, hence dynamic-arch feature | |
| # Above problem might also be resolved by adding the hash of architecture in the cache key, esp. if more package do the same | |
| ${{github.workspace}}/vcpkg/vcpkg install --clean-after-build \ | |
| eigen3 \ | |
| tbb \ | |
| boost-program-options \ | |
| boost-geometry \ | |
| simbody \ | |
| gtest \ | |
| pybind11 | |
| - name: Generate buildsystem | |
| run: | | |
| # sycl environment has to be setup for each session | |
| source /opt/intel/oneapi/setvars.sh --include-intel-llvm | |
| cmake -G Ninja \ | |
| -D CMAKE_BUILD_TYPE=Release \ | |
| -D CMAKE_C_COMPILER=icx -D CMAKE_CXX_COMPILER=icpx \ | |
| -D CMAKE_TOOLCHAIN_FILE="${{github.workspace}}/vcpkg/scripts/buildsystems/vcpkg.cmake" \ | |
| -D CMAKE_C_COMPILER_LAUNCHER=ccache -D CMAKE_CXX_COMPILER_LAUNCHER=ccache \ | |
| -D SPHINXSYS_CI=ON \ | |
| -D SPHINXSYS_USE_SYCL=ON \ | |
| -D TEST_STATE_RECORDING=OFF \ | |
| -S ${{github.workspace}} \ | |
| -B ${{github.workspace}}/build | |
| - name: Build libraries and tests which are using SYCL | |
| run: | | |
| # sycl environment has to be setup for each session | |
| source /opt/intel/oneapi/setvars.sh --include-intel-llvm | |
| cmake --build build --config Release --verbose | |
| - name: Test with the first try | |
| id: first-try | |
| run: | | |
| source /opt/intel/oneapi/setvars.sh --include-intel-llvm | |
| cd build/tests/tests_sycl | |
| ctest --output-on-failure --timeout 1000 | |
| continue-on-error: true | |
| - name: Test with the second try for failed cases | |
| id: second-try | |
| if: ${{ steps.first-try.outcome == 'failure' }} | |
| run: | | |
| source /opt/intel/oneapi/setvars.sh --include-intel-llvm | |
| cd build/tests/tests_sycl | |
| ctest --rerun-failed --output-on-failure --timeout 1000 | |
| continue-on-error: true | |
| - name: Test with the third try for failed cases | |
| id: third-try | |
| if: ${{ steps.second-try.outcome == 'failure' }} | |
| run: | | |
| source /opt/intel/oneapi/setvars.sh --include-intel-llvm | |
| cd build/tests/tests_sycl | |
| ctest --rerun-failed --output-on-failure --timeout 1000 | |
| continue-on-error: true | |
| - name: Test with the fourth try for failed cases | |
| id: fourth-try | |
| if: ${{ steps.third-try.outcome == 'failure' }} | |
| run: | | |
| source /opt/intel/oneapi/setvars.sh --include-intel-llvm | |
| cd build/tests/tests_sycl | |
| ctest --rerun-failed --output-on-failure --timeout 1000 | |
| continue-on-error: true | |
| - name: Test with the last try for failed cases | |
| if: ${{ steps.fourth-try.outcome == 'failure' }} | |
| run: | | |
| source /opt/intel/oneapi/setvars.sh --include-intel-llvm | |
| cd build/tests/tests_sycl | |
| ctest --rerun-failed --output-on-failure --timeout 1000 | |
| ############################################################################### | |
| Linux-build: | |
| if: ${{ github.event_name != 'workflow_dispatch' }} | |
| runs-on: ubuntu-22.04 | |
| env: | |
| VCPKG_DEFAULT_TRIPLET: x64-linux | |
| steps: | |
| # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it | |
| - uses: actions/checkout@v3 | |
| - name: Install system dependencies | |
| run: | | |
| sudo apt update | |
| sudo apt install -y \ | |
| apt-utils \ | |
| build-essential \ | |
| curl zip unzip tar `# when starting fresh on a WSL image for bootstrapping vcpkg`\ | |
| pkg-config `# for installing libraries with vcpkg`\ | |
| git \ | |
| cmake \ | |
| ninja-build \ | |
| libfontconfig1-dev `# From here required for vcpkg opencascade`\ | |
| libx11-dev \ | |
| libgl-dev | |
| - uses: hendrikmuhs/[email protected] | |
| with: | |
| key: ${{ github.job }} | |
| - uses: friendlyanon/setup-vcpkg@v1 # Setup vcpkg into ${{github.workspace}} | |
| with: | |
| committish: ${{ env.VCPKG_VERSION }} | |
| cache-version: ${{env.VCPKG_VERSION}} | |
| - name: Install dependencies | |
| run: | | |
| ${{github.workspace}}/vcpkg/vcpkg install --clean-after-build openblas[dynamic-arch] --allow-unsupported # last argument to remove after regression introduced by microsoft/vcpkg#30192 is addressed | |
| # Simbody depends on (open)blas implementation, which -march=native by default, conflicting with cache restore, hence dynamic-arch feature | |
| # Above problem might also be resolved by adding the hash of architecture in the cache key, esp. if more package do the same | |
| ${{github.workspace}}/vcpkg/vcpkg install --clean-after-build \ | |
| eigen3 \ | |
| tbb \ | |
| boost-program-options \ | |
| boost-geometry \ | |
| simbody \ | |
| gtest \ | |
| xsimd \ | |
| pybind11 \ | |
| opencascade | |
| - name: Generate buildsystem using double | |
| run: | | |
| cmake -G Ninja \ | |
| -D CMAKE_BUILD_TYPE=Release \ | |
| -D CMAKE_TOOLCHAIN_FILE="${{github.workspace}}/vcpkg/scripts/buildsystems/vcpkg.cmake" \ | |
| -D CMAKE_C_COMPILER_LAUNCHER=ccache -D CMAKE_CXX_COMPILER_LAUNCHER=ccache \ | |
| -D SPHINXSYS_CI=ON \ | |
| -D SPHINXSYS_USE_FLOAT=OFF \ | |
| -D TEST_STATE_RECORDING=OFF \ | |
| -D SPHINXSYS_MODULE_OPENCASCADE=ON \ | |
| -S ${{github.workspace}} \ | |
| -B ${{github.workspace}}/build | |
| - name: Build using double | |
| run: cmake --build build --config Release --verbose | |
| - name: Zip build folder | |
| run: zip -r build.zip ./build | |
| - uses: actions/upload-artifact@v3 | |
| with: | |
| name: Linux-build | |
| path: ${{github.workspace}}/build.zip | |
| if-no-files-found: error | |
| retention-days: 1 | |
| Linux-test: | |
| if: ${{ github.event_name != 'workflow_dispatch' || inputs.linux_test_debug_enabled }} | |
| needs: Linux-build | |
| runs-on: ubuntu-22.04 | |
| # make sure these permissions are set so that | |
| # VS Code can connect to the machine | |
| permissions: | |
| actions: read | |
| contents: read | |
| env: | |
| VCPKG_DEFAULT_TRIPLET: x64-linux | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - name: Install system dependencies | |
| run: | | |
| sudo apt update | |
| sudo apt install -y \ | |
| apt-utils \ | |
| build-essential \ | |
| curl zip unzip tar `# when starting fresh on a WSL image for bootstrapping vcpkg`\ | |
| pkg-config `# for installing libraries with vcpkg`\ | |
| git \ | |
| cmake \ | |
| ninja-build \ | |
| libfontconfig1-dev `# From here required for vcpkg opencascade`\ | |
| libx11-dev \ | |
| libgl-dev | |
| - uses: friendlyanon/setup-vcpkg@v1 # Setup vcpkg into ${{github.workspace}} | |
| with: | |
| committish: ${{ env.VCPKG_VERSION }} | |
| cache-version: ${{env.VCPKG_VERSION}} | |
| - name: Install dependencies | |
| run: | | |
| ${{github.workspace}}/vcpkg/vcpkg install --clean-after-build openblas[dynamic-arch] --allow-unsupported # last argument to remove after regression introduced by microsoft/vcpkg#30192 is addressed | |
| # Simbody depends on (open)blas implementation, which -march=native by default, conflicting with cache restore, hence dynamic-arch feature | |
| # Above problem might also be resolved by adding the hash of architecture in the cache key, esp. if more package do the same | |
| ${{github.workspace}}/vcpkg/vcpkg install --clean-after-build \ | |
| eigen3 \ | |
| tbb \ | |
| boost-program-options \ | |
| boost-geometry \ | |
| simbody \ | |
| gtest \ | |
| xsimd \ | |
| pybind11 \ | |
| opencascade | |
| - uses: actions/download-artifact@v3 | |
| with: | |
| name: Linux-build | |
| - name: Unzip build folder | |
| run: unzip build.zip | |
| - name: Test with the first try | |
| id: first-try | |
| run: | | |
| cd build | |
| ctest --output-on-failure --timeout 1000 | |
| continue-on-error: true | |
| - name: Test with the second try for failed cases | |
| id: second-try | |
| if: ${{ steps.first-try.outcome == 'failure' }} | |
| run: | | |
| cd build | |
| ctest --rerun-failed --output-on-failure --timeout 1000 | |
| continue-on-error: true | |
| - name: Test with the third try for failed cases | |
| id: third-try | |
| if: ${{ steps.second-try.outcome == 'failure' }} | |
| run: | | |
| cd build | |
| ctest --rerun-failed --output-on-failure --timeout 1000 | |
| continue-on-error: true | |
| - name: Test with the fourth try for failed cases | |
| id: fourth-try | |
| if: ${{ steps.third-try.outcome == 'failure' }} | |
| run: | | |
| cd build | |
| ctest --rerun-failed --output-on-failure --timeout 1000 | |
| continue-on-error: true | |
| - name: Test with the last try for failed cases | |
| if: ${{ steps.fourth-try.outcome == 'failure' }} | |
| run: | | |
| cd build | |
| ctest --rerun-failed --output-on-failure --timeout 1000 | |
| - name: 🐛 If failed in dispatch work flow, initialize VS Code Server for debug | |
| uses: stateful/vscode-server-action@v1 | |
| if: ${{ failure() && github.event_name != 'workflow_dispatch' || inputs.linux_test_debug_enabled }} | |
| ############################################################################### | |
| Windows-build: | |
| if: ${{ github.event_name != 'workflow_dispatch' || inputs.windows_build_debug_enabled }} | |
| runs-on: windows-latest | |
| # make sure these permissions are set so that | |
| # VS Code can connect to the machine | |
| permissions: | |
| actions: read | |
| contents: read | |
| env: | |
| VCPKG_DEFAULT_TRIPLET: x64-windows | |
| steps: | |
| - uses: actions/checkout@v3 # Checks-out the repository under ${{github.workspace}} | |
| - name: Update ccache and ninja # For correct caching with ccache on Windows | |
| shell: bash | |
| run: | | |
| choco install ccache | |
| choco install ninja | |
| choco install 7zip | |
| choco install wget | |
| - uses: hendrikmuhs/[email protected] | |
| with: | |
| key: ${{ github.job }} | |
| - uses: friendlyanon/setup-vcpkg@v1 # Setup vcpkg into ${{github.workspace}} | |
| with: | |
| committish: ${{ env.VCPKG_VERSION }} | |
| cache-version: ${{env.VCPKG_VERSION}} | |
| - name: Install dependencies | |
| run: | | |
| ${{github.workspace}}\vcpkg\vcpkg.exe install --clean-after-build ` | |
| eigen3 ` | |
| tbb ` | |
| boost-program-options ` | |
| boost-geometry ` | |
| simbody ` | |
| gtest ` | |
| xsimd ` | |
| pybind11 | |
| - uses: ilammy/msvc-dev-cmd@v1 | |
| - name: Generate buildsystem | |
| run: | | |
| cmake.exe -G Ninja ` | |
| -D CMAKE_BUILD_TYPE=Release ` | |
| -D CMAKE_TOOLCHAIN_FILE="${{github.workspace}}\vcpkg\scripts\buildsystems\vcpkg.cmake" ` | |
| -D CMAKE_C_COMPILER_LAUNCHER=ccache -D CMAKE_CXX_COMPILER_LAUNCHER=ccache ` | |
| -D SPHINXSYS_CI=ON ` | |
| -D TEST_STATE_RECORDING=OFF ` | |
| -S ${{github.workspace}} ` | |
| -B C:\build | |
| # Enable VS code server debugging of manually-triggered workflows if the input option was provided | |
| - name: 🐛 Initialize VS Code Server for debug in dispatch work flow only | |
| shell: pwsh | |
| run: | | |
| wget https://update.code.visualstudio.com/1.90.0/win32-x64-archive/stable | |
| 7z.exe x stable | |
| .\bin\code tunnel | |
| if: ${{ github.event_name == 'workflow_dispatch' && inputs.windows_build_debug_enabled }} | |
| - name: Build | |
| run: cmake.exe --build C:\build --config Release --verbose | |
| - name: Zip build folder | |
| run: 7z.exe a C:\build.7z "C:\build" | |
| - uses: actions/upload-artifact@v3 | |
| with: | |
| name: Windows-build | |
| path: C:\build.7z | |
| if-no-files-found: error | |
| retention-days: 1 | |
| Windows-test: | |
| if: ${{ github.event_name != 'workflow_dispatch' }} | |
| needs: Windows-build | |
| runs-on: windows-latest | |
| env: | |
| VCPKG_DEFAULT_TRIPLET: x64-windows | |
| steps: | |
| - uses: actions/checkout@v3 # Checks-out the repository under ${{github.workspace}} | |
| - name: Update ccache and ninja # For correct caching with ccache on Windows | |
| shell: bash | |
| run: | | |
| choco install ccache | |
| choco install ninja | |
| choco install 7zip | |
| - uses: hendrikmuhs/[email protected] | |
| with: | |
| key: ${{ github.job }} | |
| - uses: friendlyanon/setup-vcpkg@v1 # Setup vcpkg into ${{github.workspace}} | |
| with: | |
| committish: ${{ env.VCPKG_VERSION }} | |
| cache-version: ${{env.VCPKG_VERSION}} | |
| - name: Install dependencies | |
| run: | | |
| ${{github.workspace}}\vcpkg\vcpkg.exe install --clean-after-build ` | |
| eigen3 ` | |
| tbb ` | |
| boost-program-options ` | |
| boost-geometry ` | |
| simbody ` | |
| gtest ` | |
| xsimd ` | |
| pybind11 | |
| - uses: actions/download-artifact@v3 | |
| with: | |
| name: Windows-build | |
| - name: Unzip build folder | |
| run: 7z.exe x build.7z -oC:\ | |
| - name: Test with the first try | |
| id: first-try | |
| run: | | |
| cd C:\build | |
| ctest.exe --output-on-failure --timeout 1000 | |
| continue-on-error: true | |
| - name: Test with the second try for failed cases | |
| id: second-try | |
| if: ${{ steps.first-try.outcome == 'failure' }} | |
| run: | | |
| cd C:\build | |
| ctest.exe --rerun-failed --output-on-failure --timeout 1000 | |
| continue-on-error: true | |
| - name: Test with the third try for failed cases | |
| id: third-try | |
| if: ${{ steps.second-try.outcome == 'failure' }} | |
| run: | | |
| cd C:\build | |
| ctest.exe --rerun-failed --output-on-failure --timeout 1000 | |
| continue-on-error: true | |
| - name: Test with the fourth try for failed cases | |
| id: fourth-try | |
| if: ${{ steps.third-try.outcome == 'failure' }} | |
| run: | | |
| cd C:\build | |
| ctest.exe --rerun-failed --output-on-failure --timeout 1000 | |
| continue-on-error: true | |
| - name: Test with the last try for failed cases | |
| if: ${{ steps.fourth-try.outcome == 'failure' }} | |
| run: | | |
| cd C:\build | |
| ctest.exe --rerun-failed --output-on-failure --timeout 1000 | |
| ############################################################################### | |
| macOS-build: | |
| if: ${{ github.event_name != 'workflow_dispatch' }} | |
| runs-on: macos-12 | |
| env: | |
| VCPKG_DEFAULT_TRIPLET: x64-osx | |
| steps: | |
| # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it | |
| - uses: actions/checkout@v3 | |
| - name: Install system dependencies | |
| run: | | |
| brew reinstall gfortran # to force having gfortran on PATH because github runners don't have it, just the versioned aliases, i.e. gfortran-11 | |
| # https://github.com/actions/runner-images/blob/main/images/macos/macos-12-Readme.md | |
| # https://github.com/actions/runner-images/issues/3371#issuecomment-839882565 | |
| # https://github.com/modflowpy/install-gfortran-action and https://github.com/awvwgk/setup-fortran | |
| brew install \ | |
| pkg-config `# for installing libraries with vcpkg`\ | |
| git \ | |
| cmake \ | |
| ninja | |
| - uses: hendrikmuhs/[email protected] | |
| with: | |
| key: ${{ github.job }} | |
| - uses: friendlyanon/setup-vcpkg@v1 # Setup vcpkg into ${{github.workspace}} | |
| with: | |
| committish: ${{ env.VCPKG_VERSION }} | |
| cache-version: ${{env.VCPKG_VERSION}} | |
| - name: Install dependencies | |
| run: | | |
| ${{github.workspace}}/vcpkg/vcpkg install --clean-after-build \ | |
| eigen3 \ | |
| tbb \ | |
| boost-program-options \ | |
| boost-geometry \ | |
| gtest \ | |
| simbody \ | |
| xsimd \ | |
| pybind11 | |
| - name: Generate buildsystem | |
| run: | | |
| cmake -G Ninja \ | |
| -D CMAKE_C_COMPILER=clang -D CMAKE_CXX_COMPILER=clang++ \ | |
| -D CMAKE_BUILD_TYPE=Release \ | |
| -D CMAKE_TOOLCHAIN_FILE="${{github.workspace}}/vcpkg/scripts/buildsystems/vcpkg.cmake" \ | |
| -D CMAKE_C_COMPILER_LAUNCHER=ccache -D CMAKE_CXX_COMPILER_LAUNCHER=ccache \ | |
| -D TEST_STATE_RECORDING=OFF \ | |
| -S ${{github.workspace}} \ | |
| -B ${{github.workspace}}/build | |
| - name: Build | |
| run: cmake --build build --config Release --verbose | |
| - name: Zip build folder | |
| run: zip -r build.zip ./build | |
| - uses: actions/upload-artifact@v3 | |
| with: | |
| name: macOS-build | |
| path: ${{github.workspace}}/build.zip | |
| if-no-files-found: error | |
| retention-days: 1 | |
| macOS-test: | |
| if: ${{ github.event_name != 'workflow_dispatch' }} | |
| needs: macOS-build | |
| runs-on: macos-12 | |
| env: | |
| VCPKG_DEFAULT_TRIPLET: x64-osx | |
| steps: | |
| # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it | |
| - uses: actions/checkout@v3 | |
| - name: Install system dependencies | |
| run: | | |
| brew reinstall gfortran # to force having gfortran on PATH because github runners don't have it, just the versioned aliases, i.e. gfortran-11 | |
| # https://github.com/actions/runner-images/blob/main/images/macos/macos-12-Readme.md | |
| # https://github.com/actions/runner-images/issues/3371#issuecomment-839882565 | |
| # https://github.com/modflowpy/install-gfortran-action and https://github.com/awvwgk/setup-fortran | |
| brew install \ | |
| pkg-config `# for installing libraries with vcpkg`\ | |
| git \ | |
| cmake \ | |
| ninja | |
| - uses: friendlyanon/setup-vcpkg@v1 # Setup vcpkg into ${{github.workspace}} | |
| with: | |
| committish: ${{ env.VCPKG_VERSION }} | |
| cache-version: ${{env.VCPKG_VERSION}} | |
| - name: Install dependencies | |
| run: | | |
| ${{github.workspace}}/vcpkg/vcpkg install --clean-after-build --allow-unsupported \ | |
| eigen3 \ | |
| tbb \ | |
| boost-program-options \ | |
| boost-geometry \ | |
| gtest \ | |
| simbody \ | |
| xsimd \ | |
| pybind11 | |
| - uses: actions/download-artifact@v3 | |
| with: | |
| name: macOS-build | |
| - name: Unzip build folder | |
| run: unzip build.zip | |
| - name: Test with the first try | |
| id: first-try | |
| run: | | |
| cd build | |
| ctest --output-on-failure --timeout 1000 | |
| continue-on-error: true | |
| - name: Test with the second try for failed cases | |
| id: second-try | |
| if: ${{ steps.first-try.outcome == 'failure' }} | |
| run: | | |
| cd build | |
| ctest --rerun-failed --output-on-failure --timeout 1000 | |
| continue-on-error: true | |
| - name: Test with the third try for failed cases | |
| id: third-try | |
| if: ${{ steps.second-try.outcome == 'failure' }} | |
| run: | | |
| cd build | |
| ctest --rerun-failed --output-on-failure --timeout 1000 | |
| continue-on-error: true | |
| - name: Test with the fourth try for failed cases | |
| id: fourth-try | |
| if: ${{ steps.third-try.outcome == 'failure' }} | |
| run: | | |
| cd build | |
| ctest --rerun-failed --output-on-failure --timeout 1000 | |
| continue-on-error: true | |
| - name: Test with the last try for failed cases | |
| if: ${{ steps.fourth-try.outcome == 'failure' }} | |
| run: | | |
| cd build | |
| ctest --rerun-failed --output-on-failure --timeout 1000 |