CI: add *BSD #5162
Workflow file for this run
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # badge: https://github.com/borgbackup/borg/workflows/CI/badge.svg?branch=master | |
| name: CI | |
| on: | |
| push: | |
| branches: [ master ] | |
| tags: | |
| - '2.*' | |
| pull_request: | |
| branches: [ master ] | |
| paths: | |
| - '**.py' | |
| - '**.pyx' | |
| - '**.c' | |
| - '**.h' | |
| - '**.yml' | |
| - '**.toml' | |
| - '**.cfg' | |
| - '**.ini' | |
| - 'requirements.d/*' | |
| - '!docs/**' | |
| jobs: | |
| lint: | |
| runs-on: ubuntu-22.04 | |
| timeout-minutes: 5 | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: chartboost/ruff-action@v1 | |
| security: | |
| runs-on: ubuntu-24.04 | |
| timeout-minutes: 5 | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Set up Python | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: '3.10' | |
| - name: Install dependencies | |
| run: | | |
| python -m pip install --upgrade pip | |
| pip install bandit[toml] | |
| - name: Run Bandit | |
| run: | | |
| bandit -r src/borg -c pyproject.toml | |
| asan_ubsan: | |
| runs-on: ubuntu-24.04 | |
| timeout-minutes: 25 | |
| needs: [security] | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| # Just fetching one commit is not enough for setuptools-scm, so we fetch all. | |
| fetch-depth: 0 | |
| fetch-tags: true | |
| - name: Set up Python | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: '3.12' | |
| - name: Install system packages | |
| run: | | |
| sudo apt-get update | |
| sudo apt-get install -y pkg-config build-essential | |
| sudo apt-get install -y libssl-dev libacl1-dev libxxhash-dev liblz4-dev libzstd-dev | |
| - name: Install Python dependencies | |
| run: | | |
| python -m pip install --upgrade pip | |
| pip install -r requirements.d/development.txt | |
| - name: Build Borg with ASan/UBSan | |
| # Build the C/Cython extensions with AddressSanitizer and UndefinedBehaviorSanitizer enabled. | |
| # How this works: | |
| # - The -fsanitize=address,undefined flags inject runtime checks into our native code. If a bug is hit | |
| # (e.g., buffer overflow, use-after-free, out-of-bounds, or undefined behavior), the sanitizer prints | |
| # a detailed error report to stderr, including a stack trace, and forces the process to exit with | |
| # non-zero status. In CI, this will fail the step/job so you will notice. | |
| # - ASAN_OPTIONS/UBSAN_OPTIONS configure the sanitizers' runtime behavior (see below for meanings). | |
| env: | |
| CFLAGS: "-O1 -g -fno-omit-frame-pointer -fsanitize=address,undefined" | |
| CXXFLAGS: "-O1 -g -fno-omit-frame-pointer -fsanitize=address,undefined" | |
| LDFLAGS: "-fsanitize=address,undefined" | |
| # ASAN_OPTIONS controls AddressSanitizer runtime tweaks: | |
| # - detect_leaks=0: Disable LeakSanitizer to avoid false positives with CPython/pymalloc in short-lived tests. | |
| # - strict_string_checks=1: Make invalid string operations (e.g., over-reads) more likely to be detected. | |
| # - check_initialization_order=1: Catch uses that depend on static initialization order (C++). | |
| # - detect_stack_use_after_return=1: Detect stack-use-after-return via stack poisoning (may increase overhead). | |
| ASAN_OPTIONS: "detect_leaks=0:strict_string_checks=1:check_initialization_order=1:detect_stack_use_after_return=1" | |
| # UBSAN_OPTIONS controls UndefinedBehaviorSanitizer runtime: | |
| # - print_stacktrace=1: Include a stack trace for UB reports to ease debugging. | |
| # Note: UBSan is recoverable by default (process may continue after reporting). If you want CI to | |
| # abort immediately and fail on the first UB, add `halt_on_error=1` (e.g., UBSAN_OPTIONS="print_stacktrace=1:halt_on_error=1"). | |
| UBSAN_OPTIONS: "print_stacktrace=1" | |
| # PYTHONDEVMODE enables additional Python runtime checks and warnings. | |
| PYTHONDEVMODE: "1" | |
| run: pip install -e . | |
| - name: Run tests under sanitizers | |
| env: | |
| ASAN_OPTIONS: "detect_leaks=0:strict_string_checks=1:check_initialization_order=1:detect_stack_use_after_return=1" | |
| UBSAN_OPTIONS: "print_stacktrace=1" | |
| PYTHONDEVMODE: "1" | |
| # Ensure the ASan runtime is loaded first to avoid "ASan runtime does not come first" warnings. | |
| # We discover libasan/libubsan paths via gcc and preload them for the Python test process. | |
| # the remote tests are slow and likely won't find anything useful | |
| run: | | |
| set -euo pipefail | |
| export LD_PRELOAD="$(gcc -print-file-name=libasan.so):$(gcc -print-file-name=libubsan.so)" | |
| echo "Using LD_PRELOAD=$LD_PRELOAD" | |
| pytest -v --benchmark-skip -k "not remote" | |
| posix_tests: | |
| needs: [lint, security] | |
| permissions: | |
| contents: read | |
| id-token: write | |
| attestations: write | |
| strategy: | |
| fail-fast: true | |
| # noinspection YAMLSchemaValidation | |
| matrix: >- | |
| ${{ fromJSON( | |
| github.event_name == 'pull_request' && '{ | |
| "include": [ | |
| {"os": "ubuntu-22.04", "python-version": "3.10", "toxenv": "mypy"}, | |
| {"os": "ubuntu-22.04", "python-version": "3.11", "toxenv": "docs"}, | |
| {"os": "ubuntu-22.04", "python-version": "3.10", "toxenv": "py310-fuse2"}, | |
| {"os": "ubuntu-24.04", "python-version": "3.14", "toxenv": "py314-fuse3"} | |
| ] | |
| }' || '{ | |
| "include": [ | |
| {"os": "ubuntu-22.04", "python-version": "3.10", "toxenv": "mypy"}, | |
| {"os": "ubuntu-22.04", "python-version": "3.11", "toxenv": "docs"}, | |
| {"os": "ubuntu-22.04", "python-version": "3.10", "toxenv": "py310-fuse2"}, | |
| {"os": "ubuntu-22.04", "python-version": "3.11", "toxenv": "py311-fuse2", "binary": "borg-linux-glibc235-x86_64-gh"}, | |
| {"os": "ubuntu-22.04-arm", "python-version": "3.11", "toxenv": "py311-fuse2", "binary": "borg-linux-glibc235-arm64-gh"}, | |
| {"os": "ubuntu-24.04", "python-version": "3.12", "toxenv": "py312-fuse3"}, | |
| {"os": "ubuntu-24.04", "python-version": "3.13", "toxenv": "py313-fuse3"}, | |
| {"os": "ubuntu-24.04", "python-version": "3.14", "toxenv": "py314-fuse3"}, | |
| {"os": "macos-13", "python-version": "3.11", "toxenv": "py311-none", "binary": "borg-macos-13-x86_64-gh"}, | |
| {"os": "macos-14", "python-version": "3.11", "toxenv": "py311-none", "binary": "borg-macos-14-arm64-gh"} | |
| ] | |
| }' | |
| ) }} | |
| env: | |
| TOXENV: ${{ matrix.toxenv }} | |
| runs-on: ${{ matrix.os }} | |
| timeout-minutes: 120 | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| # Just fetching one commit is not enough for setuptools-scm, so we fetch all. | |
| fetch-depth: 0 | |
| fetch-tags: true | |
| - name: Set up Python ${{ matrix.python-version }} | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: ${{ matrix.python-version }} | |
| - name: Cache pip | |
| uses: actions/cache@v4 | |
| with: | |
| path: ~/.cache/pip | |
| key: ${{ runner.os }}-pip-${{ hashFiles('requirements.d/development.txt') }} | |
| restore-keys: | | |
| ${{ runner.os }}-pip- | |
| ${{ runner.os }}- | |
| - name: Install Linux packages | |
| if: ${{ runner.os == 'Linux' }} | |
| run: | | |
| sudo apt-get update | |
| sudo apt-get install -y pkg-config build-essential | |
| sudo apt-get install -y libssl-dev libacl1-dev libxxhash-dev liblz4-dev libzstd-dev | |
| sudo apt-get install -y libfuse-dev fuse || true # Required for Python llfuse module | |
| sudo apt-get install -y libfuse3-dev fuse3 || true # Required for Python pyfuse3 module | |
| sudo apt-get install -y bash zsh fish # for shell completion tests | |
| sudo apt-get install -y rclone openssh-server curl | |
| - name: Install macOS packages | |
| if: ${{ runner.os == 'macOS' }} | |
| run: | | |
| brew unlink [email protected] || true | |
| brew bundle install | |
| - name: Configure OpenSSH SFTP server (test only) | |
| if: ${{ runner.os == 'Linux' }} | |
| run: | | |
| sudo mkdir -p /run/sshd | |
| sudo useradd -m -s /bin/bash sftpuser || true | |
| # Create SSH key for the CI user and authorize it for sftpuser | |
| mkdir -p ~/.ssh | |
| chmod 700 ~/.ssh | |
| test -f ~/.ssh/id_ed25519 || ssh-keygen -t ed25519 -N '' -f ~/.ssh/id_ed25519 | |
| sudo mkdir -p /home/sftpuser/.ssh | |
| sudo chmod 700 /home/sftpuser/.ssh | |
| sudo cp ~/.ssh/id_ed25519.pub /home/sftpuser/.ssh/authorized_keys | |
| sudo chown -R sftpuser:sftpuser /home/sftpuser/.ssh | |
| sudo chmod 600 /home/sftpuser/.ssh/authorized_keys | |
| # Allow publickey auth and enable Subsystem sftp | |
| sudo sed -i 's/^#\?PasswordAuthentication .*/PasswordAuthentication no/' /etc/ssh/sshd_config | |
| sudo sed -i 's/^#\?PubkeyAuthentication .*/PubkeyAuthentication yes/' /etc/ssh/sshd_config | |
| if ! grep -q '^Subsystem sftp' /etc/ssh/sshd_config; then echo 'Subsystem sftp /usr/lib/openssh/sftp-server' | sudo tee -a /etc/ssh/sshd_config; fi | |
| # Ensure host keys exist to avoid slow generation on first sshd start | |
| sudo ssh-keygen -A | |
| # Start sshd (listen on default 22 inside runner) | |
| sudo /usr/sbin/sshd -D & | |
| # Add host key to known_hosts so paramiko trusts it | |
| ssh-keyscan -H localhost 127.0.0.1 | tee -a ~/.ssh/known_hosts | |
| # Start ssh-agent and add our key so paramiko can use the agent | |
| eval "$(ssh-agent -s)" | |
| ssh-add ~/.ssh/id_ed25519 | |
| # Export SFTP test URL for tox via GITHUB_ENV | |
| echo "BORG_TEST_SFTP_REPO=sftp://sftpuser@localhost:22/borg/sftp-repo" >> $GITHUB_ENV | |
| - name: Install and configure MinIO S3 server (test only) | |
| if: ${{ runner.os == 'Linux' }} | |
| run: | | |
| set -e | |
| arch=$(uname -m) | |
| case "$arch" in | |
| x86_64|amd64) srv_url=https://dl.min.io/server/minio/release/linux-amd64/minio; cli_url=https://dl.min.io/client/mc/release/linux-amd64/mc ;; | |
| aarch64|arm64) srv_url=https://dl.min.io/server/minio/release/linux-arm64/minio; cli_url=https://dl.min.io/client/mc/release/linux-arm64/mc ;; | |
| *) echo "Unsupported arch: $arch"; exit 1 ;; | |
| esac | |
| curl -fsSL -o /usr/local/bin/minio "$srv_url" | |
| curl -fsSL -o /usr/local/bin/mc "$cli_url" | |
| sudo chmod +x /usr/local/bin/minio /usr/local/bin/mc | |
| export PATH=/usr/local/bin:$PATH | |
| # Start MinIO on :9000 with default credentials (minioadmin/minioadmin) | |
| MINIO_DIR="$GITHUB_WORKSPACE/.minio-data" | |
| MINIO_LOG="$GITHUB_WORKSPACE/.minio.log" | |
| mkdir -p "$MINIO_DIR" | |
| nohup minio server "$MINIO_DIR" --address ":9000" >"$MINIO_LOG" 2>&1 & | |
| # Wait for MinIO port to be ready | |
| for i in $(seq 1 60); do (echo > /dev/tcp/127.0.0.1/9000) >/dev/null 2>&1 && break; sleep 1; done | |
| # Configure client and create bucket | |
| mc alias set local http://127.0.0.1:9000 minioadmin minioadmin | |
| mc mb --ignore-existing local/borg | |
| # Export S3 test URL for tox via GITHUB_ENV | |
| echo "BORG_TEST_S3_REPO=s3:minioadmin:minioadmin@http://127.0.0.1:9000/borg/s3-repo" >> $GITHUB_ENV | |
| - name: Install Python requirements | |
| run: | | |
| python -m pip install --upgrade pip setuptools wheel | |
| pip install -r requirements.d/development.txt | |
| - name: Install borgbackup | |
| run: | | |
| pip install -e . | |
| - name: run tox env | |
| env: | |
| XDISTN: "4" | |
| run: | | |
| # do not use fakeroot, but run as root. avoids the dreaded EISDIR sporadic failures. see #2482. | |
| #sudo -E bash -c "tox -e py" | |
| tox --skip-missing-interpreters | |
| - name: Upload coverage to Codecov | |
| uses: codecov/codecov-action@v4 | |
| env: | |
| OS: ${{ runner.os }} | |
| python: ${{ matrix.python-version }} | |
| with: | |
| token: ${{ secrets.CODECOV_TOKEN }} | |
| env_vars: OS, python | |
| - name: Build Borg fat binaries (${{ matrix.binary }}) | |
| if: ${{ matrix.binary && startsWith(github.ref, 'refs/tags/') }} | |
| run: | | |
| pip install 'pyinstaller==6.14.2' | |
| mkdir -p dist/binary | |
| pyinstaller --clean --distpath=dist/binary scripts/borg.exe.spec | |
| - name: Smoke-test the built binary (${{ matrix.binary }}) | |
| if: ${{ matrix.binary && startsWith(github.ref, 'refs/tags/') }} | |
| run: | | |
| pushd dist/binary | |
| echo "single-file binary" | |
| chmod +x borg.exe | |
| ./borg.exe -V | |
| echo "single-directory binary" | |
| chmod +x borg-dir/borg.exe | |
| ./borg-dir/borg.exe -V | |
| tar czf borg.tgz borg-dir | |
| popd | |
| - name: Prepare binaries (${{ matrix.binary }}) | |
| if: ${{ matrix.binary && startsWith(github.ref, 'refs/tags/') }} | |
| run: | | |
| mkdir -p artifacts | |
| if [ -f dist/binary/borg.exe ]; then | |
| cp dist/binary/borg.exe artifacts/${{ matrix.binary }} | |
| fi | |
| if [ -f dist/binary/borg.tgz ]; then | |
| cp dist/binary/borg.tgz artifacts/${{ matrix.binary }}.tgz | |
| fi | |
| echo "binary files" | |
| ls -l artifacts/ | |
| - name: Attest binaries provenance (${{ matrix.binary }}) | |
| if: ${{ matrix.binary && startsWith(github.ref, 'refs/tags/') }} | |
| uses: actions/attest-build-provenance@v3 | |
| with: | |
| subject-path: 'artifacts/*' | |
| - name: Upload binaries (${{ matrix.binary }}) | |
| if: ${{ matrix.binary && startsWith(github.ref, 'refs/tags/') }} | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: ${{ matrix.binary }} | |
| path: artifacts/* | |
| if-no-files-found: error | |
| windows_tests: | |
| if: false # can be used to temporarily disable the build | |
| runs-on: windows-latest | |
| timeout-minutes: 120 | |
| needs: posix_tests | |
| env: | |
| PY_COLORS: 1 | |
| defaults: | |
| run: | |
| shell: msys2 {0} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 | |
| - uses: msys2/setup-msys2@v2 | |
| with: | |
| msystem: UCRT64 | |
| update: true | |
| - name: Install system packages | |
| run: ./scripts/msys2-install-deps development | |
| - name: Build python venv | |
| run: | | |
| # building cffi / argon2-cffi in the venv fails, so we try to use the system packages | |
| python -m venv --system-site-packages env | |
| . env/bin/activate | |
| # python -m pip install --upgrade pip | |
| # pip install --upgrade setuptools build wheel | |
| pip install pyinstaller==6.14.2 | |
| - name: Build | |
| run: | | |
| # build borg.exe | |
| . env/bin/activate | |
| pip install -e . | |
| pyinstaller -y scripts/borg.exe.spec | |
| # build sdist and wheel in dist/... | |
| python -m build | |
| - uses: actions/upload-artifact@v4 | |
| with: | |
| name: borg-windows | |
| path: dist/borg.exe | |
| - name: Run tests | |
| run: | | |
| ./dist/borg.exe -V | |
| . env/bin/activate | |
| borg -V | |
| python -m pytest -n4 --benchmark-skip -vv -rs -k "not remote" | |
| - name: Upload coverage to Codecov | |
| uses: codecov/codecov-action@v4 | |
| env: | |
| OS: ${{ runner.os }} | |
| python: '3.11' | |
| with: | |
| token: ${{ secrets.CODECOV_TOKEN }} | |
| env_vars: OS, python |