Switch to S3 for vcpkg cache

.github/workflows/ci.yml

@@ -31,6 +31,14 @@ jobs:
     runs-on: ubuntu-latest
     env:
       UV_NO_EDITABLE: true
+      VCPKG_BINARY_SOURCES: ${{ vars.VCPKG_BINARY_SOURCES }}
+      AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+      AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+      AWS_ENDPOINT_URL: ${{ secrets.AWS_ENDPOINT_URL }}
+      # only required until R2 supports CRC64NVME checksums
+      # https://community.cloudflare.com/t/an-error-occurred-internalerror-when-calling-the-putobject-operation/764905/11
+      AWS_REQUEST_CHECKSUM_CALCULATION: when_required
+      AWS_RESPONSE_CHECKSUM_VALIDATION: when_required
     steps:
       - uses: actions/checkout@v4
         name: Checkout
@@ -40,20 +48,12 @@ jobs:
       # Setup C++ dependencies
       - run: ./cpp/setup_pipelines.sh
         name: Clone pipelines & vcpkg
-      - uses: TAServers/vcpkg-cache@v3
-        name: Setup vcpkg cache
-        id: vcpkg-cache
-        with:
-          token: ${{ secrets.GITHUB_TOKEN }}
 
       # Setup Python dependencies
       - name: Install uv
         uses: astral-sh/setup-uv@v5
       - name: Install the project
-        run: uv sync --dev --verbose --no-editable
-        env:
-          VCPKG_FEATURE_FLAGS: "binarycaching" # Possibly redundant, but explicitly sets the binary caching feature flag
-          VCPKG_BINARY_SOURCES: "clear;files,${{ steps.vcpkg-cache.outputs.path }},readwrite"
+        run: uv sync --dev --verbose
 
       # - name: Setup tmate session
       #   uses: mxschmitt/action-tmate@v3
@@ -107,6 +107,15 @@ jobs:
       fail-fast: false
       matrix:
         os: [ubuntu-latest, macos-13, macos-14]
+    env:
+      VCPKG_BINARY_SOURCES: ${{ vars.VCPKG_BINARY_SOURCES }}
+      AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+      AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+      AWS_ENDPOINT_URL: ${{ secrets.AWS_ENDPOINT_URL }}
+      # only required until R2 supports CRC64NVME checksums
+      # https://community.cloudflare.com/t/an-error-occurred-internalerror-when-calling-the-putobject-operation/764905/11
+      AWS_REQUEST_CHECKSUM_CALCULATION: when_required
+      AWS_RESPONSE_CHECKSUM_VALIDATION: when_required
     steps:
       - uses: actions/checkout@v4
         name: Checkout
@@ -115,25 +124,11 @@ jobs:
       - run: ./cpp/setup_pipelines.sh
         name: Clone pipelines & vcpkg
 
-      # Restore vcpkg cache
-      - uses: TAServers/vcpkg-cache@v3
-        name: Setup vcpkg cache
-        id: vcpkg-cache
-        with:
-          token: ${{ secrets.GITHUB_TOKEN }}
-
       - uses: pypa/[email protected]
         name: Build wheel
         env:
           CIBW_BUILD: "cp311-manylinux_x86_64 cp311-macosx_x86_64 cp311-macosx_arm64"
-          # for vcpkg caches
-          CIBW_ENVIRONMENT_MACOS: >
-            MACOSX_DEPLOYMENT_TARGET="11"
-            VCPKG_FEATURE_FLAGS="binarycaching" 
-            VCPKG_BINARY_SOURCES="clear;files,${{ steps.vcpkg-cache.outputs.path }},readwrite"
-          CIBW_ENVIRONMENT_LINUX: >
-            VCPKG_FEATURE_FLAGS="binarycaching" 
-            VCPKG_BINARY_SOURCES="clear;files,/host${{ steps.vcpkg-cache.outputs.path }},readwrite"
+          CIBW_ENVIRONMENT_PASS_LINUX: VCPKG_BINARY_SOURCES AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY AWS_ENDPOINT_URL AWS_REQUEST_CHECKSUM_CALCULATION AWS_RESPONSE_CHECKSUM_VALIDATION
 
       # save artifacts for testing
       - uses: actions/upload-artifact@v4

.github/workflows/release.yml

@@ -10,49 +10,28 @@ jobs:
   build_wheels:
     runs-on: ${{ matrix.os }}
     env:
-      VCPKG_DEFAULT_BINARY_CACHE: "${{ github.workspace }}/.vcpkg_cache"
+      VCPKG_BINARY_SOURCES: ${{ vars.VCPKG_BINARY_SOURCES }}
+      AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+      AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+      AWS_ENDPOINT_URL: ${{ secrets.AWS_ENDPOINT_URL }}
+      # only required until R2 supports CRC64NVME checksums
+      # https://community.cloudflare.com/t/an-error-occurred-internalerror-when-calling-the-putobject-operation/764905/11
+      AWS_REQUEST_CHECKSUM_CALCULATION: when_required
+      AWS_RESPONSE_CHECKSUM_VALIDATION: when_required
     strategy:
       matrix:
         os: [ubuntu-latest, macos-13, macos-14]
     steps:
       - uses: actions/checkout@v4
         name: Checkout
 
-      # Let vcpkg store caches in github actions
-      - name: Restore vcpkg cache
-        id: cache
-        uses: actions/cache/restore@v4
-        with:
-          path: "${{ env.VCPKG_DEFAULT_BINARY_CACHE }}"
-          key: vcpkg-${{ matrix.os }}
-      - name: Ensure vcpkg cache directory exists
-        run: mkdir -p "${{ env.VCPKG_DEFAULT_BINARY_CACHE }}"
-
       # Do all the building
       - run: ./cpp/setup_pipelines.sh
         name: Clone pipelines & vcpkg
       - uses: pypa/[email protected]
         name: Build wheel
         env:
-          # for vcpkg caches
-          CIBW_ENVIRONMENT_LINUX : VCPKG_DEFAULT_BINARY_CACHE="/host${{ env.VCPKG_DEFAULT_BINARY_CACHE }}"
-
-      # Delete the old cache on hit to emulate a cache update. See
-      # https://github.com/actions/cache/issues/342.
-      - name: Delete old cache
-        env:
-          GH_TOKEN: ${{ github.token }}
-        if: steps.cache.outputs.cache-hit 
-        # Using `--repo` makes it so that this step doesn't require checking out the repo first.
-        run: gh cache delete --repo ${{ github.repository }} ${{ steps.cache.outputs.cache-primary-key }}
-
-      # cache vcpkg
-      - name: Save vcpkg cache
-        if: always()
-        uses: actions/cache/save@v4
-        with:
-          path: "${{ env.VCPKG_DEFAULT_BINARY_CACHE }}"
-          key: vcpkg-${{ matrix.os }}
+          CIBW_ENVIRONMENT_PASS_LINUX: VCPKG_BINARY_SOURCES AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY AWS_ENDPOINT_URL AWS_REQUEST_CHECKSUM_CALCULATION AWS_RESPONSE_CHECKSUM_VALIDATION
 
       # upload to use in other jobs
       - uses: actions/upload-artifact@v4
@@ -102,6 +81,14 @@ jobs:
     runs-on: ubuntu-latest
     env:
       UV_NO_EDITABLE: true
+      VCPKG_BINARY_SOURCES: ${{ vars.VCPKG_BINARY_SOURCES }}
+      AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+      AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+      AWS_ENDPOINT_URL: ${{ secrets.AWS_ENDPOINT_URL }}
+      # only required until R2 supports CRC64NVME checksums
+      # https://community.cloudflare.com/t/an-error-occurred-internalerror-when-calling-the-putobject-operation/764905/11
+      AWS_REQUEST_CHECKSUM_CALCULATION: when_required
+      AWS_RESPONSE_CHECKSUM_VALIDATION: when_required
     steps:
       - uses: actions/checkout@v4
         name: Checkout
@@ -111,20 +98,12 @@ jobs:
       # Setup C++ dependencies
       - run: ./cpp/setup_pipelines.sh
         name: Clone pipelines & vcpkg
-      - uses: TAServers/vcpkg-cache@v3
-        name: Setup vcpkg cache
-        id: vcpkg-cache
-        with:
-          token: ${{ secrets.GITHUB_TOKEN }}
 
       # Setup Python dependencies
       - name: Install uv
         uses: astral-sh/setup-uv@v5
       - name: Install the project
         run: uv sync --dev --verbose
-        env:
-          VCPKG_FEATURE_FLAGS: "binarycaching" # Possibly redundant, but explicitly sets the binary caching feature flag
-          VCPKG_BINARY_SOURCES: "clear;files,${{ steps.vcpkg-cache.outputs.path }},readwrite"
 
       # Setup git
       - name: Configure Git user
@@ -134,6 +113,6 @@ jobs:
 
       # Finally deploy docs
       - name: Get version
-        run: echo "EVALIO_VERSION=$(python -c 'import evalio; print(evalio.__version__)')" >> $GITHUB_ENV
+        run: echo "EVALIO_VERSION=$(python -c 'import evalio; print(evalio.__version__)')" >> "$GITHUB_ENV"
       - name: Deploy docs
         run: uv run mike deploy --push --update-aliases $EVALIO_VERSION stable

pyproject.toml

@@ -57,7 +57,12 @@ test-command = "evalio ls pipelines && pytest {project}/tests"
 build-verbosity = 1
 
 [tool.cibuildwheel.linux]
-before-all = "yum -y install curl zip unzip tar"
+before-all = """
+    yum -y install curl zip unzip tar
+    curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip"
+    unzip awscliv2.zip
+    ./aws/install
+"""
 
 # tbb requires at least 10.13, pcl 10.15. We bump to 11 for simplicity.
 [tool.cibuildwheel.macos.environment]