install diff-filter-build

fatteneder · fatteneder · commit 36db0ee6e248 · 2024-11-02T18:53:44.000+01:00
diff --git a/pipelines/main/launch_unsigned_jobs.yml b/pipelines/main/launch_unsigned_jobs.yml
@@ -12,6 +12,18 @@
 # and only need to touch the webui configuration when we need to alter
 # something about the privileged steps.
 
+common:
+  - diff-filter-build_plugin: &diff-filter-build
+      https://github.com/fatteneder/diff-filter-buildkite-plugin#main:
+        name: build
+        ignore:
+          - "*.md"
+          - "*.json"
+          - "CITATION.*"
+          - ".gitignore"
+          - ".clangd"
+          - ".mailmap"
+
 steps:
   - group: "Build"
     notify:
@@ -23,29 +35,32 @@ steps:
           - JuliaCI/external-buildkite#v1:
               version: "./.buildkite-external-version"
               repo_url: "https://github.com/JuliaCI/julia-buildkite"
+          - *diff-filter-build
         commands: |
-          # Launch Linux build jobs
-          GROUP="Build" \
-              ALLOW_FAIL="false" \
-              bash .buildkite/utilities/arches_pipeline_upload.sh \
-              .buildkite/pipelines/main/platforms/build_linux.arches \
-              .buildkite/pipelines/main/platforms/build_linux.yml
-          # Launch macOS packaging jobs
-          GROUP="Build" \
-              ALLOW_FAIL="false" \
-              bash .buildkite/utilities/arches_pipeline_upload.sh \
-              .buildkite/pipelines/main/platforms/build_macos.arches \
-              .buildkite/pipelines/main/platforms/build_macos.yml
-          GROUP="Build" \
-              ALLOW_FAIL="false" \
-              bash .buildkite/utilities/arches_pipeline_upload.sh \
-              .buildkite/pipelines/main/platforms/build_freebsd.arches \
-              .buildkite/pipelines/main/platforms/build_freebsd.yml
-          GROUP="Build" \
-              ALLOW_FAIL="false" \
-              bash .buildkite/utilities/arches_pipeline_upload.sh \
-              .buildkite/pipelines/main/platforms/build_windows.arches \
-              .buildkite/pipelines/main/platforms/build_windows.yml
+          if [[ $${BUILDKITE_PLUGIN_DIFF_FILTER_TRIGGERED_BUILD} == 1 ]]; then
+            # Launch Linux build jobs
+            GROUP="Build" \
+                ALLOW_FAIL="false" \
+                bash .buildkite/utilities/arches_pipeline_upload.sh \
+                .buildkite/pipelines/main/platforms/build_linux.arches \
+                .buildkite/pipelines/main/platforms/build_linux.yml
+            # Launch macOS packaging jobs
+            GROUP="Build" \
+                ALLOW_FAIL="false" \
+                bash .buildkite/utilities/arches_pipeline_upload.sh \
+                .buildkite/pipelines/main/platforms/build_macos.arches \
+                .buildkite/pipelines/main/platforms/build_macos.yml
+            GROUP="Build" \
+                ALLOW_FAIL="false" \
+                bash .buildkite/utilities/arches_pipeline_upload.sh \
+                .buildkite/pipelines/main/platforms/build_freebsd.arches \
+                .buildkite/pipelines/main/platforms/build_freebsd.yml
+            GROUP="Build" \
+                ALLOW_FAIL="false" \
+                bash .buildkite/utilities/arches_pipeline_upload.sh \
+                .buildkite/pipelines/main/platforms/build_windows.arches \
+                .buildkite/pipelines/main/platforms/build_windows.yml
+          fi
         agents:
           queue: "julia"
           os: "linux"
@@ -59,16 +74,19 @@ steps:
           - JuliaCI/external-buildkite#v1:
               version: "./.buildkite-external-version"
               repo_url: "https://github.com/JuliaCI/julia-buildkite"
+          - *diff-filter-build
         commands: |
-          buildkite-agent pipeline upload .buildkite/pipelines/main/misc/analyzegc.yml
-          buildkite-agent pipeline upload .buildkite/pipelines/main/misc/doctest.yml
-          buildkite-agent pipeline upload .buildkite/pipelines/main/misc/pdf_docs/build_pdf_docs.yml
-          buildkite-agent pipeline upload .buildkite/pipelines/main/misc/embedding.yml
-          buildkite-agent pipeline upload .buildkite/pipelines/main/misc/llvmpasses.yml
-          # buildkite-agent pipeline upload .buildkite/pipelines/main/misc/whitespace.yml # Currently runs in GitHub Actions instead of Buildkite
+          if [[ $${BUILDKITE_PLUGIN_DIFF_FILTER_TRIGGERED_BUILD} == 1 ]]; then
+            buildkite-agent pipeline upload .buildkite/pipelines/main/misc/analyzegc.yml
+            buildkite-agent pipeline upload .buildkite/pipelines/main/misc/doctest.yml
+            buildkite-agent pipeline upload .buildkite/pipelines/main/misc/pdf_docs/build_pdf_docs.yml
+            buildkite-agent pipeline upload .buildkite/pipelines/main/misc/embedding.yml
+            buildkite-agent pipeline upload .buildkite/pipelines/main/misc/llvmpasses.yml
+            # buildkite-agent pipeline upload .buildkite/pipelines/main/misc/whitespace.yml # Currently runs in GitHub Actions instead of Buildkite
 
-          buildkite-agent pipeline upload .buildkite/pipelines/main/misc/sanitizers/asan.yml
-          buildkite-agent pipeline upload .buildkite/pipelines/main/misc/sanitizers/tsan.yml
+            buildkite-agent pipeline upload .buildkite/pipelines/main/misc/sanitizers/asan.yml
+            buildkite-agent pipeline upload .buildkite/pipelines/main/misc/sanitizers/tsan.yml
+          fi
         agents:
           queue: "julia"
           os: "linux"
@@ -82,36 +100,39 @@ steps:
           - JuliaCI/external-buildkite#v1:
               version: "./.buildkite-external-version"
               repo_url: "https://github.com/JuliaCI/julia-buildkite"
+          - *diff-filter-build
         commands: |
           export ALLOW_FAIL="false"
-          buildkite-agent pipeline upload .buildkite/pipelines/main/misc/gcext.yml
-          buildkite-agent pipeline upload .buildkite/pipelines/main/misc/test_revise.yml
+          if [[ $${BUILDKITE_PLUGIN_DIFF_FILTER_TRIGGERED_BUILD} == 1 ]]; then
+            buildkite-agent pipeline upload .buildkite/pipelines/main/misc/gcext.yml
+            buildkite-agent pipeline upload .buildkite/pipelines/main/misc/test_revise.yml
 
-          # Launch Linux test jobs
-          GROUP="Test" \
-              bash .buildkite/utilities/arches_pipeline_upload.sh \
-              .buildkite/pipelines/main/platforms/test_linux.arches \
-              .buildkite/pipelines/main/platforms/test_linux.yml
-          # Launch i686-linux-gnu test jobs
-          GROUP="Test" \
-              bash .buildkite/utilities/arches_pipeline_upload.sh \
-              .buildkite/pipelines/main/platforms/test_linux.i686.arches \
-              .buildkite/pipelines/main/platforms/test_linux.i686.yml
-          # Launch macOS test jobs
-          GROUP="Test" \
-              bash .buildkite/utilities/arches_pipeline_upload.sh \
-              .buildkite/pipelines/main/platforms/test_macos.arches \
-              .buildkite/pipelines/main/platforms/test_macos.yml
-          # Launch FreeBSD test jobs
-          GROUP="Test" \
-              bash .buildkite/utilities/arches_pipeline_upload.sh \
-              .buildkite/pipelines/main/platforms/test_freebsd.arches \
-              .buildkite/pipelines/main/platforms/test_freebsd.yml
-          # Launch windows test jobs
-          GROUP="Test" \
-              bash .buildkite/utilities/arches_pipeline_upload.sh \
-              .buildkite/pipelines/main/platforms/test_windows.arches \
-              .buildkite/pipelines/main/platforms/test_windows.yml
+            # Launch Linux test jobs
+            GROUP="Test" \
+                bash .buildkite/utilities/arches_pipeline_upload.sh \
+                .buildkite/pipelines/main/platforms/test_linux.arches \
+                .buildkite/pipelines/main/platforms/test_linux.yml
+            # Launch i686-linux-gnu test jobs
+            GROUP="Test" \
+                bash .buildkite/utilities/arches_pipeline_upload.sh \
+                .buildkite/pipelines/main/platforms/test_linux.i686.arches \
+                .buildkite/pipelines/main/platforms/test_linux.i686.yml
+            # Launch macOS test jobs
+            GROUP="Test" \
+                bash .buildkite/utilities/arches_pipeline_upload.sh \
+                .buildkite/pipelines/main/platforms/test_macos.arches \
+                .buildkite/pipelines/main/platforms/test_macos.yml
+            # Launch FreeBSD test jobs
+            GROUP="Test" \
+                bash .buildkite/utilities/arches_pipeline_upload.sh \
+                .buildkite/pipelines/main/platforms/test_freebsd.arches \
+                .buildkite/pipelines/main/platforms/test_freebsd.yml
+            # Launch windows test jobs
+            GROUP="Test" \
+                bash .buildkite/utilities/arches_pipeline_upload.sh \
+                .buildkite/pipelines/main/platforms/test_windows.arches \
+                .buildkite/pipelines/main/platforms/test_windows.yml
+          fi
         agents:
           queue: "julia"
           os: "linux"
@@ -122,18 +143,21 @@ steps:
           - JuliaCI/external-buildkite#v1:
               version: "./.buildkite-external-version"
               repo_url: "https://github.com/JuliaCI/julia-buildkite"
+          - *diff-filter-build
         commands: |
           export ALLOW_FAIL="true"
-          # Launch Linux allowed-to-fail build jobs
-          GROUP="Allow Fail" \
-              bash .buildkite/utilities/arches_pipeline_upload.sh \
-              .buildkite/pipelines/main/platforms/build_linux.soft_fail.arches \
-              .buildkite/pipelines/main/platforms/build_linux.yml
-          # Launch macOS allowed-to-fail build jobs
-          GROUP="Allow Fail" \
-              bash .buildkite/utilities/arches_pipeline_upload.sh \
-              .buildkite/pipelines/main/platforms/build_macos.soft_fail.arches \
-              .buildkite/pipelines/main/platforms/build_macos.yml
+          if [[ $${BUILDKITE_PLUGIN_DIFF_FILTER_TRIGGERED_BUILD} == 1 ]]; then
+            # Launch Linux allowed-to-fail build jobs
+            GROUP="Allow Fail" \
+                bash .buildkite/utilities/arches_pipeline_upload.sh \
+                .buildkite/pipelines/main/platforms/build_linux.soft_fail.arches \
+                .buildkite/pipelines/main/platforms/build_linux.yml
+            # Launch macOS allowed-to-fail build jobs
+            GROUP="Allow Fail" \
+                bash .buildkite/utilities/arches_pipeline_upload.sh \
+                .buildkite/pipelines/main/platforms/build_macos.soft_fail.arches \
+                .buildkite/pipelines/main/platforms/build_macos.yml
+          fi
         agents:
           queue: "julia"
           os: "linux"
@@ -142,29 +166,31 @@ steps:
           - JuliaCI/external-buildkite#v1:
               version: "./.buildkite-external-version"
               repo_url: "https://github.com/JuliaCI/julia-buildkite"
+          - *diff-filter-build
         commands: |
           export ALLOW_FAIL="true"
-
-          # Launch Linux allowed-to-fail test jobs
-          GROUP="Allow Fail" \
-              bash .buildkite/utilities/arches_pipeline_upload.sh \
-              .buildkite/pipelines/main/platforms/test_linux.soft_fail.arches \
-              .buildkite/pipelines/main/platforms/test_linux.yml
-          # Launch macOS allowed-to-fail test jobs
-          GROUP="Allow Fail" \
-              bash .buildkite/utilities/arches_pipeline_upload.sh \
-              .buildkite/pipelines/main/platforms/test_macos.soft_fail.arches \
-              .buildkite/pipelines/main/platforms/test_macos.yml
-          # Launch FreeBSD allowed-to-fail jobs
-          GROUP="Allow Fail" \
-              bash .buildkite/utilities/arches_pipeline_upload.sh \
-              .buildkite/pipelines/main/platforms/test_freebsd.soft_fail.arches \
-              .buildkite/pipelines/main/platforms/test_freebsd.yml
-          # Launch windows allowed-to-fail test jobs
-          GROUP="Allow Fail" \
-              bash .buildkite/utilities/arches_pipeline_upload.sh \
-              .buildkite/pipelines/main/platforms/test_windows.soft_fail.arches \
-              .buildkite/pipelines/main/platforms/test_windows.yml
+          if [[ $${BUILDKITE_PLUGIN_DIFF_FILTER_TRIGGERED_BUILD} == 1 ]]; then
+            # Launch Linux allowed-to-fail test jobs
+            GROUP="Allow Fail" \
+                bash .buildkite/utilities/arches_pipeline_upload.sh \
+                .buildkite/pipelines/main/platforms/test_linux.soft_fail.arches \
+                .buildkite/pipelines/main/platforms/test_linux.yml
+            # Launch macOS allowed-to-fail test jobs
+            GROUP="Allow Fail" \
+                bash .buildkite/utilities/arches_pipeline_upload.sh \
+                .buildkite/pipelines/main/platforms/test_macos.soft_fail.arches \
+                .buildkite/pipelines/main/platforms/test_macos.yml
+            # Launch FreeBSD allowed-to-fail jobs
+            GROUP="Allow Fail" \
+                bash .buildkite/utilities/arches_pipeline_upload.sh \
+                .buildkite/pipelines/main/platforms/test_freebsd.soft_fail.arches \
+                .buildkite/pipelines/main/platforms/test_freebsd.yml
+            # Launch windows allowed-to-fail test jobs
+            GROUP="Allow Fail" \
+                bash .buildkite/utilities/arches_pipeline_upload.sh \
+                .buildkite/pipelines/main/platforms/test_windows.soft_fail.arches \
+                .buildkite/pipelines/main/platforms/test_windows.yml
+          fi
         agents:
           queue: "julia"
           os: "linux"
diff --git a/pipelines/scheduled/launch_unsigned_jobs.yml b/pipelines/scheduled/launch_unsigned_jobs.yml
@@ -12,6 +12,18 @@
 # and only need to touch the webui configuration when we need to alter
 # something about the privileged steps.
 
+common:
+  - diff-filter-build_plugin: &diff-filter-build
+      https://github.com/fatteneder/diff-filter-buildkite-plugin#main:
+        name: build
+        ignore:
+          - "*.md"
+          - "*.json"
+          - "CITATION.*"
+          - ".gitignore"
+          - ".clangd"
+          - ".mailmap"
+
 steps:
   - group: "Source Build"
     steps:
@@ -20,12 +32,15 @@ steps:
           - JuliaCI/external-buildkite#v1:
               version: "./.buildkite-external-version"
               repo_url: "https://github.com/JuliaCI/julia-buildkite"
+          - *diff-filter-build
         commands: |
-          GROUP="Source Build" \
-              ALLOW_FAIL="false" \
-              bash .buildkite/utilities/arches_pipeline_upload.sh \
-              .buildkite/pipelines/scheduled/platforms/build_linux.schedule.arches \
-              .buildkite/pipelines/main/platforms/build_linux.yml
+          if [[ $${BUILDKITE_PLUGIN_DIFF_FILTER_TRIGGERED_BUILD} == 1 ]]; then
+            GROUP="Source Build" \
+                ALLOW_FAIL="false" \
+                bash .buildkite/utilities/arches_pipeline_upload.sh \
+                .buildkite/pipelines/scheduled/platforms/build_linux.schedule.arches \
+                .buildkite/pipelines/main/platforms/build_linux.yml
+          fi
         agents:
           queue: "julia"
           os: "linux"
@@ -37,11 +52,13 @@ steps:
               version: "./.buildkite-external-version"
               repo_url: "https://github.com/JuliaCI/julia-buildkite"
         commands: |
-          GROUP="Source Tests (Allow Fail)" \
-              ALLOW_FAIL="true" \
-              bash .buildkite/utilities/arches_pipeline_upload.sh \
-              .buildkite/pipelines/scheduled/platforms/test_linux.schedule.arches \
-              .buildkite/pipelines/main/platforms/test_linux.yml
+          if [[ $${BUILDKITE_PLUGIN_DIFF_FILTER_TRIGGERED_BUILD} == 1 ]]; then
+            GROUP="Source Tests (Allow Fail)" \
+                ALLOW_FAIL="true" \
+                bash .buildkite/utilities/arches_pipeline_upload.sh \
+                .buildkite/pipelines/scheduled/platforms/test_linux.schedule.arches \
+                .buildkite/pipelines/main/platforms/test_linux.yml
+          fi
         agents:
           queue: "julia"
           os: "linux"
@@ -53,20 +70,22 @@ steps:
               version: "./.buildkite-external-version"
               repo_url: "https://github.com/JuliaCI/julia-buildkite"
         commands: |
-          GROUP="no_GPL" \
-              ALLOW_FAIL="false" \
-              bash .buildkite/utilities/arches_pipeline_upload.sh \
-              .buildkite/pipelines/scheduled/platforms/build_linux.no_gpl.arches \
-              .buildkite/pipelines/main/platforms/build_linux.yml
-          GROUP="no_GPL" \
-              ALLOW_FAIL="false" \
-              bash .buildkite/utilities/arches_pipeline_upload.sh \
-              .buildkite/pipelines/scheduled/platforms/build_macos.no_gpl.arches \
-              .buildkite/pipelines/main/platforms/build_macos.yml
-          GROUP="no_GPL" \
-              ALLOW_FAIL="false" \
-              bash .buildkite/utilities/arches_pipeline_upload.sh \
-              .buildkite/pipelines/scheduled/platforms/build_windows.no_gpl.arches \
-              .buildkite/pipelines/main/platforms/build_windows.yml
+          if [[ $${BUILDKITE_PLUGIN_DIFF_FILTER_TRIGGERED_BUILD} == 1 ]]; then
+            GROUP="no_GPL" \
+                ALLOW_FAIL="false" \
+                bash .buildkite/utilities/arches_pipeline_upload.sh \
+                .buildkite/pipelines/scheduled/platforms/build_linux.no_gpl.arches \
+                .buildkite/pipelines/main/platforms/build_linux.yml
+            GROUP="no_GPL" \
+                ALLOW_FAIL="false" \
+                bash .buildkite/utilities/arches_pipeline_upload.sh \
+                .buildkite/pipelines/scheduled/platforms/build_macos.no_gpl.arches \
+                .buildkite/pipelines/main/platforms/build_macos.yml
+            GROUP="no_GPL" \
+                ALLOW_FAIL="false" \
+                bash .buildkite/utilities/arches_pipeline_upload.sh \
+                .buildkite/pipelines/scheduled/platforms/build_windows.no_gpl.arches \
+                .buildkite/pipelines/main/platforms/build_windows.yml
+          fi
         agents:
           queue: "julia"
