From 7a0401fb8ee05addf1f69fcec70af3894ce27260 Mon Sep 17 00:00:00 2001
From: Etienne Perot <eperot@google.com>
Date: Tue, 22 Oct 2024 18:00:52 -0700
Subject: [PATCH] Run CUDA tests as part of GPU tests.

This runs in continuous mode only.

PiperOrigin-RevId: 688755446
---
 .buildkite/pipeline.yaml | 10 ++++++++++
 .buildkite/release.yaml  |  9 +++++++++
 test/gpu/cuda_test.go    |  2 +-
 3 files changed, 20 insertions(+), 1 deletion(-)

diff --git a/.buildkite/pipeline.yaml b/.buildkite/pipeline.yaml
index 030a99d924..f0b6b34b64 100644
--- a/.buildkite/pipeline.yaml
+++ b/.buildkite/pipeline.yaml
@@ -190,6 +190,16 @@ steps:
       - make cos-gpu-all-tests
     agents:
       queue: cos-canary-gpu
+  - <<: *common
+    <<: *source_test_continuous
+    label: ":fish: CUDA tests"
+    # This is its own test rather than being part of the GPU tests,
+    # because it takes around 15 minutes to run.
+    commands:
+      - make sudo TARGETS=//tools/gpu:main ARGS="install --latest" || cat /var/log/nvidia-installer.log
+      - make sudo TARGETS=//test/gpu:cuda_test
+    agents:
+      queue: gpu
   - <<: *common
     <<: *source_test_continuous
     label: ":screwdriver: All GPU Drivers Test"
diff --git a/.buildkite/release.yaml b/.buildkite/release.yaml
index e913feb685..cb96a60df9 100644
--- a/.buildkite/release.yaml
+++ b/.buildkite/release.yaml
@@ -73,3 +73,12 @@ steps:
       - tools/gpu/all_drivers_test.sh
     agents:
       queue: gpu
+  - <<: *common
+    label: ":fish: CUDA tests in compatibility-verifying mode"
+    # This is its own test rather than being part of the GPU tests,
+    # because it takes around an hour to run.
+    commands:
+      - make sudo TARGETS=//tools/gpu:main ARGS="install --latest" || cat /var/log/nvidia-installer.log
+      - make sudo TARGETS=//test/gpu:cuda_test ARGS="--cuda_verify_compatibility=true"
+    agents:
+      queue: gpu
diff --git a/test/gpu/cuda_test.go b/test/gpu/cuda_test.go
index dec522b0fa..03906e68b1 100644
--- a/test/gpu/cuda_test.go
+++ b/test/gpu/cuda_test.go
@@ -745,7 +745,7 @@ func TestCUDA(t *testing.T) {
 		durationPct := 100.0 * float64(startedAgo) / float64(testDeadline.Sub(testStart))
 		testLog(t, "[Timing] %d/%d tests (%.1f%%) finished executing. Test started %v ago, deadline in %v (%.1f%%).", testsDone, numTests, donePct, startedAgo.Truncate(time.Second), deadlineIn.Truncate(time.Second), durationPct)
 		if len(failedTests) > 0 {
-			testLog(t, "[Failed] %d test failed: %v", len(failedTests), strings.Join(failedTests, ", "))
+			testLog(t, "[Failed] %d test(s) failed: %v", len(failedTests), strings.Join(failedTests, ", "))
 		}
 		testLog(t, "[Pool] %v", cp.String())
 	}