[no merge, testing premerge issues]

.bazelrc

@@ -1,12 +1,11 @@
 # Must be first. Enables build:windows, build:linux, build:macos, build:freebsd, build:openbsd
 build --enable_platform_specific_config
 
-build:linux --workspace_status_command="bash ./bazel/workspace_status.sh"
-
-# Provides users an option to turn on strict action env.
-# TODO(aslonnie): make this default; fix the python tests..
+build --incompatible_strict_action_env
 build:strict --incompatible_strict_action_env
 
+build:linux --workspace_status_command="bash ./bazel/workspace_status.sh"
+
 # To distinguish different incompatible environments.
 build --action_env=RAY_BUILD_ENV
 
@@ -28,6 +27,7 @@ build:macos --cxxopt="-std=c++17"
 build:clang-cl --cxxopt="-std=c++17"
 build:msvc-cl --cxxopt="/std:c++17"
 build:windows --cxxopt="/std:c++17"
+build:windows --cxxopt="/utf-8"
 # This workaround is needed to prevent Bazel from compiling the same file twice (once PIC and once not).
 build:linux --copt="-fPIC"
 build:macos --copt="-fPIC"
@@ -81,8 +81,6 @@ build:iwyu --output_groups=report
 build:windows --attempt_to_print_relative_paths
 # Save disk space by hardlinking cache hits instead of copying
 build:windows --experimental_repository_cache_hardlinks
-# Clean the environment before building, to make builds more deterministic
-build:windows --incompatible_strict_action_env
 # For colored output (seems necessary on Windows)
 build:windows --color=yes
 # For compiler colored output (seems necessary on Windows)
@@ -165,6 +163,18 @@ test:ci-base --test_output=errors
 test:ci-base --test_verbose_timeout_warnings
 test:ci-base --flaky_test_attempts=3
 
+# Sending in PATH is required for tests to run on CI, after we enable
+# --incompatible_strict_action_env, until we either convert all Python tests to
+# hermetic tests -- which not only requires pinning all Python dependencies with bazel,
+# but also requires building ray(test) wheel with bazel. Alternatively, we can
+# also stop using bazel test to run ray's Python tests.
+#
+# This PATH test_env is intentionally not enabled on non-CI so that C/C++
+# tests, which are all hermetic, can build, test and cache as intended, ray
+# Python developers do not really use bazel test to run tests locally, but more
+# often just run tests with "pytest" directly.
+test:ci-base --test_env=PATH
+
 build:ci --color=yes
 build:ci --curses=no
 build:ci --keep_going
@@ -220,6 +230,9 @@ build:cgroup --sandbox_writable_path=/sys/fs/cgroup --config=llvm
 # ci/env/install-llvm-dependencies.sh
 try-import %workspace%/.llvm-local.bazelrc
 
+# Allow users to define custom options.
+try-import %workspace%/.user.bazelrc
+
 # Even with sandbox mode bazel prioritizes system headers over the ones in the sandbox.
 # It picks up the system headers when someone has protobuf installed via Homebrew.
 # Work around for https://github.com/bazelbuild/bazel/issues/8053

.buildkite/_forge.rayci.yml

@@ -1,100 +1,8 @@
 group: forge
+sort_key: "_forge"
 steps:
   - name: forge
     wanda: ci/docker/forge.wanda.yaml
 
   - name: manylinux
     wanda: ci/docker/manylinux.wanda.yaml
-
-  - name: raycudabase
-    label: "wanda: ray.py{{matrix.python}}.cu{{matrix.cuda}}.base"
-    tags:
-      - python_dependencies
-      - docker
-    wanda: ci/docker/ray.cuda.base.wanda.yaml
-    matrix:
-      setup:
-        python:
-          - "3.9"
-          - "3.10"
-          - "3.11"
-          - "3.12"
-        cuda:
-          - "11.7.1-cudnn8"
-          - "11.8.0-cudnn8"
-          - "12.1.1-cudnn8"
-          - "12.3.2-cudnn9"
-          - "12.4.1-cudnn"
-          - "12.5.1-cudnn"
-          - "12.8.1-cudnn"
-    env:
-      PYTHON_VERSION: "{{matrix.python}}"
-      CUDA_VERSION: "{{matrix.cuda}}"
-
-
-  - name: raycpubase
-    label: "wanda: ray.py{{matrix}}.cpu.base"
-    tags:
-      - python_dependencies
-      - python
-      - docker
-      - tune
-      - serve
-    wanda: ci/docker/ray.cpu.base.wanda.yaml
-    matrix:
-      - "3.9"
-      - "3.10"
-      - "3.11"
-      - "3.12"
-    env:
-      PYTHON_VERSION: "{{matrix}}"
-
-  - name: ray-llmbase
-    label: "wanda: ray-llm.py{{matrix.python}}.cu{{matrix.cuda}}.base"
-    tags:
-      - python_dependencies
-      - docker
-    wanda: ci/docker/ray-llm.base.wanda.yaml
-    depends_on: raycudabase
-    matrix:
-      setup:
-        python:
-          - "3.11"
-        cuda:
-          - "12.8.1-cudnn"
-    env:
-      PYTHON_VERSION: "{{matrix.python}}"
-      CUDA_VERSION: "{{matrix.cuda}}"
-
-  - name: ray-mlcudabase
-    label: "wanda: ray-ml.py{{matrix.python}}.cu{{matrix.cuda}}.base"
-    tags:
-      - python_dependencies
-      - docker
-    wanda: ci/docker/ray-ml.cuda.base.wanda.yaml
-    depends_on: raycudabase
-    matrix:
-      setup:
-        python:
-          - "3.9"
-          - "3.10"
-          - "3.11"
-        cuda:
-          - "12.1.1-cudnn8"
-    env:
-      PYTHON_VERSION: "{{matrix.python}}"
-      CUDA_VERSION: "{{matrix.cuda}}"
-
-  - name: ray-mlcpubase
-    label: "wanda: ray-ml.py{{matrix}}.cpu.base"
-    tags:
-      - python_dependencies
-      - docker
-    wanda: ci/docker/ray-ml.cpu.base.wanda.yaml
-    depends_on: raycpubase
-    matrix:
-      - "3.9"
-      - "3.10"
-      - "3.11"
-    env:
-      PYTHON_VERSION: "{{matrix}}"

.buildkite/_images.rayci.yml

@@ -0,0 +1,225 @@
+group: images
+sort_key: "_images"
+steps:
+  - name: raycpubase
+    label: "wanda: ray-py{{matrix}}-cpu-base"
+    tags:
+      - python_dependencies
+      - docker
+    wanda: docker/base-deps/cpu.wanda.yaml
+    matrix:
+      - "3.9"
+      - "3.10"
+      - "3.11"
+      - "3.12"
+    env:
+      PYTHON_VERSION: "{{matrix}}"
+      ARCH_SUFFIX: ""
+
+  - name: raycpubaseextra
+    label: "wanda: ray-py{{matrix}}-cpu-base-extra"
+    wanda: docker/base-extra/cpu.wanda.yaml
+    matrix:
+      - "3.9"
+      - "3.10"
+      - "3.11"
+      - "3.12"
+    env:
+      PYTHON_VERSION: "{{matrix}}"
+      IMAGE_TYPE: "ray"
+      ARCH_SUFFIX: ""
+    depends_on: raycpubase
+
+  - name: raycudabase
+    label: "wanda: ray-py{{matrix.python}}-cu{{matrix.cuda}}-base"
+    tags:
+      - python_dependencies
+      - docker
+    wanda: docker/base-deps/cuda.wanda.yaml
+    matrix:
+      setup:
+        python:
+          - "3.9"
+          - "3.10"
+          - "3.11"
+          - "3.12"
+        cuda:
+          - "11.7.1-cudnn8"
+          - "11.8.0-cudnn8"
+          - "12.1.1-cudnn8"
+          - "12.3.2-cudnn9"
+          - "12.4.1-cudnn"
+          - "12.5.1-cudnn"
+          - "12.6.3-cudnn"
+          - "12.8.1-cudnn"
+    env:
+      PYTHON_VERSION: "{{matrix.python}}"
+      CUDA_VERSION: "{{matrix.cuda}}"
+      ARCH_SUFFIX: ""
+
+  - name: raycudabaseextra
+    label: "wanda: ray-py{{matrix.python}}-cu{{matrix.cuda}}-base-extra"
+    wanda: docker/base-extra/cuda.wanda.yaml
+    matrix:
+      setup:
+        python:
+          - "3.9"
+          - "3.10"
+          - "3.11"
+          - "3.12"
+        cuda:
+          - "11.7.1-cudnn8"
+          - "11.8.0-cudnn8"
+          - "12.1.1-cudnn8"
+          - "12.3.2-cudnn9"
+          - "12.4.1-cudnn"
+          - "12.5.1-cudnn"
+          - "12.6.3-cudnn"
+          - "12.8.1-cudnn"
+    env:
+      PYTHON_VERSION: "{{matrix.python}}"
+      CUDA_VERSION: "{{matrix.cuda}}"
+      IMAGE_TYPE: "ray"
+      ARCH_SUFFIX: ""
+    depends_on: raycudabase
+
+  - name: ray-llmbase
+    label: "wanda: ray-llm-py{{matrix.python}}-cu{{matrix.cuda}}-base"
+    tags:
+      - python_dependencies
+      - docker
+    wanda: docker/ray-llm/cuda.wanda.yaml
+    depends_on: raycudabase
+    matrix:
+      setup:
+        python:
+          - "3.11"
+        cuda:
+          - "12.8.1-cudnn"
+    env:
+      PYTHON_VERSION: "{{matrix.python}}"
+      CUDA_VERSION: "{{matrix.cuda}}"
+
+  - name: ray-llmbaseextra
+    label: "wanda: ray-llm-py{{matrix.python}}-cu{{matrix.cuda}}-base-extra"
+    wanda: docker/base-extra/cuda.wanda.yaml
+    matrix:
+      setup:
+        python:
+          - "3.11"
+        cuda:
+          - "12.8.1-cudnn"
+    env:
+      PYTHON_VERSION: "{{matrix.python}}"
+      CUDA_VERSION: "{{matrix.cuda}}"
+      IMAGE_TYPE: "ray-llm"
+      ARCH_SUFFIX: ""
+    depends_on: ray-llmbase
+
+  - name: ray-mlcpubase
+    label: "wanda: ray-ml-py{{matrix}}-cpu-base"
+    tags:
+      - python_dependencies
+      - docker
+    wanda: docker/ray-ml/cpu.wanda.yaml
+    depends_on: raycpubase
+    matrix:
+      - "3.9"
+      - "3.10"
+      - "3.11"
+    env:
+      PYTHON_VERSION: "{{matrix}}"
+
+  - name: ray-mlcpubaseextra
+    label: "wanda: ray-ml-py{{matrix}}-cpu-base-extra"
+    wanda: docker/base-extra/cpu.wanda.yaml
+    matrix:
+      - "3.9"
+      - "3.10"
+      - "3.11"
+    env:
+      PYTHON_VERSION: "{{matrix}}"
+      IMAGE_TYPE: "ray-ml"
+      ARCH_SUFFIX: ""
+    depends_on: ray-mlcpubase
+
+  - name: ray-mlcudabase
+    label: "wanda: ray-ml-py{{matrix.python}}-cu{{matrix.cuda}}-base"
+    tags:
+      - python_dependencies
+      - docker
+    wanda: docker/ray-ml/cuda.wanda.yaml
+    depends_on: raycudabase
+    matrix:
+      setup:
+        python:
+          - "3.9"
+          - "3.10"
+          - "3.11"
+        cuda:
+          - "12.1.1-cudnn8"
+    env:
+      PYTHON_VERSION: "{{matrix.python}}"
+      CUDA_VERSION: "{{matrix.cuda}}"
+
+  - name: ray-mlcudabaseextra
+    label: "wanda: ray-ml-py{{matrix.python}}-cu{{matrix.cuda}}-base-extra"
+    wanda: docker/base-extra/cuda.wanda.yaml
+    matrix:
+      setup:
+        python:
+          - "3.9"
+          - "3.10"
+          - "3.11"
+        cuda:
+          - "12.1.1-cudnn8"
+    env:
+      PYTHON_VERSION: "{{matrix.python}}"
+      CUDA_VERSION: "{{matrix.cuda}}"
+      IMAGE_TYPE: "ray-ml"
+      ARCH_SUFFIX: ""
+    depends_on: ray-mlcudabase
+
+  - name: ray-slimcpubase
+    label: "wanda: ray-slim-py{{matrix}}-cpu-base"
+    tags:
+      - python_dependencies
+      - docker
+      - skip-on-release-tests
+    wanda: docker/base-slim/cpu.wanda.yaml
+    matrix:
+      - "3.9"
+      - "3.10"
+      - "3.11"
+      - "3.12"
+    env:
+      PYTHON_VERSION: "{{matrix}}"
+      ARCH_SUFFIX: ""
+
+  - name: ray-slimcudabase
+    label: "wanda: ray-slim-py{{matrix.python}}-cu{{matrix.cuda}}-base"
+    tags:
+      - python_dependencies
+      - docker
+      - skip-on-release-tests
+    wanda: docker/base-slim/cuda.wanda.yaml
+    matrix:
+      setup:
+        python:
+          - "3.9"
+          - "3.10"
+          - "3.11"
+          - "3.12"
+        cuda:
+          - "11.7.1"
+          - "11.8.0"
+          - "12.1.1"
+          - "12.3.2"
+          - "12.4.1"
+          - "12.5.1"
+          - "12.6.3"
+          - "12.8.1"
+    env:
+      PYTHON_VERSION: "{{matrix.python}}"
+      CUDA_VERSION: "{{matrix.cuda}}"
+      ARCH_SUFFIX: ""