Merge branch 'master' into fix-detached-actor-killed-on-owner-exit

Author: rueian

.buildkite/BUILD.bazel

@@ -0,0 +1,12 @@
+load("@rules_python//python:defs.bzl", "py_binary")
+load("@py_deps_buildkite//:requirements.bzl", ci_require = "requirement")
+
+py_binary(
+    name = "copy_files",
+    srcs = ["copy_files.py"],
+    visibility = ["//visibility:private"],
+    deps = [
+        ci_require("boto3"),
+        ci_require("aws_requests_auth"),
+    ],
+)

.buildkite/build.rayci.yml

@@ -3,6 +3,7 @@ steps:
   - label: ":tapioca: build: wheel {{matrix}} (x86_64)"
     key: linux_wheels
     tags:
+      - release_wheels
       - linux_wheels
       - oss
     instance_type: large
@@ -22,7 +23,7 @@ steps:
     tags:
       - linux_wheels
       - oss
-    instance_type: medium
+    instance_type: large
     commands:
       - bazel run //ci/ray_ci:build_in_docker -- wheel --build-type debug --upload
     depends_on:
@@ -55,6 +56,7 @@ steps:
       - skip-on-premerge
 
   - label: ":tapioca: build: ray py{{matrix}} docker (x86_64)"
+    key: ray_images
     tags:
       - python_dependencies
       - docker
@@ -126,3 +128,16 @@ steps:
       - docker run -ti --rm rayproject/ray:dev python -c "import ray; print(ray.__version__)"
     depends_on:
       - forge
+
+  - label: ":tapioca: generate nightly indexes"
+    instance_type: small
+    tags:
+      - docker
+      - oss
+      - skip-on-premerge
+    commands:
+      - bazel run //ci/ray_ci/automation:generate_index -- --prefix nightly
+    depends_on:
+      - ray_images
+      - ray_images_aarch64
+      - forge

.buildkite/core.rayci.yml

@@ -56,11 +56,20 @@ steps:
       - bazel run //ci/ray_ci:test_in_docker -- //python/ray/tests/... //python/ray/_common/tests/... //python/ray/dag/... //python/ray/autoscaler/v2/... core
         --install-mask all-ray-libraries
         --workers "$${BUILDKITE_PARALLEL_JOB_COUNT}" --worker-id "$${BUILDKITE_PARALLEL_JOB}" --parallelism-per-worker 3
-        --except-tags debug_tests,asan_tests,post_wheel_build,ha_integration,mem_pressure,tmpfs,container,manual,use_all_core,multi_gpu
+        --except-tags debug_tests,asan_tests,post_wheel_build,ha_integration,mem_pressure,tmpfs,container,manual,multi_gpu,spark_on_ray,ray_client,compiled_graphs,dask
+        --install-mask all-ray-libraries
+
+  - label: ":ray: core: cgraph python tests"
+    tags:
+      - compiled_graphs
+    instance_type: large
+    commands:
+      - bazel run //ci/ray_ci:test_in_docker -- //python/ray/dag/... core
+        --install-mask all-ray-libraries
+        --workers "$${BUILDKITE_PARALLEL_JOB_COUNT}" --worker-id "$${BUILDKITE_PARALLEL_JOB}" --parallelism-per-worker 3
+        --only-tags compiled_graphs
+        --except-tags multi_gpu
         --install-mask all-ray-libraries
-      - bazel run //ci/ray_ci:test_in_docker -- //python/ray/tests/... //python/ray/dag/... //python/ray/autoscaler/v2/... core
-        --workers "$${BUILDKITE_PARALLEL_JOB_COUNT}" --worker-id "$${BUILDKITE_PARALLEL_JOB}"
-        --only-tags use_all_core --skip-ray-installation
 
   - label: ":ray: core: python {{matrix.python}} tests ({{matrix.worker_id}})"
     if: build.pull_request.labels includes "continuous-build" || pipeline.id == "0189e759-8c96-4302-b6b5-b4274406bf89" || pipeline.id == "018f4f1e-1b73-4906-9802-92422e3badaa"
@@ -73,18 +82,23 @@ steps:
         --install-mask all-ray-libraries
         --workers 4 --worker-id "{{matrix.worker_id}}" --parallelism-per-worker 3
         --python-version {{matrix.python}}
-        --except-tags debug_tests,asan_tests,post_wheel_build,ha_integration,mem_pressure,tmpfs,container,manual,use_all_core,multi_gpu
-      - bazel run //ci/ray_ci:test_in_docker -- //python/ray/tests/... //python/ray/dag/... //python/ray/autoscaler/v2/... core
-        --install-mask all-ray-libraries
-        --workers 4 --worker-id "{{matrix.worker_id}}" --parallelism-per-worker 3
-        --python-version {{matrix.python}}
-        --only-tags use_all_core --skip-ray-installation
+        --except-tags debug_tests,asan_tests,post_wheel_build,ha_integration,mem_pressure,tmpfs,container,manual,multi_gpu,spark_on_ray,ray_client,dask
     depends_on: corebuild-multipy
     matrix:
       setup:
         python: ["3.12"]
         worker_id: ["0", "1", "2", "3"]
 
+  - label: ":ray: core: ray client tests"
+    tags:
+      - ray_client
+    instance_type: large
+    commands:
+      - bazel run //ci/ray_ci:test_in_docker -- //python/ray/tests/...   core
+        --install-mask all-ray-libraries
+        --workers "$${BUILDKITE_PARALLEL_JOB_COUNT}" --worker-id "$${BUILDKITE_PARALLEL_JOB}" --parallelism-per-worker 3
+        --only-tags ray_client
+
   - label: ":ray: core: redis tests"
     tags:
       - python
@@ -97,10 +111,7 @@ steps:
         --install-mask all-ray-libraries
         --workers "$${BUILDKITE_PARALLEL_JOB_COUNT}" --worker-id "$${BUILDKITE_PARALLEL_JOB}" --parallelism-per-worker 3
         --test-env=TEST_EXTERNAL_REDIS=1
-        --except-tags debug_tests,asan_tests,post_wheel_build,ha_integration,mem_pressure,tmpfs,container,manual,use_all_core,multi_gpu
-      - bazel run //ci/ray_ci:test_in_docker -- //python/ray/tests/... //python/ray/dag/... //python/ray/autoscaler/v2/... core
-        --workers "$${BUILDKITE_PARALLEL_JOB_COUNT}" --worker-id "$${BUILDKITE_PARALLEL_JOB}"
-        --only-tags use_all_core --skip-ray-installation
+        --except-tags debug_tests,asan_tests,post_wheel_build,ha_integration,mem_pressure,tmpfs,container,manual,multi_gpu,spark_on_ray,ray_client,dask
 
   - label: ":ray: core: memory pressure tests"
     tags:
@@ -176,8 +187,12 @@ steps:
         --except-tags gpu
         --skip-ray-installation
 
-  - label: ":ray: core: data tests"
-    tags: python
+  - label: ":ray: core: dask & modin tests"
+    tags:
+      # These tests are only triggered on premerge if there are changes under
+      # `ray/util/dask/`. This is not technically related to modin, but modin tests can
+      # run postmerge-only and are too small for their own build.
+      - dask
     instance_type: medium
     commands:
       - bazel run //ci/ray_ci:test_in_docker --
@@ -202,8 +217,10 @@ steps:
         "$${RAYCI_WORK_REPO}":"$${RAYCI_BUILD_ID}"-corebuild /bin/bash -iecuo pipefail
         "./python/ray/dashboard/tests/run_ui_tests.sh"
 
-  - label: ":ray: core: debug test"
-    tags: python
+  - label: ":ray: core: debug tests"
+    tags:
+      - python
+      - skip-on-premerge
     instance_type: medium
     commands:
       - bazel run //ci/ray_ci:test_in_docker -- //python/ray/tests/... core
@@ -216,6 +233,7 @@ steps:
   - label: ":ray: core: asan tests"
     tags:
       - python
+      - skip-on-premerge
     instance_type: large
     commands:
       - bazel run //ci/ray_ci:test_in_docker -- //python/ray/tests/... core
@@ -306,12 +324,21 @@ steps:
       - bazel run //ci/ray_ci:test_in_docker -- //:all //src/... core --except-tags=cgroup --build-type clang
         --cache-test-results --parallelism-per-worker 2
 
+  # block on premerge and microcheck
+  - block: "run cpp sanitizer tests"
+    if: build.env("BUILDKITE_PIPELINE_ID") == "0189942e-0876-4b8f-80a4-617f988ec59b" || build.env("BUILDKITE_PIPELINE_ID") == "018f4f1e-1b73-4906-9802-92422e3badaa"
+    key: block-core-cpp-sanitizer-tests
+    depends_on: []
+
   - label: ":ray: core: cpp asan tests"
     tags: core_cpp
     instance_type: medium
     commands:
       - bazel run //ci/ray_ci:test_in_docker -- //:all //src/... core --except-tags=cgroup
         --build-type asan-clang --cache-test-results --parallelism-per-worker 2
+    depends_on:
+      - block-core-cpp-sanitizer-tests
+      - corebuild
 
   - label: ":ray: core: cpp ubsan tests"
     tags: core_cpp
@@ -320,6 +347,9 @@ steps:
       - bazel run //ci/ray_ci:test_in_docker -- //:all //src/... core
         --build-type ubsan --except-tags no_ubsan,cgroup
         --cache-test-results --parallelism-per-worker 2
+    depends_on:
+      - block-core-cpp-sanitizer-tests
+      - corebuild
 
   - label: ":ray: core: cpp tsan tests"
     tags: core_cpp
@@ -328,17 +358,23 @@ steps:
       - bazel run //ci/ray_ci:test_in_docker -- //:all //src/... core
         --build-type tsan-clang --except-tags no_tsan,cgroup
         --cache-test-results --parallelism-per-worker 2
+    depends_on:
+      - block-core-cpp-sanitizer-tests
+      - corebuild
 
   - label: ":ray: core: flaky cpp tests"
     key: core_flaky_cpp_tests
     tags:
       - python
+      - flaky
       - skip-on-premerge
     instance_type: large
     soft_fail: true
     commands:
       - bazel run //ci/ray_ci:test_in_docker -- //:all //src/... core
         --run-flaky-tests --build-type clang
+    depends_on:
+      - corebuild
 
   - label: ":ray: core: flaky tests"
     key: core_flaky_tests
@@ -353,6 +389,8 @@ steps:
         --install-mask all-ray-libraries
         --run-flaky-tests
         --except-tags multi_gpu
+    depends_on:
+      - corebuild
 
   - label: ":ray: core: flaky gpu tests"
     key: core_flaky_gpu_tests
@@ -417,6 +455,21 @@ steps:
       - raycpubase
       - corebuild
 
+  - label: ":core: core: spark-on-ray tests"
+    tags:
+      - spark_on_ray
+    instance_type: medium
+    commands:
+      - bazel run //ci/ray_ci:test_in_docker -- //python/ray/tests/... core
+        --build-type debug
+        --test-env=RAY_ON_SPARK_BACKGROUND_JOB_STARTUP_WAIT=1
+        --test-env=RAY_ON_SPARK_RAY_WORKER_NODE_STARTUP_INTERVAL=5
+        --parallelism-per-worker 3
+        --only-tags spark_on_ray
+        --except-tags kubernetes
+    depends_on:
+      - corebuild
+
   # block on premerge and microcheck
   - block: "run multi gpu tests"
     if: build.env("BUILDKITE_PIPELINE_ID") == "0189942e-0876-4b8f-80a4-617f988ec59b" || build.env("BUILDKITE_PIPELINE_ID") == "018f4f1e-1b73-4906-9802-92422e3badaa"
@@ -426,7 +479,7 @@ steps:
   - label: ":ray: core: multi gpu tests"
     key: core-multi-gpu-tests
     tags:
-      - accelerated_dag
+      - compiled_graphs
       - gpu
     instance_type: gpu-large
     # we're running some cgraph doc tests here as well since they need gpus

.buildkite/lint.rayci.yml

@@ -24,6 +24,17 @@ steps:
       - test_coverage
       - documentation_style
 
+  - label: ":lint-roller: pre-commit pydoclint"
+    key: pydoclint-small
+    tags:
+      - oss
+      - lint
+      - always
+    depends_on:
+      - forge
+    commands:
+      - ./ci/lint/lint.sh pre_commit_pydoclint
+
   - label: ":lint-roller: lint: {{matrix}}"
     tags:
       - oss

.buildkite/linux_aarch64.rayci.yml

@@ -59,6 +59,7 @@ steps:
   - label: ":tapioca: build: wheel {{matrix}} (aarch64)"
     tags:
       - linux_wheels
+      - release_wheels
       - oss
     instance_type: medium-arm64
     commands:
@@ -75,6 +76,7 @@ steps:
     job_env: forge-aarch64
 
   - label: ":tapioca: build: ray py{{matrix}} docker (aarch64)"
+    key: ray_images_aarch64
     tags:
       - python_dependencies
       - docker

.buildkite/macos/macos.rayci.yml

@@ -12,6 +12,7 @@ steps:
     tags:
       - macos_wheels
       - python_dependencies
+      - release_wheels
     job_env: MACOS
     instance_type: macos
     commands:
@@ -23,6 +24,7 @@ steps:
     tags:
       - macos_wheels
       - python_dependencies
+      - release_wheels
     job_env: MACOS
     instance_type: macos-arm64
     commands:

.buildkite/ml.rayci.yml

@@ -148,7 +148,7 @@ steps:
     commands:
       - bazel run //ci/ray_ci:test_in_docker -- //python/ray/tune/... ml
         --parallelism-per-worker 3
-        --except-tags soft_imports,gpu_only,rllib
+        --except-tags doctest,soft_imports,gpu_only,rllib
     depends_on: [ "mlbuild", "forge" ]
 
   - label: ":train: ml: tune soft import tests"
@@ -166,7 +166,7 @@ steps:
     commands:
       - bazel run //ci/ray_ci:test_in_docker -- //python/ray/air/... ml
         --parallelism-per-worker 3
-        --except-tags gpu
+        --except-tags gpu,doctest
       - bazel run //ci/ray_ci:test_in_docker -- //python/ray/data/... ml
         --parallelism-per-worker 3
         --only-tags ray_air

.buildkite/others.rayci.yml

@@ -78,7 +78,6 @@ steps:
       - bazel run //ci/ray_ci/automation:determine_microcheck_tests -- {{matrix}} 100 --test-prefix windows:__ --consider-master-branch --production
     matrix:
       - "core"
-      - "serverless"
       - "serve"
       - "data"
       - "ml"

.buildkite/release-automation/forge_arm64.Dockerfile

@@ -15,11 +15,11 @@ apt-get install -y curl zip clang-12
 
 ln -s /usr/bin/clang-12 /usr/bin/clang
 
-# Install miniconda
-curl -sfL https://repo.anaconda.com/miniconda/Miniconda3-py311_24.4.0-0-Linux-aarch64.sh > /tmp/miniconda.sh
-bash /tmp/miniconda.sh -b -u -p /usr/local/bin/miniconda3
-rm /tmp/miniconda.sh
-/usr/local/bin/miniconda3/bin/conda init bash
+# Install miniforge3
+curl -sfL https://github.com/conda-forge/miniforge/releases/download/25.3.0-1/Miniforge3-25.3.0-1-Linux-aarch64.sh > /tmp/miniforge3.sh
+bash /tmp/miniforge3.sh -b -u -p /usr/local/bin/miniforge3
+rm /tmp/miniforge3.sh
+/usr/local/bin/miniforge3/bin/conda init bash
 
 # Install Bazelisk
 curl -L https://github.com/bazelbuild/bazelisk/releases/download/v1.19.0/bazelisk-linux-arm64 --output /usr/local/bin/bazelisk

.buildkite/release-automation/forge_x86_64.Dockerfile

@@ -19,11 +19,11 @@ addgroup --gid 993 docker
 
 ln -s /usr/bin/clang-12 /usr/bin/clang
 
-# Install miniconda
-curl -sfL https://repo.anaconda.com/miniconda/Miniconda3-py311_24.4.0-0-Linux-x86_64.sh > /tmp/miniconda.sh
-bash /tmp/miniconda.sh -b -u -p /usr/local/bin/miniconda3
-rm /tmp/miniconda.sh
-/usr/local/bin/miniconda3/bin/conda init bash
+# Install miniforge3
+curl -sfL https://github.com/conda-forge/miniforge/releases/download/25.3.0-1/Miniforge3-25.3.0-1-Linux-x86_64.sh > /tmp/miniforge3.sh
+bash /tmp/miniforge3.sh -b -u -p /usr/local/bin/miniforge3
+rm /tmp/miniforge3.sh
+/usr/local/bin/miniforge3/bin/conda init bash
 
 # Install Bazelisk
 curl -L https://github.com/bazelbuild/bazelisk/releases/download/v1.19.0/bazelisk-linux-amd64 --output /usr/local/bin/bazelisk

.buildkite/release-automation/verify-linux-wheels.sh

@@ -14,8 +14,8 @@ fi
 
 export PYTHON_VERSION
 
-export PATH="/usr/local/bin/miniconda3/bin:${PATH}"
-source "/usr/local/bin/miniconda3/etc/profile.d/conda.sh"
+export PATH="/usr/local/bin/miniforge3/bin:${PATH}"
+source "/usr/local/bin/miniforge3/etc/profile.d/conda.sh"
 
 conda create -n rayio python="${PYTHON_VERSION}" -y