Skip to content

Commit f787281

Browse files
shrshishrshi
authored
Merge branch 'branch-25.04' into pylibcudf-json-reader-builder-opts
2 parents 04b4369 + 1a891e6 commit f787281

File tree

317 files changed

+4244
-3863
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

317 files changed

+4244
-3863
lines changed

‎.devcontainer/cuda12.5-conda/devcontainer.json renamed to ‎.devcontainer/cuda12.8-conda/devcontainer.json

+36-26
Original file line numberDiff line numberDiff line change
@@ -3,55 +3,65 @@
33
"context": "${localWorkspaceFolder}/.devcontainer",
44
"dockerfile": "${localWorkspaceFolder}/.devcontainer/Dockerfile",
55
"args": {
6-
"CUDA": "12.5",
6+
"CUDA": "12.8",
77
"PYTHON_PACKAGE_MANAGER": "conda",
88
"BASE": "rapidsai/devcontainers:25.04-cpp-mambaforge-ubuntu22.04"
99
}
1010
},
1111
"runArgs": [
1212
"--rm",
1313
"--name",
14-
"${localEnv:USER:anon}-rapids-${localWorkspaceFolderBasename}-25.04-cuda12.5-conda"
14+
"${localEnv:USER:anon}-rapids-${localWorkspaceFolderBasename}-25.04-cuda12.8-conda"
1515
],
16-
"hostRequirements": {"gpu": "optional"},
16+
"hostRequirements": {
17+
"gpu": "optional"
18+
},
1719
"features": {
1820
"ghcr.io/rapidsai/devcontainers/features/cuda:25.4": {
19-
"version": "12.5",
20-
"installCompilers": false,
21-
"installProfilers": true,
22-
"installDevPackages": false,
23-
"installcuDNN": false,
24-
"installcuTensor": false,
25-
"installNCCL": false,
26-
"installCUDARuntime": false,
27-
"installNVRTC": false,
28-
"installOpenCL": false,
29-
"installcuBLAS": false,
30-
"installcuSPARSE": false,
31-
"installcuFFT": false,
32-
"installcuFile": false,
33-
"installcuRAND": false,
34-
"installcuSOLVER": false,
35-
"installNPP": false,
36-
"installnvJPEG": false,
37-
"pruneStaticLibs": true
38-
},
21+
"version": "12.8",
22+
"installCompilers": false,
23+
"installProfilers": true,
24+
"installDevPackages": false,
25+
"installcuDNN": false,
26+
"installcuTensor": false,
27+
"installNCCL": false,
28+
"installCUDARuntime": false,
29+
"installNVRTC": false,
30+
"installOpenCL": false,
31+
"installcuBLAS": false,
32+
"installcuSPARSE": false,
33+
"installcuFFT": false,
34+
"installcuFile": false,
35+
"installcuRAND": false,
36+
"installcuSOLVER": false,
37+
"installNPP": false,
38+
"installnvJPEG": false,
39+
"pruneStaticLibs": true
40+
},
3941
"ghcr.io/rapidsai/devcontainers/features/rapids-build-utils:25.4": {}
4042
},
4143
"overrideFeatureInstallOrder": [
4244
"ghcr.io/rapidsai/devcontainers/features/cuda",
4345
"ghcr.io/rapidsai/devcontainers/features/rapids-build-utils"
4446
],
45-
"initializeCommand": ["/bin/bash", "-c", "mkdir -m 0755 -p ${localWorkspaceFolder}/../.{aws,cache,config,conda/pkgs,conda/${localWorkspaceFolderBasename}-cuda12.5-envs}"],
46-
"postAttachCommand": ["/bin/bash", "-c", "if [ ${CODESPACES:-false} = 'true' ]; then . devcontainer-utils-post-attach-command; . rapids-post-attach-command; fi"],
47+
"initializeCommand": [
48+
"/bin/bash",
49+
"-c",
50+
"mkdir -m 0755 -p ${localWorkspaceFolder}/../.{aws,cache,config,conda/pkgs,conda/${localWorkspaceFolderBasename}-cuda12.8-envs}"
51+
],
52+
"postAttachCommand": [
53+
"/bin/bash",
54+
"-c",
55+
"if [ ${CODESPACES:-false} = 'true' ]; then . devcontainer-utils-post-attach-command; . rapids-post-attach-command; fi"
56+
],
4757
"workspaceFolder": "/home/coder",
4858
"workspaceMount": "source=${localWorkspaceFolder},target=/home/coder/cudf,type=bind,consistency=consistent",
4959
"mounts": [
5060
"source=${localWorkspaceFolder}/../.aws,target=/home/coder/.aws,type=bind,consistency=consistent",
5161
"source=${localWorkspaceFolder}/../.cache,target=/home/coder/.cache,type=bind,consistency=consistent",
5262
"source=${localWorkspaceFolder}/../.config,target=/home/coder/.config,type=bind,consistency=consistent",
5363
"source=${localWorkspaceFolder}/../.conda/pkgs,target=/home/coder/.conda/pkgs,type=bind,consistency=consistent",
54-
"source=${localWorkspaceFolder}/../.conda/${localWorkspaceFolderBasename}-cuda12.5-envs,target=/home/coder/.conda/envs,type=bind,consistency=consistent"
64+
"source=${localWorkspaceFolder}/../.conda/${localWorkspaceFolderBasename}-cuda12.8-envs,target=/home/coder/.conda/envs,type=bind,consistency=consistent"
5565
],
5666
"customizations": {
5767
"vscode": {

‎.devcontainer/cuda12.5-pip/devcontainer.json renamed to ‎.devcontainer/cuda12.8-pip/devcontainer.json

+17-7
Original file line numberDiff line numberDiff line change
@@ -3,32 +3,42 @@
33
"context": "${localWorkspaceFolder}/.devcontainer",
44
"dockerfile": "${localWorkspaceFolder}/.devcontainer/Dockerfile",
55
"args": {
6-
"CUDA": "12.5",
6+
"CUDA": "12.8",
77
"PYTHON_PACKAGE_MANAGER": "pip",
8-
"BASE": "rapidsai/devcontainers:25.04-cpp-cuda12.5-ubuntu22.04"
8+
"BASE": "rapidsai/devcontainers:25.04-cpp-cuda12.8-ubuntu22.04"
99
}
1010
},
1111
"runArgs": [
1212
"--rm",
1313
"--name",
14-
"${localEnv:USER:anon}-rapids-${localWorkspaceFolderBasename}-25.04-cuda12.5-pip"
14+
"${localEnv:USER:anon}-rapids-${localWorkspaceFolderBasename}-25.04-cuda12.8-pip"
1515
],
16-
"hostRequirements": {"gpu": "optional"},
16+
"hostRequirements": {
17+
"gpu": "optional"
18+
},
1719
"features": {
1820
"ghcr.io/rapidsai/devcontainers/features/rapids-build-utils:25.4": {}
1921
},
2022
"overrideFeatureInstallOrder": [
2123
"ghcr.io/rapidsai/devcontainers/features/rapids-build-utils"
2224
],
23-
"initializeCommand": ["/bin/bash", "-c", "mkdir -m 0755 -p ${localWorkspaceFolder}/../.{aws,cache,config/pip,local/share/${localWorkspaceFolderBasename}-cuda12.5-venvs}"],
24-
"postAttachCommand": ["/bin/bash", "-c", "if [ ${CODESPACES:-false} = 'true' ]; then . devcontainer-utils-post-attach-command; . rapids-post-attach-command; fi"],
25+
"initializeCommand": [
26+
"/bin/bash",
27+
"-c",
28+
"mkdir -m 0755 -p ${localWorkspaceFolder}/../.{aws,cache,config/pip,local/share/${localWorkspaceFolderBasename}-cuda12.8-venvs}"
29+
],
30+
"postAttachCommand": [
31+
"/bin/bash",
32+
"-c",
33+
"if [ ${CODESPACES:-false} = 'true' ]; then . devcontainer-utils-post-attach-command; . rapids-post-attach-command; fi"
34+
],
2535
"workspaceFolder": "/home/coder",
2636
"workspaceMount": "source=${localWorkspaceFolder},target=/home/coder/cudf,type=bind,consistency=consistent",
2737
"mounts": [
2838
"source=${localWorkspaceFolder}/../.aws,target=/home/coder/.aws,type=bind,consistency=consistent",
2939
"source=${localWorkspaceFolder}/../.cache,target=/home/coder/.cache,type=bind,consistency=consistent",
3040
"source=${localWorkspaceFolder}/../.config,target=/home/coder/.config,type=bind,consistency=consistent",
31-
"source=${localWorkspaceFolder}/../.local/share/${localWorkspaceFolderBasename}-cuda12.5-venvs,target=/home/coder/.local/share/venvs,type=bind,consistency=consistent"
41+
"source=${localWorkspaceFolder}/../.local/share/${localWorkspaceFolderBasename}-cuda12.8-venvs,target=/home/coder/.local/share/venvs,type=bind,consistency=consistent"
3242
],
3343
"customizations": {
3444
"vscode": {

‎.github/workflows/auto-assign.yml

+1
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@ on:
1010
jobs:
1111
add_assignees:
1212
runs-on: ubuntu-latest
13+
if: ${{ ! github.event.pull_request.merged }}
1314
steps:
1415
- uses: actions-ecosystem/action-add-assignees@v1
1516
with:

‎.github/workflows/build.yaml

+1-1
Original file line numberDiff line numberDiff line change
@@ -64,7 +64,7 @@ jobs:
6464
build_type: ${{ inputs.build_type || 'branch' }}
6565
container_image: "rapidsai/ci-conda:latest"
6666
date: ${{ inputs.date }}
67-
node_type: "gpu-v100-latest-1"
67+
node_type: "gpu-l4-latest-1"
6868
run_script: "ci/build_docs.sh"
6969
sha: ${{ inputs.sha }}
7070
wheel-build-libcudf:

‎.github/workflows/pr.yaml

+28-6
Original file line numberDiff line numberDiff line change
@@ -41,6 +41,7 @@ jobs:
4141
- pandas-tests
4242
- pandas-tests-diff
4343
- telemetry-setup
44+
- third-party-integration-tests-cudf-pandas
4445
secrets: inherit
4546
uses: rapidsai/shared-workflows/.github/workflows/[email protected]
4647
if: always()
@@ -126,20 +127,22 @@ jobs:
126127
uses: rapidsai/shared-workflows/.github/workflows/[email protected]
127128
with:
128129
enable_check_generated_files: false
129-
ignored_pr_jobs: "telemetry-summarize"
130+
ignored_pr_jobs: "telemetry-summarize spark-rapids-jni"
130131
conda-cpp-build:
131132
needs: checks
132133
secrets: inherit
133134
uses: rapidsai/shared-workflows/.github/workflows/[email protected]
134135
with:
135136
build_type: pull-request
137+
node_type: "cpu16"
136138
cpp-linters:
137139
secrets: inherit
138140
needs: checks
139141
uses: rapidsai/shared-workflows/.github/workflows/[email protected]
140142
with:
141143
build_type: pull-request
142144
run_script: "ci/cpp_linters.sh"
145+
node_type: "cpu16"
143146
conda-cpp-checks:
144147
needs: conda-cpp-build
145148
secrets: inherit
@@ -184,7 +187,7 @@ jobs:
184187
if: fromJSON(needs.changed-files.outputs.changed_file_groups).test_java
185188
with:
186189
build_type: pull-request
187-
node_type: "gpu-v100-latest-1"
190+
node_type: "gpu-l4-latest-1"
188191
arch: "amd64"
189192
container_image: "rapidsai/ci-conda:latest"
190193
run_script: "ci/test_java.sh"
@@ -205,7 +208,7 @@ jobs:
205208
if: fromJSON(needs.changed-files.outputs.changed_file_groups).test_notebooks
206209
with:
207210
build_type: pull-request
208-
node_type: "gpu-v100-latest-1"
211+
node_type: "gpu-l4-latest-1"
209212
arch: "amd64"
210213
container_image: "rapidsai/ci-conda:latest"
211214
run_script: "ci/test_notebooks.sh"
@@ -215,7 +218,7 @@ jobs:
215218
uses: rapidsai/shared-workflows/.github/workflows/[email protected]
216219
with:
217220
build_type: pull-request
218-
node_type: "gpu-v100-latest-1"
221+
node_type: "gpu-l4-latest-1"
219222
arch: "amd64"
220223
container_image: "rapidsai/ci-conda:latest"
221224
run_script: "ci/build_docs.sh"
@@ -227,6 +230,7 @@ jobs:
227230
# build for every combination of arch and CUDA version, but only for the latest Python
228231
matrix_filter: group_by([.ARCH, (.CUDA_VER|split(".")|map(tonumber)|.[0])]) | map(max_by(.PY_VER|split(".")|map(tonumber)))
229232
build_type: pull-request
233+
node_type: "cpu16"
230234
script: "ci/build_wheel_libcudf.sh"
231235
wheel-build-pylibcudf:
232236
needs: [checks, wheel-build-libcudf]
@@ -302,8 +306,9 @@ jobs:
302306
needs: telemetry-setup
303307
uses: rapidsai/shared-workflows/.github/workflows/[email protected]
304308
with:
309+
node_type: "cpu32"
305310
arch: '["amd64"]'
306-
cuda: '["12.5"]'
311+
cuda: '["12.8"]'
307312
build_command: |
308313
sccache -z;
309314
build-all -DBUILD_BENCHMARKS=ON --verbose;
@@ -318,6 +323,19 @@ jobs:
318323
matrix_filter: map(select(.ARCH == "amd64")) | group_by(.CUDA_VER|split(".")|map(tonumber)|.[0]) | map(max_by([(.PY_VER|split(".")|map(tonumber)), (.CUDA_VER|split(".")|map(tonumber))]))
319324
build_type: pull-request
320325
script: ci/cudf_pandas_scripts/run_tests.sh
326+
third-party-integration-tests-cudf-pandas:
327+
needs: conda-python-build
328+
secrets: inherit
329+
uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@nvks-runners
330+
with:
331+
build_type: pull-request
332+
branch: ${{ inputs.branch }}
333+
date: ${{ inputs.date }}
334+
sha: ${{ inputs.sha }}
335+
node_type: "gpu-l4-latest-1"
336+
container_image: "rapidsai/ci-conda:latest"
337+
run_script: |
338+
ci/cudf_pandas_scripts/third-party-integration/test.sh python/cudf/cudf_pandas_tests/third_party_integration_tests/dependencies.yaml
321339
pandas-tests:
322340
# run the Pandas unit tests using PR branch
323341
needs: [wheel-build-cudf, changed-files]
@@ -336,9 +354,13 @@ jobs:
336354
needs: pandas-tests
337355
uses: rapidsai/shared-workflows/.github/workflows/[email protected]
338356
with:
339-
node_type: cpu4
357+
node_type: "cpu4"
340358
build_type: pull-request
341359
run_script: "ci/cudf_pandas_scripts/pandas-tests/diff.sh"
360+
spark-rapids-jni:
361+
needs: changed-files
362+
uses: ./.github/workflows/spark-rapids-jni.yaml
363+
if: fromJSON(needs.changed-files.outputs.changed_file_groups).test_java
342364

343365
telemetry-summarize:
344366
# This job must use a self-hosted runner to record telemetry traces.

‎.github/workflows/spark-rapids-jni.yaml

+22
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,22 @@
1+
name: spark-rapids-jni
2+
3+
on:
4+
workflow_call:
5+
6+
jobs:
7+
spark-rapids-jni-build:
8+
runs-on: linux-amd64-cpu8
9+
container:
10+
image: rapidsai/ci-spark-rapids-jni:rockylinux8-cuda12.2.0
11+
steps:
12+
- uses: actions/checkout@v4
13+
with:
14+
repository: NVIDIA/spark-rapids-jni
15+
submodules: recursive
16+
- uses: actions/checkout@v4
17+
with:
18+
path: thirdparty/cudf
19+
- name: "Build spark-rapids-jni"
20+
run: |
21+
mkdir target
22+
GPU_ARCHS=90 LIBCUDF_DEPENDENCY_MODE=latest USE_GDS=on scl enable gcc-toolset-11 build/buildcpp.sh

0 commit comments

Comments
 (0)