From 68c4285717bd1150c234e5a6e7f8bad7fa5550e2 Mon Sep 17 00:00:00 2001 From: Peter Andreas Entschev Date: Thu, 21 Nov 2024 01:10:50 +0100 Subject: [PATCH 1/2] Add `pynvml` as a dependency for `dask-cudf` (#17386) https://github.com/rapidsai/cudf/pull/17250 started using `pynvml` but did not add the proper dependency, this change fixes the missing dependency. Authors: - Peter Andreas Entschev (https://github.com/pentschev) - https://github.com/jakirkham Approvers: - GALI PREM SAGAR (https://github.com/galipremsagar) - https://github.com/jakirkham URL: https://github.com/rapidsai/cudf/pull/17386 --- conda/environments/all_cuda-118_arch-x86_64.yaml | 1 + conda/environments/all_cuda-125_arch-x86_64.yaml | 1 + conda/recipes/dask-cudf/meta.yaml | 1 + dependencies.yaml | 1 + python/dask_cudf/dask_cudf/io/parquet.py | 2 +- python/dask_cudf/pyproject.toml | 1 + 6 files changed, 6 insertions(+), 1 deletion(-) diff --git a/conda/environments/all_cuda-118_arch-x86_64.yaml b/conda/environments/all_cuda-118_arch-x86_64.yaml index d21497c4def..1ec002d3ec6 100644 --- a/conda/environments/all_cuda-118_arch-x86_64.yaml +++ b/conda/environments/all_cuda-118_arch-x86_64.yaml @@ -71,6 +71,7 @@ dependencies: - ptxcompiler - pyarrow>=14.0.0,<19.0.0a0 - pydata-sphinx-theme!=0.14.2 +- pynvml>=11.4.1,<12.0.0a0 - pytest-benchmark - pytest-cases>=3.8.2 - pytest-cov diff --git a/conda/environments/all_cuda-125_arch-x86_64.yaml b/conda/environments/all_cuda-125_arch-x86_64.yaml index 400c1195e00..b6d1cf75721 100644 --- a/conda/environments/all_cuda-125_arch-x86_64.yaml +++ b/conda/environments/all_cuda-125_arch-x86_64.yaml @@ -69,6 +69,7 @@ dependencies: - pyarrow>=14.0.0,<19.0.0a0 - pydata-sphinx-theme!=0.14.2 - pynvjitlink>=0.0.0a0 +- pynvml>=11.4.1,<12.0.0a0 - pytest-benchmark - pytest-cases>=3.8.2 - pytest-cov diff --git a/conda/recipes/dask-cudf/meta.yaml b/conda/recipes/dask-cudf/meta.yaml index 1e6c0a35a09..74ecded8ead 100644 --- a/conda/recipes/dask-cudf/meta.yaml +++ b/conda/recipes/dask-cudf/meta.yaml @@ -43,6 +43,7 @@ requirements: run: - python - cudf ={{ version }} + - pynvml >=11.4.1,<12.0.0a0 - rapids-dask-dependency ={{ minor_version }} - {{ pin_compatible('cuda-version', max_pin='x', min_pin='x') }} diff --git a/dependencies.yaml b/dependencies.yaml index 682aaa612b4..6c38d1c290a 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -758,6 +758,7 @@ dependencies: common: - output_types: [conda, requirements, pyproject] packages: + - pynvml>=11.4.1,<12.0.0a0 - rapids-dask-dependency==24.12.*,>=0.0.0a0 run_custreamz: common: diff --git a/python/dask_cudf/dask_cudf/io/parquet.py b/python/dask_cudf/dask_cudf/io/parquet.py index bf8fae552c2..bbedd046760 100644 --- a/python/dask_cudf/dask_cudf/io/parquet.py +++ b/python/dask_cudf/dask_cudf/io/parquet.py @@ -55,7 +55,7 @@ def _get_device_size(): handle = pynvml.nvmlDeviceGetHandleByIndex(int(index)) return pynvml.nvmlDeviceGetMemoryInfo(handle).total - except (ImportError, ValueError): + except ValueError: # Fall back to a conservative 8GiB default return 8 * 1024**3 diff --git a/python/dask_cudf/pyproject.toml b/python/dask_cudf/pyproject.toml index 07d9143db36..5dac70cc295 100644 --- a/python/dask_cudf/pyproject.toml +++ b/python/dask_cudf/pyproject.toml @@ -24,6 +24,7 @@ dependencies = [ "fsspec>=0.6.0", "numpy>=1.23,<3.0a0", "pandas>=2.0,<2.2.4dev0", + "pynvml>=11.4.1,<12.0.0a0", "rapids-dask-dependency==24.12.*,>=0.0.0a0", ] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. classifiers = [ From 0d9e577ccaab0d72f1b216fbe068afd7a0fd887e Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Wed, 20 Nov 2024 18:47:01 -0800 Subject: [PATCH 2/2] Ignore errors when testing glibc versions (#17389) This is likely the easiest fix for avoiding CI errors from this part of the code. Authors: - Vyas Ramasubramani (https://github.com/vyasr) Approvers: - GALI PREM SAGAR (https://github.com/galipremsagar) - Bradley Dice (https://github.com/bdice) URL: https://github.com/rapidsai/cudf/pull/17389 --- ci/run_cudf_polars_polars_tests.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/ci/run_cudf_polars_polars_tests.sh b/ci/run_cudf_polars_polars_tests.sh index b1bfac2a1dd..c851f65d4f6 100755 --- a/ci/run_cudf_polars_polars_tests.sh +++ b/ci/run_cudf_polars_polars_tests.sh @@ -28,8 +28,11 @@ if [[ $(arch) == "aarch64" ]]; then DESELECTED_TESTS+=("tests/unit/operations/test_join.py::test_join_4_columns_with_validity") else # Ensure that we don't run dbgen when it uses newer symbols than supported by the glibc version in the CI image. + # Allow errors since any of these commands could produce empty results that would cause the script to fail. + set +e glibc_minor_version=$(ldd --version | head -1 | grep -o "[0-9]\.[0-9]\+" | tail -1 | cut -d '.' -f2) latest_glibc_symbol_found=$(nm py-polars/tests/benchmark/data/pdsh/dbgen/dbgen | grep GLIBC | grep -o "[0-9]\.[0-9]\+" | sort --version-sort | tail -1 | cut -d "." -f 2) + set -e if [[ ${glibc_minor_version} -lt ${latest_glibc_symbol_found} ]]; then DESELECTED_TESTS+=("tests/benchmark/test_pdsh.py::test_pdsh") fi