Merge remote-tracking branch 'upstream/main' into apachegh-43683-pandas-string-dtype

Author: jorisvandenbossche

.env

@@ -57,7 +57,7 @@ CLANG_TOOLS=14
 CUDA=11.2.2
 DASK=latest
 DOTNET=8.0
-GCC_VERSION=""
+GCC=
 HDFS=3.2.1
 JDK=11
 KARTOTHEK=latest

.github/workflows/csharp.yml

@@ -54,7 +54,7 @@ jobs:
         dotnet: ['8.0.x']
     steps:
       - name: Install C#
-        uses: actions/setup-dotnet@v4.1.0
+        uses: actions/setup-dotnet@v4.2.0
         with:
           dotnet-version: ${{ matrix.dotnet }}
       - name: Setup Python
@@ -86,7 +86,7 @@ jobs:
         dotnet: ['8.0.x']
     steps:
       - name: Install C#
-        uses: actions/setup-dotnet@v4.1.0
+        uses: actions/setup-dotnet@v4.2.0
         with:
           dotnet-version: ${{ matrix.dotnet }}
       - name: Checkout Arrow
@@ -113,7 +113,7 @@ jobs:
         dotnet: ['8.0.x']
     steps:
       - name: Install C#
-        uses: actions/setup-dotnet@v4.1.0
+        uses: actions/setup-dotnet@v4.2.0
         with:
           dotnet-version: ${{ matrix.dotnet }}
       - name: Setup Python

.github/workflows/dev.yml

@@ -117,7 +117,7 @@ jobs:
         with:
           ruby-version: ruby
       - name: Install .NET
-        uses: actions/setup-dotnet@3e891b0cb619bf60e2c25674b222b8940e2c1c25 # v4.1.0
+        uses: actions/setup-dotnet@87b7050bc53ea08284295505d98d2aa94301e852 # v4.2.0
         with:
           dotnet-version: '8.0.x'
       - name: Install Dependencies

.github/workflows/pr_review_trigger.yml

@@ -29,7 +29,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: "Upload PR review Payload"
-        uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4.4.3
+        uses: actions/upload-artifact@6f51ac03b9356f520e9adb1b1b7802705f340c2b # v4.5.0
         with:
           path: "${{ github.event_path }}"
           name: "pr_review_payload"

.github/workflows/r.yml

@@ -177,7 +177,7 @@ jobs:
         if: always()
       - name: Save the test output
         if: always()
-        uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4.4.3
+        uses: actions/upload-artifact@6f51ac03b9356f520e9adb1b1b7802705f340c2b # v4.5.0
         with:
           name: test-output-${{ matrix.ubuntu }}-${{ matrix.r }}
           path: r/check/arrow.Rcheck/tests/testthat.Rout*
@@ -237,7 +237,7 @@ jobs:
         if: always()
       - name: Save the test output
         if: always()
-        uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4.4.3
+        uses: actions/upload-artifact@6f51ac03b9356f520e9adb1b1b7802705f340c2b # v4.5.0
         with:
           name: test-output-bundled
           path: r/check/arrow.Rcheck/tests/testthat.Rout*
@@ -299,7 +299,7 @@ jobs:
         # So that they're unique when multiple are downloaded in the next step
         shell: bash
         run: mv libarrow.zip libarrow-rtools${{ matrix.config.rtools }}-${{ matrix.config.arch }}.zip
-      - uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4.4.3
+      - uses: actions/upload-artifact@6f51ac03b9356f520e9adb1b1b7802705f340c2b # v4.5.0
         with:
           name: libarrow-rtools${{ matrix.config.rtools }}-${{ matrix.config.arch }}.zip
           path: libarrow-rtools${{ matrix.config.rtools }}-${{ matrix.config.arch }}.zip

.pre-commit-config.yaml

@@ -182,4 +182,5 @@ repos:
           (
           ?^ci/scripts/c_glib_build\.sh$|
           ?^ci/scripts/c_glib_test\.sh$|
+          ?^c_glib/test/run-test\.sh$|
           )

c_glib/test/run-test.sh

@@ -17,7 +17,7 @@
 # specific language governing permissions and limitations
 # under the License.
 
-test_dir="$(cd $(dirname $0); pwd)"
+test_dir="$(cd "$(dirname "$0")" && pwd)"
 build_dir="$(cd .; pwd)"
 
 modules=(
@@ -47,7 +47,7 @@ if [ "${BUILD}" != "no" ]; then
 fi
 
 for module in "${modules[@]}"; do
-  MODULE_TYPELIB_DIR_VAR_NAME="$(echo ${module} | tr a-z- A-Z_)_TYPELIB_DIR"
+  MODULE_TYPELIB_DIR_VAR_NAME="$(echo "${module}" | tr a-z- A-Z_)_TYPELIB_DIR"
   module_typelib_dir=$(eval "echo \${${MODULE_TYPELIB_DIR_VAR_NAME}}")
   if [ -z "${module_typelib_dir}" ]; then
     module_typelib_dir="${build_dir}/${module}"
@@ -74,4 +74,4 @@ case "${DEBUGGER}" in
     DEBUGGER_ARGS+=(--)
     ;;
 esac
-${DEBUGGER} "${DEBUGGER_ARGS[@]}" "${RUBY}" ${test_dir}/run-test.rb "$@"
+${DEBUGGER} "${DEBUGGER_ARGS[@]}" "${RUBY}" "${test_dir}"/run-test.rb "$@"

ci/appveyor-cpp-build.bat

@@ -34,7 +34,6 @@ IF "%ARROW_DEBUG_MEMORY_POOL%"=="" (
 set CMAKE_BUILD_PARALLEL_LEVEL=%NUMBER_OF_PROCESSORS%
 set CTEST_PARALLEL_LEVEL=%NUMBER_OF_PROCESSORS%
 
-
 call activate arrow
 
 @rem The "main" C++ build script for Windows CI
@@ -113,12 +112,12 @@ ctest --output-on-failure || exit /B
 
 popd
 
+pushd python
+
 @rem
 @rem Build and install pyarrow
 @rem
 
-pushd python
-
 set PYARROW_CMAKE_GENERATOR=%GENERATOR%
 set PYARROW_CXXFLAGS=%ARROW_CXXFLAGS%
 set PYARROW_PARALLEL=2
@@ -137,6 +136,12 @@ set ARROW_HOME=%CONDA_PREFIX%\Library
 @rem ARROW-3075; pkgconfig is broken for Parquet for now
 set PARQUET_HOME=%CONDA_PREFIX%\Library
 
+pip install --no-deps --no-build-isolation -vv --editable .
+
+@rem
+@rem Run pyarrow tests
+@rem
+
 @rem Download IANA Timezone Database to a non-standard location to
 @rem test the configurability of the timezone database path
 curl https://data.iana.org/time-zones/releases/tzdata2024b.tar.gz --output tzdata.tar.gz || exit /B
@@ -150,12 +155,9 @@ rmdir /s /q %USERPROFILE%\Downloads\tzdata
 @rem (only needed for testing purposes)
 set PYARROW_TZDATA_PATH=%USERPROFILE%\Downloads\test\tzdata
 
-python setup.py develop -q || exit /B
-
+set AWS_EC2_METADATA_DISABLED=true
 set PYTHONDEVMODE=1
 
-py.test -r sxX --durations=15 --pyargs pyarrow.tests || exit /B
+python -m pytest -r sxX --durations=15 pyarrow/tests || exit /B
 
-@rem
-@rem Wheels are built and tested separately (see ARROW-5142).
-@rem
+popd

ci/appveyor-cpp-setup.bat

@@ -17,7 +17,13 @@
 
 @echo on
 
-set "PATH=C:\Miniconda38-x64;C:\Miniconda38-x64\Scripts;C:\Miniconda38-x64\Library\bin;%PATH%"
+@rem
+@rem The miniconda install on AppVeyor is very outdated, use Mambaforge instead
+@rem
+
+appveyor DownloadFile https://github.com/conda-forge/miniforge/releases/download/24.9.2-0/Mambaforge-Windows-x86_64.exe || exit /B
+start /wait "" Mambaforge-Windows-x86_64.exe /InstallationType=JustMe /RegisterPython=0 /S /D=C:\Mambaforge
+set "PATH=C:\Mambaforge\scripts;C:\Mambaforge\condabin;%PATH%"
 
 @rem
 @rem Avoid picking up AppVeyor-installed OpenSSL (linker errors with gRPC)
@@ -33,26 +39,15 @@ rd /s /q C:\OpenSSL-v30-Win32
 rd /s /q C:\OpenSSL-v30-Win64
 
 @rem
-@rem Configure miniconda
+@rem Configure conda
 @rem
 conda config --set auto_update_conda false
-conda config --set show_channel_urls True
+conda config --set show_channel_urls true
+conda config --set always_yes true
 @rem Help with SSL timeouts to S3
 conda config --set remote_connect_timeout_secs 12
-@rem Workaround for ARROW-13636
-conda config --append disallowed_packages pypy3
-conda info -a
-
-@rem
-@rem Install Python to the base environment
-@rem
-conda install -q -y -c conda-forge python=%PYTHON% || exit /B
 
-@rem Can't use conda-libmamba-solver 2.0.0
-conda config --set solver classic
-
-@rem Update for newer CA certificates
-conda update -q -y -c conda-forge --all || exit /B
+conda info -a || exit /B
 
 @rem
 @rem Create conda environment
@@ -66,11 +61,8 @@ if "%ARROW_BUILD_GANDIVA%" == "ON" (
 )
 @rem Install pre-built "toolchain" packages for faster builds
 set CONDA_PACKAGES=%CONDA_PACKAGES% --file=ci\conda_env_cpp.txt
-@rem Force conda to use conda-forge
-conda config --add channels conda-forge
-conda config --remove channels defaults
 @rem Arrow conda environment
-conda create -n arrow -y -c conda-forge ^
+conda create -n arrow ^
   --file=ci\conda_env_python.txt ^
   %CONDA_PACKAGES%  ^
   "ccache" ^
@@ -97,7 +89,6 @@ if "%ARROW_S3%" == "ON" (
   appveyor DownloadFile https://dl.min.io/server/minio/release/windows-amd64/archive/minio.RELEASE.2024-09-13T20-26-02Z -FileName C:\Windows\Minio.exe || exit /B
 )
 
-
 @rem
 @rem Download IANA Timezone Database for unit tests
 @rem

ci/docker/conda-cpp.dockerfile

@@ -48,7 +48,7 @@ ENV PIPX_BASE_PYTHON=/opt/conda/bin/python3
 COPY ci/scripts/install_gcs_testbench.sh /arrow/ci/scripts
 RUN /arrow/ci/scripts/install_gcs_testbench.sh default
 
-# Ensure npm, node and azurite are on path. npm and node are required to install azurite, which will then need to 
+# Ensure npm, node and azurite are on path. npm and node are required to install azurite, which will then need to
 # be on the path for the tests to run.
 ENV PATH=/opt/conda/envs/arrow/bin:$PATH
 
@@ -68,6 +68,7 @@ ENV ARROW_ACERO=ON \
     ARROW_GANDIVA=ON \
     ARROW_GCS=ON \
     ARROW_HOME=$CONDA_PREFIX \
+    ARROW_JEMALLOC=ON \
     ARROW_ORC=ON \
     ARROW_PARQUET=ON \
     ARROW_S3=ON \