fix: test Pyodide with cibuildwheel and fix 32-bit issue (#3270)

* ci: use cibuildwheel for pyodide test Signed-off-by: Henry Schreiner <[email protected]> * fix: enforce exceptions on pyodide Signed-off-by: Henry Schreiner <[email protected]> * fix: added extra check for emscripten * refactor: use 32-bit check on non-Windows Signed-off-by: Henry Schreiner <[email protected]> --------- Signed-off-by: Henry Schreiner <[email protected]> Co-authored-by: Andres Rios Tascon <[email protected]>
scikit-hep · Jan 24, 2025 · 08c7789 · 08c7789
1 parent b18ca47
commit 08c7789
Show file tree

Hide file tree

Showing 8 changed files with 90 additions and 40 deletions.
diff --git a/.github/workflows/build-wheels.yml b/.github/workflows/build-wheels.yml
@@ -109,7 +109,6 @@ jobs:
         CIBW_BUILD: "${{ matrix.build }}*"
         CIBW_ARCHS: ${{ matrix.arch }}
       with:
-        config-file: cibuildwheel.toml
         package-dir: awkward-cpp
 
     - name: Check metadata

diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml
@@ -17,6 +17,35 @@ env:
   SOURCE_DATE_EPOCH: "1668811211"
 
 jobs:
+  awkward-cpp-wasm:
+    name: Build C++ WASM
+    runs-on: ubuntu-22.04
+    steps:
+    - uses: actions/checkout@v4
+      with:
+        submodules: true
+
+    - name: Setup Python
+      uses: actions/setup-python@v5
+      with:
+        python-version: "3.12"
+
+    - name: Generate build files
+      run: pipx run nox -s prepare -- --headers --signatures --tests
+
+    - uses: pypa/[email protected]
+      with:
+        package-dir: awkward-cpp
+      env:
+        CIBW_PLATFORM: pyodide
+
+    - name: Upload wheel
+      uses: actions/upload-artifact@v4
+      with:
+        name: awkward-cpp-wasm
+        path: awkward-cpp/dist/awkward*wasm32.whl
+
+
   awkward-cpp-x86-64:
     runs-on: ubuntu-22.04
     name: Build C++ x86

diff --git a/.github/workflows/packaging-test.yml b/.github/workflows/packaging-test.yml
@@ -73,15 +73,13 @@ jobs:
         CIBW_ARCHS_MACOS: universal2
         CIBW_BUILD: cp39-win_amd64 cp310-manylinux_x86_64 cp39-macosx_universal2
       with:
-        config-file: cibuildwheel.toml
         package-dir: awkward-cpp
 
     - uses: pypa/[email protected]
       if: matrix.os == 'ubuntu-latest'
       env:
         CIBW_BUILD: cp312-manylinux_x86_64
       with:
-        config-file: cibuildwheel.toml
         package-dir: awkward-cpp
 
     - name: Upload wheels

diff --git a/awkward-cpp/CMakeLists.txt b/awkward-cpp/CMakeLists.txt
@@ -58,6 +58,18 @@ target_include_directories(awkward-parent INTERFACE rapidjson/include)
 add_subdirectory(header-only EXCLUDE_FROM_ALL)
 target_link_libraries(awkward-parent INTERFACE awkward::growable-buffer)
 
+# WASM needs exceptions enabled
+if(CMAKE_SYSTEM_NAME MATCHES Emscripten)
+  set_property(
+    TARGET awkward-parent
+    APPEND
+    PROPERTY INTERFACE_LINK_OPTIONS -fexceptions)
+  set_property(
+    TARGET awkward-parent
+    APPEND
+    PROPERTY INTERFACE_COMPILE_OPTIONS -fexceptions)
+endif()
+
 # First tier: cpu-kernels
 add_library(awkward-cpu-kernels SHARED ${CPU_KERNEL_SOURCES})
 target_link_libraries(awkward-cpu-kernels PUBLIC awkward-parent)

diff --git a/awkward-cpp/pyproject.toml b/awkward-cpp/pyproject.toml
@@ -73,3 +73,38 @@ sdist.include = [
 [[tool.scikit-build.overrides]]
 if.platform-system = "darwin"
 cmake.version = ">=3.29"
+
+
+[tool.cibuildwheel]
+build-frontend = "build[uv]"
+test-requires = ["pytest>=6", "."]
+test-command = ["echo {project}:", "ls {project}", "echo {package}:", "ls {package}",
+"""
+pytest -p no:cacheprovider \
+       {project}/tests \
+       {package}/tests-cpu-kernels \
+       {package}/tests-cpu-kernels-explicit \
+       {package}/tests-spec \
+       {package}/tests-spec-explicit
+"""]
+skip = [
+    "pp*win*",
+    "pp*-manylinux_i686",
+]
+test-skip = [
+    "*universal2:arm64",
+    "pp*",
+    "*musllinux*",
+]
+build-verbosity = 1
+
+[tool.cibuildwheel.environment]
+PIP_ONLY_BINARY = "cmake,numpy"
+
+[[tool.cibuildwheel.overrides]]
+select = "cp313*"
+environment.PYO3_USE_ABI3_FORWARD_COMPATIBILITY = "1"
+
+[[tool.cibuildwheel.overrides]]
+select = "*pyodide*"
+build-frontend = {name = "build", args = ["--exports", "whole_archive"]}
diff --git a/awkward-cpp/src/libawkward/util.cpp b/awkward-cpp/src/libawkward/util.cpp
@@ -69,13 +69,13 @@ namespace awkward {
       case dtype::int16:
         return "h";
       case dtype::int32:
-#if defined _MSC_VER || defined __i386__
+#if defined _MSC_VER || INTPTR_MAX == INT32_MAX
         return "l";
 #else
         return "i";
 #endif
       case dtype::int64:
-#if defined _MSC_VER || defined __i386__
+#if defined _MSC_VER || INTPTR_MAX == INT32_MAX
         return "q";
 #else
         return "l";
@@ -85,13 +85,13 @@ namespace awkward {
       case dtype::uint16:
         return "H";
       case dtype::uint32:
-#if defined _MSC_VER || defined __i386__
+#if defined _MSC_VER || defined INTPTR_MAX == INT32_MAX
         return "L";
 #else
         return "I";
 #endif
       case dtype::uint64:
-#if defined _MSC_VER || defined __i386__
+#if defined _MSC_VER || defined INTPTR_MAX == INT32_MAX
         return "Q";
 #else
         return "L";

diff --git a/cibuildwheel.toml b/cibuildwheel.toml
diff --git a/tests/test_2682_custom_pickler.py b/tests/test_2682_custom_pickler.py
@@ -2,11 +2,9 @@
 
 from __future__ import annotations
 
-import multiprocessing
 import os
 import pickle
 import sys
-from concurrent.futures import ProcessPoolExecutor
 
 if sys.version_info < (3, 12):
     import importlib_metadata
@@ -22,10 +20,16 @@ def has_entry_point():
     return bool(importlib_metadata.entry_points(group="awkward.pickle.reduce").names)
 
 
-pytestmark = pytest.mark.skipif(
-    has_entry_point(),
-    reason="Custom pickler is already registered!",
-)
+if sys.platform.startswith("emscripten"):
+    pytestmark = pytest.mark.skip
+else:
+    import multiprocessing
+    from concurrent.futures import ProcessPoolExecutor
+
+    pytestmark = pytest.mark.skipif(
+        has_entry_point(),
+        reason="Custom pickler is already registered!",
+    )
 
 
 def _init_process_with_pickler(pickler_source: str, tmp_path):