From 600bf5a2e32b3a945dae5195b702fbd8226383d4 Mon Sep 17 00:00:00 2001 From: Thomas Li <47963215+lithomas1@users.noreply.github.com> Date: Wed, 30 Apr 2025 20:00:11 -0400 Subject: [PATCH 1/8] BLD: Try using shared memory utilities in Cython to reduce wheel sizes --- meson.build | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/meson.build b/meson.build index 66583095a6e77..2487fe8732947 100644 --- a/meson.build +++ b/meson.build @@ -47,6 +47,27 @@ endif cy = meson.get_compiler('cython') if cy.version().version_compare('>=3.1.0') add_project_arguments('-Xfreethreading_compatible=true', language: 'cython') + + # Use shared utility code to reduce wheel sizes + # copied from https://github.com/scikit-learn/scikit-learn/pull/31151/files + cy = find_program(cy.cmd_array()[0]) + cython_shared_src = custom_target( + install: false, + output: '_cyutility.c', + command: [ + cy, '-3', '--fast-fail', + '--generate-shared=' + meson.current_build_dir()/'_cyutility.c' + ], + ) + + py.extension_module('_cyutility', + cython_shared_src, + subdir: 'pandas/_libs', + cython_args: cython_args, + install: true, + ) + + add_project_arguments('--shared=pandas._libs._cyutility'', language: 'cython') endif # Needed by pandas.test() when it looks for the pytest ini options From 6368144232ec496ccc6a1cd189ccce89493f40b4 Mon Sep 17 00:00:00 2001 From: Thomas Li <47963215+lithomas1@users.noreply.github.com> Date: Wed, 30 Apr 2025 20:03:45 -0400 Subject: [PATCH 2/8] fix syntax --- meson.build | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/meson.build b/meson.build index 2487fe8732947..41aae8dcf5b0f 100644 --- a/meson.build +++ b/meson.build @@ -63,11 +63,10 @@ if cy.version().version_compare('>=3.1.0') py.extension_module('_cyutility', cython_shared_src, subdir: 'pandas/_libs', - cython_args: cython_args, install: true, ) - add_project_arguments('--shared=pandas._libs._cyutility'', language: 'cython') + add_project_arguments('--shared=pandas._libs._cyutility', language: 'cython') endif # Needed by pandas.test() when it looks for the pytest ini options From cf81ec74c889edf3f9ad529120045885e070a421 Mon Sep 17 00:00:00 2001 From: Thomas Li <47963215+lithomas1@users.noreply.github.com> Date: Wed, 30 Apr 2025 20:11:47 -0400 Subject: [PATCH 3/8] maybe fix? --- meson.build | 2 -- pandas/_libs/meson.build | 6 ++++++ pandas/_libs/tslibs/meson.build | 6 ++++++ pandas/_libs/window/meson.build | 4 ++-- 4 files changed, 14 insertions(+), 4 deletions(-) diff --git a/meson.build b/meson.build index 41aae8dcf5b0f..ab3196fc44a7f 100644 --- a/meson.build +++ b/meson.build @@ -65,8 +65,6 @@ if cy.version().version_compare('>=3.1.0') subdir: 'pandas/_libs', install: true, ) - - add_project_arguments('--shared=pandas._libs._cyutility', language: 'cython') endif # Needed by pandas.test() when it looks for the pytest ini options diff --git a/pandas/_libs/meson.build b/pandas/_libs/meson.build index a50976767928a..33fc65e5034d0 100644 --- a/pandas/_libs/meson.build +++ b/pandas/_libs/meson.build @@ -148,6 +148,12 @@ if get_option('buildtype') == 'debug' cython_args += ['--gdb'] endif +# Use shared utility code to reduce wheel sizes +# copied from https://github.com/scikit-learn/scikit-learn/pull/31151/files +if cy.version().version_compare('>=3.1.0') + cython_args += ['--shared=pandas._libs._cyutility'] +endif + foreach ext_name, ext_dict : libs_sources py.extension_module( ext_name, diff --git a/pandas/_libs/tslibs/meson.build b/pandas/_libs/tslibs/meson.build index 052a8568b76af..ac43dc7db5fb7 100644 --- a/pandas/_libs/tslibs/meson.build +++ b/pandas/_libs/tslibs/meson.build @@ -28,6 +28,12 @@ if get_option('buildtype') == 'debug' cython_args += ['--gdb'] endif +# Use shared utility code to reduce wheel sizes +# copied from https://github.com/scikit-learn/scikit-learn/pull/31151/files +if cy.version().version_compare('>=3.1.0') + cython_args += ['--shared=pandas._libs._cyutility'] +endif + foreach ext_name, ext_dict : tslibs_sources py.extension_module( ext_name, diff --git a/pandas/_libs/window/meson.build b/pandas/_libs/window/meson.build index 1d49bba47e139..17ec8758f9303 100644 --- a/pandas/_libs/window/meson.build +++ b/pandas/_libs/window/meson.build @@ -1,7 +1,7 @@ py.extension_module( 'aggregations', ['aggregations.pyx'], - cython_args: ['-X always_allow_keywords=true'], + cython_args: ['-X always_allow_keywords=true --shared=pandas._libs._cyutility'], include_directories: [inc_np, inc_pd], subdir: 'pandas/_libs/window', override_options: ['cython_language=cpp'], @@ -11,7 +11,7 @@ py.extension_module( py.extension_module( 'indexers', ['indexers.pyx'], - cython_args: ['-X always_allow_keywords=true'], + cython_args: ['-X always_allow_keywords=true --shared=pandas._libs._cyutility'], include_directories: [inc_np, inc_pd], subdir: 'pandas/_libs/window', install: true, From 03b632121c242d0d98e52e83a7ff4dc1efc6e6b5 Mon Sep 17 00:00:00 2001 From: Thomas Li <47963215+lithomas1@users.noreply.github.com> Date: Wed, 30 Apr 2025 20:20:44 -0400 Subject: [PATCH 4/8] switch keyword order? --- pandas/_libs/window/meson.build | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pandas/_libs/window/meson.build b/pandas/_libs/window/meson.build index 17ec8758f9303..4f306bf85a380 100644 --- a/pandas/_libs/window/meson.build +++ b/pandas/_libs/window/meson.build @@ -1,7 +1,7 @@ py.extension_module( 'aggregations', ['aggregations.pyx'], - cython_args: ['-X always_allow_keywords=true --shared=pandas._libs._cyutility'], + cython_args: ['--shared=pandas._libs._cyutility -X always_allow_keywords=true'], include_directories: [inc_np, inc_pd], subdir: 'pandas/_libs/window', override_options: ['cython_language=cpp'], @@ -11,7 +11,7 @@ py.extension_module( py.extension_module( 'indexers', ['indexers.pyx'], - cython_args: ['-X always_allow_keywords=true --shared=pandas._libs._cyutility'], + cython_args: ['--shared=pandas._libs._cyutility -X always_allow_keywords=true'], include_directories: [inc_np, inc_pd], subdir: 'pandas/_libs/window', install: true, From 684e8c59ab50e31aeafd922d3162fec35af6a7fb Mon Sep 17 00:00:00 2001 From: Thomas Li <47963215+lithomas1@users.noreply.github.com> Date: Wed, 30 Apr 2025 20:26:42 -0400 Subject: [PATCH 5/8] try putting as list? --- pandas/_libs/window/meson.build | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pandas/_libs/window/meson.build b/pandas/_libs/window/meson.build index 4f306bf85a380..78169e781bc47 100644 --- a/pandas/_libs/window/meson.build +++ b/pandas/_libs/window/meson.build @@ -1,7 +1,7 @@ py.extension_module( 'aggregations', ['aggregations.pyx'], - cython_args: ['--shared=pandas._libs._cyutility -X always_allow_keywords=true'], + cython_args: ['-X always_allow_keywords=true', '--shared=pandas._libs._cyutility'], include_directories: [inc_np, inc_pd], subdir: 'pandas/_libs/window', override_options: ['cython_language=cpp'], @@ -11,7 +11,7 @@ py.extension_module( py.extension_module( 'indexers', ['indexers.pyx'], - cython_args: ['--shared=pandas._libs._cyutility -X always_allow_keywords=true'], + cython_args: ['-X always_allow_keywords=true', '--shared=pandas._libs._cyutility'], include_directories: [inc_np, inc_pd], subdir: 'pandas/_libs/window', install: true, From be7ad4e7bb23e6779874c3f9c193abc23998c5c4 Mon Sep 17 00:00:00 2001 From: Thomas Li <47963215+lithomas1@users.noreply.github.com> Date: Wed, 30 Apr 2025 20:31:06 -0400 Subject: [PATCH 6/8] try the cython rc --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 3f7b6a672e1b0..a2b51aca0178b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -5,7 +5,7 @@ requires = [ "meson-python>=0.13.1", "meson>=1.2.1,<2", "wheel", - "Cython~=3.0.5", # Note: sync with setup.py, environment.yml and asv.conf.json + "Cython==3.1.0rc1", # Note: sync with setup.py, environment.yml and asv.conf.json # Force numpy higher than 2.0rc1, so that built wheels are compatible # with both numpy 1 and 2 "numpy>=2.0.0rc1", From 573c05c7c878c489f5d8fc776cd9009909917d05 Mon Sep 17 00:00:00 2001 From: Thomas Li <47963215+lithomas1@users.noreply.github.com> Date: Wed, 14 May 2025 16:56:10 -0400 Subject: [PATCH 7/8] add more version checks --- pandas/_libs/window/meson.build | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/pandas/_libs/window/meson.build b/pandas/_libs/window/meson.build index 78169e781bc47..8c00a98b1241a 100644 --- a/pandas/_libs/window/meson.build +++ b/pandas/_libs/window/meson.build @@ -1,7 +1,14 @@ +cy_args = ['-X always_allow_keywords=true'] +# Use shared utility code to reduce wheel sizes +# copied from https://github.com/scikit-learn/scikit-learn/pull/31151/files +if cy.version().version_compare('>=3.1.0') + cython_args += ['--shared=pandas._libs._cyutility'] +endif + py.extension_module( 'aggregations', ['aggregations.pyx'], - cython_args: ['-X always_allow_keywords=true', '--shared=pandas._libs._cyutility'], + cython_args: cy_args, include_directories: [inc_np, inc_pd], subdir: 'pandas/_libs/window', override_options: ['cython_language=cpp'], @@ -11,7 +18,7 @@ py.extension_module( py.extension_module( 'indexers', ['indexers.pyx'], - cython_args: ['-X always_allow_keywords=true', '--shared=pandas._libs._cyutility'], + cython_args: cy_args, include_directories: [inc_np, inc_pd], subdir: 'pandas/_libs/window', install: true, From 9cba28450af96a2eb37c87975a392f32aabbce79 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 14 May 2025 21:30:55 +0000 Subject: [PATCH 8/8] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- meson.build | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/meson.build b/meson.build index ab3196fc44a7f..6bf88cc772a59 100644 --- a/meson.build +++ b/meson.build @@ -55,12 +55,15 @@ if cy.version().version_compare('>=3.1.0') install: false, output: '_cyutility.c', command: [ - cy, '-3', '--fast-fail', - '--generate-shared=' + meson.current_build_dir()/'_cyutility.c' + cy, + '-3', + '--fast-fail', + '--generate-shared=' + meson.current_build_dir() / '_cyutility.c', ], ) - py.extension_module('_cyutility', + py.extension_module( + '_cyutility', cython_shared_src, subdir: 'pandas/_libs', install: true,