Skip to content

Commit 94548c7

Browse files
Jinyu-WJinyu Wangchaosddp
authored
Update pymaro build wheel logic (#587)
* add libffi-dev, rust toolchain to linux build wheel * try updating build wheel * update python version in build wheel script * ignore install requirements in build wheel workflow * fix python versions in build wheel * Update build wheel action (#586) * avoid install unused package while building wheels * move source package to windows, as it will cause permission issue on linux * fix manylinux python version * remove duplicated definition --------- Co-authored-by: Jinyu Wang <[email protected]> Co-authored-by: Chaos Yu <[email protected]>
1 parent 52ae7a4 commit 94548c7

13 files changed

+120
-114
lines changed

‎.github/workflows/build_wheel.yml

Lines changed: 10 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -41,26 +41,29 @@ jobs:
4141
run: |
4242
python setup.py bdist_wheel
4343
44+
- name: Build source package on Windows
45+
if: runner.os == 'Windows' && matrix.python-version == '3.7'
46+
run: |
47+
python setup.py sdist
48+
4449
- name: Build manylinux wheel
4550
if: runner.os == 'Linux' && matrix.python-version == '3.7'
4651
uses: RalfG/[email protected]_x86_64
52+
env:
53+
GITHUB_BUILD_ACTION: True
4754
with:
48-
python-versions: 'cp37-cp37m cp38-cp38m cp39-cp39m'
55+
python-versions: 'cp37-cp37m cp38-cp38 cp39-cp39'
4956
build-requirements: 'numpy'
50-
pip-wheel-args: '-w ./wheelhouse' # save wheel packages to wheelhouse folder
57+
system-packages: 'libffi-devel'
5158
pre-build-command: 'export LD_LIBRARY_PATH=/usr/local/lib/:$LD_LIBRARY_PATH'
59+
pip-wheel-args: '-w ./wheelhouse' # save wheel packages to wheelhouse folder
5260

5361
- name: Move valid packages to dist folder for manylinux
5462
if: runner.os == 'Linux' && matrix.python-version == '3.7'
5563
run: |
5664
mkdir -p dist
5765
cp wheelhouse/pymaro-*-manylinux*.whl dist
5866
59-
- name: Build source package on linux
60-
if: runner.os == 'Linux' && matrix.python-version == '3.7'
61-
run: |
62-
python setup.py sdist
63-
6467
- name: Upload linux wheel to artifact
6568
uses: actions/upload-artifact@v2
6669
with:

‎scripts/build_maro.bat

Lines changed: 10 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,10 @@
1-
@ECHO OFF
2-
3-
rem script to build maro locally on Windows, usually for development
4-
5-
chdir "%~dp0.."
6-
7-
rem compile cython files
8-
call scripts\compile_cython.bat
9-
10-
python setup.py build_ext -i
1+
@ECHO OFF
2+
3+
rem script to build maro locally on Windows, usually for development
4+
5+
chdir "%~dp0.."
6+
7+
rem compile cython files
8+
call scripts\compile_cython.bat
9+
10+
python setup.py build_ext -i

‎scripts/build_maro.sh

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -10,4 +10,4 @@ fi
1010
# compile cython files first
1111
bash ./scripts/compile_cython.sh
1212

13-
python setup.py build_ext -i
13+
python setup.py build_ext -i

‎scripts/build_sdist.bat

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,3 @@
1-
chdir "%~dp0.."
2-
3-
python setup.py sdist
1+
chdir "%~dp0.."
2+
3+
python setup.py sdist

‎scripts/build_sdist.sh

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -10,4 +10,4 @@ fi
1010

1111
bash ./scripts/compile_cython.sh
1212

13-
python setup.py sdist
13+
python setup.py sdist

‎scripts/build_wheel.bat

Lines changed: 14 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -1,14 +1,14 @@
1-
2-
rem script to build wheel package on Windows
3-
rem NOTE: Before building the wheels, please make sure you have setup-up the environment.
4-
rem for python 3.6/3.7 we need vs++14
5-
6-
chdir "%~dp0.."
7-
8-
call scripts\compile_cython.bat
9-
10-
pip install -r maro/requirements.build.txt
11-
pip install wheel
12-
pip install --upgrade setuptools
13-
14-
python setup.py bdist_wheel
1+
2+
rem script to build wheel package on Windows
3+
rem NOTE: Before building the wheels, please make sure you have setup-up the environment.
4+
rem for python 3.6/3.7 we need vs++14
5+
6+
chdir "%~dp0.."
7+
8+
call scripts\compile_cython.bat
9+
10+
pip install -r maro/requirements.build.txt
11+
pip install wheel
12+
pip install --upgrade setuptools
13+
14+
python setup.py bdist_wheel

‎scripts/build_wheel.sh

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,7 @@ set -e -x
88
cd /maro
99

1010
# Compile wheels
11-
for PYVER in 6 7; do
11+
for PYVER in 7 8 9; do
1212
PYBIN="/opt/python/cp3${PYVER}-cp3${PYVER}m/bin"
1313

1414
"${PYBIN}/pip" install -r maro/requirements.build.txt

‎scripts/install_maro.bat

Lines changed: 17 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -1,17 +1,17 @@
1-
@ECHO OFF
2-
3-
rem Script to install MARO in editable mode on Windows,
4-
rem usually for development.
5-
6-
chdir "%~dp0.."
7-
8-
rem Install dependencies.
9-
pip install -r .\maro\requirements.build.txt
10-
11-
rem Compile cython files.
12-
call .\scripts\compile_cython.bat
13-
14-
call .\scripts\install_torch.bat
15-
16-
rem Install MARO in editable mode.
17-
pip install -e .
1+
@ECHO OFF
2+
3+
rem Script to install MARO in editable mode on Windows,
4+
rem usually for development.
5+
6+
chdir "%~dp0.."
7+
8+
rem Install dependencies.
9+
pip install -r .\maro\requirements.build.txt
10+
11+
rem Compile cython files.
12+
call .\scripts\compile_cython.bat
13+
14+
call .\scripts\install_torch.bat
15+
16+
rem Install MARO in editable mode.
17+
pip install -e .

‎scripts/install_torch.bat

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1 +1 @@
1-
pip install torch===1.6.0 torchvision===0.7.0 -f https://download.pytorch.org/whl/torch_stable.html
1+
pip install torch===1.6.0 torchvision===0.7.0 -f https://download.pytorch.org/whl/torch_stable.html

‎scripts/run_tests.bat

Lines changed: 17 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -1,17 +1,17 @@
1-
rem script to run all the test script under tests folder which the file name match test_xxxx.py
2-
3-
chdir "%~dp0.."
4-
5-
set "PYTHONPAH=."
6-
7-
call scripts/build_maro.bat
8-
9-
rem install requirements
10-
11-
pip install -r ./tests/requirements.test.txt
12-
13-
rem show coverage
14-
15-
coverage run --rcfile=./tests/.coveragerc
16-
17-
coverage report --rcfile=./tests/.coveragerc
1+
rem script to run all the test script under tests folder which the file name match test_xxxx.py
2+
3+
chdir "%~dp0.."
4+
5+
set "PYTHONPAH=."
6+
7+
call scripts/build_maro.bat
8+
9+
rem install requirements
10+
11+
pip install -r ./tests/requirements.test.txt
12+
13+
rem show coverage
14+
15+
coverage run --rcfile=./tests/.coveragerc
16+
17+
coverage report --rcfile=./tests/.coveragerc

‎scripts/run_tests.sh

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -17,4 +17,4 @@ pip install -r ./tests/requirements.test.txt
1717

1818
coverage run --rcfile=./tests/.coveragerc
1919

20-
coverage report --rcfile=./tests/.coveragerc
20+
coverage report --rcfile=./tests/.coveragerc

‎scripts/show_test_coverage_html.bat

Lines changed: 19 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -1,19 +1,19 @@
1-
2-
chdir "%~dp0.."
3-
4-
rem remove old htmlcov
5-
@RD /S /Q htmlcov
6-
7-
rem generate html
8-
9-
coverage html
10-
11-
rem host html
12-
13-
cd htmlcov
14-
15-
REM python -m http.server 8888
16-
17-
start python -m http.server 8888
18-
19-
start "" "http://localhost:8888"
1+
2+
chdir "%~dp0.."
3+
4+
rem remove old htmlcov
5+
@RD /S /Q htmlcov
6+
7+
rem generate html
8+
9+
coverage html
10+
11+
rem host html
12+
13+
cd htmlcov
14+
15+
REM python -m http.server 8888
16+
17+
start python -m http.server 8888
18+
19+
start "" "http://localhost:8888"

‎setup.py

Lines changed: 25 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -25,7 +25,6 @@
2525
# Set environment variable to skip deployment process of MARO
2626
os.environ["SKIP_DEPLOYMENT"] = "TRUE"
2727

28-
2928
# root path to backend
3029
BASE_SRC_PATH = "./maro/backends"
3130
# backend module name
@@ -51,7 +50,6 @@
5150
),
5251
)
5352

54-
5553
include_dirs.append(numpy.get_include())
5654

5755
extensions.append(
@@ -84,9 +82,31 @@
8482
),
8583
)
8684

87-
specific_requires = []
85+
# It is not necessary to install these packages when using manylinux action, as we only need numpy to build wheels
86+
# NOTE: install following package will cause build error in current(2023-03-14) manylinux image
87+
if "GITHUB_BUILD_ACTION" in os.environ:
88+
install_requires = []
89+
else:
90+
install_requires = [
91+
# TODO: use a helper function to collect these
92+
"holidays>=0.10.3",
93+
"numpy>=1.19.5",
94+
"pandas>=0.25.3",
95+
"paramiko>=2.9.2",
96+
"ptvsd>=4.3.2",
97+
"python_dateutil>=2.8.1",
98+
"PyYAML>=5.4.1",
99+
"pyzmq>=19.0.2",
100+
"redis>=3.5.3",
101+
"requests>=2.25.1",
102+
"scipy>=1.7.0",
103+
"tabulate>=0.8.5",
104+
"torch>=1.6.0, <1.14.0",
105+
"tornado>=6.1",
106+
]
107+
88108
if sys.version.startswith("3.6"):
89-
specific_requires.append("dataclasses>=0.5")
109+
install_requires.append("dataclasses>=0.5")
90110

91111
readme = io.open("./maro/README.rst", encoding="utf-8").read()
92112

@@ -133,24 +153,7 @@
133153
setup_requires=[
134154
"numpy<1.20.0",
135155
],
136-
install_requires=[
137-
# TODO: use a helper function to collect these
138-
"holidays>=0.10.3",
139-
"numpy>=1.19.5",
140-
"pandas>=0.25.3",
141-
"paramiko>=2.9.2",
142-
"ptvsd>=4.3.2",
143-
"python_dateutil>=2.8.1",
144-
"PyYAML>=5.4.1",
145-
"pyzmq>=19.0.2",
146-
"redis>=3.5.3",
147-
"requests>=2.25.1",
148-
"scipy>=1.7.0",
149-
"tabulate>=0.8.5",
150-
"torch>=1.6.0, <1.14.0",
151-
"tornado>=6.1",
152-
]
153-
+ specific_requires,
156+
install_requires=install_requires,
154157
entry_points={
155158
"console_scripts": [
156159
"maro=maro.cli.maro:main",

0 commit comments

Comments
 (0)