Skip to content

Commit 7b33aa3

Browse files
authored
add paddlenlp_ops build script (#3081)
1 parent 5f9be9e commit 7b33aa3

File tree

1 file changed

+84
-0
lines changed

1 file changed

+84
-0
lines changed
Lines changed: 84 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,84 @@
1+
#!/usr/bin/env bash
2+
3+
# Copyright (c) 2025 PaddlePaddle Authors. All Rights Reserved.
4+
#
5+
# Licensed under the Apache License, Version 2.0 (the "License");
6+
# you may not use this file except in compliance with the License.
7+
# You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing, software
12+
# distributed under the License is distributed on an "AS IS" BASIS,
13+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
# See the License for the specific language governing permissions and
15+
# limitations under the License.
16+
17+
set -e
18+
ROOT_DIR=$PWD/
19+
nlp_dir=${ROOT_DIR}/PaddleNLP
20+
upload_path=${ROOT_DIR}/Bos/upload
21+
mkdir -p ${upload_path}
22+
paddle_whl=$1
23+
24+
########### 判断是否进行编译 ###########
25+
get_diff_case(){
26+
export FLAGS_build_enable=false
27+
cd ${nlp_dir}
28+
git diff --name-only HEAD~1 HEAD
29+
for file_name in `git diff --name-only HEAD~1 HEAD`;do
30+
arr_file_name=(${file_name//// })
31+
if [[ ${arr_file_name[0]} == "csrc" ]];then
32+
FLAGS_build_enable=true
33+
else
34+
continue
35+
fi
36+
done
37+
38+
if [[ ${FLAGS_build_enable} == true ]];then
39+
continue
40+
else
41+
exit 0
42+
fi
43+
}
44+
# get_diff_case
45+
46+
47+
# get python3.10
48+
wget -q https://paddlenlp.bj.bcebos.com/PaddleNLP_CI/PaddleNLP-Build-paddlenlp_ops/set_env.tar.gz
49+
mkdir -p ~/miniconda3/envs/set_env
50+
tar -zxf set_env.tar.gz -C ~/miniconda3/envs/set_env
51+
wget -q https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh -O miniconda.sh
52+
bash miniconda.sh -b -u -p /root/miniconda3
53+
source /root/miniconda3/etc/profile.d/conda.sh
54+
conda activate set_env
55+
python -m pip config set global.index-url http://pip.baidu.com/root/baidu/+simple/
56+
python -m pip config set install.trusted-host pip.baidu.com
57+
58+
# set LD_LIBRARY_PATH for cuda12.8
59+
# cuda_version=$(echo "$paddle_whl" | grep -oP 'Cuda\K[0-9.]+')
60+
# if [[ "$cuda_version" == "12.8" ]]; then
61+
# export LD_LIBRARY_PATH=/usr/local/lib/python3.10/dist-packages/nvidia/cusparse/lib/:${LD_LIBRARY_PATH}
62+
# fi
63+
64+
echo -e " ---- Install paddlepaddle-gpu ----"
65+
python -m pip install --user ${paddle_whl};
66+
python -c "import paddle;print('paddle');print(paddle.__version__);print(paddle.version.show())"
67+
echo " ---- Install paddlenlp_develop ----"
68+
python -m pip install --pre --upgrade paddlenlp -f https://www.paddlepaddle.org.cn/whl/paddlenlp.html --no-cache-dir
69+
python -c "import paddlenlp; print('paddlenlp commit:',paddlenlp.version.commit)"
70+
71+
72+
cd ${nlp_dir}/csrc
73+
bash tools/build_wheel.sh all
74+
set +e
75+
#python -c "import paddlenlp_ops"
76+
#ll ${nlp_dir}/csrc/gpu_dist
77+
78+
# for https://www.paddlepaddle.org.cn/whl/paddlenlp.html
79+
cp ${nlp_dir}/csrc/gpu_dist/p****.whl ${upload_path}/
80+
python -m pip install bce-python-sdk==0.8.74 --trusted-host pip.baidu-int.com --force-reinstall
81+
cd ${upload_path} && ls -A "${upload_path}"
82+
cd ${ROOT_DIR}/Bos && python upload.py ${upload_path} 'paddlenlp/wheels'
83+
rm -rf ${upload_path}
84+
echo -e " ---- upload wheels SUCCESS ----"

0 commit comments

Comments
 (0)