[PaddleNLP]update case for all_list (#3042)

Liujie0926 · web-flow · commit ca07101efe83 · 2025-05-09T15:23:01.000+08:00
* fix path for PaddleNLP * fix * Revert "fix" This reverts commit c7b3441. * fix * update case for all_list * update * fix error * fix requirements * fix cmd of document * fix cmd * fix * fix requirements * clean install * fix install
diff --git a/models_restruct/PaddleNLP/cases/applications^information_extraction^document.yaml b/models_restruct/PaddleNLP/cases/applications^information_extraction^document.yaml
@@ -21,23 +21,25 @@ case:
         params:
           - --device gpu
           - --max_steps 25
-          - --logging_steps 1
+          - --logging_steps 5
+          - --save_steps 25
           - --eval_steps 25
           - --seed 42
           - --model_name_or_path uie-x-base
           - --output_dir ./checkpoint/model_best
           - --train_path data/train.txt
           - --dev_path data/dev.txt
           - --max_seq_len 512
-          - --per_device_train_batch_size  2
-          - --per_device_eval_batch_size 2
-          - --num_train_epochs 20
+          - --per_device_train_batch_size  8
+          - --per_device_eval_batch_size 8
+          - --num_train_epochs 10
           - --learning_rate 1e-5
           - --do_train
           - --do_eval
           - --do_export
           - --export_model_dir ./checkpoint/model_best
           - --overwrite_output_dir
+          - --disable_tqdm True
           - --metric_for_best_model eval_f1
           - --load_best_model_at_end  True
           - --save_total_limit 1
diff --git a/models_restruct/PaddleNLP/cases/examples^language_model^convbert.yaml b/models_restruct/PaddleNLP/cases/examples^language_model^convbert.yaml
@@ -3,7 +3,7 @@ case:
     train:
       -
         name: single
-        path: slm/examples/language_model/convbert
+        path: slm/model_zoo/convbert
         cmd: python -u run_glue.py
         params:
           - --model_type convbert
@@ -27,7 +27,7 @@ case:
     train:
       -
         name: single
-        path: slm/examples/language_model/convbert
+        path: slm/model_zoo/convbert
         cmd: python -u run_glue.py
         params:
           - --model_type convbert
diff --git a/models_restruct/PaddleNLP/cases/examples^language_model^luke.yaml b/models_restruct/PaddleNLP/cases/examples^language_model^luke.yaml
@@ -3,15 +3,16 @@ case:
     train:
       -
         name: prepare
-        path: slm/examples/language_model/luke
+        path: slm/model_zoo/luke
         cmd: mkdir data && cd data && wget https://data.deepai.org/squad1.1.zip --no-check-certificate
       -
         name: prepare_data
-        path: slm/examples/language_model/luke/data
+        path: slm/model_zoo/luke/data
         cmd: unzip squad1.1.zip && mv train-v1.1.json train.json && mv dev-v1.1.json dev.json
       -
         name: run_squad
-        path: slm/examples/language_model/luke
+        path: slm/model_zoo/luke
+
         cmd: python -m paddle.distributed.launch run_squad.py
         params:
           - --model_type luke
@@ -27,7 +28,7 @@ case:
           - --save_step 2
       -
         name: run_open_entity.py
-        path: slm/examples/language_model/luke
+        path: slm/model_zoo/luke
         cmd: python -m paddle.distributed.launch run_open_entity.py
         params:
           - --model_type luke-large
diff --git a/models_restruct/PaddleNLP/cases/examples^language_model^megatronbert.yaml b/models_restruct/PaddleNLP/cases/examples^language_model^megatronbert.yaml
@@ -3,32 +3,29 @@ case:
     train:
       -
         name: run_squad
-        path: slm/examples/language_model/luke
+        path: slm/model_zoo/megatronbert
         cmd: python -m paddle.distributed.launch run_squad.py
         params:
-          - --model_type luke
-          - --device gpu
-          - --learning_rate 15e-6
-          - --num_train_epochs 1
-          - --batch_size 8
-          - --do_predict
           - --do_train
-          - --model_name_or_path luke-large
-          - --logging_steps 1
-          - --max_steps 2
-          - --save_step 2
+          - --version_2_with_negative
+          - --do_predict
+          - --batch_size=8
+          - --model_name_or_path=megatronbert-cased
+          - --learning_rate=1e-5
+          - --output_dir=output/
+          - --device=gpu
+          - --num_train_epochs=1
       -
-        name: run_open_entity
-        path: slm/examples/language_model/luke
-        cmd: python -m paddle.distributed.launch run_open_entity.py
+        name: run_glue
+        path: slm/model_zoo/megatronbert
+        cmd: python -m paddle.distributed.launch run_glue.py
         params:
-          - --model_type luke-large
-          - --data_dir data/
-          - --output_dir ./output
-          - --device gpu
-          - --learning_rate 1e-5
-          - --num_train_epochs 1
-          - --batch_size 2
+          - --task_name=mnli
+          - --output_dir=output/
+          - --model_name_or_path=megatronbert-cased
+          - --learning_rate=1e-5
+          - --device=gpu
+          - --num_train_epochs=1
     eval: skipped
     infer: skipped
     export: skipped
diff --git a/models_restruct/PaddleNLP/cases/examples^language_model^rembert.yaml b/models_restruct/PaddleNLP/cases/examples^language_model^rembert.yaml
@@ -2,22 +2,26 @@ case:
   linux:
     train:
       -
-        name: run_thucnews
-        path: slm/examples/language_model/roformer
-        cmd: python -m paddle.distributed.launch run_thucnews.py
+        name: prepare
+        path: slm/model_zoo/rembert
+        cmd: mkdir data && cd data && wget https://dl.fbaipublicfiles.com/XNLI/XNLI-MT-1.0.zip --no-check-certificate \
+        && unzip XNLI-MT-1.0.zip && rm XNLI-MT-1.0.zip && mv XNLI-MT-1.0/multinli/multinli.train.en.tsv ./
+      -
+        name: run_main
+        path: slm/model_zoo/rembert
+        cmd: python -m paddle.distributed.launch main.py
         params:
-          - --model_type roformer
-          - --model_name_or_path roformer-chinese-base
-          - --max_seq_length 256
-          - --batch_size 64
-          - --learning_rate 2e-5
-          - --num_train_epochs 3
-          - --logging_steps 1
-          - --save_steps 2
-          - --output_dir ./thucnews/
+          - --model_type rembert
+          - --data_dir data/
+          - --output_dir output/
           - --device gpu
-          - --use_amp True
-          - --max_steps 2
+          - --learning_rate 1e-5
+          - --num_train_epochs 3
+          - --train_batch_size 16
+          - --do_train
+          - --do_eval
+          - --task xnli
+          - --eval_step 2
     eval: skipped
     infer: skipped
     export: skipped
diff --git a/models_restruct/PaddleNLP/cases/examples^language_model^roformer.yaml b/models_restruct/PaddleNLP/cases/examples^language_model^roformer.yaml
@@ -3,7 +3,7 @@ case:
     train:
       -
         name: run_thucnews
-        path: slm/examples/language_model/roformer
+        path: slm/model_zoo/roformer
         cmd: python -m paddle.distributed.launch run_thucnews.py
         params:
           - --model_type roformer
@@ -20,7 +20,7 @@ case:
           - --max_steps 2
       -
         name: run_cail2019_scm
-        path: slm/examples/language_model/roformer
+        path: slm/model_zoo/roformer
         cmd: python -m paddle.distributed.launch run_cail2019_scm.py
         params:
           - --model_type roformer_mean_pooling
@@ -44,7 +44,7 @@ case:
     train:
       -
         name: run_thucnews
-        path: slm/examples/language_model/roformer
+        path: slm/model_zoo/roformer
         cmd: python -m paddle.distributed.launch run_thucnews.py
         params:
           - --model_type roformer
@@ -61,7 +61,7 @@ case:
           - --max_steps 2
       -
         name: run_cail2019_scm
-        path: slm/examples/language_model/roformer
+        path: slm/model_zoo/roformer
         cmd: python -m paddle.distributed.launch run_cail2019_scm.py
         params:
           - --model_type roformer_mean_pooling
diff --git a/models_restruct/PaddleNLP/cases/examples^language_model^t5.yaml b/models_restruct/PaddleNLP/cases/examples^language_model^t5.yaml
@@ -3,7 +3,7 @@ case:
     train:
       -
         name: train_rte
-        path: slm/examples/language_model/t5
+        path: slm/model_zoo/t5
         cmd: python run_glue.py
         params:
           - --model_name_or_path t5-base
@@ -22,7 +22,7 @@ case:
           - --output_dir outputs/rte/
       -
         name: trainer_finetune_rte
-        path: slm/examples/language_model/t5
+        path: slm/model_zoo/t5
         cmd: python -m paddle.distributed.launch run_glue_trainer.py
         params:
           - --model_name_or_path t5-base
@@ -43,7 +43,7 @@ case:
           - --max_steps 2
       -
         name: trainer_finetune_cluewsc2020
-        path: slm/examples/language_model/t5
+        path: slm/model_zoo/t5
         cmd: python -m paddle.distributed.launch run_glue_trainer.py
         params:
           - --model_name_or_path Langboat/mengzi-t5-base-mt
@@ -68,7 +68,7 @@ case:
     train:
       -
         name: train_rte
-        path: slm/examples/language_model/t5
+        path: slm/model_zoo/t5
         cmd: python run_glue.py
         params:
           - --model_name_or_path t5-base
@@ -87,7 +87,7 @@ case:
           - --output_dir outputs/rte/
       -
         name: trainer_finetune_rte
-        path: slm/examples/language_model/t5
+        path: slm/model_zoo/t5
         cmd: python -m paddle.distributed.launch run_glue_trainer.py
         params:
           - --model_name_or_path t5-base
@@ -106,9 +106,10 @@ case:
           - --fp16_opt_level "O1"
           - --output_dir outputs/rte/
           - --max_steps 2
+          - --overwrite_output_dir
       -
         name: trainer_finetune_cluewsc2020
-        path: slm/examples/language_model/t5
+        path: slm/model_zoo/t5
         cmd: python -m paddle.distributed.launch run_glue_trainer.py
         params:
           - --model_name_or_path Langboat/mengzi-t5-base-mt
@@ -124,6 +125,7 @@ case:
           - --save_total_limit 3
           - --output_dir outputs/clue/cluewsc2020
           - --max_steps 2
+          - --overwrite_output_dir
     eval: skipped
     infer: skipped
     export: skipped
diff --git a/models_restruct/PaddleNLP/cases/examples^language_model^xlm.yaml b/models_restruct/PaddleNLP/cases/examples^language_model^xlm.yaml
@@ -3,15 +3,15 @@ case:
     train:
       -
         name: clone_kytea
-        path: slm/examples/language_model/xlm
+        path: slm/model_zoo/xlm
         cmd: git clone https://github.com/neubig/kytea.git
       -
         name: build_kytea
-        path: slm/examples/language_model/xlm/kytea
+        path: slm/model_zoo/xlm/kytea
         cmd: autoreconf -i && ./configure --prefix=$HOME/local && make && make install
       -
         name: train_xnli
-        path: slm/examples/language_model/xlm
+        path: slm/model_zoo/xlm
         cmd: python xnli_train.py
         params:
           - --batch_size 8
@@ -23,7 +23,7 @@ case:
     eval:
       -
         name: eval_xnli
-        path: slm/examples/language_model/xlm
+        path: slm/model_zoo/xlm
         cmd: python xnli_eval.py
         params:
           - --batch_size 8
diff --git a/models_restruct/PaddleNLP/cases/examples^multimodal^layoutlm.yaml b/models_restruct/PaddleNLP/cases/examples^multimodal^layoutlm.yaml
@@ -10,7 +10,7 @@ case:
         path: slm/examples/multimodal/layoutlm
         cmd:   python train_funsd.py
         params:
-          - -data_dir "./data/"
+          - --data_dir "./data/"
           - --model_name_or_path "layoutlm-base-uncased"
           - --do_lower_case
           - --max_seq_length 512
diff --git a/models_restruct/PaddleNLP/cases/model_zoo^ernie-tiny.yaml b/models_restruct/PaddleNLP/cases/model_zoo^ernie-tiny.yaml
@@ -3,11 +3,11 @@ case:
     train:
       -
         name: prepare
-        path: slm/model_zoo/ernie-tiny/data
+        path: slm/model_zoo/ernie-3.0-tiny/data
         cmd: wget https://paddle-qa.bj.bcebos.com/paddlenlp/ernie-tiny.tar.gz && tar -xzvf ernie-tiny.tar.gz && cp -r ernie-tiny/* ./
       -
         name: fintune
-        path: slm/model_zoo/ernie-tiny/
+        path: slm/model_zoo/ernie-3.0-tiny/
         cmd: python run_train.py
         params:
           - --device gpu
@@ -39,7 +39,7 @@ case:
             evaluation: "="
       -
         name: compress
-        path: slm/model_zoo/ernie-tiny/
+        path: slm/model_zoo/ernie-3.0-tiny/
         cmd: python run_train.py
         params:
           - --do_compress
@@ -75,7 +75,7 @@ case:
     eval:
       -
         name: eval
-        path: slm/model_zoo/ernie-tiny/
+        path: slm/model_zoo/ernie-3.0-tiny/
         cmd: python run_eval.py
         params:
           - --device gpu
diff --git a/models_restruct/PaddleNLP/cases/model_zoo^ernie-vil2.0.yaml b/models_restruct/PaddleNLP/cases/model_zoo^ernie-vil2.0.yaml
@@ -64,7 +64,7 @@ case:
       -
         name: export
         path: slm/model_zoo/ernie-vil2.0
-        cmd: export_model.py
+        cmd: python export_model.py
         params:
           - --model_path=output_pd/checkpoint-2/
           - --output_path=./infer_model/
diff --git a/models_restruct/PaddleNLP/diy_build/PaddleNLP_Build.py b/models_restruct/PaddleNLP/diy_build/PaddleNLP_Build.py
@@ -64,12 +64,9 @@ def build_paddlenlp(self):
 
         if platform == "linux" or platform == "linux_convergence":
             os.system("python -m pip install -U setuptools -i https://mirror.baidu.com/pypi/simple")
-            os.system("python -m pip install nltk h5py")
             os.system("python -m pip install --user -r requirements_nlp.txt -i https://mirror.baidu.com/pypi/simple")
             os.system("python -m pip uninstall protobuf -y")
-            os.system("python -m pip uninstall protobuf -y")
-            os.system("python -m pip uninstall protobuf -y")
-            os.system("python -m pip install protobuf==3.20.2")
+            os.system("python -m pip install protobuf==3.20.2 -i https://mirror.baidu.com/pypi/simple")
             # os.system("python -m pip install {}".format(paddle_whl))  # install paddle for lac requirement paddle>=1.6
         else:
             os.system("python -m pip install  --user -r requirements_win.txt -i https://mirror.baidu.com/pypi/simple")
@@ -139,8 +136,7 @@ def build_paddlenlp(self):
         os.chdir(path_now)
 
         os.system("python -m pip uninstall protobuf -y")
-        os.system("python -m pip uninstall protobuf -y")
-        os.system("python -m pip install protobuf==3.20.2")
+        os.system("python -m pip install protobuf==3.20.2 -i https://mirror.baidu.com/pypi/simple")
         os.system("python -m pip list")
         import paddle
 
diff --git a/models_restruct/PaddleNLP/requirements_nlp.txt b/models_restruct/PaddleNLP/requirements_nlp.txt
@@ -1,7 +1,7 @@
 numpy<=1.26.4
 regex
-nltk==3.5
-xlrd==1.2.0
+nltk
+xlrd
 pybind11
 tool_helpers
 editdistance
@@ -27,3 +27,4 @@ evaluate==0.2.2
 sacremoses
 pythainlp
 kytea
+pgl
diff --git a/models_restruct/PaddleNLP/tools/PaddleNLP_ALL_new_list b/models_restruct/PaddleNLP/tools/PaddleNLP_ALL_new_list
