Skip to content

Commit cee683c

Browse files
huawei-noah-adminhuawei-noah-admin
authored
Merge pull request #75 from henryxiao1997/master
add the codes of PERT project
2 parents 0fa6663 + 6087411 commit cee683c

37 files changed

+53786
-0
lines changed

‎noahime/PERT/Configs/bert_config_base_py.json‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
{
2+
"hidden_size": 768,
3+
"hidden_act": "gelu",
4+
"initializer_range": 0.02,
5+
"vocab_size": 404,
6+
"hidden_dropout_prob": 0.1,
7+
"num_attention_heads": 12,
8+
"type_vocab_size": 2,
9+
"max_position_embeddings": 16,
10+
"num_hidden_layers": 12,
11+
"intermediate_size": 3072,
12+
"attention_probs_dropout_prob": 0.1,
13+
"num_labels": 6727,
14+
"seq_length": 16,
15+
"max_relative_position": 64,
16+
"use_relative_position": true
17+
}

‎noahime/PERT/Configs/bert_config_large_nezha_py.json‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
{
2+
"attention_probs_dropout_prob": 0.1,
3+
"hidden_act": "gelu",
4+
"hidden_dropout_prob": 0.1,
5+
"hidden_size": 1024,
6+
"initializer_range": 0.02,
7+
"intermediate_size": 4096,
8+
"max_position_embeddings": 16,
9+
"num_attention_heads": 16,
10+
"num_hidden_layers": 24,
11+
"type_vocab_size": 2,
12+
"vocab_size": 404,
13+
"num_labels": 6727,
14+
"seq_length": 16,
15+
"max_relative_position": 64,
16+
"use_relative_position": true
17+
}

‎noahime/PERT/Configs/bert_config_medium_nezha_py.json‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
{
2+
"hidden_size": 512,
3+
"hidden_act": "gelu",
4+
"initializer_range": 0.02,
5+
"vocab_size": 404,
6+
"hidden_dropout_prob": 0.1,
7+
"num_attention_heads": 8,
8+
"type_vocab_size": 2,
9+
"max_position_embeddings": 512,
10+
"num_hidden_layers": 8,
11+
"intermediate_size": 2048,
12+
"attention_probs_dropout_prob": 0.1,
13+
"num_labels": 6727,
14+
"seq_length": 16,
15+
"max_relative_position": 64,
16+
"use_relative_position": true
17+
}

‎noahime/PERT/Configs/bert_config_mini_nezha_py.json‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
{
2+
"hidden_size": 256,
3+
"hidden_act": "gelu",
4+
"initializer_range": 0.02,
5+
"vocab_size": 404,
6+
"hidden_dropout_prob": 0.1,
7+
"num_attention_heads": 4,
8+
"type_vocab_size": 2,
9+
"max_position_embeddings": 16,
10+
"num_hidden_layers": 4,
11+
"intermediate_size": 1024,
12+
"attention_probs_dropout_prob": 0.1,
13+
"num_labels": 6727,
14+
"seq_length": 16,
15+
"max_relative_position": 64,
16+
"use_relative_position": true
17+
}

‎noahime/PERT/Configs/bert_config_small_nezha_py.json‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
{
2+
"hidden_size": 512,
3+
"hidden_act": "gelu",
4+
"initializer_range": 0.02,
5+
"vocab_size": 404,
6+
"hidden_dropout_prob": 0.1,
7+
"num_attention_heads": 8,
8+
"type_vocab_size": 2,
9+
"max_position_embeddings": 16,
10+
"num_hidden_layers": 4,
11+
"intermediate_size": 2048,
12+
"attention_probs_dropout_prob": 0.1,
13+
"num_labels": 6727,
14+
"seq_length": 16,
15+
"max_relative_position": 64,
16+
"use_relative_position": true
17+
}

‎noahime/PERT/Configs/bert_config_tiny_nezha_py.json‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
{
2+
"hidden_size": 128,
3+
"hidden_act": "gelu",
4+
"initializer_range": 0.02,
5+
"hidden_dropout_prob": 0.1,
6+
"num_attention_heads": 2,
7+
"type_vocab_size": 2,
8+
"max_position_embeddings": 16,
9+
"num_hidden_layers": 2,
10+
"intermediate_size": 512,
11+
"attention_probs_dropout_prob": 0.1,
12+
"vocab_size": 404,
13+
"num_labels": 6727,
14+
"seq_length": 16,
15+
"max_relative_position": 64,
16+
"use_relative_position": true
17+
}

0 commit comments

Comments
 (0)