Thanks to visit codestin.com
Credit goes to github.com

Skip to content

Commit baa47b6

Browse files
w5688414w5688414
authored
Add ERNIE 3.0 based RocketQA Ranker models (PaddlePaddle#3019)
* Add ERNIE 3.0 based RocketQA Ranker models * Update Docs Introduction
1 parent 8980d64 commit baa47b6

3 files changed

Lines changed: 129 additions & 0 deletions

File tree

โ€Ždocs/model_zoo/transformers/ERNIE/contents.rstโ€Ž

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -52,5 +52,25 @@ ERNIEๆจกๅž‹ๆฑ‡ๆ€ป
5252
| | | 12-heads, 18M parameters. |
5353
| | | Trained on Chinese text. |
5454
+----------------------------------------------------------------------------------+--------------+----------------------------------------------------------------------------------+
55+
|``rocketqa-base-cross-encoder`` | Chinese | 12-layer, 768-hidden, |
56+
| | | 12-heads, 118M parameters. |
57+
| | | Trained on DuReader retrieval text. |
58+
+----------------------------------------------------------------------------------+--------------+----------------------------------------------------------------------------------+
59+
|``rocketqa-medium-cross-encoder`` | Chinese | 6-layer, 768-hidden, |
60+
| | | 12-heads, 75M parameters. |
61+
| | | Trained on DuReader retrieval text. |
62+
+----------------------------------------------------------------------------------+--------------+----------------------------------------------------------------------------------+
63+
|``rocketqa-mini-cross-encoder`` | Chinese | 6-layer, 384-hidden, |
64+
| | | 12-heads, 27M parameters. |
65+
| | | Trained on DuReader retrieval text. |
66+
+----------------------------------------------------------------------------------+--------------+----------------------------------------------------------------------------------+
67+
|``rocketqa-micro-cross-encoder`` | Chinese | 4-layer, 384-hidden, |
68+
| | | 12-heads, 23M parameters. |
69+
| | | Trained on DuReader retrieval text. |
70+
+----------------------------------------------------------------------------------+--------------+----------------------------------------------------------------------------------+
71+
|``rocketqa-nano-cross-encoder`` | Chinese | 4-layer, 312-hidden, |
72+
| | | 12-heads, 18M parameters. |
73+
| | | Trained on DuReader retrieval text. |
74+
+----------------------------------------------------------------------------------+--------------+----------------------------------------------------------------------------------+
5575

5676
.. _zhui/ernie-1.0-cluecorpussmall: https://github.com/PaddlePaddle/PaddleNLP/tree/develop/community/zhui/ernie-1.0-cluecorpussmall

โ€Žpaddlenlp/transformers/ernie/modeling.pyโ€Ž

Lines changed: 84 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -425,6 +425,80 @@ class ErniePretrainedModel(PretrainedModel):
425425
"use_task_id": True,
426426
"vocab_size": 40000
427427
},
428+
"rocketqa-base-cross-encoder": {
429+
"attention_probs_dropout_prob": 0.1,
430+
"hidden_act": "gelu",
431+
"hidden_dropout_prob": 0.1,
432+
"hidden_size": 768,
433+
"initializer_range": 0.02,
434+
"max_position_embeddings": 2048,
435+
"num_attention_heads": 12,
436+
"num_hidden_layers": 12,
437+
"task_type_vocab_size": 3,
438+
"type_vocab_size": 4,
439+
"use_task_id": True,
440+
"vocab_size": 40000
441+
},
442+
"rocketqa-medium-cross-encoder": {
443+
"attention_probs_dropout_prob": 0.1,
444+
"hidden_act": "gelu",
445+
"hidden_dropout_prob": 0.1,
446+
"hidden_size": 768,
447+
"intermediate_size": 3072,
448+
"initializer_range": 0.02,
449+
"max_position_embeddings": 2048,
450+
"num_attention_heads": 12,
451+
"num_hidden_layers": 6,
452+
"task_type_vocab_size": 16,
453+
"type_vocab_size": 4,
454+
"use_task_id": True,
455+
"vocab_size": 40000
456+
},
457+
"rocketqa-mini-cross-encoder": {
458+
"attention_probs_dropout_prob": 0.1,
459+
"hidden_act": "gelu",
460+
"hidden_dropout_prob": 0.1,
461+
"hidden_size": 384,
462+
"intermediate_size": 1536,
463+
"initializer_range": 0.02,
464+
"max_position_embeddings": 2048,
465+
"num_attention_heads": 12,
466+
"num_hidden_layers": 6,
467+
"task_type_vocab_size": 16,
468+
"type_vocab_size": 4,
469+
"use_task_id": True,
470+
"vocab_size": 40000
471+
},
472+
"rocketqa-micro-cross-encoder": {
473+
"attention_probs_dropout_prob": 0.1,
474+
"hidden_act": "gelu",
475+
"hidden_dropout_prob": 0.1,
476+
"hidden_size": 384,
477+
"intermediate_size": 1536,
478+
"initializer_range": 0.02,
479+
"max_position_embeddings": 2048,
480+
"num_attention_heads": 12,
481+
"num_hidden_layers": 4,
482+
"task_type_vocab_size": 16,
483+
"type_vocab_size": 4,
484+
"use_task_id": True,
485+
"vocab_size": 40000
486+
},
487+
"rocketqa-nano-cross-encoder": {
488+
"attention_probs_dropout_prob": 0.1,
489+
"hidden_act": "gelu",
490+
"hidden_dropout_prob": 0.1,
491+
"hidden_size": 312,
492+
"intermediate_size": 1248,
493+
"initializer_range": 0.02,
494+
"max_position_embeddings": 2048,
495+
"num_attention_heads": 12,
496+
"num_hidden_layers": 4,
497+
"task_type_vocab_size": 16,
498+
"type_vocab_size": 4,
499+
"use_task_id": True,
500+
"vocab_size": 40000
501+
},
428502
}
429503
resource_files_names = {"model_state": "model_state.pdparams"}
430504
pretrained_resource_files_map = {
@@ -472,6 +546,16 @@ class ErniePretrainedModel(PretrainedModel):
472546
"https://bj.bcebos.com/paddlenlp/models/transformers/ernie_3.0/ernie_3.0_micro_zh.pdparams",
473547
"ernie-3.0-nano-zh":
474548
"https://bj.bcebos.com/paddlenlp/models/transformers/ernie_3.0/ernie_3.0_nano_zh.pdparams",
549+
"rocketqa-base-cross-encoder":
550+
"https://paddlenlp.bj.bcebos.com/models/transformers/rocketqa/rocketqa-base-cross-encoder.pdparams",
551+
"rocketqa-medium-cross-encoder":
552+
"https://paddlenlp.bj.bcebos.com/models/transformers/rocketqa/rocketqa-medium-cross-encoder.pdparams",
553+
"rocketqa-mini-cross-encoder":
554+
"https://paddlenlp.bj.bcebos.com/models/transformers/rocketqa/rocketqa-mini-cross-encoder.pdparams",
555+
"rocketqa-micro-cross-encoder":
556+
"https://paddlenlp.bj.bcebos.com/models/transformers/rocketqa/rocketqa-micro-cross-encoder.pdparams",
557+
"rocketqa-nano-cross-encoder":
558+
"https://paddlenlp.bj.bcebos.com/models/transformers/rocketqa/rocketqa-nano-cross-encoder.pdparams",
475559
}
476560
}
477561
base_model_prefix = "ernie"

โ€Žpaddlenlp/transformers/ernie/tokenizer.pyโ€Ž

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -127,6 +127,16 @@ class ErnieTokenizer(PretrainedTokenizer):
127127
"https://bj.bcebos.com/paddlenlp/models/transformers/ernie_3.0/ernie_3.0_micro_zh_vocab.txt",
128128
"ernie-3.0-nano-zh":
129129
"https://bj.bcebos.com/paddlenlp/models/transformers/ernie_3.0/ernie_3.0_nano_zh_vocab.txt",
130+
"rocketqa-base-cross-encoder":
131+
"https://bj.bcebos.com/paddlenlp/models/transformers/ernie_3.0/ernie_3.0_base_zh_vocab.txt",
132+
"rocketqa-medium-cross-encoder":
133+
"https://bj.bcebos.com/paddlenlp/models/transformers/ernie_3.0/ernie_3.0_medium_zh_vocab.txt",
134+
"rocketqa-mini-cross-encoder":
135+
"https://bj.bcebos.com/paddlenlp/models/transformers/ernie_3.0/ernie_3.0_mini_zh_vocab.txt",
136+
"rocketqa-micro-cross-encoder":
137+
"https://bj.bcebos.com/paddlenlp/models/transformers/ernie_3.0/ernie_3.0_micro_zh_vocab.txt",
138+
"rocketqa-nano-cross-encoder":
139+
"https://bj.bcebos.com/paddlenlp/models/transformers/ernie_3.0/ernie_3.0_nano_zh_vocab.txt",
130140
}
131141
}
132142
pretrained_init_configuration = {
@@ -202,6 +212,21 @@ class ErnieTokenizer(PretrainedTokenizer):
202212
"ernie-3.0-nano-zh": {
203213
"do_lower_case": True
204214
},
215+
"rocketqa-base-cross-encoder": {
216+
"do_lower_case": True
217+
},
218+
"rocketqa-medium-cross-encoder": {
219+
"do_lower_case": True
220+
},
221+
"rocketqa-mini-cross-encoder": {
222+
"do_lower_case": True
223+
},
224+
"rocketqa-micro-cross-encoder": {
225+
"do_lower_case": True
226+
},
227+
"rocketqa-nano-cross-encoder": {
228+
"do_lower_case": True
229+
},
205230
}
206231

207232
def __init__(self,

0 commit comments

Comments
ย (0)