[PaddlePaddle Hackathon] 第51题

community/iverxin/bert-base-japanese-char-whole-word-masking/README.md

@@ -0,0 +1,5 @@
+## bert-base-japanese
+12 repeating layers, 768-hidden, 12-heads. 
+
+This version of the model processes input texts with word-level  tokenization based on the IPA dictionary, followed by character-level  tokenization. Additionally, the model is trained with the whole word masking enabled  for the masked language modeling (MLM) objective.
+[reference](https://huggingface.co/cl-tohoku/bert-base-japanese-char-whole-word-masking) 

community/iverxin/bert-base-japanese-char-whole-word-masking/files.json

@@ -0,0 +1,3 @@
+{
+  "bert-base-japanese-char-whole-word-masking": "https://paddlenlp.bj.bcebos.com/models/transformers/community/iverxin/bert-base-japanese-char-whole-word-masking/bert-base-japanese-char-whole-word-masking.pdparams"
+}

community/iverxin/bert-base-japanese-char/README.md

@@ -0,0 +1,5 @@
+## bert-base-japanese
+12 repeating layers, 768-hidden, 12-heads.
+
+This version of the model processes input texts with word-level  tokenization based on the IPA dictionary, followed by character-level  tokenization. 
+[reference](https://huggingface.co/cl-tohoku/bert-base-japanese-char)

community/iverxin/bert-base-japanese-char/files.json

@@ -0,0 +1,3 @@
+{
+  "bert-base-japanese-char": "https://paddlenlp.bj.bcebos.com/models/transformers/community/iverxin/bert-base-japanese-char/bert-base-japanese-char.pdparams"
+}

community/iverxin/bert-base-japanese-whole-word-masking/README.md

@@ -0,0 +1,5 @@
+## bert-base-japanese
+12 repeating layers, 768-hidden, 12-heads. 
+
+This version of the model processes input texts with word-level  tokenization based on the IPA dictionary, followed by the WordPiece  subword tokenization. Additionally, the model is trained with the whole word masking enabled  for the masked language modeling (MLM) objective. 
+[reference](https://huggingface.co/cl-tohoku/bert-base-japanese-whole-word-masking)

community/iverxin/bert-base-japanese-whole-word-masking/files.json

@@ -0,0 +1,3 @@
+{
+  "bert-base-japanese-whole-word-masking": "https://paddlenlp.bj.bcebos.com/models/transformers/community/iverxin/bert-base-japanese-whole-word-masking/bert-base-japanese-whole-word-masking.pdparams"
+}

community/iverxin/bert-base-japanese/README.md

@@ -0,0 +1,6 @@
+## bert-base-japanese
+12 repeating layers, 768-hidden, 12-heads. 
+
+This version of the model processes input texts with word-level  tokenization based on the IPA dictionary, followed by the WordPiece  subword tokenization. 
+[reference](https://huggingface.co/cl-tohoku/bert-base-japanese)
+

community/iverxin/bert-base-japanese/files.json

@@ -0,0 +1,3 @@
+{
+  "bert-base-japanese": "https://paddlenlp.bj.bcebos.com/models/transformers/community/iverxin/bert-base-japanese/bert-base-japanese.pdparams"
+}

paddlenlp/transformers/__init__.py

@@ -18,6 +18,7 @@
 
 from .bert.modeling import *
 from .bert.tokenizer import *
+from .bert_japanese.tokenizer import *
 from .ernie.modeling import *
 from .ernie.tokenizer import *
 from .gpt.modeling import *

paddlenlp/transformers/bert/modeling.py

@@ -270,6 +270,62 @@ class BertPretrainedModel(PretrainedModel):
             "initializer_range": 0.02,
             "pad_token_id": 0,
         },
+        "bert-base-japanese": {
+            "vocab_size": 32000,
+            "hidden_size": 768,
+            "num_hidden_layers": 12,
+            "num_attention_heads": 12,
+            "intermediate_size": 3072,
+            "hidden_act": "gelu",
+            "hidden_dropout_prob": 0.1,
+            "attention_probs_dropout_prob": 0.1,
+            "max_position_embeddings": 512,
+            "type_vocab_size": 2,
+            "initializer_range": 0.02,
+            "pad_token_id": 0,
+        },
+        "bert-base-japanese-whole-word-masking": {
+            "vocab_size": 30522,
+            "hidden_size": 768,
+            "num_hidden_layers": 12,
+            "num_attention_heads": 12,
+            "intermediate_size": 3072,
+            "hidden_act": "gelu",
+            "hidden_dropout_prob": 0.1,
+            "attention_probs_dropout_prob": 0.1,
+            "max_position_embeddings": 512,
+            "type_vocab_size": 2,
+            "initializer_range": 0.02,
+            "pad_token_id": 0,
+        },
+        "bert-base-japanese-char ": {
+            "vocab_size": 4000,
+            "hidden_size": 768,
+            "num_hidden_layers": 12,
+            "num_attention_heads": 12,
+            "intermediate_size": 3072,
+            "hidden_act": "gelu",
+            "hidden_dropout_prob": 0.1,
+            "attention_probs_dropout_prob": 0.1,
+            "max_position_embeddings": 512,
+            "type_vocab_size": 2,
+            "initializer_range": 0.02,
+            "pad_token_id": 0,
+        },
+        "bert-base-japanese-char-whole-word-masking": {
+            "vocab_size": 4000,
+            "hidden_size": 768,
+            "num_hidden_layers": 12,
+            "num_attention_heads": 12,
+            "intermediate_size": 3072,
+            "hidden_act": "gelu",
+            "hidden_dropout_prob": 0.1,
+            "attention_probs_dropout_prob": 0.1,
+            "max_position_embeddings": 512,
+            "type_vocab_size": 2,
+            "initializer_range": 0.02,
+            "pad_token_id": 0,
+        }
     }
     resource_files_names = {"model_state": "model_state.pdparams"}
     pretrained_resource_files_map = {
@@ -298,6 +354,14 @@ class BertPretrainedModel(PretrainedModel):
             "https://paddlenlp.bj.bcebos.com/models/transformers/macbert/macbert-large-chinese.pdparams",
             "simbert-base-chinese":
             "https://paddlenlp.bj.bcebos.com/models/transformers/simbert/simbert-base-chinese-v1.pdparams",
+            "bert-base-japanese":
+            "https://paddlenlp.bj.bcebos.com/models/transformers/community/iverxin/bert-base-japanese/bert-base-japanese.pdparams",
+            "bert-base-japanese-whole-word-masking":
+            "https://paddlenlp.bj.bcebos.com/models/transformers/community/iverxin/bert-base-japanese-whole-word-masking/bert-base-japanese-whole-word-masking.pdparams",
+            "bert-base-japanese-char":
+            "https://paddlenlp.bj.bcebos.com/models/transformers/community/iverxin/bert-base-japanese-char/bert-base-japanese-char.pdparams",
+            "bert-base-japanese-char-whole-word-masking":
+            "https://paddlenlp.bj.bcebos.com/models/transformers/community/iverxin/bert-base-japanese-char-whole-word-masking/bert-base-japanese-char-whole-word-masking.pdparams",
         }
     }
     base_model_prefix = "bert"

paddlenlp/transformers/bert/tokenizer.py

@@ -14,16 +14,16 @@
 # limitations under the License.
 
 import copy
-import io
-import json
 import os
-import six
 import unicodedata
+import collections
 
 from .. import PretrainedTokenizer
 from ..tokenizer_utils import convert_to_unicode, whitespace_tokenize, _is_whitespace, _is_control, _is_punctuation
 
-__all__ = ['BasicTokenizer', 'BertTokenizer', 'WordpieceTokenizer']
+__all__ = [
+    'BasicTokenizer', 'BertTokenizer', 'WordpieceTokenizer',
+]
 
 
 class BasicTokenizer(object):
@@ -296,7 +296,7 @@ class BertTokenizer(PretrainedTokenizer):
             print(inputs)
 
             '''
-            {'input_ids': [101, 2002, 2001, 1037, 13997, 11510, 102], 'token_type_ids': [0, 0, 0, 0, 0, 0, 0]}
+            ['he', 'was', 'a', 'puppet', '##eer']
             '''
 
     """
@@ -327,6 +327,14 @@ class BertTokenizer(PretrainedTokenizer):
             "https://paddle-hapi.bj.bcebos.com/models/bert/bert-base-chinese-vocab.txt",
             "simbert-base-chinese":
             "https://paddlenlp.bj.bcebos.com/models/transformers/simbert/vocab.txt",
+            "bert-base-japanese":
+            "https://huggingface.co/cl-tohoku/bert-base-japanese/resolve/main/vocab.txt",
+            "bert-base-japanese-whole-word-masking":
+            "https://huggingface.co/cl-tohoku/bert-base-japanese-whole-word-masking/resolve/main/vocab.txt",
+            "bert-base-japanese-char":
+            "https://huggingface.co/cl-tohoku/bert-base-japanese-char/resolve/main/vocab.txt",
+            "bert-base-japanese-char-whole-word-masking":
+            "https://huggingface.co/cl-tohoku/bert-base-japanese-char-whole-word-masking/resolve/main/vocab.txt"
         }
     }
     pretrained_init_configuration = {
@@ -366,6 +374,26 @@ class BertTokenizer(PretrainedTokenizer):
         "simbert-base-chinese": {
             "do_lower_case": True
         },
+        "bert-base-japanese": {
+            "do_lower_case": False,
+            "word_tokenizer_type": "mecab",
+            "subword_tokenizer_type": "wordpiece",
+        },
+        "bert-base-japanese-whole-word-masking": {
+            "do_lower_case": False,
+            "word_tokenizer_type": "mecab",
+            "subword_tokenizer_type": "wordpiece",
+        },
+        "bert-base-japanese-char": {
+            "do_lower_case": False,
+            "word_tokenizer_type": "mecab",
+            "subword_tokenizer_type": "character",
+        },
+        "bert-base-japanese-char-whole-word-masking": {
+            "do_lower_case": False,
+            "word_tokenizer_type": "mecab",
+            "subword_tokenizer_type": "character",
+        },
     }
     padding_side = 'right'
 
@@ -554,7 +582,7 @@ def create_token_type_ids_from_sequences(self,
             0 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1
             | first sequence    | second sequence |
 
-        If :obj:`token_ids_1` is :obj:`None`, this method only returns the first portion of the mask (0s).
+        If `token_ids_1` is `None`, this method only returns the first portion of the mask (0s).
 
         Args:
             token_ids_0 (List[int]):
@@ -605,4 +633,4 @@ def get_special_tokens_mask(self,
         if token_ids_1 is not None:
             return [1] + ([0] * len(token_ids_0)) + [1] + (
                 [0] * len(token_ids_1)) + [1]
-        return [1] + ([0] * len(token_ids_0)) + [1]
+        return [1] + ([0] * len(token_ids_0)) + [1]

paddlenlp/transformers/bert_japanese/convert_bert_japanese_params.py

@@ -0,0 +1,69 @@
+import paddle
+import torch
+import numpy as np
+from paddle.utils.download import get_path_from_url
+
+model_names = [
+    "bert-base-japanese", "bert-base-japanese-whole-word-masking",
+    "bert-base-japanese-char", "bert-base-japanese-char-whole-word-masking"
+]
+
+for model_name in model_names:
+    torch_model_url = "https://huggingface.co/cl-tohoku/%s/resolve/main/pytorch_model.bin" % model_name
+    torch_model_path = get_path_from_url(torch_model_url, '../bert')
+    torch_state_dict = torch.load(torch_model_path)
+
+    paddle_model_path = "%s.pdparams" % model_name
+    paddle_state_dict = {}
+
+    # State_dict's keys mapping: from torch to paddle
+    keys_dict = {
+        # about embeddings
+        "embeddings.LayerNorm.gamma": "embeddings.layer_norm.weight",
+        "embeddings.LayerNorm.beta": "embeddings.layer_norm.bias",
+
+        # about encoder layer
+        'encoder.layer': 'encoder.layers',
+        'attention.self.query': 'self_attn.q_proj',
+        'attention.self.key': 'self_attn.k_proj',
+        'attention.self.value': 'self_attn.v_proj',
+        'attention.output.dense': 'self_attn.out_proj',
+        'attention.output.LayerNorm.gamma': 'norm1.weight',
+        'attention.output.LayerNorm.beta': 'norm1.bias',
+        'intermediate.dense': 'linear1',
+        'output.dense': 'linear2',
+        'output.LayerNorm.gamma': 'norm2.weight',
+        'output.LayerNorm.beta': 'norm2.bias',
+
+        # about cls predictions
+        'cls.predictions.transform.dense': 'cls.predictions.transform',
+        'cls.predictions.decoder.weight': 'cls.predictions.decoder_weight',
+        'cls.predictions.transform.LayerNorm.gamma':
+        'cls.predictions.layer_norm.weight',
+        'cls.predictions.transform.LayerNorm.beta':
+        'cls.predictions.layer_norm.bias',
+        'cls.predictions.bias': 'cls.predictions.decoder_bias'
+    }
+
+    for torch_key in torch_state_dict:
+        paddle_key = torch_key
+        for k in keys_dict:
+            if k in paddle_key:
+                paddle_key = paddle_key.replace(k, keys_dict[k])
+
+        if ('linear' in paddle_key) or ('proj' in paddle_key) or (
+                'vocab' in paddle_key and 'weight' in paddle_key) or (
+                    "dense.weight" in paddle_key) or (
+                        'transform.weight' in paddle_key) or (
+                            'seq_relationship.weight' in paddle_key):
+            paddle_state_dict[paddle_key] = paddle.to_tensor(torch_state_dict[
+                torch_key].cpu().numpy().transpose())
+        else:
+            paddle_state_dict[paddle_key] = paddle.to_tensor(torch_state_dict[
+                torch_key].cpu().numpy())
+
+        print("torch: ", torch_key, "\t", torch_state_dict[torch_key].shape)
+        print("paddle: ", paddle_key, "\t", paddle_state_dict[paddle_key].shape,
+              "\n")
+
+    paddle.save(paddle_state_dict, paddle_model_path)