Fix multi process training (PaddlePaddle#92)

* fix embedding examples * fix multi process training error * add sentencepiece import * remove wmt14ende dataset
wangxicoding · Mar 9, 2021 · 6fe6476 · 6fe6476
1 parent 11619cb
commit 6fe6476
Show file tree

Hide file tree

Showing 6 changed files with 12 additions and 7 deletions.
diff --git a/examples/language_model/bigbird/README.md b/examples/language_model/bigbird/README.md
@@ -21,6 +21,12 @@
 
     本项目依赖于 PaddlePaddle 2.0.1 及以上版本或适当的develop版本，请参考 [安装指南](https://www.paddlepaddle.org.cn/install/quick) 进行安装。
 
+* Sentencepiece 安装
+
+   ```shell
+   pip install sentencepiece
+   ```
+
 * PaddleNLP 安装
 
    ```shell

diff --git a/examples/language_model/bigbird/run_classifier.py b/examples/language_model/bigbird/run_classifier.py
@@ -126,7 +126,7 @@ def main():
     optimizer = paddle.optimizer.Adam(
         parameters=model.parameters(),
         learning_rate=args.learning_rate,
-        epsilon=1e-6)
+        epsilon=1e-7)
 
     # Finetune the classification model
     do_train(model, criterion, metric, optimizer, train_data_loader,

diff --git a/examples/word_embedding/data.py b/examples/word_embedding/data.py
@@ -70,7 +70,7 @@ def convert_example(example, vocab, unk_token_id=1, is_test=False):
     valid_length = len(input_ids)
 
     if not is_test:
-        label = np.array(example["labels"], dtype="int64")
+        label = np.array(example["label"], dtype="int64")
         return input_ids, valid_length, label
     else:
         return input_ids, valid_length

diff --git a/examples/word_embedding/train.py b/examples/word_embedding/train.py
@@ -145,7 +145,7 @@ def forward(self, text, seq_len=None):
         vocab['[PAD]'] = len(vocab)
     # Loads dataset.
     train_ds, dev_ds, test_ds = load_dataset(
-        "chnsenticorp", splits=["train", "dev", "test"], lazy=False)
+        "chnsenticorp", splits=["train", "dev", "test"])
 
     # Constructs the newtork.
     model = BoWModel(

diff --git a/paddlenlp/datasets/experimental/__init__.py b/paddlenlp/datasets/experimental/__init__.py
@@ -28,6 +28,4 @@
 from .drcd import *
 from .dureader_robust import *
 from .glue import *
-from .wmt14ende import *
 from .cnndm import *
-
diff --git a/paddlenlp/transformers/bigbird/modeling.py b/paddlenlp/transformers/bigbird/modeling.py
@@ -474,8 +474,9 @@ def forward(self, prediction_scores, seq_relationship_score,
         masked_lm_loss = paddle.transpose(masked_lm_loss, [1, 0])
         masked_lm_loss = paddle.sum(masked_lm_loss * masked_lm_weights) / (
             paddle.sum(masked_lm_weights) + 1e-5)
+        scale = 1.0
         if not self.use_nsp:
-            return masked_lm_loss
+            scale = 0.0
         next_sentence_loss = paddle.nn.functional.softmax_with_cross_entropy(
             seq_relationship_score, next_sentence_labels)
-        return masked_lm_loss + paddle.mean(next_sentence_loss)
+        return masked_lm_loss + paddle.mean(next_sentence_loss) * scale