From 87177b7e6a77ef151a627ec2d9a37d8935f09987 Mon Sep 17 00:00:00 2001 From: Lysandre Date: Tue, 6 Oct 2020 23:08:13 +0200 Subject: [PATCH 1/4] stash --- tests/test_modeling_tf_bert.py | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/tests/test_modeling_tf_bert.py b/tests/test_modeling_tf_bert.py index 7fbdb08c87c703..c6c87f4600a892 100644 --- a/tests/test_modeling_tf_bert.py +++ b/tests/test_modeling_tf_bert.py @@ -17,7 +17,7 @@ import unittest from transformers import BertConfig, is_tf_available -from transformers.testing_utils import require_tf +from transformers.testing_utils import require_tf, slow from .test_configuration_common import ConfigTester from .test_modeling_tf_common import TFModelTesterMixin, ids_tensor @@ -328,3 +328,21 @@ def test_custom_load_tf_weights(self): self.assertEqual(sorted(output_loading_info["unexpected_keys"]), ["mlm___cls", "nsp___cls"]) for layer in output_loading_info["missing_keys"]: self.assertTrue(layer.split("_")[0] in ["dropout", "classifier"]) + + +class TFBertModelIntegrationTest(unittest.TestCase): + @slow + def test_inference_masked_lm(self): + model = TFBertForPreTraining.from_pretrained("lysandre/tiny-bert-random") + input_ids = tf.constant([[0, 1, 2, 3, 4, 5]]) + output = model(input_ids)[0] + + expected_shape = [1, 6, 10] + self.assertEqual(output.shape, expected_shape) + + expected_slice = tf.constant( + [[[-0.10651821, - 0.03419777, 0.03701617], + [0.00911148 , - 0.04026008 , - 0.05476241], + [0.11873573 , 0.05936486 , - 0.00904015]]] + ) + tf.debugging.assert_near(output[:, :3, :3], expected_slice, atol=1e-4) From 0ee8c2f8936208d0cde5337d6bd54050fec95d75 Mon Sep 17 00:00:00 2001 From: Lysandre Date: Tue, 6 Oct 2020 23:21:06 +0200 Subject: [PATCH 2/4] TF Integration testing for ELECTRA, BERT, Longformer --- src/transformers/modeling_electra.py | 2 +- src/transformers/modeling_tf_electra.py | 2 +- tests/test_modeling_tf_bert.py | 14 ++++++++++---- tests/test_modeling_tf_electra.py | 16 ++++++++++++++++ tests/test_modeling_tf_longformer.py | 22 ++++++++++++++++++++++ 5 files changed, 50 insertions(+), 6 deletions(-) diff --git a/src/transformers/modeling_electra.py b/src/transformers/modeling_electra.py index 31a7fd9ecd1639..c423b6bb38039f 100644 --- a/src/transformers/modeling_electra.py +++ b/src/transformers/modeling_electra.py @@ -504,7 +504,7 @@ def __init__(self, config): def forward(self, discriminator_hidden_states): hidden_states = self.dense(discriminator_hidden_states) hidden_states = get_activation(self.config.hidden_act)(hidden_states) - logits = self.dense_prediction(hidden_states).squeeze() + logits = self.dense_prediction(hidden_states).squeeze(-1) return logits diff --git a/src/transformers/modeling_tf_electra.py b/src/transformers/modeling_tf_electra.py index c680042a8aec62..00a725081a1169 100644 --- a/src/transformers/modeling_tf_electra.py +++ b/src/transformers/modeling_tf_electra.py @@ -418,7 +418,7 @@ def __init__(self, config, **kwargs): def call(self, discriminator_hidden_states, training=False): hidden_states = self.dense(discriminator_hidden_states) hidden_states = get_tf_activation(self.config.hidden_act)(hidden_states) - logits = tf.squeeze(self.dense_prediction(hidden_states)) + logits = tf.squeeze(self.dense_prediction(hidden_states), -1) return logits diff --git a/tests/test_modeling_tf_bert.py b/tests/test_modeling_tf_bert.py index c6c87f4600a892..d63d3520d8c7c3 100644 --- a/tests/test_modeling_tf_bert.py +++ b/tests/test_modeling_tf_bert.py @@ -333,16 +333,22 @@ def test_custom_load_tf_weights(self): class TFBertModelIntegrationTest(unittest.TestCase): @slow def test_inference_masked_lm(self): - model = TFBertForPreTraining.from_pretrained("lysandre/tiny-bert-random") + model = TFBertForPreTraining.from_pretrained("lysandre/tiny-bert-random", use_cdn=False) input_ids = tf.constant([[0, 1, 2, 3, 4, 5]]) output = model(input_ids)[0] expected_shape = [1, 6, 10] self.assertEqual(output.shape, expected_shape) + print(output[:, :3, :3]) + expected_slice = tf.constant( - [[[-0.10651821, - 0.03419777, 0.03701617], - [0.00911148 , - 0.04026008 , - 0.05476241], - [0.11873573 , 0.05936486 , - 0.00904015]]] + [ + [ + [0.03706957, 0.10124919, 0.03616843], + [-0.06099961, 0.02266058, 0.00601412], + [-0.06066202, 0.05684517, 0.02038802], + ] + ] ) tf.debugging.assert_near(output[:, :3, :3], expected_slice, atol=1e-4) diff --git a/tests/test_modeling_tf_electra.py b/tests/test_modeling_tf_electra.py index 2c1daf4557e288..fcf07ef7ef8cc5 100644 --- a/tests/test_modeling_tf_electra.py +++ b/tests/test_modeling_tf_electra.py @@ -248,3 +248,19 @@ def test_model_from_pretrained(self): for model_name in ["google/electra-small-discriminator"]: model = TFElectraModel.from_pretrained(model_name) self.assertIsNotNone(model) + + +class TFElectraModelIntegrationTest(unittest.TestCase): + @slow + def test_inference_masked_lm(self): + model = TFElectraForPreTraining.from_pretrained("lysandre/tiny-electra-random", use_cdn=False) + input_ids = tf.constant([[0, 1, 2, 3, 4, 5]]) + output = model(input_ids)[0] + + expected_shape = [1, 6] + self.assertEqual(output.shape, expected_shape) + + print(output[:, :3]) + + expected_slice = tf.constant([[-0.24651965, 0.8835437, 1.823782]]) + tf.debugging.assert_near(output[:, :3], expected_slice, atol=1e-4) diff --git a/tests/test_modeling_tf_longformer.py b/tests/test_modeling_tf_longformer.py index b0bd9bb260cc5f..317e5442a4b51a 100644 --- a/tests/test_modeling_tf_longformer.py +++ b/tests/test_modeling_tf_longformer.py @@ -548,3 +548,25 @@ def test_inference_masked_lm_long(self): tf.debugging.assert_near(tf.reduce_mean(loss), expected_loss, rtol=1e-4) tf.debugging.assert_near(tf.reduce_sum(prediction_scores), expected_prediction_scores_sum, rtol=1e-4) tf.debugging.assert_near(tf.reduce_mean(prediction_scores), expected_prediction_scores_mean, rtol=1e-4) + + @slow + def test_inference_masked_lm(self): + model = TFLongformerForMaskedLM.from_pretrained("lysandre/tiny-longformer-random", use_cdn=False) + input_ids = tf.constant([[0, 1, 2, 3, 4, 5]]) + output = model(input_ids)[0] + + expected_shape = [1, 6, 10] + self.assertEqual(output.shape, expected_shape) + + print(output[:, :3, :3]) + + expected_slice = tf.constant( + [ + [ + [-0.04926379, 0.0367098, 0.02099686], + [0.03940692, 0.01547744, -0.01448723], + [0.03495252, -0.05900355, -0.01675752], + ] + ] + ) + tf.debugging.assert_near(output[:, :3, :3], expected_slice, atol=1e-4) From cf3557d8da0320c23ee073eec8e4a6a28cf58e83 Mon Sep 17 00:00:00 2001 From: Lysandre Date: Tue, 6 Oct 2020 23:30:56 +0200 Subject: [PATCH 3/4] Trigger slow tests --- .github/workflows/self-push.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.github/workflows/self-push.yml b/.github/workflows/self-push.yml index eabd6b9e1c753e..5c84b3b87dd605 100644 --- a/.github/workflows/self-push.yml +++ b/.github/workflows/self-push.yml @@ -2,9 +2,7 @@ name: Self-hosted runner (push) on: push: - branches: - - master - paths: + paths: - "src/**" - "tests/**" - ".github/**" From 4724258d8b0e8951153485397f94bed0f491072f Mon Sep 17 00:00:00 2001 From: Lysandre Debut Date: Wed, 7 Oct 2020 11:25:33 +0200 Subject: [PATCH 4/4] Apply suggestions from code review --- tests/test_modeling_tf_bert.py | 2 +- tests/test_modeling_tf_electra.py | 2 +- tests/test_modeling_tf_longformer.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/test_modeling_tf_bert.py b/tests/test_modeling_tf_bert.py index d63d3520d8c7c3..e94b756b757dfc 100644 --- a/tests/test_modeling_tf_bert.py +++ b/tests/test_modeling_tf_bert.py @@ -333,7 +333,7 @@ def test_custom_load_tf_weights(self): class TFBertModelIntegrationTest(unittest.TestCase): @slow def test_inference_masked_lm(self): - model = TFBertForPreTraining.from_pretrained("lysandre/tiny-bert-random", use_cdn=False) + model = TFBertForPreTraining.from_pretrained("lysandre/tiny-bert-random") input_ids = tf.constant([[0, 1, 2, 3, 4, 5]]) output = model(input_ids)[0] diff --git a/tests/test_modeling_tf_electra.py b/tests/test_modeling_tf_electra.py index fcf07ef7ef8cc5..95a570a6a52634 100644 --- a/tests/test_modeling_tf_electra.py +++ b/tests/test_modeling_tf_electra.py @@ -253,7 +253,7 @@ def test_model_from_pretrained(self): class TFElectraModelIntegrationTest(unittest.TestCase): @slow def test_inference_masked_lm(self): - model = TFElectraForPreTraining.from_pretrained("lysandre/tiny-electra-random", use_cdn=False) + model = TFElectraForPreTraining.from_pretrained("lysandre/tiny-electra-random") input_ids = tf.constant([[0, 1, 2, 3, 4, 5]]) output = model(input_ids)[0] diff --git a/tests/test_modeling_tf_longformer.py b/tests/test_modeling_tf_longformer.py index 317e5442a4b51a..6e2aa26754da65 100644 --- a/tests/test_modeling_tf_longformer.py +++ b/tests/test_modeling_tf_longformer.py @@ -551,7 +551,7 @@ def test_inference_masked_lm_long(self): @slow def test_inference_masked_lm(self): - model = TFLongformerForMaskedLM.from_pretrained("lysandre/tiny-longformer-random", use_cdn=False) + model = TFLongformerForMaskedLM.from_pretrained("lysandre/tiny-longformer-random") input_ids = tf.constant([[0, 1, 2, 3, 4, 5]]) output = model(input_ids)[0]