From b30c4167057f148883c72dd280cc4401512736f1 Mon Sep 17 00:00:00 2001 From: Julien Plu Date: Tue, 10 Nov 2020 10:22:29 +0100 Subject: [PATCH 1/3] Add auto next sentence prediction --- src/transformers/modeling_tf_auto.py | 95 ++++++++++++++++++++++++++++ 1 file changed, 95 insertions(+) diff --git a/src/transformers/modeling_tf_auto.py b/src/transformers/modeling_tf_auto.py index 02bc7846245e..bbe266add010 100644 --- a/src/transformers/modeling_tf_auto.py +++ b/src/transformers/modeling_tf_auto.py @@ -61,6 +61,7 @@ from .modeling_tf_bert import ( TFBertForMaskedLM, TFBertForMultipleChoice, + TFBertForNextSentencePrediction, TFBertForPreTraining, TFBertForQuestionAnswering, TFBertForSequenceClassification, @@ -355,6 +356,12 @@ ] ) +TF_MODEL_FOR_NEXT_SENTENCE_PREDICTION_MAPPING = OrderedDict( + [ + (BertConfig, TFBertForNextSentencePrediction), + ] +) + TF_AUTO_MODEL_PRETRAINED_DOCSTRING = r""" @@ -1412,3 +1419,91 @@ def from_pretrained(cls, pretrained_model_name_or_path, *model_args, **kwargs): ", ".join(c.__name__ for c in TF_MODEL_FOR_MULTIPLE_CHOICE_MAPPING.keys()), ) ) + + +class TFAutoModelForNextSentencePrediction: + r""" + This is a generic model class that will be instantiated as one of the model classes of the library---with a next + sentence prediction head---when created with the + :meth:`~transformers.TFAutoModelForNextSentencePrediction.from_pretrained` class method or the + :meth:`~transformers.TFAutoModelForNextSentencePrediction.from_config` class method. This class cannot be + instantiated directly using ``__init__()`` (throws an error). + """ + + def __init__(self): + raise EnvironmentError( + "TFAutoModelForNextSentencePrediction is designed to be instantiated " + "using the `TFAutoModelForNextSentencePrediction.from_pretrained(pretrained_model_name_or_path)` or " + "`TFAutoModelForNextSentencePrediction.from_config(config)` methods." + ) + + @classmethod + @replace_list_option_in_docstrings(TF_MODEL_FOR_NEXT_SENTENCE_PREDICTION_MAPPING, use_model_types=False) + def from_config(cls, config): + r""" + Instantiates one of the model classes of the library---with a next sentence prediction head---from a + configuration. + Note: + Loading a model from its configuration file does **not** load the model weights. It only affects the + model's configuration. Use :meth:`~transformers.TFAutoModelForNextSentencePrediction.from_pretrained` to + load the model weights. + Args: + config (:class:`~transformers.PretrainedConfig`): + The model class to instantiate is selected based on the configuration class: + List options + Examples:: + >>> from transformers import AutoConfig, TFAutoModelForNextSentencePrediction + >>> # Download configuration from S3 and cache. + >>> config = AutoConfig.from_pretrained('bert-base-uncased') + >>> model = TFAutoModelForNextSentencePrediction.from_config(config) + """ + if type(config) in TF_MODEL_FOR_NEXT_SENTENCE_PREDICTION_MAPPING.keys(): + return TF_MODEL_FOR_NEXT_SENTENCE_PREDICTION_MAPPING[type(config)](config) + raise ValueError( + "Unrecognized configuration class {} for this kind of TFAutoModel: {}.\n" + "Model type should be one of {}.".format( + config.__class__, + cls.__name__, + ", ".join(c.__name__ for c in TF_MODEL_FOR_NEXT_SENTENCE_PREDICTION_MAPPING.keys()), + ) + ) + + @classmethod + @replace_list_option_in_docstrings(TF_MODEL_FOR_NEXT_SENTENCE_PREDICTION_MAPPING) + @add_start_docstrings( + "Instantiate one of the model classes of the library---with a next sentence prediction head---from a " + "pretrained model.", + TF_AUTO_MODEL_PRETRAINED_DOCSTRING, + ) + def from_pretrained(cls, pretrained_model_name_or_path, *model_args, **kwargs): + r""" + Examples:: + >>> from transformers import AutoConfig, TFAutoModelForNextSentencePrediction + >>> # Download model and configuration from S3 and cache. + >>> model = TFAutoModelForNextSentencePrediction.from_pretrained('bert-base-uncased') + >>> # Update configuration during loading + >>> model = TFAutoModelForNextSentencePrediction.from_pretrained('bert-base-uncased', output_attentions=True) + >>> model.config.output_attentions + True + >>> # Loading from a PyTorch checkpoint file instead of a TensorFlow model (slower) + >>> config = AutoConfig.from_json_file('./pt_model/bert_pt_model_config.json') + >>> model = TFAutoModelForNextSentencePrediction.from_pretrained('./pt_model/bert_pytorch_model.bin', from_pt=True, config=config) + """ + config = kwargs.pop("config", None) + if not isinstance(config, PretrainedConfig): + config, kwargs = AutoConfig.from_pretrained( + pretrained_model_name_or_path, return_unused_kwargs=True, **kwargs + ) + + if type(config) in TF_MODEL_FOR_NEXT_SENTENCE_PREDICTION_MAPPING.keys(): + return TF_MODEL_FOR_NEXT_SENTENCE_PREDICTION_MAPPING[type(config)].from_pretrained( + pretrained_model_name_or_path, *model_args, config=config, **kwargs + ) + raise ValueError( + "Unrecognized configuration class {} for this kind of TFAutoModel: {}.\n" + "Model type should be one of {}.".format( + config.__class__, + cls.__name__, + ", ".join(c.__name__ for c in TF_MODEL_FOR_NEXT_SENTENCE_PREDICTION_MAPPING.keys()), + ) + ) \ No newline at end of file From a30c3f88731341cfb0ee93409a3fb0648ce2d260 Mon Sep 17 00:00:00 2001 From: Julien Plu Date: Tue, 10 Nov 2020 10:39:16 +0100 Subject: [PATCH 2/3] Fix style --- src/transformers/modeling_tf_auto.py | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/src/transformers/modeling_tf_auto.py b/src/transformers/modeling_tf_auto.py index bbe266add010..71fadd0eb9f1 100644 --- a/src/transformers/modeling_tf_auto.py +++ b/src/transformers/modeling_tf_auto.py @@ -1423,11 +1423,12 @@ def from_pretrained(cls, pretrained_model_name_or_path, *model_args, **kwargs): class TFAutoModelForNextSentencePrediction: r""" - This is a generic model class that will be instantiated as one of the model classes of the library---with a next - sentence prediction head---when created with the + This is a generic model class that will be instantiated as one of the model classes of the library---with a + multiple choice classification head---when created with the when created with the :meth:`~transformers.TFAutoModelForNextSentencePrediction.from_pretrained` class method or the - :meth:`~transformers.TFAutoModelForNextSentencePrediction.from_config` class method. This class cannot be - instantiated directly using ``__init__()`` (throws an error). + :meth:`~transformers.TFAutoModelForNextSentencePrediction.from_config` class method. + + This class cannot be instantiated directly using ``__init__()`` (throws an error). """ def __init__(self): @@ -1443,15 +1444,20 @@ def from_config(cls, config): r""" Instantiates one of the model classes of the library---with a next sentence prediction head---from a configuration. + Note: Loading a model from its configuration file does **not** load the model weights. It only affects the model's configuration. Use :meth:`~transformers.TFAutoModelForNextSentencePrediction.from_pretrained` to load the model weights. + Args: config (:class:`~transformers.PretrainedConfig`): The model class to instantiate is selected based on the configuration class: + List options + Examples:: + >>> from transformers import AutoConfig, TFAutoModelForNextSentencePrediction >>> # Download configuration from S3 and cache. >>> config = AutoConfig.from_pretrained('bert-base-uncased') @@ -1478,13 +1484,17 @@ def from_config(cls, config): def from_pretrained(cls, pretrained_model_name_or_path, *model_args, **kwargs): r""" Examples:: + >>> from transformers import AutoConfig, TFAutoModelForNextSentencePrediction + >>> # Download model and configuration from S3 and cache. >>> model = TFAutoModelForNextSentencePrediction.from_pretrained('bert-base-uncased') + >>> # Update configuration during loading >>> model = TFAutoModelForNextSentencePrediction.from_pretrained('bert-base-uncased', output_attentions=True) >>> model.config.output_attentions True + >>> # Loading from a PyTorch checkpoint file instead of a TensorFlow model (slower) >>> config = AutoConfig.from_json_file('./pt_model/bert_pt_model_config.json') >>> model = TFAutoModelForNextSentencePrediction.from_pretrained('./pt_model/bert_pytorch_model.bin', from_pt=True, config=config) @@ -1506,4 +1516,4 @@ def from_pretrained(cls, pretrained_model_name_or_path, *model_args, **kwargs): cls.__name__, ", ".join(c.__name__ for c in TF_MODEL_FOR_NEXT_SENTENCE_PREDICTION_MAPPING.keys()), ) - ) \ No newline at end of file + ) From 3fd78dd3330fdff1faf7e8490cc01526fc4f8700 Mon Sep 17 00:00:00 2001 From: Julien Plu Date: Tue, 10 Nov 2020 13:55:20 +0100 Subject: [PATCH 3/3] Add mobilebert next sentence prediction --- src/transformers/modeling_tf_auto.py | 2 ++ utils/check_repo.py | 2 -- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/transformers/modeling_tf_auto.py b/src/transformers/modeling_tf_auto.py index 71fadd0eb9f1..dcf694e33952 100644 --- a/src/transformers/modeling_tf_auto.py +++ b/src/transformers/modeling_tf_auto.py @@ -121,6 +121,7 @@ from .modeling_tf_mobilebert import ( TFMobileBertForMaskedLM, TFMobileBertForMultipleChoice, + TFMobileBertForNextSentencePrediction, TFMobileBertForPreTraining, TFMobileBertForQuestionAnswering, TFMobileBertForSequenceClassification, @@ -359,6 +360,7 @@ TF_MODEL_FOR_NEXT_SENTENCE_PREDICTION_MAPPING = OrderedDict( [ (BertConfig, TFBertForNextSentencePrediction), + (MobileBertConfig, TFMobileBertForNextSentencePrediction), ] ) diff --git a/utils/check_repo.py b/utils/check_repo.py index f81b4c9fef39..a563ff9471a2 100644 --- a/utils/check_repo.py +++ b/utils/check_repo.py @@ -87,10 +87,8 @@ "RagSequenceForGeneration", "RagTokenForGeneration", "T5Stack", - "TFBertForNextSentencePrediction", "TFFunnelBaseModel", "TFGPT2DoubleHeadsModel", - "TFMobileBertForNextSentencePrediction", "TFOpenAIGPTDoubleHeadsModel", "XLMForQuestionAnswering", "XLMProphetNetDecoder",