From 531336bbfd2a97cf800f610d971d6ec0a1578752 Mon Sep 17 00:00:00 2001 From: Sylvain Gugger <35901082+sgugger@users.noreply.github.com> Date: Tue, 18 Jan 2022 07:28:53 -0500 Subject: [PATCH] Fix deprecation warnings for int div (#15180) * Fix deprecation warnings for int div Co-authored-by: mgoldey * Fix import * ensure that tensor output is python scalar * make backward compatible * make code more readable * adapt test functions Co-authored-by: mgoldey Co-authored-by: Patrick von Platen --- .../run_wav2vec2_pretraining_no_trainer.py | 2 ++ src/transformers/modeling_utils.py | 11 +++++++ .../models/hubert/modeling_hubert.py | 4 +-- src/transformers/models/sew/modeling_sew.py | 4 +-- .../models/sew_d/modeling_sew_d.py | 4 +-- .../models/unispeech/modeling_unispeech.py | 4 +-- .../unispeech_sat/modeling_unispeech_sat.py | 4 +-- .../models/wav2vec2/modeling_wav2vec2.py | 4 +-- .../models/wavlm/modeling_wavlm.py | 4 +-- tests/test_modeling_wav2vec2.py | 32 +++++++++---------- 10 files changed, 43 insertions(+), 30 deletions(-) diff --git a/examples/pytorch/speech-pretraining/run_wav2vec2_pretraining_no_trainer.py b/examples/pytorch/speech-pretraining/run_wav2vec2_pretraining_no_trainer.py index 2a3883a2d3b6..c7134cb9ea1a 100755 --- a/examples/pytorch/speech-pretraining/run_wav2vec2_pretraining_no_trainer.py +++ b/examples/pytorch/speech-pretraining/run_wav2vec2_pretraining_no_trainer.py @@ -302,6 +302,8 @@ def __call__(self, features: List[Dict[str, Union[List[int], torch.Tensor]]]) -> batch_size = batch["input_values"].shape[0] mask_indices_seq_length = self.model._get_feat_extract_output_lengths(batch["input_values"].shape[-1]) + # make sure masked sequence length is a Python scalar + mask_indices_seq_length = int(mask_indices_seq_length) # make sure that no loss is computed on padded inputs if batch.get("attention_mask") is not None: diff --git a/src/transformers/modeling_utils.py b/src/transformers/modeling_utils.py index 324046cc6a53..dfc9855a908e 100644 --- a/src/transformers/modeling_utils.py +++ b/src/transformers/modeling_utils.py @@ -23,6 +23,7 @@ from typing import Any, Callable, Dict, List, Optional, Set, Tuple, Union import torch +from packaging import version from torch import Tensor, device, nn from torch.nn import CrossEntropyLoss @@ -2362,3 +2363,13 @@ def forward(self, hidden_states): return torch.cat(output_chunks, dim=chunk_dim) return forward_fn(*input_tensors) + + +def torch_int_div(tensor1, tensor2): + """ + A function that performs integer division across different versions of PyTorch. + """ + if version.parse(torch.__version__) < version.parse("1.8.0"): + return tensor1 // tensor2 + else: + return torch.div(tensor1, tensor2, rounding_mode="floor") diff --git a/src/transformers/models/hubert/modeling_hubert.py b/src/transformers/models/hubert/modeling_hubert.py index c317bf324770..02ef5dfb53ec 100755 --- a/src/transformers/models/hubert/modeling_hubert.py +++ b/src/transformers/models/hubert/modeling_hubert.py @@ -33,7 +33,7 @@ replace_return_docstrings, ) from ...modeling_outputs import BaseModelOutput, CausalLMOutput, SequenceClassifierOutput -from ...modeling_utils import PreTrainedModel +from ...modeling_utils import PreTrainedModel, torch_int_div from ...utils import logging from .configuration_hubert import HubertConfig @@ -829,7 +829,7 @@ def _get_feat_extract_output_lengths(self, input_lengths: Union[torch.LongTensor def _conv_out_length(input_length, kernel_size, stride): # 1D convolutional layer output length formula taken # from https://pytorch.org/docs/stable/generated/torch.nn.Conv1d.html - return (input_length - kernel_size) // stride + 1 + return torch_int_div(input_length - kernel_size, stride) + 1 for kernel_size, stride in zip(self.config.conv_kernel, self.config.conv_stride): input_lengths = _conv_out_length(input_lengths, kernel_size, stride) diff --git a/src/transformers/models/sew/modeling_sew.py b/src/transformers/models/sew/modeling_sew.py index ae283baf39ec..7d60b777f3e7 100644 --- a/src/transformers/models/sew/modeling_sew.py +++ b/src/transformers/models/sew/modeling_sew.py @@ -29,7 +29,7 @@ from ...activations import ACT2FN from ...file_utils import add_code_sample_docstrings, add_start_docstrings, add_start_docstrings_to_model_forward from ...modeling_outputs import BaseModelOutput, CausalLMOutput, SequenceClassifierOutput -from ...modeling_utils import PreTrainedModel +from ...modeling_utils import PreTrainedModel, torch_int_div from ...utils import logging from .configuration_sew import SEWConfig @@ -735,7 +735,7 @@ def _get_feat_extract_output_lengths(self, input_lengths: Union[torch.LongTensor def _conv_out_length(input_length, kernel_size, stride): # 1D convolutional layer output length formula taken # from https://pytorch.org/docs/stable/generated/torch.nn.Conv1d.html - return (input_length - kernel_size) // stride + 1 + return torch_int_div(input_length - kernel_size, stride) + 1 for kernel_size, stride in zip(self.config.conv_kernel, self.config.conv_stride): input_lengths = _conv_out_length(input_lengths, kernel_size, stride) diff --git a/src/transformers/models/sew_d/modeling_sew_d.py b/src/transformers/models/sew_d/modeling_sew_d.py index be02a122441b..8a2571fa144d 100644 --- a/src/transformers/models/sew_d/modeling_sew_d.py +++ b/src/transformers/models/sew_d/modeling_sew_d.py @@ -30,7 +30,7 @@ from ...activations import ACT2FN from ...file_utils import add_code_sample_docstrings, add_start_docstrings, add_start_docstrings_to_model_forward from ...modeling_outputs import BaseModelOutput, CausalLMOutput, SequenceClassifierOutput -from ...modeling_utils import PreTrainedModel +from ...modeling_utils import PreTrainedModel, torch_int_div from ...utils import logging from .configuration_sew_d import SEWDConfig @@ -1266,7 +1266,7 @@ def _get_feat_extract_output_lengths(self, input_lengths: Union[torch.LongTensor def _conv_out_length(input_length, kernel_size, stride): # 1D convolutional layer output length formula taken # from https://pytorch.org/docs/stable/generated/torch.nn.Conv1d.html - return (input_length - kernel_size) // stride + 1 + return torch_int_div(input_length - kernel_size, stride) + 1 for kernel_size, stride in zip(self.config.conv_kernel, self.config.conv_stride): input_lengths = _conv_out_length(input_lengths, kernel_size, stride) diff --git a/src/transformers/models/unispeech/modeling_unispeech.py b/src/transformers/models/unispeech/modeling_unispeech.py index 81ef40653a86..1d27debadaa5 100755 --- a/src/transformers/models/unispeech/modeling_unispeech.py +++ b/src/transformers/models/unispeech/modeling_unispeech.py @@ -35,7 +35,7 @@ replace_return_docstrings, ) from ...modeling_outputs import BaseModelOutput, CausalLMOutput, SequenceClassifierOutput -from ...modeling_utils import PreTrainedModel +from ...modeling_utils import PreTrainedModel, torch_int_div from ...utils import logging from .configuration_unispeech import UniSpeechConfig @@ -969,7 +969,7 @@ def _get_feat_extract_output_lengths(self, input_lengths: Union[torch.LongTensor def _conv_out_length(input_length, kernel_size, stride): # 1D convolutional layer output length formula taken # from https://pytorch.org/docs/stable/generated/torch.nn.Conv1d.html - return (input_length - kernel_size) // stride + 1 + return torch_int_div(input_length - kernel_size, stride) + 1 for kernel_size, stride in zip(self.config.conv_kernel, self.config.conv_stride): input_lengths = _conv_out_length(input_lengths, kernel_size, stride) diff --git a/src/transformers/models/unispeech_sat/modeling_unispeech_sat.py b/src/transformers/models/unispeech_sat/modeling_unispeech_sat.py index b8fcbabe9753..e29d58377053 100755 --- a/src/transformers/models/unispeech_sat/modeling_unispeech_sat.py +++ b/src/transformers/models/unispeech_sat/modeling_unispeech_sat.py @@ -35,7 +35,7 @@ replace_return_docstrings, ) from ...modeling_outputs import BaseModelOutput, CausalLMOutput, SequenceClassifierOutput, TokenClassifierOutput -from ...modeling_utils import PreTrainedModel +from ...modeling_utils import PreTrainedModel, torch_int_div from ...utils import logging from .configuration_unispeech_sat import UniSpeechSatConfig @@ -1003,7 +1003,7 @@ def _get_feat_extract_output_lengths(self, input_lengths: Union[torch.LongTensor def _conv_out_length(input_length, kernel_size, stride): # 1D convolutional layer output length formula taken # from https://pytorch.org/docs/stable/generated/torch.nn.Conv1d.html - return (input_length - kernel_size) // stride + 1 + return torch_int_div(input_length - kernel_size, stride) + 1 for kernel_size, stride in zip(self.config.conv_kernel, self.config.conv_stride): input_lengths = _conv_out_length(input_lengths, kernel_size, stride) diff --git a/src/transformers/models/wav2vec2/modeling_wav2vec2.py b/src/transformers/models/wav2vec2/modeling_wav2vec2.py index b82df05b8f7f..15fb37943e96 100755 --- a/src/transformers/models/wav2vec2/modeling_wav2vec2.py +++ b/src/transformers/models/wav2vec2/modeling_wav2vec2.py @@ -41,7 +41,7 @@ SequenceClassifierOutput, TokenClassifierOutput, ) -from ...modeling_utils import PreTrainedModel +from ...modeling_utils import PreTrainedModel, torch_int_div from ...utils import logging from .configuration_wav2vec2 import Wav2Vec2Config @@ -1104,7 +1104,7 @@ def _get_feat_extract_output_lengths( def _conv_out_length(input_length, kernel_size, stride): # 1D convolutional layer output length formula taken # from https://pytorch.org/docs/stable/generated/torch.nn.Conv1d.html - return (input_length - kernel_size) // stride + 1 + return torch_int_div(input_length - kernel_size, stride) + 1 for kernel_size, stride in zip(self.config.conv_kernel, self.config.conv_stride): input_lengths = _conv_out_length(input_lengths, kernel_size, stride) diff --git a/src/transformers/models/wavlm/modeling_wavlm.py b/src/transformers/models/wavlm/modeling_wavlm.py index 344d196167f5..8b9d02df73dc 100755 --- a/src/transformers/models/wavlm/modeling_wavlm.py +++ b/src/transformers/models/wavlm/modeling_wavlm.py @@ -35,7 +35,7 @@ add_start_docstrings_to_model_forward, ) from ...modeling_outputs import BaseModelOutput, CausalLMOutput, SequenceClassifierOutput, TokenClassifierOutput -from ...modeling_utils import PreTrainedModel +from ...modeling_utils import PreTrainedModel, torch_int_div from ...utils import logging from .configuration_wavlm import WavLMConfig @@ -1057,7 +1057,7 @@ def _get_feat_extract_output_lengths( def _conv_out_length(input_length, kernel_size, stride): # 1D convolutional layer output length formula taken # from https://pytorch.org/docs/stable/generated/torch.nn.Conv1d.html - return (input_length - kernel_size) // stride + 1 + return torch_int_div(input_length - kernel_size, stride) + 1 for kernel_size, stride in zip(self.config.conv_kernel, self.config.conv_stride): input_lengths = _conv_out_length(input_lengths, kernel_size, stride) diff --git a/tests/test_modeling_wav2vec2.py b/tests/test_modeling_wav2vec2.py index 7b37a5d82cf1..51a371b225ee 100644 --- a/tests/test_modeling_wav2vec2.py +++ b/tests/test_modeling_wav2vec2.py @@ -794,10 +794,10 @@ def test_model_for_pretraining(self): config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common() model = Wav2Vec2ForPreTraining(config).to(torch_device) - features_shape = ( - inputs_dict["input_values"].shape[0], - model._get_feat_extract_output_lengths(inputs_dict["input_values"].shape[1]), - ) + batch_size = inputs_dict["input_values"].shape[0] + feature_seq_length = int(model._get_feat_extract_output_lengths(inputs_dict["input_values"].shape[1])) + + features_shape = (batch_size, feature_seq_length) mask_time_indices = _compute_mask_indices( features_shape, @@ -1158,10 +1158,10 @@ def test_inference_integration(self): inputs_dict = feature_extractor(input_speech, return_tensors="pt", padding=True) - features_shape = ( - inputs_dict["input_values"].shape[0], - model._get_feat_extract_output_lengths(torch.tensor(inputs_dict["input_values"].shape[1])), - ) + batch_size = inputs_dict["input_values"].shape[0] + feature_seq_length = int(model._get_feat_extract_output_lengths(inputs_dict["input_values"].shape[1])) + + features_shape = (batch_size, feature_seq_length) np.random.seed(4) mask_time_indices = _compute_mask_indices( @@ -1208,10 +1208,10 @@ def test_inference_pretrained(self): inputs_dict = feature_extractor(input_speech, return_tensors="pt", padding=True) - features_shape = ( - inputs_dict["input_values"].shape[0], - model._get_feat_extract_output_lengths(torch.tensor(inputs_dict["input_values"].shape[1])), - ) + batch_size = inputs_dict["input_values"].shape[0] + feature_seq_length = int(model._get_feat_extract_output_lengths(inputs_dict["input_values"].shape[1])) + + features_shape = (batch_size, feature_seq_length) torch.manual_seed(0) mask_time_indices = _compute_mask_indices( @@ -1279,10 +1279,10 @@ def test_loss_pretraining(self): inputs_dict = feature_extractor(input_speech, return_tensors="pt", padding=True) - features_shape = ( - inputs_dict["input_values"].shape[0], - model._get_feat_extract_output_lengths(inputs_dict["input_values"].shape[1]), - ) + batch_size = inputs_dict["input_values"].shape[0] + feature_seq_length = int(model._get_feat_extract_output_lengths(inputs_dict["input_values"].shape[1])) + + features_shape = (batch_size, feature_seq_length) torch.manual_seed(0) np.random.seed(0)