Merge branch 'bugfix/calibration_negative_input' of https://github.co…

…m/PyTorchLightning/metrics into bugfix/calibration_negative_input

.github/assistant.py

@@ -59,7 +59,7 @@ def prune_packages(req_file: str, *pkgs: str) -> None:
  lines = [ln for ln in lines if not ln.startswith(pkg)]
  logging.info(lines)
 
- with open(req_file, "w") as fp:
+ with open(req_file, "w", encoding="utf-8") as fp:
  fp.writelines(lines)
 
  @staticmethod
@@ -71,17 +71,17 @@ def set_min_torch_by_python(fpath: str = "requirements.txt") -> None:
  with open(fpath) as fp:
  req = fp.read()
  req = re.sub(r"torch>=[\d\.]+", f"torch>={LUT_PYTHON_TORCH[py_ver]}", req)
- with open(fpath, "w") as fp:
+ with open(fpath, "w", encoding="utf-8") as fp:
  fp.write(req)
 
  @staticmethod
  def replace_min_requirements(fpath: str) -> None:
  """Replace all `>=` by `==` in given file."""
  logging.info(f"processing: {fpath}")
- with open(fpath) as fp:
+ with open(fpath, encoding="utf-8") as fp:
  req = fp.read()
  req = req.replace(">=", "==")
- with open(fpath, "w") as fp:
+ with open(fpath, "w", encoding="utf-8") as fp:
  fp.write(req)
 
  @staticmethod

CHANGELOG.md

@@ -41,6 +41,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - Removed deprecated `compute_on_step` argument in Regression ([#967](https://github.com/PyTorchLightning/metrics/pull/967))
 
 
+- Removed deprecated `compute_on_step` argument in Image ([#979](https://github.com/PyTorchLightning/metrics/pull/979))
+
+
 ### Fixed
 
 - Fixed "Sort currently does not support bool dtype on CUDA" error in MAP for empty preds ([#983](https://github.com/PyTorchLightning/metrics/pull/983))
@@ -49,7 +52,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - Fixed `BinnedPrecisionRecallCurve` when `thresholds` argument is not provided ([#968](https://github.com/PyTorchLightning/metrics/pull/968))
 
 
-- Fixed `CalibrationError` to work on logit input ([]())
+- Fixed `CalibrationError` to work on logit input ([#985](https://github.com/PyTorchLightning/metrics/pull/985))
 
 
 ## [0.8.0] - 2022-04-14

docs/source/conf.py

@@ -65,7 +65,7 @@ def _transform_changelog(path_in: str, path_out: str) -> None:
  elif ln.startswith("### "):
  ln = ln.replace("###", f"### {chlog_ver} -")
  chlog_lines[i] = ln
- with open(path_out, "w") as fp:
+ with open(path_out, "w", encoding="utf-8") as fp:
  fp.writelines(chlog_lines)
 
 

tests/audio/test_sdr.py

@@ -87,7 +87,7 @@ def test_sdr(self, preds, target, sk_metric, ddp, dist_sync_on_step):
  SignalDistortionRatio,
  sk_metric=partial(average_metric, metric_func=sk_metric),
  dist_sync_on_step=dist_sync_on_step,
- metric_args=dict(),
+ metric_args={},
  )
 
  def test_sdr_functional(self, preds, target, sk_metric):
@@ -96,7 +96,7 @@ def test_sdr_functional(self, preds, target, sk_metric):
  target,
  signal_distortion_ratio,
  sk_metric,
- metric_args=dict(),
+ metric_args={},
  )
 
  def test_sdr_differentiability(self, preds, target, sk_metric):
@@ -105,7 +105,7 @@ def test_sdr_differentiability(self, preds, target, sk_metric):
  target=target,
  metric_module=SignalDistortionRatio,
  metric_functional=signal_distortion_ratio,
- metric_args=dict(),
+ metric_args={},
  )
 
  @pytest.mark.skipif(
@@ -117,7 +117,7 @@ def test_sdr_half_cpu(self, preds, target, sk_metric):
  target=target,
  metric_module=SignalDistortionRatio,
  metric_functional=signal_distortion_ratio,
- metric_args=dict(),
+ metric_args={},
  )
 
  @pytest.mark.skipif(not torch.cuda.is_available(), reason="test requires cuda")
@@ -127,7 +127,7 @@ def test_sdr_half_gpu(self, preds, target, sk_metric):
  target=target,
  metric_module=SignalDistortionRatio,
  metric_functional=signal_distortion_ratio,
- metric_args=dict(),
+ metric_args={},
  )
 
 

tests/detection/test_map.py

@@ -377,7 +377,7 @@ def test_error_on_wrong_input():
  metric.update([], torch.Tensor()) # type: ignore
 
  with pytest.raises(ValueError, match="Expected argument `preds` and `target` to have the same length"):
- metric.update([dict()], [dict(), dict()])
+ metric.update([{}], [{}, {}])
 
  with pytest.raises(ValueError, match="Expected all dicts in `preds` to contain the `boxes` key"):
  metric.update(

torchmetrics/functional/classification/auroc.py

@@ -111,9 +111,8 @@ def _auroc_compute(
  # max_fpr parameter is only support for binary
  if mode != DataType.BINARY:
  raise ValueError(
- f"Partial AUC computation not available in"
- f" multilabel/multiclass setting, 'max_fpr' must be"
- f" set to `None`, received `{max_fpr}`."
+ "Partial AUC computation not available in multilabel/multiclass setting,"
+ f" 'max_fpr' must be set to `None`, received `{max_fpr}`."
  )
 
  # calculate fpr, tpr
@@ -172,7 +171,7 @@ def _auroc_compute(
 
  allowed_average = (AverageMethod.NONE.value, AverageMethod.MACRO.value, AverageMethod.WEIGHTED.value)
  raise ValueError(
- f"Argument `average` expected to be one of the following:" f" {allowed_average} but got {average}"
+ f"Argument `average` expected to be one of the following: {allowed_average} but got {average}"
  )
 
  return _auc_compute_without_check(fpr, tpr, 1.0)

torchmetrics/functional/text/rouge.py

@@ -458,7 +458,7 @@ def rouge_score(
  stemmer = nltk.stem.porter.PorterStemmer() if use_stemmer else None
 
  if not isinstance(rouge_keys, tuple):
- rouge_keys = tuple([rouge_keys])
+ rouge_keys = (rouge_keys,)
  for key in rouge_keys:
  if key not in ALLOWED_ROUGE_KEYS.keys():
  raise ValueError(f"Got unknown rouge key {key}. Expected to be one of {list(ALLOWED_ROUGE_KEYS.keys())}")

torchmetrics/functional/text/squad.py

@@ -133,7 +133,7 @@ def _squad_input_check(
  _fn_answer = lambda tgt: dict(
  answers=[dict(text=txt) for txt in tgt["answers"]["text"]], id=tgt["id"] # type: ignore
  )
- targets_dict = [dict(paragraphs=[dict(qas=[_fn_answer(target) for target in targets])])]
+ targets_dict = [{"paragraphs": [{"qas": [_fn_answer(target) for target in targets]}]}]
  return preds_dict, targets_dict
 
 

torchmetrics/image/fid.py

@@ -148,10 +148,6 @@ class FrechetInceptionDistance(Metric):
  is installed. Either install as ``pip install torchmetrics[image]`` or
  ``pip install torch-fidelity``
 
- .. note:: the ``forward`` method can be used but ``compute_on_step`` is disabled by default (oppesit of
- all other metrics) as this metric does not really make sense to calculate on a single batch. This
- means that by default ``forward`` will just call ``update`` underneat.
-
  Args:
  feature:
  Either an integer or ``nn.Module``:
@@ -164,13 +160,6 @@ class FrechetInceptionDistance(Metric):
  reset_real_features: Whether to also reset the real features. Since in many cases the real dataset does not
  change, the features can cached them to avoid recomputing them which is costly. Set this to ``False`` if
  your dataset does not change.
-
- compute_on_step:
- Forward only calls ``update()`` and returns None if this is set to False.
-
- .. deprecated:: v0.8
- Argument has no use anymore and will be removed v0.9.
-
  kwargs: Additional keyword arguments, see :ref:`Metric kwargs` for more info.
 
  References:
@@ -215,10 +204,9 @@ def __init__(
  self,
  feature: Union[int, torch.nn.Module] = 2048,
  reset_real_features: bool = True,
- compute_on_step: Optional[bool] = None,
  **kwargs: Dict[str, Any],
  ) -> None:
- super().__init__(compute_on_step=compute_on_step, **kwargs)
+ super().__init__(**kwargs)
 
  rank_zero_warn(
  "Metric `FrechetInceptionDistance` will save all extracted features in buffer."

torchmetrics/image/inception.py

@@ -11,7 +11,7 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-from typing import Any, Dict, List, Optional, Tuple, Union
+from typing import Any, Dict, List, Tuple, Union
 
 import torch
 from torch import Tensor
@@ -46,10 +46,6 @@ class InceptionScore(Metric):
  is installed. Either install as ``pip install torchmetrics[image]`` or
  ``pip install torch-fidelity``
 
- .. note:: the ``forward`` method can be used but ``compute_on_step`` is disabled by default (oppesit of
- all other metrics) as this metric does not really make sense to calculate on a single batch. This
- means that by default ``forward`` will just call ``update`` underneat.
-
  Args:
  feature:
  Either an str, integer or ``nn.Module``:
@@ -60,13 +56,6 @@ class InceptionScore(Metric):
  an ``[N,d]`` matrix where ``N`` is the batch size and ``d`` is the feature size.
 
  splits: integer determining how many splits the inception score calculation should be split among
-
- compute_on_step:
- Forward only calls ``update()`` and returns None if this is set to False.
-
- .. deprecated:: v0.8
- Argument has no use anymore and will be removed v0.9.
-
  kwargs: Additional keyword arguments, see :ref:`Metric kwargs` for more info.
 
  References:
@@ -105,10 +94,9 @@ def __init__(
  self,
  feature: Union[str, int, torch.nn.Module] = "logits_unbiased",
  splits: int = 10,
- compute_on_step: Optional[bool] = None,
  **kwargs: Dict[str, Any],
  ) -> None:
- super().__init__(compute_on_step=compute_on_step, **kwargs)
+ super().__init__(**kwargs)
 
  rank_zero_warn(
  "Metric `InceptionScore` will save all extracted features in buffer."

torchmetrics/image/kid.py

@@ -89,10 +89,6 @@ class KernelInceptionDistance(Metric):
  is installed. Either install as ``pip install torchmetrics[image]`` or
  ``pip install torch-fidelity``
 
- .. note:: the ``forward`` method can be used but ``compute_on_step`` is disabled by default (oppesit of
- all other metrics) as this metric does not really make sense to calculate on a single batch. This
- means that by default ``forward`` will just call ``update`` underneat.
-
  Args:
  feature: Either an str, integer or ``nn.Module``:
 
@@ -109,12 +105,6 @@ class KernelInceptionDistance(Metric):
  reset_real_features: Whether to also reset the real features. Since in many cases the real dataset does not
  change, the features can cached them to avoid recomputing them which is costly. Set this to ``False`` if
  your dataset does not change.
- compute_on_step:
- Forward only calls ``update()`` and returns None if this is set to False.
-
- .. deprecated:: v0.8
- Argument has no use anymore and will be removed v0.9.
-
  kwargs: Additional keyword arguments, see :ref:`Metric kwargs` for more info.
 
  References:
@@ -173,10 +163,9 @@ def __init__(
  gamma: Optional[float] = None, # type: ignore
  coef: float = 1.0,
  reset_real_features: bool = True,
- compute_on_step: Optional[bool] = None,
  **kwargs: Dict[str, Any],
  ) -> None:
- super().__init__(compute_on_step=compute_on_step, **kwargs)
+ super().__init__(**kwargs)
 
  rank_zero_warn(
  "Metric `Kernel Inception Distance` will save all extracted features in buffer."

torchmetrics/image/lpip.py

@@ -11,7 +11,7 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-from typing import Any, Dict, List, Optional
+from typing import Any, Dict, List
 
 import torch
 from torch import Tensor
@@ -59,12 +59,6 @@ class LearnedPerceptualImagePatchSimilarity(Metric):
  Args:
  net_type: str indicating backbone network type to use. Choose between `'alex'`, `'vgg'` or `'squeeze'`
  reduction: str indicating how to reduce over the batch dimension. Choose between `'sum'` or `'mean'`.
- compute_on_step:
- Forward only calls ``update()`` and returns None if this is set to False.
-
- .. deprecated:: v0.8
- Argument has no use anymore and will be removed v0.9.
-
  kwargs: Additional keyword arguments, see :ref:`Metric kwargs` for more info.
 
  Raises:
@@ -98,10 +92,9 @@ def __init__(
  self,
  net_type: str = "alex",
  reduction: Literal["sum", "mean"] = "mean",
- compute_on_step: Optional[bool] = None,
  **kwargs: Dict[str, Any],
  ) -> None:
- super().__init__(compute_on_step=compute_on_step, **kwargs)
+ super().__init__(**kwargs)
 
  if not _LPIPS_AVAILABLE:
  raise ModuleNotFoundError(

torchmetrics/image/psnr.py

@@ -44,12 +44,6 @@ class PeakSignalNoiseRatio(Metric):
  dim:
  Dimensions to reduce PSNR scores over, provided as either an integer or a list of integers. Default is
  None meaning scores will be reduced across all dimensions and all batches.
- compute_on_step:
- Forward only calls ``update()`` and returns None if this is set to False.
-
- .. deprecated:: v0.8
- Argument has no use anymore and will be removed v0.9.
-
  kwargs: Additional keyword arguments, see :ref:`Metric kwargs` for more info.
 
  Raises:
@@ -78,10 +72,9 @@ def __init__(
  base: float = 10.0,
  reduction: Literal["elementwise_mean", "sum", "none", None] = "elementwise_mean",
  dim: Optional[Union[int, Tuple[int, ...]]] = None,
- compute_on_step: Optional[bool] = None,
  **kwargs: Dict[str, Any],
  ) -> None:
- super().__init__(compute_on_step=compute_on_step, **kwargs)
+ super().__init__(**kwargs)
 
  if dim is None and reduction != "elementwise_mean":
  rank_zero_warn(f"The `reduction={reduction}` will not have any effect when `dim` is None.")

torchmetrics/image/ssim.py

@@ -47,12 +47,6 @@ class StructuralSimilarityIndexMeasure(Metric):
  return_contrast_sensitivity: If true, the constant term is returned as a second argument.
  The luminance term can be obtained with luminance=ssim/contrast
  Mutually exclusive with ``return_full_image``
- compute_on_step:
- Forward only calls ``update()`` and returns None if this is set to False.
-
- .. deprecated:: v0.8
- Argument has no use anymore and will be removed v0.9.
-
  kwargs: Additional keyword arguments, see :ref:`Metric kwargs` for more info.
 
  Return:
@@ -81,12 +75,11 @@ def __init__(
  data_range: Optional[float] = None,
  k1: float = 0.01,
  k2: float = 0.03,
- compute_on_step: Optional[bool] = None,
  return_full_image: bool = False,
  return_contrast_sensitivity: bool = False,
  **kwargs: Dict[str, Any],
  ) -> None:
- super().__init__(compute_on_step=compute_on_step, **kwargs)
+ super().__init__(**kwargs)
  rank_zero_warn(
  "Metric `SSIM` will save all targets and"
  " predictions in buffer. For large datasets this may lead"
@@ -157,12 +150,6 @@ class MultiScaleStructuralSimilarityIndexMeasure(Metric):
  normalize: When MultiScaleStructuralSimilarityIndexMeasure loss is used for training, it is desirable to use
  normalizes to improve the training stability. This `normalize` argument is out of scope of the original
  implementation [1], and it is adapted from https://github.com/jorge-pessoa/pytorch-msssim instead.
- compute_on_step:
- Forward only calls ``update()`` and returns None if this is set to False.
-
- .. deprecated:: v0.8
- Argument has no use anymore and will be removed v0.9.
-
  kwargs: Additional keyword arguments, see :ref:`Metric kwargs` for more info.
 
  Return:
@@ -206,10 +193,9 @@ def __init__(
  k2: float = 0.03,
  betas: Tuple[float, ...] = (0.0448, 0.2856, 0.3001, 0.2363, 0.1333),
  normalize: Literal["relu", "simple", None] = None,
- compute_on_step: Optional[bool] = None,
  **kwargs: Dict[str, Any],
  ) -> None:
- super().__init__(compute_on_step=compute_on_step, **kwargs)
+ super().__init__(**kwargs)
  rank_zero_warn(
  "Metric `MS_SSIM` will save all targets and"
  " predictions in buffer. For large datasets this may lead"
@@ -219,7 +205,7 @@ def __init__(
  self.add_state("preds", default=[], dist_reduce_fx="cat")
  self.add_state("target", default=[], dist_reduce_fx="cat")
 
- if not (isinstance(kernel_size, Sequence) or isinstance(kernel_size, int)):
+ if not (isinstance(kernel_size, (Sequence, int))):
  raise ValueError(
  f"Argument `kernel_size` expected to be an sequence or an int, or a single int. Got {kernel_size}"
  )