Fix the use of logits in calibration error

CHANGELOG.md

@@ -52,7 +52,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - Fixed `BinnedPrecisionRecallCurve` when `thresholds` argument is not provided ([#968](https://github.com/PyTorchLightning/metrics/pull/968))
 
 
--
+- Fixed `CalibrationError` to work on logit input ([#985](https://github.com/PyTorchLightning/metrics/pull/985))
 
 
 ## [0.8.0] - 2022-04-14

tests/classification/test_calibration_error.py

@@ -3,8 +3,10 @@
 
 import numpy as np
 import pytest
+from scipy.special import softmax as _softmax
 
-from tests.classification.inputs import _input_binary_prob
+from tests.classification.inputs import _input_binary_logits, _input_binary_prob
+from tests.classification.inputs import _input_multiclass_logits as _input_mcls_logits
 from tests.classification.inputs import _input_multiclass_prob as _input_mcls_prob
 from tests.classification.inputs import _input_multidim_multiclass_prob as _input_mdmc_prob
 from tests.classification.inputs import _input_multilabel_prob as _input_mlb_prob
@@ -24,8 +26,12 @@
 def _sk_calibration(preds, target, n_bins, norm, debias=False):
  _, _, mode = _input_format_classification(preds, target, threshold=THRESHOLD)
  sk_preds, sk_target = preds.numpy(), target.numpy()
-
+ if mode == DataType.BINARY:
+ if not np.logical_and(0 <= sk_preds, sk_preds <= 1).all():
+ sk_preds = 1.0 / (1 + np.exp(-sk_preds)) # sigmoid transform
  if mode == DataType.MULTICLASS:
+ if not np.logical_and(0 <= sk_preds, sk_preds <= 1).all():
+ sk_preds = _softmax(sk_preds, axis=1)
  # binary label is whether or not the predicted class is correct
  sk_target = np.equal(np.argmax(sk_preds, axis=1), sk_target)
  sk_preds = np.max(sk_preds, axis=1)
@@ -46,7 +52,9 @@ def _sk_calibration(preds, target, n_bins, norm, debias=False):
  "preds, target",
  [
  (_input_binary_prob.preds, _input_binary_prob.target),
+ (_input_binary_logits.preds, _input_binary_logits.target),
  (_input_mcls_prob.preds, _input_mcls_prob.target),
+ (_input_mcls_logits.preds, _input_mcls_logits.target),
  (_input_mdmc_prob.preds, _input_mdmc_prob.target),
  ],
 )

torchmetrics/functional/classification/calibration_error.py

@@ -143,8 +143,12 @@ def _ce_update(preds: Tensor, target: Tensor) -> Tuple[Tensor, Tensor]:
  _, _, mode = _input_format_classification(preds, target)
 
  if mode == DataType.BINARY:
+ if not ((0 <= preds) * (preds <= 1)).all():
+ preds = preds.sigmoid()
  confidences, accuracies = preds, target
  elif mode == DataType.MULTICLASS:
+ if not ((0 <= preds) * (preds <= 1)).all():
+ preds = preds.softmax(dim=1)
  confidences, predictions = preds.max(dim=1)
  accuracies = predictions.eq(target)
  elif mode == DataType.MULTIDIM_MULTICLASS: