Add plot to detection

CHANGELOG.md

@@ -23,6 +23,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
  [#1480](https://github.com/Lightning-AI/metrics/pull/1480),
  [#1490](https://github.com/Lightning-AI/metrics/pull/1490),
  [#1581](https://github.com/Lightning-AI/metrics/pull/1581),
+ [#1585](https://github.com/Lightning-AI/metrics/pull/1585),
  [#1593](https://github.com/Lightning-AI/metrics/pull/1593),
 )
 

docs/source/conf.py

@@ -404,6 +404,7 @@ def _get_version_str():
 import os
 import torch
 
+from torch import Tensor
 from torchmetrics import Metric
 
 """

examples/plotting.py

@@ -362,6 +362,37 @@ def universal_image_quality_index():
  return fig, ax
 
 
+def mean_average_precision():
+ """Plot MAP metric."""
+ from torchmetrics.detection.mean_ap import MeanAveragePrecision
+
+ preds = lambda: [
+ {
+ "boxes": torch.tensor([[258.0, 41.0, 606.0, 285.0]]) + torch.randint(10, (1, 4)),
+ "scores": torch.tensor([0.536]) + 0.1 * torch.rand(1),
+ "labels": torch.tensor([0]),
+ }
+ ]
+ target = [
+ {
+ "boxes": torch.tensor([[214.0, 41.0, 562.0, 285.0]]),
+ "labels": torch.tensor([0]),
+ }
+ ]
+
+ # plot single value
+ metric = MeanAveragePrecision()
+ metric.update(preds(), target)
+ fig, ax = metric.plot()
+
+ # plot multiple values
+ metric = MeanAveragePrecision()
+ vals = [metric(preds(), target) for _ in range(10)]
+ fig, ax = metric.plot(vals)
+
+ return fig, ax
+
+
 if __name__ == "__main__":
  metrics_func = {
  "accuracy": accuracy_example,
@@ -373,6 +404,7 @@ def universal_image_quality_index():
  "si-snr": si_snr_example,
  "stoi": stoi_example,
  "mean_squared_error": mean_squared_error_example,
+ "mean_average_precision": mean_average_precision,
  "confusion_matrix": confusion_matrix_example,
  "spectral_distortion_index": spectral_distortion_index_example,
  "error_relative_global_dimensionless_synthesis": error_relative_global_dimensionless_synthesis,

requirements/docs.txt

@@ -15,3 +15,4 @@ sphinx-copybutton>=0.3
 -r integrate.txt
 -r visual.txt
 -r audio.txt
+-r detection.txt

src/torchmetrics/aggregation.py

@@ -20,7 +20,7 @@
 from torchmetrics.utilities import rank_zero_warn
 from torchmetrics.utilities.data import dim_zero_cat
 from torchmetrics.utilities.imports import _MATPLOTLIB_AVAILABLE
-from torchmetrics.utilities.plot import _AX_TYPE, _PLOT_OUT_TYPE, plot_single_or_multi_val
+from torchmetrics.utilities.plot import _AX_TYPE, _PLOT_OUT_TYPE
 
 if not _MATPLOTLIB_AVAILABLE:
  __doctest_skip__ = ["SumMetric.plot", "MeanMetric.plot", "MaxMetric.plot", "MinMetric.plot"]

src/torchmetrics/audio/pesq.py

@@ -18,7 +18,7 @@
 from torchmetrics.functional.audio.pesq import perceptual_evaluation_speech_quality
 from torchmetrics.metric import Metric
 from torchmetrics.utilities.imports import _MATPLOTLIB_AVAILABLE, _PESQ_AVAILABLE
-from torchmetrics.utilities.plot import _AX_TYPE, _PLOT_OUT_TYPE, plot_single_or_multi_val
+from torchmetrics.utilities.plot import _AX_TYPE, _PLOT_OUT_TYPE
 
 __doctest_requires__ = {"PerceptualEvaluationSpeechQuality": ["pesq"]}
 

src/torchmetrics/audio/pit.py

@@ -19,7 +19,7 @@
 from torchmetrics.functional.audio.pit import permutation_invariant_training
 from torchmetrics.metric import Metric
 from torchmetrics.utilities.imports import _MATPLOTLIB_AVAILABLE
-from torchmetrics.utilities.plot import _AX_TYPE, _PLOT_OUT_TYPE, plot_single_or_multi_val
+from torchmetrics.utilities.plot import _AX_TYPE, _PLOT_OUT_TYPE
 
 __doctest_requires__ = {"PermutationInvariantTraining": ["pit"]}
 

src/torchmetrics/audio/sdr.py

@@ -18,7 +18,7 @@
 from torchmetrics.functional.audio.sdr import scale_invariant_signal_distortion_ratio, signal_distortion_ratio
 from torchmetrics.metric import Metric
 from torchmetrics.utilities.imports import _MATPLOTLIB_AVAILABLE
-from torchmetrics.utilities.plot import _AX_TYPE, _PLOT_OUT_TYPE, plot_single_or_multi_val
+from torchmetrics.utilities.plot import _AX_TYPE, _PLOT_OUT_TYPE
 
 __doctest_requires__ = {"SignalDistortionRatio": ["fast_bss_eval"]}
 

src/torchmetrics/audio/snr.py

@@ -18,7 +18,7 @@
 from torchmetrics.functional.audio.snr import scale_invariant_signal_noise_ratio, signal_noise_ratio
 from torchmetrics.metric import Metric
 from torchmetrics.utilities.imports import _MATPLOTLIB_AVAILABLE
-from torchmetrics.utilities.plot import _AX_TYPE, _PLOT_OUT_TYPE, plot_single_or_multi_val
+from torchmetrics.utilities.plot import _AX_TYPE, _PLOT_OUT_TYPE
 
 if not _MATPLOTLIB_AVAILABLE:
  __doctest_skip__ = ["SignalNoiseRatio.plot", "ScaleInvariantSignalNoiseRatio.plot"]

src/torchmetrics/audio/stoi.py

@@ -18,7 +18,7 @@
 from torchmetrics.functional.audio.stoi import short_time_objective_intelligibility
 from torchmetrics.metric import Metric
 from torchmetrics.utilities.imports import _MATPLOTLIB_AVAILABLE, _PYSTOI_AVAILABLE
-from torchmetrics.utilities.plot import _AX_TYPE, _PLOT_OUT_TYPE, plot_single_or_multi_val
+from torchmetrics.utilities.plot import _AX_TYPE, _PLOT_OUT_TYPE
 
 __doctest_requires__ = {"ShortTimeObjectiveIntelligibility": ["pystoi"]}
 

src/torchmetrics/classification/accuracy.py

@@ -20,7 +20,7 @@
 from torchmetrics.metric import Metric
 from torchmetrics.utilities.enums import ClassificationTask
 from torchmetrics.utilities.imports import _MATPLOTLIB_AVAILABLE
-from torchmetrics.utilities.plot import _AX_TYPE, _PLOT_OUT_TYPE, plot_single_or_multi_val
+from torchmetrics.utilities.plot import _AX_TYPE, _PLOT_OUT_TYPE
 
 if not _MATPLOTLIB_AVAILABLE:
  __doctest_skip__ = ["BinaryAccuracy.plot", "MulticlassAccuracy.plot", "MultilabelAccuracy.plot"]

src/torchmetrics/classification/auroc.py

@@ -33,7 +33,7 @@
 from torchmetrics.utilities.data import dim_zero_cat
 from torchmetrics.utilities.enums import ClassificationTask
 from torchmetrics.utilities.imports import _MATPLOTLIB_AVAILABLE
-from torchmetrics.utilities.plot import _AX_TYPE, _PLOT_OUT_TYPE, plot_single_or_multi_val
+from torchmetrics.utilities.plot import _AX_TYPE, _PLOT_OUT_TYPE
 
 if not _MATPLOTLIB_AVAILABLE:
  __doctest_skip__ = ["BinaryAUROC.plot", "MulticlassAUROC.plot", "MultilabelAUROC.plot"]

src/torchmetrics/detection/mean_ap.py

@@ -20,19 +20,23 @@
 
 from torchmetrics.metric import Metric
 from torchmetrics.utilities.data import _cumsum
-from torchmetrics.utilities.imports import _PYCOCOTOOLS_AVAILABLE, _TORCHVISION_GREATER_EQUAL_0_8
+from torchmetrics.utilities.imports import _MATPLOTLIB_AVAILABLE, _PYCOCOTOOLS_AVAILABLE, _TORCHVISION_GREATER_EQUAL_0_8
+from torchmetrics.utilities.plot import _AX_TYPE, _PLOT_OUT_TYPE
+
+if not _MATPLOTLIB_AVAILABLE:
+ __doctest_skip__ = ["MeanAveragePrecision.plot"]
 
 if _TORCHVISION_GREATER_EQUAL_0_8:
  from torchvision.ops import box_area, box_convert, box_iou
 else:
  box_convert = box_iou = box_area = None
- __doctest_skip__ = ["MeanAveragePrecision"]
+ __doctest_skip__ = ["MeanAveragePrecision.plot", "MeanAveragePrecision"]
 
 if _PYCOCOTOOLS_AVAILABLE:
  import pycocotools.mask as mask_utils
 else:
  mask_utils = None
- __doctest_skip__ = ["MeanAveragePrecision"]
+ __doctest_skip__ = ["MeanAveragePrecision.plot", "MeanAveragePrecision"]
 
 
 log = logging.getLogger(__name__)
@@ -912,3 +916,61 @@ def compute(self) -> dict:
  metrics[f"mar_{self.max_detection_thresholds[-1]}_per_class"] = mar_max_dets_per_class_values
  metrics.classes = torch.tensor(classes, dtype=torch.int)
  return metrics
+
+ def plot(
+ self, val: Optional[Union[Dict[str, Tensor], Sequence[Dict[str, Tensor]]]] = None, ax: Optional[_AX_TYPE] = None
+ ) -> _PLOT_OUT_TYPE:
+ """Plot a single or multiple values from the metric.
+
+ Args:
+ val: Either a single result from calling `metric.forward` or `metric.compute` or a list of these results.
+ If no value is provided, will automatically call `metric.compute` and plot that result.
+ ax: An matplotlib axis object. If provided will add plot to that axis
+
+ Returns:
+ Figure object and Axes object
+
+ Raises:
+ ModuleNotFoundError:
+ If `matplotlib` is not installed
+
+ .. plot::
+ :scale: 75
+
+ >>> from torch import tensor
+ >>> from torchmetrics.detection.mean_ap import MeanAveragePrecision
+ >>> preds = [dict(
+ ... boxes=tensor([[258.0, 41.0, 606.0, 285.0]]),
+ ... scores=tensor([0.536]),
+ ... labels=tensor([0]),
+ ... )]
+ >>> target = [dict(
+ ... boxes=tensor([[214.0, 41.0, 562.0, 285.0]]),
+ ... labels=tensor([0]),
+ ... )]
+ >>> metric = MeanAveragePrecision()
+ >>> metric.update(preds, target)
+ >>> fig_, ax_ = metric.plot()
+
+ .. plot::
+ :scale: 75
+
+ >>> # Example plotting multiple values
+ >>> import torch
+ >>> from torchmetrics.detection.mean_ap import MeanAveragePrecision
+ >>> preds = lambda: [dict(
+ ... boxes=torch.tensor([[258.0, 41.0, 606.0, 285.0]]) + torch.randint(10, (1,4)),
+ ... scores=torch.tensor([0.536]) + 0.1*torch.rand(1),
+ ... labels=torch.tensor([0]),
+ ... )]
+ >>> target = [dict(
+ ... boxes=torch.tensor([[214.0, 41.0, 562.0, 285.0]]),
+ ... labels=torch.tensor([0]),
+ ... )]
+ >>> metric = MeanAveragePrecision()
+ >>> vals = []
+ >>> for _ in range(20):
+ ... vals.append(metric(preds(), target))
+ >>> fig_, ax_ = metric.plot(vals)
+ """
+ return self._plot(val, ax)

src/torchmetrics/detection/panoptic_quality.py

@@ -12,7 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 import warnings
-from typing import Any, Collection
+from typing import Any, Collection, Optional, Sequence, Union
 
 import torch
 from torch import Tensor
@@ -27,6 +27,11 @@
  _validate_inputs,
 )
 from torchmetrics.metric import Metric
+from torchmetrics.utilities.imports import _MATPLOTLIB_AVAILABLE
+from torchmetrics.utilities.plot import _AX_TYPE, _PLOT_OUT_TYPE
+
+if not _MATPLOTLIB_AVAILABLE:
+ __doctest_skip__ = ["PanopticQuality.plot"]
 
 
 class PanopticQuality(Metric):
@@ -59,8 +64,9 @@ class PanopticQuality(Metric):
  TypeError:
  If ``things``, ``stuffs`` contain non-integer ``category_id``.
 
- Example:
+ Example:ty
  >>> from torch import tensor
+ >>> from torchmetrics import PanopticQuality
  >>> preds = tensor([[[[6, 0], [0, 0], [6, 0], [6, 0]],
  ... [[0, 0], [0, 0], [6, 0], [0, 1]],
  ... [[0, 0], [0, 0], [6, 0], [0, 1]],
@@ -146,3 +152,63 @@ def update(self, preds: Tensor, target: Tensor) -> None:
  def compute(self) -> Tensor:
  """Compute panoptic quality based on inputs passed in to ``update`` previously."""
  return _panoptic_quality_compute(self.iou_sum, self.true_positives, self.false_positives, self.false_negatives)
+
+ def plot(
+ self, val: Optional[Union[Tensor, Sequence[Tensor]]] = None, ax: Optional[_AX_TYPE] = None
+ ) -> _PLOT_OUT_TYPE:
+ """Plot a single or multiple values from the metric.
+
+ Args:
+ val: Either a single result from calling `metric.forward` or `metric.compute` or a list of these results.
+ If no value is provided, will automatically call `metric.compute` and plot that result.
+ ax: An matplotlib axis object. If provided will add plot to that axis
+
+ Returns:
+ Figure object and Axes object
+
+ Raises:
+ ModuleNotFoundError:
+ If `matplotlib` is not installed
+
+ .. plot::
+ :scale: 75
+
+ >>> from torch import tensor
+ >>> from torchmetrics import PanopticQuality
+ >>> preds = tensor([[[[6, 0], [0, 0], [6, 0], [6, 0]],
+ ... [[0, 0], [0, 0], [6, 0], [0, 1]],
+ ... [[0, 0], [0, 0], [6, 0], [0, 1]],
+ ... [[0, 0], [7, 0], [6, 0], [1, 0]],
+ ... [[0, 0], [7, 0], [7, 0], [7, 0]]]])
+ >>> target = tensor([[[[6, 0], [0, 1], [6, 0], [0, 1]],
+ ... [[0, 1], [0, 1], [6, 0], [0, 1]],
+ ... [[0, 1], [0, 1], [6, 0], [1, 0]],
+ ... [[0, 1], [7, 0], [1, 0], [1, 0]],
+ ... [[0, 1], [7, 0], [7, 0], [7, 0]]]])
+ >>> metric = PanopticQuality(things = {0, 1}, stuffs = {6, 7})
+ >>> metric.update(preds, target)
+ >>> fig_, ax_ = metric.plot()
+
+ .. plot::
+ :scale: 75
+
+ >>> # Example plotting multiple values
+ >>> from torch import tensor
+ >>> from torchmetrics import PanopticQuality
+ >>> preds = tensor([[[[6, 0], [0, 0], [6, 0], [6, 0]],
+ ... [[0, 0], [0, 0], [6, 0], [0, 1]],
+ ... [[0, 0], [0, 0], [6, 0], [0, 1]],
+ ... [[0, 0], [7, 0], [6, 0], [1, 0]],
+ ... [[0, 0], [7, 0], [7, 0], [7, 0]]]])
+ >>> target = tensor([[[[6, 0], [0, 1], [6, 0], [0, 1]],
+ ... [[0, 1], [0, 1], [6, 0], [0, 1]],
+ ... [[0, 1], [0, 1], [6, 0], [1, 0]],
+ ... [[0, 1], [7, 0], [1, 0], [1, 0]],
+ ... [[0, 1], [7, 0], [7, 0], [7, 0]]]])
+ >>> metric = PanopticQuality(things = {0, 1}, stuffs = {6, 7})
+ >>> vals = []
+ >>> for _ in range(20):
+ ... vals.append(metric(preds, target))
+ >>> fig_, ax_ = metric.plot(vals)
+ """
+ return self._plot(val, ax)

src/torchmetrics/image/d_lambda.py

@@ -21,7 +21,7 @@
 from torchmetrics.utilities import rank_zero_warn
 from torchmetrics.utilities.data import dim_zero_cat
 from torchmetrics.utilities.imports import _MATPLOTLIB_AVAILABLE
-from torchmetrics.utilities.plot import _AX_TYPE, _PLOT_OUT_TYPE, plot_single_or_multi_val
+from torchmetrics.utilities.plot import _AX_TYPE, _PLOT_OUT_TYPE
 
 if not _MATPLOTLIB_AVAILABLE:
  __doctest_skip__ = ["SpectralDistortionIndex.plot"]

src/torchmetrics/image/ergas.py

@@ -22,7 +22,7 @@
 from torchmetrics.utilities import rank_zero_warn
 from torchmetrics.utilities.data import dim_zero_cat
 from torchmetrics.utilities.imports import _MATPLOTLIB_AVAILABLE
-from torchmetrics.utilities.plot import _AX_TYPE, _PLOT_OUT_TYPE, plot_single_or_multi_val
+from torchmetrics.utilities.plot import _AX_TYPE, _PLOT_OUT_TYPE
 
 if not _MATPLOTLIB_AVAILABLE:
  __doctest_skip__ = ["ErrorRelativeGlobalDimensionlessSynthesis.plot"]

src/torchmetrics/image/psnr.py

@@ -21,7 +21,7 @@
 from torchmetrics.metric import Metric
 from torchmetrics.utilities import rank_zero_warn
 from torchmetrics.utilities.imports import _MATPLOTLIB_AVAILABLE
-from torchmetrics.utilities.plot import _AX_TYPE, _PLOT_OUT_TYPE, plot_single_or_multi_val
+from torchmetrics.utilities.plot import _AX_TYPE, _PLOT_OUT_TYPE
 
 if not _MATPLOTLIB_AVAILABLE:
  __doctest_skip__ = ["PeakSignalNoiseRatio.plot"]

src/torchmetrics/image/sam.py

@@ -21,7 +21,7 @@
 from torchmetrics.utilities import rank_zero_warn
 from torchmetrics.utilities.data import dim_zero_cat
 from torchmetrics.utilities.imports import _MATPLOTLIB_AVAILABLE
-from torchmetrics.utilities.plot import _AX_TYPE, _PLOT_OUT_TYPE, plot_single_or_multi_val
+from torchmetrics.utilities.plot import _AX_TYPE, _PLOT_OUT_TYPE
 
 if not _MATPLOTLIB_AVAILABLE:
  __doctest_skip__ = ["SpectralAngleMapper.plot"]

src/torchmetrics/image/ssim.py

@@ -21,7 +21,7 @@
 from torchmetrics.metric import Metric
 from torchmetrics.utilities.data import dim_zero_cat
 from torchmetrics.utilities.imports import _MATPLOTLIB_AVAILABLE
-from torchmetrics.utilities.plot import _AX_TYPE, _PLOT_OUT_TYPE, plot_single_or_multi_val
+from torchmetrics.utilities.plot import _AX_TYPE, _PLOT_OUT_TYPE
 
 if not _MATPLOTLIB_AVAILABLE:
  __doctest_skip__ = ["StructuralSimilarityIndexMeasure.plot", "MultiScaleStructuralSimilarityIndexMeasure.plot"]

src/torchmetrics/image/uqi.py

@@ -21,7 +21,7 @@
 from torchmetrics.utilities import rank_zero_warn
 from torchmetrics.utilities.data import dim_zero_cat
 from torchmetrics.utilities.imports import _MATPLOTLIB_AVAILABLE
-from torchmetrics.utilities.plot import _AX_TYPE, _PLOT_OUT_TYPE, plot_single_or_multi_val
+from torchmetrics.utilities.plot import _AX_TYPE, _PLOT_OUT_TYPE
 
 if not _MATPLOTLIB_AVAILABLE:
  __doctest_skip__ = ["UniversalImageQualityIndex.plot"]

src/torchmetrics/regression/mse.py

@@ -18,7 +18,7 @@
 from torchmetrics.functional.regression.mse import _mean_squared_error_compute, _mean_squared_error_update
 from torchmetrics.metric import Metric
 from torchmetrics.utilities.imports import _MATPLOTLIB_AVAILABLE
-from torchmetrics.utilities.plot import _AX_TYPE, _PLOT_OUT_TYPE, plot_single_or_multi_val
+from torchmetrics.utilities.plot import _AX_TYPE, _PLOT_OUT_TYPE
 
 if not _MATPLOTLIB_AVAILABLE:
  __doctest_skip__ = ["MeanSquaredError.plot"]