Multilabel Ranking metrics

CHANGELOG.md

@@ -11,6 +11,16 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ### Added
 
+
+- Added `CoverageError` to classification metrics ([#787](https://github.com/PyTorchLightning/metrics/pull/787))
+
+
+- Added `LabelRankingAveragePrecision` ([#787](https://github.com/PyTorchLightning/metrics/pull/787))
+
+
+- Added `LabelRankingLoss` ([#787](https://github.com/PyTorchLightning/metrics/pull/787))
+
+
 - Added support for `MetricCollection` in `MetricTracker` ([#718](https://github.com/PyTorchLightning/metrics/pull/718))
 
 

docs/source/references/functional.rst

@@ -112,6 +112,13 @@ confusion_matrix [func]
  :noindex:
 
 
+coverage_error [func]
+~~~~~~~~~~~~~~~~~~~~~
+
+.. autofunction:: torchmetrics.functional.coverage_error
+ :noindex:
+
+
 dice_score [func]
 ~~~~~~~~~~~~~~~~~
 
@@ -161,6 +168,20 @@ kl_divergence [func]
  :noindex:
 
 
+label_ranking_average_precision [func]
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+.. autofunction:: torchmetrics.functional.label_ranking_average_precision
+ :noindex:
+
+
+label_ranking_loss [func]
+~~~~~~~~~~~~~~~~~~~~~~~~~
+
+.. autofunction:: torchmetrics.functional.label_ranking_loss
+ :noindex:
+
+
 matthews_corrcoef [func]
 ~~~~~~~~~~~~~~~~~~~~~~~~
 

docs/source/references/modules.rst

@@ -274,6 +274,12 @@ ConfusionMatrix
 .. autoclass:: torchmetrics.ConfusionMatrix
  :noindex:
 
+CoverageError
+~~~~~~~~~~~~~
+
+.. autoclass:: torchmetrics.CoverageError
+ :noindex:
+
 F1Score
 ~~~~~~~
 
@@ -310,6 +316,18 @@ KLDivergence
 .. autoclass:: torchmetrics.KLDivergence
  :noindex:
 
+LabelRankingAveragePrecision
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+.. autoclass:: torchmetrics.LabelRankingAveragePrecision
+ :noindex:
+
+LabelRankingLoss
+~~~~~~~~~~~~~~~~
+
+.. autoclass:: torchmetrics.LabelRankingLoss
+ :noindex:
+
 MatthewsCorrCoef
 ~~~~~~~~~~~~~~~~
 

tests/classification/test_ranking.py

@@ -0,0 +1,101 @@
+# Copyright The PyTorch Lightning team.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import pytest
+import torch
+from sklearn.metrics import coverage_error as sk_coverage_error
+from sklearn.metrics import label_ranking_average_precision_score as sk_label_ranking
+from sklearn.metrics import label_ranking_loss as sk_label_ranking_loss
+
+from tests.classification.inputs import _input_multilabel_logits as _input_mlb_logits
+from tests.classification.inputs import _input_multilabel_prob as _input_mlb_prob
+from tests.helpers import seed_all
+from tests.helpers.testers import BATCH_SIZE, NUM_BATCHES, MetricTester
+from torchmetrics.classification.ranking import CoverageError, LabelRankingAveragePrecision, LabelRankingLoss
+from torchmetrics.functional.classification.ranking import (
+ coverage_error,
+ label_ranking_average_precision,
+ label_ranking_loss,
+)
+
+seed_all(42)
+
+
+def _sk_coverage_error(preds, target, sample_weight=None):
+ if sample_weight is not None:
+ sample_weight = sample_weight.numpy()
+ return sk_coverage_error(target.numpy(), preds.numpy(), sample_weight=sample_weight)
+
+
+def _sk_label_ranking(preds, target, sample_weight=None):
+ if sample_weight is not None:
+ sample_weight = sample_weight.numpy()
+ return sk_label_ranking(target.numpy(), preds.numpy(), sample_weight=sample_weight)
+
+
+def _sk_label_ranking_loss(preds, target, sample_weight=None):
+ if sample_weight is not None:
+ sample_weight = sample_weight.numpy()
+ return sk_label_ranking_loss(target.numpy(), preds.numpy(), sample_weight=sample_weight)
+
+
+@pytest.mark.parametrize(
+ "metric, functional_metric, sk_metric",
+ [
+ (CoverageError, coverage_error, _sk_coverage_error),
+ (LabelRankingAveragePrecision, label_ranking_average_precision, _sk_label_ranking),
+ (LabelRankingLoss, label_ranking_loss, _sk_label_ranking_loss),
+ ],
+)
+@pytest.mark.parametrize(
+ "preds, target",
+ [
+ (_input_mlb_logits.preds, _input_mlb_logits.target),
+ (_input_mlb_prob.preds, _input_mlb_prob.target),
+ ],
+)
+@pytest.mark.parametrize("sample_weight", [None, torch.rand(NUM_BATCHES, BATCH_SIZE)])
+class TestRanking(MetricTester):
+ @pytest.mark.parametrize("ddp", [False, True])
+ @pytest.mark.parametrize("dist_sync_on_step", [False, True])
+ def test_ranking_class(
+ self, ddp, dist_sync_on_step, preds, target, metric, functional_metric, sk_metric, sample_weight
+ ):
+ self.run_class_metric_test(
+ ddp=ddp,
+ preds=preds,
+ target=target,
+ metric_class=metric,
+ sk_metric=sk_metric,
+ dist_sync_on_step=dist_sync_on_step,
+ fragment_kwargs=True,
+ sample_weight=sample_weight,
+ )
+
+ def test_ranking_functional(self, preds, target, metric, functional_metric, sk_metric, sample_weight):
+ self.run_functional_metric_test(
+ preds,
+ target,
+ metric_functional=functional_metric,
+ sk_metric=sk_metric,
+ fragment_kwargs=True,
+ sample_weight=sample_weight,
+ )
+
+ def test_ranking_differentiability(self, preds, target, metric, functional_metric, sk_metric, sample_weight):
+ self.run_differentiability_test(
+ preds=preds,
+ target=target,
+ metric_module=metric,
+ metric_functional=functional_metric,
+ )

torchmetrics/__init__.py

@@ -2,8 +2,6 @@
 import logging as __logging
 import os
 
-from torchmetrics.__about__ import * # noqa: F401, F403
-
 _logger = __logging.getLogger("torchmetrics")
 _logger.addHandler(__logging.StreamHandler())
 _logger.setLevel(__logging.INFO)
@@ -32,12 +30,15 @@
  CalibrationError,
  CohenKappa,
  ConfusionMatrix,
+ CoverageError,
  F1Score,
  FBetaScore,
  HammingDistance,
  HingeLoss,
  JaccardIndex,
  KLDivergence,
+ LabelRankingAveragePrecision,
+ LabelRankingLoss,
  MatthewsCorrCoef,
  Precision,
  PrecisionRecallCurve,
@@ -115,6 +116,7 @@
  "CohenKappa",
  "ConfusionMatrix",
  "CosineSimilarity",
+ "CoverageError",
  "TweedieDevianceScore",
  "ExplainedVariance",
  "ExtendedEditDistance",
@@ -124,6 +126,8 @@
  "HingeLoss",
  "JaccardIndex",
  "KLDivergence",
+ "LabelRankingAveragePrecision",
+ "LabelRankingLoss",
  "MatthewsCorrCoef",
  "MaxMetric",
  "MeanAbsoluteError",

torchmetrics/classification/__init__.py

@@ -29,6 +29,11 @@
 from torchmetrics.classification.matthews_corrcoef import MatthewsCorrCoef # noqa: F401
 from torchmetrics.classification.precision_recall import Precision, Recall # noqa: F401
 from torchmetrics.classification.precision_recall_curve import PrecisionRecallCurve # noqa: F401
+from torchmetrics.classification.ranking import ( # noqa: F401
+ CoverageError,
+ LabelRankingAveragePrecision,
+ LabelRankingLoss,
+)
 from torchmetrics.classification.roc import ROC # noqa: F401
 from torchmetrics.classification.specificity import Specificity # noqa: F401
 from torchmetrics.classification.stat_scores import StatScores # noqa: F401