From de88e2de5183248e025bc6c04f0d41ae018a4b96 Mon Sep 17 00:00:00 2001 From: Xavier Sumba Date: Mon, 10 Jan 2022 16:35:28 -0500 Subject: [PATCH 1/6] r/fbeta/fbeta_score --- docs/source/references/functional.rst | 4 +- docs/source/references/modules.rst | 6 +-- tests/classification/test_f_beta.py | 20 ++++---- torchmetrics/__init__.py | 2 + torchmetrics/classification/__init__.py | 2 +- torchmetrics/classification/f_beta.py | 48 +++++++++++++++++-- torchmetrics/functional/__init__.py | 3 +- .../functional/classification/__init__.py | 2 +- .../functional/classification/f_beta.py | 30 ++++++++++-- 9 files changed, 91 insertions(+), 26 deletions(-) diff --git a/docs/source/references/functional.rst b/docs/source/references/functional.rst index b7a4817f682..1ee860004a4 100644 --- a/docs/source/references/functional.rst +++ b/docs/source/references/functional.rst @@ -126,10 +126,10 @@ f1 [func] :noindex: -fbeta [func] +fbeta_score [func] ~~~~~~~~~~~~~~~~~~ -.. autofunction:: torchmetrics.functional.fbeta +.. autofunction:: torchmetrics.functional.fbeta_score :noindex: hamming_distance [func] diff --git a/docs/source/references/modules.rst b/docs/source/references/modules.rst index 5e7028a33c0..e2e06c056e6 100644 --- a/docs/source/references/modules.rst +++ b/docs/source/references/modules.rst @@ -286,10 +286,10 @@ F1Score .. autoclass:: torchmetrics.F1Score :noindex: -FBeta -~~~~~ +FBetaScore +~~~~~~~~~~ -.. autoclass:: torchmetrics.FBeta +.. autoclass:: torchmetrics.FBetaScore :noindex: HammingDistance diff --git a/tests/classification/test_f_beta.py b/tests/classification/test_f_beta.py index 582ce6847e9..7348b9bed1e 100644 --- a/tests/classification/test_f_beta.py +++ b/tests/classification/test_f_beta.py @@ -32,9 +32,9 @@ from tests.classification.inputs import _input_multilabel_prob as _input_mlb_prob from tests.helpers import seed_all from tests.helpers.testers import NUM_BATCHES, NUM_CLASSES, THRESHOLD, MetricTester -from torchmetrics import F1Score, FBeta, Metric +from torchmetrics import F1Score, FBetaScore, Metric from torchmetrics.functional import f1_score as f1_score_pl -from torchmetrics.functional import fbeta +from torchmetrics.functional import fbeta_score as fbeta_score_pl from torchmetrics.utilities.checks import _input_format_classification from torchmetrics.utilities.enums import AverageMethod @@ -93,7 +93,7 @@ def _sk_fbeta_f1_multidim_multiclass( @pytest.mark.parametrize( "metric_class, metric_fn", [ - (partial(FBeta, beta=2.0), partial(fbeta, beta=2.0)), + (partial(FBetaScore, beta=2.0), partial(fbeta_score_pl, beta=2.0)), (F1Score, f1_score_pl), ], ) @@ -129,7 +129,7 @@ def test_wrong_params(metric_class, metric_fn, average, mdmc_average, num_classe @pytest.mark.parametrize( "metric_class, metric_fn", [ - (partial(FBeta, beta=2.0), partial(fbeta, beta=2.0)), + (partial(FBetaScore, beta=2.0), partial(fbeta_score_pl, beta=2.0)), (F1Score, f1_score_pl), ], ) @@ -151,7 +151,7 @@ def test_zero_division(metric_class, metric_fn): @pytest.mark.parametrize( "metric_class, metric_fn", [ - (partial(FBeta, beta=2.0), partial(fbeta, beta=2.0)), + (partial(FBetaScore, beta=2.0), partial(fbeta_score_pl, beta=2.0)), (F1Score, f1_score_pl), ], ) @@ -182,7 +182,7 @@ def test_no_support(metric_class, metric_fn): @pytest.mark.parametrize( "metric_class, metric_fn", [ - (partial(FBeta, beta=2.0), partial(fbeta, beta=2.0)), + (partial(FBetaScore, beta=2.0), partial(fbeta_score_pl, beta=2.0)), (F1Score, f1_score_pl), ], ) @@ -210,7 +210,7 @@ def test_class_not_present(metric_class, metric_fn, ignore_index, expected): @pytest.mark.parametrize( "metric_class, metric_fn, sk_fn", [ - (partial(FBeta, beta=2.0), partial(fbeta, beta=2.0), partial(fbeta_score, beta=2.0)), + (partial(FBetaScore, beta=2.0), partial(fbeta_score_pl, beta=2.0), partial(fbeta_score, beta=2.0)), (F1Score, f1_score_pl, f1_score), ], ) @@ -397,8 +397,8 @@ def test_fbeta_f1_differentiability( @pytest.mark.parametrize( "metric_class, metric_fn", [ - (partial(FBeta, beta=2.0), partial(fbeta, beta=2.0)), - (F1Score, fbeta), + (partial(FBetaScore, beta=2.0), partial(fbeta_score_pl, beta=2.0)), + (F1Score, fbeta_score_pl), ], ) @pytest.mark.parametrize( @@ -441,7 +441,7 @@ def test_top_k( @pytest.mark.parametrize( "metric_class, metric_functional, sk_fn", [ - (partial(FBeta, beta=2.0), partial(fbeta, beta=2.0), partial(fbeta_score, beta=2.0)), + (partial(FBetaScore, beta=2.0), partial(fbeta_score_pl, beta=2.0), partial(fbeta_score, beta=2.0)), (F1Score, f1_score_pl, f1_score), ], ) diff --git a/torchmetrics/__init__.py b/torchmetrics/__init__.py index 56293457904..899f845e661 100644 --- a/torchmetrics/__init__.py +++ b/torchmetrics/__init__.py @@ -39,6 +39,7 @@ ConfusionMatrix, F1Score, FBeta, + FBetaScore, HammingDistance, Hinge, HingeLoss, @@ -118,6 +119,7 @@ "F1", "F1Score", "FBeta", + "FBetaScore", "HammingDistance", "Hinge", "HingeLoss", diff --git a/torchmetrics/classification/__init__.py b/torchmetrics/classification/__init__.py index 327d9a7aa51..afde28c9062 100644 --- a/torchmetrics/classification/__init__.py +++ b/torchmetrics/classification/__init__.py @@ -21,7 +21,7 @@ from torchmetrics.classification.calibration_error import CalibrationError # noqa: F401 from torchmetrics.classification.cohen_kappa import CohenKappa # noqa: F401 from torchmetrics.classification.confusion_matrix import ConfusionMatrix # noqa: F401 -from torchmetrics.classification.f_beta import F1, F1Score, FBeta # noqa: F401 +from torchmetrics.classification.f_beta import F1, F1Score, FBeta, FBetaScore # noqa: F401 from torchmetrics.classification.hamming_distance import HammingDistance # noqa: F401 from torchmetrics.classification.hinge import Hinge, HingeLoss # noqa: F401 from torchmetrics.classification.iou import IoU # noqa: F401 diff --git a/torchmetrics/classification/f_beta.py b/torchmetrics/classification/f_beta.py index e36c0b71434..f696b96d37d 100644 --- a/torchmetrics/classification/f_beta.py +++ b/torchmetrics/classification/f_beta.py @@ -22,7 +22,7 @@ from torchmetrics.utilities.enums import AverageMethod -class FBeta(StatScores): +class FBetaScore(StatScores): r""" Computes `F-score`_, specifically: @@ -124,10 +124,10 @@ class FBeta(StatScores): If ``average`` is none of ``"micro"``, ``"macro"``, ``"weighted"``, ``"none"``, ``None``. Example: - >>> from torchmetrics import FBeta + >>> from torchmetrics import FBetaScore >>> target = torch.tensor([0, 1, 2, 0, 1, 2]) >>> preds = torch.tensor([0, 2, 1, 0, 0, 1]) - >>> f_beta = FBeta(num_classes=3, beta=0.5) + >>> f_beta = FBetaScore(num_classes=3, beta=0.5) >>> f_beta(preds, target) tensor(0.3333) @@ -174,8 +174,48 @@ def compute(self) -> Tensor: tp, fp, tn, fn = self._get_final_stats() return _fbeta_compute(tp, fp, tn, fn, self.beta, self.ignore_index, self.average, self.mdmc_reduce) +class FBeta(FBetaScore): + r""" + Computes `F-score`_, specifically: + + .. deprecated:: v0.7 + Use :class:`torchmetrics.FBetaScore`. Will be removed in v0.8. + """ + + def __init__( + self, + num_classes: Optional[int] = None, + beta: float = 1.0, + threshold: float = 0.5, + average: str = "micro", + mdmc_average: Optional[str] = None, + ignore_index: Optional[int] = None, + top_k: Optional[int] = None, + multiclass: Optional[bool] = None, + compute_on_step: bool = True, + dist_sync_on_step: bool = False, + process_group: Optional[Any] = None, + dist_sync_fn: Callable = None, + ) -> None: + warn("`FBeta` was renamed to `FBetaScore` in v0.7 and it will be removed in v0.8", DeprecationWarning) + super().__init__( + num_classes=num_classes, + beta=beta, + threshold=threshold, + average=average, + mdmc_average=mdmc_average, + ignore_index=ignore_index, + top_k=top_k, + multiclass=multiclass, + compute_on_step=compute_on_step, + dist_sync_on_step=dist_sync_on_step, + process_group=process_group, + dist_sync_fn=dist_sync_fn, + ) + + -class F1Score(FBeta): +class F1Score(FBetaScore): """Computes F1 metric. F1 metrics correspond to a harmonic mean of the precision and recall scores. Works with binary, multiclass, and multilabel data. Accepts logits or probabilities from a model diff --git a/torchmetrics/functional/__init__.py b/torchmetrics/functional/__init__.py index 7d6f9942bf2..2b13101d2da 100644 --- a/torchmetrics/functional/__init__.py +++ b/torchmetrics/functional/__init__.py @@ -24,7 +24,7 @@ from torchmetrics.functional.classification.cohen_kappa import cohen_kappa from torchmetrics.functional.classification.confusion_matrix import confusion_matrix from torchmetrics.functional.classification.dice import dice_score -from torchmetrics.functional.classification.f_beta import f1, f1_score, fbeta +from torchmetrics.functional.classification.f_beta import f1, f1_score, fbeta, fbeta_score from torchmetrics.functional.classification.hamming_distance import hamming_distance from torchmetrics.functional.classification.hinge import hinge, hinge_loss from torchmetrics.functional.classification.iou import iou # noqa: F401 @@ -96,6 +96,7 @@ "f1", "f1_score", "fbeta", + "fbeta_score", "hamming_distance", "hinge", "hinge_loss", diff --git a/torchmetrics/functional/classification/__init__.py b/torchmetrics/functional/classification/__init__.py index 5c7995e3215..14395425c16 100644 --- a/torchmetrics/functional/classification/__init__.py +++ b/torchmetrics/functional/classification/__init__.py @@ -19,7 +19,7 @@ from torchmetrics.functional.classification.cohen_kappa import cohen_kappa # noqa: F401 from torchmetrics.functional.classification.confusion_matrix import confusion_matrix # noqa: F401 from torchmetrics.functional.classification.dice import dice_score # noqa: F401 -from torchmetrics.functional.classification.f_beta import f1, f1_score, fbeta # noqa: F401 +from torchmetrics.functional.classification.f_beta import f1, f1_score, fbeta, fbeta_score # noqa: F401 from torchmetrics.functional.classification.hamming_distance import hamming_distance # noqa: F401 from torchmetrics.functional.classification.hinge import hinge, hinge_loss # noqa: F401 from torchmetrics.functional.classification.jaccard import jaccard_index # noqa: F401 diff --git a/torchmetrics/functional/classification/f_beta.py b/torchmetrics/functional/classification/f_beta.py index e8c8410edc1..4194f421fb9 100644 --- a/torchmetrics/functional/classification/f_beta.py +++ b/torchmetrics/functional/classification/f_beta.py @@ -109,7 +109,7 @@ def _fbeta_compute( ) -def fbeta( +def fbeta_score( preds: Tensor, target: Tensor, beta: float = 1.0, @@ -208,10 +208,10 @@ def fbeta( of classes Example: - >>> from torchmetrics.functional import fbeta + >>> from torchmetrics.functional import fbeta_score >>> target = torch.tensor([0, 1, 2, 0, 1, 2]) >>> preds = torch.tensor([0, 2, 1, 0, 0, 1]) - >>> fbeta(preds, target, num_classes=3, beta=0.5) + >>> fbeta_score(preds, target, num_classes=3, beta=0.5) tensor(0.3333) """ @@ -244,6 +244,28 @@ def fbeta( return _fbeta_compute(tp, fp, tn, fn, beta, ignore_index, average, mdmc_average) +def fbeta( + preds: Tensor, + target: Tensor, + beta: float = 1.0, + average: str = "micro", + mdmc_average: Optional[str] = None, + ignore_index: Optional[int] = None, + num_classes: Optional[int] = None, + threshold: float = 0.5, + top_k: Optional[int] = None, + multiclass: Optional[bool] = None, +) -> Tensor: + r""" + Computes f_beta metric. + + .. deprecated:: v0.7 + Use :class:`torchmetrics.functional.f1_score`. Will be removed in v0.8. + """ + warn("`f1` was renamed to `f1_score` in v0.7 and it will be removed in v0.8", DeprecationWarning) + return fbeta_score(preds, target, beta, average, mdmc_average, ignore_index, num_classes, threshold, top_k, multiclass) + + def f1_score( preds: Tensor, target: Tensor, @@ -349,7 +371,7 @@ def f1_score( >>> f1_score(preds, target, num_classes=3) tensor(0.3333) """ - return fbeta(preds, target, 1.0, average, mdmc_average, ignore_index, num_classes, threshold, top_k, multiclass) + return fbeta_score(preds, target, 1.0, average, mdmc_average, ignore_index, num_classes, threshold, top_k, multiclass) def f1( From f840411506ccdaf53291ad39810145565d2554c0 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 10 Jan 2022 21:37:31 +0000 Subject: [PATCH 2/6] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- torchmetrics/classification/f_beta.py | 2 +- torchmetrics/functional/classification/f_beta.py | 8 ++++++-- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/torchmetrics/classification/f_beta.py b/torchmetrics/classification/f_beta.py index f696b96d37d..dedc722b4f8 100644 --- a/torchmetrics/classification/f_beta.py +++ b/torchmetrics/classification/f_beta.py @@ -174,6 +174,7 @@ def compute(self) -> Tensor: tp, fp, tn, fn = self._get_final_stats() return _fbeta_compute(tp, fp, tn, fn, self.beta, self.ignore_index, self.average, self.mdmc_reduce) + class FBeta(FBetaScore): r""" Computes `F-score`_, specifically: @@ -214,7 +215,6 @@ def __init__( ) - class F1Score(FBetaScore): """Computes F1 metric. F1 metrics correspond to a harmonic mean of the precision and recall scores. diff --git a/torchmetrics/functional/classification/f_beta.py b/torchmetrics/functional/classification/f_beta.py index 4194f421fb9..ae9cbe71358 100644 --- a/torchmetrics/functional/classification/f_beta.py +++ b/torchmetrics/functional/classification/f_beta.py @@ -263,7 +263,9 @@ def fbeta( Use :class:`torchmetrics.functional.f1_score`. Will be removed in v0.8. """ warn("`f1` was renamed to `f1_score` in v0.7 and it will be removed in v0.8", DeprecationWarning) - return fbeta_score(preds, target, beta, average, mdmc_average, ignore_index, num_classes, threshold, top_k, multiclass) + return fbeta_score( + preds, target, beta, average, mdmc_average, ignore_index, num_classes, threshold, top_k, multiclass + ) def f1_score( @@ -371,7 +373,9 @@ def f1_score( >>> f1_score(preds, target, num_classes=3) tensor(0.3333) """ - return fbeta_score(preds, target, 1.0, average, mdmc_average, ignore_index, num_classes, threshold, top_k, multiclass) + return fbeta_score( + preds, target, 1.0, average, mdmc_average, ignore_index, num_classes, threshold, top_k, multiclass + ) def f1( From 6aaeef2d675427c2be48d32dd99b44edcafd9ec9 Mon Sep 17 00:00:00 2001 From: Xavier Sumba Date: Mon, 10 Jan 2022 16:38:55 -0500 Subject: [PATCH 3/6] update changelog --- CHANGELOG.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index db69e788c12..f1fee0318cf 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -91,6 +91,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 * `torchmetrics.functional.hinge` -> `torchmetrics.functional.hinge_loss` * `torchmetrics.Hinge` -> `torchmetrics.HingeLoss` +- Renamed F-Beta metrics: ([#729](https://github.com/PyTorchLightning/metrics/pull/729)) + * `torchmetrics.functional.fbeta` -> `torchmetrics.functional.fbeta_score` + * `torchmetrics.FBeta` -> `torchmetrics.FBetaScore` + ### Removed From c89ce111c21cc78f2f837760520b7c34773262bd Mon Sep 17 00:00:00 2001 From: Xavier Sumba Date: Mon, 10 Jan 2022 16:41:57 -0500 Subject: [PATCH 4/6] update PR number in changelog --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f1fee0318cf..ea049b0224f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -91,7 +91,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 * `torchmetrics.functional.hinge` -> `torchmetrics.functional.hinge_loss` * `torchmetrics.Hinge` -> `torchmetrics.HingeLoss` -- Renamed F-Beta metrics: ([#729](https://github.com/PyTorchLightning/metrics/pull/729)) +- Renamed F-Beta metrics: ([#740](https://github.com/PyTorchLightning/metrics/pull/740)) * `torchmetrics.functional.fbeta` -> `torchmetrics.functional.fbeta_score` * `torchmetrics.FBeta` -> `torchmetrics.FBetaScore` From 5837224a384f625233a9e44f513682369bcdd3a6 Mon Sep 17 00:00:00 2001 From: Jirka Borovec Date: Mon, 10 Jan 2022 22:58:33 +0100 Subject: [PATCH 5/6] Apply suggestions from code review --- torchmetrics/classification/f_beta.py | 4 ++++ torchmetrics/functional/classification/f_beta.py | 3 +++ 2 files changed, 7 insertions(+) diff --git a/torchmetrics/classification/f_beta.py b/torchmetrics/classification/f_beta.py index dedc722b4f8..7cb58268a1a 100644 --- a/torchmetrics/classification/f_beta.py +++ b/torchmetrics/classification/f_beta.py @@ -181,6 +181,10 @@ class FBeta(FBetaScore): .. deprecated:: v0.7 Use :class:`torchmetrics.FBetaScore`. Will be removed in v0.8. + Example: + >>> f_beta = FBetaScore(num_classes=3, beta=0.5) + >>> f_beta(torch.tensor([0, 2, 1, 0, 0, 1]), torch.tensor([0, 1, 2, 0, 1, 2])) + tensor(0.3333) """ def __init__( diff --git a/torchmetrics/functional/classification/f_beta.py b/torchmetrics/functional/classification/f_beta.py index ae9cbe71358..9ff877803d3 100644 --- a/torchmetrics/functional/classification/f_beta.py +++ b/torchmetrics/functional/classification/f_beta.py @@ -261,6 +261,9 @@ def fbeta( .. deprecated:: v0.7 Use :class:`torchmetrics.functional.f1_score`. Will be removed in v0.8. + Example: + >>> fbeta(torch.tensor([0, 2, 1, 0, 0, 1]), torch.tensor([0, 1, 2, 0, 1, 2]), num_classes=3, beta=0.5) + tensor(0.3333) """ warn("`f1` was renamed to `f1_score` in v0.7 and it will be removed in v0.8", DeprecationWarning) return fbeta_score( From faffa8603e003d5d7b9fffa35aeae24e15a6ef74 Mon Sep 17 00:00:00 2001 From: Jirka Borovec Date: Tue, 11 Jan 2022 13:19:28 +0100 Subject: [PATCH 6/6] Apply suggestions from code review Co-authored-by: Ethan Harris --- torchmetrics/classification/f_beta.py | 3 ++- torchmetrics/functional/classification/f_beta.py | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/torchmetrics/classification/f_beta.py b/torchmetrics/classification/f_beta.py index 7cb58268a1a..04bda4ff462 100644 --- a/torchmetrics/classification/f_beta.py +++ b/torchmetrics/classification/f_beta.py @@ -181,7 +181,8 @@ class FBeta(FBetaScore): .. deprecated:: v0.7 Use :class:`torchmetrics.FBetaScore`. Will be removed in v0.8. - Example: + + Example:: >>> f_beta = FBetaScore(num_classes=3, beta=0.5) >>> f_beta(torch.tensor([0, 2, 1, 0, 0, 1]), torch.tensor([0, 1, 2, 0, 1, 2])) tensor(0.3333) diff --git a/torchmetrics/functional/classification/f_beta.py b/torchmetrics/functional/classification/f_beta.py index 9ff877803d3..b6b709d5888 100644 --- a/torchmetrics/functional/classification/f_beta.py +++ b/torchmetrics/functional/classification/f_beta.py @@ -261,7 +261,8 @@ def fbeta( .. deprecated:: v0.7 Use :class:`torchmetrics.functional.f1_score`. Will be removed in v0.8. - Example: + + Example:: >>> fbeta(torch.tensor([0, 2, 1, 0, 0, 1]), torch.tensor([0, 1, 2, 0, 1, 2]), num_classes=3, beta=0.5) tensor(0.3333) """