Merge branch 'master' into deprecation-decorators

Devanshu24 · web-flow · commit cd9daa142159 · 2021-02-02T00:12:36.000+05:30
diff --git a/.github/workflows/code-style.yml b/.github/workflows/code-style.yml
@@ -22,10 +22,10 @@ jobs:
   code-style:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@master
+      - uses: actions/checkout@v2
       - uses: actions/setup-python@v2
         with:
-          python-version: "3.7"
+          python-version: "3.8"
       - run: |
           python -m pip install autopep8 "black==19.10b0" "isort==4.3.21"
           isort -rc .
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
@@ -203,7 +203,17 @@ git merge upstream/master
 ### Writing documentation
 
 Ignite uses [Google style](http://sphinxcontrib-napoleon.readthedocs.io/en/latest/example_google.html)
-for formatting docstrings. Length of line inside docstrings block must be limited to 120 characters.
+for formatting docstrings and
+
+- [`.. versionadded::`] directive for adding new classes, class methods, functions,
+- [`.. versionchanged::`] directive for adding new arguments, changing internal behaviours, fixing bugs and
+- [`.. deprecated::`] directive for deprecations.
+
+Length of line inside docstrings block must be limited to 120 characters.
+
+[`.. versionadded::`]: https://www.sphinx-doc.org/en/master/usage/restructuredtext/directives.html#directive-versionadded
+[`.. versionchanged::`]: https://www.sphinx-doc.org/en/master/usage/restructuredtext/directives.html#directive-versionchanged
+[`.. deprecated::`]: https://www.sphinx-doc.org/en/master/usage/restructuredtext/directives.html#directive-deprecated
 
 #### Local documentation building and deploying
 
diff --git a/docs/source/conf.py b/docs/source/conf.py
@@ -193,7 +193,10 @@
 # -- Options for intersphinx extension ---------------------------------------
 
 # Example configuration for intersphinx: refer to the Python standard library.
-intersphinx_mapping = {"https://docs.python.org/3/": None}
+intersphinx_mapping = {
+    "python": ("https://docs.python.org/3", None),
+    "torch": ("https://pytorch.org/docs/stable/", None),
+}
 
 # -- Options for todo extension ----------------------------------------------
 
diff --git a/ignite/handlers/checkpoint.py b/ignite/handlers/checkpoint.py
@@ -98,6 +98,8 @@ class Checkpoint(Serializable):
             details.
         include_self (bool): Whether to include the `state_dict` of this object in the checkpoint. If `True`, then
             there must not be another object in ``to_save`` with key ``checkpointer``.
+        greater_or_equal (bool): if `True`, the latest equally scored model is stored. Otherwise, the first model.
+            Default, `False`.
 
     .. _DistributedDataParallel: https://pytorch.org/docs/stable/generated/
         torch.nn.parallel.DistributedDataParallel.html
@@ -245,6 +247,8 @@ def score_function(engine):
             trainer.run(data_loader, max_epochs=10)
             > ["best_model_9_val_acc=0.77.pt", "best_model_10_val_acc=0.78.pt", ]
 
+    .. versionchanged:: 0.4.3
+        Added ``greater_or_equal`` parameter.
     """
 
     Item = NamedTuple("Item", [("priority", int), ("filename", str)])
@@ -261,6 +265,7 @@ def __init__(
         global_step_transform: Optional[Callable] = None,
         filename_pattern: Optional[str] = None,
         include_self: bool = False,
+        greater_or_equal: bool = False,
     ) -> None:
 
         if to_save is not None:  # for compatibility with ModelCheckpoint
@@ -301,6 +306,7 @@ def __init__(
         self.filename_pattern = filename_pattern
         self._saved = []  # type: List["Checkpoint.Item"]
         self.include_self = include_self
+        self.greater_or_equal = greater_or_equal
 
     def reset(self) -> None:
         """Method to reset saved checkpoint names.
@@ -339,6 +345,12 @@ def _check_lt_n_saved(self, or_equal: bool = False) -> bool:
             return True
         return len(self._saved) < self.n_saved + int(or_equal)
 
+    def _compare_fn(self, new: Union[int, float]) -> bool:
+        if self.greater_or_equal:
+            return new >= self._saved[0].priority
+        else:
+            return new > self._saved[0].priority
+
     def __call__(self, engine: Engine) -> None:
 
         global_step = None
@@ -354,7 +366,7 @@ def __call__(self, engine: Engine) -> None:
                 global_step = engine.state.get_event_attrib_value(Events.ITERATION_COMPLETED)
             priority = global_step
 
-        if self._check_lt_n_saved() or self._saved[0].priority < priority:
+        if self._check_lt_n_saved() or self._compare_fn(priority):
 
             priority_str = f"{priority}" if isinstance(priority, numbers.Integral) else f"{priority:.4f}"
 
diff --git a/ignite/metrics/accuracy.py b/ignite/metrics/accuracy.py
@@ -92,8 +92,12 @@ def _check_type(self, output: Sequence[torch.Tensor]) -> None:
 
 
 class Accuracy(_BaseClassification):
-    """
-    Calculates the accuracy for binary, multiclass and multilabel data.
+    r"""Calculates the accuracy for binary, multiclass and multilabel data.
+
+    .. math:: \text{Accuracy} = \frac{ TP + TN }{ TP + TN + FP + FN }
+
+    where :math:`\text{TP}` is true positives, :math:`\text{TN}` is true negatives,
+    :math:`\text{FP}` is false positives and :math:`\text{FN}` is false negatives.
 
     - ``update`` must receive output of the form ``(y_pred, y)`` or ``{'y_pred': y_pred, 'y': y}``.
     - `y_pred` must be in the following shape (batch_size, num_categories, ...) or (batch_size, ...).
diff --git a/ignite/metrics/confusion_matrix.py b/ignite/metrics/confusion_matrix.py
@@ -130,7 +130,9 @@ def normalize(matrix: torch.Tensor, average: str) -> torch.Tensor:
 
 
 def IoU(cm: ConfusionMatrix, ignore_index: Optional[int] = None) -> MetricsLambda:
-    """Calculates Intersection over Union using :class:`~ignite.metrics.ConfusionMatrix` metric.
+    r"""Calculates Intersection over Union using :class:`~ignite.metrics.ConfusionMatrix` metric.
+
+    .. math:: \text{J}(A, B) = \frac{ \lvert A \cap B \rvert }{ \lvert A \cup B \rvert }
 
     Args:
         cm (ConfusionMatrix): instance of confusion matrix metric
diff --git a/ignite/metrics/fbeta.py b/ignite/metrics/fbeta.py
@@ -17,7 +17,13 @@ def Fbeta(
     output_transform: Optional[Callable] = None,
     device: Union[str, torch.device] = torch.device("cpu"),
 ) -> MetricsLambda:
-    """Calculates F-beta score
+    r"""Calculates F-beta score.
+
+    .. math::
+        F_\beta = \left( 1 + \beta^2 \right) * \frac{ \text{precision} * \text{recall} }
+        { \left( \beta^2 * \text{precision} \right) + \text{recall} }
+
+    where :math:`\beta` is a positive real factor.
 
     Args:
         beta (float): weight of precision in harmonic mean
diff --git a/ignite/metrics/mean_absolute_error.py b/ignite/metrics/mean_absolute_error.py
@@ -9,8 +9,11 @@
 
 
 class MeanAbsoluteError(Metric):
-    """
-    Calculates the mean absolute error.
+    r"""Calculates `the mean absolute error <https://en.wikipedia.org/wiki/Mean_absolute_error>`_.
+
+    .. math:: \text{MAE} = \frac{1}{N} \sum_{i=1}^N \lvert y_{i} - x_{i} \rvert
+
+    where :math:`y_{i}` is the prediction tensor and :math:`x_{i}` is ground true tensor.
 
     - ``update`` must receive output of the form ``(y_pred, y)`` or ``{'y_pred': y_pred, 'y': y}``.
     """
diff --git a/ignite/metrics/mean_pairwise_distance.py b/ignite/metrics/mean_pairwise_distance.py
@@ -10,8 +10,8 @@
 
 
 class MeanPairwiseDistance(Metric):
-    """
-    Calculates the mean pairwise distance: average of pairwise distances computed on provided batches.
+    """Calculates the mean :class:`~torch.nn.PairwiseDistance`.
+    Average of pairwise distances computed on provided batches.
 
     - ``update`` must receive output of the form ``(y_pred, y)`` or ``{'y_pred': y_pred, 'y': y}``.
     """
diff --git a/ignite/metrics/mean_squared_error.py b/ignite/metrics/mean_squared_error.py
@@ -9,8 +9,11 @@
 
 
 class MeanSquaredError(Metric):
-    """
-    Calculates the mean squared error.
+    r"""Calculates the `mean squared error <https://en.wikipedia.org/wiki/Mean_squared_error>`_.
+
+    .. math:: \text{MSE} = \frac{1}{N} \sum_{i=1}^N \left(y_{i} - x_{i} \right)^2
+
+    where :math:`y_{i}` is the prediction tensor and :math:`x_{i}` is ground true tensor.
 
     - ``update`` must receive output of the form ``(y_pred, y)`` or ``{'y_pred': y_pred, 'y': y}``.
     """
diff --git a/ignite/metrics/precision.py b/ignite/metrics/precision.py
@@ -69,8 +69,11 @@ def compute(self) -> Union[torch.Tensor, float]:
 
 
 class Precision(_BasePrecisionRecall):
-    """
-    Calculates precision for binary and multiclass data.
+    r"""Calculates precision for binary and multiclass data.
+
+    .. math:: \text{Precision} = \frac{ TP }{ TP + FP }
+
+    where :math:`\text{TP}` is true positives and :math:`\text{FP}` is false positives.
 
     - ``update`` must receive output of the form ``(y_pred, y)`` or ``{'y_pred': y_pred, 'y': y}``.
     - `y_pred` must be in the following shape (batch_size, num_categories, ...) or (batch_size, ...).
diff --git a/ignite/metrics/psnr.py b/ignite/metrics/psnr.py
@@ -12,21 +12,16 @@ class PSNR(Metric):
     r"""Computes average `Peak signal-to-noise ratio (PSNR) <https://en.wikipedia.org/wiki/Peak_signal-to-noise_ratio>`_.
 
     .. math::
-        \text{PSNR}(i, j) = 10 * \log_{10}\left(\frac{ MAX_{i}^2 }{ \text{ MSE } }\right)
+        \text{PSNR}(I, J) = 10 * \log_{10}\left(\frac{ MAX_{I}^2 }{ \text{ MSE } }\right)
 
     where :math:`\text{MSE}` is `mean squared error <https://en.wikipedia.org/wiki/Mean_squared_error>`_.
 
-    Note:
-        A few things to note:
-
-        - `y_pred` and `y` **must** have (batch_size, ...) shape.
-        - `y_pred` and `y` **must** have same dtype and same shape.
-        - If `y_pred` and `y` have almost identical values, the result will be infinity.
+    - `y_pred` and `y` **must** have (batch_size, ...) shape.
+    - `y_pred` and `y` **must** have same dtype and same shape.
 
     Args:
-        data_range (int or float, optional): The data range of the target image (distance between minimum
-            and maximum possible values). If not provided, it will be estimated from the input
-            data type: ``1.0`` for float/double tensor or ``255`` for unsigned 8-bit tensor.
+        data_range (int or float): The data range of the target image (distance between minimum
+            and maximum possible values).
             For other data types, please set the data range, otherwise an exception will be raised.
         output_transform (callable, optional): A callable that is used to transform the Engine’s
             process_function’s output into the form expected by the metric.
@@ -46,27 +41,42 @@ def process_function(engine, batch):
             # ...
             return y_pred, y
         engine = Engine(process_function)
-        psnr = PSNR()
+        psnr = PSNR(data_range=1.0)
         psnr.attach(engine, "psnr")
         # ...
         state = engine.run(data)
         print(f"PSNR: {state.metrics['psnr']}")
 
+    This metric by default accepts Grayscale or RGB images. But if you have YCbCr or YUV images, only
+    Y channel is needed for computing PSNR. And, this can be done with ``output_transform``. For instance,
+
+    .. code-block:: python
+
+        def get_y_channel(output):
+            y_pred, y = output
+            # y_pred and y are (B, 3, H, W) and YCbCr or YUV images
+            # let's select y channel
+            return y_pred[:, 0, ...], y[:, 0, ...]
+
+        psnr = PSNR(data_range=219, output_transform=get_y_channel)
+        psnr.attach(engine, "psnr")
+        # ...
+        state = engine.run(data)
+        print(f"PSNR: {state.metrics['psrn']}")
+
     .. versionadded:: 0.5.0
     """
 
     def __init__(
         self,
-        data_range: Optional[Union[int, float]] = None,
+        data_range: Union[int, float],
         output_transform: Callable = lambda x: x,
         device: Union[str, torch.device] = torch.device("cpu"),
     ):
         super().__init__(output_transform=output_transform, device=device)
         self.data_range = data_range
 
-    def _check_shape_dtype_drange(
-        self, output: Sequence[torch.Tensor], data_range: Union[int, float, None]
-    ) -> Union[int, float]:
+    def _check_shape_dtype(self, output: Sequence[torch.Tensor]) -> None:
         y_pred, y = output
         if y_pred.dtype != y.dtype:
             raise TypeError(
@@ -78,40 +88,19 @@ def _check_shape_dtype_drange(
                 f"Expected y_pred and y to have the same shape. Got y_pred: {y_pred.shape} and y: {y.shape}."
             )
 
-        if data_range is None:
-            try:
-                dmin, dmax = _dtype_range[y.dtype]
-            except KeyError:
-                raise ValueError(
-                    "Range for this dtype cannot be automatically estimated. Please manually specify the data_range."
-                )
-            true_min, true_max = y.min(), y.max()
-            if true_max > dmax or true_min < dmin:
-                raise ValueError(
-                    "y has intensity values outside the range expected "
-                    "for its data type. Please manually specify the data_range."
-                )
-            if true_min >= 0:
-                # most common case (255 for uint8, 1 for float)
-                data_range = dmax
-            else:
-                data_range = dmax - dmin
-        return data_range
-
     @reinit__is_reduced
     def reset(self) -> None:
         self._sum_of_batchwise_psnr = torch.tensor(0.0, dtype=torch.float64, device=self._device)
         self._num_examples = 0
 
     @reinit__is_reduced
     def update(self, output: Sequence[torch.Tensor]) -> None:
-        data_range = self.data_range
-        data_range = self._check_shape_dtype_drange(output, data_range)
+        self._check_shape_dtype(output)
         y_pred, y = output[0].detach(), output[1].detach()
 
         dim = tuple(range(1, y.ndim))
         mse_error = torch.pow(y_pred.double() - y.view_as(y_pred).double(), 2).mean(dim=dim)
-        self._sum_of_batchwise_psnr += torch.sum(10.0 * torch.log10(data_range ** 2 / mse_error)).to(
+        self._sum_of_batchwise_psnr += torch.sum(10.0 * torch.log10(self.data_range ** 2 / (mse_error + 1e-10))).to(
             device=self._device
         )
         self._num_examples += y.shape[0]
@@ -121,11 +110,3 @@ def compute(self) -> torch.Tensor:
         if self._num_examples == 0:
             raise NotComputableError("PSNR must have at least one example before it can be computed.")
         return self._sum_of_batchwise_psnr / self._num_examples
-
-
-_dtype_range = {
-    torch.uint8: (0, 255),
-    torch.float16: (-1.0, 1.0),
-    torch.float32: (-1.0, 1.0),
-    torch.float64: (-1.0, 1.0),
-}
diff --git a/ignite/metrics/recall.py b/ignite/metrics/recall.py
@@ -10,8 +10,11 @@
 
 
 class Recall(_BasePrecisionRecall):
-    """
-    Calculates recall for binary and multiclass data.
+    r"""Calculates recall for binary and multiclass data.
+
+    .. math:: \text{Recall} = \frac{ TP }{ TP + FN }
+
+    where :math:`\text{TP}` is true positives and :math:`\text{FN}` is false negatives.
 
     - ``update`` must receive output of the form ``(y_pred, y)`` or ``{'y_pred': y_pred, 'y': y}``.
     - `y_pred` must be in the following shape (batch_size, num_categories, ...) or (batch_size, ...).
diff --git a/ignite/metrics/root_mean_squared_error.py b/ignite/metrics/root_mean_squared_error.py
@@ -9,8 +9,11 @@
 
 
 class RootMeanSquaredError(MeanSquaredError):
-    """
-    Calculates the root mean squared error.
+    r"""Calculates the `root mean squared error <https://en.wikipedia.org/wiki/Root-mean-square_deviation>`_.
+
+    .. math:: \text{RMSE} = \sqrt{ \frac{1}{N} \sum_{i=1}^N \left(y_{i} - x_{i} \right)^2 }
+
+    where :math:`y_{i}` is the prediction tensor and :math:`x_{i}` is ground true tensor.
 
     - ``update`` must receive output of the form (y_pred, y) or `{'y_pred': y_pred, 'y': y}`.
     """
diff --git a/tests/ignite/handlers/test_checkpoint.py b/tests/ignite/handlers/test_checkpoint.py
@@ -1526,3 +1526,36 @@ def test_checkpoint_reset_with_engine(dirname):
     expected += [f"{_PREFIX}_{name}_{i}.pt" for i in [1 * 2, 2 * 2]]
     assert sorted(os.listdir(dirname)) == sorted(expected)
     assert "PREFIX_model_4.pt" in handler.last_checkpoint
+
+
+def test_greater_or_equal():
+    scores = iter([1, 2, 2, 2])
+
+    def score_function(_):
+        return next(scores)
+
+    class Saver:
+        def __init__(self):
+            self.counter = 0
+
+        def __call__(self, c, f, m):
+            if self.counter == 0:
+                assert f == "model_1.pt"
+            else:
+                assert f == "model_2.pt"
+            self.counter += 1
+
+    handler = Saver()
+
+    checkpointer = Checkpoint(
+        to_save={"model": DummyModel()},
+        save_handler=handler,
+        score_function=score_function,
+        n_saved=2,
+        greater_or_equal=True,
+    )
+    trainer = Engine(lambda e, b: None)
+
+    for _ in range(4):
+        checkpointer(trainer)
+    assert handler.counter == 4
diff --git a/tests/ignite/metrics/test_psnr.py b/tests/ignite/metrics/test_psnr.py
