[bugfix] Resolve metrics not being properly resetted on validation epoch end

CHANGELOG.md

@@ -417,6 +417,10 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/).
 - Fixed `lr_find` to generate same results on multiple calls ([#9704](https://github.com/PyTorchLightning/pytorch-lightning/pull/9704))
 
 
+- Fixed `reset` metrics on validation epoch end ([#9717](https://github.com/PyTorchLightning/pytorch-lightning/pull/9717))
+
+
+
 ## [1.4.8] - 2021-09-22
 
 - Fixed error reporting in DDP process reconciliation when processes are launched by an external agent ([#9389](https://github.com/PyTorchLightning/pytorch-lightning/pull/9389))

pytorch_lightning/loops/dataloader/evaluation_loop.py

@@ -214,7 +214,7 @@ def _on_evaluation_end(self, *args: Any, **kwargs: Any) -> None:
             self.trainer.call_hook("on_validation_end", *args, **kwargs)
 
         # reset any `torchmetrics.Metric` and the logger connector state
-        self.trainer.logger_connector.reset_results(metrics=True)
+        self.trainer.logger_connector.reset_results()
 
     def _on_evaluation_epoch_start(self, *args: Any, **kwargs: Any) -> None:
         """Runs ``on_epoch_start`` and ``on_{validation/test}_epoch_start`` hooks."""

tests/trainer/logging_/test_eval_loop_logging.py

@@ -602,3 +602,69 @@ def validation_step(self, batch, batch_idx):
     )
 
     trainer.fit(model)
+
+
+@pytest.mark.parametrize("val_check_interval", [0.5, 1.0])
+def test_multiple_dataloader_reset(val_check_interval, tmpdir):
+    class TestModel(BoringModel):
+        def training_step(self, batch, batch_idx):
+            out = super().training_step(batch, batch_idx)
+            value = 1 + batch_idx
+            if self.current_epoch != 0:
+                value *= 10
+            self.log("batch_idx", value, on_step=True, on_epoch=True, prog_bar=True)
+            return out
+
+        def training_epoch_end(self, outputs):
+            if val_check_interval == 1.0:
+                metrics = self.trainer.progress_bar_metrics
+                if self.current_epoch == 0:
+                    assert metrics["batch_idx_epoch"] == (15 / 5.0)
+                else:
+                    assert metrics["batch_idx_epoch"] == (150 / 5.0)
+
+        def validation_step(self, batch, batch_idx, dataloader_idx):
+            value = (1 + batch_idx) * (2 if dataloader_idx == 1 else 1)
+            if self.current_epoch != 0:
+                value *= 10
+            self.log("val_loss", value, on_step=False, on_epoch=True, prog_bar=True, logger=True)
+            return value
+
+        def validation_epoch_end(self, outputs):
+            if self.current_epoch == 0:
+                assert sum(outputs[0]) / 5 == 3
+                assert sum(outputs[1]) / 5 == 6
+            else:
+                assert sum(outputs[0]) / 5 == 30
+                assert sum(outputs[1]) / 5 == 60
+
+            tot_loss = torch.tensor(0.0)
+            for loss in outputs:
+                tot_loss += sum(loss) / len(loss)
+            tot_loss = tot_loss / len(outputs)
+            if self.current_epoch == 0:
+                assert tot_loss == (3 + 6) / 2
+            else:
+                assert tot_loss == (30 + 60) / 2
+            self.log("tot_val_loss", tot_loss, prog_bar=True, logger=True)
+            assert self.trainer._results["validation_step.val_loss.0"].cumulated_batch_size == 5
+            assert self.trainer._results["validation_step.val_loss.1"].cumulated_batch_size == 5
+
+        def configure_optimizers(self):
+            return torch.optim.SGD(self.layer.parameters(), lr=0.1)
+
+        def val_dataloader(self):
+            return [super().val_dataloader(), super().val_dataloader()]
+
+    model = TestModel()
+    trainer = Trainer(
+        default_root_dir=tmpdir,
+        limit_train_batches=5,
+        limit_val_batches=5,
+        num_sanity_val_steps=0,
+        val_check_interval=val_check_interval,
+        max_epochs=3,
+        log_every_n_steps=1,
+        weights_summary=None,
+    )
+    trainer.fit(model)