refine eval checking and logging

Author: HydrogenSulfate

ppsci/solver/eval.py

@@ -25,7 +25,6 @@
 from paddle import io
 
 from ppsci.solver import printer
-from ppsci.utils import logger
 from ppsci.utils import misc
 
 if TYPE_CHECKING:
@@ -91,11 +90,6 @@ def _eval_by_dataset(
             input_dict, label_dict, weight_dict = batch
             reader_cost = time.perf_counter() - reader_tic
 
-            # NOTE: eliminate first 5 step for warmup
-            if iter_id == 5:
-                for key in solver.eval_time_info:
-                    solver.eval_time_info[key].reset()
-
             for v in input_dict.values():
                 if hasattr(v, "stop_gradient"):
                     v.stop_gradient = False
@@ -168,11 +162,6 @@ def _eval_by_dataset(
         for metric_name, metric_func in _validator.metric.items():
             # NOTE: compute metric with entire output and label
             metric_dict = metric_func(all_output, all_label)
-            if metric_name in metric_dict_group:
-                logger.warning(
-                    f"Metric name({metric_name}) already exists, please ensure "
-                    "all metric names are unique over all validators."
-                )
             metric_dict_group[metric_name] = {
                 k: float(v) for k, v in metric_dict.items()
             }
@@ -227,11 +216,6 @@ def _eval_by_batch(
             input_dict, label_dict, weight_dict = batch
             reader_cost = time.perf_counter() - reader_tic
 
-            # NOTE: eliminate first 5 step for warmup
-            if iter_id == 5:
-                for key in solver.eval_time_info:
-                    solver.eval_time_info[key].reset()
-
             batch_size = next(iter(input_dict.values())).shape[0]
             for v in input_dict.values():
                 if hasattr(v, "stop_gradient"):
@@ -287,11 +271,6 @@ def _eval_by_batch(
 
         # concatenate all metric and discard metric of padded sample(s)
         for metric_name, metric_dict in metric_dict_group.items():
-            # if metric_name in metric_dict_group:
-            #     logger.warning(
-            #         f"Metric name({metric_name}) already exists, please ensure "
-            #         "all metric names are unique over all validators."
-            #     )
             for var_name, metric_value in metric_dict.items():
                 # NOTE: concat single metric(scalar) list into metric vector
                 metric_value = paddle.concat(metric_value)[:num_samples]

ppsci/solver/printer.py

@@ -149,8 +149,8 @@ def log_eval_info(
         )
 
     # reset time information after printing
-    for key in solver.train_time_info:
-        solver.train_time_info[key].reset()
+    for key in solver.eval_time_info:
+        solver.eval_time_info[key].reset()
 
     # logger.scalar(
     #     {

ppsci/solver/solver.py

@@ -266,6 +266,18 @@ def __init__(
                         f"{self.compute_metric_by_batch} when compute_metric_by_batch="
                         f"{self.compute_metric_by_batch}."
                     )
+            # check metric name uniqueness over all validators
+            _count = {}
+            for _validator in validator.values():
+                for metric_name in _validator.metric:
+                    if metric_name in _count:
+                        logger.warning(
+                            f"Metric name({metric_name}) is duplicated, please ensure "
+                            "all metric names are unique over all given validators."
+                        )
+                    _count[metric_name] = 1
+            del _count
+
         # whether set `stop_gradient=True` for every Tensor if no differentiation involved during evaluation
         if not cfg:
             self.eval_with_no_grad = eval_with_no_grad