Improved metrics (#1952)

CHANGELOG.md

@@ -41,6 +41,12 @@ Copy and pasting the git commit messages is __NOT__ enough.
 - Modified naming of benchmark used in NeurIPS 2022 from driving-smarts-competition-env to driving-smarts-v2022.
 - Sstudio generated scenario vehicle traffic ids are now shortened.
 - ChaseViaPoints zoo agent uses unconstrained path change command, instead of being constrained to [-1, 0, +1] path change commands used previously. 
+- Made the metrics module configurable by supplying parameters through a `Params` class.
+- Neighborhood vehicles which should be excluded from the `dist_to_obstacles` cost function can be specified through `Params`. This would be useful in certain tasks, like the vehicle-following task where the distance to the lead vehicle should not be included in the computation of the `dist_to_obstacles` cost function.
+- Unified the computation of `dist_to_destination` (previously known as `completion`) and `steps` (i.e., time taken) as functions inside the cost functions module, instead of computing them separately in a different module.
+- In the metrics module, the records which is the raw metrics data and the scoring which is the formula to compute the final results are now separated to provided greater flexibility for applying metrics to different environments.
+- Benchmark listing may specify specialised metric formula for each benchmark.
+- Changed `benchmark_runner_v0.py` to only average records across scenarios that share the same environment. Records are not averaged across different environments, because the scoring formula may differ in different environments.
 ### Deprecated
 ### Fixed
 - Fixed issues related to waypoints in junctions on Argoverse maps. Waypoints will now be generated for all paths leading through the lane(s) the vehicle is on.
@@ -49,6 +55,7 @@ Copy and pasting the git commit messages is __NOT__ enough.
 - Fixed an issue where building sumo scenarios would sometimes stall.
 - `VehicleIndex` no longer segfaults when attempting to `repr()` it.
 - Fixed issues related to waypoints in SUMO maps. Waypoints in junctions should now return all possible paths through the junction.
+- Fixed CI tests for metrics.
 ### Removed
 - Removed the deprecated `waymo_browser` utility.
 - Removed camera observation `created_at` attribute from metadata to make observation completely reproducible.

smarts/benchmark/driving_smarts/v2023/config_3.yaml

@@ -20,4 +20,4 @@ benchmark:
         - scenarios/sumo/platoon/merge_exit_sumo_t_agents_1
       kwargs:
         seed: 42
-      # metric_formula: smarts/benchmark/driving_smarts/v2023/metric_formula_platoon.py
+      metric_formula: smarts/benchmark/driving_smarts/v2023/metric_formula_platoon.py

smarts/benchmark/driving_smarts/v2023/metric_formula_platoon.py

@@ -0,0 +1,156 @@
+# MIT License
+
+# Copyright (C) 2023. Huawei Technologies Co., Ltd. All rights reserved.
+
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+
+import functools
+from typing import Dict
+
+import numpy as np
+
+from smarts.env.gymnasium.wrappers.metric.costs import Costs
+from smarts.env.gymnasium.wrappers.metric.formula import FormulaBase, Score
+from smarts.env.gymnasium.wrappers.metric.params import (
+    Comfort,
+    DistToObstacles,
+    GapBetweenVehicles,
+    Params,
+    Steps,
+)
+from smarts.env.gymnasium.wrappers.metric.types import Record
+from smarts.env.gymnasium.wrappers.metric.utils import (
+    add_dataclass,
+    divide,
+    op_dataclass,
+)
+
+
+class Formula(FormulaBase):
+    """Sets the (i) cost function parameters, and (ii) score computation formula,
+    for an environment.
+    """
+
+    def __init__(self):
+        pass
+
+    def params(self) -> Params:
+        """Return parameters to configure and initialize cost functions.
+
+        Returns:
+            Params: Cost function parameters.
+        """
+        params = Params(
+            comfort=Comfort(
+                active=False,
+            ), # TODO: Activate after implementing comfort cost function.
+            dist_to_obstacles=DistToObstacles(
+                active=False,
+            ),
+            gap_between_vehicles=GapBetweenVehicles(
+                active=False,
+                interest="Leader-007",
+            ), # TODO: Activate after implmenting gap_between_vehicles cost function.
+            steps=Steps(
+                active=False,
+            ),
+        )
+        return params
+
+    def score(self, records_sum: Dict[str, Dict[str, Record]]) -> Score:
+        """
+        Computes several sub-component scores and one total combined score named
+        "Overall" on the wrapped environment.
+
+        +-------------------+--------+-----------------------------------------------------------+
+        |                   | Range  | Remarks                                                   |
+        +===================+========+===========================================================+
+        | Overall           | [0, 1] | Total score. The higher, the better.                      |
+        +-------------------+--------+-----------------------------------------------------------+
+        | DistToDestination | [0, 1] | Remaining distance to destination. The lower, the better. |
+        +-------------------+--------+-----------------------------------------------------------+
+        | GapBetweenVehicles| [0, 1] | Gap between vehicles in a platoon. The higher, the better.|
+        +-------------------+--------+-----------------------------------------------------------+
+        | Humanness         | [0, 1] | Humanness indicator. The higher, the better.              |
+        +-------------------+--------+-----------------------------------------------------------+
+        | Rules             | [0, 1] | Traffic rules compliance. The higher, the better.         |
+        +-------------------+--------+-----------------------------------------------------------+
+
+        Returns:
+            Score: Contains "Overall", "DistToDestination", "GapBetweenVehicles",
+            "Humanness", and "Rules" scores.
+        """
+
+        costs_total = Costs()
+        episodes = 0
+        for scen, val in records_sum.items():
+            # Number of agents in scenario.
+            agents_in_scenario = len(val.keys())
+            costs_list, counts_list = zip(
+                *[(record.costs, record.counts) for agent, record in val.items()]
+            )
+            # Sum costs over all agents in scenario.
+            costs_sum_agent: Costs = functools.reduce(
+                lambda a, b: add_dataclass(a, b), costs_list
+            )
+            # Average costs over number of agents in scenario.
+            costs_mean_agent = op_dataclass(costs_sum_agent, agents_in_scenario, divide)
+            # Sum costs over all scenarios.
+            costs_total = add_dataclass(costs_total, costs_mean_agent)
+            # Increment total number of episodes.
+            episodes += counts_list[0].episodes
+
+        # Average costs over total number of episodes.
+        costs_final = op_dataclass(costs_total, episodes, divide)
+
+        # Compute sub-components of score.
+        dist_to_destination = costs_final.dist_to_destination
+        humanness = _humanness(costs=costs_final)
+        rules = _rules(costs=costs_final)
+        gap_between_vehicles = costs_final.gap_between_vehicles
+        overall = (
+            0.50 * (1 - dist_to_destination)
+            + 0.25 * gap_between_vehicles
+            + 0.20 * humanness
+            + 0.05 * rules
+        )
+
+        return Score(
+            {
+                "overall": overall,
+                "dist_to_destination": dist_to_destination,
+                "gap_between_vehicles": gap_between_vehicles,
+                "humanness": humanness,
+                "rules": rules,
+            }
+        )
+
+
+def _humanness(costs: Costs) -> float:
+    humanness = np.array(
+        [costs.jerk_linear, costs.lane_center_offset]
+    )
+    humanness = np.mean(humanness, dtype=float)
+    return 1 - humanness
+
+
+def _rules(costs: Costs) -> float:
+    rules = np.array([costs.speed_limit, costs.wrong_way])
+    rules = np.mean(rules, dtype=float)
+    return 1 - rules