From 0492dec8ab446a954d2f3f158ffb9ea9d756a8f3 Mon Sep 17 00:00:00 2001 From: Benjamin Wilson Date: Wed, 27 Apr 2022 13:10:03 -0400 Subject: [PATCH] 3D Detection Eval docstrings + typing fixes. (#40) * Add evaluation. * Small fixes. * Add unit tests + eval mask fixes. * Update detection unit tests. * Fix typing + update pyproject. * Run autoflake. * Add ROI pruning. * Remove arg. * Fix typo. * Speed up argoverse maps. * Speed up evaluation. * Small fixes. * Fix lint. * Small lint fixes. * Fix filtering. * Small fixes. * Fix enums. * Remove unused lines. * Mypy fixes. * Fix click mypy error. * Pytype fixes. * Fix pytype. * Remove pytype. * Small typing fixes. * Add unit tests. * Fix typing. * Remove click typing issue. * Fix mypy. * Detection eval speed up. * Rewrite detection eval for major speedup. * Typing fixes. * Typing fixes. * Switch from record arrays to numpy arrays. * Temp changes. * Improve readability. * Add comments. * Modularize evaluate. * Additional speedups. * Cleanup code. * Additional speedup. * Add roi pruning back. * Add multiprocessing. * Add verbosity. * Mypy fixes. * Update cuboid fields. * Lint fixes. * Fix map tutorial issues. * Add test log. * Revert strings. * Remove outputs. * Address missing detection edge cases. * Address jhony comments. * Update docstring. * Clean docstrings. * Change roi method. * Clean up roi method. * Update roi returns. * Autoflake.: * Fix lint. * Fix lint. * Update detection limiting logic. * Fix indexing. * Fix tuple return. * Update CI. * Add ROI unit tests. * Remove val identity. * Fix import. * Remove unused import. * Update column names. * Update eval.py * Update detection eval. * Update docstrings. * Revert docstring. * Fix formatting. * Small fixes. * Update ci.yml * Update ci.yml * Update ci.yml * Update ci.yml * Update ci.yml * Update ci.yml Co-authored-by: Benjamin Wilson --- .github/workflows/ci.yml | 2 +- src/av2/evaluation/detection/eval.py | 31 ++++++++++++++++----------- src/av2/evaluation/detection/utils.py | 2 -- src/av2/geometry/utm.py | 4 ++-- tests/geometry/test_utm.py | 2 +- 5 files changed, 23 insertions(+), 18 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 13f2f83e..ff39f09b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -12,7 +12,7 @@ jobs: python_version: ["3.8", "3.9", "3.10"] venv_backend: - ["virtualenv", "mamba"] # Testing two different resolvers (pip, mamba). + ["virtualenv"] defaults: run: shell: bash -l {0} diff --git a/src/av2/evaluation/detection/eval.py b/src/av2/evaluation/detection/eval.py index 3ae29cf2..7d95ff36 100644 --- a/src/av2/evaluation/detection/eval.py +++ b/src/av2/evaluation/detection/eval.py @@ -52,11 +52,11 @@ e.g. AP, ATE, ASE, AOE, CDS by default. """ import logging +from multiprocessing import get_context from typing import Dict, Final, List, Optional, Tuple import numpy as np import pandas as pd -from joblib import Parallel, delayed from av2.evaluation.detection.constants import NUM_DECIMALS, MetricNames, TruePositiveErrorNames from av2.evaluation.detection.utils import ( @@ -95,7 +95,7 @@ def evaluate( Each sweep is processed independently, computing assignment between detections and ground truth annotations. Args: - dts: (N,15) Table of detections. + dts: (N,14) Table of detections. gts: (M,15) Table of ground truth annotations. cfg: Detection configuration. n_jobs: Number of jobs running concurrently during evaluation. @@ -106,23 +106,30 @@ def evaluate( Raises: RuntimeError: If accumulation fails. + ValueError: If ROI pruning is enabled but a dataset directory is not specified. """ + if cfg.eval_only_roi_instances and cfg.dataset_dir is None: + raise ValueError( + "ROI pruning has been enabled, but the dataset directory has not be specified. " + "Please set `dataset_directory` to the split root, e.g. av2/sensor/val." + ) + # Sort both the detections and annotations by lexicographic order for grouping. dts = dts.sort_values(list(UUID_COLUMN_NAMES)) gts = gts.sort_values(list(UUID_COLUMN_NAMES)) - dts_npy: NDArrayFloat = dts.loc[:, DTS_COLUMN_NAMES].to_numpy() - gts_npy: NDArrayFloat = gts.loc[:, GTS_COLUMN_NAMES].to_numpy() + dts_npy: NDArrayFloat = dts[list(DTS_COLUMN_NAMES)].to_numpy() + gts_npy: NDArrayFloat = gts[list(GTS_COLUMN_NAMES)].to_numpy() - dts_uuids: List[str] = dts.loc[:, UUID_COLUMN_NAMES].to_numpy().astype(str).tolist() - gts_uuids: List[str] = gts.loc[:, UUID_COLUMN_NAMES].to_numpy().astype(str).tolist() + dts_uuids: List[str] = dts[list(UUID_COLUMN_NAMES)].to_numpy().tolist() + gts_uuids: List[str] = gts[list(UUID_COLUMN_NAMES)].to_numpy().tolist() # We merge the unique identifier -- the tuple of ("log_id", "timestamp_ns", "category") # into a single string to optimize the subsequent grouping operation. # `groupby_mapping` produces a mapping from the uuid to the group of detections / annotations # which fall into that group. - uuid_to_dts = groupby([":".join(x) for x in dts_uuids], dts_npy) - uuid_to_gts = groupby([":".join(x) for x in gts_uuids], gts_npy) + uuid_to_dts = groupby([":".join(map(str, x)) for x in dts_uuids], dts_npy) + uuid_to_gts = groupby([":".join(map(str, x)) for x in gts_uuids], gts_npy) log_id_to_avm: Optional[Dict[str, ArgoverseStaticMap]] = None log_id_to_timestamped_poses: Optional[Dict[str, TimestampedCitySE3EgoPoses]] = None @@ -154,9 +161,9 @@ def evaluate( args_list.append(args) logger.info("Starting evaluation ...") - outputs: Optional[List[Tuple[NDArrayFloat, NDArrayFloat]]] = Parallel(n_jobs=n_jobs, verbose=1)( - delayed(accumulate)(*args) for args in args_list - ) + with get_context("spawn").Pool(processes=n_jobs) as p: + outputs: Optional[List[Tuple[NDArrayFloat, NDArrayFloat]]] = p.starmap(accumulate, args_list) + if outputs is None: raise RuntimeError("Accumulation has failed! Please check the integrity of your detections and annotations.") dts_list, gts_list = zip(*outputs) @@ -182,7 +189,7 @@ def summarize_metrics( """Calculate and print the 3D object detection metrics. Args: - dts: (N,15) Table of detections. + dts: (N,14) Table of detections. gts: (M,15) Table of ground truth annotations. cfg: Detection configuration. diff --git a/src/av2/evaluation/detection/utils.py b/src/av2/evaluation/detection/utils.py index 20004c00..0489c3fe 100644 --- a/src/av2/evaluation/detection/utils.py +++ b/src/av2/evaluation/detection/utils.py @@ -57,7 +57,6 @@ class DetectionCfg: max_range_m: Max distance (under a specific metric in meters) for a detection or ground truth cuboid to be considered for evaluation. num_recall_samples: Number of recall points to sample uniformly in [0, 1]. - splits: Tuple of split names to evaluate. tp_threshold_m: Center distance threshold for the true positive metrics (in meters). """ @@ -70,7 +69,6 @@ class DetectionCfg: max_num_dts_per_category: int = 100 max_range_m: float = 200.0 num_recall_samples: int = 100 - split: str = "val" tp_threshold_m: float = 2.0 @property diff --git a/src/av2/geometry/utm.py b/src/av2/geometry/utm.py index 035dd681..16f54404 100644 --- a/src/av2/geometry/utm.py +++ b/src/av2/geometry/utm.py @@ -7,7 +7,7 @@ WGS84: https://en.wikipedia.org/wiki/World_Geodetic_System """ -from enum import unique, Enum +from enum import Enum, unique from typing import Dict, Final, Tuple, Union import numpy as np @@ -83,7 +83,7 @@ def convert_city_coords_to_utm(points_city: Union[NDArrayFloat, NDArrayInt], cit latitude, longitude = CITY_ORIGIN_LATLONG_DICT[city_name] # get (easting, northing) of origin origin_utm = convert_gps_to_utm(latitude=latitude, longitude=longitude, city_name=city_name) - + points_utm: NDArrayFloat = points_city.astype(float) + np.array(origin_utm, dtype=float) return points_utm diff --git a/tests/geometry/test_utm.py b/tests/geometry/test_utm.py index 283783a2..45ed9960 100644 --- a/tests/geometry/test_utm.py +++ b/tests/geometry/test_utm.py @@ -5,7 +5,7 @@ import numpy as np import av2.geometry.utm as geo_utils -from av2.geometry.utm import CityName, CITY_ORIGIN_LATLONG_DICT +from av2.geometry.utm import CityName from av2.utils.typing import NDArrayFloat