Introduce Validator plugin type

datumaro/cli/contexts/project/__init__.py

@@ -18,7 +18,7 @@
 from datumaro.components.project import \
     PROJECT_DEFAULT_CONFIG as DEFAULT_CONFIG
 from datumaro.components.project import Environment, Project
-from datumaro.components.validator import validate_annotations, TaskType
+from datumaro.components.validator import Validator, TaskType
 from datumaro.util import error_rollback
 
 from ...util import (CliException, MultilineFormatter, add_subparser,
@@ -801,8 +801,7 @@ def build_validate_parser(parser_ctor=argparse.ArgumentParser):
         """,
         formatter_class=MultilineFormatter)
 
-    parser.add_argument('task_type',
-        choices=[task_type.name for task_type in TaskType],
+    parser.add_argument('-t', '--task_type', choices=[task_type.name for task_type in TaskType],
         help="Task type for validation")
     parser.add_argument('-s', '--subset', dest='subset_name', default=None,
         help="Subset to validate (default: None)")
@@ -816,19 +815,20 @@ def build_validate_parser(parser_ctor=argparse.ArgumentParser):
 
 def validate_command(args):
     project = load_project(args.project_dir)
-    task_type = args.task_type
-    subset_name = args.subset_name
-    dst_file_name = f'validation_results-{task_type}'
+    dst_file_name = f'validation_results-{args.task_type}'
 
     dataset = project.make_dataset()
-    if subset_name is not None:
-        dataset = dataset.get_subset(subset_name)
-        dst_file_name += f'-{subset_name}'
+    if args.subset_name is not None:
+        dataset = dataset.get_subset(args.subset_name)
+        dst_file_name += f'-{args.subset_name}'
+
+    validator = project.env.validators['dataset'](args.task_type)
 
     extra_args = {}
-    from datumaro.components.validator import _Validator
-    extra_args = _Validator.parse_cmdline(args.extra_args)
-    validation_results = validate_annotations(dataset, task_type, **extra_args)
+    if hasattr(validator, 'parse_cmdline'):
+        extra_args = validator.parse_cmdline(args.extra_args)
+    validator.set_extra_args(extra_args)
+    validation_results = validator.validate_annotations(dataset)
 
     def numpy_encoder(obj):
         if isinstance(obj, np.generic):
@@ -875,4 +875,4 @@ def build_parser(parser_ctor=argparse.ArgumentParser):
     add_subparser(subparsers, 'stats', build_stats_parser)
     add_subparser(subparsers, 'validate', build_validate_parser)
 
-    return parser
+    return parser

datumaro/components/cli_plugin.py

@@ -45,6 +45,6 @@ def parse_cmdline(cls, args=None):
         return args
 
 def remove_plugin_type(s):
-    for t in {'transform', 'extractor', 'converter', 'launcher', 'importer'}:
+    for t in {'transform', 'extractor', 'converter', 'launcher', 'importer', 'validator'}:
         s = s.replace('_' + t, '')
     return s

datumaro/components/environment.py

@@ -149,6 +149,7 @@ def __init__(self, config=None):
         from datumaro.components.extractor import (Importer, Extractor,
             Transform)
         from datumaro.components.launcher import Launcher
+        from datumaro.components.validator import Validator
         self.extractors = PluginRegistry(
             builtin=select(builtin, Extractor),
             local=select(custom, Extractor)
@@ -172,6 +173,10 @@ def __init__(self, config=None):
             builtin=select(builtin, Transform),
             local=select(custom, Transform)
         )
+        self.validators = PluginRegistry(
+            builtin=select(builtin, Validator),
+            local=select(custom, Validator)
+        )
 
     @staticmethod
     def _find_plugins(plugins_dir):
@@ -262,7 +267,8 @@ def _load_plugins2(cls, plugins_dir):
         from datumaro.components.extractor import (Extractor, Importer,
             Transform)
         from datumaro.components.launcher import Launcher
-        types = [Extractor, Converter, Importer, Launcher, Transform]
+        from datumaro.components.validator import Validator
+        types = [Extractor, Converter, Importer, Launcher, Transform, Validator]
 
         return cls._load_plugins(plugins_dir, types)