Merge branch 'develop' into feature/download-uc-merced-and-eurosat

.gitattributes

@@ -1,6 +1,7 @@
 *               text=auto whitespace=trailing-space,space-before-tab,-indent-with-non-tab,tab-in-indent,tabwidth=4
 
 .git*           text export-ignore
+Makefile        text whitespace=-tab-in-indent
 
 *.txt           text
 *.htm           text

.github/workflows/github_pages.yml

@@ -56,4 +56,5 @@ jobs:
         with:
           github_token: ${{ secrets.GITHUB_TOKEN }}
           publish_dir: ./public
+          destination_dir: legacy
           force_orphan: true

.github/workflows/nightly_check.yml

@@ -24,7 +24,7 @@ jobs:
       - name: Installing dependencies
         run: |
           python -m pip install -r tests/requirements.txt
-          python -m pip install -e '.[default,tf]'
+          python -m pip install -e '.[default,tf,tfds-dev]'
       - name: Nightly regression testing
         run: |
           python -m pytest -v --html=nightly_regression_test_report.html

.github/workflows/pr_check.yml

@@ -38,7 +38,7 @@ jobs:
       - name: Installing dependencies
         run: |
           python -m pip install -r tests/requirements.txt
-          python -m pip install -e '.[default,tf]'
+          python -m pip install -e '.[default,tf,tfds-dev]'
       - name: Unit testing
         run: |
           python -m pytest -v tests/unit/ --cov

.github/workflows/weekly_check.yml

@@ -27,7 +27,7 @@ jobs:
       - name: Installing dependencies
         run: |
           python -m pip install -r tests/requirements.txt
-          python -m pip install -e '.[default,tf]'
+          python -m pip install -e '.[default,tf,tfds-dev]'
       - name: Stability testing
         run: |
           python -m pytest --html=stability_test_report.html --minutes 5

CHANGELOG.md

@@ -13,6 +13,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
   (<https://github.com/openvinotoolkit/datumaro/pull/901>, <https://github.com/openvinotoolkit/datumaro/pull/906>)
 - Refactor download CLI commands
   (<https://github.com/openvinotoolkit/datumaro/pull/909>)
+- Refactor CLI commands w/ and w/o project
+  (<https://github.com/openvinotoolkit/datumaro/pull/910>)
 - Add tfds:uc_merced and tfds:eurosat download
   (<https://github.com/openvinotoolkit/datumaro/pull/914>)
 

datumaro/cli/__main__.py

@@ -16,8 +16,9 @@
     send_command_success_info,
 )
 from ..version import __version__
-from . import commands, contexts
-from .util import add_subparser
+from . import contexts
+from .commands import get_non_project_commands, get_project_commands
+from .util import add_subparser, make_subcommands_help
 from .util.errors import CliException
 
 _log_levels = {
@@ -67,16 +68,7 @@ def _define_loglevel_option(parser):
         return parser
 
 
-def _make_subcommands_help(commands, help_line_start=0):
-    desc = ""
-    for command_name, _, command_help in commands:
-        desc += ("  %-" + str(max(0, help_line_start - 2 - 1)) + "s%s\n") % (
-            command_name,
-            command_help,
-        )
-    return desc
-
-
+# TODO: revisit during CLI refactoring
 def _get_known_contexts():
     return [
         ("model", contexts.model, "Actions with models"),
@@ -86,42 +78,9 @@ def _get_known_contexts():
     ]
 
 
-def _get_known_commands():
-    return [
-        ("Project modification:", None, ""),
-        ("add", commands.add, "Add dataset"),
-        ("create", commands.create, "Create empty project"),
-        ("import", commands.import_, "Import dataset"),
-        ("remove", commands.remove, "Remove dataset"),
-        ("", None, ""),
-        ("Project versioning:", None, ""),
-        ("checkout", commands.checkout, "Switch to another branch or revision"),
-        ("commit", commands.commit, "Commit changes in tracked files"),
-        ("log", commands.log, "List history"),
-        ("status", commands.status, "Display current status"),
-        ("", None, ""),
-        ("Dataset operations:", None, ""),
-        ("convert", commands.convert, "Convert dataset between formats"),
-        ("detect-format", commands.detect_format, "Detect the format of a dataset"),
-        ("diff", commands.diff, "Compare datasets"),
-        ("download", commands.download, "Download a publicly available dataset"),
-        ("explain", commands.explain, "Run Explainable AI algorithm for model"),
-        ("export", commands.export, "Export dataset in some format"),
-        ("filter", commands.filter, "Filter dataset items"),
-        ("generate", commands.generate, "Generate synthetic dataset"),
-        ("info", commands.info, "Print dataset info"),
-        ("merge", commands.merge, "Merge datasets"),
-        ("patch", commands.patch, "Update dataset from another one"),
-        ("stats", commands.stats, "Compute dataset statistics"),
-        ("transform", commands.transform, "Modify dataset items"),
-        ("validate", commands.validate, "Validate dataset"),
-        ("search", commands.search, "Search similar datasetitems of query"),
-    ]
-
-
 def _get_sensitive_args():
     known_contexts = _get_known_contexts()
-    known_commands = _get_known_commands()
+    known_commands = get_project_commands() + get_non_project_commands()
 
     res = {}
     for _, command, _ in known_contexts + known_commands:
@@ -142,7 +101,7 @@ def make_parser():
     _LogManager._define_loglevel_option(parser)
 
     known_contexts = _get_known_contexts()
-    known_commands = _get_known_commands()
+    known_commands = get_non_project_commands()
 
     # Argparse doesn't support subparser groups:
     # https://stackoverflow.com/questions/32017020/grouping-argparse-subparser-arguments
@@ -151,12 +110,12 @@ def make_parser():
     subcommands_desc = ""
     if known_contexts:
         subcommands_desc += "Contexts:\n"
-        subcommands_desc += _make_subcommands_help(known_contexts, help_line_start)
+        subcommands_desc += make_subcommands_help(known_contexts, help_line_start)
     if known_commands:
         if subcommands_desc:
             subcommands_desc += "\n"
-        subcommands_desc += "Commands:\n"
-        subcommands_desc += _make_subcommands_help(known_commands, help_line_start)
+        subcommands_desc += "Basic Commands:\n"
+        subcommands_desc += make_subcommands_help(known_commands, help_line_start)
     if subcommands_desc:
         subcommands_desc += (
             "\nRun '%s COMMAND --help' for more information on a command." % parser.prog

datumaro/cli/commands/__init__.py

@@ -4,28 +4,24 @@
 
 # pylint: disable=redefined-builtin
 
-from . import (
-    add,
-    checkout,
-    commit,
-    convert,
-    create,
-    detect_format,
-    diff,
-    explain,
-    export,
-    filter,
-    generate,
-    import_,
-    info,
-    log,
-    merge,
-    patch,
-    remove,
-    search,
-    stats,
-    status,
-    transform,
-    validate,
-)
-from .no_project import download
+from . import convert, detect_format, download, explain, filter, generate, merge, patch, search
+from .require_project import get_project_commands
+
+__all__ = [
+    "get_non_project_commands",
+    "get_project_commands",
+]
+
+
+def get_non_project_commands():
+    return [
+        ("convert", convert, "Convert dataset between formats"),
+        ("detect-format", detect_format, "Detect the format of a dataset"),
+        ("download", download, "Download a publicly available dataset"),
+        ("explain", explain, "Run Explainable AI algorithm for model"),
+        ("filter", filter, "Filter dataset items"),
+        ("generate", generate, "Generate synthetic dataset"),
+        ("merge", merge, "Merge datasets"),
+        ("patch", patch, "Update dataset from another one"),
+        ("search", search, "Search similar datasetitems of query"),
+    ]

datumaro/cli/commands/convert.py

@@ -11,10 +11,9 @@
 from datumaro.components.project import Environment
 from datumaro.util.os_util import make_file_name
 
-from ..contexts.project import FilterModes
 from ..util import MultilineFormatter
 from ..util.errors import CliException
-from ..util.project import generate_next_file_name
+from ..util.project import FilterModes, generate_next_file_name
 
 
 def build_parser(parser_ctor=argparse.ArgumentParser):

datumaro/cli/commands/no_project/download.py → datumaro/cli/commands/download.py

@@ -19,9 +19,9 @@
 from datumaro.util import dump_json
 from datumaro.util.os_util import make_file_name
 
-from ...util import MultilineFormatter
-from ...util.errors import CliException
-from ...util.project import generate_next_file_name
+from ..util import MultilineFormatter
+from ..util.errors import CliException
+from ..util.project import generate_next_file_name
 
 
 def build_parser(parser_ctor=argparse.ArgumentParser):

datumaro/cli/commands/filter.py

@@ -13,10 +13,9 @@
 from datumaro.util import str_to_bool
 from datumaro.util.scope import scope_add, scoped
 
-from ..contexts.project import FilterModes
 from ..util import MultilineFormatter
 from ..util.errors import CliException
-from ..util.project import load_project, parse_full_revpath
+from ..util.project import FilterModes, load_project, parse_full_revpath
 
 
 def build_parser(parser_ctor=argparse.ArgumentParser):

datumaro/cli/commands/require_project/__init__.py

@@ -0,0 +1,30 @@
+# Copyright (C) 2023 Intel Corporation
+#
+# SPDX-License-Identifier: MIT
+
+from . import dataset_operations, modification, versioning
+
+
+def get_project_commands():
+    return [
+        ("Project modification:", None, ""),
+        ("add", modification.add, "Add dataset"),
+        ("create", modification.create, "Create empty project"),
+        ("import", modification.import_, "Import dataset"),
+        ("remove", modification.remove, "Remove dataset"),
+        ("", None, ""),
+        ("Project versioning:", None, ""),
+        ("checkout", versioning.checkout, "Switch to another branch or revision"),
+        ("commit", versioning.commit, "Commit changes in tracked files"),
+        ("log", versioning.log, "List history"),
+        ("status", versioning.status, "Display current status"),
+        ("pinfo", versioning.info, "Print project info"),
+        ("", None, ""),
+        ("Dataset operations:", None, ""),
+        ("diff", dataset_operations.diff, "Compare datasets"),
+        ("export", dataset_operations.export, "Export dataset in some format"),
+        ("stats", dataset_operations.stats, "Compute dataset statistics"),
+        ("transform", dataset_operations.transform, "Modify dataset items"),
+        ("validate", dataset_operations.validate, "Validate dataset"),
+        ("dinfo", dataset_operations.info, "Print dataset info"),
+    ]

datumaro/cli/commands/require_project/dataset_operations/__init__.py

@@ -0,0 +1,5 @@
+# Copyright (C) 2023 Intel Corporation
+#
+# SPDX-License-Identifier: MIT
+
+from . import diff, export, info, stats, transform, validate

datumaro/cli/commands/diff.py → datumaro/cli/commands/require_project/dataset_operations/diff.py

@@ -14,10 +14,10 @@
 from datumaro.util.os_util import rmtree
 from datumaro.util.scope import on_error_do, scope_add, scoped
 
-from ..contexts.project.diff import DiffVisualizer
-from ..util import MultilineFormatter
-from ..util.errors import CliException
-from ..util.project import generate_next_file_name, load_project, parse_full_revpath
+from ....util import MultilineFormatter
+from ....util.diff import DiffVisualizer
+from ....util.errors import CliException
+from ....util.project import generate_next_file_name, load_project, parse_full_revpath
 
 
 class ComparisonMethod(Enum):