ls shell-like wildcards

CHANGELOG.md

@@ -24,6 +24,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 * Fix UnicodeEncodeError in non-Unicode terminals by prioritizing stdout encoding
 * When listing licenses in `license` command only show licenses of `b2` and its dependencies
 
+### Changes
+* Wildcard style of the `ls` commands is now shell-like instead of glob-like
+
+### Removed
+* Remove parameter `--recursive` from `ls` command
+
 ## [3.9.0] - 2023-04-28
 
 ### Added

README.md

@@ -69,7 +69,7 @@ b2 list-buckets [-h] [--json]
 b2 list-keys [-h] [--long]
 b2 list-parts [-h] largeFileId
 b2 list-unfinished-large-files [-h] bucketName
-b2 ls [-h] [--long] [--json] [--replication] [--versions] [--recursive] [--withWildcard] bucketName [folderName]
+b2 ls [-h] [--long] [--json] [--replication] [--versions] [--withWildcard] bucketName [folderName]
 b2 rm [-h] [--dryRun] [--threads THREADS] [--queueSize QUEUESIZE] [--noProgress] [--failFast] [--versions] [--recursive] [--withWildcard] bucketName [folderName]
 b2 make-url [-h] fileId
 b2 make-friendly-url [-h] bucketName fileName

b2/console_tool.py

@@ -1738,26 +1738,31 @@ class AbstractLsCommand(Command, metaclass=ABCMeta):
     The ``--versions`` option selects all versions of each file, not
     just the most recent.
 
-    The ``--recursive`` option will descend into folders, and will select
-    only files, not folders.
-
-    The ``--withWildcard`` option will allow using ``*``, ``?`` and ```[]```
-    characters in ``folderName`` as a greedy wildcard, single character
-    wildcard and range of characters. It requires the ``--recursive`` option.
     Remember to quote ``folderName`` to avoid shell expansion.
     """
+    wildcard_style = 'shell'  # TODO B2-13 remove with rm changes implementation
 
     @classmethod
     def _setup_parser(cls, parser):
         parser.add_argument('--versions', action='store_true')
-        parser.add_argument('--recursive', action='store_true')
         parser.add_argument('--withWildcard', action='store_true')
         parser.add_argument('bucketName').completer = bucket_name_completer
         parser.add_argument('folderName', nargs='?').completer = file_name_completer
 
     def run(self, args):
         generator = self._get_ls_generator(args)
 
+        try:
+            file_version, folder_name = next(generator)
+        except StopIteration:
+            if args.folderName is not None:
+                # specific path requested but not found, exit with error
+                print(f"No such file or directory: {args.folderName}", file=sys.stderr)
+                return 1
+        else:
+            self._print_file_version(args, file_version, folder_name)
+
+        # print remaining items
         for file_version, folder_name in generator:
             self._print_file_version(args, file_version, folder_name)
 
@@ -1775,13 +1780,13 @@ def _get_ls_generator(self, args):
         start_file_name = args.folderName or ''
 
         bucket = self.api.get_bucket_by_name(args.bucketName)
-
         try:
             yield from bucket.ls(
                 start_file_name,
                 latest_only=not args.versions,
                 recursive=args.recursive,
                 with_wildcard=args.withWildcard,
+                wildcard_style=self.wildcard_style,
             )
         except ValueError as error:
             # Wrap these errors into B2Error. At the time of writing there's
@@ -1808,6 +1813,10 @@ class Ls(AbstractLsCommand):
 
     The ``--replication`` option adds replication status
 
+    The ``--withWildcard`` option will allow using ``*``, ```**```, ``?``, ``[]``, ``{{}}``
+    characters in ``folderName`` as a non-greedy wildcard, greedy-wildcard, single character
+    wildcard, range of characters, and sequence of characters respectively.
+
     {ABSTRACTLSCOMMAND}
 
     Examples
@@ -1818,25 +1827,30 @@ class Ls(AbstractLsCommand):
         characters are not expanded by the shell.
 
 
-    List csv and tsv files (in any directory, in the whole bucket):
+    List csv and tsv files (in the current directory):
 
     .. code-block::
 
-        {NAME} ls --recursive --withWildcard bucketName "*.[ct]sv"
+        {NAME} ls bucketName "*.[ct]sv"
 
+    List csv and tsv files (in any directory, in the whole bucket):
+
+    .. code-block::
 
-    List all info.txt files from buckets bX, where X is any character:
+        {NAME} ls bucketName "**/*.[ct]sv"
+
+    List all info.txt files from directories bX, where X is any character:
 
     .. code-block::
 
-        {NAME} ls --recursive --withWildcard bucketName "b?/info.txt"
+        {NAME} ls bucketName "b?/info.txt"
 
 
-    List all pdf files from buckets b0 to b9 (including sub-directories):
+    List all pdf files from directories b0 to b9 (including sub-directories):
 
     .. code-block::
 
-        {NAME} ls --recursive --withWildcard bucketName "b[0-9]/*.pdf"
+        {NAME} ls bucketName "b[0-9]/*.pdf"
 
 
     Requires capability:
@@ -1850,12 +1864,14 @@ class Ls(AbstractLsCommand):
 
     @classmethod
     def _setup_parser(cls, parser):
-        parser.add_argument('--long', action='store_true')
+        parser.add_argument('-l', '--long', action='store_true')
         parser.add_argument('--json', action='store_true')
         parser.add_argument('--replication', action='store_true')
         super()._setup_parser(parser)
 
     def run(self, args):
+        # if not folder is specified, list root folder only
+        args.recursive = args.folderName is not None
         if args.json:
             # TODO: Make this work for an infinite generation.
             #   Use `_print_file_version` to print a single `file_version` and manage the external JSON list
@@ -1928,6 +1944,12 @@ class Rm(AbstractLsCommand):
 
     Progress is displayed on the console unless ``--noProgress`` is specified.
 
+    The ``--recursive`` option will descend into folders, deleting all files
+
+    The ``--withWildcard`` option will allow using ``*``, ```**```, ``?``, ``[]``, ``{{}}``
+    characters in ``folderName`` as a non-greedy wildcard, greedy-wildcard, single character
+    wildcard, range of characters, and sequence of characters respectively. It requires the ``--recursive`` option.
+
     {ABSTRACTLSCOMMAND}
 
     The ``--dryRun`` option prints all the files that would be affected by
@@ -1961,26 +1983,32 @@ class Rm(AbstractLsCommand):
         Use with caution. Running examples presented below can cause data-loss.
 
 
-    Remove all csv and tsv files (in any directory, in the whole bucket):
+    Remove all csv and tsv files in the root of the bucket:
 
     .. code-block::
 
         {NAME} rm --recursive --withWildcard bucketName "*.[ct]sv"
 
 
-    Remove all info.txt files from buckets bX, where X is any character:
+    Remove all info.txt files from folders bX, where X is any character:
 
     .. code-block::
 
         {NAME} rm --recursive --withWildcard bucketName "b?/info.txt"
 
 
-    Remove all pdf files from buckets b0 to b9 (including sub-directories):
+    Remove all pdf files from folders b0 to b9 (including sub-directories):
 
     .. code-block::
 
         {NAME} rm --recursive --withWildcard bucketName "b[0-9]/*.pdf"
 
+    Remove all pdf files from folders b00, b11, b22
+
+    .. code-block::
+
+        {NAME} rm --recursive --withWildcard bucketName "b{{00,11,22}}/*.pdf"
+
 
     Requires capability:
 
@@ -1990,6 +2018,7 @@ class Rm(AbstractLsCommand):
 
     DEFAULT_THREADS = 10
     PROGRESS_REPORT_CLASS = ProgressReport
+    wildcard_style = 'glob'  # TODO B2-13 remove with rm changes implementation
 
     class SubmitThread(threading.Thread):
         END_MARKER = object()
@@ -2045,6 +2074,12 @@ def _run_removal(self, executor: Executor):
 
             self.reporter.end_total()
 
+            if not self.reporter.total_count and self.args.folderName is not None:
+                # specific path requested but not found, exit with error
+                self.messages_queue.put(
+                    (self.ERROR_TAG, None, f"No such file or directory: `{self.args.folderName}`")
+                )
+
         def _removal_done(self, future: Future) -> None:
             with self.mapping_lock:
                 file_version = self.futures_mapping.pop(future)
@@ -2069,6 +2104,7 @@ def _removal_done(self, future: Future) -> None:
 
     @classmethod
     def _setup_parser(cls, parser):
+        parser.add_argument('-r', '--recursive', action='store_true')
         parser.add_argument('--dryRun', action='store_true')
         parser.add_argument('--threads', type=int, default=cls.DEFAULT_THREADS)
         parser.add_argument(
@@ -2082,7 +2118,7 @@ def _setup_parser(cls, parser):
         parser.add_argument('--failFast', action='store_true')
         super()._setup_parser(parser)
 
-    def run(self, args):
+    def run(self, args: argparse.Namespace) -> int:
         if args.dryRun:
             return super().run(args)
 
@@ -2102,8 +2138,11 @@ def run(self, args):
                 event_type, *data = queue_entry
                 if event_type == submit_thread.ERROR_TAG:
                     file_version, error = data
-                    message = f'Deletion of file "{file_version.file_name}" ' \
-                              f'({file_version.id_}) failed: {str(error)}'
+                    if file_version:
+                        message = f'Deletion of file "{file_version.file_name}" ' \
+                                  f'({file_version.id_}) failed: {str(error)}'
+                    else:
+                        message = str(error)
                     reporter.print_completion(message)
 
                     failed_on_any_file = True

test/integration/helpers.py

@@ -218,7 +218,7 @@ def clean_bucket(self, bucket: Union[Bucket, str]):
                 self.api.update_file_legal_hold(
                     file_version_info.id_, file_version_info.file_name, LegalHold.OFF
                 )
-            print('Removing file version:', file_version_info.id_)
+            print('Removing file version:', file_version_info.id_, file_version_info.file_name)
             try:
                 self.api.delete_file_version(file_version_info.id_, file_version_info.file_name)
             except FileNotPresent:
@@ -444,7 +444,7 @@ def should_succeed(
             assert re.search(expected_pattern, stdout), \
             f'did not match pattern="{expected_pattern}", stdout="{stdout}"'
 
-        return stdout
+        return "\n".join([x for x in stdout.split('\n') if not x.strip().startswith('DEBUG:')])
 
     def should_succeed_json(self, args, additional_env: Optional[dict] = None):
         """
@@ -482,7 +482,10 @@ def reauthorize(self, check_key_capabilities=False):
             assert not missing_capabilities, f'it appears that the raw_api integration test is being run with a non-full key. Missing capabilities: {missing_capabilities}'
 
     def list_file_versions(self, bucket_name):
-        return self.should_succeed_json(['ls', '--json', '--recursive', '--versions', bucket_name])
+        # TODO B2-13 replace with `find` command
+        return self.should_succeed_json(
+            ['ls', '--json', '--withWildcard', '--versions', bucket_name, '**']
+        )
 
 
 class TempDir:

test/integration/test_b2_command_line.py

@@ -119,7 +119,7 @@ def test_basic(b2_tool, bucket_name):
     # with_wildcard allows us to target a single file. rm will be removed, rm1 will be left alone
     b2_tool.should_succeed(['rm', '--recursive', '--withWildcard', bucket_name, 'rm'])
     list_of_files = b2_tool.should_succeed_json(
-        ['ls', '--json', '--recursive', '--withWildcard', bucket_name, 'rm*']
+        ['ls', '--withWildcard', '--json', bucket_name, 'rm*']
     )
     should_equal(['rm1'], [f['fileName'] for f in list_of_files])
     b2_tool.should_succeed(['rm', '--recursive', '--withWildcard', bucket_name, 'rm1'])
@@ -131,11 +131,15 @@ def test_basic(b2_tool, bucket_name):
 
     b2_tool.should_succeed(['hide-file', bucket_name, 'c'])
 
-    list_of_files = b2_tool.should_succeed_json(['ls', '--json', '--recursive', bucket_name])
+    # TODO B2-13 replace with `find` command
+    list_of_files = b2_tool.should_succeed_json(
+        ['ls', '--json', '--withWildcard', bucket_name, '**']
+    )
     should_equal(['a', 'b/1', 'b/2', 'd'], [f['fileName'] for f in list_of_files])
 
+    # TODO B2-13 replace with `find` command
     list_of_files = b2_tool.should_succeed_json(
-        ['ls', '--json', '--recursive', '--versions', bucket_name]
+        ['ls', '--json', '--withWildcard', '--versions', bucket_name, '**']
     )
     should_equal(['a', 'a', 'b/1', 'b/2', 'c', 'c', 'd'], [f['fileName'] for f in list_of_files])
     should_equal(
@@ -147,9 +151,7 @@ def test_basic(b2_tool, bucket_name):
 
     first_c_version = list_of_files[4]
     second_c_version = list_of_files[5]
-    list_of_files = b2_tool.should_succeed_json(
-        ['ls', '--json', '--recursive', '--versions', bucket_name, 'c']
-    )
+    list_of_files = b2_tool.should_succeed_json(['ls', '--json', '--versions', bucket_name, 'c'])
     should_equal([], [f['fileName'] for f in list_of_files])
 
     b2_tool.should_succeed(['copy-file-by-id', first_a_version['fileId'], bucket_name, 'x'])
@@ -1143,7 +1145,10 @@ def test_sse_b2(b2_tool, bucket_name):
             ]
         )
 
-    list_of_files = b2_tool.should_succeed_json(['ls', '--json', '--recursive', bucket_name])
+    # TODO B2-13 replace with `find` command
+    list_of_files = b2_tool.should_succeed_json(
+        ['ls', '--json', '--withWildcard', bucket_name, '**']
+    )
     should_equal(
         [{
             'algorithm': 'AES256',
@@ -1170,7 +1175,10 @@ def test_sse_b2(b2_tool, bucket_name):
         ['copy-file-by-id', not_encrypted_version['fileId'], bucket_name, 'copied_not_encrypted']
     )
 
-    list_of_files = b2_tool.should_succeed_json(['ls', '--json', '--recursive', bucket_name])
+    # TODO B2-13 replace with `find` command
+    list_of_files = b2_tool.should_succeed_json(
+        ['ls', '--json', '--withWildcard', bucket_name, '**']
+    )
     should_equal(
         [{
             'algorithm': 'AES256',
@@ -1398,7 +1406,7 @@ def test_sse_c(b2_tool, bucket_name):
             'B2_DESTINATION_SSE_C_KEY_ID': 'another-user-generated-key-id',
         }
     )
-    list_of_files = b2_tool.should_succeed_json(['ls', '--json', '--recursive', bucket_name])
+    list_of_files = b2_tool.should_succeed_json(['ls', '--json', bucket_name])
     should_equal(
         [
             {