bpo-42973: argparse: mixing optional and positional

Doc/library/argparse.rst

@@ -961,6 +961,28 @@ values are:
      usage: PROG [-h] foo [foo ...]
      PROG: error: the following arguments are required: foo
 
+.. index:: single: **; in argparse module
+
+* ``'**'``. This value is only valid for positional arguments. Like ``'*'``,
+  all command-line args are gathered into a list. In contrast to ``'*'``,
+  all arguments are consumed, even if they are interspersed with optional
+  arguments. For example::
+
+     >>> parser = argparse.ArgumentParser()
+     >>> parser.add_argument('--foo', action='store_true')
+     >>> parser.add_argument('bar', nargs='**', action='extend')
+     >>> parser.parse_args('arg1 arg2 --foo arg3 arg4'.split())
+     Namespace(foo=True, bar=['arg1', 'arg2', 'arg3', 'arg4'])
+
+  Note that the ``action`` associated with the positional argument is executed
+  for each group of command-line args separated by optional args, so using the
+  default ``'store'`` action will not give expected results. In the example
+  above, if ``action`` was ``'store'``, ``bar`` in the returned namespace would
+  be just ``['args3', 'args4']``. Positional argument with ``nargs='**'``, if
+  preset, must be the last positional argument.
+
+  .. versionadded:: 3.10
+
 If the ``nargs`` keyword argument is not provided, the number of arguments consumed
 is determined by the action_.  Generally this means a single command-line argument
 will be consumed and a single item (not a list) will be produced.
@@ -1342,6 +1364,8 @@ behavior::
    >>> parser.parse_args('--foo XXX'.split())
    Namespace(bar='XXX')
 
+.. _action-classes:
+
 Action classes
 ^^^^^^^^^^^^^^
 

Lib/argparse.py

@@ -82,6 +82,7 @@
     'REMAINDER',
     'SUPPRESS',
     'ZERO_OR_MORE',
+    'AS_MANY_AS_POSSIBLE'
 ]
 
 
@@ -95,6 +96,7 @@
 
 OPTIONAL = '?'
 ZERO_OR_MORE = '*'
+AS_MANY_AS_POSSIBLE = '**'
 ONE_OR_MORE = '+'
 PARSER = 'A...'
 REMAINDER = '...'
@@ -590,7 +592,8 @@ def _format_args(self, action, default_metavar):
             result = '%s' % get_metavar(1)
         elif action.nargs == OPTIONAL:
             result = '[%s]' % get_metavar(1)
-        elif action.nargs == ZERO_OR_MORE:
+        elif (action.nargs == ZERO_OR_MORE
+                or action.nargs == AS_MANY_AS_POSSIBLE):
             metavar = get_metavar(1)
             if len(metavar) == 2:
                 result = '[%s [%s ...]]' % metavar
@@ -692,7 +695,7 @@ def _get_help_string(self, action):
         help = action.help
         if '%(default)' not in action.help:
             if action.default is not SUPPRESS:
-                defaulting_nargs = [OPTIONAL, ZERO_OR_MORE]
+                defaulting_nargs = [OPTIONAL, ZERO_OR_MORE, AS_MANY_AS_POSSIBLE]
                 if action.option_strings or action.nargs in defaulting_nargs:
                     help += ' (default: %(default)s)'
         return help
@@ -781,6 +784,7 @@ class Action(_AttributeHolder):
                 - N (an integer) consumes N arguments (and produces a list)
                 - '?' consumes zero or one arguments
                 - '*' consumes zero or more arguments (and produces a list)
+                - '**' consumes all remaining positional arguments
                 - '+' consumes one or more arguments (and produces a list)
             Note that the difference between the default and nargs=1 is that
             with the default, a single value will be produced, while with
@@ -1515,7 +1519,8 @@ def _get_positional_kwargs(self, dest, **kwargs):
 
         # mark positional arguments as required if at least one is
         # always required
-        if kwargs.get('nargs') not in [OPTIONAL, ZERO_OR_MORE]:
+        if kwargs.get('nargs') not in [OPTIONAL, ZERO_OR_MORE,
+                                       AS_MANY_AS_POSSIBLE]:
             kwargs['required'] = True
         if kwargs.get('nargs') == ZERO_OR_MORE and 'default' not in kwargs:
             kwargs['required'] = True
@@ -1541,6 +1546,11 @@ def _get_optional_kwargs(self, *args, **kwargs):
             if len(option_string) > 1 and option_string[1] in self.prefix_chars:
                 long_option_strings.append(option_string)
 
+        # nargs='**' is invalid for optionals
+        if kwargs.get('nargs') == AS_MANY_AS_POSSIBLE:
+            msg = _("'nargs=\"**\"' is invalid for optionals")
+            raise TypeError(msg)
+
         # infer destination, '--foo-bar' -> 'foo_bar' and '-x' -> 'x'
         dest = kwargs.pop('dest', None)
         if dest is None:
@@ -2010,17 +2020,23 @@ def consume_positionals(start_index):
             match_partial = self._match_arguments_partial
             selected_pattern = arg_strings_pattern[start_index:]
             arg_counts = match_partial(positionals, selected_pattern)
+            action_index = 0
 
             # slice off the appropriate arg strings for each Positional
             # and add the Positional and its args to the list
-            for action, arg_count in zip(positionals, arg_counts):
+            for arg_count in arg_counts:
+                action = positionals[action_index]
                 args = arg_strings[start_index: start_index + arg_count]
                 start_index += arg_count
                 take_action(action, args)
+                # if positional action nargs is '**',
+                # never remove it from actions list
+                if action.nargs != AS_MANY_AS_POSSIBLE:
+                    action_index += 1
 
             # slice off the Positionals that we just parsed and return the
             # index at which the Positionals' string args stopped
-            positionals[:] = positionals[len(arg_counts):]
+            positionals[:] = positionals[action_index:]
             return start_index
 
         # consume Positionals and Optionals alternately, until we have
@@ -2290,7 +2306,7 @@ def _get_nargs_pattern(self, action):
             nargs_pattern = '(-*A?-*)'
 
         # allow zero or more arguments
-        elif nargs == ZERO_OR_MORE:
+        elif nargs == ZERO_OR_MORE or nargs == AS_MANY_AS_POSSIBLE:
             nargs_pattern = '(-*[A-]*)'
 
         # allow one or more arguments

Lib/test/test_argparse.py

@@ -5012,6 +5012,25 @@ def test_mixed(self):
         self.assertEqual(NS(v=3, spam=True, badger="B"), args)
         self.assertEqual(["C", "--foo", "4"], extras)
 
+    def test_nongreedy(self):
+        parser = argparse.ArgumentParser()
+        parser.add_argument('first', nargs='*', action='extend')
+        parser.add_argument('second')
+
+        argv = ['arg1', 'arg2', 'arg3']
+        args = parser.parse_args(argv)
+        self.assertEqual(NS(first=['arg1', 'arg2'], second='arg3'), args)
+
+    def test_greedy(self):
+        parser = argparse.ArgumentParser()
+        parser.add_argument('--foo', action='store')
+        parser.add_argument('first', nargs='**', action='extend')
+
+        argv = ['arg1', 'arg2', '--foo', 'bar', 'arg3', 'arg4']
+        args = parser.parse_args(argv)
+        self.assertEqual(NS(foo='bar',
+                            first=['arg1', 'arg2', 'arg3', 'arg4']), args)
+
 # ===========================
 # parse_intermixed_args tests
 # ===========================

Misc/NEWS.d/next/Library/2021-01-27-11-16-30.bpo-42973.sHk-kI.rst

@@ -0,0 +1 @@
+New possible value ``'**'`` for ``nargs`` parameter of :func:`ArgumentParser.add_argument` in :mod:`argparse`.