bpo-14191 Add parse_intermixed_args. (#3319)

This adds support for parsing a command line where options and positionals are intermixed as is common in many unix commands. This is paul.j3's patch with a few tweaks.

Author: bitdancer

Doc/library/argparse.rst

@@ -1985,6 +1985,45 @@ Exiting methods
    This method prints a usage message including the *message* to the
    standard error and terminates the program with a status code of 2.
 
+
+Intermixed parsing
+^^^^^^^^^^^^^^^^^^
+
+.. method:: ArgumentParser.parse_intermixed_args(args=None, namespace=None)
+.. method:: ArgumentParser.parse_known_intermixed_args(args=None, namespace=None)
+
+A number of Unix commands allow the user to intermix optional arguments with
+positional arguments.  The :meth:`~ArgumentParser.parse_intermixed_args`
+and :meth:`~ArgumentParser.parse_known_intermixed_args` methods
+support this parsing style.
+
+These parsers do not support all the argparse features, and will raise
+exceptions if unsupported features are used.  In particular, subparsers,
+``argparse.REMAINDER``, and mutually exclusive groups that include both
+optionals and positionals are not supported.
+
+The following example shows the difference between
+:meth:`~ArgumentParser.parse_known_args` and
+:meth:`~ArgumentParser.parse_intermixed_args`: the former returns ``['2',
+'3']`` as unparsed arguments, while the latter collects all the positionals
+into ``rest``.  ::
+
+   >>> parser = argparse.ArgumentParser()
+   >>> parser.add_argument('--foo')
+   >>> parser.add_argument('cmd')
+   >>> parser.add_argument('rest', nargs='*', type=int)
+   >>> parser.parse_known_args('doit 1 --foo bar 2 3'.split())
+   (Namespace(cmd='doit', foo='bar', rest=[1]), ['2', '3'])
+   >>> parser.parse_intermixed_args('doit 1 --foo bar 2 3'.split())
+   Namespace(cmd='doit', foo='bar', rest=[1, 2, 3])
+
+:meth:`~ArgumentParser.parse_known_intermixed_args` returns a two item tuple
+containing the populated namespace and the list of remaining argument strings.
+:meth:`~ArgumentParser.parse_intermixed_args` raises an error if there are any
+remaining unparsed argument strings.
+
+.. versionadded:: 3.7
+
 .. _upgrading-optparse-code:
 
 Upgrading optparse code
@@ -2018,9 +2057,8 @@ A partial upgrade path from :mod:`optparse` to :mod:`argparse`:
   called ``options``, now in the :mod:`argparse` context is called ``args``.
 
 * Replace :meth:`optparse.OptionParser.disable_interspersed_args`
-  by setting ``nargs`` of a positional argument to `argparse.REMAINDER`_, or
-  use :meth:`~ArgumentParser.parse_known_args` to collect unparsed argument
-  strings in a separate list.
+  by using :meth:`~ArgumentParser.parse_intermixed_args` instead of
+  :meth:`~ArgumentParser.parse_args`.
 
 * Replace callback actions and the ``callback_*`` keyword arguments with
   ``type`` or ``action`` arguments.

Doc/whatsnew/3.7.rst

@@ -140,6 +140,15 @@ Improved Modules
 ================
 
 
+argparse
+--------
+
+The :meth:`~argparse.ArgumentParser.parse_intermixed_args` supports letting
+the user intermix options and positional arguments on the command line,
+as is possible in many unix commands.  It supports most but not all
+argparse features.  (Contributed by paul.j3 in :issue:`14191`.)
+
+
 binascii
 --------
 

Lib/argparse.py

@@ -587,6 +587,8 @@ def _format_args(self, action, default_metavar):
             result = '...'
         elif action.nargs == PARSER:
             result = '%s ...' % get_metavar(1)
+        elif action.nargs == SUPPRESS:
+            result = ''
         else:
             formats = ['%s' for _ in range(action.nargs)]
             result = ' '.join(formats) % get_metavar(action.nargs)
@@ -2212,6 +2214,10 @@ def _get_nargs_pattern(self, action):
         elif nargs == PARSER:
             nargs_pattern = '(-*A[-AO]*)'
 
+        # suppress action, like nargs=0
+        elif nargs == SUPPRESS:
+            nargs_pattern = '(-*-*)'
+
         # all others should be integers
         else:
             nargs_pattern = '(-*%s-*)' % '-*'.join('A' * nargs)
@@ -2224,6 +2230,91 @@ def _get_nargs_pattern(self, action):
         # return the pattern
         return nargs_pattern
 
+    # ========================
+    # Alt command line argument parsing, allowing free intermix
+    # ========================
+
+    def parse_intermixed_args(self, args=None, namespace=None):
+        args, argv = self.parse_known_intermixed_args(args, namespace)
+        if argv:
+            msg = _('unrecognized arguments: %s')
+            self.error(msg % ' '.join(argv))
+        return args
+
+    def parse_known_intermixed_args(self, args=None, namespace=None):
+        # returns a namespace and list of extras
+        #
+        # positional can be freely intermixed with optionals.  optionals are
+        # first parsed with all positional arguments deactivated.  The 'extras'
+        # are then parsed.  If the parser definition is incompatible with the
+        # intermixed assumptions (e.g. use of REMAINDER, subparsers) a
+        # TypeError is raised.
+        #
+        # positionals are 'deactivated' by setting nargs and default to
+        # SUPPRESS.  This blocks the addition of that positional to the
+        # namespace
+
+        positionals = self._get_positional_actions()
+        a = [action for action in positionals
+             if action.nargs in [PARSER, REMAINDER]]
+        if a:
+            raise TypeError('parse_intermixed_args: positional arg'
+                            ' with nargs=%s'%a[0].nargs)
+
+        if [action.dest for group in self._mutually_exclusive_groups
+            for action in group._group_actions if action in positionals]:
+            raise TypeError('parse_intermixed_args: positional in'
+                            ' mutuallyExclusiveGroup')
+
+        try:
+            save_usage = self.usage
+            try:
+                if self.usage is None:
+                    # capture the full usage for use in error messages
+                    self.usage = self.format_usage()[7:]
+                for action in positionals:
+                    # deactivate positionals
+                    action.save_nargs = action.nargs
+                    # action.nargs = 0
+                    action.nargs = SUPPRESS
+                    action.save_default = action.default
+                    action.default = SUPPRESS
+                namespace, remaining_args = self.parse_known_args(args,
+                                                                  namespace)
+                for action in positionals:
+                    # remove the empty positional values from namespace
+                    if (hasattr(namespace, action.dest)
+                            and getattr(namespace, action.dest)==[]):
+                        from warnings import warn
+                        warn('Do not expect %s in %s' % (action.dest, namespace))
+                        delattr(namespace, action.dest)
+            finally:
+                # restore nargs and usage before exiting
+                for action in positionals:
+                    action.nargs = action.save_nargs
+                    action.default = action.save_default
+            optionals = self._get_optional_actions()
+            try:
+                # parse positionals.  optionals aren't normally required, but
+                # they could be, so make sure they aren't.
+                for action in optionals:
+                    action.save_required = action.required
+                    action.required = False
+                for group in self._mutually_exclusive_groups:
+                    group.save_required = group.required
+                    group.required = False
+                namespace, extras = self.parse_known_args(remaining_args,
+                                                          namespace)
+            finally:
+                # restore parser values before exiting
+                for action in optionals:
+                    action.required = action.save_required
+                for group in self._mutually_exclusive_groups:
+                    group.required = group.save_required
+        finally:
+            self.usage = save_usage
+        return namespace, extras
+
     # ========================
     # Value conversion methods
     # ========================
@@ -2270,6 +2361,10 @@ def _get_values(self, action, arg_strings):
             value = [self._get_value(action, v) for v in arg_strings]
             self._check_value(action, value[0])
 
+        # SUPPRESS argument does not put anything in the namespace
+        elif action.nargs == SUPPRESS:
+            value = SUPPRESS
+
         # all other types of nargs produce a list
         else:
             value = [self._get_value(action, v) for v in arg_strings]

Lib/test/test_argparse.py

@@ -4804,6 +4804,93 @@ def test_mixed(self):
         self.assertEqual(NS(v=3, spam=True, badger="B"), args)
         self.assertEqual(["C", "--foo", "4"], extras)
 
+# ===========================
+# parse_intermixed_args tests
+# ===========================
+
+class TestIntermixedArgs(TestCase):
+    def test_basic(self):
+        # test parsing intermixed optionals and positionals
+        parser = argparse.ArgumentParser(prog='PROG')
+        parser.add_argument('--foo', dest='foo')
+        bar = parser.add_argument('--bar', dest='bar', required=True)
+        parser.add_argument('cmd')
+        parser.add_argument('rest', nargs='*', type=int)
+        argv = 'cmd --foo x 1 --bar y 2 3'.split()
+        args = parser.parse_intermixed_args(argv)
+        # rest gets [1,2,3] despite the foo and bar strings
+        self.assertEqual(NS(bar='y', cmd='cmd', foo='x', rest=[1, 2, 3]), args)
+
+        args, extras = parser.parse_known_args(argv)
+        # cannot parse the '1,2,3'
+        self.assertEqual(NS(bar='y', cmd='cmd', foo='x', rest=[]), args)
+        self.assertEqual(["1", "2", "3"], extras)
+
+        argv = 'cmd --foo x 1 --error 2 --bar y 3'.split()
+        args, extras = parser.parse_known_intermixed_args(argv)
+        # unknown optionals go into extras
+        self.assertEqual(NS(bar='y', cmd='cmd', foo='x', rest=[1]), args)
+        self.assertEqual(['--error', '2', '3'], extras)
+
+        # restores attributes that were temporarily changed
+        self.assertIsNone(parser.usage)
+        self.assertEqual(bar.required, True)
+
+    def test_remainder(self):
+        # Intermixed and remainder are incompatible
+        parser = ErrorRaisingArgumentParser(prog='PROG')
+        parser.add_argument('-z')
+        parser.add_argument('x')
+        parser.add_argument('y', nargs='...')
+        argv = 'X A B -z Z'.split()
+        # intermixed fails with '...' (also 'A...')
+        # self.assertRaises(TypeError, parser.parse_intermixed_args, argv)
+        with self.assertRaises(TypeError) as cm:
+            parser.parse_intermixed_args(argv)
+        self.assertRegex(str(cm.exception), r'\.\.\.')
+
+    def test_exclusive(self):
+        # mutually exclusive group; intermixed works fine
+        parser = ErrorRaisingArgumentParser(prog='PROG')
+        group = parser.add_mutually_exclusive_group(required=True)
+        group.add_argument('--foo', action='store_true', help='FOO')
+        group.add_argument('--spam', help='SPAM')
+        parser.add_argument('badger', nargs='*', default='X', help='BADGER')
+        args = parser.parse_intermixed_args('1 --foo 2'.split())
+        self.assertEqual(NS(badger=['1', '2'], foo=True, spam=None), args)
+        self.assertRaises(ArgumentParserError, parser.parse_intermixed_args, '1 2'.split())
+        self.assertEqual(group.required, True)
+
+    def test_exclusive_incompatible(self):
+        # mutually exclusive group including positional - fail
+        parser = ErrorRaisingArgumentParser(prog='PROG')
+        group = parser.add_mutually_exclusive_group(required=True)
+        group.add_argument('--foo', action='store_true', help='FOO')
+        group.add_argument('--spam', help='SPAM')
+        group.add_argument('badger', nargs='*', default='X', help='BADGER')
+        self.assertRaises(TypeError, parser.parse_intermixed_args, [])
+        self.assertEqual(group.required, True)
+
+class TestIntermixedMessageContentError(TestCase):
+    # case where Intermixed gives different error message
+    # error is raised by 1st parsing step
+    def test_missing_argument_name_in_message(self):
+        parser = ErrorRaisingArgumentParser(prog='PROG', usage='')
+        parser.add_argument('req_pos', type=str)
+        parser.add_argument('-req_opt', type=int, required=True)
+
+        with self.assertRaises(ArgumentParserError) as cm:
+            parser.parse_args([])
+        msg = str(cm.exception)
+        self.assertRegex(msg, 'req_pos')
+        self.assertRegex(msg, 'req_opt')
+
+        with self.assertRaises(ArgumentParserError) as cm:
+            parser.parse_intermixed_args([])
+        msg = str(cm.exception)
+        self.assertNotRegex(msg, 'req_pos')
+        self.assertRegex(msg, 'req_opt')
+
 # ==========================
 # add_argument metavar tests
 # ==========================

Misc/NEWS.d/next/Library/2017-09-05-17-43-00.bpo-14191.vhh2xx.rst

@@ -0,0 +1,3 @@
+A new function ``argparse.ArgumentParser.parse_intermixed_args`` provides the
+ability to parse command lines where there user intermixes options and
+positional arguments.