Merge pull request #56360 from Ch3LL/mac_virtual

Use virtualenv 20.0.10 for macosx tests

requirements/static/darwin.in

@@ -25,7 +25,7 @@ rfc3987
 salttesting==2017.6.1
 strict_rfc3339
 supervisor==3.3.5; python_version < '3'
-virtualenv
+virtualenv==20.0.10
 watchdog
 yamlordereddictloader
 vcert~=0.7.0

requirements/static/py2.7/darwin.txt

@@ -5,6 +5,7 @@
 #    pip-compile -o requirements/static/py2.7/darwin.txt -v pkg/osx/req.txt pkg/osx/req_ext.txt requirements/base.txt requirements/zeromq.txt requirements/pytest.txt requirements/static/darwin.in
 #
 apache-libcloud==2.4.0
+appdirs==1.4.3            # via virtualenv
 argh==0.26.2              # via watchdog
 asn1crypto==1.3.0         # via certvalidator, cryptography, oscrypto
 atomicwrites==1.3.0       # via pytest
@@ -30,16 +31,18 @@ cherrypy==17.4.1
 click==7.0
 clustershell==1.8.1
 configparser==4.0.2       # via importlib-metadata
-contextlib2==0.5.5        # via cherrypy, importlib-metadata
+contextlib2==0.6.0.post1  # via cherrypy, importlib-metadata, importlib-resources, virtualenv
 cookies==2.2.1            # via responses
 croniter==0.3.29
 cryptography==2.6.1
+distlib==0.3.0            # via virtualenv
 dnspython==1.16.0
 docker-pycreds==0.4.0     # via docker
 docker==3.7.2
 docutils==0.14            # via botocore
 ecdsa==0.13.3             # via python-jose
 enum34==1.1.6
+filelock==3.0.12          # via virtualenv
 funcsigs==1.0.2           # via mock, pytest
 functools32==3.2.3.post2  # via jsonschema
 future==0.17.1            # via python-jose
@@ -50,7 +53,8 @@ gitdb==0.6.4
 gitpython==2.1.11
 google-auth==1.6.3        # via kubernetes
 idna==2.8
-importlib-metadata==0.23  # via pluggy, pytest
+importlib-metadata==0.23  # via importlib-resources, pluggy, pytest, virtualenv
+importlib-resources==1.3.1  # via virtualenv
 ipaddress==1.0.22
 jaraco.functools==2.0     # via tempora
 jinja2==2.10.1
@@ -77,7 +81,7 @@ netaddr==0.7.19           # via junos-eznc
 oscrypto==1.2.0           # via certvalidator
 packaging==19.2           # via pytest
 paramiko==2.4.2           # via junos-eznc, ncclient, scp
-pathlib2==2.3.3           # via importlib-metadata, pytest
+pathlib2==2.3.3           # via importlib-metadata, importlib-resources, pytest, virtualenv
 pathtools==0.1.2          # via watchdog
 pluggy==0.13.1            # via pytest
 portend==2.4              # via cherrypy
@@ -116,16 +120,17 @@ scp==0.13.2               # via junos-eznc
 selectors2==2.0.1         # via ncclient
 setproctitle==1.1.10
 singledispatch==3.4.0.3 ; python_version < "3.4"
-six==1.12.0               # via bcrypt, cheroot, cherrypy, cryptography, docker, docker-pycreds, google-auth, junos-eznc, kubernetes, mock, more-itertools, moto, ncclient, packaging, pathlib2, pynacl, pyopenssl, pytest, python-dateutil, python-jose, pyvmomi, responses, salttesting, singledispatch, tempora, vcert, websocket-client
+six==1.12.0               # via bcrypt, cheroot, cherrypy, cryptography, docker, docker-pycreds, google-auth, junos-eznc, kubernetes, mock, more-itertools, moto, ncclient, packaging, pathlib2, pynacl, pyopenssl, pytest, python-dateutil, python-jose, pyvmomi, responses, salttesting, singledispatch, tempora, vcert, virtualenv, websocket-client
 smmap2==2.0.5             # via gitdb2
 smmap==0.9.0
 strict-rfc3339==0.7
 supervisor==3.3.5 ; python_version < "3"
 tempora==1.14.1           # via portend
 timelib==0.2.4
+typing==3.7.4.1           # via importlib-resources
 urllib3==1.24.2           # via botocore, kubernetes, python-etcd, requests
 vcert==0.7.3
-virtualenv==16.4.3
+virtualenv==20.0.10
 vultr==1.0.1
 watchdog==0.9.0
 wcwidth==0.1.7            # via pytest
@@ -136,6 +141,6 @@ wrapt==1.11.1             # via aws-xray-sdk
 xmltodict==0.12.0         # via moto
 yamlordereddictloader==0.4.0
 zc.lockfile==1.4          # via cherrypy
-zipp==0.6.0               # via importlib-metadata
+zipp==0.6.0               # via importlib-metadata, importlib-resources
 # Passthrough dependencies from pkg/osx/req.txt
 pyobjc==5.1.2

requirements/static/py3.5/darwin.txt

@@ -5,6 +5,7 @@
 #    pip-compile -o requirements/static/py3.5/darwin.txt -v pkg/osx/req.txt pkg/osx/req_ext.txt requirements/base.txt requirements/zeromq.txt requirements/pytest.txt requirements/static/darwin.in
 #
 apache-libcloud==2.4.0
+appdirs==1.4.3            # via virtualenv
 argh==0.26.2              # via watchdog
 asn1crypto==1.3.0         # via certvalidator, cryptography, oscrypto
 atomicwrites==1.3.0       # via pytest
@@ -29,20 +30,23 @@ clustershell==1.8.1
 contextlib2==0.5.5        # via cherrypy
 croniter==0.3.29
 cryptography==2.6.1
+distlib==0.3.0            # via virtualenv
 dnspython==1.16.0
 docker-pycreds==0.4.0     # via docker
 docker==3.7.2
 docutils==0.14            # via botocore
 ecdsa==0.13.3             # via python-jose
 enum34==1.1.6
+filelock==3.0.12          # via virtualenv
 future==0.17.1            # via python-jose
 genshi==0.7.3
 gitdb2==2.0.5             # via gitpython
 gitdb==0.6.4
 gitpython==2.1.11
 google-auth==1.6.3        # via kubernetes
 idna==2.8
-importlib-metadata==0.23  # via pluggy, pytest
+importlib-metadata==0.23  # via importlib-resources, pluggy, pytest, virtualenv
+importlib-resources==1.3.1  # via virtualenv
 ipaddress==1.0.22
 jaraco.functools==2.0     # via tempora
 jinja2==2.10.1
@@ -104,15 +108,15 @@ s3transfer==0.2.0         # via boto3
 salttesting==2017.6.1
 scp==0.13.2               # via junos-eznc
 setproctitle==1.1.10
-six==1.12.0               # via bcrypt, cheroot, cherrypy, cryptography, docker, docker-pycreds, google-auth, junos-eznc, kubernetes, mock, more-itertools, moto, ncclient, packaging, pathlib2, pynacl, pyopenssl, pytest, python-dateutil, python-jose, pyvmomi, responses, salttesting, tempora, vcert, websocket-client
+six==1.12.0               # via bcrypt, cheroot, cherrypy, cryptography, docker, docker-pycreds, google-auth, junos-eznc, kubernetes, mock, more-itertools, moto, ncclient, packaging, pathlib2, pynacl, pyopenssl, pytest, python-dateutil, python-jose, pyvmomi, responses, salttesting, tempora, vcert, virtualenv, websocket-client
 smmap2==2.0.5             # via gitdb2
 smmap==0.9.0
 strict-rfc3339==0.7
 tempora==1.14.1           # via portend
 timelib==0.2.4
 urllib3==1.24.2           # via botocore, kubernetes, python-etcd, requests
 vcert==0.7.3
-virtualenv==16.4.3
+virtualenv==20.0.10
 vultr==1.0.1
 watchdog==0.9.0
 wcwidth==0.1.7            # via pytest
@@ -122,6 +126,6 @@ wrapt==1.11.1             # via aws-xray-sdk
 xmltodict==0.12.0         # via moto
 yamlordereddictloader==0.4.0
 zc.lockfile==1.4          # via cherrypy
-zipp==0.6.0               # via importlib-metadata
+zipp==0.6.0               # via importlib-metadata, importlib-resources
 # Passthrough dependencies from pkg/osx/req.txt
 pyobjc==5.1.2

requirements/static/py3.6/darwin.txt

@@ -5,6 +5,7 @@
 #    pip-compile -o requirements/static/py3.6/darwin.txt -v pkg/osx/req.txt pkg/osx/req_ext.txt requirements/base.txt requirements/zeromq.txt requirements/pytest.txt requirements/static/darwin.in
 #
 apache-libcloud==2.4.0
+appdirs==1.4.3            # via virtualenv
 argh==0.26.2              # via watchdog
 asn1crypto==1.3.0         # via certvalidator, cryptography, oscrypto
 atomicwrites==1.3.0       # via pytest
@@ -29,20 +30,23 @@ clustershell==1.8.1
 contextlib2==0.5.5        # via cherrypy
 croniter==0.3.29
 cryptography==2.6.1
+distlib==0.3.0            # via virtualenv
 dnspython==1.16.0
 docker-pycreds==0.4.0     # via docker
 docker==3.7.2
 docutils==0.14            # via botocore
 ecdsa==0.13.3             # via python-jose
 enum34==1.1.6
+filelock==3.0.12          # via virtualenv
 future==0.17.1            # via python-jose
 genshi==0.7.3
 gitdb2==2.0.5             # via gitpython
 gitdb==0.6.4
 gitpython==2.1.11
 google-auth==1.6.3        # via kubernetes
 idna==2.8
-importlib-metadata==0.23  # via pluggy, pytest
+importlib-metadata==0.23  # via importlib-resources, pluggy, pytest, virtualenv
+importlib-resources==1.3.1  # via virtualenv
 ipaddress==1.0.22
 jaraco.functools==2.0     # via tempora
 jinja2==2.10.1
@@ -103,15 +107,15 @@ s3transfer==0.2.0         # via boto3
 salttesting==2017.6.1
 scp==0.13.2               # via junos-eznc
 setproctitle==1.1.10
-six==1.12.0               # via bcrypt, cheroot, cherrypy, cryptography, docker, docker-pycreds, google-auth, junos-eznc, kubernetes, mock, more-itertools, moto, ncclient, packaging, pynacl, pyopenssl, pytest, python-dateutil, python-jose, pyvmomi, responses, salttesting, tempora, vcert, websocket-client
+six==1.12.0               # via bcrypt, cheroot, cherrypy, cryptography, docker, docker-pycreds, google-auth, junos-eznc, kubernetes, mock, more-itertools, moto, ncclient, packaging, pynacl, pyopenssl, pytest, python-dateutil, python-jose, pyvmomi, responses, salttesting, tempora, vcert, virtualenv, websocket-client
 smmap2==2.0.5             # via gitdb2
 smmap==0.9.0
 strict-rfc3339==0.7
 tempora==1.14.1           # via portend
 timelib==0.2.4
 urllib3==1.24.2           # via botocore, kubernetes, python-etcd, requests
 vcert==0.7.3
-virtualenv==16.4.3
+virtualenv==20.0.10
 vultr==1.0.1
 watchdog==0.9.0
 wcwidth==0.1.7            # via pytest
@@ -121,6 +125,6 @@ wrapt==1.11.1             # via aws-xray-sdk
 xmltodict==0.12.0         # via moto
 yamlordereddictloader==0.4.0
 zc.lockfile==1.4          # via cherrypy
-zipp==0.6.0               # via importlib-metadata
+zipp==0.6.0               # via importlib-metadata, importlib-resources
 # Passthrough dependencies from pkg/osx/req.txt
 pyobjc==5.1.2

requirements/static/py3.7/darwin.txt

@@ -5,6 +5,7 @@
 #    pip-compile -o requirements/static/py3.7/darwin.txt -v pkg/osx/req.txt pkg/osx/req_ext.txt requirements/base.txt requirements/zeromq.txt requirements/pytest.txt requirements/static/darwin.in
 #
 apache-libcloud==2.4.0
+appdirs==1.4.3            # via virtualenv
 argh==0.26.2              # via watchdog
 asn1crypto==1.3.0         # via certvalidator, cryptography, oscrypto
 atomicwrites==1.3.0       # via pytest
@@ -29,20 +30,22 @@ clustershell==1.8.1
 contextlib2==0.5.5        # via cherrypy
 croniter==0.3.29
 cryptography==2.6.1
+distlib==0.3.0            # via virtualenv
 dnspython==1.16.0
 docker-pycreds==0.4.0     # via docker
 docker==3.7.2
 docutils==0.14            # via botocore
 ecdsa==0.13.3             # via python-jose
 enum34==1.1.6
+filelock==3.0.12          # via virtualenv
 future==0.17.1            # via python-jose
 genshi==0.7.3
 gitdb2==2.0.5             # via gitpython
 gitdb==0.6.4
 gitpython==2.1.11
 google-auth==1.6.3        # via kubernetes
 idna==2.8
-importlib-metadata==0.23  # via pluggy, pytest
+importlib-metadata==0.23  # via pluggy, pytest, virtualenv
 ipaddress==1.0.22
 jaraco.functools==2.0     # via tempora
 jinja2==2.10.1
@@ -103,15 +106,15 @@ s3transfer==0.2.0         # via boto3
 salttesting==2017.6.1
 scp==0.13.2               # via junos-eznc
 setproctitle==1.1.10
-six==1.12.0               # via bcrypt, cheroot, cherrypy, cryptography, docker, docker-pycreds, google-auth, junos-eznc, kubernetes, mock, more-itertools, moto, ncclient, packaging, pynacl, pyopenssl, pytest, python-dateutil, python-jose, pyvmomi, responses, salttesting, tempora, vcert, websocket-client
+six==1.12.0               # via bcrypt, cheroot, cherrypy, cryptography, docker, docker-pycreds, google-auth, junos-eznc, kubernetes, mock, more-itertools, moto, ncclient, packaging, pynacl, pyopenssl, pytest, python-dateutil, python-jose, pyvmomi, responses, salttesting, tempora, vcert, virtualenv, websocket-client
 smmap2==2.0.5             # via gitdb2
 smmap==0.9.0
 strict-rfc3339==0.7
 tempora==1.14.1           # via portend
 timelib==0.2.4
 urllib3==1.24.2           # via botocore, kubernetes, python-etcd, requests
 vcert==0.7.3
-virtualenv==16.4.3
+virtualenv==20.0.10
 vultr==1.0.1
 watchdog==0.9.0
 wcwidth==0.1.7            # via pytest

salt/modules/virtualenv_mod.py

@@ -43,6 +43,37 @@ def __virtual__():
     return __virtualname__
 
 
+def virtualenv_ver(venv_bin, user=None, **kwargs):
+    '''
+    return virtualenv version if exists
+    '''
+    # Virtualenv package
+    try:
+        import virtualenv
+        version = getattr(virtualenv, '__version__', None)
+        if not version:
+            version = virtualenv.virtualenv_version
+        virtualenv_version_info = tuple(
+            [int(i) for i in version.split('rc')[0].split('.')]
+        )
+    except ImportError:
+        # Unable to import?? Let's parse the version from the console
+        version_cmd = [venv_bin, '--version']
+        ret = __salt__['cmd.run_all'](
+                version_cmd, runas=user, python_shell=False, **kwargs
+            )
+        if ret['retcode'] > 0 or not ret['stdout'].strip():
+            raise CommandExecutionError(
+                'Unable to get the virtualenv version output using \'{0}\'. '
+                'Returned data: {1}'.format(version_cmd, ret)
+            )
+        virtualenv_version_info = tuple(
+            [int(i) for i in
+             ret['stdout'].strip().split('rc')[0].split('.')]
+        )
+    return virtualenv_version_info
+
+
 def create(path,
            venv_bin=None,
            system_site_packages=False,
@@ -164,29 +195,7 @@ def create(path,
             )
         # <---- Stop the user if pyvenv only options are used ----------------
 
-        # Virtualenv package
-        try:
-            import virtualenv
-            version = getattr(virtualenv, '__version__',
-                              virtualenv.virtualenv_version)
-            virtualenv_version_info = tuple(
-                [int(i) for i in version.split('rc')[0].split('.')]
-            )
-        except ImportError:
-            # Unable to import?? Let's parse the version from the console
-            version_cmd = [venv_bin, '--version']
-            ret = __salt__['cmd.run_all'](
-                    version_cmd, runas=user, python_shell=False, **kwargs
-                )
-            if ret['retcode'] > 0 or not ret['stdout'].strip():
-                raise CommandExecutionError(
-                    'Unable to get the virtualenv version output using \'{0}\'. '
-                    'Returned data: {1}'.format(version_cmd, ret)
-                )
-            virtualenv_version_info = tuple(
-                [int(i) for i in
-                 ret['stdout'].strip().split('rc')[0].split('.')]
-            )
+        virtualenv_version_info = virtualenv_ver(venv_bin, user=user, **kwargs)
 
         if distribute:
             if virtualenv_version_info >= (1, 10):

tests/integration/states/test_pip_state.py

@@ -405,8 +405,12 @@ def test_issue_6833_pip_upgrade_pip(self):
                 pprint.pformat(ret)
             )
         )
+        import salt.modules.virtualenv_mod
+        msg = 'New python executable'
+        if salt.modules.virtualenv_mod.virtualenv_ver(venv_dir) >= (20, 0, 2):
+            msg = 'created virtual environment'
         self.assertIn(
-            'New python executable',
+            msg,
             ret['stdout'],
             msg='Expected STDOUT did not match. Full return dictionary:\n{}'.format(
                 pprint.pformat(ret)

tests/unit/modules/test_virtualenv_mod.py

@@ -350,3 +350,31 @@ def test_symlinks_argument(self):
                 runas=None,
                 python_shell=False
             )
+
+    def test_virtualenv_ver(self):
+        '''
+        test virtualenv_ver when there is no ImportError
+        '''
+        ret = virtualenv_mod.virtualenv_ver(venv_bin='pyvenv')
+        assert ret == (1, 9, 1)
+
+    def test_virtualenv_ver_importerror(self):
+        '''
+        test virtualenv_ver when there is an ImportError
+        '''
+        with ForceImportErrorOn('virtualenv'):
+            mock_ver = MagicMock(return_value={'retcode': 0, 'stdout': '1.9.1'})
+            with patch.dict(virtualenv_mod.__salt__, {'cmd.run_all': mock_ver}):
+                ret = virtualenv_mod.virtualenv_ver(venv_bin='pyenv')
+        assert ret == (1, 9, 1)
+
+    def test_virtualenv_ver_importerror_cmd_error(self):
+        '''
+        test virtualenv_ver when there is an ImportError
+        and virtualenv --version does not return anything
+        '''
+        with ForceImportErrorOn('virtualenv'):
+            mock_ver = MagicMock(return_value={'retcode': 0, 'stdout': ''})
+            with patch.dict(virtualenv_mod.__salt__, {'cmd.run_all': mock_ver}):
+                with self.assertRaises(CommandExecutionError):
+                    virtualenv_mod.virtualenv_ver(venv_bin='pyenv')

tests/unit/modules/test_zcbuildout.py

@@ -27,6 +27,7 @@
 import salt.utils.path
 import salt.utils.platform
 import salt.modules.zcbuildout as buildout
+import salt.modules.virtualenv_mod
 import salt.modules.cmdmod as cmd
 from salt.ext import six
 
@@ -466,6 +467,9 @@ def test_buildout_bootstrap(self):
 
     @requires_network()
     def test_run_buildout(self):
+        if salt.modules.virtualenv_mod.virtualenv_ver(self.ppy_st) >= (20, 0, 0):
+            self.skipTest("Skiping until upstream resolved https://github.com/pypa/virtualenv/issues/1715")
+
         b_dir = os.path.join(self.tdir, 'b')
         ret = buildout.bootstrap(b_dir, buildout_ver=2, python=self.py_st)
         self.assertTrue(ret['status'])
@@ -477,6 +481,9 @@ def test_run_buildout(self):
 
     @requires_network()
     def test_buildout(self):
+        if salt.modules.virtualenv_mod.virtualenv_ver(self.ppy_st) >= (20, 0, 0):
+            self.skipTest("Skiping until upstream resolved https://github.com/pypa/virtualenv/issues/1715")
+
         b_dir = os.path.join(self.tdir, 'b')
         ret = buildout.buildout(b_dir, buildout_ver=2, python=self.py_st)
         self.assertTrue(ret['status'])