Fix tests failures and errors when detected on VM execution from Salt…

… Shaker (saltstack#636)

* test_chmod: fix test expectation

* test_pkg: Adjust package expectation for SUSE family

* test_docker_network: Skip non-supported operation for SUSE family

* Fix tests failing due wrong docker-py version

* test_version: skip test in packaged scenario when setup.py is missing

* Fix issue related to docker version used during testing

* Fix test errors when setup.py is not available

* test_loader: do not run if setup.py is missing

* test_install: Fix test errors when setup.py is not available

* test_master: use a right service name expected on SUSE family

* test_jinja_filters: prevent test failure when which binary is not available

* Prevent errors when x509 utils cannot be loaded

* test_thin: skip test if virtualenv binary is missing

tests/integration/pillar/test_git_pillar.py

@@ -79,6 +79,7 @@
     PYGIT2_VERSION,
     FileserverConfigError,
 )
+from salt.utils.versions import Version
 from tests.support.gitfs import (  # pylint: disable=unused-import
     PASSWORD,
     USERNAME,
@@ -101,11 +102,20 @@
 except Exception:  # pylint: disable=broad-except
     HAS_PYGIT2 = False
 
+docker = pytest.importorskip("docker")
+
 INSIDE_CONTAINER = os.getenv("HOSTNAME", "") == "salt-test-container"
+
 pytestmark = [
     SKIP_INITIAL_PHOTONOS_FAILURES,
     pytest.mark.skip_on_platforms(windows=True, darwin=True),
-    pytest.mark.skipif(INSIDE_CONTAINER, reason="Communication problems between containers."),
+    pytest.mark.skipif(
+        INSIDE_CONTAINER, reason="Communication problems between containers."
+    ),
+    pytest.mark.skipif(
+        Version(docker.__version__) < Version("4.0.0"),
+        reason="Test does not work in this version of docker-py",
+    ),
 ]
 
 

tests/pytests/functional/cache/test_consul.py

@@ -8,6 +8,7 @@
 
 import salt.cache
 import salt.loader
+from salt.utils.versions import Version
 from tests.pytests.functional.cache.helpers import run_common_cache_tests
 
 docker = pytest.importorskip("docker")
@@ -21,6 +22,10 @@
     pytest.mark.slow_test,
     pytest.mark.skip_if_binaries_missing("dockerd"),
     pytest.mark.skipif(INSIDE_CONTAINER, reason="Cannot run in a container"),
+    pytest.mark.skipif(
+        Version(docker.__version__) < Version("4.0.0"),
+        reason="Test does not work in this version of docker-py",
+    ),
 ]
 
 

tests/pytests/functional/cache/test_mysql.py

@@ -5,6 +5,7 @@
 
 import salt.cache
 import salt.loader
+from salt.utils.versions import Version
 from tests.pytests.functional.cache.helpers import run_common_cache_tests
 from tests.support.pytest.mysql import *  # pylint: disable=wildcard-import,unused-wildcard-import
 
@@ -18,6 +19,10 @@
     pytest.mark.slow_test,
     pytest.mark.skip_if_binaries_missing("dockerd"),
     pytest.mark.skipif(INSIDE_CONTAINER, reason="Cannot run in a container"),
+    pytest.mark.skipif(
+        Version(docker.__version__) < Version("4.0.0"),
+        reason="Test does not work in this version of docker-py",
+    ),
 ]
 
 

tests/pytests/functional/loader/test_loader.py

@@ -1,15 +1,24 @@
 import json
+import os
 
 import pytest
 
 from salt.utils.versions import Version
 from tests.support.helpers import SaltVirtualEnv
 from tests.support.pytest.helpers import FakeSaltExtension
+from tests.support.runtests import RUNTIME_VARS
+
+MISSING_SETUP_PY_FILE = not os.path.exists(
+    os.path.join(RUNTIME_VARS.CODE_DIR, "setup.py")
+)
 
 pytestmark = [
     # These are slow because they create a virtualenv and install salt in it
     pytest.mark.slow_test,
     pytest.mark.timeout_unless_on_windows(240),
+    pytest.mark.skipif(
+        MISSING_SETUP_PY_FILE, reason="This test only work if setup.py is available"
+    ),
 ]
 
 

tests/pytests/functional/modules/state/test_jinja_filters.py

@@ -987,9 +987,9 @@ def _filter_id(value):
         ),
         Filter(
             name="which",
-            expected={"ret": salt.utils.path.which("which")},
+            expected={"ret": salt.utils.path.which("ls")},
             sls="""
-            {% set result = 'which' | which() %}
+            {% set result = 'ls' | which() %}
             test:
               module.run:
                 - name: test.echo

tests/pytests/functional/modules/test_pkg.py

@@ -72,6 +72,8 @@ def pkg_name(grains):
             _pkg = "units"
     elif grains["os_family"] == "Debian":
         _pkg = "ifenslave"
+    elif grains["os_family"] == "Suse":
+        _pkg = "wget"
     return _pkg
 
 

tests/pytests/functional/states/rabbitmq/conftest.py

@@ -5,8 +5,19 @@
 import pytest
 from saltfactories.utils import random_string
 
+from salt.utils.versions import Version
+
 log = logging.getLogger(__name__)
 
+docker = pytest.importorskip("docker")
+
+pytestmark = [
+    pytest.mark.skipif(
+        Version(docker.__version__) < Version("4.0.0"),
+        reason="Test does not work in this version of docker-py",
+    ),
+]
+
 
 @attr.s(kw_only=True, slots=True)
 class RabbitMQImage:

tests/pytests/functional/states/rabbitmq/test_cluster.py

@@ -9,8 +9,9 @@
 
 import salt.modules.rabbitmq as rabbitmq
 import salt.states.rabbitmq_cluster as rabbitmq_cluster
+from salt.utils.versions import Version
 
-pytest.importorskip("docker")
+docker = pytest.importorskip("docker")
 
 log = logging.getLogger(__name__)
 
@@ -22,6 +23,10 @@
         "docker", "dockerd", reason="Docker not installed"
     ),
     pytest.mark.skipif(INSIDE_CONTAINER, reason="Cannot run in a container"),
+    pytest.mark.skipif(
+        Version(docker.__version__) < Version("4.0.0"),
+        reason="Test does not work in this version of docker-py",
+    ),
 ]
 
 

tests/pytests/functional/states/rabbitmq/test_plugin.py

@@ -9,11 +9,13 @@
 
 import salt.modules.rabbitmq as rabbitmq
 import salt.states.rabbitmq_plugin as rabbitmq_plugin
+from salt.utils.versions import Version
 from tests.support.mock import patch
 
 log = logging.getLogger(__name__)
 
-pytest.importorskip("docker")
+docker = pytest.importorskip("docker")
+
 
 INSIDE_CONTAINER = os.getenv("HOSTNAME", "") == "salt-test-container"
 
@@ -23,6 +25,10 @@
         "docker", "dockerd", reason="Docker not installed"
     ),
     pytest.mark.skipif(INSIDE_CONTAINER, reason="Cannot run in a container"),
+    pytest.mark.skipif(
+        Version(docker.__version__) < Version("4.0.0"),
+        reason="Test does not work in this version of docker-py",
+    ),
 ]
 
 

tests/pytests/functional/states/rabbitmq/test_policy.py

@@ -9,11 +9,12 @@
 
 import salt.modules.rabbitmq as rabbitmq
 import salt.states.rabbitmq_policy as rabbitmq_policy
+from salt.utils.versions import Version
 from tests.support.mock import MagicMock, patch
 
 log = logging.getLogger(__name__)
 
-pytest.importorskip("docker")
+docker = pytest.importorskip("docker")
 
 INSIDE_CONTAINER = os.getenv("HOSTNAME", "") == "salt-test-container"
 
@@ -23,6 +24,10 @@
         "docker", "dockerd", reason="Docker not installed"
     ),
     pytest.mark.skipif(INSIDE_CONTAINER, reason="Cannot run in a container"),
+    pytest.mark.skipif(
+        Version(docker.__version__) < Version("4.0.0"),
+        reason="Test does not work in this version of docker-py",
+    ),
 ]
 
 

tests/pytests/functional/states/rabbitmq/test_upstream.py

@@ -9,10 +9,11 @@
 
 import salt.modules.rabbitmq as rabbitmq
 import salt.states.rabbitmq_upstream as rabbitmq_upstream
+from salt.utils.versions import Version
 
 log = logging.getLogger(__name__)
 
-pytest.importorskip("docker")
+docker = pytest.importorskip("docker")
 
 INSIDE_CONTAINER = os.getenv("HOSTNAME", "") == "salt-test-container"
 
@@ -22,6 +23,10 @@
         "docker", "dockerd", reason="Docker not installed"
     ),
     pytest.mark.skipif(INSIDE_CONTAINER, reason="Cannot run in a container"),
+    pytest.mark.skipif(
+        Version(docker.__version__) < Version("4.0.0"),
+        reason="Test does not work in this version of docker-py",
+    ),
 ]
 
 

tests/pytests/functional/states/rabbitmq/test_user.py

@@ -9,10 +9,11 @@
 
 import salt.modules.rabbitmq as rabbitmq
 import salt.states.rabbitmq_user as rabbitmq_user
+from salt.utils.versions import Version
 
 log = logging.getLogger(__name__)
 
-pytest.importorskip("docker")
+docker = pytest.importorskip("docker")
 
 INSIDE_CONTAINER = os.getenv("HOSTNAME", "") == "salt-test-container"
 
@@ -22,6 +23,10 @@
         "docker", "dockerd", reason="Docker not installed"
     ),
     pytest.mark.skipif(INSIDE_CONTAINER, reason="Cannot run in a container"),
+    pytest.mark.skipif(
+        Version(docker.__version__) < Version("4.0.0"),
+        reason="Test does not work in this version of docker-py",
+    ),
 ]
 
 

tests/pytests/functional/states/rabbitmq/test_vhost.py

@@ -9,10 +9,11 @@
 
 import salt.modules.rabbitmq as rabbitmq
 import salt.states.rabbitmq_vhost as rabbitmq_vhost
+from salt.utils.versions import Version
 
 log = logging.getLogger(__name__)
 
-pytest.importorskip("docker")
+docker = pytest.importorskip("docker")
 
 INSIDE_CONTAINER = os.getenv("HOSTNAME", "") == "salt-test-container"
 
@@ -22,6 +23,10 @@
         "docker", "dockerd", reason="Docker not installed"
     ),
     pytest.mark.skipif(INSIDE_CONTAINER, reason="Cannot run in a container"),
+    pytest.mark.skipif(
+        Version(docker.__version__) < Version("4.0.0"),
+        reason="Test does not work in this version of docker-py",
+    ),
 ]
 
 

tests/pytests/functional/states/test_pkg.py

@@ -73,7 +73,7 @@ def PKG_TARGETS(grains):
         else:
             _PKG_TARGETS = ["units", "zsh-html"]
     elif grains["os_family"] == "Suse":
-        _PKG_TARGETS = ["lynx", "htop"]
+        _PKG_TARGETS = ["iotop", "screen"]
     return _PKG_TARGETS
 
 

tests/pytests/functional/test_version.py

@@ -1,15 +1,24 @@
 import json
 import logging
+import os
 
 import pytest
 
 from tests.support.helpers import SaltVirtualEnv
 from tests.support.pytest.helpers import FakeSaltExtension
+from tests.support.runtests import RUNTIME_VARS
+
+MISSING_SETUP_PY_FILE = not os.path.exists(
+    os.path.join(RUNTIME_VARS.CODE_DIR, "setup.py")
+)
 
 pytestmark = [
     # These are slow because they create a virtualenv and install salt in it
     pytest.mark.slow_test,
     pytest.mark.timeout_unless_on_windows(240),
+    pytest.mark.skipif(
+        MISSING_SETUP_PY_FILE, reason="This test only work if setup.py is available"
+    ),
 ]
 
 log = logging.getLogger(__name__)

tests/pytests/integration/modules/test_x509_v2.py

@@ -11,8 +11,8 @@
 import pytest
 from saltfactories.utils import random_string
 
-import salt.utils.x509 as x509util
 from tests.conftest import FIPS_TESTRUN
+x509util = pytest.importorskip("salt.utils.x509")
 
 try:
     import cryptography

tests/pytests/integration/ssh/test_log.py

@@ -9,9 +9,10 @@
 import pytest
 from saltfactories.utils import random_string
 
+from salt.utils.versions import Version
 from tests.support.helpers import Keys
 
-pytest.importorskip("docker")
+docker = pytest.importorskip("docker")
 
 INSIDE_CONTAINER = os.getenv("HOSTNAME", "") == "salt-test-container"
 
@@ -21,6 +22,10 @@
     pytest.mark.slow_test,
     pytest.mark.skip_if_binaries_missing("dockerd"),
     pytest.mark.skipif(INSIDE_CONTAINER, reason="Cannot run in a container"),
+    pytest.mark.skipif(
+        Version(docker.__version__) < Version("4.0.0"),
+        reason="Test does not work in this version of docker-py",
+    ),
 ]
 
 

tests/pytests/integration/ssh/test_master.py

@@ -23,7 +23,7 @@ def test_service(salt_ssh_cli, grains):
     os_release = grains["osrelease"]
     if os_family == "RedHat":
         service = "crond"
-    elif os_family == "Arch":
+    elif os_family in ["Suse", "Arch"]:
         service = "sshd"
     elif os_family == "MacOS":
         service = "org.ntp.ntpd"

tests/pytests/integration/ssh/test_py_versions.py

@@ -10,9 +10,10 @@
 import pytest
 from saltfactories.utils import random_string
 
+from salt.utils.versions import Version
 from tests.support.helpers import Keys
 
-pytest.importorskip("docker")
+docker = pytest.importorskip("docker")
 
 INSIDE_CONTAINER = os.getenv("HOSTNAME", "") == "salt-test-container"
 
@@ -22,6 +23,10 @@
     pytest.mark.slow_test,
     pytest.mark.skip_if_binaries_missing("dockerd"),
     pytest.mark.skipif(INSIDE_CONTAINER, reason="Cannot run in a container"),
+    pytest.mark.skipif(
+        Version(docker.__version__) < Version("4.0.0"),
+        reason="Test does not work in this version of docker-py",
+    ),
 ]