Fix tests failures and errors when detected on VM execution from Salt…

… Shaker (saltstack#636)

* test_chmod: fix test expectation

* test_pkg: Adjust package expectation for SUSE family

* test_docker_network: Skip non-supported operation for SUSE family

* Fix tests failing due wrong docker-py version

* test_version: skip test in packaged scenario when setup.py is missing

* Fix issue related to docker version used during testing

* Fix test errors when setup.py is not available

* test_loader: do not run if setup.py is missing

* test_install: Fix test errors when setup.py is not available

* test_master: use a right service name expected on SUSE family

* test_jinja_filters: prevent test failure when which binary is not available

* Prevent errors when x509 utils cannot be loaded

* test_thin: skip test if virtualenv binary is missing

tests/integration/pillar/test_git_pillar.py

@@ -79,6 +79,7 @@
     PYGIT2_VERSION,
     FileserverConfigError,
 )
+from salt.utils.versions import Version
 from tests.support.gitfs import (  # pylint: disable=unused-import
     PASSWORD,
     USERNAME,
@@ -101,11 +102,20 @@
 except Exception:  # pylint: disable=broad-except
     HAS_PYGIT2 = False
 
+docker = pytest.importorskip("docker")
+
 INSIDE_CONTAINER = os.getenv("HOSTNAME", "") == "salt-test-container"
+
 pytestmark = [
     SKIP_INITIAL_PHOTONOS_FAILURES,
     pytest.mark.skip_on_platforms(windows=True, darwin=True),
-    pytest.mark.skipif(INSIDE_CONTAINER, reason="Communication problems between containers."),
+    pytest.mark.skipif(
+        INSIDE_CONTAINER, reason="Communication problems between containers."
+    ),
+    pytest.mark.skipif(
+        Version(docker.__version__) < Version("4.0.0"),
+        reason="Test does not work in this version of docker-py",
+    ),
 ]
 
 

tests/pytests/functional/cache/test_consul.py

@@ -8,6 +8,7 @@
 
 import salt.cache
 import salt.loader
+from salt.utils.versions import Version
 from tests.pytests.functional.cache.helpers import run_common_cache_tests
 
 docker = pytest.importorskip("docker")
@@ -20,6 +21,10 @@
     pytest.mark.slow_test,
     pytest.mark.skip_if_binaries_missing("dockerd"),
     pytest.mark.skipif(INSIDE_CONTAINER, reason="Cannot run in a container"),
+    pytest.mark.skipif(
+        Version(docker.__version__) < Version("4.0.0"),
+        reason="Test does not work in this version of docker-py",
+    ),
 ]
 
 

tests/pytests/functional/cache/test_mysql.py

@@ -5,6 +5,7 @@
 
 import salt.cache
 import salt.loader
+from salt.utils.versions import Version
 from tests.pytests.functional.cache.helpers import run_common_cache_tests
 from tests.support.pytest.mysql import *  # pylint: disable=wildcard-import,unused-wildcard-import
 
@@ -18,6 +19,10 @@
     pytest.mark.slow_test,
     pytest.mark.skip_if_binaries_missing("dockerd"),
     pytest.mark.skipif(INSIDE_CONTAINER, reason="Cannot run in a container"),
+    pytest.mark.skipif(
+        Version(docker.__version__) < Version("4.0.0"),
+        reason="Test does not work in this version of docker-py",
+    ),
 ]
 
 

tests/pytests/functional/loader/test_loader.py

@@ -1,14 +1,23 @@
 import json
+import os
 
 import pytest
 
 from salt.utils.versions import Version
 from tests.support.helpers import SaltVirtualEnv
 from tests.support.pytest.helpers import FakeSaltExtension
+from tests.support.runtests import RUNTIME_VARS
+
+MISSING_SETUP_PY_FILE = not os.path.exists(
+    os.path.join(RUNTIME_VARS.CODE_DIR, "setup.py")
+)
 
 pytestmark = [
     # These are slow because they create a virtualenv and install salt in it
     pytest.mark.slow_test,
+    pytest.mark.skipif(
+        MISSING_SETUP_PY_FILE, reason="This test only work if setup.py is available"
+    ),
 ]
 
 

tests/pytests/functional/modules/state/test_jinja_filters.py

@@ -798,9 +798,9 @@ def _filter_id(value):
         ),
         Filter(
             name="which",
-            expected={"ret": salt.utils.path.which("which")},
+            expected={"ret": salt.utils.path.which("ls")},
             sls="""
-            {% set result = 'which' | which() %}
+            {% set result = 'ls' | which() %}
             test:
               module.run:
                 - name: test.echo

tests/pytests/functional/modules/test_cmdmod.py

@@ -105,7 +105,7 @@ def test_run(cmdmod):
             template="jinja",
             python_shell=True,
         )
-        == "func-tests-minion"
+        == "func-tests-minion-opts"
     )
     assert cmdmod.run("grep f", stdin="one\ntwo\nthree\nfour\nfive\n") == "four\nfive"
     assert cmdmod.run('echo "a=b" | sed -e s/=/:/g', python_shell=True) == "a:b"

tests/pytests/functional/modules/test_dockermod.py

@@ -8,7 +8,9 @@
 from saltfactories.utils import random_string
 from saltfactories.utils.functional import StateResult
 
-pytest.importorskip("docker")
+from salt.utils.versions import Version
+
+docker = pytest.importorskip("docker")
 
 log = logging.getLogger(__name__)
 
@@ -18,6 +20,10 @@
     pytest.mark.slow_test,
     pytest.mark.skip_if_binaries_missing("docker", "dockerd", check_all=False),
     pytest.mark.skipif(INSIDE_CONTAINER, reason="Cannot run inside a container"),
+    pytest.mark.skipif(
+        Version(docker.__version__) < Version("4.0.0"),
+        reason="Test does not work in this version of docker-py",
+    ),
 ]
 
 

tests/pytests/functional/modules/test_pkg.py

@@ -67,6 +67,8 @@ def test_pkg(grains):
             _pkg = "units"
     elif grains["os_family"] == "Debian":
         _pkg = "ifenslave"
+    elif grains["os_family"] == "Suse":
+        _pkg = "wget"
     return _pkg
 
 

tests/pytests/functional/modules/test_swarm.py

@@ -20,7 +20,11 @@
 def docker_version(shell, grains):
     ret = shell.run("docker", "--version")
     assert ret.returncode == 0
-    return salt.utils.versions.Version(ret.stdout.split(",")[0].split()[-1].strip())
+    # Example output:
+    # Docker version 24.0.7-ce, build 311b9ff0aa93
+    return salt.utils.versions.Version(
+        ret.stdout.split(",")[0].split()[-1].split("-")[0].strip()
+    )
 
 
 @pytest.fixture

tests/pytests/functional/states/rabbitmq/conftest.py

@@ -5,8 +5,19 @@
 import pytest
 from saltfactories.utils import random_string
 
+from salt.utils.versions import Version
+
 log = logging.getLogger(__name__)
 
+docker = pytest.importorskip("docker")
+
+pytestmark = [
+    pytest.mark.skipif(
+        Version(docker.__version__) < Version("4.0.0"),
+        reason="Test does not work in this version of docker-py",
+    ),
+]
+
 
 @attr.s(kw_only=True, slots=True)
 class RabbitMQImage:

tests/pytests/functional/states/rabbitmq/test_cluster.py

@@ -9,8 +9,9 @@
 
 import salt.modules.rabbitmq as rabbitmq
 import salt.states.rabbitmq_cluster as rabbitmq_cluster
+from salt.utils.versions import Version
 
-pytest.importorskip("docker")
+docker = pytest.importorskip("docker")
 
 log = logging.getLogger(__name__)
 
@@ -22,6 +23,10 @@
         "docker", "dockerd", reason="Docker not installed"
     ),
     pytest.mark.skipif(INSIDE_CONTAINER, reason="Cannot run in a container"),
+    pytest.mark.skipif(
+        Version(docker.__version__) < Version("4.0.0"),
+        reason="Test does not work in this version of docker-py",
+    ),
 ]
 
 

tests/pytests/functional/states/rabbitmq/test_plugin.py

@@ -9,11 +9,13 @@
 
 import salt.modules.rabbitmq as rabbitmq
 import salt.states.rabbitmq_plugin as rabbitmq_plugin
+from salt.utils.versions import Version
 from tests.support.mock import patch
 
 log = logging.getLogger(__name__)
 
-pytest.importorskip("docker")
+docker = pytest.importorskip("docker")
+
 
 INSIDE_CONTAINER = os.getenv("HOSTNAME", "") == "salt-test-container"
 
@@ -23,6 +25,10 @@
         "docker", "dockerd", reason="Docker not installed"
     ),
     pytest.mark.skipif(INSIDE_CONTAINER, reason="Cannot run in a container"),
+    pytest.mark.skipif(
+        Version(docker.__version__) < Version("4.0.0"),
+        reason="Test does not work in this version of docker-py",
+    ),
 ]
 
 

tests/pytests/functional/states/rabbitmq/test_policy.py

@@ -9,11 +9,12 @@
 
 import salt.modules.rabbitmq as rabbitmq
 import salt.states.rabbitmq_policy as rabbitmq_policy
+from salt.utils.versions import Version
 from tests.support.mock import MagicMock, patch
 
 log = logging.getLogger(__name__)
 
-pytest.importorskip("docker")
+docker = pytest.importorskip("docker")
 
 INSIDE_CONTAINER = os.getenv("HOSTNAME", "") == "salt-test-container"
 
@@ -23,6 +24,10 @@
         "docker", "dockerd", reason="Docker not installed"
     ),
     pytest.mark.skipif(INSIDE_CONTAINER, reason="Cannot run in a container"),
+    pytest.mark.skipif(
+        Version(docker.__version__) < Version("4.0.0"),
+        reason="Test does not work in this version of docker-py",
+    ),
 ]
 
 

tests/pytests/functional/states/rabbitmq/test_upstream.py

@@ -9,10 +9,11 @@
 
 import salt.modules.rabbitmq as rabbitmq
 import salt.states.rabbitmq_upstream as rabbitmq_upstream
+from salt.utils.versions import Version
 
 log = logging.getLogger(__name__)
 
-pytest.importorskip("docker")
+docker = pytest.importorskip("docker")
 
 INSIDE_CONTAINER = os.getenv("HOSTNAME", "") == "salt-test-container"
 
@@ -22,6 +23,10 @@
         "docker", "dockerd", reason="Docker not installed"
     ),
     pytest.mark.skipif(INSIDE_CONTAINER, reason="Cannot run in a container"),
+    pytest.mark.skipif(
+        Version(docker.__version__) < Version("4.0.0"),
+        reason="Test does not work in this version of docker-py",
+    ),
 ]
 
 

tests/pytests/functional/states/rabbitmq/test_user.py

@@ -9,10 +9,11 @@
 
 import salt.modules.rabbitmq as rabbitmq
 import salt.states.rabbitmq_user as rabbitmq_user
+from salt.utils.versions import Version
 
 log = logging.getLogger(__name__)
 
-pytest.importorskip("docker")
+docker = pytest.importorskip("docker")
 
 INSIDE_CONTAINER = os.getenv("HOSTNAME", "") == "salt-test-container"
 
@@ -22,6 +23,10 @@
         "docker", "dockerd", reason="Docker not installed"
     ),
     pytest.mark.skipif(INSIDE_CONTAINER, reason="Cannot run in a container"),
+    pytest.mark.skipif(
+        Version(docker.__version__) < Version("4.0.0"),
+        reason="Test does not work in this version of docker-py",
+    ),
 ]
 
 

tests/pytests/functional/states/rabbitmq/test_vhost.py

@@ -9,10 +9,11 @@
 
 import salt.modules.rabbitmq as rabbitmq
 import salt.states.rabbitmq_vhost as rabbitmq_vhost
+from salt.utils.versions import Version
 
 log = logging.getLogger(__name__)
 
-pytest.importorskip("docker")
+docker = pytest.importorskip("docker")
 
 INSIDE_CONTAINER = os.getenv("HOSTNAME", "") == "salt-test-container"
 
@@ -22,6 +23,10 @@
         "docker", "dockerd", reason="Docker not installed"
     ),
     pytest.mark.skipif(INSIDE_CONTAINER, reason="Cannot run in a container"),
+    pytest.mark.skipif(
+        Version(docker.__version__) < Version("4.0.0"),
+        reason="Test does not work in this version of docker-py",
+    ),
 ]
 
 

tests/pytests/functional/states/test_docker_network.py

@@ -220,10 +220,15 @@ def test_present_with_containers(network, docker, docker_network, container):
 
 
 @pytest.mark.parametrize("reconnect", [True, False])
-def test_present_with_reconnect(network, docker, docker_network, container, reconnect):
+def test_present_with_reconnect(
+    network, docker, docker_network, container, reconnect, grains
+):
     """
     Test reconnecting with containers not passed to state
     """
+    if grains["os_family"] == "Suse":
+        pytest.skip("This test is failing for SUSE family")
+
     with network() as net:
         ret = docker_network.present(name=net.name, driver="bridge")
         assert ret.result is True

tests/pytests/functional/states/test_pkg.py

@@ -55,7 +55,7 @@ def PKG_TARGETS(grains):
         else:
             _PKG_TARGETS = ["units", "zsh-html"]
     elif grains["os_family"] == "Suse":
-        _PKG_TARGETS = ["lynx", "htop"]
+        _PKG_TARGETS = ["iotop", "screen"]
     return _PKG_TARGETS
 
 

tests/pytests/functional/test_version.py

@@ -1,14 +1,23 @@
 import json
 import logging
+import os
 
 import pytest
 
 from tests.support.helpers import SaltVirtualEnv
 from tests.support.pytest.helpers import FakeSaltExtension
+from tests.support.runtests import RUNTIME_VARS
+
+MISSING_SETUP_PY_FILE = not os.path.exists(
+    os.path.join(RUNTIME_VARS.CODE_DIR, "setup.py")
+)
 
 pytestmark = [
     # These are slow because they create a virtualenv and install salt in it
     pytest.mark.slow_test,
+    pytest.mark.skipif(
+        MISSING_SETUP_PY_FILE, reason="This test only work if setup.py is available"
+    ),
 ]
 
 log = logging.getLogger(__name__)

tests/pytests/integration/modules/test_virt.py

@@ -9,6 +9,7 @@
 import pytest
 
 import salt.version
+from salt.utils.versions import Version
 from tests.support.virt import SaltVirtMinionContainerFactory
 
 docker = pytest.importorskip("docker")
@@ -21,6 +22,10 @@
     pytest.mark.slow_test,
     pytest.mark.skip_if_binaries_missing("docker"),
     pytest.mark.skipif(INSIDE_CONTAINER, reason="Cannot run in a container"),
+    pytest.mark.skipif(
+        Version(docker.__version__) < Version("4.0.0"),
+        reason="Test does not work in this version of docker-py",
+    ),
 ]
 
 

tests/pytests/integration/modules/test_x509_v2.py

@@ -11,7 +11,7 @@
 import pytest
 from saltfactories.utils import random_string
 
-import salt.utils.x509 as x509util
+x509util = pytest.importorskip("salt.utils.x509")
 
 try:
     import cryptography