feat: provide support for Python 3.9 for OS-dependent libraries

docs/advanced/os-dependent_libraries.md

@@ -30,6 +30,10 @@ and your pip command should look like this:<br>
 A dot in the platform part indicates that a given distribution supports several platforms.
 In this case, "**.**" in **manylinux_2_17_x86_64.manylinux2014_x86_64** means this distribution supports both **manylinux_2_17_x86_64** and **manylinux2014_x86_64**.
 
+Currently supported python versions are "3.7" and "3.9". Syntax without dots ("37" or "39") is also supported.
+
+If there are libraries specified for different python versions, they must have different target paths.
+
 For more informations, see [.whl](https://www.youtube.com/watch?v=4L0Jb3Ku81s) and [manylinux platform](https://www.youtube.com/watch?v=80j-MRtHMek).
 
 ### Usage

splunk_add_on_ucc_framework/commands/rest_builder/builder.py

@@ -24,7 +24,6 @@
 
 __all__ = ["RestBuilder"]
 
-
 _import_declare_content = """
 import os
 import sys
@@ -43,6 +42,7 @@
 bindir = os.path.dirname(os.path.realpath(os.path.dirname(__file__)))
 libdir = os.path.join(bindir, "lib")
 platform = sys.platform
+python_version = "".join(str(x) for x in sys.version_info[:2])
 """
 
 
@@ -54,25 +54,42 @@ def _generate_import_declare_test(
     if not libraries:
         return base_content
 
-    paths = get_paths_to_add(libraries)
-    os_lib_part = _import_declare_os_lib_content
-    for lib_os, targets in paths.items():
-        if lib_os == "windows":
-            os_lib_part += 'if platform.startswith("win"):\n'
-            for target in targets:
-                os_lib_part += get_insert_to_syspath_str(target)
-        elif lib_os == "darwin":
-            os_lib_part += 'if platform.startswith("darwin"):\n'
-            for target in targets:
-                os_lib_part += get_insert_to_syspath_str(target)
-        else:
-            os_lib_part += 'if platform.startswith("linux"):\n'
-            for target in targets:
-                os_lib_part += get_insert_to_syspath_str(target)
+    base_content += _import_declare_os_lib_content
+    os_lib_part = ""
+
+    paths = group_libs_by_python_version_and_platform(libraries)
+    for python_version, os_specific_paths in paths.items():
+        os_lib_part += f'\nif python_version == "{python_version}":\n'
+        for lib_os, targets in os_specific_paths.items():
+            if lib_os == "windows":
+                os_lib_part += '\tif platform.startswith("win"):\n'
+                for target in targets:
+                    os_lib_part += get_insert_to_syspath_str(target)
+            elif lib_os == "darwin":
+                os_lib_part += '\tif platform.startswith("darwin"):\n'
+                for target in targets:
+                    os_lib_part += get_insert_to_syspath_str(target)
+            else:
+                os_lib_part += '\tif platform.startswith("linux"):\n'
+                for target in targets:
+                    os_lib_part += get_insert_to_syspath_str(target)
 
     return base_content + os_lib_part
 
 
+def group_libs_by_python_version_and_platform(
+    libraries: List[OSDependentLibraryConfig],
+) -> Dict[str, Dict[str, Set[str]]]:
+    """Returns os specific paths grouped by python version and platform"""
+    python_versions = {lib.python_version for lib in libraries}
+    os_specific_paths = {}
+    for python_version in python_versions:
+        os_specific_paths[python_version] = get_paths_to_add(
+            [lib for lib in libraries if lib.python_version == python_version]
+        )
+    return os_specific_paths
+
+
 def get_paths_to_add(libraries: List[OSDependentLibraryConfig]) -> Dict[str, Set[str]]:
     result: Dict[str, Set[str]] = {}
     for library in libraries:
@@ -83,7 +100,7 @@ def get_paths_to_add(libraries: List[OSDependentLibraryConfig]) -> Dict[str, Set
 
 
 def get_insert_to_syspath_str(target: str) -> str:
-    return f'\tsys.path.insert(0, os.path.join(libdir, "{target}"))\n'
+    return f'\t\tsys.path.insert(0, os.path.join(libdir, "{target}"))\n'
 
 
 class _RestBuilderOutput:

splunk_add_on_ucc_framework/global_config.py

@@ -52,8 +52,16 @@ def from_dict(cls, **kwargs: Any) -> "OSDependentLibraryConfig":
         }
         deps_flag = "" if result.get("dependencies") else "--no-deps"
         result.update({"deps_flag": deps_flag})
+        result.update(
+            {"python_version": cls._format_python_version(result["python_version"])}
+        )
         return cls(**result)
 
+    @staticmethod
+    def _format_python_version(python_version: str) -> str:
+        """Remove all non-numeric characters from the python version string to simplify processing"""
+        return "".join(x for x in python_version if x.isnumeric())
+
 
 class GlobalConfig:
     def __init__(self, global_config_path: str) -> None:

splunk_add_on_ucc_framework/install_python_libraries.py

@@ -227,6 +227,8 @@ def install_os_dependent_libraries(
         return cleanup_libraries
 
     logger.info("Installing os-dependentLibraries.")
+
+    validate_conflicting_paths(os_libraries)
     for os_lib in os_libraries:
         if os_lib.dependencies is False and not _pip_is_lib_installed(
             installer=installer,
@@ -272,3 +274,15 @@ def install_os_dependent_libraries(
             sys.exit("Package building process interrupted.")
         cleanup_libraries.add(os_lib.name)
     return cleanup_libraries
+
+
+def validate_conflicting_paths(libs: List[OSDependentLibraryConfig]) -> bool:
+    name_target_pairs = [(lib.name, lib.target) for lib in libs]
+    conflicts = {x for x in name_target_pairs if name_target_pairs.count(x) > 1}
+    if conflicts:
+        logger.error(
+            f"Found conflicting paths for libraries: {conflicts}. "
+            "Please make sure that the paths are unique."
+        )
+        raise CouldNotInstallRequirements
+    return True

splunk_add_on_ucc_framework/schema/schema.json

@@ -1924,7 +1924,29 @@
               },
               "python_version": {
                 "type": "string",
-                "description": "Python version compatible with the library."
+                "description": "Python version compatible with the library.",
+                "anyOf": [
+                  {
+                    "const": "3.7",
+                    "type": "string",
+                    "description": "Exactly: 3.7"
+                  },
+                  {
+                    "const": "37",
+                    "type": "string",
+                    "description": "Alternative syntax for 3.7"
+                  },
+                  {
+                    "const": "3.9",
+                    "type": "string",
+                    "description": "Exactly: 3.9"
+                  },
+                  {
+                    "const": "39",
+                    "type": "string",
+                    "description": "Alternative syntax for 3.9"
+                  }
+                ]
               },
               "target": {
                 "type": "string",

tests/unit/commands/rest_builder/test_generate_import_declare_test.py

@@ -5,7 +5,10 @@
     GlobalConfigBuilderSchema,
 )
 from splunk_add_on_ucc_framework import global_config as gc
-from splunk_add_on_ucc_framework.commands.rest_builder.builder import RestBuilder
+from splunk_add_on_ucc_framework.commands.rest_builder.builder import (
+    RestBuilder,
+    group_libs_by_python_version_and_platform,
+)
 
 base_file_content = [
     "\n",
@@ -27,12 +30,15 @@
     "bindir = os.path.dirname(os.path.realpath(os.path.dirname(__file__)))\n",
     'libdir = os.path.join(bindir, "lib")\n',
     "platform = sys.platform\n",
-    'if platform.startswith("win"):\n',
-    '\tsys.path.insert(0, os.path.join(libdir, "3rdparty/windows"))\n',
-    'if platform.startswith("darwin"):\n',
-    '\tsys.path.insert(0, os.path.join(libdir, "3rdparty/darwin"))\n',
-    'if platform.startswith("linux"):\n',
-    '\tsys.path.insert(0, os.path.join(libdir, "3rdparty/linux"))\n',
+    'python_version = "".join(str(x) for x in sys.version_info[:2])\n',
+    "\n",
+    'if python_version == "37":\n',
+    '\tif platform.startswith("win"):\n',
+    '\t\tsys.path.insert(0, os.path.join(libdir, "3rdparty/windows"))\n',
+    '\tif platform.startswith("darwin"):\n',
+    '\t\tsys.path.insert(0, os.path.join(libdir, "3rdparty/darwin"))\n',
+    '\tif platform.startswith("linux"):\n',
+    '\t\tsys.path.insert(0, os.path.join(libdir, "3rdparty/linux"))\n',
 ]
 
 
@@ -73,3 +79,48 @@ def run_rest_builder_build(global_config, tmp_lib_path):
 
     os.listdir(tmp_lib_path)
     os.walk(tmp_lib_path)
+
+
+def test_group_libs_by_python_version_and_platform_single_version(
+    os_dependent_library_config,
+):
+    libraries = [
+        os_dependent_library_config(
+            name="lib1", python_version="37", os="linux", target="/path/to/lib1"
+        ),
+        os_dependent_library_config(
+            name="lib2", python_version="37", os="linux", target="/path/to/lib2"
+        ),
+    ]
+    expected = {"37": {"linux": {"/path/to/lib1", "/path/to/lib2"}}}
+    assert group_libs_by_python_version_and_platform(libraries) == expected
+
+
+def test_group_libs_by_python_version_and_platform_multiple_versions(
+    os_dependent_library_config,
+):
+    libraries = [
+        os_dependent_library_config(
+            name="lib1", python_version="37", os="linux", target="/path/to/lib1"
+        ),
+        os_dependent_library_config(
+            name="lib2", python_version="38", os="linux", target="/path/to/lib2"
+        ),
+    ]
+    expected = {"37": {"linux": {"/path/to/lib1"}}, "38": {"linux": {"/path/to/lib2"}}}
+    assert group_libs_by_python_version_and_platform(libraries) == expected
+
+
+def test_group_libs_by_python_version_and_platform_multiple_os(
+    os_dependent_library_config,
+):
+    libraries = [
+        os_dependent_library_config(
+            name="lib1", python_version="37", os="linux", target="/path/to/lib1"
+        ),
+        os_dependent_library_config(
+            name="lib2", python_version="37", os="windows", target="C:\\path\\to\\lib2"
+        ),
+    ]
+    expected = {"37": {"linux": {"/path/to/lib1"}, "windows": {"C:\\path\\to\\lib2"}}}
+    assert group_libs_by_python_version_and_platform(libraries) == expected

tests/unit/conftest.py

@@ -6,6 +6,7 @@
 from splunk_add_on_ucc_framework import global_config as global_config_lib
 from splunk_add_on_ucc_framework import app_manifest as app_manifest_lib
 import tests.unit.helpers as helpers
+from splunk_add_on_ucc_framework.global_config import OSDependentLibraryConfig
 
 
 @pytest.fixture
@@ -53,6 +54,19 @@ def global_config_only_logging() -> global_config_lib.GlobalConfig:
     return global_config
 
 
+@pytest.fixture()
+def os_dependent_library_config():
+    return lambda name="lib1", python_version="37", target="t", os="os": OSDependentLibraryConfig(
+        name=name,
+        version="version",
+        python_version=python_version,
+        platform="platform",
+        target=target,
+        os=os,
+        deps_flag="deps_flag",
+    )
+
+
 @pytest.fixture
 def monkeypatch(monkeypatch):
     """

tests/unit/test_install_python_libraries.py

@@ -1,9 +1,11 @@
 import os
 import stat
+from typing import List
 from unittest import mock
 
 import pytest
 import tests.unit.helpers as helpers
+from splunk_add_on_ucc_framework.global_config import OSDependentLibraryConfig
 
 from splunk_add_on_ucc_framework.install_python_libraries import (
     CouldNotInstallRequirements,
@@ -13,6 +15,7 @@
     install_python_libraries,
     remove_execute_bit,
     remove_packages,
+    validate_conflicting_paths,
 )
 
 from splunk_add_on_ucc_framework import global_config as gc
@@ -31,25 +34,25 @@
         ("solnlib==5.0.0\nsplunktaucclib==6.0.0\n", True),
         (
             """splunktalib==2.2.6; python_version >= "3.7" and python_version < "4.0" \
-    --hash=sha256:bba70ac7407cdedcb45437cb152ac0e43aae16b978031308e6bec548d3543119 \
-    --hash=sha256:8d58d697a842319b4c675557b0cc4a9c68e8d909389a98ed240e2bb4ff358d31
-splunktaucclib==5.0.7; python_version >= "3.7" and python_version < "4.0" \
-    --hash=sha256:3ddc1276c41c809c16ae810cb20e9eb4abd2f94dba5ddf460cf9c49b50f659ac \
-    --hash=sha256:a1e3f710fcb0b24dff8913e6e5df0d36f0693b7f3ed7c0a9a43b08372b08eb90""",
+        --hash=sha256:bba70ac7407cdedcb45437cb152ac0e43aae16b978031308e6bec548d3543119 \
+        --hash=sha256:8d58d697a842319b4c675557b0cc4a9c68e8d909389a98ed240e2bb4ff358d31
+    splunktaucclib==5.0.7; python_version >= "3.7" and python_version < "4.0" \
+        --hash=sha256:3ddc1276c41c809c16ae810cb20e9eb4abd2f94dba5ddf460cf9c49b50f659ac \
+        --hash=sha256:a1e3f710fcb0b24dff8913e6e5df0d36f0693b7f3ed7c0a9a43b08372b08eb90""",
             True,
         ),
         (
             """splunktaucclib==5.0.7; python_version >= "3.7" and python_version < "4.0" \
-    --hash=sha256:3ddc1276c41c809c16ae810cb20e9eb4abd2f94dba5ddf460cf9c49b50f659ac \
-    --hash=sha256:a1e3f710fcb0b24dff8913e6e5df0d36f0693b7f3ed7c0a9a43b08372b08eb90""",
+        --hash=sha256:3ddc1276c41c809c16ae810cb20e9eb4abd2f94dba5ddf460cf9c49b50f659ac \
+        --hash=sha256:a1e3f710fcb0b24dff8913e6e5df0d36f0693b7f3ed7c0a9a43b08372b08eb90""",
             True,
         ),
         (
             """sortedcontainers==2.4.0; python_version >= "3.7" and python_version < "4.0" \
-    --hash=sha256:a163dcaede0f1c021485e957a39245190e74249897e2ae4b2aa38595db237ee0 \
-    --hash=sha256:25caa5a06cc30b6b83d11423433f65d1f9d76c4c6a0c90e3379eaa43b9bfdb88
-splunk-sdk==1.7.1 \
-    --hash=sha256:4d0de12a87395f28f2a0c90b179882072a39a1f09a3ec9e79ce0de7a16220fe1""",
+        --hash=sha256:a163dcaede0f1c021485e957a39245190e74249897e2ae4b2aa38595db237ee0 \
+        --hash=sha256:25caa5a06cc30b6b83d11423433f65d1f9d76c4c6a0c90e3379eaa43b9bfdb88
+    splunk-sdk==1.7.1 \
+        --hash=sha256:4d0de12a87395f28f2a0c90b179882072a39a1f09a3ec9e79ce0de7a16220fe1""",
             False,
         ),
     ],
@@ -474,3 +477,33 @@ def test_install_libraries_legacy_resolver_with_wrong_pip(caplog):
         "Please remove '--pip-legacy-resolver' from your build command or use a different version of pip e.g. 23.2.1"
     )
     assert expected_msg in caplog.text
+
+
+def test_validate_conflicting_paths_no_conflict(os_dependent_library_config):
+    libs: List[OSDependentLibraryConfig] = [
+        os_dependent_library_config(name="lib1", target="path1"),
+        os_dependent_library_config(name="lib2", target="path2"),
+    ]
+    assert validate_conflicting_paths(libs)
+
+
+def test_validate_conflicting_paths_with_conflict(os_dependent_library_config, caplog):
+    libs: List[OSDependentLibraryConfig] = [
+        os_dependent_library_config(name="lib1", target="path1"),
+        os_dependent_library_config(name="lib1", target="path1"),
+        os_dependent_library_config(name="lib1", target="path2"),
+        os_dependent_library_config(name="lib2", target="path2"),
+        os_dependent_library_config(name="lib3", target="path2"),
+        os_dependent_library_config(name="lib3", target="path2"),
+    ]
+    with pytest.raises(CouldNotInstallRequirements):
+        validate_conflicting_paths(libs)
+
+    assert "('lib1', 'path1')" in caplog.text
+    assert "('lib3', 'path2')" in caplog.text
+    assert "('lib2', 'path2')" not in caplog.text
+
+
+def test_validate_conflicting_paths_empty_list():
+    libs: List[OSDependentLibraryConfig] = []
+    assert validate_conflicting_paths(libs)