Add "instruments-any" feature: unblock multi-target instrumentations

while fixing dependency conflict breakage (SQUASH)

Author: jeremydvoss

CHANGELOG.md

@@ -11,6 +11,18 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## Unreleased
 
+### Fixed
+
+- `opentelemetry-instrumentation`: Fix dependency conflict detection when instrumented packages are not installed by moving check back to before instrumentors are loaded. Add "instruments-any" feature for instrumentations that target multiple packages.
+  ([#3610](https://github.com/open-telemetry/opentelemetry-python-contrib/pull/3610))
+
+### Added
+
+- `opentelemetry-instrumentation-psycopg2` Utilize instruments-any functionality.
+  ([#3610](https://github.com/open-telemetry/opentelemetry-python-contrib/pull/3612))
+- `opentelemetry-instrumentation-kafka-python` Utilize instruments-any functionality.
+  ([#3610](https://github.com/open-telemetry/opentelemetry-python-contrib/pull/3612))
+
 ## Version 1.35.0/0.56b0 (2025-07-11)
 
 ### Added

CONTRIBUTING.md

@@ -325,10 +325,14 @@ Below is a checklist of things to be mindful of when implementing a new instrume
 ### Update supported instrumentation package versions
 
 - Navigate to the **instrumentation package directory:**
-  - Update **`pyproject.toml`** file by modifying _instruments_ entry in the `[project.optional-dependencies]` section with the new version constraint
-  - Update `_instruments` variable in instrumentation **`package.py`** file with the new version constraint
+  - Update **`pyproject.toml`** file by modifying `instruments` or `instruments-any` entry in the `[project.optional-dependencies]` section with the new version constraint
+  - Update `_instruments` or `_instruments_any` variable in instrumentation **`package.py`** file with the new version constraint
 - At the **root of the project directory**, run `tox -e generate` to regenerate necessary files
 
+Please note that `instruments-any` is an optional field that can be used instead of or in addition to `instruments`. While `instruments` is a list of dependencies, _all_ of which are expected by the instrumentation, `instruments-any` is a list _any_ of which but not all are expected.
+
+<!-- See https://github.com/open-telemetry/opentelemetry-python-contrib/pull/3610 for details on instruments-any -->
+
 If you're adding support for a new version of the instrumentation package, follow these additional steps:
 
 - At the **instrumentation package directory:** Add new test-requirements.txt file with the respective package version required for testing

instrumentation/opentelemetry-instrumentation-fastapi/tests/test_fastapi_instrumentation.py

@@ -40,7 +40,6 @@
 )
 from opentelemetry.instrumentation.dependencies import (
     DependencyConflict,
-    DependencyConflictError,
 )
 from opentelemetry.sdk.metrics.export import (
     HistogramDataPoint,
@@ -1102,40 +1101,34 @@ def test_instruments_with_fastapi_installed(self, mock_logger):
             [self._instrumentation_loaded_successfully_call()]
         )
 
+    @patch(
+        "opentelemetry.instrumentation.auto_instrumentation._load.get_dist_dependency_conflicts"
+    )
     @patch("opentelemetry.instrumentation.auto_instrumentation._load._logger")
-    def test_instruments_with_old_fastapi_installed(self, mock_logger):  # pylint: disable=no-self-use
+    def test_instruments_with_old_fastapi_installed(
+        self, mock_logger, mock_dep
+    ):  # pylint: disable=no-self-use
         dependency_conflict = DependencyConflict("0.58", "0.57")
         mock_distro = Mock()
-        mock_distro.load_instrumentor.side_effect = DependencyConflictError(
-            dependency_conflict
-        )
+        mock_dep.return_value = dependency_conflict
         _load_instrumentors(mock_distro)
-        self.assertEqual(len(mock_distro.load_instrumentor.call_args_list), 1)
-        (ep,) = mock_distro.load_instrumentor.call_args.args
-        self.assertEqual(ep.name, "fastapi")
-        assert (
-            self._instrumentation_loaded_successfully_call()
-            not in mock_logger.debug.call_args_list
-        )
+        mock_distro.load_instrumentor.assert_not_called()
         mock_logger.debug.assert_has_calls(
             [self._instrumentation_failed_to_load_call(dependency_conflict)]
         )
 
+    @patch(
+        "opentelemetry.instrumentation.auto_instrumentation._load.get_dist_dependency_conflicts"
+    )
     @patch("opentelemetry.instrumentation.auto_instrumentation._load._logger")
-    def test_instruments_without_fastapi_installed(self, mock_logger):  # pylint: disable=no-self-use
+    def test_instruments_without_fastapi_installed(
+        self, mock_logger, mock_dep
+    ):  # pylint: disable=no-self-use
         dependency_conflict = DependencyConflict("0.58", None)
         mock_distro = Mock()
-        mock_distro.load_instrumentor.side_effect = DependencyConflictError(
-            dependency_conflict
-        )
+        mock_dep.return_value = dependency_conflict
         _load_instrumentors(mock_distro)
-        self.assertEqual(len(mock_distro.load_instrumentor.call_args_list), 1)
-        (ep,) = mock_distro.load_instrumentor.call_args.args
-        self.assertEqual(ep.name, "fastapi")
-        assert (
-            self._instrumentation_loaded_successfully_call()
-            not in mock_logger.debug.call_args_list
-        )
+        mock_distro.load_instrumentor.assert_not_called()
         mock_logger.debug.assert_has_calls(
             [self._instrumentation_failed_to_load_call(dependency_conflict)]
         )

instrumentation/opentelemetry-instrumentation-kafka-python/pyproject.toml

@@ -31,7 +31,8 @@ dependencies = [
 ]
 
 [project.optional-dependencies]
-instruments = [
+instruments = []
+instruments-any = [
   "kafka-python >= 2.0, < 3.0",
   "kafka-python-ng >= 2.0, < 3.0"
 ]

instrumentation/opentelemetry-instrumentation-kafka-python/src/opentelemetry/instrumentation/kafka/package.py

@@ -16,4 +16,5 @@
 _instruments_kafka_python = "kafka-python >= 2.0, < 3.0"
 _instruments_kafka_python_ng = "kafka-python-ng >= 2.0, < 3.0"
 
-_instruments = (_instruments_kafka_python, _instruments_kafka_python_ng)
+_instruments = ()
+_instruments_any = (_instruments_kafka_python, _instruments_kafka_python_ng)

instrumentation/opentelemetry-instrumentation-psycopg2/pyproject.toml

@@ -31,7 +31,8 @@ dependencies = [
 ]
 
 [project.optional-dependencies]
-instruments = [
+instruments = []
+instruments-any = [
   "psycopg2 >= 2.7.3.1",
   "psycopg2-binary >= 2.7.3.1",
 ]

instrumentation/opentelemetry-instrumentation-psycopg2/src/opentelemetry/instrumentation/psycopg2/package.py

@@ -16,7 +16,8 @@
 _instruments_psycopg2 = "psycopg2 >= 2.7.3.1"
 _instruments_psycopg2_binary = "psycopg2-binary >= 2.7.3.1"
 
-_instruments = (
+_instruments = ()
+_instruments_any = (
     _instruments_psycopg2,
     _instruments_psycopg2_binary,
 )

instrumentation/opentelemetry-instrumentation-psycopg2/tests/test_psycopg2_instrumentation.py

@@ -137,12 +137,16 @@ def _distribution(name):
     # Note there is only one test here but it is run twice in tox
     # once with the psycopg2 package installed and once with
     # psycopg2-binary installed.
+    @patch(
+        "opentelemetry.instrumentation.auto_instrumentation._load.get_dist_dependency_conflicts"
+    )
     @patch("opentelemetry.instrumentation.auto_instrumentation._load._logger")
-    def test_instruments_with_psycopg2_installed(self, mock_logger):
+    def test_instruments_with_psycopg2_installed(self, mock_logger, mock_dep):
         def _instrumentation_loaded_successfully_call():
             return call("Instrumented %s", "psycopg2")
 
         mock_distro = Mock()
+        mock_dep.return_value = None
         mock_distro.load_instrumentor.return_value = None
         _load_instrumentors(mock_distro)
         self.assertEqual(len(mock_distro.load_instrumentor.call_args_list), 1)

opentelemetry-instrumentation/src/opentelemetry/instrumentation/auto_instrumentation/_load.py

@@ -12,22 +12,51 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+from functools import cached_property
 from logging import getLogger
 from os import environ
 
-from opentelemetry.instrumentation.dependencies import DependencyConflictError
+from opentelemetry.instrumentation.dependencies import (
+    DependencyConflictError,
+    get_dist_dependency_conflicts,
+)
 from opentelemetry.instrumentation.distro import BaseDistro, DefaultDistro
 from opentelemetry.instrumentation.environment_variables import (
     OTEL_PYTHON_CONFIGURATOR,
     OTEL_PYTHON_DISABLED_INSTRUMENTATIONS,
     OTEL_PYTHON_DISTRO,
 )
 from opentelemetry.instrumentation.version import __version__
-from opentelemetry.util._importlib_metadata import entry_points
+from opentelemetry.util._importlib_metadata import (
+    EntryPoint,
+    distributions,
+    entry_points,
+)
 
 _logger = getLogger(__name__)
 
 
+class _EntryPointDistFinder:
+    @cached_property
+    def _mapping(self):
+        return {
+            self._key_for(ep): dist
+            for dist in distributions()
+            for ep in dist.entry_points
+        }
+
+    def dist_for(self, entry_point: EntryPoint):
+        dist = getattr(entry_point, "dist", None)
+        if dist:
+            return dist
+
+        return self._mapping.get(self._key_for(entry_point))
+
+    @staticmethod
+    def _key_for(entry_point: EntryPoint):
+        return f"{entry_point.group}:{entry_point.name}:{entry_point.value}"
+
+
 def _load_distro() -> BaseDistro:
     distro_name = environ.get(OTEL_PYTHON_DISTRO, None)
     for entry_point in entry_points(group="opentelemetry_distro"):
@@ -55,6 +84,7 @@ def _load_distro() -> BaseDistro:
 
 def _load_instrumentors(distro):
     package_to_exclude = environ.get(OTEL_PYTHON_DISABLED_INSTRUMENTATIONS, [])
+    entry_point_finder = _EntryPointDistFinder()
     if isinstance(package_to_exclude, str):
         package_to_exclude = package_to_exclude.split(",")
         # to handle users entering "requests , flask" or "requests, flask" with spaces
@@ -71,11 +101,24 @@ def _load_instrumentors(distro):
             continue
 
         try:
-            distro.load_instrumentor(
-                entry_point, raise_exception_on_conflict=True
-            )
+            entry_point_dist = entry_point_finder.dist_for(entry_point)
+            conflict = get_dist_dependency_conflicts(entry_point_dist)
+            if conflict:
+                _logger.debug(
+                    "Skipping instrumentation %s: %s",
+                    entry_point.name,
+                    conflict,
+                )
+                continue
+
+            # tell instrumentation to not run dep checks again as we already did it above
+            distro.load_instrumentor(entry_point, skip_dep_check=True)
             _logger.debug("Instrumented %s", entry_point.name)
         except DependencyConflictError as exc:
+            # Dependency conflicts are generally caught from get_dist_dependency_conflicts
+            # returning a DependencyConflict. Keeping this error handling in case custom
+            # distro and instrumentor behavior raises a DependencyConflictError later.
+            # See https://github.com/open-telemetry/opentelemetry-python-contrib/pull/3610
             _logger.debug(
                 "Skipping instrumentation %s: %s",
                 entry_point.name,

opentelemetry-instrumentation/src/opentelemetry/instrumentation/dependencies.py

@@ -20,6 +20,7 @@
 from packaging.requirements import InvalidRequirement, Requirement
 
 from opentelemetry.util._importlib_metadata import (
+    Distribution,
     PackageNotFoundError,
     version,
 )
@@ -28,14 +29,44 @@
 
 
 class DependencyConflict:
+    """Represents a dependency conflict in OpenTelemetry instrumentation.
+
+    This class is used to track conflicts between required dependencies and the
+    actual installed packages. It supports two scenarios:
+
+    1. Standard conflicts where all dependencies are required
+    2. Either/or conflicts where only one of a set of dependencies is required
+
+    Attributes:
+        required: The required dependency specification that conflicts with what's installed.
+        found: The actual dependency that was found installed (if any).
+        required_any: Collection of dependency specifications where any one would satisfy
+            the requirement (for either/or scenarios).
+        found_any: Collection of actual dependencies found for either/or scenarios.
+    """
+
     required: str | None = None
     found: str | None = None
-
-    def __init__(self, required: str | None, found: str | None = None):
+    # The following fields are used when an instrumentation requires any of a set of dependencies rather than all.
+    required_any: Collection[str] = None
+    found_any: Collection[str] = None
+
+    def __init__(
+        self,
+        required: str | None = None,
+        found: str | None = None,
+        required_any: Collection[str] = None,
+        found_any: Collection[str] = None,
+    ):
         self.required = required
         self.found = found
+        # The following fields are used when an instrumentation requires any of a set of dependencies rather than all.
+        self.required_any = required_any
+        self.found_any = found_any
 
     def __str__(self):
+        if not self.required and (self.required_any or self.found_any):
+            return f'DependencyConflict: requested any of the following: "{self.required_any}" but found: "{self.found_any}"'
         return f'DependencyConflict: requested: "{self.required}" but found: "{self.found}"'
 
 
@@ -49,8 +80,39 @@ def __str__(self):
         return str(self.conflict)
 
 
+def get_dist_dependency_conflicts(
+    dist: Distribution,
+) -> DependencyConflict | None:
+    instrumentation_deps = []
+    instrumentation_any_deps = []
+    extra = "extra"
+    instruments = "instruments"
+    instruments_marker = {extra: instruments}
+    instruments_any = "instruments-any"
+    instruments_any_marker = {extra: instruments_any}
+    if dist.requires:
+        for dep in dist.requires:
+            if extra not in dep:
+                continue
+            if instruments not in dep and instruments_any not in dep:
+                continue
+
+            req = Requirement(dep)
+            if req.marker.evaluate(instruments_marker):  # type: ignore
+                instrumentation_deps.append(req)  # type: ignore
+            if req.marker.evaluate(instruments_any_marker):  # type: ignore
+                instrumentation_any_deps.append(req)  # type: ignore
+    return get_dependency_conflicts(
+        instrumentation_deps, instrumentation_any_deps
+    )  # type: ignore
+
+
 def get_dependency_conflicts(
-    deps: Collection[str | Requirement],
+    deps: Collection[
+        str | Requirement
+    ],  # Dependencies all of which are required
+    deps_any: Collection[str | Requirement]
+    | None = None,  # Dependencies any of which are required
 ) -> DependencyConflict | None:
     for dep in deps:
         if isinstance(dep, Requirement):
@@ -73,4 +135,53 @@ def get_dependency_conflicts(
 
         if not req.specifier.contains(dist_version):
             return DependencyConflict(dep, f"{req.name} {dist_version}")
+
+    # If all the dependencies in "instruments" are present, check "instruments-any" for conflicts.
+    if deps_any:
+        return _get_dependency_conflicts_any(deps_any)
+    return None
+
+
+# This is a helper functions designed to ease reading and meet linting requirements.
+def _get_dependency_conflicts_any(
+    deps_any: Collection[str | Requirement],
+) -> DependencyConflict | None:
+    if not deps_any:
+        return None
+    is_dependency_conflict = True
+    required_any: Collection[str] = []
+    found_any: Collection[str] = []
+    for dep in deps_any:
+        if isinstance(dep, Requirement):
+            req = dep
+        else:
+            try:
+                req = Requirement(dep)
+            except InvalidRequirement as exc:
+                logger.warning(
+                    'error parsing dependency, reporting as a conflict: "%s" - %s',
+                    dep,
+                    exc,
+                )
+                return DependencyConflict(dep)
+
+        try:
+            dist_version = version(req.name)
+        except PackageNotFoundError:
+            required_any.append(str(dep))
+            continue
+
+        if req.specifier.contains(dist_version):
+            # Since only one of the instrumentation_any dependencies is required, there is no dependency conflict.
+            is_dependency_conflict = False
+            break
+        # If the version does not match, add it to the list of unfulfilled requirement options.
+        required_any.append(str(dep))
+        found_any.append(f"{req.name} {dist_version}")
+
+    if is_dependency_conflict:
+        return DependencyConflict(
+            required_any=required_any,
+            found_any=found_any,
+        )
     return None