From 972db6fb067e8fd1d777761b7bb7a349d4bf867d Mon Sep 17 00:00:00 2001 From: Harsh Modi Date: Wed, 6 Sep 2023 11:16:47 -0400 Subject: [PATCH] instrument OSBS base task This will allow us to get traces of all HTTP requests made in different tasks Signed-off-by: Harsh Modi --- atomic_reactor/constants.py | 2 + .../source_containers_user_params.json | 8 ++ atomic_reactor/schemas/user_params.json | 7 ++ atomic_reactor/tasks/common.py | 44 ++++++- atomic_reactor/tasks/plugin_based.py | 1 - otel-requirements.in | 8 ++ otel-requirements.txt | 112 ++++++++++++++++++ requirements-devel.in | 1 + requirements-devel.txt | 111 +++++++++++++++-- requirements.in | 1 + test.sh | 5 + 11 files changed, 288 insertions(+), 12 deletions(-) create mode 100644 otel-requirements.in create mode 100644 otel-requirements.txt diff --git a/atomic_reactor/constants.py b/atomic_reactor/constants.py index 66f374c8d..8f3510579 100644 --- a/atomic_reactor/constants.py +++ b/atomic_reactor/constants.py @@ -76,6 +76,8 @@ IMAGE_TYPE_OCI = 'oci' IMAGE_TYPE_OCI_TAR = 'oci-tar' +OTEL_SERVICE_NAME = 'osbs' + PLUGIN_KOJI_PROMOTE_PLUGIN_KEY = 'koji_promote' PLUGIN_KOJI_IMPORT_PLUGIN_KEY = 'koji_import' PLUGIN_KOJI_IMPORT_SOURCE_CONTAINER_PLUGIN_KEY = 'koji_import_source_container' diff --git a/atomic_reactor/schemas/source_containers_user_params.json b/atomic_reactor/schemas/source_containers_user_params.json index 27378282a..f1bf7882f 100644 --- a/atomic_reactor/schemas/source_containers_user_params.json +++ b/atomic_reactor/schemas/source_containers_user_params.json @@ -10,6 +10,14 @@ "kind": {"type": "string"}, "koji_target": {"type": "string"}, "koji_task_id": {"type": "integer"}, + "opentelemetry_info": { + "type": "object", + "properties": { + "traceparent": {"type": ["string", "null"]}, + "otel_url": {"type": ["string", "null"]} + }, + "additionalProperties": false + }, "reactor_config_map": {"type": "string"}, "scratch": {"type": "boolean"}, "signing_intent": {"type": "string"}, diff --git a/atomic_reactor/schemas/user_params.json b/atomic_reactor/schemas/user_params.json index 1ceacab30..d25ccfef4 100644 --- a/atomic_reactor/schemas/user_params.json +++ b/atomic_reactor/schemas/user_params.json @@ -28,6 +28,13 @@ "koji_task_id": {"type": "integer"}, "name": {"type": "string"}, "operator_csv_modifications_url": {"type": "string"}, + "opentelemetry_info": { + "type": "object", + "properties": { + "traceparent": {"type": ["string", "null"]}, + "otel_url": {"type": ["string", "null"]} + } + }, "platform": {"type": "string"}, "platforms": { "type": "array", diff --git a/atomic_reactor/tasks/common.py b/atomic_reactor/tasks/common.py index 8840d14d2..71b5a0abb 100644 --- a/atomic_reactor/tasks/common.py +++ b/atomic_reactor/tasks/common.py @@ -7,20 +7,28 @@ """ import abc -import signal import json +import logging +import os +import signal from dataclasses import dataclass +from functools import cached_property from pathlib import Path from typing import Dict, Any, ClassVar, Generic, TypeVar, Optional -from functools import cached_property + +from opentelemetry.instrumentation.requests import RequestsInstrumentor +from otel_extensions import TelemetryOptions, init_telemetry_provider, get_tracer from atomic_reactor import config from atomic_reactor import dirs from atomic_reactor import inner from atomic_reactor import source from atomic_reactor import util +from atomic_reactor.constants import OTEL_SERVICE_NAME from atomic_reactor.plugin import TaskCanceledException +logger = logging.getLogger(__name__) + def write_task_result(output_file, msg): with open(output_file, 'w') as f: @@ -88,6 +96,7 @@ class Task(abc.ABC, Generic[ParamsT]): ignore_sigterm: ClassVar[bool] = False # Automatically save context data before exiting? (Note: do not use for parallel tasks) autosave_context_data: ClassVar[bool] = True + task_name = 'default' def __init__(self, params: ParamsT): """Initialize a Task.""" @@ -122,7 +131,36 @@ def run(self, *args, **kwargs): else: signal.signal(signal.SIGTERM, self.throw_task_canceled_exception) - result = self.execute(*args, **kwargs) + opentelemetry_info = self._params.user_params.get('opentelemetry_info', None) + if opentelemetry_info: + traceparent = opentelemetry_info.get('traceparent', None) + otel_url = opentelemetry_info.get('otel_url', None) + + span_exporter = '' + otel_protocol = 'http/protobuf' + if not otel_url: + otel_protocol = 'custom' + span_exporter = '"opentelemetry.sdk.trace.export.ConsoleSpanExporter"' + + if traceparent: + os.environ['TRACEPARENT'] = traceparent + logger.info('traceparent is set to %s', traceparent) + otel_options = TelemetryOptions( + OTEL_SERVICE_NAME=OTEL_SERVICE_NAME, + OTEL_EXPORTER_CUSTOM_SPAN_EXPORTER_TYPE=span_exporter, + OTEL_EXPORTER_OTLP_ENDPOINT=otel_url, + OTEL_EXPORTER_OTLP_PROTOCOL=otel_protocol, + ) + init_telemetry_provider(otel_options) + + RequestsInstrumentor().instrument() + + span_name = self.task_name + if hasattr(self._params, 'platform'): + span_name += '_' + self._params.platform + tracer = get_tracer(module_name=span_name, service_name=OTEL_SERVICE_NAME) + with tracer.start_as_current_span(span_name): + result = self.execute(*args, **kwargs) if self._params.task_result: write_task_result(self._params.task_result, json.dumps(result)) diff --git a/atomic_reactor/tasks/plugin_based.py b/atomic_reactor/tasks/plugin_based.py index 85bec23e0..4dbc3f3c5 100644 --- a/atomic_reactor/tasks/plugin_based.py +++ b/atomic_reactor/tasks/plugin_based.py @@ -31,7 +31,6 @@ class PluginBasedTask(Task[ParamsT]): # {"name": "add_filesystem", "args": {...}} # Refer to plugins.json schema for the details. plugins_conf: ClassVar[List[Dict[str, Any]]] = [] - task_name = 'default' def prepare_workflow(self) -> inner.DockerBuildWorkflow: """Fully initialize the workflow instance to be used for running the list of plugins.""" diff --git a/otel-requirements.in b/otel-requirements.in new file mode 100644 index 000000000..cc32fc36c --- /dev/null +++ b/otel-requirements.in @@ -0,0 +1,8 @@ +calver +hatchling +opentelemetry-api +opentelemetry-exporter-otlp +opentelemetry-instrumentation-requests +opentelemetry-sdk +otel-extensions +urllib3 < 2.0 diff --git a/otel-requirements.txt b/otel-requirements.txt new file mode 100644 index 000000000..a791fc431 --- /dev/null +++ b/otel-requirements.txt @@ -0,0 +1,112 @@ +# +# This file is autogenerated by pip-compile with Python 3.8 +# by the following command: +# +# pip-compile --output-file=otel-requirements.txt otel-requirements.in +# +backoff==2.2.1 + # via + # opentelemetry-exporter-otlp-proto-grpc + # opentelemetry-exporter-otlp-proto-http +calver==2022.6.26 + # via -r otel-requirements.in +certifi==2023.7.22 + # via requests +charset-normalizer==3.2.0 + # via requests +deprecated==1.2.14 + # via + # opentelemetry-api + # opentelemetry-exporter-otlp-proto-grpc + # opentelemetry-exporter-otlp-proto-http +editables==0.5 + # via hatchling +googleapis-common-protos==1.60.0 + # via + # opentelemetry-exporter-otlp-proto-grpc + # opentelemetry-exporter-otlp-proto-http +grpcio==1.57.0 + # via opentelemetry-exporter-otlp-proto-grpc +hatchling==1.18.0 + # via -r otel-requirements.in +idna==3.4 + # via requests +importlib-metadata==6.8.0 + # via opentelemetry-api +opentelemetry-api==1.19.0 + # via + # -r otel-requirements.in + # opentelemetry-exporter-otlp-proto-grpc + # opentelemetry-exporter-otlp-proto-http + # opentelemetry-instrumentation + # opentelemetry-instrumentation-requests + # opentelemetry-sdk + # otel-extensions +opentelemetry-exporter-otlp==1.19.0 + # via -r otel-requirements.in +opentelemetry-exporter-otlp-proto-common==1.19.0 + # via + # opentelemetry-exporter-otlp-proto-grpc + # opentelemetry-exporter-otlp-proto-http +opentelemetry-exporter-otlp-proto-grpc==1.19.0 + # via opentelemetry-exporter-otlp +opentelemetry-exporter-otlp-proto-http==1.19.0 + # via opentelemetry-exporter-otlp +opentelemetry-instrumentation==0.40b0 + # via opentelemetry-instrumentation-requests +opentelemetry-instrumentation-requests==0.40b0 + # via -r otel-requirements.in +opentelemetry-proto==1.19.0 + # via + # opentelemetry-exporter-otlp-proto-common + # opentelemetry-exporter-otlp-proto-grpc + # opentelemetry-exporter-otlp-proto-http +opentelemetry-sdk==1.19.0 + # via + # -r otel-requirements.in + # opentelemetry-exporter-otlp-proto-grpc + # opentelemetry-exporter-otlp-proto-http + # otel-extensions +opentelemetry-semantic-conventions==0.40b0 + # via + # opentelemetry-instrumentation-requests + # opentelemetry-sdk +opentelemetry-util-http==0.40b0 + # via opentelemetry-instrumentation-requests +otel-extensions==0.2.5 + # via -r otel-requirements.in +packaging==23.1 + # via hatchling +pathspec==0.11.2 + # via hatchling +pluggy==1.3.0 + # via hatchling +protobuf==4.24.2 + # via + # googleapis-common-protos + # opentelemetry-proto +pydantic==1.10.12 + # via otel-extensions +requests==2.31.0 + # via opentelemetry-exporter-otlp-proto-http +tomli==2.0.1 + # via hatchling +trove-classifiers==2023.8.7 + # via hatchling +typing-extensions==4.7.1 + # via + # opentelemetry-sdk + # pydantic +urllib3==1.26.16 + # via + # -r otel-requirements.in + # requests +wrapt==1.15.0 + # via + # deprecated + # opentelemetry-instrumentation +zipp==3.16.2 + # via importlib-metadata + +# The following packages are considered to be unsafe in a requirements file: +# setuptools diff --git a/requirements-devel.in b/requirements-devel.in index ce9b90c6f..8f426023f 100644 --- a/requirements-devel.in +++ b/requirements-devel.in @@ -1,4 +1,5 @@ -r requirements.in +-r otel-requirements.in -r tests/requirements.in pyflakes pep8 diff --git a/requirements-devel.txt b/requirements-devel.txt index 8d1a188a0..b98ac182c 100644 --- a/requirements-devel.txt +++ b/requirements-devel.txt @@ -9,9 +9,14 @@ attrs==21.2.0 # jsonschema # pytest backoff==1.11.1 - # via -r requirements.in + # via + # -r requirements.in + # opentelemetry-exporter-otlp-proto-grpc + # opentelemetry-exporter-otlp-proto-http bcrypt==3.2.0 # via paramiko +calver==2022.6.26 + # via -r otel-requirements.in certifi==2022.12.7 # via requests cffi==1.15.0 @@ -32,10 +37,17 @@ cryptography==3.4.8 # requests-kerberos decorator==5.1.0 # via gssapi +deprecated==1.2.14 + # via + # opentelemetry-api + # opentelemetry-exporter-otlp-proto-grpc + # opentelemetry-exporter-otlp-proto-http dockerfile-parse==1.2.0 # via # -r requirements.in # osbs-client +editables==0.5 + # via hatchling execnet==1.9.0 # via pytest-xdist flake8==4.0.1 @@ -44,12 +56,22 @@ flatpak-module-tools==0.14 # via -r requirements.in flexmock==0.11.1 # via -r tests/requirements.in +googleapis-common-protos==1.60.0 + # via + # opentelemetry-exporter-otlp-proto-grpc + # opentelemetry-exporter-otlp-proto-http +grpcio==1.57.0 + # via opentelemetry-exporter-otlp-proto-grpc gssapi==1.7.2 # via # pyspnego # requests-gssapi +hatchling==1.18.0 + # via -r otel-requirements.in idna==3.3 # via requests +importlib-metadata==6.8.0 + # via opentelemetry-api iniconfig==1.1.1 # via pytest jsonschema==4.1.2 @@ -62,16 +84,68 @@ krb5==0.2.0 # via pyspnego mccabe==0.6.1 # via flake8 +opentelemetry-api==1.19.0 + # via + # -r otel-requirements.in + # opentelemetry-exporter-otlp-proto-grpc + # opentelemetry-exporter-otlp-proto-http + # opentelemetry-instrumentation + # opentelemetry-instrumentation-requests + # opentelemetry-sdk + # otel-extensions +opentelemetry-exporter-otlp==1.19.0 + # via -r otel-requirements.in +opentelemetry-exporter-otlp-proto-common==1.19.0 + # via + # opentelemetry-exporter-otlp-proto-grpc + # opentelemetry-exporter-otlp-proto-http +opentelemetry-exporter-otlp-proto-grpc==1.19.0 + # via opentelemetry-exporter-otlp +opentelemetry-exporter-otlp-proto-http==1.19.0 + # via opentelemetry-exporter-otlp +opentelemetry-instrumentation==0.40b0 + # via opentelemetry-instrumentation-requests +opentelemetry-instrumentation-requests==0.40b0 + # via -r otel-requirements.in +opentelemetry-proto==1.19.0 + # via + # opentelemetry-exporter-otlp-proto-common + # opentelemetry-exporter-otlp-proto-grpc + # opentelemetry-exporter-otlp-proto-http +opentelemetry-sdk==1.19.0 + # via + # -r otel-requirements.in + # opentelemetry-exporter-otlp-proto-grpc + # opentelemetry-exporter-otlp-proto-http + # otel-extensions +opentelemetry-semantic-conventions==0.40b0 + # via + # opentelemetry-instrumentation-requests + # opentelemetry-sdk +opentelemetry-util-http==0.40b0 + # via opentelemetry-instrumentation-requests osbs-client @ git+https://github.com/containerbuildsystem/osbs-client@2bd03f4e0e5edc474b6236c5c128620d988f79a3 # via -r requirements.in -packaging==21.2 - # via pytest +otel-extensions==0.2.5 + # via -r otel-requirements.in +packaging==23.1 + # via + # hatchling + # pytest paramiko==2.10.3 # via -r requirements.in +pathspec==0.11.2 + # via hatchling pep8==1.7.1 # via -r requirements-devel.in pluggy==1.0.0 - # via pytest + # via + # hatchling + # pytest +protobuf==4.24.2 + # via + # googleapis-common-protos + # opentelemetry-proto py==1.10.0 # via # pytest @@ -82,6 +156,8 @@ pycodestyle==2.8.0 # via flake8 pycparser==2.20 # via cffi +pydantic==1.10.12 + # via otel-extensions pyflakes==2.4.0 # via # -r requirements-devel.in @@ -90,8 +166,6 @@ pygobject==3.42.0 # via -r requirements.in pynacl==1.5.0 # via paramiko -pyparsing==2.4.7 - # via packaging pypng==0.0.21 # via -r tests/requirements.in pyrsistent==0.18.0 @@ -130,6 +204,7 @@ requests==2.26.0 # via # -r requirements.in # koji + # opentelemetry-exporter-otlp-proto-http # osbs-client # requests-gssapi # requests-kerberos @@ -160,6 +235,26 @@ six==1.16.0 toml==0.10.2 # via pytest tomli==1.2.2 - # via coverage + # via + # coverage + # hatchling +trove-classifiers==2023.8.7 + # via hatchling +typing-extensions==4.7.1 + # via + # opentelemetry-sdk + # pydantic urllib3==1.26.7 - # via requests + # via + # -r otel-requirements.in + # -r requirements.in + # requests +wrapt==1.15.0 + # via + # deprecated + # opentelemetry-instrumentation +zipp==3.16.2 + # via importlib-metadata + +# The following packages are considered to be unsafe in a requirements file: +# setuptools diff --git a/requirements.in b/requirements.in index 8769f9092..115d331d3 100644 --- a/requirements.in +++ b/requirements.in @@ -12,3 +12,4 @@ requests koji PyGObject reflink +urllib3 < 2.0 diff --git a/test.sh b/test.sh index bdc856fd1..ffbff4fb2 100755 --- a/test.sh +++ b/test.sh @@ -5,6 +5,7 @@ set -eux ENGINE=${ENGINE:="podman"} OS=${OS:="centos"} OS_VERSION=${OS_VERSION:="8"} +OTEL_ENABLED=${OTEL_ENABLED:=true} PYTHON_VERSION=${PYTHON_VERSION:="3.8"} ACTION=${ACTION:="test"} CONTAINER_NAME="atomic-reactor-$OS-$OS_VERSION-py$PYTHON_VERSION" @@ -84,6 +85,10 @@ function setup_osbs() { # avoid conflict with PyGoObject on fedora $RUN "${PIP_INST[@]}" -r requirements.txt --ignore-installed + if [[ $OTEL_ENABLED == true ]]; then + $RUN "${PIP_INST[@]}" -r otel-requirements.txt --ignore-installed + fi + # Setuptools install atomic-reactor from source $RUN $PYTHON setup.py install