diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5e9d8fba8a..982cfd4c3e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -601,6 +601,16 @@ jobs: run: echo "INFRAHUB_DB_BACKUP_PORT=$(shuf -n 1 -i 10000-30000)" >> $GITHUB_ENV - name: Select vmagent port run: echo "VMAGENT_PORT=$(shuf -n 1 -i 10000-30000)" >> $GITHUB_ENV + + - name: Enable tracing + run: echo "INFRAHUB_TRACE_ENABLE=true" >> $GITHUB_ENV + - name: Set tracing configuration + run: echo "INFRAHUB_TRACE_INSECURE=false" >> $GITHUB_ENV + - name: Set tracing configuration + run: echo "INFRAHUB_TRACE_EXPORTER_ENDPOINT=${{ secrets.TRACING_ENDPOINT }}" >> $GITHUB_ENV + - name: Set tracing configuration + run: echo "OTEL_RESOURCE_ATTRIBUTES=github.run_id=${GITHUB_RUN_ID}" >> $GITHUB_ENV + - name: "Store start time" run: echo TEST_START_TIME=$(date +%s)000 >> $GITHUB_ENV diff --git a/backend/infrahub/cli/git_agent.py b/backend/infrahub/cli/git_agent.py index e35d6673eb..2afc44ec25 100644 --- a/backend/infrahub/cli/git_agent.py +++ b/backend/infrahub/cli/git_agent.py @@ -73,7 +73,7 @@ async def _start(debug: bool, port: int) -> None: service="infrahub-git-agent", version=__version__, exporter_type=config.SETTINGS.trace.exporter_type, - exporter_endpoint=config.SETTINGS.trace.trace_endpoint, + exporter_endpoint=config.SETTINGS.trace.exporter_endpoint, exporter_protocol=config.SETTINGS.trace.exporter_protocol, ) diff --git a/backend/infrahub/config.py b/backend/infrahub/config.py index a968f56d1d..ea0c6d1c17 100644 --- a/backend/infrahub/config.py +++ b/backend/infrahub/config.py @@ -279,35 +279,6 @@ class TraceSettings(BaseSettings): default=TraceTransportProtocol.GRPC, description="Protocol to be used for exporting traces" ) exporter_endpoint: Optional[str] = Field(default=None, description="OTLP endpoint for exporting traces") - exporter_port: Optional[int] = Field( - default=None, ge=1, le=65535, description="Specified if running on a non default port (4317)" - ) - - @property - def service_port(self) -> int: - if self.exporter_protocol == TraceTransportProtocol.GRPC: - default_port = 4317 - elif self.exporter_protocol == TraceTransportProtocol.HTTP_PROTOBUF: - default_port = 4318 - else: - default_port = 4317 - - return self.exporter_port or default_port - - @property - def trace_endpoint(self) -> Optional[str]: - if not self.exporter_endpoint: - return None - if self.insecure: - scheme = "http://" - else: - scheme = "https://" - endpoint = str(self.exporter_endpoint) + ":" + str(self.service_port) - - if self.exporter_protocol == TraceTransportProtocol.HTTP_PROTOBUF: - endpoint += "/v1/traces" - - return scheme + endpoint @dataclass diff --git a/backend/infrahub/server.py b/backend/infrahub/server.py index 30dfe2361e..5b98f7641c 100644 --- a/backend/infrahub/server.py +++ b/backend/infrahub/server.py @@ -47,7 +47,7 @@ async def app_initialization(application: FastAPI) -> None: service="infrahub-server", version=__version__, exporter_type=config.SETTINGS.trace.exporter_type, - exporter_endpoint=config.SETTINGS.trace.trace_endpoint, + exporter_endpoint=config.SETTINGS.trace.exporter_endpoint, exporter_protocol=config.SETTINGS.trace.exporter_protocol, ) diff --git a/backend/infrahub/trace.py b/backend/infrahub/trace.py index 3a8588a43d..c332380b61 100644 --- a/backend/infrahub/trace.py +++ b/backend/infrahub/trace.py @@ -1,3 +1,5 @@ +import os + from opentelemetry import trace from opentelemetry.exporter.otlp.proto.grpc.trace_exporter import ( OTLPSpanExporter as GRPCSpanExporter, @@ -10,36 +12,13 @@ from opentelemetry.sdk.trace.export import BatchSpanProcessor, ConsoleSpanExporter from opentelemetry.trace import StatusCode +from infrahub.worker import WORKER_IDENTITY + def get_current_span_with_context() -> trace.Span: return trace.get_current_span() -def create_tracer_provider( - service: str, version: str, exporter_type: str, exporter_endpoint: str = None, exporter_protocol: str = None -) -> TracerProvider: - # Create a BatchSpanProcessor exporter based on the type - if exporter_type == "console": - exporter = ConsoleSpanExporter() - elif exporter_type == "otlp": - if not exporter_endpoint: - raise ValueError("Exporter type is set to otlp but endpoint is not set") - if exporter_protocol == "http/protobuf": - exporter = HTTPSpanExporter(endpoint=exporter_endpoint) - elif exporter_protocol == "grpc": - exporter = GRPCSpanExporter(endpoint=exporter_endpoint) - else: - raise ValueError("Exporter type unsupported by Infrahub") - - # Resource can be required for some backends, e.g. Jaeger - resource = Resource(attributes={"service.name": service, "service.version": version}) - span_processor = BatchSpanProcessor(exporter) - tracer_provider = TracerProvider(resource=resource) - tracer_provider.add_span_processor(span_processor) - - return tracer_provider - - def get_traceid() -> str: current_span = get_current_span_with_context() trace_id = current_span.get_span_context().trace_id @@ -75,6 +54,42 @@ def add_span_exception(exception: Exception) -> None: current_span.record_exception(exception) +def create_tracer_provider( + service: str, version: str, exporter_type: str, exporter_endpoint: str = None, exporter_protocol: str = None +) -> TracerProvider: + # Create a BatchSpanProcessor exporter based on the type + if exporter_type == "console": + exporter = ConsoleSpanExporter() + elif exporter_type == "otlp": + if not exporter_endpoint: + raise ValueError("Exporter type is set to otlp but endpoint is not set") + if exporter_protocol == "http/protobuf": + exporter = HTTPSpanExporter(endpoint=exporter_endpoint) + elif exporter_protocol == "grpc": + exporter = GRPCSpanExporter(endpoint=exporter_endpoint) + else: + raise ValueError("Exporter type unsupported by Infrahub") + + extra_attributes = {} + if os.getenv("OTEL_RESOURCE_ATTRIBUTES"): + extra_attributes = dict(attr.split("=") for attr in os.getenv("OTEL_RESOURCE_ATTRIBUTES").split(",")) + + # Resource can be required for some backends, e.g. Jaeger + resource = Resource( + attributes={ + "service.name": service, + "service.version": version, + "worker.id": WORKER_IDENTITY, + **extra_attributes, + } + ) + span_processor = BatchSpanProcessor(exporter) + tracer_provider = TracerProvider(resource=resource) + tracer_provider.add_span_processor(span_processor) + + return tracer_provider + + def configure_trace( service: str, version: str, exporter_type: str, exporter_endpoint: str | None = None, exporter_protocol: str = None ) -> None: diff --git a/development/docker-compose.yml b/development/docker-compose.yml index 2281185917..5a30ae3526 100644 --- a/development/docker-compose.yml +++ b/development/docker-compose.yml @@ -24,6 +24,11 @@ services: - "INFRAHUB_SECURITY_INITIAL_ADMIN_TOKEN=06438eb2-8019-4776-878c-0941b1f1d1ec" - "INFRAHUB_SECURITY_SECRET_KEY=327f747f-efac-42be-9e73-999f08f86b92" - "INFRAHUB_ALLOW_ANONYMOUS_ACCESS=true" + - "INFRAHUB_TRACE_ENABLE=${INFRAHUB_TRACE_ENABLE:-false}" + - "INFRAHUB_TRACE_INSECURE=${INFRAHUB_TRACE_INSECURE:-true}" + - "INFRAHUB_TRACE_EXPORTER_TYPE=${INFRAHUB_TRACE_EXPORTER_TYPE:-otlp}" + - "INFRAHUB_TRACE_EXPORTER_ENDPOINT=${INFRAHUB_TRACE_EXPORTER_ENDPOINT:-http://jaeger:4317}" + - "OTEL_RESOURCE_ATTRIBUTES=${OTEL_RESOURCE_ATTRIBUTES:-}" - "INFRAHUB_DB_TYPE=${INFRAHUB_DB_TYPE}" volumes: - "storage_data:/opt/infrahub/storage" @@ -58,6 +63,11 @@ services: - "INFRAHUB_LOG_LEVEL=DEBUG" - "INFRAHUB_SDK_API_TOKEN=06438eb2-8019-4776-878c-0941b1f1d1ec" - "INFRAHUB_SDK_TIMEOUT=20" + - "INFRAHUB_TRACE_ENABLE=${INFRAHUB_TRACE_ENABLE:-false}" + - "INFRAHUB_TRACE_INSECURE=${INFRAHUB_TRACE_INSECURE:-true}" + - "INFRAHUB_TRACE_EXPORTER_TYPE=${INFRAHUB_TRACE_EXPORTER_TYPE:-otlp}" + - "INFRAHUB_TRACE_EXPORTER_ENDPOINT=${INFRAHUB_TRACE_EXPORTER_ENDPOINT:-http://jaeger:4317}" + - "OTEL_RESOURCE_ATTRIBUTES=${OTEL_RESOURCE_ATTRIBUTES:-}" - "INFRAHUB_DB_TYPE=${INFRAHUB_DB_TYPE}" volumes: - "git_data:/opt/infrahub/git" diff --git a/development/infrahub.toml b/development/infrahub.toml index c8aec827fc..3a22ebe91a 100644 --- a/development/infrahub.toml +++ b/development/infrahub.toml @@ -25,14 +25,5 @@ driver = "local" [storage.local] path = "/opt/infrahub/storage" -[trace] -enable = false -insecure = "True" -exporter_type = "otlp" -exporter_protocol = "grpc" -exporter_endpoint = "jaeger" -exporter_port = 4317 - - # [experimental_features] # pull_request = true diff --git a/pyproject.toml b/pyproject.toml index 355498f9e6..e890f5c16f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -366,10 +366,6 @@ ignore_errors = true module = "infrahub.message_bus.operations.*" ignore_errors = true -[[tool.mypy.overrides]] -module = "infrahub.server" -ignore_errors = true - [[tool.mypy.overrides]] module = "infrahub.tasks.registry" ignore_errors = true