feat(ci): enable tracing during e2e tests

Signed-off-by: Fatih Acar <fatih@opsmill.com>

.github/workflows/ci.yml

@@ -606,6 +606,16 @@ jobs:
         run: echo "INFRAHUB_DB_BACKUP_PORT=$(shuf -n 1 -i 10000-30000)" >> $GITHUB_ENV
       - name: Select vmagent port
         run: echo "VMAGENT_PORT=$(shuf -n 1 -i 10000-30000)" >> $GITHUB_ENV
+
+      - name: Enable tracing
+        run: echo "INFRAHUB_TRACE_ENABLE=true" >> $GITHUB_ENV
+      - name: Set tracing configuration
+        run: echo "INFRAHUB_TRACE_INSECURE=false" >> $GITHUB_ENV
+      - name: Set tracing configuration
+        run: echo "INFRAHUB_TRACE_EXPORTER_ENDPOINT=${{ secrets.TRACING_ENDPOINT }}" >> $GITHUB_ENV
+      - name: Set tracing configuration
+        run: echo "OTEL_RESOURCE_ATTRIBUTES=github.run_id=${GITHUB_RUN_ID}" >> $GITHUB_ENV
+
       - name: "Store start time"
         run: echo TEST_START_TIME=$(date +%s)000 >> $GITHUB_ENV
 

backend/infrahub/cli/git_agent.py

@@ -73,7 +73,7 @@ async def _start(debug: bool, port: int) -> None:
             service="infrahub-git-agent",
             version=__version__,
             exporter_type=config.SETTINGS.trace.exporter_type,
-            exporter_endpoint=config.SETTINGS.trace.trace_endpoint,
+            exporter_endpoint=config.SETTINGS.trace.exporter_endpoint,
             exporter_protocol=config.SETTINGS.trace.exporter_protocol,
         )
 

backend/infrahub/config.py

@@ -279,35 +279,6 @@ class TraceSettings(BaseSettings):
         default=TraceTransportProtocol.GRPC, description="Protocol to be used for exporting traces"
     )
     exporter_endpoint: Optional[str] = Field(default=None, description="OTLP endpoint for exporting traces")
-    exporter_port: Optional[int] = Field(
-        default=None, ge=1, le=65535, description="Specified if running on a non default port (4317)"
-    )
-
-    @property
-    def service_port(self) -> int:
-        if self.exporter_protocol == TraceTransportProtocol.GRPC:
-            default_port = 4317
-        elif self.exporter_protocol == TraceTransportProtocol.HTTP_PROTOBUF:
-            default_port = 4318
-        else:
-            default_port = 4317
-
-        return self.exporter_port or default_port
-
-    @property
-    def trace_endpoint(self) -> Optional[str]:
-        if not self.exporter_endpoint:
-            return None
-        if self.insecure:
-            scheme = "http://"
-        else:
-            scheme = "https://"
-        endpoint = str(self.exporter_endpoint) + ":" + str(self.service_port)
-
-        if self.exporter_protocol == TraceTransportProtocol.HTTP_PROTOBUF:
-            endpoint += "/v1/traces"
-
-        return scheme + endpoint
 
 
 @dataclass

backend/infrahub/server.py

@@ -47,7 +47,7 @@ async def app_initialization(application: FastAPI) -> None:
             service="infrahub-server",
             version=__version__,
             exporter_type=config.SETTINGS.trace.exporter_type,
-            exporter_endpoint=config.SETTINGS.trace.trace_endpoint,
+            exporter_endpoint=config.SETTINGS.trace.exporter_endpoint,
             exporter_protocol=config.SETTINGS.trace.exporter_protocol,
         )
 

backend/infrahub/trace.py

@@ -1,3 +1,5 @@
+import os
+
 from opentelemetry import trace
 from opentelemetry.exporter.otlp.proto.grpc.trace_exporter import (
     OTLPSpanExporter as GRPCSpanExporter,
@@ -10,36 +12,13 @@
 from opentelemetry.sdk.trace.export import BatchSpanProcessor, ConsoleSpanExporter
 from opentelemetry.trace import StatusCode
 
+from infrahub.worker import WORKER_IDENTITY
+
 
 def get_current_span_with_context() -> trace.Span:
     return trace.get_current_span()
 
 
-def create_tracer_provider(
-    service: str, version: str, exporter_type: str, exporter_endpoint: str = None, exporter_protocol: str = None
-) -> TracerProvider:
-    # Create a BatchSpanProcessor exporter based on the type
-    if exporter_type == "console":
-        exporter = ConsoleSpanExporter()
-    elif exporter_type == "otlp":
-        if not exporter_endpoint:
-            raise ValueError("Exporter type is set to otlp but endpoint is not set")
-        if exporter_protocol == "http/protobuf":
-            exporter = HTTPSpanExporter(endpoint=exporter_endpoint)
-        elif exporter_protocol == "grpc":
-            exporter = GRPCSpanExporter(endpoint=exporter_endpoint)
-    else:
-        raise ValueError("Exporter type unsupported by Infrahub")
-
-    # Resource can be required for some backends, e.g. Jaeger
-    resource = Resource(attributes={"service.name": service, "service.version": version})
-    span_processor = BatchSpanProcessor(exporter)
-    tracer_provider = TracerProvider(resource=resource)
-    tracer_provider.add_span_processor(span_processor)
-
-    return tracer_provider
-
-
 def get_traceid() -> str:
     current_span = get_current_span_with_context()
     trace_id = current_span.get_span_context().trace_id
@@ -75,6 +54,42 @@ def add_span_exception(exception: Exception) -> None:
         current_span.record_exception(exception)
 
 
+def create_tracer_provider(
+    service: str, version: str, exporter_type: str, exporter_endpoint: str = None, exporter_protocol: str = None
+) -> TracerProvider:
+    # Create a BatchSpanProcessor exporter based on the type
+    if exporter_type == "console":
+        exporter = ConsoleSpanExporter()
+    elif exporter_type == "otlp":
+        if not exporter_endpoint:
+            raise ValueError("Exporter type is set to otlp but endpoint is not set")
+        if exporter_protocol == "http/protobuf":
+            exporter = HTTPSpanExporter(endpoint=exporter_endpoint)
+        elif exporter_protocol == "grpc":
+            exporter = GRPCSpanExporter(endpoint=exporter_endpoint)
+    else:
+        raise ValueError("Exporter type unsupported by Infrahub")
+
+    extra_attributes = {}
+    if os.getenv("OTEL_RESOURCE_ATTRIBUTES"):
+        extra_attributes = dict(attr.split("=") for attr in os.getenv("OTEL_RESOURCE_ATTRIBUTES").split(","))
+
+    # Resource can be required for some backends, e.g. Jaeger
+    resource = Resource(
+        attributes={
+            "service.name": service,
+            "service.version": version,
+            "worker.id": WORKER_IDENTITY,
+            **extra_attributes,
+        }
+    )
+    span_processor = BatchSpanProcessor(exporter)
+    tracer_provider = TracerProvider(resource=resource)
+    tracer_provider.add_span_processor(span_processor)
+
+    return tracer_provider
+
+
 def configure_trace(
     service: str, version: str, exporter_type: str, exporter_endpoint: str | None = None, exporter_protocol: str = None
 ) -> None:

development/docker-compose.yml

@@ -24,6 +24,11 @@ services:
       - "INFRAHUB_SECURITY_INITIAL_ADMIN_TOKEN=06438eb2-8019-4776-878c-0941b1f1d1ec"
       - "INFRAHUB_SECURITY_SECRET_KEY=327f747f-efac-42be-9e73-999f08f86b92"
       - "INFRAHUB_ALLOW_ANONYMOUS_ACCESS=true"
+      - "INFRAHUB_TRACE_ENABLE=${INFRAHUB_TRACE_ENABLE:-false}"
+      - "INFRAHUB_TRACE_INSECURE=${INFRAHUB_TRACE_INSECURE:-true}"
+      - "INFRAHUB_TRACE_EXPORTER_TYPE=${INFRAHUB_TRACE_EXPORTER_TYPE:-otlp}"
+      - "INFRAHUB_TRACE_EXPORTER_ENDPOINT=${INFRAHUB_TRACE_EXPORTER_ENDPOINT:-http://jaeger:4317}"
+      - "OTEL_RESOURCE_ATTRIBUTES=${OTEL_RESOURCE_ATTRIBUTES:-}"
       - "INFRAHUB_DB_TYPE=${INFRAHUB_DB_TYPE}"
     volumes:
       - "storage_data:/opt/infrahub/storage"
@@ -58,6 +63,11 @@ services:
       - "INFRAHUB_LOG_LEVEL=DEBUG"
       - "INFRAHUB_SDK_API_TOKEN=06438eb2-8019-4776-878c-0941b1f1d1ec"
       - "INFRAHUB_SDK_TIMEOUT=20"
+      - "INFRAHUB_TRACE_ENABLE=${INFRAHUB_TRACE_ENABLE:-false}"
+      - "INFRAHUB_TRACE_INSECURE=${INFRAHUB_TRACE_INSECURE:-true}"
+      - "INFRAHUB_TRACE_EXPORTER_TYPE=${INFRAHUB_TRACE_EXPORTER_TYPE:-otlp}"
+      - "INFRAHUB_TRACE_EXPORTER_ENDPOINT=${INFRAHUB_TRACE_EXPORTER_ENDPOINT:-http://jaeger:4317}"
+      - "OTEL_RESOURCE_ATTRIBUTES=${OTEL_RESOURCE_ATTRIBUTES:-}"
       - "INFRAHUB_DB_TYPE=${INFRAHUB_DB_TYPE}"
     volumes:
       - "git_data:/opt/infrahub/git"

development/infrahub.toml

@@ -25,14 +25,5 @@ driver = "local"
 [storage.local]
 path = "/opt/infrahub/storage"
 
-[trace]
-enable = false
-insecure = "True"
-exporter_type = "otlp"
-exporter_protocol = "grpc"
-exporter_endpoint = "jaeger"
-exporter_port = 4317
-
-
 # [experimental_features]
 # pull_request = true

pyproject.toml

@@ -380,10 +380,6 @@ ignore_errors = true
 module = "infrahub.message_bus.operations.*"
 ignore_errors = true
 
-[[tool.mypy.overrides]]
-module = "infrahub.server"
-ignore_errors = true
-
 [[tool.mypy.overrides]]
 module = "infrahub.tasks.registry"
 ignore_errors = true