Weird opentelemetry-collector errors

[ghevge]

I'm trying to set up opentelemetry-collector with prometheus, tempo and loki on a dockerize jave springboot 3 application.

I've managed to start all the containers, but once I generet some activity on my app, otel collector start to generate errors like bellow:

otel-collector_1  | 2023-08-30T15:24:45.452Z    error   prometheusexporter@v0.82.0/log.go:23    error gathering metrics: 5 error(s) occurred:
otel-collector_1  | * collected metric log4j2_events label:{name:"application" value:"testapp"} label:{name:"job" value:"unknown_service"} label:{name:"label1" value:"value1"} label:{name:"level" value:"debug"} counter:{value:599} has help "Number of debug level log events" but should have "Number of trace level log events"
otel-collector_1  | * collected metric log4j2_events label:{name:"application" value:"testapp"} label:{name:"job" value:"unknown_service"} label:{name:"label1" value:"value1"} label:{name:"level" value:"warn"} counter:{value:2} has help "Number of warn level log events" but should have "Number of trace level log events"
otel-collector_1  | * collected metric log4j2_events label:{name:"application" value:"testapp"} label:{name:"job" value:"unknown_service"} label:{name:"label1" value:"value1"} label:{name:"level" value:"fatal"} counter:{value:0} has help "Number of fatal level log events" but should have "Number of trace level log events"
otel-collector_1  | * collected metric log4j2_events label:{name:"application" value:"testapp"} label:{name:"job" value:"unknown_service"} label:{name:"label1" value:"value1"} label:{name:"level" value:"error"} counter:{value:0} has help "Number of error level log events" but should have "Number of trace level log events"
otel-collector_1  | * collected metric log4j2_events label:{name:"application" value:"testapp"} label:{name:"job" value:"unknown_service"} label:{name:"label1" value:"value1"} label:{name:"level" value:"info"} counter:{value:12} has help "Number of info level log events" but should have "Number of trace level log events"
otel-collector_1  |     {"kind": "exporter", "data_type": "metrics", "name": "prometheus"}
otel-collector_1  | github.com/open-telemetry/opentelemetry-collector-contrib/exporter/prometheusexporter.(*promLogger).Println
otel-collector_1  |     github.com/open-telemetry/opentelemetry-collector-contrib/exporter/prometheusexporter@v0.82.0/log.go:23
otel-collector_1  | github.com/prometheus/client_golang/prometheus/promhttp.HandlerForTransactional.func1
otel-collector_1  |     github.com/prometheus/client_golang@v1.16.0/prometheus/promhttp/http.go:144
otel-collector_1  | net/http.HandlerFunc.ServeHTTP
otel-collector_1  |     net/http/server.go:2122
otel-collector_1  | net/http.(*ServeMux).ServeHTTP
otel-collector_1  |     net/http/server.go:2500
otel-collector_1  | go.opentelemetry.io/collector/config/confighttp.(*decompressor).ServeHTTP
otel-collector_1  |     go.opentelemetry.io/collector/config/confighttp@v0.82.0/compression.go:147
otel-collector_1  | go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp.(*Handler).ServeHTTP
otel-collector_1  |     go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp@v0.42.0/handler.go:212
otel-collector_1  | go.opentelemetry.io/collector/config/confighttp.(*clientInfoHandler).ServeHTTP
otel-collector_1  |     go.opentelemetry.io/collector/config/confighttp@v0.82.0/clientinfohandler.go:28
otel-collector_1  | net/http.serverHandler.ServeHTTP
otel-collector_1  |     net/http/server.go:2936
otel-collector_1  | net/http.(*conn).serve
otel-collector_1  |     net/http/server.go:1995

Any idea what could be causing this?

My docker-compose looks something like this:

........
version: '3.8'

services:
 otel-collector:
    image: otel/opentelemetry-collector-contrib:0.82.0
    restart: always
    command:
      - --config=/etc/otelcol-contrib/otel-collector.yml
    volumes:
      - /data/configs/otel-collector.yml:/etc/otelcol-contrib/otel-collector.yml
    ports:
      - "1888:1888" # pprof extension
      - "8888:8888" # Prometheus metrics exposed by the collector
      - "8889:8889" # Prometheus exporter metrics
      - "13133:13133" # health_check extension
      - "4317:4317" # OTLP gRPC receiver
      - "4318:4318" # OTLP http receiver
      - "55679:55679" # zpages extension
    networks:
      mw-network:
        aliases:
          - otel-collector

  prometheus:
    container_name: prometheus
    image: prom/prometheus
    restart: always
    command:
      - --config.file=/etc/prometheus/prometheus.yml
    ports:
      - "9090:9090"
    volumes:
      - /data/configs/prometheus.yml:/etc/prometheus/prometheus.yml
    networks:
      mw-network:
        aliases:
          - prometheus

  loki:
    image: grafana/loki:latest
    ports:
      - 3100:3100
    command: -config.file=/etc/loki/local-config.yaml
    networks:
      mw-network:
        aliases:
          - loki

  tempo:
    image: grafana/tempo:latest
    command: [ "-config.file=/etc/tempo.yml" ]
    volumes:
      - /data/configs/tempo.yml:/etc/tempo.yml
    ports:
      - "3200:3200"   # tempo
      - "4317"  # otlp grpc
    networks:
      mw-network:
        aliases:
          - tempo

  grafana:
    image: grafana/grafana-enterprise
    container_name: grafana
    restart: unless-stopped
    ports:
      - "3000:3000"
    volumes:
      - /data/grafana:/var/lib/grafana 
..........

Otel config file looks like this:

receivers:
  otlp:
    protocols:
      http:
      grpc:

processors:
  # batch metrics before sending to reduce API usage
  batch:

exporters:
  logging:
    loglevel: debug

  loki:
    endpoint: "http://loki:3100/loki/api/v1/push"

  prometheus:
    endpoint: "0.0.0.0:8889"
    const_labels:
      label1: value1

  otlp:
    endpoint: tempo:4317
    tls:
      insecure: true

# https://github.com/open-telemetry/opentelemetry-collector/blob/main/extension/README.md
extensions:
  # responsible for responding to health check calls on behalf of the collector.
  health_check:
  # fetches the collector’s performance data
  pprof:
  # serves as an http endpoint that provides live debugging data about instrumented components.
  zpages:

service:
  extensions: [health_check, pprof, zpages]
  pipelines:
    metrics:
      receivers: [otlp]
      processors: [batch]
      exporters: [prometheus]
    traces:
      receivers: [otlp]
      processors: [batch]
      exporters: [otlp]
    logs:
      receivers: [otlp]
      exporters: [loki]

Java config class:

@Configuration
public class OpenTelemetryConfig {
    @Bean
    OpenTelemetry openTelemetry(SdkLoggerProvider sdkLoggerProvider, SdkTracerProvider sdkTracerProvider, ContextPropagators contextPropagators) {
        OpenTelemetrySdk openTelemetrySdk = OpenTelemetrySdk.builder()
                .setLoggerProvider(sdkLoggerProvider)
                .setTracerProvider(sdkTracerProvider)
                .setPropagators(contextPropagators)
                .build();
        OpenTelemetryAppender.install(openTelemetrySdk);
        return openTelemetrySdk;
    }

    @Bean
    SdkLoggerProvider otelSdkLoggerProvider(Environment environment, ObjectProvider<LogRecordProcessor> logRecordProcessors) {
        String applicationName = environment.getProperty("spring.application.name", "application");
        Resource springResource = Resource.create(Attributes.of(ResourceAttributes.SERVICE_NAME, applicationName));
        SdkLoggerProviderBuilder builder = SdkLoggerProvider.builder()
                .setResource(Resource.getDefault().merge(springResource));
        logRecordProcessors.orderedStream().forEach(builder::addLogRecordProcessor);
        return builder.build();
    }

    @Bean
    LogRecordProcessor otelLogRecordProcessor() {
        return BatchLogRecordProcessor
                .builder(OtlpGrpcLogRecordExporter.builder()
                                                  .setEndpoint("http://otel-collector:4317")
                                                  .build())
                .build();
    }
}

application.yaml:

logging:
  config: classpath:/log4j2-spring.xml

management:
  endpoints:
    web:
      exposure:
        include: metrics
  otlp:
    metrics:
      export:
        url: http://otel-collector:4318/v1/metrics
        step: 10s
    tracing:
      endpoint: http://otel-collector:4318/v1/traces
  tracing:
    sampling:
      probability: 1.0

spring:
  application:
    name: kelteu
...............

[atoulme]

Closing as duplicate of open-telemetry/opentelemetry-collector-contrib#26343