Add custom JMX metric collection

apps/rest-services/java/calendar/README.md

@@ -78,3 +78,7 @@ Install calendar in K8s with DD SDK
 ```
 helm install -n otel-ingest calendar-dd-java ./calendar-dd/k8s/ --set image.repository=dineshgurumurthydd/calendar-java --set image.tag=otel-0.1,node_group=ng-1
 ```
+
+## JMX metrics
+
+The `jmx_metrics_config.yaml` file provides a detailed example of configuring custom JMX metrics collection using the OpenTelemetry Java agent.

apps/rest-services/java/calendar/deploys/Dockerfile.otel

@@ -8,11 +8,19 @@ RUN apt-get update -y; apt-get install curl -y
 WORKDIR /home/otel
 RUN curl -Lo opentelemetry-javaagent.jar  https://github.com/open-telemetry/opentelemetry-java-instrumentation/releases/download/v2.4.0/opentelemetry-javaagent.jar
 COPY . calendar/
+COPY ./jmx_metrics_config.yaml ./jmx_metrics_config.yaml
 
 WORKDIR /home/otel/calendar
 
 #Compile with gradle
 RUN ./gradlew build
 
-ENTRYPOINT ["java","-javaagent:../opentelemetry-javaagent.jar",  "-jar" , "build/libs/calendar-0.0.1-SNAPSHOT.jar"]
+ENV JAVA_OPTS="-Dcom.sun.management.jmxremote \
+               -Dcom.sun.management.jmxremote.port=9092 \
+               -Dcom.sun.management.jmxremote.authenticate=false \
+               -Dcom.sun.management.jmxremote.ssl=false \
+               -Dcom.sun.management.jmxremote.rmi.port=9093\
+               -Djava.rmi.server.hostname=127.0.0.1"
+ENTRYPOINT exec java ${JAVA_OPTS} -javaagent:../opentelemetry-javaagent.jar -Dotel.jmx.config=./jmx_metrics_config.yaml -jar build/libs/calendar-0.0.1-SNAPSHOT.jar
+
 EXPOSE 8080

apps/rest-services/java/calendar/deploys/docker/docker-compose-otel.yml

@@ -20,6 +20,9 @@ services:
       - OTEL_EXPORTER_OTLP_PROTOCOL=grpc
     ports:
       - "9090:9090"
+      - "9092:9092"
+      - "9093:9093"
+
   # open-telemetry collector
   otelcol:
     image: otel/opentelemetry-collector-contrib:latest

apps/rest-services/java/calendar/jmx_metrics_config.yaml

@@ -0,0 +1,14 @@
+rules:
+  - bean: com.otel.main:type=Calendar
+    mapping:
+      HitsCount:
+        metric: my.jmx-calendar.hits.count
+        type: counter
+        desc: "The number of hits to the calendar"
+        unit: "1"
+      RequestLatency:
+        metric: my.jmx-calendar.request.latency
+        type: gauge
+        desc: "The average request latency to the calendar"
+        unit: "ns"
+

apps/rest-services/java/calendar/run-otel-local.sh

@@ -6,6 +6,12 @@ export OTEL_METRICS_EXPORTER=otlp
 export OTEL_LOGS_EXPORTER=otlp
 export OTEL_EXPORTER_OTLP_ENDPOINT=http://localhost:4317
 export OTEL_EXPORTER_OTLP_PROTOCOL=grpc
+export JAVA_OPTS="-Dcom.sun.management.jmxremote \
+               -Dcom.sun.management.jmxremote.port=9092 \
+               -Dcom.sun.management.jmxremote.authenticate=false \
+               -Dcom.sun.management.jmxremote.ssl=false \
+               -Dcom.sun.management.jmxremote.rmi.port=9093\
+               -Djava.rmi.server.hostname=127.0.0.1"
 
 # Define the path to the Java agent JAR
 JAVA_AGENT_JAR="$SCRIPT_DIR/opentelemetry-javaagent.jar"
@@ -21,4 +27,6 @@ fi
 export OTEL_RESOURCE_ATTRIBUTES=service.name=my-calendar-service,deployment.environment=otel-test,host.name=calendar-host
 
 java -javaagent:$JAVA_AGENT_JAR \
+    ${JAVA_OPTS} \
+    "-Dotel.jmx.config=./jmx_metrics_config.yaml" \
 	-jar $SCRIPT_DIR/build/libs/calendar-0.0.1-SNAPSHOT.jar

apps/rest-services/java/calendar/src/main/java/com/otel/controller/CalendarController.java

@@ -19,50 +19,65 @@
 import java.util.Map;
 import java.util.Random;
 import java.util.concurrent.atomic.AtomicLong;
+import javax.management.MBeanServer;
+import javax.management.MBeanServerInvocationHandler;
+import javax.management.ObjectName;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.util.MultiValueMap;
 import org.springframework.web.bind.annotation.GetMapping;
 import org.springframework.web.bind.annotation.RequestHeader;
 import org.springframework.web.bind.annotation.RestController;
+import com.otel.main.SpringApp.CalendarMBean;
 
 @RestController
 public class CalendarController {
+
     private final Logger log = LoggerFactory.getLogger(CalendarController.class);
     private final Tracer tracer;
     private final LongCounter hitsCounter;
     private final DoubleHistogram latency;
     private final ObservableDoubleGauge activeUsersGauge;
     private final AtomicLong activeUsersCounter;
     private final Random random = new Random();
+    private final CalendarMBean calendarMBean;
 
     @Autowired
-    CalendarController(OpenTelemetry openTelemetry, String serviceName) {
-        log.info("Starting CalendarController for {}", serviceName);
+    CalendarController(OpenTelemetry openTelemetry, String serviceName, MBeanServer mBeanServer) {
         tracer = openTelemetry.getTracer(CalendarController.class.getName());
         Meter meter = openTelemetry.getMeter(CalendarController.class.getName());
         hitsCounter = meter.counterBuilder(serviceName + ".api.hits").build();
         latency = meter.histogramBuilder(serviceName + ".task.duration").build();
         activeUsersCounter = new AtomicLong();
         activeUsersGauge = meter.gaugeBuilder(serviceName + ".active.users.guage").buildWithCallback(measurement -> measurement.record(activeUsersCounter.get()));
 
-    }
+        try {
+            // Create the MBean proxy
+            ObjectName objectName = new ObjectName("com.otel.main:type=Calendar");
+            this.calendarMBean = MBeanServerInvocationHandler.newProxyInstance(mBeanServer, objectName, CalendarMBean.class, false);
+            log.info("CalendarMBean proxy initialized with instance hashcode: {}", System.identityHashCode(this.calendarMBean));
+        } catch (Exception e) {
+            throw new RuntimeException("Failed to create MBean proxy", e);
+        }
 
+        log.info("Starting CalendarController for {}", serviceName);
+    }
 
     @GetMapping("/calendar")
     public Map<String, String> getDate(@RequestHeader MultiValueMap<String, String> headers) {
         long startTime = System.currentTimeMillis();
         activeUsersCounter.incrementAndGet();
         try {
             hitsCounter.add(1);
-            String output = getDate();
-            // the correct JSON output should put this in quotes. Spring does not, so let's put quotes here
+            calendarMBean.incrementHitsCount();
+            long endTime = System.currentTimeMillis();
+            calendarMBean.addRequestLatency(endTime - startTime);
+            log.info("Generated JMX hit count: {}, latency: {}", calendarMBean.getHitsCount(), calendarMBean.getRequestLatency());String output = getDate();
+             // the correct JSON output should put this in quotes. Spring does not, so let's put quotes here
             // by hand.
             return Map.of("date", output);
         } finally {
-            long endTime = System.currentTimeMillis();
-            latency.record(endTime - startTime);
             activeUsersCounter.decrementAndGet();
         }
     }
@@ -76,8 +91,8 @@ private String getDate() {
             String output = start.format(DateTimeFormatter.ISO_LOCAL_DATE);
             span.setAttribute("date", output);
             // Add random sleep
-            Thread.sleep(random.nextLong(1,950));
-            log.info("generated date: {}", output);
+            Thread.sleep(random.nextLong(1, 950));
+            log.info("Generated date: {}", output);
             return output;
         } catch (InterruptedException e) {
             throw new RuntimeException(e);

apps/rest-services/java/calendar/src/main/java/com/otel/main/SpringApp.java

@@ -9,24 +9,90 @@ This product includes software developed at Datadog (https://www.datadoghq.com/)
 
 import io.opentelemetry.api.GlobalOpenTelemetry;
 import io.opentelemetry.api.OpenTelemetry;
-import org.springframework.beans.BeanUtils;
+import java.lang.management.ManagementFactory;
+import javax.management.MBeanServer;
+import javax.management.ObjectName;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 import org.springframework.boot.SpringApplication;
 import org.springframework.boot.autoconfigure.SpringBootApplication;
 import org.springframework.context.annotation.Bean;
+import org.springframework.context.event.ContextRefreshedEvent;
+import org.springframework.context.event.EventListener;
 
 @SpringBootApplication(scanBasePackages = "com.otel")
 public class SpringApp {
-  public static void main(String[] args) {
-    SpringApplication.run(SpringApp.class, args);
-  }
-  @Bean
-  public OpenTelemetry openTelemetry() {
-    return GlobalOpenTelemetry.get();
-  }
-
-  @Bean
-  public String serviceName() {
-    String serviceName = System.getProperty("otel.serviceName");
-    return   serviceName != null ? serviceName : "calendar";
-  }
+
+    private static final Logger log = LoggerFactory.getLogger(SpringApp.class);
+
+    public static void main(String[] args) {
+        SpringApplication.run(SpringApp.class, args);
+    }
+
+    @Bean
+    public MBeanServer mBeanServer() {
+        return ManagementFactory.getPlatformMBeanServer();
+    }
+
+    @Bean
+    public CalendarMBean calendarMBean() {
+        return new Calendar();
+    }
+
+    @EventListener(ContextRefreshedEvent.class)
+    public void registerAndInitMBean() {
+        try {
+            MBeanServer mBeanServer = mBeanServer();
+            ObjectName objectName = new ObjectName("com.otel.main:type=Calendar");
+            Calendar calendarMBean = new Calendar();
+            mBeanServer.registerMBean(calendarMBean, objectName);
+            log.info("MBean registered: {}", objectName);
+        } catch (Exception e) {
+            log.error("Error registering and initializing MBean", e);
+            throw new IllegalStateException("Failed to register and initialize CalendarMBean", e);
+        }
+    }
+
+    @Bean
+    public OpenTelemetry openTelemetry() {
+        return GlobalOpenTelemetry.get();
+    }
+
+    @Bean
+    public String serviceName() {
+        String serviceName = System.getProperty("otel.serviceName");
+        return serviceName != null ? serviceName : "calendar";
+    }
+
+    public interface CalendarMBean {
+        int getHitsCount();
+        void incrementHitsCount();
+        float getRequestLatency();
+        void addRequestLatency(float latency);
+    }
+
+    public static class Calendar implements CalendarMBean {
+        private int hitsCount = 0;
+        private float totalRequestLatency = 0;
+
+        @Override
+        public synchronized int getHitsCount() {
+            return hitsCount;
+        }
+
+        @Override
+        public synchronized void incrementHitsCount() {
+            hitsCount++;
+        }
+
+        @Override
+        public synchronized float getRequestLatency() {
+            return hitsCount == 0 ? 0 : (float) totalRequestLatency / hitsCount;
+        }
+
+        @Override
+        public synchronized void addRequestLatency(float latency) {
+            totalRequestLatency += latency;
+        }
+    }
 }