From e87e2e76989f0056149c6100685e52c074df2ac6 Mon Sep 17 00:00:00 2001 From: UTXOnly Date: Tue, 6 Aug 2024 00:08:13 -0400 Subject: [PATCH 1/8] Add custom JMX metric collection docker --- .../java/calendar/deploys/Dockerfile.otel | 12 ++- .../deploys/docker/docker-compose-otel.yml | 3 + .../java/calendar/jmx_metrics_config.yaml | 14 +++ .../otel/controller/CalendarController.java | 48 ++++++---- .../main/java/com/otel/main/SpringApp.java | 87 +++++++++++++++---- 5 files changed, 128 insertions(+), 36 deletions(-) create mode 100644 apps/rest-services/java/calendar/jmx_metrics_config.yaml diff --git a/apps/rest-services/java/calendar/deploys/Dockerfile.otel b/apps/rest-services/java/calendar/deploys/Dockerfile.otel index b04b654..4072456 100644 --- a/apps/rest-services/java/calendar/deploys/Dockerfile.otel +++ b/apps/rest-services/java/calendar/deploys/Dockerfile.otel @@ -11,8 +11,16 @@ COPY . calendar/ WORKDIR /home/otel/calendar -#Compile with gradle +COPY ./jmx_metrics_config.yaml ./jmx_metrics_config.yaml RUN ./gradlew build -ENTRYPOINT ["java","-javaagent:../opentelemetry-javaagent.jar", "-jar" , "build/libs/calendar-0.0.1-SNAPSHOT.jar"] +ENV JAVA_OPTS="-Dcom.sun.management.jmxremote \ + -Dcom.sun.management.jmxremote.port=9092 \ + -Dcom.sun.management.jmxremote.authenticate=false \ + -Dcom.sun.management.jmxremote.ssl=false \ + -Dcom.sun.management.jmxremote.rmi.port=9093\ + -Djava.rmi.server.hostname=127.0.0.1" +ENTRYPOINT exec java ${JAVA_OPTS} -javaagent:../opentelemetry-javaagent.jar -Dotel.jmx.config=./jmx_metrics_config.yaml -jar build/libs/calendar-0.0.1-SNAPSHOT.jar + +#ENTRYPOINT ["java", "-javaagent:../opentelemetry-javaagent.jar", "-Dotel.jmx.config=./jmx_metrics_config.yaml", "-jar" , "build/libs/calendar-0.0.1-SNAPSHOT.jar"] EXPOSE 8080 diff --git a/apps/rest-services/java/calendar/deploys/docker/docker-compose-otel.yml b/apps/rest-services/java/calendar/deploys/docker/docker-compose-otel.yml index d029392..4f11da4 100644 --- a/apps/rest-services/java/calendar/deploys/docker/docker-compose-otel.yml +++ b/apps/rest-services/java/calendar/deploys/docker/docker-compose-otel.yml @@ -20,6 +20,9 @@ services: - OTEL_EXPORTER_OTLP_PROTOCOL=grpc ports: - "9090:9090" + - "9092:9092" + - "9093:9093" + # open-telemetry collector otelcol: image: otel/opentelemetry-collector-contrib:latest diff --git a/apps/rest-services/java/calendar/jmx_metrics_config.yaml b/apps/rest-services/java/calendar/jmx_metrics_config.yaml new file mode 100644 index 0000000..b10e45b --- /dev/null +++ b/apps/rest-services/java/calendar/jmx_metrics_config.yaml @@ -0,0 +1,14 @@ +rules: + - bean: com.otel.main:type=Calendar + mapping: + HitsCount: + metric: my.jmx-calendar.hits.count + type: counter + desc: "The number of hits to the calendar" + unit: "1" + RequestLatency: + metric: my.jmx-calendar.request.latency + type: gauge + desc: "The average request latency to the calendar" + unit: "ns" + diff --git a/apps/rest-services/java/calendar/src/main/java/com/otel/controller/CalendarController.java b/apps/rest-services/java/calendar/src/main/java/com/otel/controller/CalendarController.java index 79f1e18..c34ccd5 100644 --- a/apps/rest-services/java/calendar/src/main/java/com/otel/controller/CalendarController.java +++ b/apps/rest-services/java/calendar/src/main/java/com/otel/controller/CalendarController.java @@ -1,7 +1,3 @@ -/* -Unless explicitly stated otherwise all files in this repository are licensed -under the Apache 2.0 License. -*/ package com.otel.controller; import io.opentelemetry.api.OpenTelemetry; @@ -13,20 +9,28 @@ import io.opentelemetry.api.trace.SpanKind; import io.opentelemetry.api.trace.Tracer; import io.opentelemetry.context.Scope; +import java.lang.management.ManagementFactory; +import javax.management.MBeanServer; +import javax.management.ObjectName; import java.time.LocalDate; import java.time.Month; import java.time.format.DateTimeFormatter; import java.util.Map; import java.util.Random; import java.util.concurrent.atomic.AtomicLong; +import javax.management.MBeanServerInvocationHandler; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.context.event.ContextRefreshedEvent; +import org.springframework.context.event.EventListener; import org.springframework.util.MultiValueMap; import org.springframework.web.bind.annotation.GetMapping; import org.springframework.web.bind.annotation.RequestHeader; import org.springframework.web.bind.annotation.RestController; +import com.otel.main.SpringApp; + @RestController public class CalendarController { private final Logger log = LoggerFactory.getLogger(CalendarController.class); @@ -36,6 +40,7 @@ public class CalendarController { private final ObservableDoubleGauge activeUsersGauge; private final AtomicLong activeUsersCounter; private final Random random = new Random(); + private SpringApp.CalendarMBean calendarMBean; @Autowired CalendarController(OpenTelemetry openTelemetry, String serviceName) { @@ -46,9 +51,20 @@ public class CalendarController { latency = meter.histogramBuilder(serviceName + ".task.duration").build(); activeUsersCounter = new AtomicLong(); activeUsersGauge = meter.gaugeBuilder(serviceName + ".active.users.guage").buildWithCallback(measurement -> measurement.record(activeUsersCounter.get())); - } - + // Initialize MBean server and ObjectName + @EventListener(ContextRefreshedEvent.class) + public void init() { + try { + MBeanServer mBeanServer = ManagementFactory.getPlatformMBeanServer(); + ObjectName objectName = new ObjectName("com.otel.main:type=Calendar"); + this.calendarMBean = MBeanServerInvocationHandler.newProxyInstance(mBeanServer, objectName, SpringApp.CalendarMBean.class, false); + log.info("CalendarMBean initialized"); + } catch (Exception e) { + log.error("Failed to initialize CalendarMBean", e); + throw new IllegalStateException("Failed to get CalendarMBean", e); + } + } @GetMapping("/calendar") public Map getDate(@RequestHeader MultiValueMap headers) { @@ -56,27 +72,29 @@ public Map getDate(@RequestHeader MultiValueMap activeUsersCounter.incrementAndGet(); try { hitsCounter.add(1); + calendarMBean.incrementHitsCount(); + + long endTime = System.currentTimeMillis(); + float latency = (endTime - startTime); + calendarMBean.addRequestLatency(latency); + + log.info("generated JMX hit count: {}, request latency: {}", calendarMBean.getHitsCount(), calendarMBean.getRequestLatency()); + String output = getDate(); - // the correct JSON output should put this in quotes. Spring does not, so let's put quotes here - // by hand. - return Map.of("date", output); + return Map.of("date", output, "latency", String.valueOf(latency)); } finally { - long endTime = System.currentTimeMillis(); - latency.record(endTime - startTime); activeUsersCounter.decrementAndGet(); } } - + private String getDate() { Span span = tracer.spanBuilder("getDate").setAttribute("peer.service", "random-date-service").setSpanKind(SpanKind.CLIENT).startSpan(); try (Scope scope = span.makeCurrent()) { - // get back a random date in the year 2022 int val = new Random().nextInt(365); LocalDate start = LocalDate.of(2022, Month.JANUARY, 1).plusDays(val); String output = start.format(DateTimeFormatter.ISO_LOCAL_DATE); span.setAttribute("date", output); - // Add random sleep - Thread.sleep(random.nextLong(1,950)); + Thread.sleep(random.nextLong(1, 950)); log.info("generated date: {}", output); return output; } catch (InterruptedException e) { diff --git a/apps/rest-services/java/calendar/src/main/java/com/otel/main/SpringApp.java b/apps/rest-services/java/calendar/src/main/java/com/otel/main/SpringApp.java index 1af65e3..0d44de4 100644 --- a/apps/rest-services/java/calendar/src/main/java/com/otel/main/SpringApp.java +++ b/apps/rest-services/java/calendar/src/main/java/com/otel/main/SpringApp.java @@ -1,32 +1,81 @@ -/* -Unless explicitly stated otherwise all files in this repository are licensed -under the Apache 2.0 License. - -This product includes software developed at Datadog (https://www.datadoghq.com/) -Copyright 2023 Datadog, Inc. - */ package com.otel.main; import io.opentelemetry.api.GlobalOpenTelemetry; import io.opentelemetry.api.OpenTelemetry; -import org.springframework.beans.BeanUtils; +import java.lang.management.ManagementFactory; +import javax.management.MBeanServer; +import javax.management.ObjectName; import org.springframework.boot.SpringApplication; import org.springframework.boot.autoconfigure.SpringBootApplication; import org.springframework.context.annotation.Bean; +import org.springframework.context.event.ContextRefreshedEvent; +import org.springframework.context.event.EventListener; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; @SpringBootApplication(scanBasePackages = "com.otel") public class SpringApp { - public static void main(String[] args) { - SpringApplication.run(SpringApp.class, args); - } - @Bean - public OpenTelemetry openTelemetry() { - return GlobalOpenTelemetry.get(); - } - @Bean - public String serviceName() { - String serviceName = System.getProperty("otel.serviceName"); - return serviceName != null ? serviceName : "calendar"; + private static final Logger log = LoggerFactory.getLogger(SpringApp.class); + + public static void main(String[] args) { + SpringApplication.run(SpringApp.class, args); + } + + @EventListener(ContextRefreshedEvent.class) + public void registerMBean() { + try { + MBeanServer mBeanServer = ManagementFactory.getPlatformMBeanServer(); + ObjectName objectName = new ObjectName("com.otel.main:type=Calendar"); + Calendar calendarMBean = new Calendar(); + mBeanServer.registerMBean(calendarMBean, objectName); + log.info("MBean registered: {}", objectName); + } catch (Exception e) { + log.error("Error registering MBean", e); + } + } + + @Bean + public OpenTelemetry openTelemetry() { + return GlobalOpenTelemetry.get(); + } + + @Bean + public String serviceName() { + String serviceName = System.getProperty("otel.serviceName"); + return serviceName != null ? serviceName : "calendar"; + } + + public interface CalendarMBean { + int getHitsCount(); + void incrementHitsCount(); + float getRequestLatency(); + void addRequestLatency(float latency); } + + public static class Calendar implements CalendarMBean { + private int hitsCount = 0; + private long totalRequestLatency = 0; + + @Override + public synchronized int getHitsCount() { + return hitsCount; + } + + @Override + public synchronized void incrementHitsCount() { + hitsCount++; + } + + @Override + public synchronized float getRequestLatency() { + return hitsCount == 0 ? 0 : (float) totalRequestLatency / hitsCount; + } + + @Override + public synchronized void addRequestLatency(float latency) { + totalRequestLatency += (long) latency; + } } + +} From f919542e7b902769578e927196f4e197589468ea Mon Sep 17 00:00:00 2001 From: UTXOnly Date: Fri, 9 Aug 2024 23:24:33 -0400 Subject: [PATCH 2/8] Add back comments --- .../otel/controller/CalendarController.java | 43 +++-------- .../main/java/com/otel/main/SpringApp.java | 77 +++++++++++-------- 2 files changed, 58 insertions(+), 62 deletions(-) diff --git a/apps/rest-services/java/calendar/src/main/java/com/otel/controller/CalendarController.java b/apps/rest-services/java/calendar/src/main/java/com/otel/controller/CalendarController.java index c34ccd5..823ce89 100644 --- a/apps/rest-services/java/calendar/src/main/java/com/otel/controller/CalendarController.java +++ b/apps/rest-services/java/calendar/src/main/java/com/otel/controller/CalendarController.java @@ -9,30 +9,24 @@ import io.opentelemetry.api.trace.SpanKind; import io.opentelemetry.api.trace.Tracer; import io.opentelemetry.context.Scope; -import java.lang.management.ManagementFactory; -import javax.management.MBeanServer; -import javax.management.ObjectName; import java.time.LocalDate; import java.time.Month; import java.time.format.DateTimeFormatter; import java.util.Map; import java.util.Random; import java.util.concurrent.atomic.AtomicLong; -import javax.management.MBeanServerInvocationHandler; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.context.event.ContextRefreshedEvent; -import org.springframework.context.event.EventListener; import org.springframework.util.MultiValueMap; import org.springframework.web.bind.annotation.GetMapping; import org.springframework.web.bind.annotation.RequestHeader; import org.springframework.web.bind.annotation.RestController; - -import com.otel.main.SpringApp; +import com.otel.main.SpringApp.CalendarMBean; @RestController public class CalendarController { + private final Logger log = LoggerFactory.getLogger(CalendarController.class); private final Tracer tracer; private final LongCounter hitsCounter; @@ -40,31 +34,19 @@ public class CalendarController { private final ObservableDoubleGauge activeUsersGauge; private final AtomicLong activeUsersCounter; private final Random random = new Random(); - private SpringApp.CalendarMBean calendarMBean; + private final CalendarMBean calendarMBean; @Autowired - CalendarController(OpenTelemetry openTelemetry, String serviceName) { + CalendarController(OpenTelemetry openTelemetry, String serviceName, CalendarMBean calendarMBean) { + this.calendarMBean = calendarMBean; log.info("Starting CalendarController for {}", serviceName); tracer = openTelemetry.getTracer(CalendarController.class.getName()); - Meter meter = openTelemetry.getMeter(CalendarController.class.getName()); + Meter meter = openTelemetry.meterBuilder(CalendarController.class.getName()).build(); hitsCounter = meter.counterBuilder(serviceName + ".api.hits").build(); latency = meter.histogramBuilder(serviceName + ".task.duration").build(); activeUsersCounter = new AtomicLong(); activeUsersGauge = meter.gaugeBuilder(serviceName + ".active.users.guage").buildWithCallback(measurement -> measurement.record(activeUsersCounter.get())); } - // Initialize MBean server and ObjectName - @EventListener(ContextRefreshedEvent.class) - public void init() { - try { - MBeanServer mBeanServer = ManagementFactory.getPlatformMBeanServer(); - ObjectName objectName = new ObjectName("com.otel.main:type=Calendar"); - this.calendarMBean = MBeanServerInvocationHandler.newProxyInstance(mBeanServer, objectName, SpringApp.CalendarMBean.class, false); - log.info("CalendarMBean initialized"); - } catch (Exception e) { - log.error("Failed to initialize CalendarMBean", e); - throw new IllegalStateException("Failed to get CalendarMBean", e); - } - } @GetMapping("/calendar") public Map getDate(@RequestHeader MultiValueMap headers) { @@ -73,20 +55,17 @@ public Map getDate(@RequestHeader MultiValueMap try { hitsCounter.add(1); calendarMBean.incrementHitsCount(); - long endTime = System.currentTimeMillis(); - float latency = (endTime - startTime); - calendarMBean.addRequestLatency(latency); - - log.info("generated JMX hit count: {}, request latency: {}", calendarMBean.getHitsCount(), calendarMBean.getRequestLatency()); - + calendarMBean.addRequestLatency(endTime - startTime); + log.info("generated JMX hit count: {}, latency: {}", calendarMBean.getHitsCount(), calendarMBean.getRequestLatency()); + String output = getDate(); - return Map.of("date", output, "latency", String.valueOf(latency)); + return Map.of("date", output); } finally { activeUsersCounter.decrementAndGet(); } } - + private String getDate() { Span span = tracer.spanBuilder("getDate").setAttribute("peer.service", "random-date-service").setSpanKind(SpanKind.CLIENT).startSpan(); try (Scope scope = span.makeCurrent()) { diff --git a/apps/rest-services/java/calendar/src/main/java/com/otel/main/SpringApp.java b/apps/rest-services/java/calendar/src/main/java/com/otel/main/SpringApp.java index 0d44de4..68f4e7a 100644 --- a/apps/rest-services/java/calendar/src/main/java/com/otel/main/SpringApp.java +++ b/apps/rest-services/java/calendar/src/main/java/com/otel/main/SpringApp.java @@ -1,17 +1,25 @@ +/* +Unless explicitly stated otherwise all files in this repository are licensed +under the Apache 2.0 License. +This product includes software developed at Datadog (https://www.datadoghq.com/) +Copyright 2023 Datadog, Inc. + */ + package com.otel.main; import io.opentelemetry.api.GlobalOpenTelemetry; import io.opentelemetry.api.OpenTelemetry; import java.lang.management.ManagementFactory; import javax.management.MBeanServer; +import javax.management.MBeanServerInvocationHandler; import javax.management.ObjectName; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import org.springframework.boot.SpringApplication; import org.springframework.boot.autoconfigure.SpringBootApplication; import org.springframework.context.annotation.Bean; import org.springframework.context.event.ContextRefreshedEvent; import org.springframework.context.event.EventListener; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; @SpringBootApplication(scanBasePackages = "com.otel") public class SpringApp { @@ -22,16 +30,26 @@ public static void main(String[] args) { SpringApplication.run(SpringApp.class, args); } + @Bean + public CalendarMBean calendarMBean() { + return new Calendar(); + } + @EventListener(ContextRefreshedEvent.class) - public void registerMBean() { + public void registerAndInitMBean() { try { MBeanServer mBeanServer = ManagementFactory.getPlatformMBeanServer(); ObjectName objectName = new ObjectName("com.otel.main:type=Calendar"); Calendar calendarMBean = new Calendar(); mBeanServer.registerMBean(calendarMBean, objectName); log.info("MBean registered: {}", objectName); + + // Initialize MBean proxy + CalendarMBean proxyMBean = MBeanServerInvocationHandler.newProxyInstance(mBeanServer, objectName, CalendarMBean.class, false); + log.info("CalendarMBean initialized"); } catch (Exception e) { - log.error("Error registering MBean", e); + log.error("Error registering and initializing MBean", e); + throw new IllegalStateException("Failed to register and initialize CalendarMBean", e); } } @@ -47,35 +65,34 @@ public String serviceName() { } public interface CalendarMBean { - int getHitsCount(); - void incrementHitsCount(); - float getRequestLatency(); - void addRequestLatency(float latency); - } - - public static class Calendar implements CalendarMBean { - private int hitsCount = 0; - private long totalRequestLatency = 0; - - @Override - public synchronized int getHitsCount() { - return hitsCount; + int getHitsCount(); + void incrementHitsCount(); + float getRequestLatency(); + void addRequestLatency(float latency); } - @Override - public synchronized void incrementHitsCount() { - hitsCount++; - } + public static class Calendar implements CalendarMBean { + private int hitsCount = 0; + private long totalRequestLatency = 0; - @Override - public synchronized float getRequestLatency() { - return hitsCount == 0 ? 0 : (float) totalRequestLatency / hitsCount; - } + @Override + public synchronized int getHitsCount() { + return hitsCount; + } + + @Override + public synchronized void incrementHitsCount() { + hitsCount++; + } - @Override - public synchronized void addRequestLatency(float latency) { - totalRequestLatency += (long) latency; + @Override + public synchronized float getRequestLatency() { + return hitsCount == 0 ? 0 : (float) totalRequestLatency / hitsCount; + } + + @Override + public synchronized void addRequestLatency(float latency) { + totalRequestLatency += latency; + } } } - -} From 6dc78cf01771916ca2bde92507643d0043a7d99f Mon Sep 17 00:00:00 2001 From: UTXOnly Date: Fri, 9 Aug 2024 23:31:08 -0400 Subject: [PATCH 3/8] Revert to gtMeter method --- .../main/java/com/otel/controller/CalendarController.java | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/apps/rest-services/java/calendar/src/main/java/com/otel/controller/CalendarController.java b/apps/rest-services/java/calendar/src/main/java/com/otel/controller/CalendarController.java index 823ce89..1bb15a2 100644 --- a/apps/rest-services/java/calendar/src/main/java/com/otel/controller/CalendarController.java +++ b/apps/rest-services/java/calendar/src/main/java/com/otel/controller/CalendarController.java @@ -1,3 +1,8 @@ +/* +Unless explicitly stated otherwise all files in this repository are licensed +under the Apache 2.0 License. +*/ + package com.otel.controller; import io.opentelemetry.api.OpenTelemetry; @@ -41,7 +46,7 @@ public class CalendarController { this.calendarMBean = calendarMBean; log.info("Starting CalendarController for {}", serviceName); tracer = openTelemetry.getTracer(CalendarController.class.getName()); - Meter meter = openTelemetry.meterBuilder(CalendarController.class.getName()).build(); + Meter meter = openTelemetry.getMeter(CalendarController.class.getName()); hitsCounter = meter.counterBuilder(serviceName + ".api.hits").build(); latency = meter.histogramBuilder(serviceName + ".task.duration").build(); activeUsersCounter = new AtomicLong(); From 9693c6682e8875a98724aefce6d7b20274000ce1 Mon Sep 17 00:00:00 2001 From: UTXOnly Date: Mon, 12 Aug 2024 23:52:48 -0400 Subject: [PATCH 4/8] Working mbean + proxy --- .../java/calendar/deploys/Dockerfile.otel | 3 +- .../otel/controller/CalendarController.java | 39 ++++++++++++------- .../main/java/com/otel/main/SpringApp.java | 14 +++---- 3 files changed, 35 insertions(+), 21 deletions(-) diff --git a/apps/rest-services/java/calendar/deploys/Dockerfile.otel b/apps/rest-services/java/calendar/deploys/Dockerfile.otel index 4072456..6d9511e 100644 --- a/apps/rest-services/java/calendar/deploys/Dockerfile.otel +++ b/apps/rest-services/java/calendar/deploys/Dockerfile.otel @@ -8,10 +8,11 @@ RUN apt-get update -y; apt-get install curl -y WORKDIR /home/otel RUN curl -Lo opentelemetry-javaagent.jar https://github.com/open-telemetry/opentelemetry-java-instrumentation/releases/download/v2.4.0/opentelemetry-javaagent.jar COPY . calendar/ +COPY ./jmx_metrics_config.yaml ./jmx_metrics_config.yaml WORKDIR /home/otel/calendar -COPY ./jmx_metrics_config.yaml ./jmx_metrics_config.yaml +#Compile with gradle RUN ./gradlew build ENV JAVA_OPTS="-Dcom.sun.management.jmxremote \ diff --git a/apps/rest-services/java/calendar/src/main/java/com/otel/controller/CalendarController.java b/apps/rest-services/java/calendar/src/main/java/com/otel/controller/CalendarController.java index 1bb15a2..3fab25f 100644 --- a/apps/rest-services/java/calendar/src/main/java/com/otel/controller/CalendarController.java +++ b/apps/rest-services/java/calendar/src/main/java/com/otel/controller/CalendarController.java @@ -2,7 +2,6 @@ Unless explicitly stated otherwise all files in this repository are licensed under the Apache 2.0 License. */ - package com.otel.controller; import io.opentelemetry.api.OpenTelemetry; @@ -20,6 +19,9 @@ import java.util.Map; import java.util.Random; import java.util.concurrent.atomic.AtomicLong; +import javax.management.MBeanServer; +import javax.management.MBeanServerInvocationHandler; +import javax.management.ObjectName; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; @@ -42,15 +44,24 @@ public class CalendarController { private final CalendarMBean calendarMBean; @Autowired - CalendarController(OpenTelemetry openTelemetry, String serviceName, CalendarMBean calendarMBean) { - this.calendarMBean = calendarMBean; - log.info("Starting CalendarController for {}", serviceName); - tracer = openTelemetry.getTracer(CalendarController.class.getName()); + CalendarController(OpenTelemetry openTelemetry, String serviceName, MBeanServer mBeanServer) { + this.tracer = openTelemetry.getTracer(CalendarController.class.getName()); Meter meter = openTelemetry.getMeter(CalendarController.class.getName()); - hitsCounter = meter.counterBuilder(serviceName + ".api.hits").build(); - latency = meter.histogramBuilder(serviceName + ".task.duration").build(); - activeUsersCounter = new AtomicLong(); - activeUsersGauge = meter.gaugeBuilder(serviceName + ".active.users.guage").buildWithCallback(measurement -> measurement.record(activeUsersCounter.get())); + this.hitsCounter = meter.counterBuilder(serviceName + ".api.hits").build(); + this.latency = meter.histogramBuilder(serviceName + ".task.duration").build(); + this.activeUsersCounter = new AtomicLong(); + this.activeUsersGauge = meter.gaugeBuilder(serviceName + ".active.users.guage").buildWithCallback(measurement -> measurement.record(activeUsersCounter.get())); + + try { + // Create the MBean proxy + ObjectName objectName = new ObjectName("com.otel.main:type=Calendar"); + this.calendarMBean = MBeanServerInvocationHandler.newProxyInstance(mBeanServer, objectName, CalendarMBean.class, false); + log.info("CalendarMBean proxy initialized with instance hashcode: {}", System.identityHashCode(this.calendarMBean)); + } catch (Exception e) { + throw new RuntimeException("Failed to create MBean proxy", e); + } + + log.info("Starting CalendarController for {}", serviceName); } @GetMapping("/calendar") @@ -62,9 +73,9 @@ public Map getDate(@RequestHeader MultiValueMap calendarMBean.incrementHitsCount(); long endTime = System.currentTimeMillis(); calendarMBean.addRequestLatency(endTime - startTime); - log.info("generated JMX hit count: {}, latency: {}", calendarMBean.getHitsCount(), calendarMBean.getRequestLatency()); - - String output = getDate(); + log.info("Generated JMX hit count: {}, latency: {}", calendarMBean.getHitsCount(), calendarMBean.getRequestLatency());String output = getDate(); + // the correct JSON output should put this in quotes. Spring does not, so let's put quotes here + // by hand. return Map.of("date", output); } finally { activeUsersCounter.decrementAndGet(); @@ -74,12 +85,14 @@ public Map getDate(@RequestHeader MultiValueMap private String getDate() { Span span = tracer.spanBuilder("getDate").setAttribute("peer.service", "random-date-service").setSpanKind(SpanKind.CLIENT).startSpan(); try (Scope scope = span.makeCurrent()) { + // get back a random date in the year 2022 int val = new Random().nextInt(365); LocalDate start = LocalDate.of(2022, Month.JANUARY, 1).plusDays(val); String output = start.format(DateTimeFormatter.ISO_LOCAL_DATE); span.setAttribute("date", output); + // Add random sleep Thread.sleep(random.nextLong(1, 950)); - log.info("generated date: {}", output); + log.info("Generated date: {}", output); return output; } catch (InterruptedException e) { throw new RuntimeException(e); diff --git a/apps/rest-services/java/calendar/src/main/java/com/otel/main/SpringApp.java b/apps/rest-services/java/calendar/src/main/java/com/otel/main/SpringApp.java index 68f4e7a..c321676 100644 --- a/apps/rest-services/java/calendar/src/main/java/com/otel/main/SpringApp.java +++ b/apps/rest-services/java/calendar/src/main/java/com/otel/main/SpringApp.java @@ -1,17 +1,16 @@ /* Unless explicitly stated otherwise all files in this repository are licensed under the Apache 2.0 License. + This product includes software developed at Datadog (https://www.datadoghq.com/) Copyright 2023 Datadog, Inc. */ - package com.otel.main; import io.opentelemetry.api.GlobalOpenTelemetry; import io.opentelemetry.api.OpenTelemetry; import java.lang.management.ManagementFactory; import javax.management.MBeanServer; -import javax.management.MBeanServerInvocationHandler; import javax.management.ObjectName; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -30,6 +29,11 @@ public static void main(String[] args) { SpringApplication.run(SpringApp.class, args); } + @Bean + public MBeanServer mBeanServer() { + return ManagementFactory.getPlatformMBeanServer(); + } + @Bean public CalendarMBean calendarMBean() { return new Calendar(); @@ -38,15 +42,11 @@ public CalendarMBean calendarMBean() { @EventListener(ContextRefreshedEvent.class) public void registerAndInitMBean() { try { - MBeanServer mBeanServer = ManagementFactory.getPlatformMBeanServer(); + MBeanServer mBeanServer = mBeanServer(); ObjectName objectName = new ObjectName("com.otel.main:type=Calendar"); Calendar calendarMBean = new Calendar(); mBeanServer.registerMBean(calendarMBean, objectName); log.info("MBean registered: {}", objectName); - - // Initialize MBean proxy - CalendarMBean proxyMBean = MBeanServerInvocationHandler.newProxyInstance(mBeanServer, objectName, CalendarMBean.class, false); - log.info("CalendarMBean initialized"); } catch (Exception e) { log.error("Error registering and initializing MBean", e); throw new IllegalStateException("Failed to register and initialize CalendarMBean", e); From ed57a7704a1479cf0729584abbfa98c5fe204fc0 Mon Sep 17 00:00:00 2001 From: UTXOnly Date: Tue, 13 Aug 2024 00:07:18 -0400 Subject: [PATCH 5/8] Change totalRequestLatency to float --- .../java/calendar/src/main/java/com/otel/main/SpringApp.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/rest-services/java/calendar/src/main/java/com/otel/main/SpringApp.java b/apps/rest-services/java/calendar/src/main/java/com/otel/main/SpringApp.java index c321676..12f47cf 100644 --- a/apps/rest-services/java/calendar/src/main/java/com/otel/main/SpringApp.java +++ b/apps/rest-services/java/calendar/src/main/java/com/otel/main/SpringApp.java @@ -73,7 +73,7 @@ public interface CalendarMBean { public static class Calendar implements CalendarMBean { private int hitsCount = 0; - private long totalRequestLatency = 0; + private float totalRequestLatency = 0; @Override public synchronized int getHitsCount() { From fc75dab65a3759e2418616bb8244d3a1b3ec50a7 Mon Sep 17 00:00:00 2001 From: UTXOnly Date: Tue, 13 Aug 2024 00:23:44 -0400 Subject: [PATCH 6/8] Update JAVA_OPTS for local script --- apps/rest-services/java/calendar/deploys/Dockerfile.otel | 1 - apps/rest-services/java/calendar/run-otel-local.sh | 8 ++++++++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/apps/rest-services/java/calendar/deploys/Dockerfile.otel b/apps/rest-services/java/calendar/deploys/Dockerfile.otel index 6d9511e..a017e9d 100644 --- a/apps/rest-services/java/calendar/deploys/Dockerfile.otel +++ b/apps/rest-services/java/calendar/deploys/Dockerfile.otel @@ -23,5 +23,4 @@ ENV JAVA_OPTS="-Dcom.sun.management.jmxremote \ -Djava.rmi.server.hostname=127.0.0.1" ENTRYPOINT exec java ${JAVA_OPTS} -javaagent:../opentelemetry-javaagent.jar -Dotel.jmx.config=./jmx_metrics_config.yaml -jar build/libs/calendar-0.0.1-SNAPSHOT.jar -#ENTRYPOINT ["java", "-javaagent:../opentelemetry-javaagent.jar", "-Dotel.jmx.config=./jmx_metrics_config.yaml", "-jar" , "build/libs/calendar-0.0.1-SNAPSHOT.jar"] EXPOSE 8080 diff --git a/apps/rest-services/java/calendar/run-otel-local.sh b/apps/rest-services/java/calendar/run-otel-local.sh index 3e8245a..b904e78 100755 --- a/apps/rest-services/java/calendar/run-otel-local.sh +++ b/apps/rest-services/java/calendar/run-otel-local.sh @@ -6,6 +6,12 @@ export OTEL_METRICS_EXPORTER=otlp export OTEL_LOGS_EXPORTER=otlp export OTEL_EXPORTER_OTLP_ENDPOINT=http://localhost:4317 export OTEL_EXPORTER_OTLP_PROTOCOL=grpc +export JAVA_OPTS="-Dcom.sun.management.jmxremote \ + -Dcom.sun.management.jmxremote.port=9092 \ + -Dcom.sun.management.jmxremote.authenticate=false \ + -Dcom.sun.management.jmxremote.ssl=false \ + -Dcom.sun.management.jmxremote.rmi.port=9093\ + -Djava.rmi.server.hostname=127.0.0.1" # Define the path to the Java agent JAR JAVA_AGENT_JAR="$SCRIPT_DIR/opentelemetry-javaagent.jar" @@ -21,4 +27,6 @@ fi export OTEL_RESOURCE_ATTRIBUTES=service.name=my-calendar-service,deployment.environment=otel-test,host.name=calendar-host java -javaagent:$JAVA_AGENT_JAR \ + ${JAVA_OPTS} \ + "-Dotel.jmx.config=./jmx_metrics_config.yaml" \ -jar $SCRIPT_DIR/build/libs/calendar-0.0.1-SNAPSHOT.jar From 53f7bf6d442a4a2a105f90de6b1a0dc2b290e5ee Mon Sep 17 00:00:00 2001 From: UTXOnly Date: Tue, 13 Aug 2024 11:03:00 -0400 Subject: [PATCH 7/8] Remove this refrence --- .../java/com/otel/controller/CalendarController.java | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/apps/rest-services/java/calendar/src/main/java/com/otel/controller/CalendarController.java b/apps/rest-services/java/calendar/src/main/java/com/otel/controller/CalendarController.java index 3fab25f..a3d9347 100644 --- a/apps/rest-services/java/calendar/src/main/java/com/otel/controller/CalendarController.java +++ b/apps/rest-services/java/calendar/src/main/java/com/otel/controller/CalendarController.java @@ -45,12 +45,12 @@ public class CalendarController { @Autowired CalendarController(OpenTelemetry openTelemetry, String serviceName, MBeanServer mBeanServer) { - this.tracer = openTelemetry.getTracer(CalendarController.class.getName()); + tracer = openTelemetry.getTracer(CalendarController.class.getName()); Meter meter = openTelemetry.getMeter(CalendarController.class.getName()); - this.hitsCounter = meter.counterBuilder(serviceName + ".api.hits").build(); - this.latency = meter.histogramBuilder(serviceName + ".task.duration").build(); - this.activeUsersCounter = new AtomicLong(); - this.activeUsersGauge = meter.gaugeBuilder(serviceName + ".active.users.guage").buildWithCallback(measurement -> measurement.record(activeUsersCounter.get())); + hitsCounter = meter.counterBuilder(serviceName + ".api.hits").build(); + latency = meter.histogramBuilder(serviceName + ".task.duration").build(); + activeUsersCounter = new AtomicLong(); + activeUsersGauge = meter.gaugeBuilder(serviceName + ".active.users.guage").buildWithCallback(measurement -> measurement.record(activeUsersCounter.get())); try { // Create the MBean proxy From 6e5d7f6c91094b99ce54f399dcd03516f1c3cfb8 Mon Sep 17 00:00:00 2001 From: UTXOnly Date: Tue, 13 Aug 2024 11:10:46 -0400 Subject: [PATCH 8/8] Update README --- apps/rest-services/java/calendar/README.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/apps/rest-services/java/calendar/README.md b/apps/rest-services/java/calendar/README.md index cfa398a..4c42afc 100644 --- a/apps/rest-services/java/calendar/README.md +++ b/apps/rest-services/java/calendar/README.md @@ -78,3 +78,7 @@ Install calendar in K8s with DD SDK ``` helm install -n otel-ingest calendar-dd-java ./calendar-dd/k8s/ --set image.repository=dineshgurumurthydd/calendar-java --set image.tag=otel-0.1,node_group=ng-1 ``` + +## JMX metrics + +The `jmx_metrics_config.yaml` file provides a detailed example of configuring custom JMX metrics collection using the OpenTelemetry Java agent.