diff --git a/apps/rest-services/java/calendar/README.md b/apps/rest-services/java/calendar/README.md index cfa398a..4c42afc 100644 --- a/apps/rest-services/java/calendar/README.md +++ b/apps/rest-services/java/calendar/README.md @@ -78,3 +78,7 @@ Install calendar in K8s with DD SDK ``` helm install -n otel-ingest calendar-dd-java ./calendar-dd/k8s/ --set image.repository=dineshgurumurthydd/calendar-java --set image.tag=otel-0.1,node_group=ng-1 ``` + +## JMX metrics + +The `jmx_metrics_config.yaml` file provides a detailed example of configuring custom JMX metrics collection using the OpenTelemetry Java agent. diff --git a/apps/rest-services/java/calendar/deploys/Dockerfile.otel b/apps/rest-services/java/calendar/deploys/Dockerfile.otel index b04b654..a017e9d 100644 --- a/apps/rest-services/java/calendar/deploys/Dockerfile.otel +++ b/apps/rest-services/java/calendar/deploys/Dockerfile.otel @@ -8,11 +8,19 @@ RUN apt-get update -y; apt-get install curl -y WORKDIR /home/otel RUN curl -Lo opentelemetry-javaagent.jar https://github.com/open-telemetry/opentelemetry-java-instrumentation/releases/download/v2.4.0/opentelemetry-javaagent.jar COPY . calendar/ +COPY ./jmx_metrics_config.yaml ./jmx_metrics_config.yaml WORKDIR /home/otel/calendar #Compile with gradle RUN ./gradlew build -ENTRYPOINT ["java","-javaagent:../opentelemetry-javaagent.jar", "-jar" , "build/libs/calendar-0.0.1-SNAPSHOT.jar"] +ENV JAVA_OPTS="-Dcom.sun.management.jmxremote \ + -Dcom.sun.management.jmxremote.port=9092 \ + -Dcom.sun.management.jmxremote.authenticate=false \ + -Dcom.sun.management.jmxremote.ssl=false \ + -Dcom.sun.management.jmxremote.rmi.port=9093\ + -Djava.rmi.server.hostname=127.0.0.1" +ENTRYPOINT exec java ${JAVA_OPTS} -javaagent:../opentelemetry-javaagent.jar -Dotel.jmx.config=./jmx_metrics_config.yaml -jar build/libs/calendar-0.0.1-SNAPSHOT.jar + EXPOSE 8080 diff --git a/apps/rest-services/java/calendar/deploys/docker/docker-compose-otel.yml b/apps/rest-services/java/calendar/deploys/docker/docker-compose-otel.yml index d029392..4f11da4 100644 --- a/apps/rest-services/java/calendar/deploys/docker/docker-compose-otel.yml +++ b/apps/rest-services/java/calendar/deploys/docker/docker-compose-otel.yml @@ -20,6 +20,9 @@ services: - OTEL_EXPORTER_OTLP_PROTOCOL=grpc ports: - "9090:9090" + - "9092:9092" + - "9093:9093" + # open-telemetry collector otelcol: image: otel/opentelemetry-collector-contrib:latest diff --git a/apps/rest-services/java/calendar/jmx_metrics_config.yaml b/apps/rest-services/java/calendar/jmx_metrics_config.yaml new file mode 100644 index 0000000..b10e45b --- /dev/null +++ b/apps/rest-services/java/calendar/jmx_metrics_config.yaml @@ -0,0 +1,14 @@ +rules: + - bean: com.otel.main:type=Calendar + mapping: + HitsCount: + metric: my.jmx-calendar.hits.count + type: counter + desc: "The number of hits to the calendar" + unit: "1" + RequestLatency: + metric: my.jmx-calendar.request.latency + type: gauge + desc: "The average request latency to the calendar" + unit: "ns" + diff --git a/apps/rest-services/java/calendar/run-otel-local.sh b/apps/rest-services/java/calendar/run-otel-local.sh index 3e8245a..b904e78 100755 --- a/apps/rest-services/java/calendar/run-otel-local.sh +++ b/apps/rest-services/java/calendar/run-otel-local.sh @@ -6,6 +6,12 @@ export OTEL_METRICS_EXPORTER=otlp export OTEL_LOGS_EXPORTER=otlp export OTEL_EXPORTER_OTLP_ENDPOINT=http://localhost:4317 export OTEL_EXPORTER_OTLP_PROTOCOL=grpc +export JAVA_OPTS="-Dcom.sun.management.jmxremote \ + -Dcom.sun.management.jmxremote.port=9092 \ + -Dcom.sun.management.jmxremote.authenticate=false \ + -Dcom.sun.management.jmxremote.ssl=false \ + -Dcom.sun.management.jmxremote.rmi.port=9093\ + -Djava.rmi.server.hostname=127.0.0.1" # Define the path to the Java agent JAR JAVA_AGENT_JAR="$SCRIPT_DIR/opentelemetry-javaagent.jar" @@ -21,4 +27,6 @@ fi export OTEL_RESOURCE_ATTRIBUTES=service.name=my-calendar-service,deployment.environment=otel-test,host.name=calendar-host java -javaagent:$JAVA_AGENT_JAR \ + ${JAVA_OPTS} \ + "-Dotel.jmx.config=./jmx_metrics_config.yaml" \ -jar $SCRIPT_DIR/build/libs/calendar-0.0.1-SNAPSHOT.jar diff --git a/apps/rest-services/java/calendar/src/main/java/com/otel/controller/CalendarController.java b/apps/rest-services/java/calendar/src/main/java/com/otel/controller/CalendarController.java index 79f1e18..a3d9347 100644 --- a/apps/rest-services/java/calendar/src/main/java/com/otel/controller/CalendarController.java +++ b/apps/rest-services/java/calendar/src/main/java/com/otel/controller/CalendarController.java @@ -19,6 +19,9 @@ import java.util.Map; import java.util.Random; import java.util.concurrent.atomic.AtomicLong; +import javax.management.MBeanServer; +import javax.management.MBeanServerInvocationHandler; +import javax.management.ObjectName; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; @@ -26,9 +29,11 @@ import org.springframework.web.bind.annotation.GetMapping; import org.springframework.web.bind.annotation.RequestHeader; import org.springframework.web.bind.annotation.RestController; +import com.otel.main.SpringApp.CalendarMBean; @RestController public class CalendarController { + private final Logger log = LoggerFactory.getLogger(CalendarController.class); private final Tracer tracer; private final LongCounter hitsCounter; @@ -36,10 +41,10 @@ public class CalendarController { private final ObservableDoubleGauge activeUsersGauge; private final AtomicLong activeUsersCounter; private final Random random = new Random(); + private final CalendarMBean calendarMBean; @Autowired - CalendarController(OpenTelemetry openTelemetry, String serviceName) { - log.info("Starting CalendarController for {}", serviceName); + CalendarController(OpenTelemetry openTelemetry, String serviceName, MBeanServer mBeanServer) { tracer = openTelemetry.getTracer(CalendarController.class.getName()); Meter meter = openTelemetry.getMeter(CalendarController.class.getName()); hitsCounter = meter.counterBuilder(serviceName + ".api.hits").build(); @@ -47,8 +52,17 @@ public class CalendarController { activeUsersCounter = new AtomicLong(); activeUsersGauge = meter.gaugeBuilder(serviceName + ".active.users.guage").buildWithCallback(measurement -> measurement.record(activeUsersCounter.get())); - } + try { + // Create the MBean proxy + ObjectName objectName = new ObjectName("com.otel.main:type=Calendar"); + this.calendarMBean = MBeanServerInvocationHandler.newProxyInstance(mBeanServer, objectName, CalendarMBean.class, false); + log.info("CalendarMBean proxy initialized with instance hashcode: {}", System.identityHashCode(this.calendarMBean)); + } catch (Exception e) { + throw new RuntimeException("Failed to create MBean proxy", e); + } + log.info("Starting CalendarController for {}", serviceName); + } @GetMapping("/calendar") public Map getDate(@RequestHeader MultiValueMap headers) { @@ -56,13 +70,14 @@ public Map getDate(@RequestHeader MultiValueMap activeUsersCounter.incrementAndGet(); try { hitsCounter.add(1); - String output = getDate(); - // the correct JSON output should put this in quotes. Spring does not, so let's put quotes here + calendarMBean.incrementHitsCount(); + long endTime = System.currentTimeMillis(); + calendarMBean.addRequestLatency(endTime - startTime); + log.info("Generated JMX hit count: {}, latency: {}", calendarMBean.getHitsCount(), calendarMBean.getRequestLatency());String output = getDate(); + // the correct JSON output should put this in quotes. Spring does not, so let's put quotes here // by hand. return Map.of("date", output); } finally { - long endTime = System.currentTimeMillis(); - latency.record(endTime - startTime); activeUsersCounter.decrementAndGet(); } } @@ -76,8 +91,8 @@ private String getDate() { String output = start.format(DateTimeFormatter.ISO_LOCAL_DATE); span.setAttribute("date", output); // Add random sleep - Thread.sleep(random.nextLong(1,950)); - log.info("generated date: {}", output); + Thread.sleep(random.nextLong(1, 950)); + log.info("Generated date: {}", output); return output; } catch (InterruptedException e) { throw new RuntimeException(e); diff --git a/apps/rest-services/java/calendar/src/main/java/com/otel/main/SpringApp.java b/apps/rest-services/java/calendar/src/main/java/com/otel/main/SpringApp.java index 1af65e3..12f47cf 100644 --- a/apps/rest-services/java/calendar/src/main/java/com/otel/main/SpringApp.java +++ b/apps/rest-services/java/calendar/src/main/java/com/otel/main/SpringApp.java @@ -9,24 +9,90 @@ This product includes software developed at Datadog (https://www.datadoghq.com/) import io.opentelemetry.api.GlobalOpenTelemetry; import io.opentelemetry.api.OpenTelemetry; -import org.springframework.beans.BeanUtils; +import java.lang.management.ManagementFactory; +import javax.management.MBeanServer; +import javax.management.ObjectName; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import org.springframework.boot.SpringApplication; import org.springframework.boot.autoconfigure.SpringBootApplication; import org.springframework.context.annotation.Bean; +import org.springframework.context.event.ContextRefreshedEvent; +import org.springframework.context.event.EventListener; @SpringBootApplication(scanBasePackages = "com.otel") public class SpringApp { - public static void main(String[] args) { - SpringApplication.run(SpringApp.class, args); - } - @Bean - public OpenTelemetry openTelemetry() { - return GlobalOpenTelemetry.get(); - } - - @Bean - public String serviceName() { - String serviceName = System.getProperty("otel.serviceName"); - return serviceName != null ? serviceName : "calendar"; - } + + private static final Logger log = LoggerFactory.getLogger(SpringApp.class); + + public static void main(String[] args) { + SpringApplication.run(SpringApp.class, args); + } + + @Bean + public MBeanServer mBeanServer() { + return ManagementFactory.getPlatformMBeanServer(); + } + + @Bean + public CalendarMBean calendarMBean() { + return new Calendar(); + } + + @EventListener(ContextRefreshedEvent.class) + public void registerAndInitMBean() { + try { + MBeanServer mBeanServer = mBeanServer(); + ObjectName objectName = new ObjectName("com.otel.main:type=Calendar"); + Calendar calendarMBean = new Calendar(); + mBeanServer.registerMBean(calendarMBean, objectName); + log.info("MBean registered: {}", objectName); + } catch (Exception e) { + log.error("Error registering and initializing MBean", e); + throw new IllegalStateException("Failed to register and initialize CalendarMBean", e); + } + } + + @Bean + public OpenTelemetry openTelemetry() { + return GlobalOpenTelemetry.get(); + } + + @Bean + public String serviceName() { + String serviceName = System.getProperty("otel.serviceName"); + return serviceName != null ? serviceName : "calendar"; + } + + public interface CalendarMBean { + int getHitsCount(); + void incrementHitsCount(); + float getRequestLatency(); + void addRequestLatency(float latency); + } + + public static class Calendar implements CalendarMBean { + private int hitsCount = 0; + private float totalRequestLatency = 0; + + @Override + public synchronized int getHitsCount() { + return hitsCount; + } + + @Override + public synchronized void incrementHitsCount() { + hitsCount++; + } + + @Override + public synchronized float getRequestLatency() { + return hitsCount == 0 ? 0 : (float) totalRequestLatency / hitsCount; + } + + @Override + public synchronized void addRequestLatency(float latency) { + totalRequestLatency += latency; + } + } }