[WIP] JMX scraper (#1445)

.github/component_owners.yml

@@ -41,6 +41,10 @@ components:
     - sfriberg
   jmx-metrics:
     - breedx-splk
+  jmx-scraper:
+    - breedx-splk
+    - robsunday
+    - sylvainjuge
   maven-extension:
     - cyrille-leclerc
     - kenfinnigan

jmx-scraper/README.md

@@ -0,0 +1,8 @@
+# JMX Metric Scraper
+
+This utility provides a way to query JMX metrics and export them to an OTLP endpoint.
+The JMX MBeans and their metric mappings are defined in YAML and reuse implementation from
+[jmx-metrics instrumentation](https://github.com/open-telemetry/opentelemetry-java-instrumentation/tree/main/instrumentation/jmx-metrics).
+
+This is currently a work-in-progress component not ready to be used in production.
+The end goal is to provide an alternative to the [JMX Gatherer](../jmx-metrics/README.md) utility.

jmx-scraper/build.gradle.kts

@@ -0,0 +1,94 @@
+plugins {
+  application
+  id("com.github.johnrengelman.shadow")
+
+  id("otel.java-conventions")
+
+  // TODO publishing disabled until component is ready to be used
+  // id("otel.publish-conventions")
+}
+
+description = "JMX metrics scraper"
+otelJava.moduleName.set("io.opentelemetry.contrib.jmxscraper")
+
+application.mainClass.set("io.opentelemetry.contrib.jmxscraper.JmxScraper")
+
+dependencies {
+  // TODO remove snapshot dependency on upstream once 2.9.0 is released
+  // api(enforcedPlatform("io.opentelemetry.instrumentation:opentelemetry-instrumentation-bom-alpha:2.9.0-SNAPSHOT-alpha",))
+
+  implementation("io.opentelemetry:opentelemetry-api")
+  implementation("io.opentelemetry:opentelemetry-sdk")
+  implementation("io.opentelemetry:opentelemetry-sdk-metrics")
+  implementation("io.opentelemetry:opentelemetry-sdk-extension-autoconfigure")
+
+  implementation("io.opentelemetry.instrumentation:opentelemetry-jmx-metrics")
+
+  testImplementation("org.junit-pioneer:junit-pioneer")
+  testImplementation("io.opentelemetry:opentelemetry-sdk-testing")
+}
+
+testing {
+  suites {
+    val integrationTest by registering(JvmTestSuite::class) {
+      dependencies {
+        implementation("org.testcontainers:junit-jupiter")
+        implementation("org.slf4j:slf4j-simple")
+        implementation("com.linecorp.armeria:armeria-junit5")
+        implementation("com.linecorp.armeria:armeria-grpc")
+        implementation("io.opentelemetry.proto:opentelemetry-proto:0.20.0-alpha")
+      }
+    }
+  }
+}
+
+tasks {
+  shadowJar {
+    mergeServiceFiles()
+
+    manifest {
+      attributes["Implementation-Version"] = project.version
+    }
+    // This should always be standalone, so remove "-all" to prevent unnecessary artifact.
+    archiveClassifier.set("")
+  }
+
+  jar {
+    archiveClassifier.set("noshadow")
+  }
+
+  withType<Test>().configureEach {
+    dependsOn(shadowJar)
+    dependsOn(named("appJar"))
+    systemProperty("shadow.jar.path", shadowJar.get().archiveFile.get().asFile.absolutePath)
+    systemProperty("app.jar.path", named<Jar>("appJar").get().archiveFile.get().asFile.absolutePath)
+    systemProperty("gradle.project.version", "${project.version}")
+  }
+
+  // Because we reconfigure publishing to only include the shadow jar, the Gradle metadata is not correct.
+  // Since we are fully bundled and have no dependencies, Gradle metadata wouldn't provide any advantage over
+  // the POM anyways so in practice we shouldn't be losing anything.
+  withType<GenerateModuleMetadata>().configureEach {
+    enabled = false
+  }
+}
+
+tasks.register<Jar>("appJar") {
+  from(sourceSets.get("integrationTest").output)
+  archiveClassifier.set("app")
+  manifest {
+    attributes["Main-Class"] = "io.opentelemetry.contrib.jmxscraper.TestApp"
+  }
+}
+
+// Don't publish non-shadowed jar (shadowJar is in shadowRuntimeElements)
+with(components["java"] as AdhocComponentWithVariants) {
+  configurations.forEach {
+    withVariantsFromConfiguration(configurations["apiElements"]) {
+      skip()
+    }
+    withVariantsFromConfiguration(configurations["runtimeElements"]) {
+      skip()
+    }
+  }
+}

jmx-scraper/src/integrationTest/java/io/opentelemetry/contrib/jmxscraper/JmxConnectorBuilderTest.java

@@ -0,0 +1,92 @@
+/*
+ * Copyright The OpenTelemetry Authors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+package io.opentelemetry.contrib.jmxscraper;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+import java.io.IOException;
+import javax.management.ObjectName;
+import javax.management.remote.JMXConnector;
+import org.junit.jupiter.api.AfterAll;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.Test;
+import org.testcontainers.containers.Network;
+
+public class JmxConnectorBuilderTest {
+
+  private static Network network;
+
+  @BeforeAll
+  static void beforeAll() {
+    network = Network.newNetwork();
+  }
+
+  @AfterAll
+  static void afterAll() {
+    network.close();
+  }
+
+  @Test
+  void noAuth() {
+    try (TestAppContainer app = new TestAppContainer().withNetwork(network).withJmxPort(9990)) {
+      app.start();
+      testConnector(
+          () -> JmxConnectorBuilder.createNew(app.getHost(), app.getMappedPort(9990)).build());
+    }
+  }
+
+  @Test
+  void loginPwdAuth() {
+    String login = "user";
+    String pwd = "t0p!Secret";
+    try (TestAppContainer app =
+        new TestAppContainer().withNetwork(network).withJmxPort(9999).withUserAuth(login, pwd)) {
+      app.start();
+      testConnector(
+          () ->
+              JmxConnectorBuilder.createNew(app.getHost(), app.getMappedPort(9999))
+                  .userCredentials(login, pwd)
+                  .build());
+    }
+  }
+
+  @Test
+  void serverSSL() {
+    // TODO: test with SSL enabled as RMI registry seems to work differently with SSL
+
+    // create keypair (public,private)
+    // create server keystore with private key
+    // configure server keystore
+    //
+    // create client truststore with public key
+    // can we configure to use a custom truststore ???
+    // connect to server
+  }
+
+  private static void testConnector(ConnectorSupplier connectorSupplier) {
+    try (JMXConnector connector = connectorSupplier.get()) {
+      assertThat(connector.getMBeanServerConnection())
+          .isNotNull()
+          .satisfies(
+              connection -> {
+                try {
+                  ObjectName name = new ObjectName(TestApp.OBJECT_NAME);
+                  Object value = connection.getAttribute(name, "IntValue");
+                  assertThat(value).isEqualTo(42);
+                } catch (Exception e) {
+                  throw new RuntimeException(e);
+                }
+              });
+
+    } catch (IOException e) {
+      throw new RuntimeException(e);
+    }
+  }
+
+  private interface ConnectorSupplier {
+    JMXConnector get() throws IOException;
+  }
+}

jmx-scraper/src/integrationTest/java/io/opentelemetry/contrib/jmxscraper/JmxScraperContainer.java

@@ -0,0 +1,107 @@
+/*
+ * Copyright The OpenTelemetry Authors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+package io.opentelemetry.contrib.jmxscraper;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+import com.google.errorprone.annotations.CanIgnoreReturnValue;
+import java.time.Duration;
+import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Locale;
+import java.util.Set;
+import org.testcontainers.containers.GenericContainer;
+import org.testcontainers.containers.wait.strategy.Wait;
+import org.testcontainers.utility.MountableFile;
+
+/** Test container that allows to execute {@link JmxScraper} in an isolated container */
+public class JmxScraperContainer extends GenericContainer<JmxScraperContainer> {
+
+  private final String endpoint;
+  private final Set<String> targetSystems;
+  private String serviceUrl;
+  private int intervalMillis;
+  private final Set<String> customYamlFiles;
+
+  public JmxScraperContainer(String otlpEndpoint) {
+    super("openjdk:8u272-jre-slim");
+
+    String scraperJarPath = System.getProperty("shadow.jar.path");
+    assertThat(scraperJarPath).isNotNull();
+
+    this.withCopyFileToContainer(MountableFile.forHostPath(scraperJarPath), "/scraper.jar")
+        .waitingFor(
+            Wait.forLogMessage(".*JMX scraping started.*", 1)
+                .withStartupTimeout(Duration.ofSeconds(10)));
+
+    this.endpoint = otlpEndpoint;
+    this.targetSystems = new HashSet<>();
+    this.customYamlFiles = new HashSet<>();
+    this.intervalMillis = 1000;
+  }
+
+  @CanIgnoreReturnValue
+  public JmxScraperContainer withTargetSystem(String targetSystem) {
+    targetSystems.add(targetSystem);
+    return this;
+  }
+
+  @CanIgnoreReturnValue
+  public JmxScraperContainer withIntervalMillis(int intervalMillis) {
+    this.intervalMillis = intervalMillis;
+    return this;
+  }
+
+  @CanIgnoreReturnValue
+  public JmxScraperContainer withService(String host, int port) {
+    // TODO: adding a way to provide 'host:port' syntax would make this easier for end users
+    this.serviceUrl =
+        String.format(
+            Locale.getDefault(), "service:jmx:rmi:///jndi/rmi://%s:%d/jmxrmi", host, port);
+    return this;
+  }
+
+  @CanIgnoreReturnValue
+  public JmxScraperContainer withCustomYaml(String yamlPath) {
+    this.customYamlFiles.add(yamlPath);
+    return this;
+  }
+
+  @Override
+  public void start() {
+    // for now only configure through JVM args
+    List<String> arguments = new ArrayList<>();
+    arguments.add("java");
+    arguments.add("-Dotel.exporter.otlp.endpoint=" + endpoint);
+
+    if (!targetSystems.isEmpty()) {
+      arguments.add("-Dotel.jmx.target.system=" + String.join(",", targetSystems));
+    }
+
+    if (serviceUrl == null) {
+      throw new IllegalStateException("Missing service URL");
+    }
+    arguments.add("-Dotel.jmx.service.url=" + serviceUrl);
+    arguments.add("-Dotel.jmx.interval.milliseconds=" + intervalMillis);
+
+    if (!customYamlFiles.isEmpty()) {
+      for (String yaml : customYamlFiles) {
+        this.withCopyFileToContainer(MountableFile.forClasspathResource(yaml), yaml);
+      }
+      arguments.add("-Dotel.jmx.config=" + String.join(",", customYamlFiles));
+    }
+
+    arguments.add("-jar");
+    arguments.add("/scraper.jar");
+
+    this.withCommand(arguments.toArray(new String[0]));
+
+    logger().info("Starting scraper with command: " + String.join(" ", arguments));
+
+    super.start();
+  }
+}

jmx-scraper/src/integrationTest/java/io/opentelemetry/contrib/jmxscraper/TestApp.java

@@ -0,0 +1,60 @@
+/*
+ * Copyright The OpenTelemetry Authors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+package io.opentelemetry.contrib.jmxscraper;
+
+import java.lang.management.ManagementFactory;
+import javax.management.MBeanServer;
+import javax.management.ObjectName;
+
+@SuppressWarnings("all")
+public class TestApp implements TestAppMXBean {
+
+  public static final String APP_STARTED_MSG = "app started";
+  public static final String OBJECT_NAME = "io.opentelemetry.test:name=TestApp";
+
+  private volatile boolean running;
+
+  public static void main(String[] args) {
+    TestApp app = TestApp.start();
+    while (app.isRunning()) {
+      try {
+        Thread.sleep(100);
+      } catch (InterruptedException e) {
+        throw new RuntimeException(e);
+      }
+    }
+  }
+
+  private TestApp() {}
+
+  static TestApp start() {
+    TestApp app = new TestApp();
+    MBeanServer mbs = ManagementFactory.getPlatformMBeanServer();
+    try {
+      ObjectName objectName = new ObjectName(OBJECT_NAME);
+      mbs.registerMBean(app, objectName);
+    } catch (Exception e) {
+      throw new RuntimeException(e);
+    }
+    app.running = true;
+    System.out.println(APP_STARTED_MSG);
+    return app;
+  }
+
+  @Override
+  public int getIntValue() {
+    return 42;
+  }
+
+  @Override
+  public void stopApp() {
+    running = false;
+  }
+
+  boolean isRunning() {
+    return running;
+  }
+}