From 704421fd6b768c9707b170afcb4b856de4c3f73d Mon Sep 17 00:00:00 2001 From: SylvainJuge <763082+SylvainJuge@users.noreply.github.com> Date: Fri, 27 Sep 2024 16:12:20 +0200 Subject: [PATCH] [WIP] JMX scraper (#1445) --- .github/component_owners.yml | 4 + jmx-scraper/README.md | 8 + jmx-scraper/build.gradle.kts | 94 +++++ .../jmxscraper/JmxConnectorBuilderTest.java | 92 +++++ .../jmxscraper/JmxScraperContainer.java | 107 ++++++ .../contrib/jmxscraper/TestApp.java | 60 ++++ .../contrib/jmxscraper/TestAppContainer.java | 126 +++++++ .../contrib/jmxscraper/TestAppMXBean.java | 14 + .../target_systems/JvmIntegrationTest.java | 31 ++ .../TargetSystemIntegrationTest.java | 164 +++++++++ .../target_systems/TomcatIntegrationTest.java | 54 +++ .../jmxscraper/ArgumentsParsingException.java | 14 + .../jmxscraper/JmxConnectorBuilder.java | 161 +++++++++ .../contrib/jmxscraper/JmxScraper.java | 137 +++++++ .../config/ConfigurationException.java | 18 + .../jmxscraper/config/JmxScraperConfig.java | 248 +++++++++++++ .../jmxscraper/internal/StringUtils.java | 26 ++ .../contrib/jmxscraper/JmxScraperTest.java | 78 ++++ .../config/JmxScraperConfigTest.java | 337 ++++++++++++++++++ .../src/test/resources/validConfig.properties | 20 ++ settings.gradle.kts | 2 + 21 files changed, 1795 insertions(+) create mode 100644 jmx-scraper/README.md create mode 100644 jmx-scraper/build.gradle.kts create mode 100644 jmx-scraper/src/integrationTest/java/io/opentelemetry/contrib/jmxscraper/JmxConnectorBuilderTest.java create mode 100644 jmx-scraper/src/integrationTest/java/io/opentelemetry/contrib/jmxscraper/JmxScraperContainer.java create mode 100644 jmx-scraper/src/integrationTest/java/io/opentelemetry/contrib/jmxscraper/TestApp.java create mode 100644 jmx-scraper/src/integrationTest/java/io/opentelemetry/contrib/jmxscraper/TestAppContainer.java create mode 100644 jmx-scraper/src/integrationTest/java/io/opentelemetry/contrib/jmxscraper/TestAppMXBean.java create mode 100644 jmx-scraper/src/integrationTest/java/io/opentelemetry/contrib/jmxscraper/target_systems/JvmIntegrationTest.java create mode 100644 jmx-scraper/src/integrationTest/java/io/opentelemetry/contrib/jmxscraper/target_systems/TargetSystemIntegrationTest.java create mode 100644 jmx-scraper/src/integrationTest/java/io/opentelemetry/contrib/jmxscraper/target_systems/TomcatIntegrationTest.java create mode 100644 jmx-scraper/src/main/java/io/opentelemetry/contrib/jmxscraper/ArgumentsParsingException.java create mode 100644 jmx-scraper/src/main/java/io/opentelemetry/contrib/jmxscraper/JmxConnectorBuilder.java create mode 100644 jmx-scraper/src/main/java/io/opentelemetry/contrib/jmxscraper/JmxScraper.java create mode 100644 jmx-scraper/src/main/java/io/opentelemetry/contrib/jmxscraper/config/ConfigurationException.java create mode 100644 jmx-scraper/src/main/java/io/opentelemetry/contrib/jmxscraper/config/JmxScraperConfig.java create mode 100644 jmx-scraper/src/main/java/io/opentelemetry/contrib/jmxscraper/internal/StringUtils.java create mode 100644 jmx-scraper/src/test/java/io/opentelemetry/contrib/jmxscraper/JmxScraperTest.java create mode 100644 jmx-scraper/src/test/java/io/opentelemetry/contrib/jmxscraper/config/JmxScraperConfigTest.java create mode 100644 jmx-scraper/src/test/resources/validConfig.properties diff --git a/.github/component_owners.yml b/.github/component_owners.yml index 552ca7edd..739e9bd7e 100644 --- a/.github/component_owners.yml +++ b/.github/component_owners.yml @@ -41,6 +41,10 @@ components: - sfriberg jmx-metrics: - breedx-splk + jmx-scraper: + - breedx-splk + - robsunday + - sylvainjuge maven-extension: - cyrille-leclerc - kenfinnigan diff --git a/jmx-scraper/README.md b/jmx-scraper/README.md new file mode 100644 index 000000000..175dd8c4d --- /dev/null +++ b/jmx-scraper/README.md @@ -0,0 +1,8 @@ +# JMX Metric Scraper + +This utility provides a way to query JMX metrics and export them to an OTLP endpoint. +The JMX MBeans and their metric mappings are defined in YAML and reuse implementation from +[jmx-metrics instrumentation](https://github.com/open-telemetry/opentelemetry-java-instrumentation/tree/main/instrumentation/jmx-metrics). + +This is currently a work-in-progress component not ready to be used in production. +The end goal is to provide an alternative to the [JMX Gatherer](../jmx-metrics/README.md) utility. diff --git a/jmx-scraper/build.gradle.kts b/jmx-scraper/build.gradle.kts new file mode 100644 index 000000000..bab7b7cb7 --- /dev/null +++ b/jmx-scraper/build.gradle.kts @@ -0,0 +1,94 @@ +plugins { + application + id("com.github.johnrengelman.shadow") + + id("otel.java-conventions") + + // TODO publishing disabled until component is ready to be used + // id("otel.publish-conventions") +} + +description = "JMX metrics scraper" +otelJava.moduleName.set("io.opentelemetry.contrib.jmxscraper") + +application.mainClass.set("io.opentelemetry.contrib.jmxscraper.JmxScraper") + +dependencies { + // TODO remove snapshot dependency on upstream once 2.9.0 is released + // api(enforcedPlatform("io.opentelemetry.instrumentation:opentelemetry-instrumentation-bom-alpha:2.9.0-SNAPSHOT-alpha",)) + + implementation("io.opentelemetry:opentelemetry-api") + implementation("io.opentelemetry:opentelemetry-sdk") + implementation("io.opentelemetry:opentelemetry-sdk-metrics") + implementation("io.opentelemetry:opentelemetry-sdk-extension-autoconfigure") + + implementation("io.opentelemetry.instrumentation:opentelemetry-jmx-metrics") + + testImplementation("org.junit-pioneer:junit-pioneer") + testImplementation("io.opentelemetry:opentelemetry-sdk-testing") +} + +testing { + suites { + val integrationTest by registering(JvmTestSuite::class) { + dependencies { + implementation("org.testcontainers:junit-jupiter") + implementation("org.slf4j:slf4j-simple") + implementation("com.linecorp.armeria:armeria-junit5") + implementation("com.linecorp.armeria:armeria-grpc") + implementation("io.opentelemetry.proto:opentelemetry-proto:0.20.0-alpha") + } + } + } +} + +tasks { + shadowJar { + mergeServiceFiles() + + manifest { + attributes["Implementation-Version"] = project.version + } + // This should always be standalone, so remove "-all" to prevent unnecessary artifact. + archiveClassifier.set("") + } + + jar { + archiveClassifier.set("noshadow") + } + + withType().configureEach { + dependsOn(shadowJar) + dependsOn(named("appJar")) + systemProperty("shadow.jar.path", shadowJar.get().archiveFile.get().asFile.absolutePath) + systemProperty("app.jar.path", named("appJar").get().archiveFile.get().asFile.absolutePath) + systemProperty("gradle.project.version", "${project.version}") + } + + // Because we reconfigure publishing to only include the shadow jar, the Gradle metadata is not correct. + // Since we are fully bundled and have no dependencies, Gradle metadata wouldn't provide any advantage over + // the POM anyways so in practice we shouldn't be losing anything. + withType().configureEach { + enabled = false + } +} + +tasks.register("appJar") { + from(sourceSets.get("integrationTest").output) + archiveClassifier.set("app") + manifest { + attributes["Main-Class"] = "io.opentelemetry.contrib.jmxscraper.TestApp" + } +} + +// Don't publish non-shadowed jar (shadowJar is in shadowRuntimeElements) +with(components["java"] as AdhocComponentWithVariants) { + configurations.forEach { + withVariantsFromConfiguration(configurations["apiElements"]) { + skip() + } + withVariantsFromConfiguration(configurations["runtimeElements"]) { + skip() + } + } +} diff --git a/jmx-scraper/src/integrationTest/java/io/opentelemetry/contrib/jmxscraper/JmxConnectorBuilderTest.java b/jmx-scraper/src/integrationTest/java/io/opentelemetry/contrib/jmxscraper/JmxConnectorBuilderTest.java new file mode 100644 index 000000000..132605242 --- /dev/null +++ b/jmx-scraper/src/integrationTest/java/io/opentelemetry/contrib/jmxscraper/JmxConnectorBuilderTest.java @@ -0,0 +1,92 @@ +/* + * Copyright The OpenTelemetry Authors + * SPDX-License-Identifier: Apache-2.0 + */ + +package io.opentelemetry.contrib.jmxscraper; + +import static org.assertj.core.api.Assertions.assertThat; + +import java.io.IOException; +import javax.management.ObjectName; +import javax.management.remote.JMXConnector; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.testcontainers.containers.Network; + +public class JmxConnectorBuilderTest { + + private static Network network; + + @BeforeAll + static void beforeAll() { + network = Network.newNetwork(); + } + + @AfterAll + static void afterAll() { + network.close(); + } + + @Test + void noAuth() { + try (TestAppContainer app = new TestAppContainer().withNetwork(network).withJmxPort(9990)) { + app.start(); + testConnector( + () -> JmxConnectorBuilder.createNew(app.getHost(), app.getMappedPort(9990)).build()); + } + } + + @Test + void loginPwdAuth() { + String login = "user"; + String pwd = "t0p!Secret"; + try (TestAppContainer app = + new TestAppContainer().withNetwork(network).withJmxPort(9999).withUserAuth(login, pwd)) { + app.start(); + testConnector( + () -> + JmxConnectorBuilder.createNew(app.getHost(), app.getMappedPort(9999)) + .userCredentials(login, pwd) + .build()); + } + } + + @Test + void serverSSL() { + // TODO: test with SSL enabled as RMI registry seems to work differently with SSL + + // create keypair (public,private) + // create server keystore with private key + // configure server keystore + // + // create client truststore with public key + // can we configure to use a custom truststore ??? + // connect to server + } + + private static void testConnector(ConnectorSupplier connectorSupplier) { + try (JMXConnector connector = connectorSupplier.get()) { + assertThat(connector.getMBeanServerConnection()) + .isNotNull() + .satisfies( + connection -> { + try { + ObjectName name = new ObjectName(TestApp.OBJECT_NAME); + Object value = connection.getAttribute(name, "IntValue"); + assertThat(value).isEqualTo(42); + } catch (Exception e) { + throw new RuntimeException(e); + } + }); + + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + private interface ConnectorSupplier { + JMXConnector get() throws IOException; + } +} diff --git a/jmx-scraper/src/integrationTest/java/io/opentelemetry/contrib/jmxscraper/JmxScraperContainer.java b/jmx-scraper/src/integrationTest/java/io/opentelemetry/contrib/jmxscraper/JmxScraperContainer.java new file mode 100644 index 000000000..f85a5ba17 --- /dev/null +++ b/jmx-scraper/src/integrationTest/java/io/opentelemetry/contrib/jmxscraper/JmxScraperContainer.java @@ -0,0 +1,107 @@ +/* + * Copyright The OpenTelemetry Authors + * SPDX-License-Identifier: Apache-2.0 + */ + +package io.opentelemetry.contrib.jmxscraper; + +import static org.assertj.core.api.Assertions.assertThat; + +import com.google.errorprone.annotations.CanIgnoreReturnValue; +import java.time.Duration; +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Locale; +import java.util.Set; +import org.testcontainers.containers.GenericContainer; +import org.testcontainers.containers.wait.strategy.Wait; +import org.testcontainers.utility.MountableFile; + +/** Test container that allows to execute {@link JmxScraper} in an isolated container */ +public class JmxScraperContainer extends GenericContainer { + + private final String endpoint; + private final Set targetSystems; + private String serviceUrl; + private int intervalMillis; + private final Set customYamlFiles; + + public JmxScraperContainer(String otlpEndpoint) { + super("openjdk:8u272-jre-slim"); + + String scraperJarPath = System.getProperty("shadow.jar.path"); + assertThat(scraperJarPath).isNotNull(); + + this.withCopyFileToContainer(MountableFile.forHostPath(scraperJarPath), "/scraper.jar") + .waitingFor( + Wait.forLogMessage(".*JMX scraping started.*", 1) + .withStartupTimeout(Duration.ofSeconds(10))); + + this.endpoint = otlpEndpoint; + this.targetSystems = new HashSet<>(); + this.customYamlFiles = new HashSet<>(); + this.intervalMillis = 1000; + } + + @CanIgnoreReturnValue + public JmxScraperContainer withTargetSystem(String targetSystem) { + targetSystems.add(targetSystem); + return this; + } + + @CanIgnoreReturnValue + public JmxScraperContainer withIntervalMillis(int intervalMillis) { + this.intervalMillis = intervalMillis; + return this; + } + + @CanIgnoreReturnValue + public JmxScraperContainer withService(String host, int port) { + // TODO: adding a way to provide 'host:port' syntax would make this easier for end users + this.serviceUrl = + String.format( + Locale.getDefault(), "service:jmx:rmi:///jndi/rmi://%s:%d/jmxrmi", host, port); + return this; + } + + @CanIgnoreReturnValue + public JmxScraperContainer withCustomYaml(String yamlPath) { + this.customYamlFiles.add(yamlPath); + return this; + } + + @Override + public void start() { + // for now only configure through JVM args + List arguments = new ArrayList<>(); + arguments.add("java"); + arguments.add("-Dotel.exporter.otlp.endpoint=" + endpoint); + + if (!targetSystems.isEmpty()) { + arguments.add("-Dotel.jmx.target.system=" + String.join(",", targetSystems)); + } + + if (serviceUrl == null) { + throw new IllegalStateException("Missing service URL"); + } + arguments.add("-Dotel.jmx.service.url=" + serviceUrl); + arguments.add("-Dotel.jmx.interval.milliseconds=" + intervalMillis); + + if (!customYamlFiles.isEmpty()) { + for (String yaml : customYamlFiles) { + this.withCopyFileToContainer(MountableFile.forClasspathResource(yaml), yaml); + } + arguments.add("-Dotel.jmx.config=" + String.join(",", customYamlFiles)); + } + + arguments.add("-jar"); + arguments.add("/scraper.jar"); + + this.withCommand(arguments.toArray(new String[0])); + + logger().info("Starting scraper with command: " + String.join(" ", arguments)); + + super.start(); + } +} diff --git a/jmx-scraper/src/integrationTest/java/io/opentelemetry/contrib/jmxscraper/TestApp.java b/jmx-scraper/src/integrationTest/java/io/opentelemetry/contrib/jmxscraper/TestApp.java new file mode 100644 index 000000000..1316ca036 --- /dev/null +++ b/jmx-scraper/src/integrationTest/java/io/opentelemetry/contrib/jmxscraper/TestApp.java @@ -0,0 +1,60 @@ +/* + * Copyright The OpenTelemetry Authors + * SPDX-License-Identifier: Apache-2.0 + */ + +package io.opentelemetry.contrib.jmxscraper; + +import java.lang.management.ManagementFactory; +import javax.management.MBeanServer; +import javax.management.ObjectName; + +@SuppressWarnings("all") +public class TestApp implements TestAppMXBean { + + public static final String APP_STARTED_MSG = "app started"; + public static final String OBJECT_NAME = "io.opentelemetry.test:name=TestApp"; + + private volatile boolean running; + + public static void main(String[] args) { + TestApp app = TestApp.start(); + while (app.isRunning()) { + try { + Thread.sleep(100); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + } + } + + private TestApp() {} + + static TestApp start() { + TestApp app = new TestApp(); + MBeanServer mbs = ManagementFactory.getPlatformMBeanServer(); + try { + ObjectName objectName = new ObjectName(OBJECT_NAME); + mbs.registerMBean(app, objectName); + } catch (Exception e) { + throw new RuntimeException(e); + } + app.running = true; + System.out.println(APP_STARTED_MSG); + return app; + } + + @Override + public int getIntValue() { + return 42; + } + + @Override + public void stopApp() { + running = false; + } + + boolean isRunning() { + return running; + } +} diff --git a/jmx-scraper/src/integrationTest/java/io/opentelemetry/contrib/jmxscraper/TestAppContainer.java b/jmx-scraper/src/integrationTest/java/io/opentelemetry/contrib/jmxscraper/TestAppContainer.java new file mode 100644 index 000000000..a38dd7ace --- /dev/null +++ b/jmx-scraper/src/integrationTest/java/io/opentelemetry/contrib/jmxscraper/TestAppContainer.java @@ -0,0 +1,126 @@ +/* + * Copyright The OpenTelemetry Authors + * SPDX-License-Identifier: Apache-2.0 + */ + +package io.opentelemetry.contrib.jmxscraper; + +import static org.assertj.core.api.Assertions.assertThat; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.time.Duration; +import java.util.HashMap; +import java.util.Map; +import java.util.stream.Collectors; +import org.testcontainers.containers.GenericContainer; +import org.testcontainers.containers.wait.strategy.Wait; +import org.testcontainers.shaded.com.google.errorprone.annotations.CanIgnoreReturnValue; +import org.testcontainers.utility.MountableFile; + +/** Test container that allows to execute {@link TestApp} in an isolated container */ +public class TestAppContainer extends GenericContainer { + + private final Map properties; + private int port; + private String login; + private String pwd; + + public TestAppContainer() { + super("openjdk:8u272-jre-slim"); + + this.properties = new HashMap<>(); + + String appJar = System.getProperty("app.jar.path"); + assertThat(Paths.get(appJar)).isNotEmptyFile().isReadable(); + + this.withCopyFileToContainer(MountableFile.forHostPath(appJar), "/app.jar") + .waitingFor( + Wait.forLogMessage(TestApp.APP_STARTED_MSG + "\\n", 1) + .withStartupTimeout(Duration.ofSeconds(5))) + .withCommand("java", "-jar", "/app.jar"); + } + + @CanIgnoreReturnValue + public TestAppContainer withJmxPort(int port) { + this.port = port; + properties.put("com.sun.management.jmxremote.port", Integer.toString(port)); + return this.withExposedPorts(port); + } + + @CanIgnoreReturnValue + public TestAppContainer withUserAuth(String login, String pwd) { + this.login = login; + this.pwd = pwd; + return this; + } + + @Override + public void start() { + + // TODO: add support for ssl + properties.put("com.sun.management.jmxremote.ssl", "false"); + + if (pwd == null) { + properties.put("com.sun.management.jmxremote.authenticate", "false"); + } else { + properties.put("com.sun.management.jmxremote.authenticate", "true"); + + Path pwdFile = createPwdFile(login, pwd); + this.withCopyFileToContainer(MountableFile.forHostPath(pwdFile), "/jmx.password"); + properties.put("com.sun.management.jmxremote.password.file", "/jmx.password"); + + Path accessFile = createAccessFile(login); + this.withCopyFileToContainer(MountableFile.forHostPath(accessFile), "/jmx.access"); + properties.put("com.sun.management.jmxremote.access.file", "/jmx.access"); + } + + String confArgs = + properties.entrySet().stream() + .map( + e -> { + String s = "-D" + e.getKey(); + if (!e.getValue().isEmpty()) { + s += "=" + e.getValue(); + } + return s; + }) + .collect(Collectors.joining(" ")); + + this.withEnv("JAVA_TOOL_OPTIONS", confArgs); + + logger().info("Test application JAVA_TOOL_OPTIONS = " + confArgs); + + super.start(); + + logger().info("Test application JMX port mapped to {}:{}", getHost(), getMappedPort(port)); + } + + private static Path createPwdFile(String login, String pwd) { + try { + Path path = Files.createTempFile("test", ".pwd"); + writeLine(path, String.format("%s %s", login, pwd)); + return path; + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + private static Path createAccessFile(String login) { + try { + Path path = Files.createTempFile("test", ".pwd"); + writeLine(path, String.format("%s %s", login, "readwrite")); + return path; + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + private static void writeLine(Path path, String line) throws IOException { + line = line + "\n"; + Files.write(path, line.getBytes(StandardCharsets.UTF_8)); + } +} diff --git a/jmx-scraper/src/integrationTest/java/io/opentelemetry/contrib/jmxscraper/TestAppMXBean.java b/jmx-scraper/src/integrationTest/java/io/opentelemetry/contrib/jmxscraper/TestAppMXBean.java new file mode 100644 index 000000000..11ea69905 --- /dev/null +++ b/jmx-scraper/src/integrationTest/java/io/opentelemetry/contrib/jmxscraper/TestAppMXBean.java @@ -0,0 +1,14 @@ +/* + * Copyright The OpenTelemetry Authors + * SPDX-License-Identifier: Apache-2.0 + */ + +package io.opentelemetry.contrib.jmxscraper; + +@SuppressWarnings("unused") +public interface TestAppMXBean { + + int getIntValue(); + + void stopApp(); +} diff --git a/jmx-scraper/src/integrationTest/java/io/opentelemetry/contrib/jmxscraper/target_systems/JvmIntegrationTest.java b/jmx-scraper/src/integrationTest/java/io/opentelemetry/contrib/jmxscraper/target_systems/JvmIntegrationTest.java new file mode 100644 index 000000000..4c240ee16 --- /dev/null +++ b/jmx-scraper/src/integrationTest/java/io/opentelemetry/contrib/jmxscraper/target_systems/JvmIntegrationTest.java @@ -0,0 +1,31 @@ +/* + * Copyright The OpenTelemetry Authors + * SPDX-License-Identifier: Apache-2.0 + */ + +package io.opentelemetry.contrib.jmxscraper.target_systems; + +import io.opentelemetry.contrib.jmxscraper.JmxScraperContainer; +import io.opentelemetry.contrib.jmxscraper.TestAppContainer; +import io.opentelemetry.proto.collector.metrics.v1.ExportMetricsServiceRequest; +import java.util.List; +import org.testcontainers.containers.GenericContainer; + +public class JvmIntegrationTest extends TargetSystemIntegrationTest { + + @Override + protected GenericContainer createTargetContainer(int jmxPort) { + // reusing test application for JVM metrics and custom yaml + return new TestAppContainer().withJmxPort(jmxPort); + } + + @Override + protected JmxScraperContainer customizeScraperContainer(JmxScraperContainer scraper) { + return scraper.withTargetSystem("jvm"); + } + + @Override + protected void verifyMetrics(List metrics) { + // TODO: Verify gathered metrics + } +} diff --git a/jmx-scraper/src/integrationTest/java/io/opentelemetry/contrib/jmxscraper/target_systems/TargetSystemIntegrationTest.java b/jmx-scraper/src/integrationTest/java/io/opentelemetry/contrib/jmxscraper/target_systems/TargetSystemIntegrationTest.java new file mode 100644 index 000000000..0552aa3bd --- /dev/null +++ b/jmx-scraper/src/integrationTest/java/io/opentelemetry/contrib/jmxscraper/target_systems/TargetSystemIntegrationTest.java @@ -0,0 +1,164 @@ +/* + * Copyright The OpenTelemetry Authors + * SPDX-License-Identifier: Apache-2.0 + */ + +package io.opentelemetry.contrib.jmxscraper.target_systems; + +import static org.assertj.core.api.Assertions.assertThat; + +import com.linecorp.armeria.server.ServerBuilder; +import com.linecorp.armeria.server.grpc.GrpcService; +import com.linecorp.armeria.testing.junit5.server.ServerExtension; +import io.grpc.stub.StreamObserver; +import io.opentelemetry.contrib.jmxscraper.JmxConnectorBuilder; +import io.opentelemetry.contrib.jmxscraper.JmxScraperContainer; +import io.opentelemetry.proto.collector.metrics.v1.ExportMetricsServiceRequest; +import io.opentelemetry.proto.collector.metrics.v1.ExportMetricsServiceResponse; +import io.opentelemetry.proto.collector.metrics.v1.MetricsServiceGrpc; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.LinkedBlockingDeque; +import javax.management.remote.JMXConnector; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.testcontainers.Testcontainers; +import org.testcontainers.containers.GenericContainer; +import org.testcontainers.containers.Network; +import org.testcontainers.containers.output.Slf4jLogConsumer; + +public abstract class TargetSystemIntegrationTest { + + private static final Logger logger = LoggerFactory.getLogger(TargetSystemIntegrationTest.class); + private static final String TARGET_SYSTEM_NETWORK_ALIAS = "targetsystem"; + private static String otlpEndpoint; + + /** + * Create target system container + * + * @param jmxPort JMX port target JVM should listen to + * @return target system container + */ + protected abstract GenericContainer createTargetContainer(int jmxPort); + + private static Network network; + private static OtlpGrpcServer otlpServer; + private GenericContainer target; + private JmxScraperContainer scraper; + + private static final String OTLP_HOST = "host.testcontainers.internal"; + private static final int JMX_PORT = 9999; + + @BeforeAll + static void beforeAll() { + network = Network.newNetwork(); + otlpServer = new OtlpGrpcServer(); + otlpServer.start(); + Testcontainers.exposeHostPorts(otlpServer.httpPort()); + otlpEndpoint = "http://" + OTLP_HOST + ":" + otlpServer.httpPort(); + } + + @AfterAll + static void afterAll() { + network.close(); + try { + otlpServer.stop().get(); + } catch (InterruptedException | ExecutionException e) { + throw new RuntimeException(e); + } + } + + @AfterEach + void afterEach() { + if (target != null && target.isRunning()) { + target.stop(); + } + if (scraper != null && scraper.isRunning()) { + scraper.stop(); + } + if (otlpServer != null) { + otlpServer.reset(); + } + } + + @Test + void endToEndTest() { + + target = + createTargetContainer(JMX_PORT) + .withLogConsumer(new Slf4jLogConsumer(logger)) + .withNetwork(network) + .withExposedPorts(JMX_PORT) + .withNetworkAliases(TARGET_SYSTEM_NETWORK_ALIAS); + target.start(); + + String targetHost = target.getHost(); + Integer targetPort = target.getMappedPort(JMX_PORT); + logger.info( + "Target system started, JMX port: {} mapped to {}:{}", JMX_PORT, targetHost, targetPort); + + // TODO : wait for metrics to be sent and add assertions on what is being captured + // for now we just test that we can connect to remote JMX using our client. + try (JMXConnector connector = JmxConnectorBuilder.createNew(targetHost, targetPort).build()) { + assertThat(connector.getMBeanServerConnection()).isNotNull(); + } catch (IOException e) { + throw new RuntimeException(e); + } + + scraper = + new JmxScraperContainer(otlpEndpoint) + .withNetwork(network) + .withService(TARGET_SYSTEM_NETWORK_ALIAS, JMX_PORT); + + scraper = customizeScraperContainer(scraper); + scraper.start(); + + verifyMetrics(otlpServer.getMetrics()); + } + + protected abstract void verifyMetrics(List metrics); + + protected JmxScraperContainer customizeScraperContainer(JmxScraperContainer scraper) { + return scraper; + } + + private static class OtlpGrpcServer extends ServerExtension { + + private final BlockingQueue metricRequests = + new LinkedBlockingDeque<>(); + + List getMetrics() { + return new ArrayList<>(metricRequests); + } + + void reset() { + metricRequests.clear(); + } + + @Override + protected void configure(ServerBuilder sb) { + sb.service( + GrpcService.builder() + .addService( + new MetricsServiceGrpc.MetricsServiceImplBase() { + @Override + public void export( + ExportMetricsServiceRequest request, + StreamObserver responseObserver) { + metricRequests.add(request); + responseObserver.onNext(ExportMetricsServiceResponse.getDefaultInstance()); + responseObserver.onCompleted(); + } + }) + .build()); + sb.http(0); + } + } +} diff --git a/jmx-scraper/src/integrationTest/java/io/opentelemetry/contrib/jmxscraper/target_systems/TomcatIntegrationTest.java b/jmx-scraper/src/integrationTest/java/io/opentelemetry/contrib/jmxscraper/target_systems/TomcatIntegrationTest.java new file mode 100644 index 000000000..ccf7e59a7 --- /dev/null +++ b/jmx-scraper/src/integrationTest/java/io/opentelemetry/contrib/jmxscraper/target_systems/TomcatIntegrationTest.java @@ -0,0 +1,54 @@ +/* + * Copyright The OpenTelemetry Authors + * SPDX-License-Identifier: Apache-2.0 + */ + +package io.opentelemetry.contrib.jmxscraper.target_systems; + +import io.opentelemetry.contrib.jmxscraper.JmxScraperContainer; +import io.opentelemetry.proto.collector.metrics.v1.ExportMetricsServiceRequest; +import java.time.Duration; +import java.util.List; +import org.testcontainers.containers.GenericContainer; +import org.testcontainers.containers.wait.strategy.Wait; +import org.testcontainers.images.builder.ImageFromDockerfile; + +public class TomcatIntegrationTest extends TargetSystemIntegrationTest { + + @Override + protected GenericContainer createTargetContainer(int jmxPort) { + return new GenericContainer<>( + new ImageFromDockerfile() + .withDockerfileFromBuilder( + builder -> + builder + .from("tomcat:9.0") + .run("rm", "-fr", "/usr/local/tomcat/webapps/ROOT") + .add( + "https://tomcat.apache.org/tomcat-9.0-doc/appdev/sample/sample.war", + "/usr/local/tomcat/webapps/ROOT.war") + .build())) + .withEnv("LOCAL_JMX", "no") + .withEnv( + "CATALINA_OPTS", + "-Dcom.sun.management.jmxremote.local.only=false" + + " -Dcom.sun.management.jmxremote.authenticate=false" + + " -Dcom.sun.management.jmxremote.ssl=false" + + " -Dcom.sun.management.jmxremote.port=" + + jmxPort + + " -Dcom.sun.management.jmxremote.rmi.port=" + + jmxPort) + .withStartupTimeout(Duration.ofMinutes(2)) + .waitingFor(Wait.forListeningPort()); + } + + @Override + protected JmxScraperContainer customizeScraperContainer(JmxScraperContainer scraper) { + return scraper.withTargetSystem("tomcat"); + } + + @Override + protected void verifyMetrics(List metrics) { + // TODO: Verify gathered metrics + } +} diff --git a/jmx-scraper/src/main/java/io/opentelemetry/contrib/jmxscraper/ArgumentsParsingException.java b/jmx-scraper/src/main/java/io/opentelemetry/contrib/jmxscraper/ArgumentsParsingException.java new file mode 100644 index 000000000..f006269cb --- /dev/null +++ b/jmx-scraper/src/main/java/io/opentelemetry/contrib/jmxscraper/ArgumentsParsingException.java @@ -0,0 +1,14 @@ +/* + * Copyright The OpenTelemetry Authors + * SPDX-License-Identifier: Apache-2.0 + */ + +package io.opentelemetry.contrib.jmxscraper; + +public class ArgumentsParsingException extends Exception { + private static final long serialVersionUID = 0L; + + public ArgumentsParsingException(String msg) { + super(msg); + } +} diff --git a/jmx-scraper/src/main/java/io/opentelemetry/contrib/jmxscraper/JmxConnectorBuilder.java b/jmx-scraper/src/main/java/io/opentelemetry/contrib/jmxscraper/JmxConnectorBuilder.java new file mode 100644 index 000000000..dd509aa13 --- /dev/null +++ b/jmx-scraper/src/main/java/io/opentelemetry/contrib/jmxscraper/JmxConnectorBuilder.java @@ -0,0 +1,161 @@ +/* + * Copyright The OpenTelemetry Authors + * SPDX-License-Identifier: Apache-2.0 + */ + +package io.opentelemetry.contrib.jmxscraper; + +import com.google.errorprone.annotations.CanIgnoreReturnValue; +import java.io.IOException; +import java.net.MalformedURLException; +import java.security.Provider; +import java.security.Security; +import java.util.HashMap; +import java.util.Locale; +import java.util.Map; +import java.util.logging.Level; +import java.util.logging.Logger; +import javax.annotation.Nullable; +import javax.management.remote.JMXConnector; +import javax.management.remote.JMXConnectorFactory; +import javax.management.remote.JMXServiceURL; +import javax.security.auth.callback.Callback; +import javax.security.auth.callback.CallbackHandler; +import javax.security.auth.callback.NameCallback; +import javax.security.auth.callback.PasswordCallback; +import javax.security.auth.callback.UnsupportedCallbackException; +import javax.security.sasl.RealmCallback; + +public class JmxConnectorBuilder { + + private static final Logger logger = Logger.getLogger(JmxConnectorBuilder.class.getName()); + + private final JMXServiceURL url; + @Nullable private String userName; + @Nullable private String password; + @Nullable private String profile; + @Nullable private String realm; + private boolean sslRegistry; + + private JmxConnectorBuilder(JMXServiceURL url) { + this.url = url; + } + + public static JmxConnectorBuilder createNew(String host, int port) { + return new JmxConnectorBuilder(buildUrl(host, port)); + } + + public static JmxConnectorBuilder createNew(String url) { + return new JmxConnectorBuilder(buildUrl(url)); + } + + @CanIgnoreReturnValue + public JmxConnectorBuilder userCredentials(String userName, String password) { + this.userName = userName; + this.password = password; + return this; + } + + @CanIgnoreReturnValue + public JmxConnectorBuilder withRemoteProfile(String profile) { + this.profile = profile; + return this; + } + + @CanIgnoreReturnValue + public JmxConnectorBuilder withRealm(String realm) { + this.realm = realm; + return this; + } + + @CanIgnoreReturnValue + public JmxConnectorBuilder withSslRegistry() { + this.sslRegistry = true; + return this; + } + + /** + * Builds JMX connector instance by connecting to the remote JMX endpoint + * + * @return JMX connector + * @throws IOException in case of communication error + */ + public JMXConnector build() throws IOException { + Map env = buildEnv(); + + try { + if (sslRegistry) { + return doConnectSslRegistry(url, env); + } + + return doConnect(url, env); + + } catch (IOException e) { + throw new IOException("Unable to connect to " + url.getHost() + ":" + url.getPort(), e); + } + } + + private Map buildEnv() { + Map env = new HashMap<>(); + if (userName != null && password != null) { + env.put(JMXConnector.CREDENTIALS, new String[] {userName, password}); + } + + if (profile != null) { + env.put("jmx.remote.profile", profile); + } + + try { + // Not all supported versions of Java contain this Provider + // Also it might not be accessible due to java.security.sasl module not accessible + Class klass = Class.forName("com.sun.security.sasl.Provider"); + Provider provider = (Provider) klass.getDeclaredConstructor().newInstance(); + Security.addProvider(provider); + + env.put( + "jmx.remote.sasl.callback.handler", + (CallbackHandler) + callbacks -> { + for (Callback callback : callbacks) { + if (callback instanceof NameCallback) { + ((NameCallback) callback).setName(userName); + } else if (callback instanceof PasswordCallback) { + char[] pwd = password == null ? null : password.toCharArray(); + ((PasswordCallback) callback).setPassword(pwd); + } else if (callback instanceof RealmCallback) { + ((RealmCallback) callback).setText(realm); + } else { + throw new UnsupportedCallbackException(callback); + } + } + }); + } catch (ReflectiveOperationException e) { + logger.log(Level.WARNING, "SASL unsupported in current environment: " + e.getMessage()); + } + return env; + } + + @SuppressWarnings("BanJNDI") + private static JMXConnector doConnect(JMXServiceURL url, Map env) + throws IOException { + return JMXConnectorFactory.connect(url, env); + } + + public JMXConnector doConnectSslRegistry(JMXServiceURL url, Map env) { + throw new IllegalStateException("TODO"); + } + + private static JMXServiceURL buildUrl(String host, int port) { + return buildUrl( + String.format( + Locale.getDefault(), "service:jmx:rmi:///jndi/rmi://%s:%d/jmxrmi", host, port)); + } + + private static JMXServiceURL buildUrl(String url) { + try { + return new JMXServiceURL(url); + } catch (MalformedURLException e) { + throw new IllegalArgumentException("invalid url", e); + } + } +} diff --git a/jmx-scraper/src/main/java/io/opentelemetry/contrib/jmxscraper/JmxScraper.java b/jmx-scraper/src/main/java/io/opentelemetry/contrib/jmxscraper/JmxScraper.java new file mode 100644 index 000000000..ebed7c780 --- /dev/null +++ b/jmx-scraper/src/main/java/io/opentelemetry/contrib/jmxscraper/JmxScraper.java @@ -0,0 +1,137 @@ +/* + * Copyright The OpenTelemetry Authors + * SPDX-License-Identifier: Apache-2.0 + */ + +package io.opentelemetry.contrib.jmxscraper; + +import io.opentelemetry.contrib.jmxscraper.config.ConfigurationException; +import io.opentelemetry.contrib.jmxscraper.config.JmxScraperConfig; +import java.io.DataInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.nio.file.Files; +import java.nio.file.Paths; +import java.util.Arrays; +import java.util.List; +import java.util.Properties; +import java.util.logging.Logger; +import javax.management.MBeanServerConnection; +import javax.management.remote.JMXConnector; + +public class JmxScraper { + private static final Logger logger = Logger.getLogger(JmxScraper.class.getName()); + private static final String CONFIG_ARG = "-config"; + + private final JmxConnectorBuilder client; + + // TODO depend on instrumentation 2.9.0 snapshot + // private final JmxMetricInsight service; + + /** + * Main method to create and run a {@link JmxScraper} instance. + * + * @param args - must be of the form "-config {jmx_config_path,'-'}" + */ + @SuppressWarnings({"SystemOut", "SystemExitOutsideMain"}) + public static void main(String[] args) { + try { + JmxScraperConfig config = + JmxScraperConfig.fromProperties(parseArgs(Arrays.asList(args)), System.getProperties()); + // propagate effective user-provided configuration to JVM system properties + config.propagateSystemProperties(); + // TODO: depend on instrumentation 2.9.0 snapshot + // service = JmxMetricInsight.createService(GlobalOpenTelemetry.get(), + // config.getIntervalMilliseconds()); + JmxScraper jmxScraper = new JmxScraper(JmxConnectorBuilder.createNew(config.getServiceUrl())); + jmxScraper.start(); + + } catch (ArgumentsParsingException e) { + System.err.println("ERROR: " + e.getMessage()); + System.err.println( + "Usage: java -jar " + + "-config "); + System.exit(1); + } catch (ConfigurationException e) { + System.err.println(e.getMessage()); + System.exit(1); + } catch (IOException e) { + System.err.println("Unable to connect " + e.getMessage()); + System.exit(2); + } + } + + /** + * Create {@link Properties} from command line options + * + * @param args application commandline arguments + */ + static Properties parseArgs(List args) + throws ArgumentsParsingException, ConfigurationException { + + if (args.isEmpty()) { + // empty properties from stdin or external file + // config could still be provided through JVM system properties + return new Properties(); + } + if (args.size() != 2) { + throw new ArgumentsParsingException("exactly two arguments expected, got " + args.size()); + } + if (!args.get(0).equalsIgnoreCase(CONFIG_ARG)) { + throw new ArgumentsParsingException("unexpected first argument must be '" + CONFIG_ARG + "'"); + } + + String path = args.get(1); + if (path.trim().equals("-")) { + return loadPropertiesFromStdin(); + } else { + return loadPropertiesFromPath(path); + } + } + + private static Properties loadPropertiesFromStdin() throws ConfigurationException { + Properties properties = new Properties(); + try (InputStream is = new DataInputStream(System.in)) { + properties.load(is); + return properties; + } catch (IOException e) { + throw new ConfigurationException("Failed to read config properties from stdin", e); + } + } + + private static Properties loadPropertiesFromPath(String path) throws ConfigurationException { + Properties properties = new Properties(); + try (InputStream is = Files.newInputStream(Paths.get(path))) { + properties.load(is); + return properties; + } catch (IOException e) { + throw new ConfigurationException("Failed to read config properties file: '" + path + "'", e); + } + } + + JmxScraper(JmxConnectorBuilder client) { + this.client = client; + } + + private void start() throws IOException { + + JMXConnector connector = client.build(); + + @SuppressWarnings("unused") + MBeanServerConnection connection = connector.getMBeanServerConnection(); + + // TODO: depend on instrumentation 2.9.0 snapshot + // MetricConfiguration metricConfig = new MetricConfiguration(); + // TODO create JMX insight config from scraper config + // service.startRemote(metricConfig, () -> Collections.singletonList(connection)); + + logger.info("JMX scraping started"); + + // TODO: wait a bit to keep the JVM running, this won't be needed once calling jmx insight + try { + Thread.sleep(5000); + } catch (InterruptedException e) { + throw new IllegalStateException(e); + } + } +} diff --git a/jmx-scraper/src/main/java/io/opentelemetry/contrib/jmxscraper/config/ConfigurationException.java b/jmx-scraper/src/main/java/io/opentelemetry/contrib/jmxscraper/config/ConfigurationException.java new file mode 100644 index 000000000..76c69998a --- /dev/null +++ b/jmx-scraper/src/main/java/io/opentelemetry/contrib/jmxscraper/config/ConfigurationException.java @@ -0,0 +1,18 @@ +/* + * Copyright The OpenTelemetry Authors + * SPDX-License-Identifier: Apache-2.0 + */ + +package io.opentelemetry.contrib.jmxscraper.config; + +public class ConfigurationException extends Exception { + private static final long serialVersionUID = 0L; + + public ConfigurationException(String message, Throwable cause) { + super(message, cause); + } + + public ConfigurationException(String message) { + super(message); + } +} diff --git a/jmx-scraper/src/main/java/io/opentelemetry/contrib/jmxscraper/config/JmxScraperConfig.java b/jmx-scraper/src/main/java/io/opentelemetry/contrib/jmxscraper/config/JmxScraperConfig.java new file mode 100644 index 000000000..edb7599fd --- /dev/null +++ b/jmx-scraper/src/main/java/io/opentelemetry/contrib/jmxscraper/config/JmxScraperConfig.java @@ -0,0 +1,248 @@ +/* + * Copyright The OpenTelemetry Authors + * SPDX-License-Identifier: Apache-2.0 + */ + +package io.opentelemetry.contrib.jmxscraper.config; + +import static io.opentelemetry.contrib.jmxscraper.internal.StringUtils.isBlank; + +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Properties; +import java.util.Set; +import java.util.stream.Collectors; + +/** This class keeps application settings */ +public class JmxScraperConfig { + + static final String SERVICE_URL = "otel.jmx.service.url"; + static final String CUSTOM_JMX_SCRAPING_CONFIG = "otel.jmx.custom.scraping.config"; + static final String TARGET_SYSTEM = "otel.jmx.target.system"; + static final String INTERVAL_MILLISECONDS = "otel.jmx.interval.milliseconds"; + static final String METRICS_EXPORTER_TYPE = "otel.metrics.exporter"; + static final String EXPORTER_INTERVAL = "otel.metric.export.interval"; + static final String REGISTRY_SSL = "otel.jmx.remote.registry.ssl"; + + static final String OTLP_ENDPOINT = "otel.exporter.otlp.endpoint"; + + static final String JMX_USERNAME = "otel.jmx.username"; + static final String JMX_PASSWORD = "otel.jmx.password"; + static final String JMX_REMOTE_PROFILE = "otel.jmx.remote.profile"; + static final String JMX_REALM = "otel.jmx.realm"; + + static final String OTLP_METRICS_EXPORTER = "otlp"; + + static final List AVAILABLE_TARGET_SYSTEMS = + Collections.unmodifiableList( + Arrays.asList( + "activemq", + "cassandra", + "hbase", + "hadoop", + "jetty", + "jvm", + "kafka", + "kafka-consumer", + "kafka-producer", + "solr", + "tomcat", + "wildfly")); + + private String serviceUrl = ""; + private String customJmxScrapingConfigPath = ""; + private Set targetSystems = Collections.emptySet(); + private int intervalMilliseconds; + private String metricsExporterType = ""; + private String otlpExporterEndpoint = ""; + private String username = ""; + private String password = ""; + private String realm = ""; + private String remoteProfile = ""; + private boolean registrySsl; + + /** Combined properties kept for initializing system properties */ + private final Properties properties; + + private JmxScraperConfig(Properties properties) { + this.properties = properties; + } + + public String getServiceUrl() { + return serviceUrl; + } + + public String getCustomJmxScrapingConfigPath() { + return customJmxScrapingConfigPath; + } + + public Set getTargetSystems() { + return targetSystems; + } + + public int getIntervalMilliseconds() { + return intervalMilliseconds; + } + + public String getMetricsExporterType() { + return metricsExporterType; + } + + public String getOtlpExporterEndpoint() { + return otlpExporterEndpoint; + } + + public String getUsername() { + return username; + } + + public String getPassword() { + return password; + } + + public String getRealm() { + return realm; + } + + public String getRemoteProfile() { + return remoteProfile; + } + + public boolean isRegistrySsl() { + return registrySsl; + } + + /** + * Builds scraper configuration from user and system properties + * + * @param userProperties user-provided configuration + * @param systemProperties system properties through '-Dxxx' JVM arguments + * @return JMX scraper configuration + * @throws ConfigurationException if there is any configuration error + */ + public static JmxScraperConfig fromProperties( + Properties userProperties, Properties systemProperties) throws ConfigurationException { + + Properties properties = new Properties(); + properties.putAll(userProperties); + + // command line takes precedence so replace any that were specified via config file properties + properties.putAll(systemProperties); + + JmxScraperConfig config = new JmxScraperConfig(properties); + + config.serviceUrl = properties.getProperty(SERVICE_URL); + config.customJmxScrapingConfigPath = properties.getProperty(CUSTOM_JMX_SCRAPING_CONFIG); + String targetSystem = + properties.getProperty(TARGET_SYSTEM, "").toLowerCase(Locale.ENGLISH).trim(); + + List targets = + Arrays.asList(isBlank(targetSystem) ? new String[0] : targetSystem.split(",")); + config.targetSystems = targets.stream().map(String::trim).collect(Collectors.toSet()); + + int interval = getProperty(properties, INTERVAL_MILLISECONDS, 0); + config.intervalMilliseconds = (interval == 0 ? 10000 : interval); + // configure SDK metric exporter interval from jmx metric interval + getAndSetPropertyIfUndefined(properties, EXPORTER_INTERVAL, config.intervalMilliseconds); + + config.metricsExporterType = + getAndSetPropertyIfUndefined(properties, METRICS_EXPORTER_TYPE, "logging"); + if (OTLP_METRICS_EXPORTER.equalsIgnoreCase(config.metricsExporterType)) { + config.otlpExporterEndpoint = + getAndSetPropertyIfUndefined(properties, OTLP_ENDPOINT, "http://localhost:4318"); + } + config.username = properties.getProperty(JMX_USERNAME); + config.password = properties.getProperty(JMX_PASSWORD); + + config.remoteProfile = properties.getProperty(JMX_REMOTE_PROFILE); + config.realm = properties.getProperty(JMX_REALM); + + config.registrySsl = Boolean.parseBoolean(properties.getProperty(REGISTRY_SSL)); + + validateConfig(config); + return config; + } + + /** + * Sets system properties from effective configuration, must be called once and early before any + * OTel SDK or SSL/TLS stack initialization. This allows to override JVM system properties using + * user-provided configuration and also to set standard OTel SDK configuration. + */ + public void propagateSystemProperties() { + for (Map.Entry entry : properties.entrySet()) { + + String key = entry.getKey().toString(); + String value = entry.getValue().toString(); + if (key.startsWith("otel.") + || key.startsWith("javax.net.ssl.keyStore") + || key.startsWith("javax.net.ssl.trustStore")) { + System.setProperty(key, value); + } + } + } + + private static int getProperty(Properties properties, String key, int defaultValue) + throws ConfigurationException { + String propVal = properties.getProperty(key); + if (propVal == null) { + return defaultValue; + } + try { + return Integer.parseInt(propVal); + } catch (NumberFormatException e) { + throw new ConfigurationException("Failed to parse " + key, e); + } + } + + /** + * Similar to getProperty(key, defaultValue) but sets the property to default if not in object. + */ + private static String getAndSetPropertyIfUndefined( + Properties properties, String key, String defaultValue) { + String propVal = properties.getProperty(key, defaultValue); + if (propVal.equals(defaultValue)) { + properties.setProperty(key, defaultValue); + } + return propVal; + } + + private static int getAndSetPropertyIfUndefined( + Properties properties, String key, int defaultValue) throws ConfigurationException { + int propVal = getProperty(properties, key, defaultValue); + if (propVal == defaultValue) { + properties.setProperty(key, String.valueOf(defaultValue)); + } + return propVal; + } + + /** Will determine if parsed config is complete, setting any applicable values and defaults. */ + private static void validateConfig(JmxScraperConfig config) throws ConfigurationException { + if (isBlank(config.serviceUrl)) { + throw new ConfigurationException(SERVICE_URL + " must be specified."); + } + + if (isBlank(config.customJmxScrapingConfigPath) && config.targetSystems.isEmpty()) { + throw new ConfigurationException( + CUSTOM_JMX_SCRAPING_CONFIG + " or " + TARGET_SYSTEM + " must be specified."); + } + + if (!config.targetSystems.isEmpty() + && !AVAILABLE_TARGET_SYSTEMS.containsAll(config.targetSystems)) { + throw new ConfigurationException( + String.format( + "%s must specify targets from %s", config.targetSystems, AVAILABLE_TARGET_SYSTEMS)); + } + + if (OTLP_METRICS_EXPORTER.equalsIgnoreCase(config.metricsExporterType) + && isBlank(config.otlpExporterEndpoint)) { + throw new ConfigurationException(OTLP_ENDPOINT + " must be specified for otlp format."); + } + + if (config.intervalMilliseconds < 0) { + throw new ConfigurationException(INTERVAL_MILLISECONDS + " must be positive."); + } + } +} diff --git a/jmx-scraper/src/main/java/io/opentelemetry/contrib/jmxscraper/internal/StringUtils.java b/jmx-scraper/src/main/java/io/opentelemetry/contrib/jmxscraper/internal/StringUtils.java new file mode 100644 index 000000000..fa12d24b4 --- /dev/null +++ b/jmx-scraper/src/main/java/io/opentelemetry/contrib/jmxscraper/internal/StringUtils.java @@ -0,0 +1,26 @@ +/* + * Copyright The OpenTelemetry Authors + * SPDX-License-Identifier: Apache-2.0 + */ + +package io.opentelemetry.contrib.jmxscraper.internal; + +import javax.annotation.Nullable; + +/** + * This class is internal and is hence not for public use. Its APIs are unstable and can change at + * any time.
This is a utility class implementing miscellaneous String operations. + */ +public final class StringUtils { + private StringUtils() {} + + /** + * Determines if a String is null or without non-whitespace chars. + * + * @param s - {@link String} to evaluate + * @return - if s is null or without non-whitespace chars. + */ + public static boolean isBlank(@Nullable String s) { + return (s == null) || s.trim().isEmpty(); + } +} diff --git a/jmx-scraper/src/test/java/io/opentelemetry/contrib/jmxscraper/JmxScraperTest.java b/jmx-scraper/src/test/java/io/opentelemetry/contrib/jmxscraper/JmxScraperTest.java new file mode 100644 index 000000000..1dfe2717d --- /dev/null +++ b/jmx-scraper/src/test/java/io/opentelemetry/contrib/jmxscraper/JmxScraperTest.java @@ -0,0 +1,78 @@ +/* + * Copyright The OpenTelemetry Authors + * SPDX-License-Identifier: Apache-2.0 + */ + +package io.opentelemetry.contrib.jmxscraper; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +import io.opentelemetry.contrib.jmxscraper.config.ConfigurationException; +import io.opentelemetry.contrib.jmxscraper.config.JmxScraperConfig; +import java.io.IOException; +import java.io.InputStream; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.Properties; +import org.junit.jupiter.api.Test; + +class JmxScraperTest { + @Test + void shouldThrowExceptionWhenInvalidCommandLineArgsProvided() { + // Given + List emptyArgs = Collections.singletonList("-nonExistentOption"); + + // When and Then + assertThatThrownBy(() -> JmxScraper.parseArgs(emptyArgs)) + .isInstanceOf(ArgumentsParsingException.class); + } + + @Test + void shouldThrowExceptionWhenTooManyCommandLineArgsProvided() { + // Given + List args = Arrays.asList("-config", "path", "-nonExistentOption"); + + // When and Then + assertThatThrownBy(() -> JmxScraper.parseArgs(args)) + .isInstanceOf(ArgumentsParsingException.class); + } + + @Test + void shouldCreateConfig_propertiesLoadedFromFile() + throws ConfigurationException, ArgumentsParsingException { + // Given + String filePath = + ClassLoader.getSystemClassLoader().getResource("validConfig.properties").getPath(); + List args = Arrays.asList("-config", filePath); + + // When + JmxScraperConfig config = + JmxScraperConfig.fromProperties(JmxScraper.parseArgs(args), new Properties()); + + // Then + assertThat(config).isNotNull(); + } + + @Test + void shouldCreateConfig_propertiesLoadedFromStdIn() + throws ConfigurationException, ArgumentsParsingException, IOException { + InputStream originalIn = System.in; + try (InputStream stream = + ClassLoader.getSystemClassLoader().getResourceAsStream("validConfig.properties")) { + // Given + System.setIn(stream); + List args = Arrays.asList("-config", "-"); + + // When + JmxScraperConfig config = + JmxScraperConfig.fromProperties(JmxScraper.parseArgs(args), new Properties()); + + // Then + assertThat(config).isNotNull(); + } finally { + System.setIn(originalIn); + } + } +} diff --git a/jmx-scraper/src/test/java/io/opentelemetry/contrib/jmxscraper/config/JmxScraperConfigTest.java b/jmx-scraper/src/test/java/io/opentelemetry/contrib/jmxscraper/config/JmxScraperConfigTest.java new file mode 100644 index 000000000..4764ada6a --- /dev/null +++ b/jmx-scraper/src/test/java/io/opentelemetry/contrib/jmxscraper/config/JmxScraperConfigTest.java @@ -0,0 +1,337 @@ +/* + * Copyright The OpenTelemetry Authors + * SPDX-License-Identifier: Apache-2.0 + */ + +package io.opentelemetry.contrib.jmxscraper.config; + +import static io.opentelemetry.contrib.jmxscraper.config.JmxScraperConfig.CUSTOM_JMX_SCRAPING_CONFIG; +import static io.opentelemetry.contrib.jmxscraper.config.JmxScraperConfig.INTERVAL_MILLISECONDS; +import static io.opentelemetry.contrib.jmxscraper.config.JmxScraperConfig.JMX_PASSWORD; +import static io.opentelemetry.contrib.jmxscraper.config.JmxScraperConfig.JMX_REALM; +import static io.opentelemetry.contrib.jmxscraper.config.JmxScraperConfig.JMX_REMOTE_PROFILE; +import static io.opentelemetry.contrib.jmxscraper.config.JmxScraperConfig.JMX_USERNAME; +import static io.opentelemetry.contrib.jmxscraper.config.JmxScraperConfig.METRICS_EXPORTER_TYPE; +import static io.opentelemetry.contrib.jmxscraper.config.JmxScraperConfig.OTLP_ENDPOINT; +import static io.opentelemetry.contrib.jmxscraper.config.JmxScraperConfig.REGISTRY_SSL; +import static io.opentelemetry.contrib.jmxscraper.config.JmxScraperConfig.SERVICE_URL; +import static io.opentelemetry.contrib.jmxscraper.config.JmxScraperConfig.TARGET_SYSTEM; +import static io.opentelemetry.contrib.jmxscraper.config.JmxScraperConfig.fromProperties; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +import java.util.Properties; +import java.util.stream.Stream; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.junitpioneer.jupiter.ClearSystemProperty; + +class JmxScraperConfigTest { + private static Properties validProperties; + + @BeforeAll + static void setUp() { + validProperties = new Properties(); + validProperties.setProperty( + SERVICE_URL, "jservice:jmx:rmi:///jndi/rmi://localhost:9010/jmxrmi"); + validProperties.setProperty(CUSTOM_JMX_SCRAPING_CONFIG, ""); + validProperties.setProperty(TARGET_SYSTEM, "tomcat, activemq"); + validProperties.setProperty(METRICS_EXPORTER_TYPE, "otlp"); + validProperties.setProperty(INTERVAL_MILLISECONDS, "1410"); + validProperties.setProperty(REGISTRY_SSL, "true"); + validProperties.setProperty(OTLP_ENDPOINT, "http://localhost:4317"); + validProperties.setProperty(JMX_USERNAME, "some-user"); + validProperties.setProperty(JMX_PASSWORD, "some-password"); + validProperties.setProperty(JMX_REMOTE_PROFILE, "some-profile"); + validProperties.setProperty(JMX_REALM, "some-realm"); + } + + @AfterEach + void afterEach() { + // make sure that no test leaked in global system properties + Stream.of(System.getProperties().keySet()) + .map(Object::toString) + .forEach( + key -> { + if (key.startsWith("otel.") || key.startsWith("javax.net.ssl.")) { + System.clearProperty(key); + } + }); + } + + @Test + void shouldCreateMinimalValidConfiguration() throws ConfigurationException { + // Given + Properties properties = new Properties(); + properties.setProperty(SERVICE_URL, "jservice:jmx:rmi:///jndi/rmi://localhost:9010/jmxrmi"); + properties.setProperty(CUSTOM_JMX_SCRAPING_CONFIG, "/file.properties"); + + // When + JmxScraperConfig config = fromProperties(properties, new Properties()); + + // Then + assertThat(config.getServiceUrl()) + .isEqualTo("jservice:jmx:rmi:///jndi/rmi://localhost:9010/jmxrmi"); + assertThat(config.getCustomJmxScrapingConfigPath()).isEqualTo("/file.properties"); + assertThat(config.getTargetSystems()).isEmpty(); + assertThat(config.getIntervalMilliseconds()).isEqualTo(10000); + assertThat(config.getMetricsExporterType()).isEqualTo("logging"); + assertThat(config.getOtlpExporterEndpoint()).isBlank(); + assertThat(config.getUsername()).isNull(); + assertThat(config.getPassword()).isNull(); + assertThat(config.getRemoteProfile()).isNull(); + assertThat(config.getRealm()).isNull(); + } + + @Test + void shouldCreateConfig_defaultOtlEndpoint() throws ConfigurationException { + // Given + Properties properties = new Properties(); + properties.setProperty(SERVICE_URL, "jservice:jmx:rmi:///jndi/rmi://localhost:9010/jmxrmi"); + properties.setProperty(CUSTOM_JMX_SCRAPING_CONFIG, "/file.properties"); + properties.setProperty(METRICS_EXPORTER_TYPE, "otlp"); + + // When + JmxScraperConfig config = fromProperties(properties, new Properties()); + + // Then + assertThat(config.getMetricsExporterType()).isEqualTo("otlp"); + assertThat(config.getOtlpExporterEndpoint()).isEqualTo("http://localhost:4318"); + } + + @Test + @ClearSystemProperty(key = "javax.net.ssl.keyStore") + @ClearSystemProperty(key = "javax.net.ssl.keyStorePassword") + @ClearSystemProperty(key = "javax.net.ssl.keyStoreType") + @ClearSystemProperty(key = "javax.net.ssl.trustStore") + @ClearSystemProperty(key = "javax.net.ssl.trustStorePassword") + @ClearSystemProperty(key = "javax.net.ssl.trustStoreType") + void shouldUseValuesFromProperties() throws ConfigurationException { + // Given + // Properties to be propagated to system, properties + Properties properties = (Properties) validProperties.clone(); + properties.setProperty("javax.net.ssl.keyStore", "/my/key/store"); + properties.setProperty("javax.net.ssl.keyStorePassword", "abc123"); + properties.setProperty("javax.net.ssl.keyStoreType", "JKS"); + properties.setProperty("javax.net.ssl.trustStore", "/my/trust/store"); + properties.setProperty("javax.net.ssl.trustStorePassword", "def456"); + properties.setProperty("javax.net.ssl.trustStoreType", "JKS"); + + assertThat(System.getProperty("javax.net.ssl.keyStore")) + .describedAs("keystore config should not be set") + .isNull(); + + // When + JmxScraperConfig config = fromProperties(properties, new Properties()); + config.propagateSystemProperties(); + + // Then + assertThat(config.getServiceUrl()) + .isEqualTo("jservice:jmx:rmi:///jndi/rmi://localhost:9010/jmxrmi"); + assertThat(config.getCustomJmxScrapingConfigPath()).isEqualTo(""); + assertThat(config.getTargetSystems()).containsOnly("tomcat", "activemq"); + assertThat(config.getIntervalMilliseconds()).isEqualTo(1410); + assertThat(config.getMetricsExporterType()).isEqualTo("otlp"); + assertThat(config.getOtlpExporterEndpoint()).isEqualTo("http://localhost:4317"); + assertThat(config.getUsername()).isEqualTo("some-user"); + assertThat(config.getPassword()).isEqualTo("some-password"); + assertThat(config.getRemoteProfile()).isEqualTo("some-profile"); + assertThat(config.getRealm()).isEqualTo("some-realm"); + assertThat(config.isRegistrySsl()).isTrue(); + + // These properties are set from the config file loading into JmxConfig + assertThat(System.getProperty("javax.net.ssl.keyStore")).isEqualTo("/my/key/store"); + assertThat(System.getProperty("javax.net.ssl.keyStorePassword")).isEqualTo("abc123"); + assertThat(System.getProperty("javax.net.ssl.keyStoreType")).isEqualTo("JKS"); + assertThat(System.getProperty("javax.net.ssl.trustStore")).isEqualTo("/my/trust/store"); + assertThat(System.getProperty("javax.net.ssl.trustStorePassword")).isEqualTo("def456"); + assertThat(System.getProperty("javax.net.ssl.trustStoreType")).isEqualTo("JKS"); + } + + @Test + @ClearSystemProperty(key = "otel.jmx.service.url") + @ClearSystemProperty(key = "javax.net.ssl.keyStorePassword") + void shouldRetainPredefinedSystemProperties() throws ConfigurationException { + // Given + // user properties to be propagated to system properties + Properties properties = (Properties) validProperties.clone(); + properties.setProperty("javax.net.ssl.keyStorePassword", "abc123"); + + // system properties + Properties systemProperties = new Properties(); + systemProperties.put("otel.jmx.service.url", "originalServiceUrl"); + systemProperties.put("javax.net.ssl.keyStorePassword", "originalPassword"); + + // When + JmxScraperConfig config = fromProperties(properties, systemProperties); + // even when effective configuration is propagated to system properties original values are kept + // due to priority of system properties over user-provided ones. + config.propagateSystemProperties(); + + // Then + assertThat(System.getProperty("otel.jmx.service.url")).isEqualTo("originalServiceUrl"); + assertThat(System.getProperty("javax.net.ssl.keyStorePassword")).isEqualTo("originalPassword"); + } + + @Test + void shouldFailValidation_missingServiceUrl() { + // Given + Properties properties = (Properties) validProperties.clone(); + properties.remove(SERVICE_URL); + + // When and Then + assertThatThrownBy(() -> fromProperties(properties, new Properties())) + .isInstanceOf(ConfigurationException.class) + .hasMessage("otel.jmx.service.url must be specified."); + } + + @Test + void shouldFailValidation_missingConfigPathAndTargetSystem() { + // Given + Properties properties = (Properties) validProperties.clone(); + properties.remove(CUSTOM_JMX_SCRAPING_CONFIG); + properties.remove(TARGET_SYSTEM); + + // When and Then + assertThatThrownBy(() -> fromProperties(properties, new Properties())) + .isInstanceOf(ConfigurationException.class) + .hasMessage("otel.jmx.custom.scraping.config or otel.jmx.target.system must be specified."); + } + + @Test + void shouldFailValidation_invalidTargetSystem() { + // Given + Properties properties = (Properties) validProperties.clone(); + properties.setProperty(TARGET_SYSTEM, "hal9000"); + + // When and Then + assertThatThrownBy(() -> fromProperties(properties, new Properties())) + .isInstanceOf(ConfigurationException.class) + .hasMessageStartingWith("[hal9000] must specify targets from "); + } + + @Test + void shouldFailValidation_blankOtlpEndpointProvided() { + // Given + Properties properties = (Properties) validProperties.clone(); + properties.setProperty(OTLP_ENDPOINT, ""); + properties.setProperty(METRICS_EXPORTER_TYPE, "otlp"); + + // When and Then + assertThatThrownBy(() -> fromProperties(properties, new Properties())) + .isInstanceOf(ConfigurationException.class) + .hasMessage("otel.exporter.otlp.endpoint must be specified for otlp format."); + } + + @Test + void shouldPassValidation_noMetricsExporterType() throws ConfigurationException { + // Given + Properties properties = (Properties) validProperties.clone(); + properties.remove(OTLP_ENDPOINT); + properties.remove(METRICS_EXPORTER_TYPE); + + // When + JmxScraperConfig config = fromProperties(properties, new Properties()); + + // Then + assertThat(config).isNotNull(); + } + + @Test + void shouldPassValidation_nonOtlpMetricsExporterType() throws ConfigurationException { + // Given + Properties properties = (Properties) validProperties.clone(); + properties.remove(OTLP_ENDPOINT); + properties.setProperty(METRICS_EXPORTER_TYPE, "logging"); + + // When + JmxScraperConfig config = fromProperties(properties, new Properties()); + + // Then + assertThat(config).isNotNull(); + } + + @Test + void shouldFailValidation_negativeInterval() { + // Given + Properties properties = (Properties) validProperties.clone(); + properties.setProperty(INTERVAL_MILLISECONDS, "-1"); + + // When and Then + assertThatThrownBy(() -> fromProperties(properties, new Properties())) + .isInstanceOf(ConfigurationException.class) + .hasMessage("otel.jmx.interval.milliseconds must be positive."); + } + + @Test + void shouldFailConfigCreation_invalidInterval() { + // Given + Properties properties = (Properties) validProperties.clone(); + properties.setProperty(INTERVAL_MILLISECONDS, "abc"); + + // When and Then + assertThatThrownBy(() -> fromProperties(properties, new Properties())) + .isInstanceOf(ConfigurationException.class) + .hasMessage("Failed to parse otel.jmx.interval.milliseconds"); + } + + // TODO: Tests below will be reimplemented + + // @Test + // @SetSystemProperty(key = "otel.jmx.service.url", value = "myServiceUrl") + // @SetSystemProperty(key = "javax.net.ssl.keyStorePassword", value = "truth") + // void propertiesFileOverride() { + // Properties props = new Properties(); + // JmxMetrics.loadPropertiesFromPath( + // props, ClassLoader.getSystemClassLoader().getResource("all.properties").getPath()); + // JmxConfig config = new JmxConfig(props); + // + // // This property should retain the system property value, not the config file value + // assertThat(config.serviceUrl).isEqualTo("myServiceUrl"); + // // These properties are set from the config file + // assertThat(config.groovyScript).isEqualTo("/my/groovy/script"); + // assertThat(config.targetSystem).isEqualTo("jvm,cassandra"); + // assertThat(config.targetSystems).containsOnly("jvm", "cassandra"); + // assertThat(config.intervalMilliseconds).isEqualTo(20000); + // assertThat(config.metricsExporterType).isEqualTo("otlp"); + // assertThat(config.otlpExporterEndpoint).isEqualTo("https://myotlpendpoint"); + // assertThat(config.prometheusExporterHost).isEqualTo("host123.domain.com"); + // assertThat(config.prometheusExporterPort).isEqualTo(67890); + // assertThat(config.username).isEqualTo("myUser\nname"); + // assertThat(config.password).isEqualTo("myPassw\\ord"); + // assertThat(config.remoteProfile).isEqualTo("SASL/DIGEST-MD5"); + // assertThat(config.realm).isEqualTo("myRealm"); + // + // // This property should retain the system property value, not the config file value + // assertThat(System.getProperty("javax.net.ssl.keyStorePassword")).isEqualTo("truth"); + // // These properties are set from the config file loading into JmxConfig + // assertThat(System.getProperty("javax.net.ssl.keyStore")).isEqualTo("/my/key/store"); + // assertThat(System.getProperty("javax.net.ssl.keyStoreType")).isEqualTo("JKS"); + // assertThat(System.getProperty("javax.net.ssl.trustStore")).isEqualTo("/my/trust/store"); + // assertThat(System.getProperty("javax.net.ssl.trustStorePassword")).isEqualTo("def456"); + // assertThat(System.getProperty("javax.net.ssl.trustStoreType")).isEqualTo("JKS"); + // } + // + // @Test + // @SetSystemProperty(key = "otel.jmx.service.url", value = "myServiceUrl") + // @SetSystemProperty(key = "otel.jmx.groovy.script", value = "myGroovyScript") + // @SetSystemProperty(key = "otel.jmx.target.system", value = "myTargetSystem") + // void canSupportScriptAndTargetSystem() { + // JmxConfig config = new JmxConfig(); + // + // assertThat(config.serviceUrl).isEqualTo("myServiceUrl"); + // assertThat(config.groovyScript).isEqualTo("myGroovyScript"); + // assertThat(config.targetSystem).isEqualTo("mytargetsystem"); + // assertThat(config.targetSystems).containsOnly("mytargetsystem"); + // } + // + // @Test + // @SetSystemProperty(key = "otel.metric.export.interval", value = "123") + // void otelMetricExportIntervalRespected() { + // JmxConfig config = new JmxConfig(); + // assertThat(config.intervalMilliseconds).isEqualTo(10000); + // assertThat(config.properties.getProperty("otel.metric.export.interval")).isEqualTo("123"); + // } + // +} diff --git a/jmx-scraper/src/test/resources/validConfig.properties b/jmx-scraper/src/test/resources/validConfig.properties new file mode 100644 index 000000000..c4c7ac092 --- /dev/null +++ b/jmx-scraper/src/test/resources/validConfig.properties @@ -0,0 +1,20 @@ +otel.jmx.service.url=service:jmx:rmi:///jndi/rmi://myhost:12345/jmxrmi +otel.jmx.custom.scraping.config=/my/scraping-config.yaml +otel.jmx.target.system=jvm,cassandra +otel.jmx.interval.milliseconds=20000 +otel.metrics.exporter=otlp +otel.metric.export.interval=1000 +otel.exporter.otlp.endpoint=https://myotlpendpoint +otel.jmx.username=myUser\n\ + name +otel.jmx.password=myPassw\\ord +otel.jmx.remote.profile=SASL/DIG\EST-MD5 +otel.jmx.realm=myRealm +otel.resource.attributes=one=two,three=four +javax.net.ssl.keyStore=/my/key/store +javax.net.ssl.keyStorePassword=abc123 +javax.net.ssl.keyStoreType=JKS +javax.net.ssl.trustStore=/my/trust/store +javax.net.ssl.trustStorePassword=def456 +javax.net.ssl.trustStoreType=JKS +otel.jmx.aggregate.across.mbeans=true diff --git a/settings.gradle.kts b/settings.gradle.kts index a127e4d63..c2a3b27e5 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -13,6 +13,7 @@ plugins { dependencyResolutionManagement { repositories { mavenCentral() + mavenLocal() } } @@ -70,6 +71,7 @@ include(":example") include(":jfr-events") include(":jfr-connection") include(":jmx-metrics") +include(":jmx-scraper") include(":maven-extension") include(":micrometer-meter-provider") include(":noop-api")