diff --git a/sdk-extensions/incubator/src/main/java/io/opentelemetry/sdk/extension/incubator/fileconfig/ConfigurationFactory.java b/sdk-extensions/incubator/src/main/java/io/opentelemetry/sdk/extension/incubator/fileconfig/ConfigurationFactory.java index 1e0dddf3074..9c51f00af9a 100644 --- a/sdk-extensions/incubator/src/main/java/io/opentelemetry/sdk/extension/incubator/fileconfig/ConfigurationFactory.java +++ b/sdk-extensions/incubator/src/main/java/io/opentelemetry/sdk/extension/incubator/fileconfig/ConfigurationFactory.java @@ -34,6 +34,9 @@ private ConfigurationFactory() {} * Parse the {@code inputStream} YAML to {@link OpenTelemetryConfiguration} and interpret the * model to create {@link OpenTelemetrySdk} instance corresponding to the configuration. * + *

Before parsing, environment variable substitution is performed as described in {@link + * ConfigurationReader.EnvSubstitutionConstructor}. + * * @param inputStream the configuration YAML * @return the {@link OpenTelemetrySdk} */ diff --git a/sdk-extensions/incubator/src/main/java/io/opentelemetry/sdk/extension/incubator/fileconfig/ConfigurationReader.java b/sdk-extensions/incubator/src/main/java/io/opentelemetry/sdk/extension/incubator/fileconfig/ConfigurationReader.java index 4fe765ee344..731bfee1770 100644 --- a/sdk-extensions/incubator/src/main/java/io/opentelemetry/sdk/extension/incubator/fileconfig/ConfigurationReader.java +++ b/sdk-extensions/incubator/src/main/java/io/opentelemetry/sdk/extension/incubator/fileconfig/ConfigurationReader.java @@ -10,23 +10,120 @@ import com.fasterxml.jackson.databind.ObjectMapper; import io.opentelemetry.sdk.extension.incubator.fileconfig.internal.model.OpenTelemetryConfiguration; import java.io.InputStream; +import java.util.Map; +import java.util.regex.MatchResult; +import java.util.regex.Matcher; +import java.util.regex.Pattern; import org.snakeyaml.engine.v2.api.Load; import org.snakeyaml.engine.v2.api.LoadSettings; +import org.snakeyaml.engine.v2.constructor.StandardConstructor; +import org.snakeyaml.engine.v2.nodes.MappingNode; +import org.yaml.snakeyaml.Yaml; final class ConfigurationReader { - private static final ObjectMapper MAPPER = - new ObjectMapper() - // Create empty object instances for keys which are present but have null values - .setDefaultSetterInfo(JsonSetter.Value.forValueNulls(Nulls.AS_EMPTY)); + private static final Pattern ENV_VARIABLE_REFERENCE = + Pattern.compile("\\$\\{env:([a-zA-Z_]+[a-zA-Z0-9_]*)}"); + + private static final ObjectMapper MAPPER; + + static { + MAPPER = + new ObjectMapper() + // Create empty object instances for keys which are present but have null values + .setDefaultSetterInfo(JsonSetter.Value.forValueNulls(Nulls.AS_EMPTY)); + // Boxed primitives which are present but have null values should be set to null, rather than + // empty instances + MAPPER.configOverride(String.class).setSetterInfo(JsonSetter.Value.forValueNulls(Nulls.SET)); + MAPPER.configOverride(Integer.class).setSetterInfo(JsonSetter.Value.forValueNulls(Nulls.SET)); + MAPPER.configOverride(Double.class).setSetterInfo(JsonSetter.Value.forValueNulls(Nulls.SET)); + MAPPER.configOverride(Boolean.class).setSetterInfo(JsonSetter.Value.forValueNulls(Nulls.SET)); + } private ConfigurationReader() {} - /** Parse the {@code configuration} YAML and return the {@link OpenTelemetryConfiguration}. */ + /** + * Parse the {@code configuration} YAML and return the {@link OpenTelemetryConfiguration}. + * + *

Before parsing, environment variable substitution is performed as described in {@link + * EnvSubstitutionConstructor}. + */ static OpenTelemetryConfiguration parse(InputStream configuration) { - LoadSettings settings = LoadSettings.builder().build(); - Load yaml = new Load(settings); - Object yamlObj = yaml.loadFromInputStream(configuration); + return parse(configuration, System.getenv()); + } + + // Visible for testing + static OpenTelemetryConfiguration parse( + InputStream configuration, Map environmentVariables) { + Object yamlObj = loadYaml(configuration, environmentVariables); return MAPPER.convertValue(yamlObj, OpenTelemetryConfiguration.class); } + + static Object loadYaml(InputStream inputStream, Map environmentVariables) { + LoadSettings settings = LoadSettings.builder().build(); + Load yaml = new Load(settings, new EnvSubstitutionConstructor(settings, environmentVariables)); + return yaml.loadFromInputStream(inputStream); + } + + /** + * {@link StandardConstructor} which substitutes environment variables. + * + *

Environment variables follow the syntax {@code ${env:VARIABLE}}, where {@code VARIABLE} is + * an environment variable matching the regular expression {@code [a-zA-Z_]+[a-zA-Z0-9_]*}. + * + *

Environment variable substitution only takes place on scalar values of maps. References to + * environment variables in keys or sets are ignored. + * + *

If a referenced environment variable is not defined, it is replaced with {@code ""}. + */ + static final class EnvSubstitutionConstructor extends StandardConstructor { + + // Yaml is not thread safe but this instance is always used on the same thread + private final Yaml yaml = new Yaml(); + private final Map environmentVariables; + + private EnvSubstitutionConstructor( + LoadSettings loadSettings, Map environmentVariables) { + super(loadSettings); + this.environmentVariables = environmentVariables; + } + + @Override + protected Map constructMapping(MappingNode node) { + // First call the super to construct mapping from MappingNode as usual + Map result = super.constructMapping(node); + + // Iterate through the map entries, and: + // 1. Identify entries which are scalar strings eligible for environment variable substitution + // 2. Apply environment variable substitution + // 3. Re-parse substituted value so it has correct type (i.e. yaml.load(newVal)) + for (Map.Entry entry : result.entrySet()) { + Object value = entry.getValue(); + if (!(value instanceof String)) { + continue; + } + + String val = (String) value; + Matcher matcher = ENV_VARIABLE_REFERENCE.matcher(val); + if (!matcher.find()) { + continue; + } + + int offset = 0; + StringBuilder newVal = new StringBuilder(); + do { + MatchResult matchResult = matcher.toMatchResult(); + String replacement = environmentVariables.getOrDefault(matcher.group(1), ""); + newVal.append(val, offset, matchResult.start()).append(replacement); + offset = matchResult.end(); + } while (matcher.find()); + if (offset != val.length()) { + newVal.append(val, offset, val.length()); + } + entry.setValue(yaml.load(newVal.toString())); + } + + return result; + } + } } diff --git a/sdk-extensions/incubator/src/test/java/io/opentelemetry/sdk/extension/incubator/fileconfig/ConfigurationReaderTest.java b/sdk-extensions/incubator/src/test/java/io/opentelemetry/sdk/extension/incubator/fileconfig/ConfigurationReaderTest.java index afdb4cac33e..3a1e81c4073 100644 --- a/sdk-extensions/incubator/src/test/java/io/opentelemetry/sdk/extension/incubator/fileconfig/ConfigurationReaderTest.java +++ b/sdk-extensions/incubator/src/test/java/io/opentelemetry/sdk/extension/incubator/fileconfig/ConfigurationReaderTest.java @@ -49,14 +49,21 @@ import java.io.FileInputStream; import java.io.IOException; import java.nio.charset.StandardCharsets; +import java.util.AbstractMap; import java.util.Arrays; import java.util.Collections; +import java.util.HashMap; +import java.util.Map; +import javax.annotation.Nullable; import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; class ConfigurationReaderTest { @Test - void read_KitchenSinkExampleFile() throws IOException { + void parse_KitchenSinkExampleFile() throws IOException { OpenTelemetryConfiguration expected = new OpenTelemetryConfiguration(); expected.withFileFormat("0.1"); @@ -283,7 +290,7 @@ void read_KitchenSinkExampleFile() throws IOException { } @Test - void nullValuesParsedToEmptyObjects() { + void parse_nullValuesParsedToEmptyObjects() { String objectPlaceholderString = "file_format: \"0.1\"\n" + "tracer_provider:\n" @@ -337,4 +344,139 @@ void nullValuesParsedToEmptyObjects() { assertThat(objectPlaceholderModel).isEqualTo(noObjectPlaceholderModel); } + + @Test + void parse_nullBoxedPrimitivesParsedToNull() { + String yaml = + "file_format:\n" // String + + "disabled:\n" // Boolean + + "attribute_limits:\n" + + " attribute_value_length_limit:\n" // Integer + + "tracer_provider:\n" + + " sampler:\n" + + " trace_id_ratio_based:\n" + + " ratio:\n"; // Double + + OpenTelemetryConfiguration model = + ConfigurationReader.parse(new ByteArrayInputStream(yaml.getBytes(StandardCharsets.UTF_8))); + + assertThat(model.getFileFormat()).isNull(); + assertThat(model.getDisabled()).isNull(); + assertThat(model.getAttributeLimits().getAttributeValueLengthLimit()).isNull(); + assertThat(model.getTracerProvider().getSampler().getTraceIdRatioBased().getRatio()).isNull(); + + assertThat(model) + .isEqualTo( + new OpenTelemetryConfiguration() + .withAttributeLimits(new AttributeLimits()) + .withTracerProvider( + new TracerProvider() + .withSampler( + new Sampler().withTraceIdRatioBased(new TraceIdRatioBased())))); + } + + @ParameterizedTest + @MethodSource("envVarSubstitutionArgs") + void envSubstituteAndLoadYaml(String rawYaml, Object expectedYamlResult) { + Map environmentVariables = new HashMap<>(); + environmentVariables.put("STR_1", "value1"); + environmentVariables.put("STR_2", "value2"); + environmentVariables.put("BOOL", "true"); + environmentVariables.put("INT", "1"); + environmentVariables.put("FLOAT", "1.1"); + + Object yaml = + ConfigurationReader.loadYaml( + new ByteArrayInputStream(rawYaml.getBytes(StandardCharsets.UTF_8)), + environmentVariables); + assertThat(yaml).isEqualTo(expectedYamlResult); + } + + @SuppressWarnings("unchecked") + private static java.util.stream.Stream envVarSubstitutionArgs() { + return java.util.stream.Stream.of( + // Simple cases + Arguments.of("key1: ${env:STR_1}\n", mapOf(entry("key1", "value1"))), + Arguments.of("key1: ${env:BOOL}\n", mapOf(entry("key1", true))), + Arguments.of("key1: ${env:INT}\n", mapOf(entry("key1", 1))), + Arguments.of("key1: ${env:FLOAT}\n", mapOf(entry("key1", 1.1))), + Arguments.of( + "key1: ${env:STR_1}\n" + "key2: value2\n", + mapOf(entry("key1", "value1"), entry("key2", "value2"))), + Arguments.of( + "key1: ${env:STR_1} value1\n" + "key2: value2\n", + mapOf(entry("key1", "value1 value1"), entry("key2", "value2"))), + // Multiple environment variables referenced + Arguments.of("key1: ${env:STR_1}${env:STR_2}\n", mapOf(entry("key1", "value1value2"))), + Arguments.of("key1: ${env:STR_1} ${env:STR_2}\n", mapOf(entry("key1", "value1 value2"))), + // Undefined environment variable + Arguments.of("key1: ${env:STR_3}\n", mapOf(entry("key1", null))), + Arguments.of("key1: ${env:STR_1} ${env:STR_3}\n", mapOf(entry("key1", "value1"))), + // Environment variable keys must match pattern: [a-zA-Z_]+[a-zA-Z0-9_]* + Arguments.of("key1: ${env:VAR&}\n", mapOf(entry("key1", "${env:VAR&}"))), + // Environment variable substitution only takes place in scalar values of maps + Arguments.of("${env:STR_1}: value1\n", mapOf(entry("${env:STR_1}", "value1"))), + Arguments.of( + "key1:\n ${env:STR_1}: value1\n", + mapOf(entry("key1", mapOf(entry("${env:STR_1}", "value1"))))), + Arguments.of( + "key1:\n - ${env:STR_1}\n", + mapOf(entry("key1", Collections.singletonList("${env:STR_1}"))))); + } + + private static Map.Entry entry(K key, @Nullable V value) { + return new AbstractMap.SimpleEntry<>(key, value); + } + + @SuppressWarnings("unchecked") + private static Map mapOf(Map.Entry... entries) { + Map result = new HashMap<>(); + for (Map.Entry entry : entries) { + result.put(entry.getKey(), entry.getValue()); + } + return result; + } + + @Test + void read_WithEnvironmentVariables() { + String yaml = + "file_format: \"0.1\"\n" + + "tracer_provider:\n" + + " processors:\n" + + " - batch:\n" + + " exporter:\n" + + " otlp:\n" + + " endpoint: ${env:OTEL_EXPORTER_OTLP_ENDPOINT}\n" + + " - batch:\n" + + " exporter:\n" + + " otlp:\n" + + " endpoint: \"${env:UNSET_ENV_VAR}\"\n"; + Map envVars = new HashMap<>(); + envVars.put("OTEL_EXPORTER_OTLP_ENDPOINT", "http://collector:4317"); + OpenTelemetryConfiguration model = + ConfigurationReader.parse( + new ByteArrayInputStream(yaml.getBytes(StandardCharsets.UTF_8)), envVars); + assertThat(model) + .isEqualTo( + new OpenTelemetryConfiguration() + .withFileFormat("0.1") + .withTracerProvider( + new TracerProvider() + .withProcessors( + Arrays.asList( + new SpanProcessor() + .withBatch( + new BatchSpanProcessor() + .withExporter( + new SpanExporter() + .withOtlp( + new Otlp() + .withEndpoint( + "http://collector:4317")))), + new SpanProcessor() + .withBatch( + new BatchSpanProcessor() + .withExporter( + new SpanExporter().withOtlp(new Otlp()))))))); + } }