From c3cc0dc2e31441990db2860296b4a9fb78691d46 Mon Sep 17 00:00:00 2001 From: Ulysses Date: Sat, 16 Nov 2024 00:12:17 +0100 Subject: [PATCH 1/2] Fix for #461 (#462) Co-authored-by: Ivan Ananev --- .../main/kotlin/com/sksamuel/hoplite/parsers/PropsParser.kt | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/hoplite-core/src/main/kotlin/com/sksamuel/hoplite/parsers/PropsParser.kt b/hoplite-core/src/main/kotlin/com/sksamuel/hoplite/parsers/PropsParser.kt index 69fd91ec..e8ea93af 100644 --- a/hoplite-core/src/main/kotlin/com/sksamuel/hoplite/parsers/PropsParser.kt +++ b/hoplite-core/src/main/kotlin/com/sksamuel/hoplite/parsers/PropsParser.kt @@ -6,13 +6,15 @@ import com.sksamuel.hoplite.PropertySource import com.sksamuel.hoplite.PropertySourceContext import com.sksamuel.hoplite.fp.valid import java.io.InputStream +import java.io.InputStreamReader +import java.nio.charset.Charset import java.util.Properties class PropsParser : Parser { override fun load(input: InputStream, source: String): Node { val props = Properties() - props.load(input) + props.load(InputStreamReader(input, Charset.forName("UTF-8"))) return props.toNode(source) } From 4a760faf2f03dd5b6fb791cf439e9eedd70c2bd2 Mon Sep 17 00:00:00 2001 From: Raman Gupta Date: Wed, 27 Nov 2024 14:02:01 -0500 Subject: [PATCH 2/2] Config aliases should match source keys and transformed path elements (#458) --- .../com/sksamuel/hoplite/DecoderContext.kt | 4 +- .../hoplite/decoder/DataClassDecoder.kt | 16 +++++- .../sksamuel/hoplite/internal/ConfigParser.kt | 3 +- .../main/kotlin/com/sksamuel/hoplite/nodes.kt | 12 ++++ .../hoplite/transformer/NodeTransformer.kt | 3 + .../hoplite/transformer/PathNormalizer.kt | 4 +- .../EnvironmentVariablesPropertySourceTest.kt | 18 ++++++ .../hoplite/decoder/DataClassDecoderTest.kt | 21 +++---- .../hoplite/decoder/EnumDecoderTest.kt | 2 + .../toml/ConfigAliasNormalizationTest.kt | 17 ++++++ .../sksamuel/hoplite/toml/ConfigAliasTest.kt | 57 +++++++++---------- .../test/resources/alias_normalization.toml | 2 + 12 files changed, 113 insertions(+), 46 deletions(-) create mode 100644 hoplite-toml/src/test/kotlin/com/sksamuel/hoplite/toml/ConfigAliasNormalizationTest.kt create mode 100644 hoplite-toml/src/test/resources/alias_normalization.toml diff --git a/hoplite-core/src/main/kotlin/com/sksamuel/hoplite/DecoderContext.kt b/hoplite-core/src/main/kotlin/com/sksamuel/hoplite/DecoderContext.kt index 0fe183b1..416d0a45 100644 --- a/hoplite-core/src/main/kotlin/com/sksamuel/hoplite/DecoderContext.kt +++ b/hoplite-core/src/main/kotlin/com/sksamuel/hoplite/DecoderContext.kt @@ -7,6 +7,7 @@ import com.sksamuel.hoplite.env.Environment import com.sksamuel.hoplite.fp.Validated import com.sksamuel.hoplite.resolver.Resolving import com.sksamuel.hoplite.resolver.context.ContextResolverMode +import com.sksamuel.hoplite.transformer.NodeTransformer import kotlin.reflect.KParameter import kotlin.reflect.KType @@ -21,6 +22,7 @@ import kotlin.reflect.KType data class DecoderContext( val decoders: DecoderRegistry, val paramMappers: List, + val nodeTransformers: List, val reporter: Reporter = Reporter(), // these are the dot paths for every config value - overrided or not, that was used val usedPaths: MutableSet = mutableSetOf(), @@ -59,7 +61,7 @@ data class DecoderContext( fun report(section: String, row: Map) = reporter.report(section, row) companion object { - val zero = DecoderContext(DecoderRegistry.zero, emptyList(), resolvers = Resolving(emptyList(), Undefined)) + val zero = DecoderContext(DecoderRegistry.zero, emptyList(), emptyList(), resolvers = Resolving(emptyList(), Undefined)) } } diff --git a/hoplite-core/src/main/kotlin/com/sksamuel/hoplite/decoder/DataClassDecoder.kt b/hoplite-core/src/main/kotlin/com/sksamuel/hoplite/decoder/DataClassDecoder.kt index 904637fd..1d662c3e 100644 --- a/hoplite-core/src/main/kotlin/com/sksamuel/hoplite/decoder/DataClassDecoder.kt +++ b/hoplite-core/src/main/kotlin/com/sksamuel/hoplite/decoder/DataClassDecoder.kt @@ -81,15 +81,27 @@ class DataClassDecoder : NullHandlingDecoder { // use parameter mappers to retrieve alternative names, then try each one in turn // until we find one that is defined val names = context.paramMappers.flatMap { it.map(param, constructor, kclass) } + + fun nameLookup(name: String): Node { + var atKey = node.atKey(name) + // check if the node has a key matching a transformed path element + if (atKey is Undefined) atKey = node.atKey( + context.nodeTransformers.fold(name) { n, transformer -> transformer.transformPathElement(n) } + ) + // also check the source key, as parameter mappers may be referring to the source name + if (atKey is Undefined) atKey = node.atSourceKey(name) + return atKey + } + // every alternative name is marked as used so strict can detect that overrides were 'used' too. names.forEach { - context.usedPaths.add(node.atKey(it).path) + context.usedPaths.add(nameLookup(it).path) } val n = names.fold(Undefined) { n, name -> if (n.isDefined) n else { usedName = name - node.atKey(name) + nameLookup(name) } } diff --git a/hoplite-core/src/main/kotlin/com/sksamuel/hoplite/internal/ConfigParser.kt b/hoplite-core/src/main/kotlin/com/sksamuel/hoplite/internal/ConfigParser.kt index 72cf55e8..2057fa02 100644 --- a/hoplite-core/src/main/kotlin/com/sksamuel/hoplite/internal/ConfigParser.kt +++ b/hoplite-core/src/main/kotlin/com/sksamuel/hoplite/internal/ConfigParser.kt @@ -70,6 +70,7 @@ class ConfigParser( return DecoderContext( decoders = decoderRegistry, paramMappers = paramMappers, + nodeTransformers = nodeTransformers, config = DecoderConfig(flattenArraysToString, resolveTypesCaseInsensitive), environment = environment, resolvers = Resolving(resolvers, root), @@ -139,7 +140,7 @@ class ConfigParser( private fun Node.prefixedNode(prefix: String?) = when { prefix == null -> this - nodeTransformers.contains(PathNormalizer) -> atPath(PathNormalizer.normalizePathElement(prefix)) + nodeTransformers.contains(PathNormalizer) -> atPath(PathNormalizer.transformPathElement(prefix)) else -> atPath(prefix) } } diff --git a/hoplite-core/src/main/kotlin/com/sksamuel/hoplite/nodes.kt b/hoplite-core/src/main/kotlin/com/sksamuel/hoplite/nodes.kt index 1ef86ce2..7f4865d2 100644 --- a/hoplite-core/src/main/kotlin/com/sksamuel/hoplite/nodes.kt +++ b/hoplite-core/src/main/kotlin/com/sksamuel/hoplite/nodes.kt @@ -28,6 +28,14 @@ sealed interface Node { */ fun atKey(key: String): Node + /** + * Returns the [PrimitiveNode] at the given source key. + * If this node is not a [MapNode] or the node contained at the + * given key is not a primitive, or the node does not contain + * the key at all, then this will return [Undefined]. + */ + fun atSourceKey(key: String): Node + operator fun get(key: String): Node = atKey(key) /** @@ -159,6 +167,7 @@ data class MapNode( ) : ContainerNode() { override val simpleName: String = "Map" override fun atKey(key: String): Node = map[key] ?: Undefined + override fun atSourceKey(key: String): Node = map.values.firstOrNull { it.sourceKey == key } ?: Undefined override fun atIndex(index: Int): Node = Undefined override val size: Int = map.size } @@ -172,12 +181,14 @@ data class ArrayNode( ) : ContainerNode() { override val simpleName: String = "List" override fun atKey(key: String): Node = Undefined + override fun atSourceKey(key: String): Node = Undefined override fun atIndex(index: Int): Node = elements.getOrElse(index) { Undefined } override val size: Int = elements.size } sealed class PrimitiveNode : Node { override fun atKey(key: String): Node = Undefined + override fun atSourceKey(key: String): Node = Undefined override fun atIndex(index: Int): Node = Undefined override val size: Int = 0 abstract val value: Any? @@ -241,6 +252,7 @@ object Undefined : Node { override val path = DotPath.root override val sourceKey: String? = null override fun atKey(key: String): Node = this + override fun atSourceKey(key: String): Node = this override fun atIndex(index: Int): Node = this override val size: Int = 0 override val meta: Map = emptyMap() diff --git a/hoplite-core/src/main/kotlin/com/sksamuel/hoplite/transformer/NodeTransformer.kt b/hoplite-core/src/main/kotlin/com/sksamuel/hoplite/transformer/NodeTransformer.kt index a0ffe18e..97982b28 100644 --- a/hoplite-core/src/main/kotlin/com/sksamuel/hoplite/transformer/NodeTransformer.kt +++ b/hoplite-core/src/main/kotlin/com/sksamuel/hoplite/transformer/NodeTransformer.kt @@ -7,5 +7,8 @@ import com.sksamuel.hoplite.* * be applied at configuration loading time. */ interface NodeTransformer { + /** Used for one of path element transformations equivalent to the node transformation. */ + fun transformPathElement(element: String): String + fun transform(node: Node, sealedTypeDiscriminatorField: String?): Node } diff --git a/hoplite-core/src/main/kotlin/com/sksamuel/hoplite/transformer/PathNormalizer.kt b/hoplite-core/src/main/kotlin/com/sksamuel/hoplite/transformer/PathNormalizer.kt index 8a360954..027230ed 100644 --- a/hoplite-core/src/main/kotlin/com/sksamuel/hoplite/transformer/PathNormalizer.kt +++ b/hoplite-core/src/main/kotlin/com/sksamuel/hoplite/transformer/PathNormalizer.kt @@ -21,7 +21,7 @@ import com.sksamuel.hoplite.* * It does NOT normalize the sealed type discriminator field for map nodes. */ object PathNormalizer : NodeTransformer { - fun normalizePathElement(element: String): String = element + override fun transformPathElement(element: String): String = element .replace("-", "") .replace("_", "") .lowercase() @@ -31,7 +31,7 @@ object PathNormalizer : NodeTransformer { val normalizedPathNode = it.withPath( it.path.copy(keys = it.path.keys.map { key -> if (it is MapNode) normalizePathElementExceptDiscriminator(key, sealedTypeDiscriminatorField) - else normalizePathElement(key) + else transformPathElement(key) }) ) when (normalizedPathNode){ diff --git a/hoplite-core/src/test/kotlin/com/sksamuel/hoplite/EnvironmentVariablesPropertySourceTest.kt b/hoplite-core/src/test/kotlin/com/sksamuel/hoplite/EnvironmentVariablesPropertySourceTest.kt index fd9bfb5d..3ca94248 100644 --- a/hoplite-core/src/test/kotlin/com/sksamuel/hoplite/EnvironmentVariablesPropertySourceTest.kt +++ b/hoplite-core/src/test/kotlin/com/sksamuel/hoplite/EnvironmentVariablesPropertySourceTest.kt @@ -30,4 +30,22 @@ class EnvironmentVariablesPropertySourceTest : FunSpec({ ) } + test("env var source should respect config aliases that need to be normalized to match") { + data class TestConfig(@ConfigAlias("fooBar") val bazBar: String) + + val config = ConfigLoader { + addPropertySource( + EnvironmentVariablesPropertySource( + useUnderscoresAsSeparator = false, + allowUppercaseNames = true, + environmentVariableMap = { + mapOf("FOO_BAR" to "fooValue") + }, + ) + ) + }.loadConfigOrThrow() + + config shouldBe TestConfig("fooValue") + } + }) diff --git a/hoplite-core/src/test/kotlin/com/sksamuel/hoplite/decoder/DataClassDecoderTest.kt b/hoplite-core/src/test/kotlin/com/sksamuel/hoplite/decoder/DataClassDecoderTest.kt index 24c8d9c6..43cfaccd 100644 --- a/hoplite-core/src/test/kotlin/com/sksamuel/hoplite/decoder/DataClassDecoderTest.kt +++ b/hoplite-core/src/test/kotlin/com/sksamuel/hoplite/decoder/DataClassDecoderTest.kt @@ -7,6 +7,7 @@ import com.sksamuel.hoplite.MapNode import com.sksamuel.hoplite.NullNode import com.sksamuel.hoplite.Pos import com.sksamuel.hoplite.StringNode +import com.sksamuel.hoplite.defaultNodeTransformers import com.sksamuel.hoplite.defaultParamMappers import com.sksamuel.hoplite.fp.valid import io.kotest.core.spec.style.StringSpec @@ -35,7 +36,7 @@ class DataClassDecoderTest : StringSpec() { DataClassDecoder().decode( node, Foo::class.createType(), - DecoderContext(defaultDecoderRegistry(), defaultParamMappers()) + DecoderContext(defaultDecoderRegistry(), defaultParamMappers(), defaultNodeTransformers()) ) shouldBe Foo("hello", 123, true).valid() } @@ -54,7 +55,7 @@ class DataClassDecoderTest : StringSpec() { DataClassDecoder().decode( node, Foo::class.createType(), - DecoderContext(defaultDecoderRegistry(), defaultParamMappers()) + DecoderContext(defaultDecoderRegistry(), defaultParamMappers(), defaultNodeTransformers()) ) shouldBe Foo(null, null, null).valid() } @@ -73,7 +74,7 @@ class DataClassDecoderTest : StringSpec() { DataClassDecoder().decode( node, Foo::class.createType(), - DecoderContext(defaultDecoderRegistry(), defaultParamMappers()) + DecoderContext(defaultDecoderRegistry(), defaultParamMappers(), defaultNodeTransformers()) ) shouldBe Foo("hello", 123, true).valid() } @@ -98,7 +99,7 @@ class DataClassDecoderTest : StringSpec() { ) DataClassDecoder().decode(node, Foo::class.createType(), - DecoderContext(defaultDecoderRegistry(), defaultParamMappers()) + DecoderContext(defaultDecoderRegistry(), defaultParamMappers(), defaultNodeTransformers()) ) shouldBe Foo(Year.of(1991), expectedDate, YearMonth.parse("2007-12"), expectedSqlTimestamp).valid() } @@ -117,7 +118,7 @@ class DataClassDecoderTest : StringSpec() { DataClassDecoder().decode( node, Foo::class.createType(), - DecoderContext(defaultDecoderRegistry(), defaultParamMappers()) + DecoderContext(defaultDecoderRegistry(), defaultParamMappers(), defaultNodeTransformers()) ) shouldBe Foo(IntRange(1, 4), LongRange(50, 60), CharRange('d', 'g')).valid() } @@ -135,7 +136,7 @@ class DataClassDecoderTest : StringSpec() { DataClassDecoder().decode( node, Foo::class.createType(), - DecoderContext(defaultDecoderRegistry(), defaultParamMappers()) + DecoderContext(defaultDecoderRegistry(), defaultParamMappers(), defaultNodeTransformers()) ) shouldBe Foo("value", "default b", false).valid() } @@ -147,7 +148,7 @@ class DataClassDecoderTest : StringSpec() { DataClassDecoder().decode( node, Foo::class.createType(), - DecoderContext(defaultDecoderRegistry(), defaultParamMappers()) + DecoderContext(defaultDecoderRegistry(), defaultParamMappers(), defaultNodeTransformers()) ) shouldBe Foo(FooEnum.SECOND).valid() } @@ -161,7 +162,7 @@ class DataClassDecoderTest : StringSpec() { DataClassDecoder().decode( node, Foo::class.createType(), - DecoderContext(defaultDecoderRegistry(), defaultParamMappers()) + DecoderContext(defaultDecoderRegistry(), defaultParamMappers(), defaultNodeTransformers()) ) shouldBe Foo(FooEnum.THIRD).valid() } @@ -183,7 +184,7 @@ class DataClassDecoderTest : StringSpec() { DataClassDecoder().decode( node, Foo::class.createType(), - DecoderContext(defaultDecoderRegistry(), defaultParamMappers()) + DecoderContext(defaultDecoderRegistry(), defaultParamMappers(), defaultNodeTransformers()) ) shouldBe Foo(FooEnum.FIRST, "MultiParamCallExpected", false).valid() } @@ -205,7 +206,7 @@ class DataClassDecoderTest : StringSpec() { DataClassDecoder().decode( node, Foo::class.createType(), - DecoderContext(defaultDecoderRegistry(), defaultParamMappers()) + DecoderContext(defaultDecoderRegistry(), defaultParamMappers(), defaultNodeTransformers()) ) shouldBe Foo(FooEnum.THIRD, true).valid() } } diff --git a/hoplite-core/src/test/kotlin/com/sksamuel/hoplite/decoder/EnumDecoderTest.kt b/hoplite-core/src/test/kotlin/com/sksamuel/hoplite/decoder/EnumDecoderTest.kt index 4e4964db..9fa625b1 100644 --- a/hoplite-core/src/test/kotlin/com/sksamuel/hoplite/decoder/EnumDecoderTest.kt +++ b/hoplite-core/src/test/kotlin/com/sksamuel/hoplite/decoder/EnumDecoderTest.kt @@ -6,6 +6,7 @@ import com.sksamuel.hoplite.DecoderContext import com.sksamuel.hoplite.Pos import com.sksamuel.hoplite.PrimitiveNode import com.sksamuel.hoplite.StringNode +import com.sksamuel.hoplite.defaultNodeTransformers import com.sksamuel.hoplite.defaultParamMappers import com.sksamuel.hoplite.fp.Validated import io.kotest.core.spec.style.BehaviorSpec @@ -65,6 +66,7 @@ class EnumDecoderTest : BehaviorSpec({ DecoderContext( decoders = defaultDecoderRegistry(), paramMappers = defaultParamMappers(), + nodeTransformers = defaultNodeTransformers(), config = DecoderConfig(flattenArraysToString = false, resolveTypesCaseInsensitive = ignoreCase) ) ) diff --git a/hoplite-toml/src/test/kotlin/com/sksamuel/hoplite/toml/ConfigAliasNormalizationTest.kt b/hoplite-toml/src/test/kotlin/com/sksamuel/hoplite/toml/ConfigAliasNormalizationTest.kt new file mode 100644 index 00000000..659d1064 --- /dev/null +++ b/hoplite-toml/src/test/kotlin/com/sksamuel/hoplite/toml/ConfigAliasNormalizationTest.kt @@ -0,0 +1,17 @@ +package com.sksamuel.hoplite.toml + +import com.sksamuel.hoplite.ConfigAlias +import com.sksamuel.hoplite.ConfigLoader +import io.kotest.core.spec.style.FunSpec +import io.kotest.matchers.shouldBe + +class ConfigAliasNormalizationTest : FunSpec({ + data class AliasConfig( + @ConfigAlias("fooBarAlias") val fooBar: String, + ) + + test("parsers should support @ConfigAlias with field normalization") { + val config = ConfigLoader().loadConfigOrThrow("/alias_normalization.toml") + config.fooBar shouldBe "Tom Preston-Werner" + } +}) diff --git a/hoplite-toml/src/test/kotlin/com/sksamuel/hoplite/toml/ConfigAliasTest.kt b/hoplite-toml/src/test/kotlin/com/sksamuel/hoplite/toml/ConfigAliasTest.kt index 3f317d22..eec63a4a 100644 --- a/hoplite-toml/src/test/kotlin/com/sksamuel/hoplite/toml/ConfigAliasTest.kt +++ b/hoplite-toml/src/test/kotlin/com/sksamuel/hoplite/toml/ConfigAliasTest.kt @@ -5,38 +5,35 @@ import com.sksamuel.hoplite.ConfigLoader import io.kotest.core.spec.style.FunSpec import io.kotest.matchers.shouldBe -data class A( - @ConfigAlias("b") @ConfigAlias("zah") val bb: String, - val c: Int -) +class ConfigAliasTest : FunSpec({ + data class A( + @ConfigAlias("b") @ConfigAlias("zah") val bb: String, + val c: Int + ) -data class D( - val e: String, - @ConfigAlias("f") val ff: Boolean -) + data class D( + val e: String, + @ConfigAlias("f") val ff: Boolean + ) -data class AliasConfig( - val a: A, - @ConfigAlias("d") val dd: D -) + data class AliasConfig( + val a: A, + @ConfigAlias("d") val dd: D + ) -class ConfigAliasTest : FunSpec() { - init { - - test("parsers should support @ConfigAlias") { - val config = ConfigLoader().loadConfigOrThrow("/alias.toml") - config.a.bb shouldBe "Tom Preston-Werner" - config.a.c shouldBe 5000 - config.dd.e shouldBe "192.168.1.1" - config.dd.ff shouldBe true - } + test("parsers should support @ConfigAlias") { + val config = ConfigLoader().loadConfigOrThrow("/alias.toml") + config.a.bb shouldBe "Tom Preston-Werner" + config.a.c shouldBe 5000 + config.dd.e shouldBe "192.168.1.1" + config.dd.ff shouldBe true + } - test("parsers should support multiple @ConfigAlias") { - val config = ConfigLoader().loadConfigOrThrow("/repeated_alias.toml") - config.a.bb shouldBe "Tom Preston-Werner" - config.a.c shouldBe 5000 - config.dd.e shouldBe "192.168.1.1" - config.dd.ff shouldBe true - } + test("parsers should support multiple @ConfigAlias") { + val config = ConfigLoader().loadConfigOrThrow("/repeated_alias.toml") + config.a.bb shouldBe "Tom Preston-Werner" + config.a.c shouldBe 5000 + config.dd.e shouldBe "192.168.1.1" + config.dd.ff shouldBe true } -} +}) diff --git a/hoplite-toml/src/test/resources/alias_normalization.toml b/hoplite-toml/src/test/resources/alias_normalization.toml new file mode 100644 index 00000000..2aa1fce5 --- /dev/null +++ b/hoplite-toml/src/test/resources/alias_normalization.toml @@ -0,0 +1,2 @@ +# This is a TOML document. +fooBarAlias = "Tom Preston-Werner"