Denormalize maps keys with alternate delimiters (#457)

hoplite-core/src/main/kotlin/com/sksamuel/hoplite/decoder/Decoder.kt

@@ -54,6 +54,7 @@ data class DotPath(val keys: List<String>) {
 
   fun with(name: String): DotPath = DotPath(keys + name)
   fun flatten() = keys.joinToString(".")
+  fun flatten(delimiter: String) = keys.joinToString(delimiter)
 }
 
 inline fun <T, reified U> Decoder<T>.map(crossinline f: (T) -> U): Decoder<U> = object : Decoder<U> {

hoplite-core/src/main/kotlin/com/sksamuel/hoplite/internal/cascade.kt

@@ -71,7 +71,7 @@ class Cascader(
               val overrides = merges.values.toList().flatMap { it.overrides }
               val elements = merges.mapValues { it.value.node }
               CascadeResult(
-                MapNode(elements, a.pos, a.path, cascade(a.value, b.value).node, a.meta, a.sourceKey),
+                MapNode(elements, a.pos, a.path, cascade(a.value, b.value).node, a.meta, a.delimiter, a.sourceKey),
                 overrides
               )
             }

hoplite-core/src/main/kotlin/com/sksamuel/hoplite/nodes.kt

@@ -14,6 +14,11 @@ sealed interface Node {
    */
   val path: DotPath
 
+  /**
+   * The original delimiter between path elements. Usually a `.` but could be something else.
+   */
+  val delimiter: String
+
   /**
    * The original source key of this node without any normalization.
    * Useful for reporting or potentially for custom decoders.
@@ -149,7 +154,7 @@ fun Node.transform(transformer: (Node) -> Node): Node = when (val transformed =
 fun <T : Node> T.denormalize(): T {
   return when (this) {
     is MapNode -> copy(map = map.mapKeys { (k, v) ->
-      (v.sourceKey ?: k).removePrefix("$sourceKey.")
+      (v.sourceKey ?: k).removePrefix("$sourceKey$delimiter")
     })
     else -> this
   } as T
@@ -163,7 +168,8 @@ data class MapNode(
   override val path: DotPath,
   val value: Node = Undefined,
   override val meta: Map<String, Any?> = emptyMap(),
-  override val sourceKey: String? = if (path == DotPath.root) null else path.flatten(),
+  override val delimiter: String = ".",
+  override val sourceKey: String? = if (path == DotPath.root) null else path.flatten(delimiter),
 ) : ContainerNode() {
   override val simpleName: String = "Map"
   override fun atKey(key: String): Node = map[key] ?: Undefined
@@ -177,7 +183,8 @@ data class ArrayNode(
   override val pos: Pos,
   override val path: DotPath,
   override val meta: Map<String, Any?> = emptyMap(),
-  override val sourceKey: String? = if (path == DotPath.root) null else path.flatten(),
+  override val delimiter: String = ".",
+  override val sourceKey: String? = if (path == DotPath.root) null else path.flatten(delimiter),
 ) : ContainerNode() {
   override val simpleName: String = "List"
   override fun atKey(key: String): Node = Undefined
@@ -199,7 +206,8 @@ data class StringNode(
   override val pos: Pos,
   override val path: DotPath,
   override val meta: Map<String, Any?> = emptyMap(),
-  override val sourceKey: String? = if (path == DotPath.root) null else path.flatten(),
+  override val delimiter: String = ".",
+  override val sourceKey: String? = if (path == DotPath.root) null else path.flatten(delimiter),
 ) : PrimitiveNode() {
   override val simpleName: String = "String"
 }
@@ -209,7 +217,8 @@ data class BooleanNode(
   override val pos: Pos,
   override val path: DotPath,
   override val meta: Map<String, Any?> = emptyMap(),
-  override val sourceKey: String? = if (path == DotPath.root) null else path.flatten(),
+  override val delimiter: String = ".",
+  override val sourceKey: String? = if (path == DotPath.root) null else path.flatten(delimiter),
 ) : PrimitiveNode() {
   override val simpleName: String = "Boolean"
 }
@@ -221,7 +230,8 @@ data class LongNode(
   override val pos: Pos,
   override val path: DotPath,
   override val meta: Map<String, Any?> = emptyMap(),
-  override val sourceKey: String? = if (path == DotPath.root) null else path.flatten(),
+  override val delimiter: String = ".",
+  override val sourceKey: String? = if (path == DotPath.root) null else path.flatten(delimiter),
 ) : NumberNode() {
   override val simpleName: String = "Long"
 }
@@ -231,7 +241,8 @@ data class DoubleNode(
   override val pos: Pos,
   override val path: DotPath,
   override val meta: Map<String, Any?> = emptyMap(),
-  override val sourceKey: String? = if (path == DotPath.root) null else path.flatten(),
+  override val delimiter: String = ".",
+  override val sourceKey: String? = if (path == DotPath.root) null else path.flatten(delimiter),
 ) : NumberNode() {
   override val simpleName: String = "Double"
 }
@@ -240,7 +251,8 @@ data class NullNode(
   override val pos: Pos,
   override val path: DotPath,
   override val meta: Map<String, Any?> = emptyMap(),
-  override val sourceKey: String? = if (path == DotPath.root) null else path.flatten(),
+  override val delimiter: String = ".",
+  override val sourceKey: String? = if (path == DotPath.root) null else path.flatten(delimiter),
 ) : PrimitiveNode() {
   override val simpleName: String = "null"
   override val value: Any? = null
@@ -251,6 +263,7 @@ object Undefined : Node {
   override val pos: Pos = Pos.NoPos
   override val path = DotPath.root
   override val sourceKey: String? = null
+  override val delimiter: String = "."
   override fun atKey(key: String): Node = this
   override fun atSourceKey(key: String): Node = this
   override fun atIndex(index: Int): Node = this

hoplite-core/src/main/kotlin/com/sksamuel/hoplite/parsers/loadProps.kt

@@ -37,6 +37,7 @@ data class Element(
   val values: MutableMap<String, Element> = hashMapOf(),
   var value: Any? = null,
   var sourceKey: String? = null,
+  var delimiter: String = ".",
 )
 
 private fun <T, K> Iterable<T>.toNode(
@@ -59,6 +60,7 @@ private fun <T, K> Iterable<T>.toNode(
         if (index == segments.size - 1) {
           it.value = value
           it.sourceKey = sourceKey.toString()
+          it.delimiter = delimiter
         }
       }
     }
@@ -74,19 +76,22 @@ private fun <T, K> Iterable<T>.toNode(
         pos = pos,
         path = path,
         value = value?.transform(path, sourceKey) ?: Undefined,
+        delimiter = delimiter,
       )
     }
     is Array<*> -> ArrayNode(
       elements = mapNotNull { it?.transform(path, parentSourceKey) },
       pos = pos,
       path = path,
       sourceKey = parentSourceKey,
+      delimiter = delimiter,
     )
     is Collection<*> -> ArrayNode(
       elements = mapNotNull { it?.transform(path, parentSourceKey) },
       pos = pos,
       path = path,
       sourceKey = parentSourceKey,
+      delimiter = delimiter,
     )
     is Map<*, *> -> MapNode(
       map = takeUnless { it.isEmpty() }?.mapNotNull { entry ->
@@ -95,8 +100,9 @@ private fun <T, K> Iterable<T>.toNode(
       pos = pos,
       path = path,
       sourceKey = parentSourceKey,
+      delimiter = delimiter,
     )
-    else -> StringNode(this.toString(), pos, path = path, emptyMap(), parentSourceKey)
+    else -> StringNode(this.toString(), pos, path = path, emptyMap(), delimiter, parentSourceKey)
   }
 
   return map.transform(DotPath.root)

hoplite-core/src/main/kotlin/com/sksamuel/hoplite/preprocessor/LookupPreprocessor.kt

@@ -49,9 +49,9 @@ object LookupPreprocessor : Preprocessor {
     fun handle(n: Node): Node = when (n) {
       is MapNode -> {
         val value = if (n.value is StringNode) replace(replace(n.value, regex1), regex2) else n.value
-        MapNode(n.map.map { (k, v) -> k to handle(v) }.toMap(), n.pos, n.path, value, sourceKey = n.sourceKey)
+        MapNode(n.map.map { (k, v) -> k to handle(v) }.toMap(), n.pos, n.path, value, delimiter = n.delimiter, sourceKey = n.sourceKey)
       }
-      is ArrayNode -> ArrayNode(n.elements.map { handle(it) }, n.pos, n.path, sourceKey = n.sourceKey)
+      is ArrayNode -> ArrayNode(n.elements.map { handle(it) }, n.pos, n.path, delimiter = n.delimiter, sourceKey = n.sourceKey)
       is StringNode -> replace(replace(n, regex1), regex2)
       else -> n
     }

hoplite-core/src/main/kotlin/com/sksamuel/hoplite/preprocessor/Preprocessor.kt

@@ -44,14 +44,14 @@ abstract class TraversingPrimitivePreprocessor : Preprocessor {
         .map { it.toMap() }.flatMap { map ->
           val value = if (node.value is PrimitiveNode) handle(node.value, context) else node.value.valid()
           value.map { v ->
-            MapNode(map, node.pos, node.path, v, sourceKey = node.sourceKey)
+            MapNode(map, node.pos, node.path, v, sourceKey = node.sourceKey, delimiter = node.delimiter)
           }
         }
     }
     is ArrayNode -> {
       node.elements.map { process(it, context) }.sequence()
         .mapInvalid { ConfigFailure.MultipleFailures(it) }
-        .map { ArrayNode(it, node.pos, node.path, sourceKey = node.sourceKey) }
+        .map { ArrayNode(it, node.pos, node.path, sourceKey = node.sourceKey, delimiter = node.delimiter) }
     }
     is PrimitiveNode -> handle(node, context)
     else -> node.valid()

hoplite-core/src/main/kotlin/com/sksamuel/hoplite/sources/EnvironmentVariableOverridePropertySource.kt

@@ -25,11 +25,13 @@ class EnvironmentVariableOverridePropertySource(
   override fun source(): String = "Env Var Overrides"
 
   override fun node(context: PropertySourceContext): ConfigResult<Node> {
+    // at the moment the delimiter is either `__` or `.` -- it can't be mixed
+    val delimiter = if (useUnderscoresAsSeparator) "__" else "."
+
     val vars = environmentVariableMap()
-      .mapKeys { if (useUnderscoresAsSeparator) it.key.replace("__", ".") else it.key }
       .filter { it.key.startsWith(Prefix) }
     return if (vars.isEmpty()) Undefined.valid() else {
-      vars.toNode("Env Var Overrides") {
+      vars.toNode("Env Var Overrides", delimiter) {
         it.removePrefix(Prefix)
       }.valid()
     }

hoplite-core/src/main/kotlin/com/sksamuel/hoplite/sources/EnvironmentVariablesPropertySource.kt

@@ -20,13 +20,15 @@ class EnvironmentVariablesPropertySource(
     val map = environmentVariableMap()
       .mapKeys { if (prefix == null) it.key else it.key.removePrefix(prefix) }
 
-    return map.toNode("env") { key ->
+    // at the moment the delimiter is either `__` or `.` -- it can't be mixed
+    val delimiter = if (useUnderscoresAsSeparator) "__" else "."
+
+    return map.toNode("env", delimiter) { key ->
       key
         .let { if (prefix == null) it else it.removePrefix(prefix) }
-        .let { if (useUnderscoresAsSeparator) it.replace("__", ".") else it }
         .let {
           if (allowUppercaseNames && Character.isUpperCase(it.codePointAt(0))) {
-            it.split(".").joinToString(separator = ".") { value ->
+            it.split(delimiter).joinToString(separator = delimiter) { value ->
               value.fold("") { acc, char ->
                 when {
                   acc.isEmpty() -> acc + char.lowercaseChar()

hoplite-core/src/test/kotlin/com/sksamuel/hoplite/PrefixTest.kt

@@ -123,7 +123,7 @@ class PrefixTest : FunSpec() {
     test("reads from default source before specified at a given prefix") {
       data class TestConfig(val a: String, val b: Int, val other: List<String>)
 
-      withEnvironment(mapOf("foo.b" to "91", "foo.other" to "Random13")) {
+      withEnvironment(mapOf("foo__b" to "91", "foo__other" to "Random13")) {
         val arguments = arrayOf(
           "--foo.a=A value",
           "--foo.b=42",

hoplite-json/src/test/kotlin/com/sksamuel/hoplite/json/EnvPropertySourceTest.kt

@@ -30,7 +30,7 @@ class EnvPropertySourceTest : FunSpec({
     data class Bar(val s: Long, val t: Long)
     data class Foo(val bar: Bar)
     data class TestConfig(val foo: Foo)
-    withEnvironment(mapOf("foo.bar.s" to "1")) {
+    withEnvironment(mapOf("foo__bar__s" to "1")) {
       ConfigLoader.builder()
         .addEnvironmentSource()
         .build()
@@ -42,7 +42,7 @@ class EnvPropertySourceTest : FunSpec({
     data class Foo(val bar: Bar)
     data class TestConfig(val foo: Foo)
     // the sys prop foo is a parent of our bar.s but since Foo maps to a data class, the prop should never be used
-    withEnvironment(mapOf("foo.bar.s" to "x", "foo" to "y")) {
+    withEnvironment(mapOf("foo__bar__s" to "x", "foo" to "y")) {
       ConfigLoader.builder()
         .addEnvironmentSource()
         .build()
@@ -54,7 +54,7 @@ class EnvPropertySourceTest : FunSpec({
     data class Foo(val bar: Bar)
     data class TestConfig(val foo: Foo)
     // the sysprop foo.bar.s has a child foo.bar.s.u which is not required, but the parent value should still be used
-    withEnvironment(mapOf("foo.bar.s" to "x", "foo.bar.s.u" to "y")) {
+    withEnvironment(mapOf("foo__bar__s" to "x", "foo__bar__s__u" to "y")) {
       ConfigLoader.builder()
         .addEnvironmentSource()
         .build()

hoplite-json/src/test/kotlin/com/sksamuel/hoplite/json/EnvPropertySourceUppercaseTest.kt

@@ -20,8 +20,8 @@ class EnvPropertySourceUppercaseTest : DescribeSpec({
               allowUppercaseNames = true,
               environmentVariableMap = {
                 mapOf(
-                  "CREDS.USERNAME" to "a",
-                  "CREDS.PASSWORD" to "c",
+                  "CREDS__USERNAME" to "a",
+                  "CREDS__PASSWORD" to "c",
                   "SOME_CAMEL_SETTING" to "c"
                 )
               }
@@ -78,8 +78,8 @@ class EnvPropertySourceUppercaseTest : DescribeSpec({
               allowUppercaseNames = true,
               environmentVariableMap = {
                 mapOf(
-                  "WIBBLE_CREDS.USERNAME" to "a",
-                  "WIBBLE_CREDS.PASSWORD" to "c",
+                  "WIBBLE_CREDS__USERNAME" to "a",
+                  "WIBBLE_CREDS__PASSWORD" to "c",
                   "WIBBLE_SOME_CAMEL_SETTING" to "c"
                 )
               },

hoplite-vavr/src/test/kotlin/com/sksamuel/hoplite/decoder/vavr/MapDecoderTest.kt

@@ -1,18 +1,56 @@
 package com.sksamuel.hoplite.decoder.vavr
 
 import com.sksamuel.hoplite.ConfigLoader
+import com.sksamuel.hoplite.sources.EnvironmentVariablesPropertySource
 import io.kotest.core.spec.style.FunSpec
 import io.kotest.matchers.shouldBe
 import io.vavr.collection.Map
 import io.vavr.kotlin.linkedHashMap
 
 class MapDecoderTest : FunSpec({
+  data class Test(val map: Map<String, String>)
 
   test("Map<String, String> decoded from yaml") {
-    data class Test(val map: Map<String, String>)
-
     val config = ConfigLoader().loadConfigOrThrow<Test>("/test_map.yml")
     config shouldBe Test(linkedHashMap("key1" to "test1", "key2" to "test2", "key-3" to "test3", "Key4" to "test4"))
   }
 
+  test("Map<String, String> decoded from environment with underscores") {
+    run {
+      ConfigLoader {
+        addPropertySource(EnvironmentVariablesPropertySource(
+          useUnderscoresAsSeparator = true,
+          allowUppercaseNames = true,
+          environmentVariableMap = {
+            mapOf(
+              "map__key1" to "test1",
+              "map__key2" to "test2",
+              "map__key-3" to "test3",
+              "map__Key4" to "test4",
+            )
+          }
+        ))
+      }.loadConfigOrThrow<Test>()
+    } shouldBe Test(linkedHashMap("key1" to "test1", "key2" to "test2", "key-3" to "test3", "Key4" to "test4"))
+  }
+
+  test("Map<String, String> decoded from environment") {
+    run {
+      ConfigLoader {
+        addPropertySource(EnvironmentVariablesPropertySource(
+          useUnderscoresAsSeparator = false,
+          allowUppercaseNames = true,
+          environmentVariableMap = {
+            mapOf(
+              "map.key1" to "test1",
+              "map.key2" to "test2",
+              "map.key-3" to "test3",
+              "map.Key4" to "test4",
+            )
+          }
+        ))
+      }.loadConfigOrThrow<Test>()
+    } shouldBe Test(linkedHashMap("key1" to "test1", "key2" to "test2", "key-3" to "test3", "Key4" to "test4"))
+  }
+
 })

hoplite-yaml/src/test/kotlin/com/sksamuel/hoplite/yaml/DenormalizedMapKeysTest.kt

@@ -77,8 +77,8 @@ class DenormalizedMapKeysTest : FunSpec({
   test("should set denormalized map keys from environment variables") {
     withEnvironment(
       mapOf(
-        "m.DC1.x-val" to "15",
-        "m.DC2.x-val" to "25"
+        "m__DC1__x-val" to "15",
+        "m__DC2__x-val" to "25"
       )
     ) {
       val config = ConfigLoaderBuilder.default()
@@ -98,8 +98,8 @@ class DenormalizedMapKeysTest : FunSpec({
   test("should set denormalized map keys from environment variables, overriding a property source") {
     withEnvironment(
       mapOf(
-        "m.DC1.x-val" to "15",
-        "m.DC2.x-val" to "25"
+        "m__DC1__x-val" to "15",
+        "m__DC2__x-val" to "25"
       )
     ) {
       val config = ConfigLoaderBuilder.default()
@@ -120,8 +120,8 @@ class DenormalizedMapKeysTest : FunSpec({
   test("should set denormalized map keys from command line arguments, overriding environment variables and property sources") {
     withEnvironment(
       mapOf(
-        "m.DC1.x-val" to "15",
-        "m.DC2.x-val" to "25"
+        "m__DC1__x-val" to "15",
+        "m__DC2__x-val" to "25"
       )
     ) {
       val config = ConfigLoaderBuilder.default()