From d9d1f0a3fc650c3b70ad4b40c95070911f5f5a8c Mon Sep 17 00:00:00 2001 From: Andrea Peruffo Date: Wed, 27 Feb 2019 12:37:37 +0000 Subject: [PATCH 01/13] Add a config strategy to load from environment variables first. --- config/checkstyle-config.xml | 2 +- config/checkstyle-suppressions.xml | 2 +- .../config/EnvFirstConfigLoadingStrategy.java | 79 +++++++++++++++++++ .../com/typesafe/config/impl/ConfigTest.scala | 47 +++++++++++ .../typesafe/config/impl/PublicApiTest.scala | 29 +++++++ 5 files changed, 157 insertions(+), 2 deletions(-) create mode 100644 config/src/main/java/com/typesafe/config/EnvFirstConfigLoadingStrategy.java diff --git a/config/checkstyle-config.xml b/config/checkstyle-config.xml index b5b71d2b6..0fc2c197a 100644 --- a/config/checkstyle-config.xml +++ b/config/checkstyle-config.xml @@ -1,7 +1,7 @@ + "https://checkstyle.org/dtds/configuration_1_3.dtd"> diff --git a/config/checkstyle-suppressions.xml b/config/checkstyle-suppressions.xml index 21fb8e51e..6fa3fdc00 100644 --- a/config/checkstyle-suppressions.xml +++ b/config/checkstyle-suppressions.xml @@ -1,6 +1,6 @@ + "https://checkstyle.org/dtds/configuration_1_3.dtd"> + * Environment variables are mangled in the following way after stripping the prefix: + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + *
Env VarConfig
_   [1 underscore]. [dot]
__  [2 underscore]- [dash]
___ [3 underscore]_ [underscore]
+ * + *

+ * A variable like: {@code CONFIG_a_b__c___d} + * is translated to a config key: {@code a.b-c_d} + * + *

+ * The prefix may be altered by defining the VM property {@code config.env_var_prefix} + */ +public class EnvFirstConfigLoadingStrategy extends DefaultConfigLoadingStrategy { + + protected static Map env = new HashMap(System.getenv()); + + @Override + public Config parseApplicationConfig(ConfigParseOptions parseOptions) { + String envVarPrefix = System.getProperty("config.env_var_prefix"); + if (envVarPrefix == null) // fallback to default + envVarPrefix = "CONFIG_"; + + Map defaultsFromEnv = new HashMap(); + for (String key : env.keySet()) { + if (key.startsWith(envVarPrefix)) { + StringBuilder builder = new StringBuilder(); + + String strippedPrefix = key.substring(envVarPrefix.length(), key.length()); + + int underscores = 0; + for (char c : strippedPrefix.toCharArray()) { + if (c == '_') { + underscores++; + } else { + switch (underscores) { + case 1: builder.append('.'); + break; + case 2: builder.append('-'); + break; + case 3: builder.append('_'); + break; + } + underscores = 0; + builder.append(c); + } + } + + String propertyKey = builder.toString(); + defaultsFromEnv.put(propertyKey, env.get(key)); + } + } + + return ConfigFactory.parseMap(defaultsFromEnv).withFallback(super.parseApplicationConfig(parseOptions)); + } +} diff --git a/config/src/test/scala/com/typesafe/config/impl/ConfigTest.scala b/config/src/test/scala/com/typesafe/config/impl/ConfigTest.scala index 1bf031ba3..f5084c79d 100644 --- a/config/src/test/scala/com/typesafe/config/impl/ConfigTest.scala +++ b/config/src/test/scala/com/typesafe/config/impl/ConfigTest.scala @@ -1090,6 +1090,53 @@ class ConfigTest extends TestUtils { assertEquals(10, resolved.getInt("bar.nested.a.q")) } + @Test + def testLoadWithEnvSubstitutions() { + TestEnvFirstStrategy.putEnvVar("CONFIG_42___a", "1") + TestEnvFirstStrategy.putEnvVar("CONFIG_a_b_c", "2") + TestEnvFirstStrategy.putEnvVar("CONFIG_a__c", "3") + TestEnvFirstStrategy.putEnvVar("CONFIG_a___c", "4") + + TestEnvFirstStrategy.putEnvVar("CONFIG_akka_version", "foo") + TestEnvFirstStrategy.putEnvVar("CONFIG_akka_event__handler__dispatcher_max__pool__size", "10") + + System.setProperty("config.strategy", classOf[EnvFirstConfigLoadingStrategy].getCanonicalName) + + try { + val loader02 = new TestClassLoader(this.getClass().getClassLoader(), + Map("reference.conf" -> resourceFile("test02.conf").toURI.toURL())) + + val loader04 = new TestClassLoader(this.getClass().getClassLoader(), + Map("reference.conf" -> resourceFile("test04.conf").toURI.toURL())) + + val conf02 = withContextClassLoader(loader02) { + ConfigFactory.load() + } + + val conf04 = withContextClassLoader(loader04) { + ConfigFactory.load() + } + + assertEquals(1, conf02.getInt("42_a")) + assertEquals(2, conf02.getInt("a.b.c")) + assertEquals(3, conf02.getInt("a-c")) + assertEquals(4, conf02.getInt("a_c")) + + assertEquals("foo", conf04.getString("akka.version")) + assertEquals(10, conf04.getInt("akka.event-handler-dispatcher.max-pool-size")) + } finally { + System.clearProperty("config.strategy") + + TestEnvFirstStrategy.removeEnvVar("CONFIG_42___a") + TestEnvFirstStrategy.removeEnvVar("CONFIG_a_b_c") + TestEnvFirstStrategy.removeEnvVar("CONFIG_a__c") + TestEnvFirstStrategy.removeEnvVar("CONFIG_a___c") + + TestEnvFirstStrategy.removeEnvVar("CONFIG_akka_version") + TestEnvFirstStrategy.removeEnvVar("CONFIG_akka_event__handler__dispatcher_max__pool__size") + } + } + @Test def renderRoundTrip() { val allBooleans = true :: false :: Nil diff --git a/config/src/test/scala/com/typesafe/config/impl/PublicApiTest.scala b/config/src/test/scala/com/typesafe/config/impl/PublicApiTest.scala index d25d43659..136ea690a 100644 --- a/config/src/test/scala/com/typesafe/config/impl/PublicApiTest.scala +++ b/config/src/test/scala/com/typesafe/config/impl/PublicApiTest.scala @@ -653,6 +653,28 @@ class PublicApiTest extends TestUtils { } } + @Test + def supportsEnvFirstConfigLoadingStrategy(): Unit = { + assertEquals("config.strategy is not set", null, System.getProperty("config.strategy")) + + TestEnvFirstStrategy.putEnvVar("CONFIG_a", "5") + System.setProperty("config.strategy", classOf[EnvFirstConfigLoadingStrategy].getCanonicalName) + + try { + val loaderA1 = new TestClassLoader(this.getClass().getClassLoader(), + Map("reference.conf" -> resourceFile("a_1.conf").toURI.toURL())) + + val configA1 = withContextClassLoader(loaderA1) { + ConfigFactory.load() + } + + assertEquals(5, configA1.getInt("a")) + } finally { + System.clearProperty("config.strategy") + TestEnvFirstStrategy.removeEnvVar("CONFIG_a") + } + } + @Test def usesContextClassLoaderForApplicationConf() { val loaderA1 = new TestClassLoader(this.getClass().getClassLoader(), @@ -1145,4 +1167,11 @@ object TestStrategy { private var invocations = 0 def getIncovations() = invocations def increment() = invocations += 1 +} + +object TestEnvFirstStrategy extends EnvFirstConfigLoadingStrategy { + def putEnvVar(key: String, value: String) = + EnvFirstConfigLoadingStrategy.env.put(key, value) + def removeEnvVar(key: String) = + EnvFirstConfigLoadingStrategy.env.remove(key) } \ No newline at end of file From 5c04377293ffcacb5c811323e6c67ad9a3d68bbe Mon Sep 17 00:00:00 2001 From: Andrea Peruffo Date: Wed, 27 Feb 2019 12:39:46 +0000 Subject: [PATCH 02/13] Fix suppressions doctype --- config/checkstyle-suppressions.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/checkstyle-suppressions.xml b/config/checkstyle-suppressions.xml index 6fa3fdc00..62d514067 100644 --- a/config/checkstyle-suppressions.xml +++ b/config/checkstyle-suppressions.xml @@ -1,6 +1,6 @@ + "https://checkstyle.org/dtds/suppressions_1_1.dtd"> Date: Wed, 27 Feb 2019 12:41:15 +0000 Subject: [PATCH 03/13] Trailing blank line --- .../src/test/scala/com/typesafe/config/impl/PublicApiTest.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/src/test/scala/com/typesafe/config/impl/PublicApiTest.scala b/config/src/test/scala/com/typesafe/config/impl/PublicApiTest.scala index 136ea690a..5eed422ac 100644 --- a/config/src/test/scala/com/typesafe/config/impl/PublicApiTest.scala +++ b/config/src/test/scala/com/typesafe/config/impl/PublicApiTest.scala @@ -1174,4 +1174,4 @@ object TestEnvFirstStrategy extends EnvFirstConfigLoadingStrategy { EnvFirstConfigLoadingStrategy.env.put(key, value) def removeEnvVar(key: String) = EnvFirstConfigLoadingStrategy.env.remove(key) -} \ No newline at end of file +} From 7fe47888d2ccb0f4cb9f29030c23c9bc34c44c3f Mon Sep 17 00:00:00 2001 From: Andrea Peruffo Date: Mon, 4 Mar 2019 14:43:12 +0000 Subject: [PATCH 04/13] Env variables resolution moved to defaultOverrides --- build.sbt | 12 ++- .../com/typesafe/config/ConfigFactory.java | 59 +++++++++++++- .../config/EnvFirstConfigLoadingStrategy.java | 79 ------------------- .../com/typesafe/config/impl/ConfigImpl.java | 52 ++++++++++++ .../com/typesafe/config/impl/ConfigTest.scala | 20 +---- .../typesafe/config/impl/PublicApiTest.scala | 25 ++---- 6 files changed, 130 insertions(+), 117 deletions(-) delete mode 100644 config/src/main/java/com/typesafe/config/EnvFirstConfigLoadingStrategy.java diff --git a/build.sbt b/build.sbt index 9d78b6aea..15260557a 100644 --- a/build.sbt +++ b/build.sbt @@ -85,7 +85,17 @@ lazy val configLib = Project("config", file("config")) Test/ run / fork := true //env vars for tests - Test / envVars ++= Map("testList.0" -> "0", "testList.1" -> "1") + Test / envVars ++= Map("testList.0" -> "0", + "testList.1" -> "1", + "CONFIG_FORCE_b" -> "5", + "CONFIG_FORCE_testList_0" -> "10", + "CONFIG_FORCE_testList_1" -> "11", + "CONFIG_FORCE_42___a" -> "1", + "CONFIG_FORCE_a_b_c" -> "2", + "CONFIG_FORCE_a__c" -> "3", + "CONFIG_FORCE_a___c" -> "4", + "CONFIG_FORCE_akka_version" -> "foo", + "CONFIG_FORCE_akka_event__handler__dispatcher_max__pool__size" -> "10") OsgiKeys.exportPackage := Seq("com.typesafe.config", "com.typesafe.config.impl") publish := sys.error("use publishSigned instead of plain publish") diff --git a/config/src/main/java/com/typesafe/config/ConfigFactory.java b/config/src/main/java/com/typesafe/config/ConfigFactory.java index 86d995e36..8405520f5 100644 --- a/config/src/main/java/com/typesafe/config/ConfigFactory.java +++ b/config/src/main/java/com/typesafe/config/ConfigFactory.java @@ -36,6 +36,7 @@ */ public final class ConfigFactory { private static final String STRATEGY_PROPERTY_NAME = "config.strategy"; + private static final String OVERRIDE_WITH_ENV_PROPERTY_NAME = "config.override_with_env_vars"; private ConfigFactory() { } @@ -383,7 +384,11 @@ public static Config defaultReference(ClassLoader loader) { * @return the default override configuration */ public static Config defaultOverrides() { - return systemProperties(); + if (!getOverrideWithEnv()) { + return systemProperties(); + } else { + return systemEnvironmentOverrides().withFallback(systemProperties()); + } } /** @@ -394,7 +399,7 @@ public static Config defaultOverrides() { * @return the default override configuration */ public static Config defaultOverrides(ClassLoader loader) { - return systemProperties(); + return defaultOverrides(); } /** @@ -549,6 +554,50 @@ public static Config systemProperties() { return ConfigImpl.systemPropertiesAsConfig(); } + /** + * Gets a Config containing the system's environment variables + * used to override configuration keys. + * Environment variables taken in considerations are starting with + * {@code CONFIG_FORCE_} + * + *

+ * Environment variables are mangled in the following way after stripping the prefix "CONFIG_FORCE_": + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + *
Env VarConfig
_   [1 underscore]. [dot]
__  [2 underscore]- [dash]
___ [3 underscore]_ [underscore]
+ * + *

+ * A variable like: {@code CONFIG_FORCE_a_b__c___d} + * is translated to a config key: {@code a.b-c_d} + * + *

+ * This method can return a global immutable singleton, so it's preferred + * over parsing system properties yourself. + *

+ * {@link #defaultOverrides} will include the system system environment variables as + * overrides if `config.override_with_env_vars` is set to `true`. + * + * @return system environment variable overrides parsed into a Config + */ + public static Config systemEnvironmentOverrides() { + return ConfigImpl.envVariablesOverridesAsConfig(); + } + /** * Gets a Config containing the system's environment variables. * This method can return a global immutable singleton. @@ -1063,4 +1112,10 @@ private static ConfigLoadingStrategy getConfigLoadingStrategy() { return new DefaultConfigLoadingStrategy(); } } + + private static Boolean getOverrideWithEnv() { + String overrideWithEnv = System.getProperties().getProperty(OVERRIDE_WITH_ENV_PROPERTY_NAME); + + return Boolean.parseBoolean(overrideWithEnv); + } } diff --git a/config/src/main/java/com/typesafe/config/EnvFirstConfigLoadingStrategy.java b/config/src/main/java/com/typesafe/config/EnvFirstConfigLoadingStrategy.java deleted file mode 100644 index 0b0948035..000000000 --- a/config/src/main/java/com/typesafe/config/EnvFirstConfigLoadingStrategy.java +++ /dev/null @@ -1,79 +0,0 @@ -package com.typesafe.config; - -import java.util.Map; -import java.util.HashMap; - -/** - * Environment variables first config loading strategy. Able to load environment variables first and fallback to {@link DefaultConfigLoadingStrategy} - * - *

- * Environment variables are mangled in the following way after stripping the prefix: - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - *
Env VarConfig
_   [1 underscore]. [dot]
__  [2 underscore]- [dash]
___ [3 underscore]_ [underscore]
- * - *

- * A variable like: {@code CONFIG_a_b__c___d} - * is translated to a config key: {@code a.b-c_d} - * - *

- * The prefix may be altered by defining the VM property {@code config.env_var_prefix} - */ -public class EnvFirstConfigLoadingStrategy extends DefaultConfigLoadingStrategy { - - protected static Map env = new HashMap(System.getenv()); - - @Override - public Config parseApplicationConfig(ConfigParseOptions parseOptions) { - String envVarPrefix = System.getProperty("config.env_var_prefix"); - if (envVarPrefix == null) // fallback to default - envVarPrefix = "CONFIG_"; - - Map defaultsFromEnv = new HashMap(); - for (String key : env.keySet()) { - if (key.startsWith(envVarPrefix)) { - StringBuilder builder = new StringBuilder(); - - String strippedPrefix = key.substring(envVarPrefix.length(), key.length()); - - int underscores = 0; - for (char c : strippedPrefix.toCharArray()) { - if (c == '_') { - underscores++; - } else { - switch (underscores) { - case 1: builder.append('.'); - break; - case 2: builder.append('-'); - break; - case 3: builder.append('_'); - break; - } - underscores = 0; - builder.append(c); - } - } - - String propertyKey = builder.toString(); - defaultsFromEnv.put(propertyKey, env.get(key)); - } - } - - return ConfigFactory.parseMap(defaultsFromEnv).withFallback(super.parseApplicationConfig(parseOptions)); - } -} diff --git a/config/src/main/java/com/typesafe/config/impl/ConfigImpl.java b/config/src/main/java/com/typesafe/config/impl/ConfigImpl.java index 9cf49913b..f9057ede5 100644 --- a/config/src/main/java/com/typesafe/config/impl/ConfigImpl.java +++ b/config/src/main/java/com/typesafe/config/impl/ConfigImpl.java @@ -32,6 +32,7 @@ * For use only by the {@link com.typesafe.config} package. */ public class ConfigImpl { + private static final String ENV_VAR_OVERRIDE_PREFIX = "CONFIG_FORCE_"; private static class LoaderCache { private Config currentSystemProperties; @@ -360,6 +361,57 @@ public static void reloadEnvVariablesConfig() { EnvVariablesHolder.envVariables = loadEnvVariables(); } + private static AbstractConfigObject loadEnvVariablesOverrides() { + Map env = new HashMap(System.getenv()); + Map result = new HashMap(System.getenv()); + for (String key : env.keySet()) { + if (key.startsWith(ENV_VAR_OVERRIDE_PREFIX)) { + StringBuilder builder = new StringBuilder(); + + String strippedPrefix = key.substring(ENV_VAR_OVERRIDE_PREFIX.length(), key.length()); + + int underscores = 0; + for (char c : strippedPrefix.toCharArray()) { + if (c == '_') { + underscores++; + } else { + switch (underscores) { + case 1: builder.append('.'); + break; + case 2: builder.append('-'); + break; + case 3: builder.append('_'); + break; + } + underscores = 0; + builder.append(c); + } + } + + String propertyKey = builder.toString(); + result.put(propertyKey, env.get(key)); + } + } + + return PropertiesParser.fromStringMap(newSimpleOrigin("env variables overrides"), result); + } + + private static class EnvVariablesOverridesHolder { + static volatile AbstractConfigObject envVariables = loadEnvVariablesOverrides(); + } + + static AbstractConfigObject envVariablesOverridesAsConfigObject() { + try { + return EnvVariablesOverridesHolder.envVariables; + } catch (ExceptionInInitializerError e) { + throw ConfigImplUtil.extractInitializerError(e); + } + } + + public static Config envVariablesOverridesAsConfig() { + return envVariablesOverridesAsConfigObject().toConfig(); + } + public static Config defaultReference(final ClassLoader loader) { return computeCachedConfig(loader, "defaultReference", new Callable() { @Override diff --git a/config/src/test/scala/com/typesafe/config/impl/ConfigTest.scala b/config/src/test/scala/com/typesafe/config/impl/ConfigTest.scala index f5084c79d..853aa249f 100644 --- a/config/src/test/scala/com/typesafe/config/impl/ConfigTest.scala +++ b/config/src/test/scala/com/typesafe/config/impl/ConfigTest.scala @@ -1092,15 +1092,7 @@ class ConfigTest extends TestUtils { @Test def testLoadWithEnvSubstitutions() { - TestEnvFirstStrategy.putEnvVar("CONFIG_42___a", "1") - TestEnvFirstStrategy.putEnvVar("CONFIG_a_b_c", "2") - TestEnvFirstStrategy.putEnvVar("CONFIG_a__c", "3") - TestEnvFirstStrategy.putEnvVar("CONFIG_a___c", "4") - - TestEnvFirstStrategy.putEnvVar("CONFIG_akka_version", "foo") - TestEnvFirstStrategy.putEnvVar("CONFIG_akka_event__handler__dispatcher_max__pool__size", "10") - - System.setProperty("config.strategy", classOf[EnvFirstConfigLoadingStrategy].getCanonicalName) + System.setProperty("config.override_with_env_vars", "true") try { val loader02 = new TestClassLoader(this.getClass().getClassLoader(), @@ -1125,15 +1117,7 @@ class ConfigTest extends TestUtils { assertEquals("foo", conf04.getString("akka.version")) assertEquals(10, conf04.getInt("akka.event-handler-dispatcher.max-pool-size")) } finally { - System.clearProperty("config.strategy") - - TestEnvFirstStrategy.removeEnvVar("CONFIG_42___a") - TestEnvFirstStrategy.removeEnvVar("CONFIG_a_b_c") - TestEnvFirstStrategy.removeEnvVar("CONFIG_a__c") - TestEnvFirstStrategy.removeEnvVar("CONFIG_a___c") - - TestEnvFirstStrategy.removeEnvVar("CONFIG_akka_version") - TestEnvFirstStrategy.removeEnvVar("CONFIG_akka_event__handler__dispatcher_max__pool__size") + System.clearProperty("config.override_with_env_vars") } } diff --git a/config/src/test/scala/com/typesafe/config/impl/PublicApiTest.scala b/config/src/test/scala/com/typesafe/config/impl/PublicApiTest.scala index 5eed422ac..4c72643ea 100644 --- a/config/src/test/scala/com/typesafe/config/impl/PublicApiTest.scala +++ b/config/src/test/scala/com/typesafe/config/impl/PublicApiTest.scala @@ -654,24 +654,22 @@ class PublicApiTest extends TestUtils { } @Test - def supportsEnvFirstConfigLoadingStrategy(): Unit = { - assertEquals("config.strategy is not set", null, System.getProperty("config.strategy")) + def loadEnvironmentVariablesOverridesIfConfigured(): Unit = { + assertEquals("config.override_with_env_vars is not set", null, System.getProperty("config.override_with_env_vars")) - TestEnvFirstStrategy.putEnvVar("CONFIG_a", "5") - System.setProperty("config.strategy", classOf[EnvFirstConfigLoadingStrategy].getCanonicalName) + System.setProperty("config.override_with_env_vars", "true") try { - val loaderA1 = new TestClassLoader(this.getClass().getClassLoader(), - Map("reference.conf" -> resourceFile("a_1.conf").toURI.toURL())) + val loaderB2 = new TestClassLoader(this.getClass().getClassLoader(), + Map("reference.conf" -> resourceFile("b_2.conf").toURI.toURL())) - val configA1 = withContextClassLoader(loaderA1) { + val configB2 = withContextClassLoader(loaderB2) { ConfigFactory.load() } - assertEquals(5, configA1.getInt("a")) + assertEquals(5, configB2.getInt("b")) } finally { - System.clearProperty("config.strategy") - TestEnvFirstStrategy.removeEnvVar("CONFIG_a") + System.clearProperty("config.override_with_env_vars") } } @@ -1168,10 +1166,3 @@ object TestStrategy { def getIncovations() = invocations def increment() = invocations += 1 } - -object TestEnvFirstStrategy extends EnvFirstConfigLoadingStrategy { - def putEnvVar(key: String, value: String) = - EnvFirstConfigLoadingStrategy.env.put(key, value) - def removeEnvVar(key: String) = - EnvFirstConfigLoadingStrategy.env.remove(key) -} From e302b9a34d09afcf4f289038e719ebc648071e1a Mon Sep 17 00:00:00 2001 From: Andrea Peruffo Date: Fri, 8 Mar 2019 14:26:47 +0000 Subject: [PATCH 05/13] fixed review comments --- .../src/main/java/com/typesafe/config/ConfigFactory.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/config/src/main/java/com/typesafe/config/ConfigFactory.java b/config/src/main/java/com/typesafe/config/ConfigFactory.java index 8405520f5..4937c8d3b 100644 --- a/config/src/main/java/com/typesafe/config/ConfigFactory.java +++ b/config/src/main/java/com/typesafe/config/ConfigFactory.java @@ -384,10 +384,10 @@ public static Config defaultReference(ClassLoader loader) { * @return the default override configuration */ public static Config defaultOverrides() { - if (!getOverrideWithEnv()) { - return systemProperties(); - } else { + if (getOverrideWithEnv()) { return systemEnvironmentOverrides().withFallback(systemProperties()); + } else { + return systemProperties(); } } @@ -589,7 +589,7 @@ public static Config systemProperties() { * This method can return a global immutable singleton, so it's preferred * over parsing system properties yourself. *

- * {@link #defaultOverrides} will include the system system environment variables as + * {@link #defaultOverrides} will include the system environment variables as * overrides if `config.override_with_env_vars` is set to `true`. * * @return system environment variable overrides parsed into a Config From c14bb77acdbebf7d9ab5db1339fcab5b0a6eed3a Mon Sep 17 00:00:00 2001 From: Andrea Peruffo Date: Fri, 8 Mar 2019 14:53:14 +0000 Subject: [PATCH 06/13] Added a description of the new property in Readme --- README.md | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/README.md b/README.md index 28d51af89..d967fd6fc 100644 --- a/README.md +++ b/README.md @@ -675,6 +675,24 @@ value just disappear if the substitution is not found: // this array could have one or two elements path = [ "a", ${?OPTIONAL_A} ] +By setting the JVM property `-Dconfig.override_with_env_vars=true` +it is possible to override any configuration value using environment +variables even if an explicit substitution is not specified. + +The environment variable value will override any pre-existing value +and also any value provided as Java property. + +With this option enabled only environment variables starting with +`CONFIG_FORCE_` are considered, and the name is mangled as follows: + + - the prefix `CONFIG_FORCE_` is stripped + - single underscore(`_`) is converted into a dot(`.`) + - double underscore(`__`) is converted into a dash(`-`) + - triple underscore(`___`) is converted into a single underscore(`_`) + +i.e. The environment variable `CONFIG_FORCE_a_b__c___d` set the +configuration key `a.b-c_d` + ### Concatenation Values _on the same line_ are concatenated (for strings and From 4571c79b512dc19ec7950b87371d445a1a306e65 Mon Sep 17 00:00:00 2001 From: Andrea Peruffo Date: Fri, 15 Mar 2019 16:40:59 +0000 Subject: [PATCH 07/13] Detail resons for _ sustitutions --- README.md | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/README.md b/README.md index d967fd6fc..23ef139e4 100644 --- a/README.md +++ b/README.md @@ -693,6 +693,15 @@ With this option enabled only environment variables starting with i.e. The environment variable `CONFIG_FORCE_a_b__c___d` set the configuration key `a.b-c_d` +Rationale the name mangling: + +Most shells (e.g. bash, sh, etc.) doesn't support any character other +than alphanumeric and `_` in environment variables names. +In HOCON the default separator is `.` so it is directly translated to a +single `_` for convenience; `-` and `_` are less often present in config +keys but they have to be representable and the only possible mapping is +`_` repeated. + ### Concatenation Values _on the same line_ are concatenated (for strings and From 87dc17d55ed035577806266822c50de8624dfb35 Mon Sep 17 00:00:00 2001 From: Andrea Peruffo Date: Thu, 21 Mar 2019 08:52:42 +0000 Subject: [PATCH 08/13] Move rationale to code and implement invalidate cache --- README.md | 9 --------- .../java/com/typesafe/config/ConfigFactory.java | 1 + .../java/com/typesafe/config/impl/ConfigImpl.java | 14 ++++++++++++++ 3 files changed, 15 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index 23ef139e4..d967fd6fc 100644 --- a/README.md +++ b/README.md @@ -693,15 +693,6 @@ With this option enabled only environment variables starting with i.e. The environment variable `CONFIG_FORCE_a_b__c___d` set the configuration key `a.b-c_d` -Rationale the name mangling: - -Most shells (e.g. bash, sh, etc.) doesn't support any character other -than alphanumeric and `_` in environment variables names. -In HOCON the default separator is `.` so it is directly translated to a -single `_` for convenience; `-` and `_` are less often present in config -keys but they have to be representable and the only possible mapping is -`_` repeated. - ### Concatenation Values _on the same line_ are concatenated (for strings and diff --git a/config/src/main/java/com/typesafe/config/ConfigFactory.java b/config/src/main/java/com/typesafe/config/ConfigFactory.java index 4937c8d3b..4b91a1ea4 100644 --- a/config/src/main/java/com/typesafe/config/ConfigFactory.java +++ b/config/src/main/java/com/typesafe/config/ConfigFactory.java @@ -502,6 +502,7 @@ public static void invalidateCaches() { // all caches ConfigImpl.reloadSystemPropertiesConfig(); ConfigImpl.reloadEnvVariablesConfig(); + ConfigImpl.reloadEnvVariablesOverridesConfig(); } /** diff --git a/config/src/main/java/com/typesafe/config/impl/ConfigImpl.java b/config/src/main/java/com/typesafe/config/impl/ConfigImpl.java index f9057ede5..ffd2ba891 100644 --- a/config/src/main/java/com/typesafe/config/impl/ConfigImpl.java +++ b/config/src/main/java/com/typesafe/config/impl/ConfigImpl.java @@ -364,6 +364,14 @@ public static void reloadEnvVariablesConfig() { private static AbstractConfigObject loadEnvVariablesOverrides() { Map env = new HashMap(System.getenv()); Map result = new HashMap(System.getenv()); + // Rationale on name mangling: + // + // Most shells (e.g. bash, sh, etc.) doesn't support any character other + // than alphanumeric and `_` in environment variables names. + // In HOCON the default separator is `.` so it is directly translated to a + // single `_` for convenience; `-` and `_` are less often present in config + // keys but they have to be representable and the only possible mapping is + // `_` repeated. for (String key : env.keySet()) { if (key.startsWith(ENV_VAR_OVERRIDE_PREFIX)) { StringBuilder builder = new StringBuilder(); @@ -412,6 +420,12 @@ public static Config envVariablesOverridesAsConfig() { return envVariablesOverridesAsConfigObject().toConfig(); } + public static void reloadEnvVariablesOverridesConfig() { + // ConfigFactory.invalidateCaches() relies on this having the side + // effect that it drops all caches + EnvVariablesOverridesHolder.envVariables = loadEnvVariablesOverrides(); + } + public static Config defaultReference(final ClassLoader loader) { return computeCachedConfig(loader, "defaultReference", new Callable() { @Override From 3a9a1d83333362f0adbd7830b40a7b10f5a137c2 Mon Sep 17 00:00:00 2001 From: Andrea Peruffo Date: Thu, 21 Mar 2019 09:02:48 +0000 Subject: [PATCH 09/13] Bump sbt version --- project/build.properties | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/project/build.properties b/project/build.properties index 72f902892..c0bab0494 100644 --- a/project/build.properties +++ b/project/build.properties @@ -1 +1 @@ -sbt.version=1.2.7 +sbt.version=1.2.8 From 90e2465f34aefe430976981fffe20db8ef1d3eed Mon Sep 17 00:00:00 2001 From: Andrea Peruffo Date: Thu, 21 Mar 2019 10:01:27 +0000 Subject: [PATCH 10/13] Remove Puppy Crawl references --- config/checkstyle-config.xml | 5 ++--- config/checkstyle-suppressions.xml | 3 ++- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/config/checkstyle-config.xml b/config/checkstyle-config.xml index 0fc2c197a..064ac59a5 100644 --- a/config/checkstyle-config.xml +++ b/config/checkstyle-config.xml @@ -1,8 +1,7 @@ - - diff --git a/config/checkstyle-suppressions.xml b/config/checkstyle-suppressions.xml index 62d514067..ca3fb2d15 100644 --- a/config/checkstyle-suppressions.xml +++ b/config/checkstyle-suppressions.xml @@ -1,5 +1,6 @@ - From 4937ee41abb6f6fb4101df55d93ca264b303a668 Mon Sep 17 00:00:00 2001 From: Andrea Peruffo Date: Thu, 21 Mar 2019 10:50:48 +0000 Subject: [PATCH 11/13] Move env mangling to ConfigImplUtil --- .../com/typesafe/config/impl/ConfigImpl.java | 36 ++------------- .../typesafe/config/impl/ConfigImplUtil.java | 46 +++++++++++++++++++ .../com/typesafe/config/impl/ConfigTest.scala | 16 +++++++ 3 files changed, 66 insertions(+), 32 deletions(-) diff --git a/config/src/main/java/com/typesafe/config/impl/ConfigImpl.java b/config/src/main/java/com/typesafe/config/impl/ConfigImpl.java index ffd2ba891..e96913287 100644 --- a/config/src/main/java/com/typesafe/config/impl/ConfigImpl.java +++ b/config/src/main/java/com/typesafe/config/impl/ConfigImpl.java @@ -361,43 +361,15 @@ public static void reloadEnvVariablesConfig() { EnvVariablesHolder.envVariables = loadEnvVariables(); } + + private static AbstractConfigObject loadEnvVariablesOverrides() { Map env = new HashMap(System.getenv()); Map result = new HashMap(System.getenv()); - // Rationale on name mangling: - // - // Most shells (e.g. bash, sh, etc.) doesn't support any character other - // than alphanumeric and `_` in environment variables names. - // In HOCON the default separator is `.` so it is directly translated to a - // single `_` for convenience; `-` and `_` are less often present in config - // keys but they have to be representable and the only possible mapping is - // `_` repeated. + for (String key : env.keySet()) { if (key.startsWith(ENV_VAR_OVERRIDE_PREFIX)) { - StringBuilder builder = new StringBuilder(); - - String strippedPrefix = key.substring(ENV_VAR_OVERRIDE_PREFIX.length(), key.length()); - - int underscores = 0; - for (char c : strippedPrefix.toCharArray()) { - if (c == '_') { - underscores++; - } else { - switch (underscores) { - case 1: builder.append('.'); - break; - case 2: builder.append('-'); - break; - case 3: builder.append('_'); - break; - } - underscores = 0; - builder.append(c); - } - } - - String propertyKey = builder.toString(); - result.put(propertyKey, env.get(key)); + result.put(ConfigImplUtil.envVariableAsProperty(key, ENV_VAR_OVERRIDE_PREFIX), env.get(key)); } } diff --git a/config/src/main/java/com/typesafe/config/impl/ConfigImplUtil.java b/config/src/main/java/com/typesafe/config/impl/ConfigImplUtil.java index 1dcc43d3d..1a8ffc1db 100644 --- a/config/src/main/java/com/typesafe/config/impl/ConfigImplUtil.java +++ b/config/src/main/java/com/typesafe/config/impl/ConfigImplUtil.java @@ -235,6 +235,52 @@ static String toCamelCase(String originalName) { return nameBuilder.toString(); } + private static char underscoreMappings(int num) { + // Rationale on name mangling: + // + // Most shells (e.g. bash, sh, etc.) doesn't support any character other + // than alphanumeric and `_` in environment variables names. + // In HOCON the default separator is `.` so it is directly translated to a + // single `_` for convenience; `-` and `_` are less often present in config + // keys but they have to be representable and the only possible mapping is + // `_` repeated. + switch (num) { + case 1: return '.'; + case 2: return '-'; + case 3: return '_'; + default: return 0; + } + } + + static String envVariableAsProperty(String variable, String prefix) throws ConfigException { + StringBuilder builder = new StringBuilder(); + + String strippedPrefix = variable.substring(prefix.length(), variable.length()); + + int underscores = 0; + for (char c : strippedPrefix.toCharArray()) { + if (c == '_') { + underscores++; + } else { + if (underscores > 0 && underscores < 4) { + builder.append(underscoreMappings(underscores)); + } else if (underscores > 3) { + throw new ConfigException.BadPath(variable, "Environment variable contains an un-mapped number of underscores."); + } + underscores = 0; + builder.append(c); + } + } + + if (underscores > 0 && underscores < 4) { + builder.append(underscoreMappings(underscores)); + } else if (underscores > 3) { + throw new ConfigException.BadPath(variable, "Environment variable contains an un-mapped number of underscores."); + } + + return builder.toString(); + } + /** * Guess configuration syntax from given filename. * diff --git a/config/src/test/scala/com/typesafe/config/impl/ConfigTest.scala b/config/src/test/scala/com/typesafe/config/impl/ConfigTest.scala index 853aa249f..eeda74e70 100644 --- a/config/src/test/scala/com/typesafe/config/impl/ConfigTest.scala +++ b/config/src/test/scala/com/typesafe/config/impl/ConfigTest.scala @@ -1090,6 +1090,22 @@ class ConfigTest extends TestUtils { assertEquals(10, resolved.getInt("bar.nested.a.q")) } + @Test + def testEnvVariablesNameMangling() { + assertEquals("a", ConfigImplUtil.envVariableAsProperty("prefix_a", "prefix_")) + assertEquals("a.b", ConfigImplUtil.envVariableAsProperty("prefix_a_b", "prefix_")) + assertEquals("a.b.c", ConfigImplUtil.envVariableAsProperty("prefix_a_b_c", "prefix_")) + assertEquals("a.b-c-d", ConfigImplUtil.envVariableAsProperty("prefix_a_b__c__d", "prefix_")) + assertEquals("a.b_c_d", ConfigImplUtil.envVariableAsProperty("prefix_a_b___c___d", "prefix_")) + + intercept[ConfigException.BadPath] { + ConfigImplUtil.envVariableAsProperty("prefix_____", "prefix_") + } + intercept[ConfigException.BadPath] { + ConfigImplUtil.envVariableAsProperty("prefix_a_b___c____d", "prefix_") + } + } + @Test def testLoadWithEnvSubstitutions() { System.setProperty("config.override_with_env_vars", "true") From fd0e4188af80900993db80dd3b14d379b8e9a937 Mon Sep 17 00:00:00 2001 From: Andrea Peruffo Date: Thu, 21 Mar 2019 10:53:10 +0000 Subject: [PATCH 12/13] minor fix --- config/checkstyle-config.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/checkstyle-config.xml b/config/checkstyle-config.xml index 064ac59a5..ae2259fa7 100644 --- a/config/checkstyle-config.xml +++ b/config/checkstyle-config.xml @@ -1,6 +1,6 @@ From 4fad11380b99be534e00270bd18b03a1c4b24afd Mon Sep 17 00:00:00 2001 From: Andrea Peruffo Date: Thu, 21 Mar 2019 10:54:31 +0000 Subject: [PATCH 13/13] fix again XML --- config/checkstyle-config.xml | 2 +- config/checkstyle-suppressions.xml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/config/checkstyle-config.xml b/config/checkstyle-config.xml index ae2259fa7..148167f31 100644 --- a/config/checkstyle-config.xml +++ b/config/checkstyle-config.xml @@ -1,5 +1,5 @@ - diff --git a/config/checkstyle-suppressions.xml b/config/checkstyle-suppressions.xml index ca3fb2d15..300751899 100644 --- a/config/checkstyle-suppressions.xml +++ b/config/checkstyle-suppressions.xml @@ -1,5 +1,5 @@ -