Merge pull request #620 from andreaTP/envConfigLoading

Add a config.strategy to enable overwrite of properties from env vars

README.md

@@ -675,6 +675,24 @@ value just disappear if the substitution is not found:
     // this array could have one or two elements
     path = [ "a", ${?OPTIONAL_A} ]
 
+By setting the JVM property `-Dconfig.override_with_env_vars=true`
+it is possible to override any configuration value using environment
+variables even if an explicit substitution is not specified.
+
+The environment variable value will override any pre-existing value
+and also any value provided as Java property.
+
+With this option enabled only environment variables starting with
+`CONFIG_FORCE_` are considered, and the name is mangled as follows:
+
+  - the prefix `CONFIG_FORCE_` is stripped
+  - single underscore(`_`) is converted into a dot(`.`)
+  - double underscore(`__`) is converted into a dash(`-`)
+  - triple underscore(`___`) is converted into a single underscore(`_`)
+
+i.e. The environment variable `CONFIG_FORCE_a_b__c___d` set the
+configuration key `a.b-c_d`
+
 ### Concatenation
 
 Values _on the same line_ are concatenated (for strings and

build.sbt

@@ -85,7 +85,17 @@ lazy val configLib =  Project("config", file("config"))
     Test/ run / fork                       := true
 
     //env vars for tests
-    Test / envVars                         ++= Map("testList.0" -> "0", "testList.1" -> "1")
+    Test / envVars                         ++= Map("testList.0" -> "0",
+      "testList.1" -> "1",
+      "CONFIG_FORCE_b" -> "5",
+      "CONFIG_FORCE_testList_0" -> "10",
+      "CONFIG_FORCE_testList_1" -> "11",
+      "CONFIG_FORCE_42___a" -> "1",
+      "CONFIG_FORCE_a_b_c" -> "2",
+      "CONFIG_FORCE_a__c" -> "3",
+      "CONFIG_FORCE_a___c" -> "4",
+      "CONFIG_FORCE_akka_version" -> "foo",
+      "CONFIG_FORCE_akka_event__handler__dispatcher_max__pool__size" -> "10")
 
     OsgiKeys.exportPackage                 := Seq("com.typesafe.config", "com.typesafe.config.impl")
     publish                                := sys.error("use publishSigned instead of plain publish")

config/checkstyle-config.xml

@@ -1,8 +1,7 @@
 <?xml version="1.0" encoding="UTF-8"?>
 <!DOCTYPE module PUBLIC
-    "-//Puppy Crawl//DTD Check Configuration 1.3//EN"
-    "http://www.puppycrawl.com/dtds/configuration_1_3.dtd">
-
+    "-//Checkstyle//DTD Check Configuration 1.3//EN"
+    "https://checkstyle.org/dtds/configuration_1_3.dtd">
 <module name="Checker">
   <module name="SuppressionFilter">
     <property name="file" value="config/checkstyle-suppressions.xml"/>

config/checkstyle-suppressions.xml

@@ -1,6 +1,7 @@
 <?xml version="1.0" encoding="UTF-8"?>
-<!DOCTYPE suppressions PUBLIC "-//Puppy Crawl//DTD Suppressions 1.1//EN"
-    "http://www.puppycrawl.com/dtds/suppressions_1_1.dtd">
+<!DOCTYPE suppressions PUBLIC
+    "-//Checkstyle//DTD Suppressions 1.1//EN"
+    "https://checkstyle.org/dtds/suppressions_1_1.dtd">
 <suppressions>
   <!-- don't care about javadoc coverage of methods in impl -->
   <suppress checks="JavadocMethod"

config/src/main/java/com/typesafe/config/ConfigFactory.java

@@ -36,6 +36,7 @@
  */
 public final class ConfigFactory {
     private static final String STRATEGY_PROPERTY_NAME = "config.strategy";
+    private static final String OVERRIDE_WITH_ENV_PROPERTY_NAME = "config.override_with_env_vars";
 
     private ConfigFactory() {
     }
@@ -383,7 +384,11 @@ public static Config defaultReference(ClassLoader loader) {
      * @return the default override configuration
      */
     public static Config defaultOverrides() {
-        return systemProperties();
+        if (getOverrideWithEnv()) {
+            return systemEnvironmentOverrides().withFallback(systemProperties());
+        } else {
+            return systemProperties();
+        }
     }
 
     /**
@@ -394,7 +399,7 @@ public static Config defaultOverrides() {
      * @return the default override configuration
      */
     public static Config defaultOverrides(ClassLoader loader) {
-        return systemProperties();
+        return defaultOverrides();
     }
 
     /**
@@ -497,6 +502,7 @@ public static void invalidateCaches() {
         // all caches
         ConfigImpl.reloadSystemPropertiesConfig();
         ConfigImpl.reloadEnvVariablesConfig();
+        ConfigImpl.reloadEnvVariablesOverridesConfig();
     }
 
     /**
@@ -549,6 +555,50 @@ public static Config systemProperties() {
         return ConfigImpl.systemPropertiesAsConfig();
     }
 
+    /**
+     * Gets a <code>Config</code> containing the system's environment variables
+     * used to override configuration keys.
+     * Environment variables taken in considerations are starting with
+     * {@code CONFIG_FORCE_}
+     * 
+     * <p>
+     * Environment variables are mangled in the following way after stripping the prefix "CONFIG_FORCE_":
+     * <table border="1">
+     * <tr>
+     *     <th bgcolor="silver">Env Var</th>
+     *     <th bgcolor="silver">Config</th>
+     * </tr>
+     * <tr>
+     *     <td>_&nbsp;&nbsp;&nbsp;[1 underscore]</td>
+     *     <td>. [dot]</td>
+     * </tr>
+     * <tr>
+     *     <td>__&nbsp;&nbsp;[2 underscore]</td>
+     *     <td>- [dash]</td>
+     *  </tr>
+     * <tr>
+     *     <td>___&nbsp;[3 underscore]</td>
+     *     <td>_ [underscore]</td>
+     * </tr>
+     * </table>
+     * 
+     * <p>
+     * A variable like: {@code CONFIG_FORCE_a_b__c___d}
+     * is translated to a config key: {@code a.b-c_d}
+     * 
+     * <p>
+     * This method can return a global immutable singleton, so it's preferred
+     * over parsing system properties yourself.
+     * <p>
+     * {@link #defaultOverrides} will include the system environment variables as
+     * overrides if `config.override_with_env_vars` is set to `true`.
+     *
+     * @return system environment variable overrides parsed into a <code>Config</code>
+     */
+    public static Config systemEnvironmentOverrides() {
+        return ConfigImpl.envVariablesOverridesAsConfig();
+    }
+
     /**
      * Gets a <code>Config</code> containing the system's environment variables.
      * This method can return a global immutable singleton.
@@ -1063,4 +1113,10 @@ private static ConfigLoadingStrategy getConfigLoadingStrategy() {
             return new DefaultConfigLoadingStrategy();
         }
     }
+
+    private static Boolean getOverrideWithEnv() {
+        String overrideWithEnv = System.getProperties().getProperty(OVERRIDE_WITH_ENV_PROPERTY_NAME);
+
+        return Boolean.parseBoolean(overrideWithEnv);
+    }
 }

config/src/main/java/com/typesafe/config/impl/ConfigImpl.java

@@ -32,6 +32,7 @@
  * For use only by the {@link com.typesafe.config} package.
  */
 public class ConfigImpl {
+    private static final String ENV_VAR_OVERRIDE_PREFIX = "CONFIG_FORCE_";
 
     private static class LoaderCache {
         private Config currentSystemProperties;
@@ -360,6 +361,43 @@ public static void reloadEnvVariablesConfig() {
         EnvVariablesHolder.envVariables = loadEnvVariables();
     }
 
+
+
+    private static AbstractConfigObject loadEnvVariablesOverrides() {
+        Map<String, String> env = new HashMap(System.getenv());
+        Map<String, String> result = new HashMap(System.getenv());
+
+        for (String key : env.keySet()) {
+            if (key.startsWith(ENV_VAR_OVERRIDE_PREFIX)) {
+                result.put(ConfigImplUtil.envVariableAsProperty(key, ENV_VAR_OVERRIDE_PREFIX), env.get(key));
+            }
+        }
+
+        return PropertiesParser.fromStringMap(newSimpleOrigin("env variables overrides"), result);
+    }
+
+    private static class EnvVariablesOverridesHolder {
+        static volatile AbstractConfigObject envVariables = loadEnvVariablesOverrides();
+    }
+
+    static AbstractConfigObject envVariablesOverridesAsConfigObject() {
+        try {
+            return EnvVariablesOverridesHolder.envVariables;
+        } catch (ExceptionInInitializerError e) {
+            throw ConfigImplUtil.extractInitializerError(e);
+        }
+    }
+
+    public static Config envVariablesOverridesAsConfig() {
+        return envVariablesOverridesAsConfigObject().toConfig();
+    }
+
+    public static void reloadEnvVariablesOverridesConfig() {
+        // ConfigFactory.invalidateCaches() relies on this having the side
+        // effect that it drops all caches
+        EnvVariablesOverridesHolder.envVariables = loadEnvVariablesOverrides();
+    }
+
     public static Config defaultReference(final ClassLoader loader) {
         return computeCachedConfig(loader, "defaultReference", new Callable<Config>() {
             @Override

config/src/main/java/com/typesafe/config/impl/ConfigImplUtil.java

@@ -235,6 +235,52 @@ static String toCamelCase(String originalName) {
         return nameBuilder.toString();
     }
 
+    private static char underscoreMappings(int num) {
+        // Rationale on name mangling:
+        //
+        // Most shells (e.g. bash, sh, etc.) doesn't support any character other
+        // than alphanumeric and `_` in environment variables names.
+        // In HOCON the default separator is `.` so it is directly translated to a
+        // single `_` for convenience; `-` and `_` are less often present in config
+        // keys but they have to be representable and the only possible mapping is
+        // `_` repeated.
+        switch (num) {
+            case 1: return '.';
+            case 2: return '-';
+            case 3: return '_';
+            default: return 0;
+        }
+    }
+
+    static String envVariableAsProperty(String variable, String prefix) throws ConfigException {
+        StringBuilder builder = new StringBuilder();
+
+        String strippedPrefix = variable.substring(prefix.length(), variable.length());
+
+        int underscores = 0;
+        for (char c : strippedPrefix.toCharArray()) {
+            if (c == '_') {
+                underscores++;
+            } else {
+                if (underscores > 0  && underscores < 4) {
+                    builder.append(underscoreMappings(underscores));
+                } else if (underscores > 3) {
+                    throw new ConfigException.BadPath(variable, "Environment variable contains an un-mapped number of underscores.");
+                }
+                underscores = 0;
+                builder.append(c);
+            }
+        }
+
+        if (underscores > 0  && underscores < 4) {
+            builder.append(underscoreMappings(underscores));
+        } else if (underscores > 3) {
+            throw new ConfigException.BadPath(variable, "Environment variable contains an un-mapped number of underscores.");
+        }
+
+        return builder.toString();
+    }
+
     /**
      * Guess configuration syntax from given filename.
      *

config/src/test/scala/com/typesafe/config/impl/ConfigTest.scala

@@ -1090,6 +1090,53 @@ class ConfigTest extends TestUtils {
         assertEquals(10, resolved.getInt("bar.nested.a.q"))
     }
 
+    @Test
+    def testEnvVariablesNameMangling() {
+        assertEquals("a", ConfigImplUtil.envVariableAsProperty("prefix_a", "prefix_"))
+        assertEquals("a.b", ConfigImplUtil.envVariableAsProperty("prefix_a_b", "prefix_"))
+        assertEquals("a.b.c", ConfigImplUtil.envVariableAsProperty("prefix_a_b_c", "prefix_"))
+        assertEquals("a.b-c-d", ConfigImplUtil.envVariableAsProperty("prefix_a_b__c__d", "prefix_"))
+        assertEquals("a.b_c_d", ConfigImplUtil.envVariableAsProperty("prefix_a_b___c___d", "prefix_"))
+
+        intercept[ConfigException.BadPath] {
+            ConfigImplUtil.envVariableAsProperty("prefix_____", "prefix_")
+        }
+        intercept[ConfigException.BadPath] {
+            ConfigImplUtil.envVariableAsProperty("prefix_a_b___c____d", "prefix_")
+        }
+    }
+
+    @Test
+    def testLoadWithEnvSubstitutions() {
+        System.setProperty("config.override_with_env_vars", "true")
+
+        try {
+            val loader02 = new TestClassLoader(this.getClass().getClassLoader(),
+                Map("reference.conf" -> resourceFile("test02.conf").toURI.toURL()))
+
+            val loader04 = new TestClassLoader(this.getClass().getClassLoader(),
+                Map("reference.conf" -> resourceFile("test04.conf").toURI.toURL()))
+
+            val conf02 = withContextClassLoader(loader02) {
+                ConfigFactory.load()
+            }
+
+            val conf04 = withContextClassLoader(loader04) {
+                ConfigFactory.load()
+            }
+
+            assertEquals(1, conf02.getInt("42_a"))
+            assertEquals(2, conf02.getInt("a.b.c"))
+            assertEquals(3, conf02.getInt("a-c"))
+            assertEquals(4, conf02.getInt("a_c"))
+
+            assertEquals("foo", conf04.getString("akka.version"))
+            assertEquals(10, conf04.getInt("akka.event-handler-dispatcher.max-pool-size"))
+        } finally {
+            System.clearProperty("config.override_with_env_vars")
+        }
+    }
+
     @Test
     def renderRoundTrip() {
         val allBooleans = true :: false :: Nil

config/src/test/scala/com/typesafe/config/impl/PublicApiTest.scala

@@ -653,6 +653,26 @@ class PublicApiTest extends TestUtils {
         }
     }
 
+    @Test
+    def loadEnvironmentVariablesOverridesIfConfigured(): Unit = {
+        assertEquals("config.override_with_env_vars is not set", null, System.getProperty("config.override_with_env_vars"))
+
+        System.setProperty("config.override_with_env_vars", "true")
+
+        try {
+            val loaderB2 = new TestClassLoader(this.getClass().getClassLoader(),
+                Map("reference.conf" -> resourceFile("b_2.conf").toURI.toURL()))
+
+            val configB2 = withContextClassLoader(loaderB2) {
+                ConfigFactory.load()
+            }
+
+            assertEquals(5, configB2.getInt("b"))
+        } finally {
+            System.clearProperty("config.override_with_env_vars")
+        }
+    }
+
     @Test
     def usesContextClassLoaderForApplicationConf() {
         val loaderA1 = new TestClassLoader(this.getClass().getClassLoader(),
@@ -1145,4 +1165,4 @@ object TestStrategy {
     private var invocations = 0
     def getIncovations() = invocations
     def increment() = invocations += 1
-}
+}

project/build.properties

@@ -1 +1 @@
-sbt.version=1.2.7
+sbt.version=1.2.8