Interpolate environment variables

compose/config/__init__.py

@@ -0,0 +1,10 @@
+from .config import (
+    DOCKER_CONFIG_KEYS,
+    ConfigDetails,
+    ConfigurationError,
+    find,
+    load,
+    parse_environment,
+    merge_environment,
+    get_service_name_from_net,
+)  # flake8: noqa

compose/config.py → compose/config/config.py

@@ -8,6 +8,13 @@
 
 from compose.cli.utils import find_candidates_in_parent_dirs
 
+from .interpolation import interpolate_environment_variables
+from .errors import (
+    ConfigurationError,
+    CircularReference,
+    ComposeFileNotFound,
+)
+
 
 DOCKER_CONFIG_KEYS = [
     'cap_add',
@@ -126,11 +133,11 @@ def get_config_path(base_dir):
 
 def load(config_details):
     dictionary, working_dir, filename = config_details
+    dictionary = interpolate_environment_variables(dictionary)
+
     service_dicts = []
 
     for service_name, service_dict in list(dictionary.items()):
-        if not isinstance(service_dict, dict):
-            raise ConfigurationError('Service "%s" doesn\'t have any configuration options. All top level keys in your docker-compose.yml must map to a dictionary of configuration options.' % service_name)
         loader = ServiceLoader(working_dir=working_dir, filename=filename)
         service_dict = loader.make_service_dict(service_name, service_dict)
         validate_paths(service_dict)
@@ -423,9 +430,9 @@ def resolve_volume_paths(volumes, working_dir=None):
 
 def resolve_volume_path(volume, working_dir):
     container_path, host_path = split_path_mapping(volume)
-    container_path = os.path.expanduser(os.path.expandvars(container_path))
+    container_path = os.path.expanduser(container_path)
     if host_path is not None:
-        host_path = os.path.expanduser(os.path.expandvars(host_path))
+        host_path = os.path.expanduser(host_path)
         return "%s:%s" % (expand_path(working_dir, host_path), container_path)
     else:
         return container_path
@@ -536,33 +543,3 @@ def load_yaml(filename):
             return yaml.safe_load(fh)
     except IOError as e:
         raise ConfigurationError(six.text_type(e))
-
-
-class ConfigurationError(Exception):
-    def __init__(self, msg):
-        self.msg = msg
-
-    def __str__(self):
-        return self.msg
-
-
-class CircularReference(ConfigurationError):
-    def __init__(self, trail):
-        self.trail = trail
-
-    @property
-    def msg(self):
-        lines = [
-            "{} in {}".format(service_name, filename)
-            for (filename, service_name) in self.trail
-        ]
-        return "Circular reference:\n  {}".format("\n  extends ".join(lines))
-
-
-class ComposeFileNotFound(ConfigurationError):
-    def __init__(self, supported_filenames):
-        super(ComposeFileNotFound, self).__init__("""
-        Can't find a suitable configuration file in this directory or any parent. Are you in the right directory?
-
-        Supported filenames: %s
-        """ % ", ".join(supported_filenames))

compose/config/errors.py

@@ -0,0 +1,28 @@
+class ConfigurationError(Exception):
+    def __init__(self, msg):
+        self.msg = msg
+
+    def __str__(self):
+        return self.msg
+
+
+class CircularReference(ConfigurationError):
+    def __init__(self, trail):
+        self.trail = trail
+
+    @property
+    def msg(self):
+        lines = [
+            "{} in {}".format(service_name, filename)
+            for (filename, service_name) in self.trail
+        ]
+        return "Circular reference:\n  {}".format("\n  extends ".join(lines))
+
+
+class ComposeFileNotFound(ConfigurationError):
+    def __init__(self, supported_filenames):
+        super(ComposeFileNotFound, self).__init__("""
+        Can't find a suitable configuration file in this directory or any parent. Are you in the right directory?
+
+        Supported filenames: %s
+        """ % ", ".join(supported_filenames))

compose/config/interpolation.py

@@ -0,0 +1,86 @@
+import os
+from string import Template
+
+import six
+
+from .errors import ConfigurationError
+
+import logging
+log = logging.getLogger(__name__)
+
+
+def interpolate_environment_variables(config):
+    return dict(
+        (service_name, process_service(service_name, service_dict))
+        for (service_name, service_dict) in config.items()
+    )
+
+
+def process_service(service_name, service_dict):
+    if not isinstance(service_dict, dict):
+        raise ConfigurationError(
+            'Service "%s" doesn\'t have any configuration options. '
+            'All top level keys in your docker-compose.yml must map '
+            'to a dictionary of configuration options.' % service_name
+        )
+
+    return dict(
+        (key, interpolate_value(service_name, key, val))
+        for (key, val) in service_dict.items()
+    )
+
+
+def interpolate_value(service_name, config_key, value):
+    try:
+        return recursive_interpolate(value)
+    except InvalidInterpolation as e:
+        raise ConfigurationError(
+            'Invalid interpolation format for "{config_key}" option '
+            'in service "{service_name}": "{string}"'
+            .format(
+                config_key=config_key,
+                service_name=service_name,
+                string=e.string,
+            )
+        )
+
+
+def recursive_interpolate(obj):
+    if isinstance(obj, six.string_types):
+        return interpolate(obj, os.environ)
+    elif isinstance(obj, dict):
+        return dict(
+            (key, recursive_interpolate(val))
+            for (key, val) in obj.items()
+        )
+    elif isinstance(obj, list):
+        return map(recursive_interpolate, obj)
+    else:
+        return obj
+
+
+def interpolate(string, mapping):
+    try:
+        return Template(string).substitute(BlankDefaultDict(mapping))
+    except ValueError:
+        raise InvalidInterpolation(string)
+
+
+class BlankDefaultDict(dict):
+    def __init__(self, mapping):
+        super(BlankDefaultDict, self).__init__(mapping)
+
+    def __getitem__(self, key):
+        try:
+            return super(BlankDefaultDict, self).__getitem__(key)
+        except KeyError:
+            log.warn(
+                "The {} variable is not set. Substituting a blank string."
+                .format(key)
+            )
+            return ""
+
+
+class InvalidInterpolation(Exception):
+    def __init__(self, string):
+        self.string = string

docs/yml.md

@@ -19,6 +19,10 @@ As with `docker run`, options specified in the Dockerfile (e.g., `CMD`,
 `EXPOSE`, `VOLUME`, `ENV`) are respected by default - you don't need to
 specify them again in `docker-compose.yml`.
 
+Values for configuration options can contain environment variables, e.g. 
+`image: postgres:${POSTGRES_VERSION}`. For more details, see the section on
+[variable substitution](#variable-substitution).
+
 ### image
 
 Tag or partial image ID. Can be local or remote - Compose will attempt to
@@ -369,6 +373,33 @@ Each of these is a single value, analogous to its
     volume_driver: mydriver
 ```
 
+## Variable substitution
+
+Your configuration options can contain environment variables. Compose uses the
+variable values from the shell environment in which `docker-compose` is run. For
+example, suppose the shell contains `POSTGRES_VERSION=9.3` and you supply this
+configuration:
+
+    db:
+      image: "postgres:${POSTGRES_VERSION}"
+
+When you run `docker-compose up` with this configuration, Compose looks for the
+`POSTGRES_VERSION` environment variable in the shell and substitutes its value
+in. For this example, Compose resolves the `image` to `postgres:9.3` before
+running the configuration.
+
+If an environment variable is not set, Compose substitutes with an empty
+string. In the example above, if `POSTGRES_VERSION` is not set, the value for
+the `image` option is `postgres:`.
+
+Both `$VARIABLE` and `${VARIABLE}` syntax are supported. Extended shell-style
+features, such as `${VARIABLE-default}` and `${VARIABLE/foo/bar}`, are not
+supported.
+
+If you need to put a literal dollar sign in a configuration value, use a double
+dollar sign (`$$`).
+
+
 ## Compose documentation
 
 - [User guide](/)

tests/fixtures/environment-interpolation/docker-compose.yml

@@ -0,0 +1,17 @@
+web:
+  # unbracketed name
+  image: $IMAGE
+
+  # array element
+  ports:
+    - "${HOST_PORT}:8000"
+
+  # dictionary item value
+  labels:
+    mylabel: "${LABEL_VALUE}"
+
+  # unset value
+  hostname: "host-${UNSET_VALUE}"
+
+  # escaped interpolation
+  command: "$${ESCAPED}"

tests/fixtures/volume-path-interpolation/docker-compose.yml

@@ -0,0 +1,5 @@
+test:
+  image: busybox
+  command: top
+  volumes:
+    - "~/${VOLUME_NAME}:/container-path"

tests/integration/cli_test.py

@@ -488,6 +488,21 @@ def test_env_file_relative_to_compose_file(self):
         self.assertEqual(len(containers), 1)
         self.assertIn("FOO=1", containers[0].get('Config.Env'))
 
+    @patch.dict(os.environ)
+    def test_home_and_env_var_in_volume_path(self):
+        os.environ['VOLUME_NAME'] = 'my-volume'
+        os.environ['HOME'] = '/tmp/home-dir'
+        expected_host_path = os.path.join(os.environ['HOME'], os.environ['VOLUME_NAME'])
+
+        self.command.base_dir = 'tests/fixtures/volume-path-interpolation'
+        self.command.dispatch(['up', '-d'], None)
+
+        container = self.project.containers(stopped=True)[0]
+        actual_host_path = container.get('Volumes')['/container-path']
+        components = actual_host_path.split('/')
+        self.assertTrue(components[-2:] == ['home-dir', 'my-volume'],
+                        msg="Last two components differ: %s, %s" % (actual_host_path, expected_host_path))
+
     def test_up_with_extends(self):
         self.command.base_dir = 'tests/fixtures/extends'
         self.command.dispatch(['up', '-d'], None)

tests/integration/service_test.py

@@ -273,24 +273,6 @@ def test_duplicate_volume_trailing_slash(self):
 
         self.assertEqual(service.containers(stopped=False), [new_container])
 
-    @patch.dict(os.environ)
-    def test_create_container_with_home_and_env_var_in_volume_path(self):
-        os.environ['VOLUME_NAME'] = 'my-volume'
-        os.environ['HOME'] = '/tmp/home-dir'
-        expected_host_path = os.path.join(os.environ['HOME'], os.environ['VOLUME_NAME'])
-
-        host_path = '~/${VOLUME_NAME}'
-        container_path = '/container-path'
-
-        service = self.create_service('db', volumes=['%s:%s' % (host_path, container_path)])
-        container = service.create_container()
-        service.start_container(container)
-
-        actual_host_path = container.get('Volumes')[container_path]
-        components = actual_host_path.split('/')
-        self.assertTrue(components[-2:] == ['home-dir', 'my-volume'],
-                        msg="Last two components differ: %s, %s" % (actual_host_path, expected_host_path))
-
     def test_create_container_with_volumes_from(self):
         volume_service = self.create_service('data')
         volume_container_1 = volume_service.create_container()

tests/integration/testcases.py

@@ -1,7 +1,7 @@
 from __future__ import unicode_literals
 from __future__ import absolute_import
 from compose.service import Service
-from compose.config import ServiceLoader
+from compose.config.config import ServiceLoader
 from compose.const import LABEL_PROJECT
 from compose.cli.docker_client import docker_client
 from compose.progress_stream import stream_output