-
Notifications
You must be signed in to change notification settings - Fork 5.3k
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Add support for environment variables with backoff values in fig.yml
This change resolves #495 by introducing a Config class inside the cli library. The Config class is responsible for loading a .yml file and evaluating its contents for environment variables. The Config immediately reloads the data as a dictionary, which reduces compatibility issues within previously written code. This behavior also prevents race conditions caused by changes to the environment during long-running fig builds. This change is valuable because it allows for a greater degree of automation and variability when working with multiple environments or deployment types that use identically built containers. We currently have to define separate services for each degree of variability we want to account for. Future steps for this change may include introducing template files, or adding access and manipulation APIs directly so that we can perpetuate the config class everywhere we are currently using bare dictionaries. Added in revision requests from @dnephin: * Terminology fix in docs/yml.md * Refactored config class into a series of functions * Updated tests to reflect functional interpolation * Improved forward compatibility Added in revision requests from @thaJeztah * Terminology fixes Added in revision requests from @aanand * Support for escaping Developer's Certificate of Origin 1.1 By making a contribution to this project, I certify that: (a) The contribution was created in whole or in part by me and I have the right to submit it under the open source license indicated in the file; or (b) The contribution is based upon previous work that, to the best of my knowledge, is covered under an appropriate open source license and I have the right under that license to submit that work with modifications, whether created in whole or in part by me, under the same open source license (unless I am permitted to submit under a different license), as indicated in the file; or (c) The contribution was provided directly to me by some other person who certified (a), (b) or (c) and I have not modified it. (d) I understand and agree that this project and the contribution are public and that a record of the contribution (including all personal information I submit with it, including my sign-off) is maintained indefinitely and may be redistributed consistent with this project or the open source license(s) involved. Signed-off-by: Robert Elwell <robert.elwell@gmail.com> Sponsored by: Lookout, Inc.
- Loading branch information
Robert Elwell
committed
Jan 15, 2015
1 parent
72095f5
commit ab81a7a
Showing
4 changed files
with
234 additions
and
9 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,73 @@ | ||
| from __future__ import unicode_literals | ||
| from __future__ import absolute_import | ||
|
|
||
| import six | ||
| import errno | ||
| import yaml | ||
| import os | ||
| import re | ||
|
|
||
| from . import errors | ||
|
|
||
|
|
||
| def resolve_environment_vars(value): | ||
| """ | ||
| Matches our environment variable pattern, replaces the value with the value in the env | ||
| A future improvement here would be to also reference a separate dictionary that keeps track of | ||
| configurable variables via flat file. | ||
| :param value: any value that is reachable by a key in a dict represented by a yaml file | ||
| :return: the value itself if not a string with an environment variable, otherwise the value specified in the env | ||
| """ | ||
| if not isinstance(value, six.string_types): | ||
| return value | ||
|
|
||
| env_regex = r'([^\\])?\$\{(?P<env_var>[^\}^:]+)(:(?P<default_val>[^\}]+))?\}' | ||
|
|
||
| match_object = re.match(env_regex, value) | ||
| if match_object is None: | ||
| return value | ||
|
|
||
| result = os.environ.get(match_object.group('env_var'), match_object.group('default_val')) | ||
|
|
||
| if result is None: | ||
| raise errors.UserError("No value for ${%s} found in environment." % (match_object.group('env_var')) | ||
| + "Please set a value in the environment or provide a default.") | ||
|
|
||
| return re.sub(env_regex, result, value) | ||
|
|
||
|
|
||
| def with_environment_vars(value): | ||
| """ | ||
| Recursively interpolates environment variables for a structured or unstructured value | ||
| :param value: a dict, list, or any other kind of value | ||
| :return: the dict with its values interpolated from the env | ||
| :rtype: dict | ||
| """ | ||
| if type(value) == dict: | ||
| return dict([(subkey, with_environment_vars(subvalue)) | ||
| for subkey, subvalue in value.items()]) | ||
| elif type(value) == list: | ||
| return [resolve_environment_vars(x) for x in value] | ||
| else: | ||
| return resolve_environment_vars(value) | ||
|
|
||
|
|
||
| def from_yaml_with_environment_vars(yaml_filename): | ||
| """ | ||
| Resolves enviornment variables in values defined by a YAML file and transformed into a dict | ||
| :param yaml_filename: the name of the yaml file | ||
| :type yaml_filename: str | ||
| :return: a dict with environment variables properly interpolated | ||
| """ | ||
| try: | ||
| with open(yaml_filename, 'r') as fh: | ||
| return with_environment_vars(yaml.safe_load(fh)) | ||
| except IOError as e: | ||
| if e.errno == errno.ENOENT: | ||
| raise errors.FigFileNotFound(os.path.basename(e.filename)) | ||
| raise errors.UserError(six.text_type(e)) |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,126 @@ | ||
| from __future__ import unicode_literals | ||
| from __future__ import absolute_import | ||
| from tests import unittest | ||
|
|
||
| from fig.cli import config | ||
| from fig.cli.errors import UserError | ||
|
|
||
| import yaml | ||
| import os | ||
|
|
||
|
|
||
| TEST_YML_DICT_WITH_DEFAULTS = yaml.load("""db: | ||
| image: postgres | ||
| web: | ||
| build: ${DJANGO_BUILD:.} | ||
| command: python manage.py runserver 0.0.0.0:8000 | ||
| volumes: | ||
| - .:/code | ||
| environment: | ||
| DJANGO_ENV: ${TEST_VALUE:production} | ||
| ports: | ||
| - ${DJANGO_PORT:"8000:8000"} | ||
| links: | ||
| - db""") | ||
|
|
||
| TEST_YML_DICT_NO_DEFAULTS = yaml.load("""db: | ||
| image: postgres | ||
| web: | ||
| build: ${TEST_VALUE} | ||
| command: python manage.py runserver 0.0.0.0:8000 | ||
| volumes: | ||
| - .:/code | ||
| ports: | ||
| - "8000:8000" | ||
| links: | ||
| - db""") | ||
|
|
||
|
|
||
| class ConfigTestCase(unittest.TestCase): | ||
|
|
||
| def setUp(self): | ||
| self.former_test_value = os.environ.get('TEST_VALUE') | ||
|
|
||
| def tearDown(self): | ||
| if self.former_test_value is not None: | ||
| os.environ['TEST_VALUE'] = self.former_test_value | ||
| elif 'TEST_VALUE' in os.environ: | ||
| del os.environ['TEST_VALUE'] | ||
|
|
||
| def test_with_resolve_environment_vars_nonmatch_values(self): | ||
| # It should just return non-string values | ||
| self.assertEqual(1, config.resolve_environment_vars(1)) | ||
| self.assertEqual([], config.resolve_environment_vars([])) | ||
| self.assertEqual({}, config.resolve_environment_vars({})) | ||
| self.assertEqual(1.234, config.resolve_environment_vars(1.234)) | ||
| self.assertEqual(None, config.resolve_environment_vars(None)) | ||
|
|
||
| # Any string that doesn't match our regex should just be returned | ||
| self.assertEqual('localhost', config.resolve_environment_vars('localhost')) | ||
|
|
||
| expected = "some-other-host" | ||
| os.environ['TEST_VALUE'] = expected | ||
|
|
||
| # Bare mentions of the environment variable shouldn't work | ||
| value = 'TEST_VALUE:foo' | ||
| self.assertEqual(value, config.resolve_environment_vars(value)) | ||
|
|
||
| value = 'TEST_VALUE' | ||
| self.assertEqual(value, config.resolve_environment_vars(value)) | ||
|
|
||
| # Incomplete pattern shouldn't work as well | ||
| for value in ['${TEST_VALUE', '$TEST_VALUE', '{TEST_VALUE}']: | ||
| self.assertEqual(value, config.resolve_environment_vars(value)) | ||
| value += ':foo' | ||
| self.assertEqual(value, config.resolve_environment_vars(value)) | ||
|
|
||
| def test_fully_interpolated_matches(self): | ||
| expected = "some-other-host" | ||
| os.environ['TEST_VALUE'] = expected | ||
|
|
||
| # if we have a basic match | ||
| self.assertEqual(expected, config.resolve_environment_vars("${TEST_VALUE}")) | ||
|
|
||
| # if we have a match with a default value | ||
| self.assertEqual(expected, config.resolve_environment_vars("${TEST_VALUE:localhost}")) | ||
|
|
||
| # escaping should prevent interpolation | ||
| escaped_no_backoff = "\${TEST_VALUE}" | ||
| escaped_with_backoff = "\${TEST_VALUE:localhost}" | ||
| self.assertEqual(escaped_no_backoff, escaped_no_backoff) | ||
| self.assertEqual(escaped_with_backoff, escaped_with_backoff) | ||
|
|
||
| # if we have no match but a default value | ||
| del os.environ['TEST_VALUE'] | ||
| self.assertEqual('localhost', config.resolve_environment_vars("${TEST_VALUE:localhost}")) | ||
|
|
||
| def test_fully_interpolated_errors(self): | ||
| if 'TEST_VALUE' in os.environ: | ||
| del os.environ['TEST_VALUE'] | ||
| self.assertRaises(UserError, config.resolve_environment_vars, "${TEST_VALUE}") | ||
|
|
||
| def test_functional_defaults_as_dict(self): | ||
| d = config.with_environment_vars(TEST_YML_DICT_WITH_DEFAULTS) | ||
|
|
||
| # tests the basic structure and functionality of defaults | ||
| self.assertEqual(d['web']['build'], '.') | ||
|
|
||
| # test that environment variables with defaults are handled in lists | ||
| self.assertEqual(d['web']['ports'][0], '"8000:8000"') | ||
|
|
||
| # test that environment variables with defaults are handled in dictionaries | ||
| self.assertEqual(d['web']['environment']['DJANGO_ENV'], 'production') | ||
|
|
||
| # test that having an environment variable set properly pulls it | ||
| os.environ['TEST_VALUE'] = 'development' | ||
| d = config.with_environment_vars(TEST_YML_DICT_WITH_DEFAULTS) | ||
| self.assertEqual(d['web']['environment']['DJANGO_ENV'], 'development') | ||
|
|
||
| def test_functional_no_defaults(self): | ||
| # test that not having defaults raises an error in a real YML situation | ||
| self.assertRaises(UserError, config.with_environment_vars, TEST_YML_DICT_NO_DEFAULTS) | ||
|
|
||
| # test that a bare environment variable is interpolated | ||
| # note that we have to reload it | ||
| os.environ['TEST_VALUE'] = '/home/ubuntu/django' | ||
| self.assertEqual(config.with_environment_vars(TEST_YML_DICT_NO_DEFAULTS)['web']['build'], '/home/ubuntu/django') |