Optional TOML configs support (#2457)

See the added docs for usage.

doc/configuration.rst

@@ -1,18 +1,35 @@
 Configuration
 =============
 
-All configuration can be done by adding configuration files. They are looked for in:
+All configuration can be done by adding configuration files.
 
- * ``/etc/luigi/client.cfg``
- * ``luigi.cfg`` (or its legacy name ``client.cfg``) in your current working directory
- * ``LUIGI_CONFIG_PATH`` environment variable
+Supported config parsers:
+* ``cfg`` (default)
+* ``toml``
 
-in increasing order of preference. The order only matters in case of key conflicts (see docs for ConfigParser.read_). These files are meant for both the client and ``luigid``. If you decide to specify your own configuration you should make sure that both the client and ``luigid`` load it properly.
+You can choose right parser via ``LUIGI_CONFIG_PARSER`` environment variable. For example, ``LUIGI_CONFIG_PARSER=toml``.
+
+Default (cfg) parser are looked for in:
+
+* ``/etc/luigi/client.cfg`` (deprecated)
+* ``/etc/luigi/luigi.cfg``
+* ``client.cfg`` (deprecated)
+* ``luigi.cfg``
+* ``LUIGI_CONFIG_PATH`` environment variable
+
+`TOML <https://github.com/toml-lang/toml>`_ parser are looked for in:
+
+* ``/etc/luigi/luigi.toml``
+* ``luigi.toml``
+* ``LUIGI_CONFIG_PATH`` environment variable
+
+Both config lists increase in priority (from low to high). The order only matters in case of key conflicts (see docs for ConfigParser.read_). These files are meant for both the client and ``luigid``. If you decide to specify your own configuration you should make sure that both the client and ``luigid`` load it properly.
 
 .. _ConfigParser.read: https://docs.python.org/3.6/library/configparser.html#configparser.ConfigParser.read
 
-The config file is broken into sections, each controlling a different part of the config. Example configuration file:
+The config file is broken into sections, each controlling a different part of the config.
 
+Example cfg config:
 
 .. code:: ini
 
@@ -23,6 +40,17 @@ The config file is broken into sections, each controlling a different part of th
  [core]
  scheduler_host=luigi-host.mycompany.foo
 
+Example toml config:
+
+.. code:: python
+
+ [hadoop]
+ version = "cdh4"
+ streaming-jar = "/usr/lib/hadoop-xyz/hadoop-streaming-xyz-123.jar"
+
+ [core]
+ scheduler_host = "luigi-host.mycompany.foo"
+
 
 .. _ParamConfigIngestion:
 

luigi/configuration/__init__.py

@@ -0,0 +1,27 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright 2012-2015 Spotify AB
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+from .cfg_parser import LuigiConfigParser
+from .core import get_config, add_config_path
+from .toml_parser import LuigiTomlParser
+
+
+__all__ = [
+ 'add_config_path',
+ 'get_config',
+ 'LuigiConfigParser',
+ 'LuigiTomlParser',
+]

luigi/configuration/base_parser.py

@@ -0,0 +1,41 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright 2012-2015 Spotify AB
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+import logging
+
+
+# IMPORTANT: don't inherit from `object`!
+# ConfigParser have some troubles in this case.
+# More info: https://stackoverflow.com/a/19323238
+class BaseParser:
+ @classmethod
+ def instance(cls, *args, **kwargs):
+ """ Singleton getter """
+ if cls._instance is None:
+ cls._instance = cls(*args, **kwargs)
+ loaded = cls._instance.reload()
+ logging.getLogger('luigi-interface').info('Loaded %r', loaded)
+
+ return cls._instance
+
+ @classmethod
+ def add_config_path(cls, path):
+ cls._config_paths.append(path)
+ cls.reload()
+
+ @classmethod
+ def reload(cls):
+ return cls.instance().read(cls._config_paths)

luigi/configuration.py → luigi/configuration/cfg_parser.py

@@ -29,7 +29,6 @@
 See :doc:`/configuration` for more info.
 """
 
-import logging
 import os
 import warnings
 
@@ -38,37 +37,19 @@
 except ImportError:
  from configparser import ConfigParser, NoOptionError, NoSectionError
 
+from .base_parser import BaseParser
 
-class LuigiConfigParser(ConfigParser):
+
+class LuigiConfigParser(BaseParser, ConfigParser):
  NO_DEFAULT = object()
+ enabled = True
  _instance = None
  _config_paths = [
  '/etc/luigi/client.cfg', # Deprecated old-style global luigi config
  '/etc/luigi/luigi.cfg',
  'client.cfg', # Deprecated old-style local luigi config
  'luigi.cfg',
  ]
- if 'LUIGI_CONFIG_PATH' in os.environ:
- config_file = os.environ['LUIGI_CONFIG_PATH']
- if not os.path.isfile(config_file):
- warnings.warn("LUIGI_CONFIG_PATH points to a file which does not exist. Invalid file: {path}".format(path=config_file))
- else:
- _config_paths.append(config_file)
-
- @classmethod
- def add_config_path(cls, path):
- cls._config_paths.append(path)
- cls.reload()
-
- @classmethod
- def instance(cls, *args, **kwargs):
- """ Singleton getter """
- if cls._instance is None:
- cls._instance = cls(*args, **kwargs)
- loaded = cls._instance.reload()
- logging.getLogger('luigi-interface').info('Loaded %r', loaded)
-
- return cls._instance
 
  @classmethod
  def reload(cls):
@@ -124,10 +105,3 @@ def set(self, section, option, value=None):
  ConfigParser.add_section(self, section)
 
  return ConfigParser.set(self, section, option, value)
-
-
-def get_config():
- """
- Convenience method (for backwards compatibility) for accessing config singleton.
- """
- return LuigiConfigParser.instance()

luigi/configuration/core.py

@@ -0,0 +1,79 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright 2012-2015 Spotify AB
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+import logging
+import os
+import warnings
+
+from .cfg_parser import LuigiConfigParser
+from .toml_parser import LuigiTomlParser
+
+
+logger = logging.getLogger('luigi-interface')
+
+
+PARSERS = {
+ 'cfg': LuigiConfigParser,
+ 'conf': LuigiConfigParser,
+ 'ini': LuigiConfigParser,
+ 'toml': LuigiTomlParser,
+}
+
+# select parser via env var
+DEFAULT_PARSER = 'cfg'
+PARSER = os.environ.get('LUIGI_CONFIG_PARSER', DEFAULT_PARSER)
+if PARSER not in PARSERS:
+ warnings.warn("Invalid parser: {parser}".format(parser=PARSER))
+ PARSER = DEFAULT_PARSER
+
+
+def get_config(parser=PARSER):
+ """Get configs singleton for parser
+ """
+
+ parser_class = PARSERS[parser]
+ if not parser_class.enabled:
+ logger.error((
+ "Parser not installed yet. "
+ "Please, install luigi with required parser:\n"
+ "pip install luigi[{parser}]"
+ ).format(parser)
+ )
+
+ return parser_class.instance()
+
+
+def add_config_path(path):
+ """Select config parser by file extension and add path into parser.
+ """
+ if not os.path.isfile(path):
+ warnings.warn("Config file does not exist: {path}".format(path=path))
+ return False
+
+ # select parser by file extension
+ _base, ext = os.path.splitext(path)
+ if ext and ext[1:] in PARSERS:
+ parser_class = PARSERS[ext[1:]]
+ else:
+ parser_class = PARSERS[PARSER]
+
+ # add config path to parser
+ parser_class.add_config_path(path)
+ return True
+
+
+if 'LUIGI_CONFIG_PATH' in os.environ:
+ add_config_path(os.environ['LUIGI_CONFIG_PATH'])

luigi/configuration/toml_parser.py

@@ -0,0 +1,82 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright 2018 Cindicator Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+import os.path
+
+try:
+ import toml
+except ImportError:
+ toml = False
+
+from .base_parser import BaseParser
+
+
+class LuigiTomlParser(BaseParser):
+ NO_DEFAULT = object()
+ enabled = bool(toml)
+ data = dict()
+ _instance = None
+ _config_paths = [
+ '/etc/luigi/luigi.toml',
+ 'luigi.toml',
+ ]
+
+ @staticmethod
+ def _update_data(data, new_data):
+ if not new_data:
+ return data
+ if not data:
+ return new_data
+ for section, content in new_data.items():
+ if section not in data:
+ data[section] = dict()
+ data[section].update(content)
+ return data
+
+ def read(self, config_paths):
+ self.data = dict()
+ for path in config_paths:
+ if os.path.isfile(path):
+ self.data = self._update_data(self.data, toml.load(path))
+ return self.data
+
+ def get(self, section, option, default=NO_DEFAULT, **kwargs):
+ try:
+ return self.data[section][option]
+ except KeyError:
+ if default is self.NO_DEFAULT:
+ raise
+ return default
+
+ def getboolean(self, section, option, default=NO_DEFAULT):
+ return self.get(section, option, default)
+
+ def getint(self, section, option, default=NO_DEFAULT):
+ return self.get(section, option, default)
+
+ def getfloat(self, section, option, default=NO_DEFAULT):
+ return self.get(section, option, default)
+
+ def getintdict(self, section):
+ return self.data.get(section, {})
+
+ def set(self, section, option, value=None):
+ if section not in self.data:
+ self.data[section] = {}
+ self.data[section][option] = value
+
+ def __getitem__(self, name):
+ return self.data[name]

luigi/contrib/s3.py

@@ -486,7 +486,7 @@ def _get_s3_config(self, key=None):
  defaults = dict(configuration.get_config().defaults())
  try:
  config = dict(configuration.get_config().items('s3'))
- except NoSectionError:
+ except (NoSectionError, KeyError):
  return {}
  # So what ports etc can be read without us having to specify all dtypes
  for k, v in six.iteritems(config):

luigi/parameter.py

@@ -168,7 +168,7 @@ def _get_value_from_config(self, section, name):
 
  try:
  value = conf.get(section, name)
- except (NoSectionError, NoOptionError):
+ except (NoSectionError, NoOptionError, KeyError):
  return _no_value
 
  return self.parse(value)

setup.py

@@ -58,6 +58,7 @@ def get_static_files(path):
  license='Apache License 2.0',
  packages=[
  'luigi',
+ 'luigi.configuration',
  'luigi.contrib',
  'luigi.contrib.hdfs',
  'luigi.tools'
@@ -75,6 +76,9 @@ def get_static_files(path):
  ]
  },
  install_requires=install_requires,
+ extras_require={
+ 'toml': ['toml<2.0.0'],
+ },
  classifiers=[
  'Development Status :: 5 - Production/Stable',
  'Environment :: Console',