Add support for dynamic configurations

docs/source/config-options.md

@@ -4,10 +4,12 @@ Jupyter Enterprise Gateway adheres to the
 [Jupyter common configuration approach](https://jupyter.readthedocs.io/en/latest/projects/config.html)
 . You can configure an instance of Enterprise Gateway using:
 
-1. A configuration file
+1. A configuration file (recommended)
 2. Command line parameters
 3. Environment variables
 
+See [Dynamic Configurables](#dynamic-configurables) for additional information.
+
 To generate a template configuration file, run the following:
 
 ```
@@ -20,7 +22,7 @@ To see the same configuration options at the command line, run the following:
 jupyter enterprisegateway --help-all
 ```
 
-A snapshot of this help appears below for ease of reference on the web.
+A snapshot of this help appears below for ease of reference on the web.  
 
 ```
 Jupyter Enterprise Gateway
@@ -135,6 +137,11 @@ EnterpriseGatewayApp options
 --EnterpriseGatewayApp.default_kernel_name=<Unicode>
     Default: ''
     Default kernel name when spawning a kernel (EG_DEFAULT_KERNEL_NAME env var)
+--EnterpriseGatewayApp.dynamic_config_interval=<Int>
+    Default: 0
+    Specifies the number of seconds configuration files are polled for changes.
+    A value of 0 or less disables dynamic config updates.
+    (EG_DYNAMIC_CONFIG_INTERVAL env var)
 --EnterpriseGatewayApp.env_process_whitelist=<List>
     Default: []
     Environment variables allowed to be inherited from the spawning process by
@@ -639,3 +646,31 @@ The following kernel-specific environment variables are managed within Enterpris
     launch script the mode of Spark context intiatilization it should apply when
     starting the spark-based kernel container.
 ```
+### Dynamic Configurables
+Enterprise Gateway now supports the ability to update configuration variables without having to
+restart Enterprise Gateway.  This enables the ability to do things like enable debug logging or 
+adjust the maximum number of kernels per user, all without having to restart Enterprise Gateway.
+
+To enable dynamic configurables configure `EnterpriseGatewayApp.dynamic_config_interval` to a
+positive value (default is 0 or disabled).  Since this is the number of seconds to poll Enterprise Gateway's configuration files,
+a value greater than 60 (1 minute) is recommended.  This functionality works for most configuration 
+values, but does have the following caveats:
+1. Any configuration variables set on the command line (CLI) or via environment variables are
+NOT eligible for dynamic updates.  This is because Jupyter gives those values priority over
+file-based configuration variables.
+2. Any configuration variables tied to background processing may not reflect their update if
+the variable is not *observed* for changes.  For example, the code behind 
+`MappingKernelManager.cull_idle_timeout` may not reflect changes to the timeout period if 
+that variable is not monitored (i.e., observed) for changes.
+3. Only `Configurables` registered by Enterprise Gateway are eligible for dynamic updates.
+Currently, that list consists of the following (and their subclasses): EnterpriseGatewayApp, 
+MappingKernelManager, KernelSpecManager, and KernelSessionManager.
+
+As a result, administrators are encouraged to configure Enterprise Gateway via configuration
+files with only static values configured via the command line or environment.
+
+Note that if `EnterpriseGatewayApp.dynamic_config_interval` is configured with a positive value
+via the configuration file (i.e., is eligible for updates) and is subsequently set to 0, then
+dynamic configuration updates will be disabled until Enterprise Gateway is restarted with a 
+positive value.  Therefore, we recommend `EnterpriseGatewayApp.dynamic_config_interval` be 
+configured via the command line or environment.

enterprise_gateway/enterprisegatewayapp.py

@@ -9,6 +9,9 @@
 import signal
 import socket
 import sys
+import time
+import weakref
+
 from distutils.util import strtobool
 
 # Install the pyzmq ioloop. This has to be done before anything else from
@@ -20,7 +23,8 @@
 from tornado import web
 from tornado.log import enable_pretty_logging, LogFormatter
 
-from traitlets import default, List, Set, Unicode, Type, Instance, Bool, CBool, Integer
+from traitlets import default, List, Set, Unicode, Type, Instance, Bool, CBool, Integer, observe
+from traitlets.config import Configurable
 from jupyter_core.application import JupyterApp, base_aliases
 from jupyter_client.kernelspec import KernelSpecManager
 from notebook.services.kernels.kernelmanager import MappingKernelManager
@@ -394,6 +398,36 @@ def max_kernels_per_user_default(self):
     def ws_ping_interval_default(self):
         return int(os.getenv(self.ws_ping_interval_env, self.ws_ping_interval_default_value))
 
+    # Dynamic Update Interval
+    dynamic_config_interval_env = 'EG_DYNAMIC_CONFIG_INTERVAL'
+    dynamic_config_interval_default_value = 0
+    dynamic_config_interval = Integer(dynamic_config_interval_default_value, min=0, config=True,
+                                      help="""Specifies the number of seconds configuration files are polled for
+                                      changes.  A value of 0 or less disables dynamic config updates.
+                                      (EG_DYNAMIC_CONFIG_INTERVAL env var)""")
+
+    @default('dynamic_config_interval')
+    def dynamic_config_interval_default(self):
+        return int(os.getenv(self.dynamic_config_interval_env, self.dynamic_config_interval_default_value))
+
+    @observe('dynamic_config_interval')
+    def dynamic_config_interval_changed(self, event):
+        prev_val = event['old']
+        self.dynamic_config_interval = event['new']
+        if self.dynamic_config_interval != prev_val:
+            # Values are different.  Stop the current poller.  If new value is > 0, start a poller.
+            if self.dynamic_config_poller:
+                self.dynamic_config_poller.stop()
+                self.dynamic_config_poller = None
+
+            if self.dynamic_config_interval <= 0:
+                self.log.warning("Dynamic configuration updates have been disabled and cannot be re-enabled "
+                                 "without restarting Enterprise Gateway!")
+            elif prev_val > 0:  # The interval has been changed, but still positive
+                self.init_dynamic_configs()  # Restart the poller
+
+    dynamic_config_poller = None
+
     kernel_spec_manager = Instance(KernelSpecManager, allow_none=True)
 
     kernel_spec_manager_class = Type(
@@ -481,6 +515,8 @@ def init_configurables(self):
 
         self.contents_manager = None  # Gateways don't use contents manager
 
+        self.init_dynamic_configs()
+
     def _create_request_handlers(self):
         """Create default Jupyter handlers and redefine them off of the
         base_url path. Assumes init_configurables() has already been called.
@@ -659,5 +695,81 @@ def _signal_stop(self, sig, frame):
         self.log.info("Received signal to terminate Enterprise Gateway.")
         self.io_loop.add_callback_from_signal(self.io_loop.stop)
 
+    _last_config_update = int(time.time())
+    _dynamic_configurables = {}
+
+    def update_dynamic_configurables(self):
+        """
+        Called periodically, this checks the set of loaded configuration files for updates.
+        If updates have been detected, reload the configuration files and update the list of
+        configurables participating in dynamic updates.
+        :return: True if updates were taken
+        """
+        updated = False
+        configs = []
+        for file in self.loaded_config_files:
+            mod_time = int(os.path.getmtime(file))
+            if mod_time > self._last_config_update:
+                self.log.debug("Config file was updated: {}!".format(file))
+                self._last_config_update = mod_time
+                updated = True
+
+        if updated:
+            # If config changes are present, reload the config files.  This will also update
+            # the Application's configuration, then update the config of each configurable
+            # from the newly loaded values.
+
+            self.load_config_file(self)
+
+            for config_name, configurable in self._dynamic_configurables.items():
+                # Since Application.load_config_file calls update_config on the Application, skip
+                # the configurable registered with self (i.e., the application).
+                if configurable is not self:
+                    configurable.update_config(self.config)
+                configs.append(config_name)
+
+            self.log.info("Configuration file changes detected.  Instances for the following "
+                          "configurables have been updated: {}".format(configs))
+        return updated
+
+    def add_dynamic_configurable(self, config_name, configurable):
+        """
+        Adds the configurable instance associated with the given name to the list of Configurables
+        that can have their configurations updated when configuration file updates are detected.
+        :param config_name: the name of the config within this application
+        :param configurable: the configurable instance corresponding to that config
+        """
+        if not isinstance(configurable, Configurable):
+            raise RuntimeError("'{}' is not a subclass of Configurable!".format(configurable))
+
+        self._dynamic_configurables[config_name] = weakref.proxy(configurable)
+
+    def init_dynamic_configs(self):
+        """
+        Initialize the set of configurables that should participate in dynamic updates.  We should
+        also log that we're performing dynamic configuration updates, along with the list of CLI
+        options - that are not privy to dynamic updates.
+        :return:
+        """
+        if self.dynamic_config_interval > 0:
+            self.add_dynamic_configurable('EnterpriseGatewayApp', self)
+            self.add_dynamic_configurable('MappingKernelManager', self.kernel_manager)
+            self.add_dynamic_configurable('KernelSpecManager', self.kernel_spec_manager)
+            self.add_dynamic_configurable('KernelSessionManager', self.kernel_session_manager)
+
+            self.log.info("Dynamic updates have been configured.  Checking every {} seconds.".
+                          format(self.dynamic_config_interval))
+
+            self.log.info("The following configuration options will not be subject to dynamic updates "
+                          "(configured via CLI):")
+            for config, options in self.cli_config.items():
+                for option, value in options.items():
+                    self.log.info("    '{}.{}': '{}'".format(config, option, value))
+
+            if self.dynamic_config_poller is None:
+                self.dynamic_config_poller = ioloop.PeriodicCallback(self.update_dynamic_configurables,
+                                                                     self.dynamic_config_interval * 1000)
+            self.dynamic_config_poller.start()
+
 
 launch_instance = EnterpriseGatewayApp.launch_instance

enterprise_gateway/tests/test_enterprise_gateway.py

@@ -2,15 +2,22 @@
 # Distributed under the terms of the Modified BSD License.
 """Tests for jupyter-enterprise-gateway."""
 
+import os
+import time
+
 from tornado.testing import gen_test
 from tornado.escape import json_decode, url_escape
+
+from ipython_genutils.tempdir import TemporaryDirectory
 from .test_handlers import TestHandlers
 
+pjoin = os.path.join
+
 
 class TestEnterpriseGateway(TestHandlers):
 
     def setUp(self):
-        super(TestHandlers, self).setUp()
+        super(TestEnterpriseGateway, self).setUp()
         # Enable debug logging if necessary
         # app = self.get_app()
         # app.settings['kernel_manager'].log.level = logging.DEBUG
@@ -139,3 +146,49 @@ def test_port_range(self):
 
         for port in port_list:
             self.assertTrue(30000 <= port <= 31000)
+
+    @gen_test
+    def test_dynamic_updates(self):
+        app = self.app  # Get the actual EnterpriseGatewayApp instance
+        s1 = time.time()
+        name = app.config_file_name + '.py'
+        with TemporaryDirectory('_1') as td1:
+            os.environ['JUPYTER_CONFIG_DIR'] = td1
+            config_file = pjoin(td1, name)
+            with open(config_file, 'w') as f:
+                f.writelines([
+                    "c.EnterpriseGatewayApp.impersonation_enabled = False\n",
+                    "c.MappingKernelManager.cull_connected = False\n"
+                ])
+            #  app.jupyter_path.append(td1)
+            app.load_config_file()
+            app.add_dynamic_configurable("EnterpriseGatewayApp", app)
+            app.add_dynamic_configurable("RemoteMappingKernelManager", app.kernel_manager)
+            with self.assertRaises(RuntimeError):
+                app.add_dynamic_configurable("Bogus", app.log)
+
+            self.assertEqual(app.impersonation_enabled, False)
+            self.assertEqual(app.kernel_manager.cull_connected, False)
+
+            # Ensure file update doesn't happen during same second as initial value.
+            # This is necessary on test systems that don't have finer-grained
+            # timestamps (of less than a second).
+            s2 = time.time()
+            if s2 - s1 < 1.0:
+                time.sleep(1.0 - (s2 - s1))
+            # update config file
+            with open(config_file, 'w') as f:
+                f.writelines([
+                    "c.EnterpriseGatewayApp.impersonation_enabled = True\n",
+                    "c.MappingKernelManager.cull_connected = True\n"
+                ])
+
+            # trigger reload and verify updates
+            app.update_dynamic_configurables()
+            self.assertEqual(app.impersonation_enabled, True)
+            self.assertEqual(app.kernel_manager.cull_connected, True)
+
+            # repeat to ensure no unexpected changes occurred
+            app.update_dynamic_configurables()
+            self.assertEqual(app.impersonation_enabled, True)
+            self.assertEqual(app.kernel_manager.cull_connected, True)

setup.py

@@ -59,7 +59,7 @@
         'pyzmq>=17.0.0',
         'requests>=2.7,<3.0',
         'tornado>=4.2.0',
-        'traitlets>=4.2.0',
+        'traitlets>=4.3.3',
         'yarn-api-client>=1.0',
     ],
     python_requires='>=3.5',