From 9fe53f9872dd755d695f0da95b2e94465674d0d5 Mon Sep 17 00:00:00 2001 From: Gorka Eguileor Date: Tue, 29 Jan 2019 17:49:38 +0100 Subject: [PATCH] Optional probe (#79) * Correctly extract multipath, name, and workers We were not correctly setting up defaults and extracting provided values for REQUEST_MULTIPATH, WORKERS, and PLUGIN_NAME. Now we follow the right order: - We have defaults defined for these parameters in EMBER_CFG - Then we use them to set X_CSI_EMBER_CONFIG defaults for missing keys - Extract Ember specific keys that should not go to cinderlib * Close #76: Allow enabling/disabling probe Seems like there are a number of sidecars that don't properly support probing, and expect probes to take less than 1 second, which isn't usually the case. We are now disabling the probe by default unless explicitly enabled on the X_CSI_EMBER_CONFIG variable using the enable_probe key. --- README.md | 28 ++++++++++++++-------------- ember_csi/base.py | 2 +- ember_csi/config.py | 17 +++++++++++++---- ember_csi/defaults.py | 5 +++-- 4 files changed, 31 insertions(+), 21 deletions(-) diff --git a/README.md b/README.md index 7d2f06a2..07f37dc6 100644 --- a/README.md +++ b/README.md @@ -104,20 +104,20 @@ For RBD we'll also need a specific package: The CSI driver is configured via environmental variables, any value that doesn't have a default is a required value. -| Name | Role | Description | Default | Example | -| -------------------------- | ---------- | ------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `CSI_ENDPOINT` | all | IP and port to bind the service | [::]:50051 | 192.168.1.22:50050 | -| `CSI_MODE` | all | Role the service should perform: controller, node, all | all | controller | -| `X_CSI_SPEC_VERSION` | all | CSI Spec version to run. Supported v0.2 and v1.0 | v0.2.0 | 0.2.0 -| `X_CSI_STORAGE_NW_IP` | node | IP address in the Node used to connect to the storage | IP resolved from Node's fqdn | 192.168.1.22 | -| `X_CSI_NODE_ID` | node | ID used by this node to identify itself to the controller | Node's fqdn | csi_test_node | -| `X_CSI_PERSISTENCE_CONFIG` | all | Configuration of the `cinderlib` metadata persistence plugin. | {"storage": "crd", "namespace": "default"} | {"storage": "db", "connection": "mysql+pymysql://root:stackdb@192.168.1.1/cinder?charset=utf8"} | -| `X_CSI_EMBER_CONFIG` | all | Global `cinderlib` configuration | {'project_id': 'io.ember-csi', 'user_id': 'io.ember-csi', 'root_helper': 'sudo', 'request_multipath': false, 'plugin_name': 'io.ember-csi', 'file_locks_path': '/var/lib/ember-csi/locks'} | {"project_id":"k8s project","user_id":"csi driver","root_helper":"sudo",'plugin_name': 'io.ember-csi.external-ceph'} | -| `X_CSI_BACKEND_CONFIG` | controller | Driver configuration | | {"name": "rbd", "driver": "RBD", "rbd_user": "cinder", "rbd_pool": "volumes", "rbd_ceph_conf": "/etc/ceph/ceph.conf", "rbd_keyring_conf": "/etc/ceph/ceph.client.cinder.keyring"} | -| `X_CSI_DEFAULT_MOUNT_FS` | node | Default mount filesystem when missing in publish calls | ext4 | btrfs | -| `X_CSI_SYSTEM_FILES` | all | All required storage driver-specific files archived in tar, tar.gz or tar.bz2 format| | /path/to/etc-ceph.tar.gz | -| `X_CSI_DEBUG_MODE` | all | Debug mode (rpdb, pdb) to use. Disabled by default. | | rpdb | -| `X_CSI_ABORT_DUPLICATES` | all | If we want to abort or queue (default) duplicated requests. | false | true | +| Name | Role | Description | Default | Example | +| -------------------------- | ---------- | ------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `CSI_ENDPOINT` | all | IP and port to bind the service | [::]:50051 | 192.168.1.22:50050 | +| `CSI_MODE` | all | Role the service should perform: controller, node, all | all | controller | +| `X_CSI_SPEC_VERSION` | all | CSI Spec version to run. Supported v0.2 and v1.0 | v0.2.0 | 0.2.0 | +| `X_CSI_STORAGE_NW_IP` | node | IP address in the Node used to connect to the storage | IP resolved from Node's fqdn | 192.168.1.22 | +| `X_CSI_NODE_ID` | node | ID used by this node to identify itself to the controller | Node's fqdn | csi_test_node | +| `X_CSI_PERSISTENCE_CONFIG` | all | Configuration of the `cinderlib` metadata persistence plugin. | {"storage": "crd", "namespace": "default"} | {"storage": "db", "connection": "mysql+pymysql://root:stackdb@192.168.1.1/cinder?charset=utf8"} | +| `X_CSI_EMBER_CONFIG` | all | Global `Ember` and `cinderlib` configuration | {"project_id": "io.ember-csi", "user_id": "io.ember-csi", "root_helper": "sudo", "request_multipath": false, "plugin_name": "io.ember-csi", "file_locks_path": "/var/lib/ember-csi/locks", "name": "io.ember-csi", "grpc_workers": 30, "enable_probe": false} | {"project_id":"k8s project","user_id":"csi driver","root_helper":"sudo","plugin_name": "io.ember-csi.external-ceph"} | +| `X_CSI_BACKEND_CONFIG` | controller | Driver configuration | | {"name": "rbd", "driver": "RBD", "rbd_user": "cinder", "rbd_pool": "volumes", "rbd_ceph_conf": "/etc/ceph/ceph.conf", "rbd_keyring_conf": "/etc/ceph/ceph.client.cinder.keyring"} | +| `X_CSI_DEFAULT_MOUNT_FS` | node | Default mount filesystem when missing in publish calls | ext4 | btrfs | +| `X_CSI_SYSTEM_FILES` | all | All required storage driver-specific files archived in tar, tar.gz or tar.bz2 format| | /path/to/etc-ceph.tar.gz | +| `X_CSI_DEBUG_MODE` | all | Debug mode (rpdb, pdb) to use. Disabled by default. | | rpdb | +| `X_CSI_ABORT_DUPLICATES` | all | If we want to abort or queue (default) duplicated requests. | false | true | The only role that has been tested at the moment is the default one, where Controller and Node servicer are executed in the same service (`CSI_MODE=all`), and other modes are expected to have issues at the moment. diff --git a/ember_csi/base.py b/ember_csi/base.py index d725c929..7def2ff6 100644 --- a/ember_csi/base.py +++ b/ember_csi/base.py @@ -162,7 +162,7 @@ def GetPluginCapabilities(self, request, context): def Probe(self, request, context): # Proving may take a couple of seconds, and attacher sidecar prior to # v0.4 will fail due to small timeout. - if self.csi_version < '1.0.0': + if not config.ENABLE_PROBE: return self.PROBE_RESP try: diff --git a/ember_csi/config.py b/ember_csi/config.py index fe8a2cb6..5e4aeaca 100644 --- a/ember_csi/config.py +++ b/ember_csi/config.py @@ -68,10 +68,8 @@ def _get_system_fs_types(): defaults.PERSISTENCE_CFG) EMBER_CONFIG = _load_json_config('X_CSI_EMBER_CONFIG', defaults.EMBER_CFG) -REQUEST_MULTIPATH = EMBER_CONFIG.pop('request_multipath', - defaults.REQUEST_MULTIPATH) -WORKERS = EMBER_CONFIG.pop('grpc_workers', defaults.WORKERS) -PLUGIN_NAME = EMBER_CONFIG.pop('plugin_name', None) or defaults.NAME +# REQUEST_MULTIPATH, WORKERS, PLUGIN_NAME, ENABLE_PROBE are set from +# EMBER_CONFIG on _set_defaults_ember_cfg BACKEND_CONFIG = _load_json_config('X_CSI_BACKEND_CONFIG') NODE_ID = os.environ.get('X_CSI_NODE_ID') or socket.getfqdn() DEFAULT_MOUNT_FS = os.environ.get('X_CSI_DEFAULT_MOUNT_FS', defaults.MOUNT_FS) @@ -166,6 +164,11 @@ def _map_backend_config(): def _set_defaults_ember_cfg(): + global REQUEST_MULTIPATH + global WORKERS + global PLUGIN_NAME + global ENABLE_PROBE + # First set defaults for missing keys for key, value in defaults.EMBER_CFG.items(): EMBER_CONFIG.setdefault(key, value) @@ -178,6 +181,12 @@ def _set_defaults_ember_cfg(): defaults.VOL_BINDS_DIR = defaults.VOL_BINDS_DIR.replace('$state_path', state_path) + # Now set global variables + REQUEST_MULTIPATH = EMBER_CONFIG.pop('request_multipath') + WORKERS = EMBER_CONFIG.pop('grpc_workers') + PLUGIN_NAME = EMBER_CONFIG.pop('plugin_name') + ENABLE_PROBE = EMBER_CONFIG.pop('enable_probe') + def _create_default_dirs_files(): def create_dir(name): diff --git a/ember_csi/defaults.py b/ember_csi/defaults.py index d4ebcc05..b0cf5cac 100644 --- a/ember_csi/defaults.py +++ b/ember_csi/defaults.py @@ -22,10 +22,12 @@ VOL_BINDS_DIR = '$state_path/vols' LOCKS_DIR = '$state_path/locks' REQUEST_MULTIPATH = False +WORKERS = 30 EMBER_CFG = {'project_id': NAME, 'user_id': NAME, 'plugin_name': NAME, 'root_helper': ROOT_HELPER, 'request_multipath': REQUEST_MULTIPATH, - 'file_locks_path': LOCKS_DIR, 'state_path': STATE_PATH} + 'file_locks_path': LOCKS_DIR, 'state_path': STATE_PATH, + 'enable_probe': False, 'grpc_workers': WORKERS} LOG_LEVELS = ('amqp=WARN,amqplib=WARN,boto=WARN,qpid=WARN,sqlalchemy=WARN,' 'suds=WARN,oslo.messaging=WARN,oslo_messaging=WARN,iso8601=WARN,' @@ -53,4 +55,3 @@ VOLUME_SIZE = 1.0 SPEC_VERSION = '0.2.0' CRD_NAMESPACE = 'default' -WORKERS = 30