Add yaml config support to ydbd_slice (#1406)

* add yaml config support to ydbd_slice

* remove nested dir for dynconfig

Author: Enjection

ydb/tools/ydbd_slice/__init__.py

@@ -13,7 +13,6 @@
 from ydb.tools.ydbd_slice import nodes, handlers, cluster_description
 from ydb.tools.ydbd_slice.kube import handlers as kube_handlers, docker
 
-
 warnings.filterwarnings("ignore", category=DeprecationWarning)
 warnings.filterwarnings("ignore", category=HTTPWarning)
 
@@ -830,7 +829,7 @@ def _run(args):
 
             manifests = kube_handlers.get_all_manifests(args.path)
             kube_handlers.manifests_ydb_set_image(args.path, manifests, image)
-            kube_handlers.slice_install(args.path, manifests, args.wait_ready)
+            kube_handlers.slice_install(args.path, manifests, args.wait_ready, args.dynamic_config_type)
 
             logger.info('kube-install finished')
         except RuntimeError as e:
@@ -857,6 +856,12 @@ def _run(args):
         help='Do not build docker image, just specify image name in manifests.',
         action='store_true',
     )
+    mode.add_argument(
+        '--dynamic-config-type',
+        help='Upload dynamic config with specified type',
+        choices=['both', 'proto', 'yaml', 'none'],
+        default='both',
+    )
     add_arguments_docker_build_with_remainder(mode, add_force_rebuild=True)
     mode.set_defaults(handler=_run)
 
@@ -872,7 +877,7 @@ def _run(args):
             manifests = kube_handlers.get_all_manifests(args.path)
             manifests = kube_handlers.manifests_ydb_filter_components(args.path, manifests, args.components)
             kube_handlers.manifests_ydb_set_image(args.path, manifests, image)
-            kube_handlers.slice_update(args.path, manifests, args.wait_ready)
+            kube_handlers.slice_update(args.path, manifests, args.wait_ready, args.dynamic_config_type)
 
             logger.info('kube-update finished')
         except RuntimeError as e:
@@ -905,6 +910,12 @@ def _run(args):
         help='Do not build docker image, just specify image name in manifests.',
         action='store_true',
     )
+    mode.add_argument(
+        '--dynamic-config-type',
+        help='Upload dynamic config with specified type',
+        choices=['both', 'proto', 'yaml', 'none'],
+        default='both',
+    )
     add_arguments_docker_build_with_remainder(mode, add_force_rebuild=True)
     mode.set_defaults(handler=_run)
 
@@ -947,7 +958,7 @@ def _run(args):
         try:
             manifests = kube_handlers.get_all_manifests(args.path)
             manifests = kube_handlers.manifests_ydb_filter_components(args.path, manifests, args.components)
-            kube_handlers.slice_start(args.path, manifests, args.wait_ready)
+            kube_handlers.slice_start(args.path, manifests, args.wait_ready, args.dynamic_config_type)
 
             logger.info('kube-start finished')
         except RuntimeError as e:
@@ -975,6 +986,12 @@ def _run(args):
         help='Wait for ydb objects ready state. Default: false',
         action='store_true',
     )
+    mode.add_argument(
+        '--dynamic-config-type',
+        help='Upload dynamic config with specified type',
+        choices=['both', 'proto', 'yaml', 'none'],
+        default='both',
+    )
     mode.set_defaults(handler=_run)
 
 
@@ -1040,7 +1057,7 @@ def _run(args):
         logger.debug("starting kube-format cmd with args '%s'", args)
         try:
             manifests = kube_handlers.get_all_manifests(args.path)
-            kube_handlers.slice_format(args.path, manifests, args.wait_ready)
+            kube_handlers.slice_format(args.path, manifests, args.wait_ready, args.dynamic_config_type)
 
             logger.info('kube-format finished')
         except RuntimeError as e:
@@ -1063,6 +1080,12 @@ def _run(args):
         help='Wait for ydb objects ready state. Default: false',
         action='store_true',
     )
+    mode.add_argument(
+        '--dynamic-config-type',
+        help='Upload dynamic config with specified type',
+        choices=['both', 'proto', 'yaml', 'none'],
+        default='both',
+    )
     mode.set_defaults(handler=_run)
 
 

ydb/tools/ydbd_slice/kube/dynconfig.py

@@ -0,0 +1,87 @@
+# small wrapper for operations with dynconfig
+
+import logging
+import os
+
+from ydb import Driver as YdbDriver, DriverConfig, AnonymousCredentials, credentials_from_env_variables
+from ydb.draft import DynamicConfigClient
+
+
+logger = logging.getLogger(__name__)
+
+
+DYNCONFIG_NAME= 'dynconfig.yaml'
+
+
+class Client():
+    def __init__(self, node_list, domain, credentials=credentials_from_env_variables(), allow_fallback_to_anonymous_credentials=True):
+        self.config = DriverConfig(
+            endpoint=f'grpc://{node_list[0]}:2135',
+            database=domain,
+            credentials=credentials,
+            endpoints=[f'grpc://{x}:2135' for x in node_list],
+        )
+        self.allow_fallback_to_anonymous_credentials = allow_fallback_to_anonymous_credentials
+        self.driver = YdbDriver(self.config)
+
+    def __enter__(self):
+        try:
+            self.driver.wait(timeout=5)
+        except TimeoutError:
+            if self.allow_fallback_to_anonymous_credentials:
+                logger.warning('Trying to fallback to anonymous credentials. ' \
+                               'To get rid of this message either set YDB_ANONYMOUS_CREDENTIALS=1 ' \
+                               'Or set correct credentials in env as it described in ydb-python-sdk.')
+                config = self.config
+                config.credentials = AnonymousCredentials()
+                self.driver = YdbDriver(config)
+                try:
+                    self.driver.wait(timeout=5)
+                except TimeoutError:
+                    logger.error('Unable to connect to static nodes for dynconfig management.')
+                    raise
+            else:
+                logger.error('Unable to connect to static nodes for dynconfig management.')
+                raise
+
+        return DynamicConfigClient(self.driver)
+
+    def __exit__(self, exception_type, exception_value, exception_traceback):
+        pass
+
+
+def get_config_path(project_path):
+    return os.path.join(project_path, DYNCONFIG_NAME)
+
+
+def get_config_abspath(project_path):
+    return os.path.abspath(get_config_path(project_path))
+
+
+def get_local_config(project_path):
+    dynconfig_file = get_config_abspath(project_path)
+    if not os.path.exists(dynconfig_file):
+        return None
+    with open(dynconfig_file, 'r') as config:
+        return config.read()
+
+
+def write_local_config(project_path, new_config):
+    dynconfig_file = get_config_abspath(project_path)
+    with open(dynconfig_file, 'w') as config:
+        config.write(new_config)
+
+
+def get_remote_config(client):
+    return client.get_config().config
+
+
+def apply_config(client, project_path, force=False, allow_unknown_fields=True):
+        config = get_local_config(project_path)
+        if force:
+            client.set_config(config, False, allow_unknown_fields)
+        else:
+            client.replace_config(config, False, allow_unknown_fields)
+        # TODO: there is small race here, we can pull back config applied after our's, but for dev slices it's fine
+        # as far as even in real production with long approve process we faced conflict only once in a three months
+        return get_remote_config(client)

ydb/tools/ydbd_slice/kube/generate.py

@@ -3,13 +3,14 @@
 import jinja2
 import library.python.resource as rs
 
-from ydb.tools.ydbd_slice.kube import cms
+from ydb.tools.ydbd_slice.kube import cms, dynconfig
 
 
 def render_resource(name, params):
     env = jinja2.Environment(
         loader=jinja2.FunctionLoader(lambda x: rs.find(x).decode()), undefined=jinja2.StrictUndefined
     )
+
     template = env.get_template(name)
     return template.render(**params)
 
@@ -50,8 +51,20 @@ def generate_legacy_configs(project_path, preferred_pool_kind='ssd'):
     )
 
 
+def generate_dynconfigs(project_path, namespace_name, cluster_uuid, preferred_pool_kind='ssd'):
+    generate_file(
+        project_path=project_path,
+        filename=dynconfig.get_config_path(project_path),
+        template='/ydbd_slice/templates/common/dynconfig.yaml',
+        template_kwargs=dict(
+            preferred_pool_kind=preferred_pool_kind,
+            cluster_uuid=cluster_uuid,
+        )
+    )
+
+
 def generate_8_node_block_4_2(project_path, user, namespace_name, nodeclaim_name, node_flavor,
-                              storage_name, database_name):
+                              storage_name, database_name, cluster_uuid=''):
     generate_file(
         project_path=project_path,
         filename=f'namespace-{namespace_name}.yaml',
@@ -95,4 +108,6 @@ def generate_8_node_block_4_2(project_path, user, namespace_name, nodeclaim_name
         )
     )
 
+    generate_dynconfigs(project_path, namespace_name, cluster_uuid)
+
     generate_legacy_configs(project_path)

ydb/tools/ydbd_slice/kube/handlers.py

@@ -2,13 +2,13 @@
 import os
 import sys
 import logging
-
+import time
 
 from collections import defaultdict
 from kubernetes.client import Configuration
 
 from ydb.tools.ydbd_slice import nodes, handlers
-from ydb.tools.ydbd_slice.kube import api, kubectl, yaml, generate, cms
+from ydb.tools.ydbd_slice.kube import api, kubectl, yaml, generate, cms, dynconfig
 
 
 logger = logging.getLogger(__name__)
@@ -148,6 +148,13 @@ def get_nodes(api_client, project_path, manifests):
     return node_list
 
 
+def get_domain(api_client, project_path, manifests):
+    for (_, _, kind, _, _, data) in manifests:
+        if kind != 'storage':
+            continue
+        return data['spec']['domain']
+
+
 def manifests_ydb_set_image(project_path, manifests, image):
     for (path, api_version, kind, namespace, name, data) in manifests:
         if not (kind in ['storage', 'database'] and api_version in ['ydb.tech/v1alpha1']):
@@ -245,9 +252,20 @@ def slice_nodeclaim_delete(api_client, project_path, manifests):
             sys.exit(e.args[0])
 
 
+def wait_for_storage(api_client, project_path, manifests):
+    for (path, api_version, kind, namespace, name, data) in manifests:
+        if not (kind in ['storage'] and api_version in ['ydb.tech/v1alpha1']):
+            continue
+        namespace = data['metadata']['namespace']
+        name = data['metadata']['name']
+        try:
+            api.wait_storage_state_ready(api_client, namespace, name)
+        except TimeoutError as e:
+            sys.exit(e.args[0])
+
 #
 # macro level ydb functions
-def slice_ydb_apply(api_client, project_path, manifests):
+def slice_ydb_apply(api_client, project_path, manifests, dynamic_config_type):
     # process storages first
     for (path, api_version, kind, namespace, name, data) in manifests:
         if not (kind in ['storage', 'database'] and api_version in ['ydb.tech/v1alpha1']):
@@ -264,28 +282,36 @@ def slice_ydb_apply(api_client, project_path, manifests):
             data['spec'] = new_data['spec']
             update_manifest(path, data)
 
-    config_items = cms.get_from_files(project_path)
-    if config_items is not None:
-        logger.debug(
-            f'found {len(config_items)} legacy cms config items, '
-            'need to wait for storage to become ready to apply configs'
-        )
-        # if configs present, then wait for storage
-        for (path, api_version, kind, namespace, name, data) in manifests:
-            if not (kind in ['storage'] and api_version in ['ydb.tech/v1alpha1']):
-                continue
-            namespace = data['metadata']['namespace']
-            name = data['metadata']['name']
-            try:
-                api.wait_storage_state_ready(api_client, namespace, name)
-            except TimeoutError as e:
-                sys.exit(e.args[0])
+    if dynamic_config_type in ['both', 'yaml']:
+        local_config = dynconfig.get_local_config(project_path)
+        if local_config is not None:
+            wait_for_storage(api_client, project_path, manifests)
+
+            node_list = get_nodes(api_client, project_path, manifests)
+            domain = get_domain(api_client, project_path, manifests)
+
+            with dynconfig.Client(node_list, domain) as dynconfig_client:
+                remote_config = dynconfig.get_remote_config(dynconfig_client)
+
+                if remote_config is None or remote_config != local_config:
+                    new_config = dynconfig.apply_config(dynconfig_client, project_path)
+                    dynconfig.write_local_config(project_path, new_config)
+
+    if dynamic_config_type in ['both', 'proto']:
+        config_items = cms.get_from_files(project_path)
+        if config_items is not None:
+            logger.debug(
+                f'found {len(config_items)} legacy cms config items, '
+                'need to wait for storage to become ready to apply configs'
+            )
+            # if configs present, then wait for storage
+            wait_for_storage(api_client, project_path, manifests)
 
-        # and apply configs
-        node_list = get_nodes(api_client, project_path, manifests)
-        if len(node_list) == 0:
-            raise RuntimeError('no nodes found, cannot apply legacy cms config items.')
-        cms.apply_legacy_cms_config_items(config_items, [f'grpc://{i}:2135' for i in node_list])
+            # and apply configs
+            node_list = get_nodes(api_client, project_path, manifests)
+            if len(node_list) == 0:
+                raise RuntimeError('no nodes found, cannot apply legacy cms config items.')
+            cms.apply_legacy_cms_config_items(config_items, [f'grpc://{i}:2135' for i in node_list])
 
     # process databases later
     for (path, api_version, kind, namespace, name, data) in manifests:
@@ -432,23 +458,23 @@ def slice_generate(project_path, user, slice_name, template, template_vars):
         sys.exit(f'Slice template {template} not implemented.')
 
 
-def slice_install(project_path, manifests, wait_ready):
+def slice_install(project_path, manifests, wait_ready, dynamic_config_type):
     with api.ApiClient() as api_client:
         slice_namespace_apply(api_client, project_path, manifests)
         slice_nodeclaim_apply(api_client, project_path, manifests)
         slice_nodeclaim_wait_ready(api_client, project_path, manifests)
         slice_ydb_delete(api_client, project_path, manifests)
         slice_ydb_storage_wait_pods_deleted(api_client, project_path, manifests)
         slice_nodeclaim_format(api_client, project_path, manifests)
-        slice_ydb_apply(api_client, project_path, manifests)
+        slice_ydb_apply(api_client, project_path, manifests, dynamic_config_type)
         slice_ydb_wait_ready(api_client, project_path, manifests, wait_ready)
 
 
-def slice_update(project_path, manifests, wait_ready):
+def slice_update(project_path, manifests, wait_ready, dynamic_config_type):
     with api.ApiClient() as api_client:
         slice_nodeclaim_apply(api_client, project_path, manifests)
         slice_nodeclaim_wait_ready(api_client, project_path, manifests)
-        slice_ydb_apply(api_client, project_path, manifests)
+        slice_ydb_apply(api_client, project_path, manifests, dynamic_config_type)
         slice_ydb_restart(api_client, project_path, manifests)
         slice_ydb_wait_ready(api_client, project_path, manifests, wait_ready)
 
@@ -458,9 +484,9 @@ def slice_stop(project_path, manifests):
         slice_ydb_delete(api_client, project_path, manifests)
 
 
-def slice_start(project_path, manifests, wait_ready):
+def slice_start(project_path, manifests, wait_ready, dynamic_config_type):
     with api.ApiClient() as api_client:
-        slice_ydb_apply(api_client, project_path, manifests)
+        slice_ydb_apply(api_client, project_path, manifests, dynamic_config_type)
         slice_ydb_wait_ready(api_client, project_path, manifests, wait_ready)
 
 
@@ -474,12 +500,12 @@ def slice_nodes(project_path, manifests):
         slice_nodeclaim_nodes(api_client, project_path, manifests)
 
 
-def slice_format(project_path, manifests, wait_ready):
+def slice_format(project_path, manifests, wait_ready, dynamic_config_type):
     with api.ApiClient() as api_client:
         slice_ydb_delete(api_client, project_path, manifests)
         slice_ydb_storage_wait_pods_deleted(api_client, project_path, manifests)
         slice_nodeclaim_format(api_client, project_path, manifests)
-        slice_ydb_apply(api_client, project_path, manifests)
+        slice_ydb_apply(api_client, project_path, manifests, dynamic_config_type)
         slice_ydb_wait_ready(api_client, project_path, manifests, wait_ready)