diff --git a/ydb/tools/ydbd_slice/__init__.py b/ydb/tools/ydbd_slice/__init__.py index 4e862f155549..d2c2ab2e6c2b 100644 --- a/ydb/tools/ydbd_slice/__init__.py +++ b/ydb/tools/ydbd_slice/__init__.py @@ -13,7 +13,6 @@ from ydb.tools.ydbd_slice import nodes, handlers, cluster_description from ydb.tools.ydbd_slice.kube import handlers as kube_handlers, docker - warnings.filterwarnings("ignore", category=DeprecationWarning) warnings.filterwarnings("ignore", category=HTTPWarning) @@ -830,7 +829,7 @@ def _run(args): manifests = kube_handlers.get_all_manifests(args.path) kube_handlers.manifests_ydb_set_image(args.path, manifests, image) - kube_handlers.slice_install(args.path, manifests, args.wait_ready) + kube_handlers.slice_install(args.path, manifests, args.wait_ready, args.dynamic_config_type) logger.info('kube-install finished') except RuntimeError as e: @@ -857,6 +856,12 @@ def _run(args): help='Do not build docker image, just specify image name in manifests.', action='store_true', ) + mode.add_argument( + '--dynamic-config-type', + help='Upload dynamic config with specified type', + choices=['both', 'proto', 'yaml', 'none'], + default='both', + ) add_arguments_docker_build_with_remainder(mode, add_force_rebuild=True) mode.set_defaults(handler=_run) @@ -872,7 +877,7 @@ def _run(args): manifests = kube_handlers.get_all_manifests(args.path) manifests = kube_handlers.manifests_ydb_filter_components(args.path, manifests, args.components) kube_handlers.manifests_ydb_set_image(args.path, manifests, image) - kube_handlers.slice_update(args.path, manifests, args.wait_ready) + kube_handlers.slice_update(args.path, manifests, args.wait_ready, args.dynamic_config_type) logger.info('kube-update finished') except RuntimeError as e: @@ -905,6 +910,12 @@ def _run(args): help='Do not build docker image, just specify image name in manifests.', action='store_true', ) + mode.add_argument( + '--dynamic-config-type', + help='Upload dynamic config with specified type', + choices=['both', 'proto', 'yaml', 'none'], + default='both', + ) add_arguments_docker_build_with_remainder(mode, add_force_rebuild=True) mode.set_defaults(handler=_run) @@ -947,7 +958,7 @@ def _run(args): try: manifests = kube_handlers.get_all_manifests(args.path) manifests = kube_handlers.manifests_ydb_filter_components(args.path, manifests, args.components) - kube_handlers.slice_start(args.path, manifests, args.wait_ready) + kube_handlers.slice_start(args.path, manifests, args.wait_ready, args.dynamic_config_type) logger.info('kube-start finished') except RuntimeError as e: @@ -975,6 +986,12 @@ def _run(args): help='Wait for ydb objects ready state. Default: false', action='store_true', ) + mode.add_argument( + '--dynamic-config-type', + help='Upload dynamic config with specified type', + choices=['both', 'proto', 'yaml', 'none'], + default='both', + ) mode.set_defaults(handler=_run) @@ -1040,7 +1057,7 @@ def _run(args): logger.debug("starting kube-format cmd with args '%s'", args) try: manifests = kube_handlers.get_all_manifests(args.path) - kube_handlers.slice_format(args.path, manifests, args.wait_ready) + kube_handlers.slice_format(args.path, manifests, args.wait_ready, args.dynamic_config_type) logger.info('kube-format finished') except RuntimeError as e: @@ -1063,6 +1080,12 @@ def _run(args): help='Wait for ydb objects ready state. Default: false', action='store_true', ) + mode.add_argument( + '--dynamic-config-type', + help='Upload dynamic config with specified type', + choices=['both', 'proto', 'yaml', 'none'], + default='both', + ) mode.set_defaults(handler=_run) diff --git a/ydb/tools/ydbd_slice/kube/dynconfig.py b/ydb/tools/ydbd_slice/kube/dynconfig.py new file mode 100644 index 000000000000..2946934bd111 --- /dev/null +++ b/ydb/tools/ydbd_slice/kube/dynconfig.py @@ -0,0 +1,87 @@ +# small wrapper for operations with dynconfig + +import logging +import os + +from ydb import Driver as YdbDriver, DriverConfig, AnonymousCredentials, credentials_from_env_variables +from ydb.draft import DynamicConfigClient + + +logger = logging.getLogger(__name__) + + +DYNCONFIG_NAME= 'dynconfig.yaml' + + +class Client(): + def __init__(self, node_list, domain, credentials=credentials_from_env_variables(), allow_fallback_to_anonymous_credentials=True): + self.config = DriverConfig( + endpoint=f'grpc://{node_list[0]}:2135', + database=domain, + credentials=credentials, + endpoints=[f'grpc://{x}:2135' for x in node_list], + ) + self.allow_fallback_to_anonymous_credentials = allow_fallback_to_anonymous_credentials + self.driver = YdbDriver(self.config) + + def __enter__(self): + try: + self.driver.wait(timeout=5) + except TimeoutError: + if self.allow_fallback_to_anonymous_credentials: + logger.warning('Trying to fallback to anonymous credentials. ' \ + 'To get rid of this message either set YDB_ANONYMOUS_CREDENTIALS=1 ' \ + 'Or set correct credentials in env as it described in ydb-python-sdk.') + config = self.config + config.credentials = AnonymousCredentials() + self.driver = YdbDriver(config) + try: + self.driver.wait(timeout=5) + except TimeoutError: + logger.error('Unable to connect to static nodes for dynconfig management.') + raise + else: + logger.error('Unable to connect to static nodes for dynconfig management.') + raise + + return DynamicConfigClient(self.driver) + + def __exit__(self, exception_type, exception_value, exception_traceback): + pass + + +def get_config_path(project_path): + return os.path.join(project_path, DYNCONFIG_NAME) + + +def get_config_abspath(project_path): + return os.path.abspath(get_config_path(project_path)) + + +def get_local_config(project_path): + dynconfig_file = get_config_abspath(project_path) + if not os.path.exists(dynconfig_file): + return None + with open(dynconfig_file, 'r') as config: + return config.read() + + +def write_local_config(project_path, new_config): + dynconfig_file = get_config_abspath(project_path) + with open(dynconfig_file, 'w') as config: + config.write(new_config) + + +def get_remote_config(client): + return client.get_config().config + + +def apply_config(client, project_path, force=False, allow_unknown_fields=True): + config = get_local_config(project_path) + if force: + client.set_config(config, False, allow_unknown_fields) + else: + client.replace_config(config, False, allow_unknown_fields) + # TODO: there is small race here, we can pull back config applied after our's, but for dev slices it's fine + # as far as even in real production with long approve process we faced conflict only once in a three months + return get_remote_config(client) diff --git a/ydb/tools/ydbd_slice/kube/generate.py b/ydb/tools/ydbd_slice/kube/generate.py index 884fa38ec9fd..b1a86b834fce 100644 --- a/ydb/tools/ydbd_slice/kube/generate.py +++ b/ydb/tools/ydbd_slice/kube/generate.py @@ -3,13 +3,14 @@ import jinja2 import library.python.resource as rs -from ydb.tools.ydbd_slice.kube import cms +from ydb.tools.ydbd_slice.kube import cms, dynconfig def render_resource(name, params): env = jinja2.Environment( loader=jinja2.FunctionLoader(lambda x: rs.find(x).decode()), undefined=jinja2.StrictUndefined ) + template = env.get_template(name) return template.render(**params) @@ -50,8 +51,20 @@ def generate_legacy_configs(project_path, preferred_pool_kind='ssd'): ) +def generate_dynconfigs(project_path, namespace_name, cluster_uuid, preferred_pool_kind='ssd'): + generate_file( + project_path=project_path, + filename=dynconfig.get_config_path(project_path), + template='/ydbd_slice/templates/common/dynconfig.yaml', + template_kwargs=dict( + preferred_pool_kind=preferred_pool_kind, + cluster_uuid=cluster_uuid, + ) + ) + + def generate_8_node_block_4_2(project_path, user, namespace_name, nodeclaim_name, node_flavor, - storage_name, database_name): + storage_name, database_name, cluster_uuid=''): generate_file( project_path=project_path, filename=f'namespace-{namespace_name}.yaml', @@ -95,4 +108,6 @@ def generate_8_node_block_4_2(project_path, user, namespace_name, nodeclaim_name ) ) + generate_dynconfigs(project_path, namespace_name, cluster_uuid) + generate_legacy_configs(project_path) diff --git a/ydb/tools/ydbd_slice/kube/handlers.py b/ydb/tools/ydbd_slice/kube/handlers.py index 3603c016fbf3..d01d9258f86e 100644 --- a/ydb/tools/ydbd_slice/kube/handlers.py +++ b/ydb/tools/ydbd_slice/kube/handlers.py @@ -2,13 +2,13 @@ import os import sys import logging - +import time from collections import defaultdict from kubernetes.client import Configuration from ydb.tools.ydbd_slice import nodes, handlers -from ydb.tools.ydbd_slice.kube import api, kubectl, yaml, generate, cms +from ydb.tools.ydbd_slice.kube import api, kubectl, yaml, generate, cms, dynconfig logger = logging.getLogger(__name__) @@ -148,6 +148,13 @@ def get_nodes(api_client, project_path, manifests): return node_list +def get_domain(api_client, project_path, manifests): + for (_, _, kind, _, _, data) in manifests: + if kind != 'storage': + continue + return data['spec']['domain'] + + def manifests_ydb_set_image(project_path, manifests, image): for (path, api_version, kind, namespace, name, data) in manifests: if not (kind in ['storage', 'database'] and api_version in ['ydb.tech/v1alpha1']): @@ -245,9 +252,20 @@ def slice_nodeclaim_delete(api_client, project_path, manifests): sys.exit(e.args[0]) +def wait_for_storage(api_client, project_path, manifests): + for (path, api_version, kind, namespace, name, data) in manifests: + if not (kind in ['storage'] and api_version in ['ydb.tech/v1alpha1']): + continue + namespace = data['metadata']['namespace'] + name = data['metadata']['name'] + try: + api.wait_storage_state_ready(api_client, namespace, name) + except TimeoutError as e: + sys.exit(e.args[0]) + # # macro level ydb functions -def slice_ydb_apply(api_client, project_path, manifests): +def slice_ydb_apply(api_client, project_path, manifests, dynamic_config_type): # process storages first for (path, api_version, kind, namespace, name, data) in manifests: if not (kind in ['storage', 'database'] and api_version in ['ydb.tech/v1alpha1']): @@ -264,28 +282,36 @@ def slice_ydb_apply(api_client, project_path, manifests): data['spec'] = new_data['spec'] update_manifest(path, data) - config_items = cms.get_from_files(project_path) - if config_items is not None: - logger.debug( - f'found {len(config_items)} legacy cms config items, ' - 'need to wait for storage to become ready to apply configs' - ) - # if configs present, then wait for storage - for (path, api_version, kind, namespace, name, data) in manifests: - if not (kind in ['storage'] and api_version in ['ydb.tech/v1alpha1']): - continue - namespace = data['metadata']['namespace'] - name = data['metadata']['name'] - try: - api.wait_storage_state_ready(api_client, namespace, name) - except TimeoutError as e: - sys.exit(e.args[0]) + if dynamic_config_type in ['both', 'yaml']: + local_config = dynconfig.get_local_config(project_path) + if local_config is not None: + wait_for_storage(api_client, project_path, manifests) + + node_list = get_nodes(api_client, project_path, manifests) + domain = get_domain(api_client, project_path, manifests) + + with dynconfig.Client(node_list, domain) as dynconfig_client: + remote_config = dynconfig.get_remote_config(dynconfig_client) + + if remote_config is None or remote_config != local_config: + new_config = dynconfig.apply_config(dynconfig_client, project_path) + dynconfig.write_local_config(project_path, new_config) + + if dynamic_config_type in ['both', 'proto']: + config_items = cms.get_from_files(project_path) + if config_items is not None: + logger.debug( + f'found {len(config_items)} legacy cms config items, ' + 'need to wait for storage to become ready to apply configs' + ) + # if configs present, then wait for storage + wait_for_storage(api_client, project_path, manifests) - # and apply configs - node_list = get_nodes(api_client, project_path, manifests) - if len(node_list) == 0: - raise RuntimeError('no nodes found, cannot apply legacy cms config items.') - cms.apply_legacy_cms_config_items(config_items, [f'grpc://{i}:2135' for i in node_list]) + # and apply configs + node_list = get_nodes(api_client, project_path, manifests) + if len(node_list) == 0: + raise RuntimeError('no nodes found, cannot apply legacy cms config items.') + cms.apply_legacy_cms_config_items(config_items, [f'grpc://{i}:2135' for i in node_list]) # process databases later for (path, api_version, kind, namespace, name, data) in manifests: @@ -432,7 +458,7 @@ def slice_generate(project_path, user, slice_name, template, template_vars): sys.exit(f'Slice template {template} not implemented.') -def slice_install(project_path, manifests, wait_ready): +def slice_install(project_path, manifests, wait_ready, dynamic_config_type): with api.ApiClient() as api_client: slice_namespace_apply(api_client, project_path, manifests) slice_nodeclaim_apply(api_client, project_path, manifests) @@ -440,15 +466,15 @@ def slice_install(project_path, manifests, wait_ready): slice_ydb_delete(api_client, project_path, manifests) slice_ydb_storage_wait_pods_deleted(api_client, project_path, manifests) slice_nodeclaim_format(api_client, project_path, manifests) - slice_ydb_apply(api_client, project_path, manifests) + slice_ydb_apply(api_client, project_path, manifests, dynamic_config_type) slice_ydb_wait_ready(api_client, project_path, manifests, wait_ready) -def slice_update(project_path, manifests, wait_ready): +def slice_update(project_path, manifests, wait_ready, dynamic_config_type): with api.ApiClient() as api_client: slice_nodeclaim_apply(api_client, project_path, manifests) slice_nodeclaim_wait_ready(api_client, project_path, manifests) - slice_ydb_apply(api_client, project_path, manifests) + slice_ydb_apply(api_client, project_path, manifests, dynamic_config_type) slice_ydb_restart(api_client, project_path, manifests) slice_ydb_wait_ready(api_client, project_path, manifests, wait_ready) @@ -458,9 +484,9 @@ def slice_stop(project_path, manifests): slice_ydb_delete(api_client, project_path, manifests) -def slice_start(project_path, manifests, wait_ready): +def slice_start(project_path, manifests, wait_ready, dynamic_config_type): with api.ApiClient() as api_client: - slice_ydb_apply(api_client, project_path, manifests) + slice_ydb_apply(api_client, project_path, manifests, dynamic_config_type) slice_ydb_wait_ready(api_client, project_path, manifests, wait_ready) @@ -474,12 +500,12 @@ def slice_nodes(project_path, manifests): slice_nodeclaim_nodes(api_client, project_path, manifests) -def slice_format(project_path, manifests, wait_ready): +def slice_format(project_path, manifests, wait_ready, dynamic_config_type): with api.ApiClient() as api_client: slice_ydb_delete(api_client, project_path, manifests) slice_ydb_storage_wait_pods_deleted(api_client, project_path, manifests) slice_nodeclaim_format(api_client, project_path, manifests) - slice_ydb_apply(api_client, project_path, manifests) + slice_ydb_apply(api_client, project_path, manifests, dynamic_config_type) slice_ydb_wait_ready(api_client, project_path, manifests, wait_ready) diff --git a/ydb/tools/ydbd_slice/kube/templates/common/dynconfig.yaml b/ydb/tools/ydbd_slice/kube/templates/common/dynconfig.yaml new file mode 100644 index 000000000000..22c4aa1400c7 --- /dev/null +++ b/ydb/tools/ydbd_slice/kube/templates/common/dynconfig.yaml @@ -0,0 +1,51 @@ +metadata: + kind: MainConfig + cluster: "{{ cluster_uuid }}" # empty by default + version: 0 +config: + yaml_config_enabled: true + table_profiles_config: + table_profiles: + - name: default + compaction_policy: default + execution_policy: default + partitioning_policy: default + storage_policy: default + replication_policy: default + caching_policy: default + compaction_policies: + - name: default + execution_policies: + - name: default + partitioning_policies: + - name: default + auto_split: true + auto_merge: false + size_to_split: 2147483648 + storage_policies: + - name: default + column_families: + - storage_config: + sys_log: + preferred_pool_kind: "{{ preferred_pool_kind }}" # fix this manually and run kube-install or kube-update if you have different pool kind + log: + preferred_pool_kind: "{{ preferred_pool_kind }}" # fix this manually and run kube-install or kube-update if you have different pool kind + data: + preferred_pool_kind: "{{ preferred_pool_kind }}" # fix this manually and run kube-install or kube-update if you have different pool kind + replication_policies: + - name: default + caching_policies: + - name: default + log_config: + sys_log: false + uaclient_config: + uri: "[fd53::1]:16400" + grpc_max_message_size: 4194304 +allowed_labels: + node_id: + type: string + host: + type: string + tenant: + type: string +selector_config: [] diff --git a/ydb/tools/ydbd_slice/ya.make b/ydb/tools/ydbd_slice/ya.make index adc5af24ec45..e57056faf6db 100644 --- a/ydb/tools/ydbd_slice/ya.make +++ b/ydb/tools/ydbd_slice/ya.make @@ -11,6 +11,7 @@ PY_SRCS( kube/api.py kube/cms.py kube/docker.py + kube/dynconfig.py kube/generate.py kube/handlers.py kube/kubectl.py @@ -21,6 +22,8 @@ PY_SRCS( PEERDIR( ydb/tools/cfg + ydb/public/sdk/python + ydb/public/sdk/python/enable_v3_new_behavior contrib/python/PyYAML contrib/python/ruamel.yaml contrib/python/kubernetes @@ -32,6 +35,7 @@ PEERDIR( RESOURCE( kube/templates/common/namespace.yaml /ydbd_slice/templates/common/namespace.yaml kube/templates/common/database.yaml /ydbd_slice/templates/common/database.yaml + kube/templates/common/dynconfig.yaml /ydbd_slice/templates/common/dynconfig.yaml kube/templates/8-node-block-4-2/nodeclaim.yaml /ydbd_slice/templates/8-node-block-4-2/nodeclaim.yaml kube/templates/8-node-block-4-2/storage.yaml /ydbd_slice/templates/8-node-block-4-2/storage.yaml kube/templates/legacy-cms-config-items/table-profile.txt /ydbd_slice/templates/legacy-cms-config-items/table-profile.txt