From 09f2ce83d5a0d2033e8218464a4be04cad8eea85 Mon Sep 17 00:00:00 2001 From: Evgeniy Zayats Date: Wed, 28 Feb 2024 22:38:52 -0500 Subject: [PATCH] env: add usage of custom config files Signed-off-by: Evgeniy Zayats --- src/neofs_testlib/env/env.py | 91 ++++++++++++++----- .../env/templates/neofs_env_config.yaml | 8 ++ 2 files changed, 76 insertions(+), 23 deletions(-) diff --git a/src/neofs_testlib/env/env.py b/src/neofs_testlib/env/env.py index d267af5..02a5870 100644 --- a/src/neofs_testlib/env/env.py +++ b/src/neofs_testlib/env/env.py @@ -206,7 +206,7 @@ def persist(self) -> str: def download_binaries(self): logger.info(f"Going to download missing binaries, if any") deploy_threads = [] - + binaries = [ (self.neofs_adm_path, "neofs_adm"), (self.neofs_cli_path, "neofs_cli"), @@ -218,7 +218,7 @@ def download_binaries(self): (self.neofs_rest_gw_path, "neofs_rest_gw"), (self.neofs_http_gw_path, "neofs_http_gw"), ] - + for binary in binaries: binary_path, binary_name = binary if not os.path.isfile(binary_path): @@ -226,15 +226,15 @@ def download_binaries(self): neofs_binary_params = self.neofs_env_config["binaries"][binary_name] deploy_threads.append( threading.Thread( - target=NeoFSEnv.download_binary, - args=( - neofs_binary_params["repo"], - neofs_binary_params["version"], - neofs_binary_params["file"], - binary_path - ) - ) + target=NeoFSEnv.download_binary, + args=( + neofs_binary_params["repo"], + neofs_binary_params["version"], + neofs_binary_params["file"], + binary_path, + ), ) + ) else: logger.info(f"'{binary_name}' already exists, will not be downloaded") @@ -253,7 +253,9 @@ def load(cls, persisted_path: str) -> "NeoFSEnv": @classmethod def simple(cls, neofs_env_config: dict = None) -> "NeoFSEnv": if not neofs_env_config: - neofs_env_config = yaml.safe_load(files("neofs_testlib.env.templates").joinpath("neofs_env_config.yaml").read_text()) + neofs_env_config = yaml.safe_load( + files("neofs_testlib.env.templates").joinpath("neofs_env_config.yaml").read_text() + ) neofs_env = NeoFSEnv(neofs_env_config=neofs_env_config) neofs_env.download_binaries() neofs_env.deploy_inner_ring_node() @@ -272,9 +274,14 @@ def simple(cls, neofs_env_config: dict = None) -> "NeoFSEnv": return neofs_env @staticmethod - def generate_config_file(config_template: str, config_path: str, **kwargs): + def generate_config_file(config_template: str, config_path: str, custom=False, **kwargs): jinja_env = jinja2.Environment() - config_template = files("neofs_testlib.env.templates").joinpath(config_template).read_text() + if custom: + config_template = Path(config_template).read_text() + else: + config_template = ( + files("neofs_testlib.env.templates").joinpath(config_template).read_text() + ) jinja_template = jinja_env.from_string(config_template) rendered_config = jinja_template.render(**kwargs) with open(config_path, mode="w") as fp: @@ -294,8 +301,7 @@ def download_binary(repo: str, version: str, file: str, target: str): resp = requests.get(download_url) if not resp.ok: raise AssertionError( - f"Can not download binary from url: {download_url}: - {resp.status_code}/{resp.reason}/{resp.json()}" + f"Can not download binary from url: {download_url}: {resp.status_code}/{resp.reason}/{resp.json()}" ) with open(target, mode="wb") as binary_file: binary_file.write(resp.content) @@ -357,8 +363,15 @@ def start(self): if self.process is not None: raise RuntimeError(f"This inner ring node instance has already been started") logger.info(f"Generating network config at: {self.network_config}") + + network_config_template = "network.yaml" + if self.neofs_env.neofs_env_config["configs_templates"]["network"] != "default": + network_config_template = self.neofs_env.neofs_env_config["configs_templates"][ + "network" + ] + NeoFSEnv.generate_config_file( - config_template="network.yaml", + config_template=network_config_template, config_path=self.network_config, morph_endpoint=self.rpc_address, alphabet_wallets_path=self.alphabet_wallet.path, @@ -369,8 +382,13 @@ def start(self): WalletType.ALPHABET, self.alphabet_wallet, network_config=self.network_config ) logger.info(f"Generating IR config at: {self.ir_node_config_path}") + + ir_config_template = "ir.yaml" + if self.neofs_env.neofs_env_config["configs_templates"]["ir"] != "default": + ir_config_template = self.neofs_env.neofs_env_config["configs_templates"]["ir"] + NeoFSEnv.generate_config_file( - config_template="ir.yaml", + config_template=ir_config_template, config_path=self.ir_node_config_path, wallet=self.alphabet_wallet, public_key=wallet_utils.get_last_public_key_from_wallet( @@ -476,8 +494,13 @@ def start(self, fresh=True): WalletType.STORAGE, self.wallet, label=f"sn{self.sn_number}" ) logger.info(f"Generating config for storage node at {self.storage_node_config_path}") + + sn_config_template = "sn.yaml" + if self.neofs_env.neofs_env_config["configs_templates"]["sn"] != "default": + sn_config_template = self.neofs_env.neofs_env_config["configs_templates"]["sn"] + NeoFSEnv.generate_config_file( - config_template="sn.yaml", + config_template=sn_config_template, config_path=self.storage_node_config_path, morph_endpoint=self.neofs_env.morph_rpc, shards=self.shards, @@ -508,8 +531,13 @@ def delete_data(self): os.remove(shard.wc_path) os.remove(self.state_file) self.shards = [Shard(), Shard()] + + sn_config_template = "sn.yaml" + if self.neofs_env.neofs_env_config["configs_templates"]["sn"] != "default": + sn_config_template = self.neofs_env.neofs_env_config["configs_templates"]["sn"] + NeoFSEnv.generate_config_file( - config_template="sn.yaml", + config_template=sn_config_template, config_path=self.storage_node_config_path, morph_endpoint=self.neofs_env.morph_rpc, shards=self.shards, @@ -524,8 +552,13 @@ def delete_metadata(self): for shard in self.shards: os.remove(shard.metabase_path) shard.metabase_path = NeoFSEnv._generate_temp_file() + + sn_config_template = "sn.yaml" + if self.neofs_env.neofs_env_config["configs_templates"]["sn"] != "default": + sn_config_template = self.neofs_env.neofs_env_config["configs_templates"]["sn"] + NeoFSEnv.generate_config_file( - config_template="sn.yaml", + config_template=sn_config_template, config_path=self.storage_node_config_path, morph_endpoint=self.neofs_env.morph_rpc, shards=self.shards, @@ -618,8 +651,12 @@ def _generate_config(self): with open(self.tls_key_path, mode="w") as fp: fp.write(tls_key_template) + s3_config_template = "s3.yaml" + if self.neofs_env.neofs_env_config["configs_templates"]["s3"] != "default": + s3_config_template = self.neofs_env.neofs_env_config["configs_templates"]["s3"] + NeoFSEnv.generate_config_file( - config_template="s3.yaml", + config_template=s3_config_template, config_path=self.config_path, address=self.address, cert_file_path=self.tls_cert_path, @@ -689,8 +726,12 @@ def start(self): logger.info(f"Launched HTTP GW: {self}") def _generate_config(self): + http_config_template = "http.yaml" + if self.neofs_env.neofs_env_config["configs_templates"]["http"] != "default": + http_config_template = self.neofs_env.neofs_env_config["configs_templates"]["http"] + NeoFSEnv.generate_config_file( - config_template="http.yaml", + config_template=http_config_template, config_path=self.config_path, address=self.address, wallet=self.wallet, @@ -758,8 +799,12 @@ def start(self): logger.info(f"Launched REST GW: {self}") def _generate_config(self): + rest_config_template = "rest.yaml" + if self.neofs_env.neofs_env_config["configs_templates"]["rest"] != "default": + rest_config_template = self.neofs_env.neofs_env_config["configs_templates"]["rest"] + NeoFSEnv.generate_config_file( - config_template="rest.yaml", + config_template=rest_config_template, config_path=self.config_path, address=self.address, wallet=self.wallet, diff --git a/src/neofs_testlib/env/templates/neofs_env_config.yaml b/src/neofs_testlib/env/templates/neofs_env_config.yaml index 6d8b8c7..4269d7a 100644 --- a/src/neofs_testlib/env/templates/neofs_env_config.yaml +++ b/src/neofs_testlib/env/templates/neofs_env_config.yaml @@ -48,3 +48,11 @@ binaries: repo: 'nspcc-dev/neo-go' version: 'v0.104.0' file: 'neo-go-linux-amd64' + +configs_templates: + network: 'default' + ir: 'default' + sn: 'default' + http: 'default' + rest: 'default' + s3: 'default'