Skip to content
This repository has been archived by the owner on Mar 29, 2024. It is now read-only.

Commit

Permalink
env: add usage of custom config files
Browse files Browse the repository at this point in the history 
Signed-off-by: Evgeniy Zayats <zayatsevgeniy@nspcc.io>
  • Loading branch information
Evgeniy Zayats committed Feb 29, 2024
1 parent 879ffe3 commit 09f2ce8
Show file tree
Hide file tree
Showing 2 changed files with 76 additions and 23 deletions.
91 changes: 68 additions & 23 deletions src/neofs_testlib/env/env.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -206,7 +206,7 @@ def persist(self) -> str:
def download_binaries(self):
logger.info(f"Going to download missing binaries, if any")
deploy_threads = []

binaries = [
(self.neofs_adm_path, "neofs_adm"),
(self.neofs_cli_path, "neofs_cli"),
Expand All @@ -218,23 +218,23 @@ def download_binaries(self):
(self.neofs_rest_gw_path, "neofs_rest_gw"),
(self.neofs_http_gw_path, "neofs_http_gw"),
]

for binary in binaries:
binary_path, binary_name = binary
if not os.path.isfile(binary_path):
logger.info(f"Will download {binary_name}")
neofs_binary_params = self.neofs_env_config["binaries"][binary_name]
deploy_threads.append(
threading.Thread(
target=NeoFSEnv.download_binary,
args=(
neofs_binary_params["repo"],
neofs_binary_params["version"],
neofs_binary_params["file"],
binary_path
)
)
target=NeoFSEnv.download_binary,
args=(
neofs_binary_params["repo"],
neofs_binary_params["version"],
neofs_binary_params["file"],
binary_path,
),
)
)
else:
logger.info(f"'{binary_name}' already exists, will not be downloaded")

Expand All @@ -253,7 +253,9 @@ def load(cls, persisted_path: str) -> "NeoFSEnv":
@classmethod
def simple(cls, neofs_env_config: dict = None) -> "NeoFSEnv":
if not neofs_env_config:
neofs_env_config = yaml.safe_load(files("neofs_testlib.env.templates").joinpath("neofs_env_config.yaml").read_text())
neofs_env_config = yaml.safe_load(
files("neofs_testlib.env.templates").joinpath("neofs_env_config.yaml").read_text()
)
neofs_env = NeoFSEnv(neofs_env_config=neofs_env_config)
neofs_env.download_binaries()
neofs_env.deploy_inner_ring_node()
Expand All @@ -272,9 +274,14 @@ def simple(cls, neofs_env_config: dict = None) -> "NeoFSEnv":
return neofs_env

@staticmethod
def generate_config_file(config_template: str, config_path: str, **kwargs):
def generate_config_file(config_template: str, config_path: str, custom=False, **kwargs):
jinja_env = jinja2.Environment()
config_template = files("neofs_testlib.env.templates").joinpath(config_template).read_text()
if custom:
config_template = Path(config_template).read_text()
else:
config_template = (
files("neofs_testlib.env.templates").joinpath(config_template).read_text()
)
jinja_template = jinja_env.from_string(config_template)
rendered_config = jinja_template.render(**kwargs)
with open(config_path, mode="w") as fp:
Expand All @@ -294,8 +301,7 @@ def download_binary(repo: str, version: str, file: str, target: str):
resp = requests.get(download_url)
if not resp.ok:
raise AssertionError(
f"Can not download binary from url: {download_url}:
{resp.status_code}/{resp.reason}/{resp.json()}"
f"Can not download binary from url: {download_url}: {resp.status_code}/{resp.reason}/{resp.json()}"
)
with open(target, mode="wb") as binary_file:
binary_file.write(resp.content)
Expand Down Expand Up @@ -357,8 +363,15 @@ def start(self):
if self.process is not None:
raise RuntimeError(f"This inner ring node instance has already been started")
logger.info(f"Generating network config at: {self.network_config}")

network_config_template = "network.yaml"
if self.neofs_env.neofs_env_config["configs_templates"]["network"] != "default":
network_config_template = self.neofs_env.neofs_env_config["configs_templates"][
"network"
]

NeoFSEnv.generate_config_file(
config_template="network.yaml",
config_template=network_config_template,
config_path=self.network_config,
morph_endpoint=self.rpc_address,
alphabet_wallets_path=self.alphabet_wallet.path,
Expand All @@ -369,8 +382,13 @@ def start(self):
WalletType.ALPHABET, self.alphabet_wallet, network_config=self.network_config
)
logger.info(f"Generating IR config at: {self.ir_node_config_path}")

ir_config_template = "ir.yaml"
if self.neofs_env.neofs_env_config["configs_templates"]["ir"] != "default":
ir_config_template = self.neofs_env.neofs_env_config["configs_templates"]["ir"]

NeoFSEnv.generate_config_file(
config_template="ir.yaml",
config_template=ir_config_template,
config_path=self.ir_node_config_path,
wallet=self.alphabet_wallet,
public_key=wallet_utils.get_last_public_key_from_wallet(
Expand Down Expand Up @@ -476,8 +494,13 @@ def start(self, fresh=True):
WalletType.STORAGE, self.wallet, label=f"sn{self.sn_number}"
)
logger.info(f"Generating config for storage node at {self.storage_node_config_path}")

sn_config_template = "sn.yaml"
if self.neofs_env.neofs_env_config["configs_templates"]["sn"] != "default":
sn_config_template = self.neofs_env.neofs_env_config["configs_templates"]["sn"]

NeoFSEnv.generate_config_file(
config_template="sn.yaml",
config_template=sn_config_template,
config_path=self.storage_node_config_path,
morph_endpoint=self.neofs_env.morph_rpc,
shards=self.shards,
Expand Down Expand Up @@ -508,8 +531,13 @@ def delete_data(self):
os.remove(shard.wc_path)
os.remove(self.state_file)
self.shards = [Shard(), Shard()]

sn_config_template = "sn.yaml"
if self.neofs_env.neofs_env_config["configs_templates"]["sn"] != "default":
sn_config_template = self.neofs_env.neofs_env_config["configs_templates"]["sn"]

NeoFSEnv.generate_config_file(
config_template="sn.yaml",
config_template=sn_config_template,
config_path=self.storage_node_config_path,
morph_endpoint=self.neofs_env.morph_rpc,
shards=self.shards,
Expand All @@ -524,8 +552,13 @@ def delete_metadata(self):
for shard in self.shards:
os.remove(shard.metabase_path)
shard.metabase_path = NeoFSEnv._generate_temp_file()

sn_config_template = "sn.yaml"
if self.neofs_env.neofs_env_config["configs_templates"]["sn"] != "default":
sn_config_template = self.neofs_env.neofs_env_config["configs_templates"]["sn"]

NeoFSEnv.generate_config_file(
config_template="sn.yaml",
config_template=sn_config_template,
config_path=self.storage_node_config_path,
morph_endpoint=self.neofs_env.morph_rpc,
shards=self.shards,
Expand Down Expand Up @@ -618,8 +651,12 @@ def _generate_config(self):
with open(self.tls_key_path, mode="w") as fp:
fp.write(tls_key_template)

s3_config_template = "s3.yaml"
if self.neofs_env.neofs_env_config["configs_templates"]["s3"] != "default":
s3_config_template = self.neofs_env.neofs_env_config["configs_templates"]["s3"]

NeoFSEnv.generate_config_file(
config_template="s3.yaml",
config_template=s3_config_template,
config_path=self.config_path,
address=self.address,
cert_file_path=self.tls_cert_path,
Expand Down Expand Up @@ -689,8 +726,12 @@ def start(self):
logger.info(f"Launched HTTP GW: {self}")

def _generate_config(self):
http_config_template = "http.yaml"
if self.neofs_env.neofs_env_config["configs_templates"]["http"] != "default":
http_config_template = self.neofs_env.neofs_env_config["configs_templates"]["http"]

NeoFSEnv.generate_config_file(
config_template="http.yaml",
config_template=http_config_template,
config_path=self.config_path,
address=self.address,
wallet=self.wallet,
Expand Down Expand Up @@ -758,8 +799,12 @@ def start(self):
logger.info(f"Launched REST GW: {self}")

def _generate_config(self):
rest_config_template = "rest.yaml"
if self.neofs_env.neofs_env_config["configs_templates"]["rest"] != "default":
rest_config_template = self.neofs_env.neofs_env_config["configs_templates"]["rest"]

NeoFSEnv.generate_config_file(
config_template="rest.yaml",
config_template=rest_config_template,
config_path=self.config_path,
address=self.address,
wallet=self.wallet,
Expand Down
8 changes: 8 additions & 0 deletions src/neofs_testlib/env/templates/neofs_env_config.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -48,3 +48,11 @@ binaries:
repo: 'nspcc-dev/neo-go'
version: 'v0.104.0'
file: 'neo-go-linux-amd64'

configs_templates:
network: 'default'
ir: 'default'
sn: 'default'
http: 'default'
rest: 'default'
s3: 'default'

0 comments on commit 09f2ce8

Please sign in to comment.