diff --git a/src/neofs_testlib/env/env.py b/src/neofs_testlib/env/env.py index d267af5..b3e4e50 100644 --- a/src/neofs_testlib/env/env.py +++ b/src/neofs_testlib/env/env.py @@ -4,6 +4,7 @@ import pickle import random import socket +import stat import string import subprocess import threading @@ -206,7 +207,7 @@ def persist(self) -> str: def download_binaries(self): logger.info(f"Going to download missing binaries, if any") deploy_threads = [] - + binaries = [ (self.neofs_adm_path, "neofs_adm"), (self.neofs_cli_path, "neofs_cli"), @@ -218,7 +219,7 @@ def download_binaries(self): (self.neofs_rest_gw_path, "neofs_rest_gw"), (self.neofs_http_gw_path, "neofs_http_gw"), ] - + for binary in binaries: binary_path, binary_name = binary if not os.path.isfile(binary_path): @@ -226,15 +227,15 @@ def download_binaries(self): neofs_binary_params = self.neofs_env_config["binaries"][binary_name] deploy_threads.append( threading.Thread( - target=NeoFSEnv.download_binary, - args=( - neofs_binary_params["repo"], - neofs_binary_params["version"], - neofs_binary_params["file"], - binary_path - ) - ) + target=NeoFSEnv.download_binary, + args=( + neofs_binary_params["repo"], + neofs_binary_params["version"], + neofs_binary_params["file"], + binary_path, + ), ) + ) else: logger.info(f"'{binary_name}' already exists, will not be downloaded") @@ -253,7 +254,9 @@ def load(cls, persisted_path: str) -> "NeoFSEnv": @classmethod def simple(cls, neofs_env_config: dict = None) -> "NeoFSEnv": if not neofs_env_config: - neofs_env_config = yaml.safe_load(files("neofs_testlib.env.templates").joinpath("neofs_env_config.yaml").read_text()) + neofs_env_config = yaml.safe_load( + files("neofs_testlib.env.templates").joinpath("neofs_env_config.yaml").read_text() + ) neofs_env = NeoFSEnv(neofs_env_config=neofs_env_config) neofs_env.download_binaries() neofs_env.deploy_inner_ring_node() @@ -272,9 +275,14 @@ def simple(cls, neofs_env_config: dict = None) -> "NeoFSEnv": return neofs_env @staticmethod - def generate_config_file(config_template: str, config_path: str, **kwargs): + def generate_config_file(config_template: str, config_path: str, custom=False, **kwargs): jinja_env = jinja2.Environment() - config_template = files("neofs_testlib.env.templates").joinpath(config_template).read_text() + if custom: + config_template = Path(config_template).read_text() + else: + config_template = ( + files("neofs_testlib.env.templates").joinpath(config_template).read_text() + ) jinja_template = jinja_env.from_string(config_template) rendered_config = jinja_template.render(**kwargs) with open(config_path, mode="w") as fp: @@ -294,11 +302,13 @@ def download_binary(repo: str, version: str, file: str, target: str): resp = requests.get(download_url) if not resp.ok: raise AssertionError( - f"Can not download binary from url: {download_url}: - {resp.status_code}/{resp.reason}/{resp.json()}" + f"Can not download binary from url: {download_url}: {resp.status_code}/{resp.reason}/{resp.json()}" ) with open(target, mode="wb") as binary_file: binary_file.write(resp.content) + # make binary executable + current_perm = os.stat(target) + os.chmod(target, current_perm.st_mode | stat.S_IEXEC) @staticmethod def _generate_temp_file(extension: str = "") -> str: @@ -357,8 +367,15 @@ def start(self): if self.process is not None: raise RuntimeError(f"This inner ring node instance has already been started") logger.info(f"Generating network config at: {self.network_config}") + + network_config_template = "network.yaml" + if self.neofs_env.neofs_env_config["configs_templates"]["network"] != "default": + network_config_template = self.neofs_env.neofs_env_config["configs_templates"][ + "network" + ] + NeoFSEnv.generate_config_file( - config_template="network.yaml", + config_template=network_config_template, config_path=self.network_config, morph_endpoint=self.rpc_address, alphabet_wallets_path=self.alphabet_wallet.path, @@ -369,8 +386,13 @@ def start(self): WalletType.ALPHABET, self.alphabet_wallet, network_config=self.network_config ) logger.info(f"Generating IR config at: {self.ir_node_config_path}") + + ir_config_template = "ir.yaml" + if self.neofs_env.neofs_env_config["configs_templates"]["ir"] != "default": + ir_config_template = self.neofs_env.neofs_env_config["configs_templates"]["ir"] + NeoFSEnv.generate_config_file( - config_template="ir.yaml", + config_template=ir_config_template, config_path=self.ir_node_config_path, wallet=self.alphabet_wallet, public_key=wallet_utils.get_last_public_key_from_wallet( @@ -476,8 +498,13 @@ def start(self, fresh=True): WalletType.STORAGE, self.wallet, label=f"sn{self.sn_number}" ) logger.info(f"Generating config for storage node at {self.storage_node_config_path}") + + sn_config_template = "sn.yaml" + if self.neofs_env.neofs_env_config["configs_templates"]["sn"] != "default": + sn_config_template = self.neofs_env.neofs_env_config["configs_templates"]["sn"] + NeoFSEnv.generate_config_file( - config_template="sn.yaml", + config_template=sn_config_template, config_path=self.storage_node_config_path, morph_endpoint=self.neofs_env.morph_rpc, shards=self.shards, @@ -508,8 +535,13 @@ def delete_data(self): os.remove(shard.wc_path) os.remove(self.state_file) self.shards = [Shard(), Shard()] + + sn_config_template = "sn.yaml" + if self.neofs_env.neofs_env_config["configs_templates"]["sn"] != "default": + sn_config_template = self.neofs_env.neofs_env_config["configs_templates"]["sn"] + NeoFSEnv.generate_config_file( - config_template="sn.yaml", + config_template=sn_config_template, config_path=self.storage_node_config_path, morph_endpoint=self.neofs_env.morph_rpc, shards=self.shards, @@ -524,8 +556,13 @@ def delete_metadata(self): for shard in self.shards: os.remove(shard.metabase_path) shard.metabase_path = NeoFSEnv._generate_temp_file() + + sn_config_template = "sn.yaml" + if self.neofs_env.neofs_env_config["configs_templates"]["sn"] != "default": + sn_config_template = self.neofs_env.neofs_env_config["configs_templates"]["sn"] + NeoFSEnv.generate_config_file( - config_template="sn.yaml", + config_template=sn_config_template, config_path=self.storage_node_config_path, morph_endpoint=self.neofs_env.morph_rpc, shards=self.shards, @@ -618,8 +655,12 @@ def _generate_config(self): with open(self.tls_key_path, mode="w") as fp: fp.write(tls_key_template) + s3_config_template = "s3.yaml" + if self.neofs_env.neofs_env_config["configs_templates"]["s3"] != "default": + s3_config_template = self.neofs_env.neofs_env_config["configs_templates"]["s3"] + NeoFSEnv.generate_config_file( - config_template="s3.yaml", + config_template=s3_config_template, config_path=self.config_path, address=self.address, cert_file_path=self.tls_cert_path, @@ -689,8 +730,12 @@ def start(self): logger.info(f"Launched HTTP GW: {self}") def _generate_config(self): + http_config_template = "http.yaml" + if self.neofs_env.neofs_env_config["configs_templates"]["http"] != "default": + http_config_template = self.neofs_env.neofs_env_config["configs_templates"]["http"] + NeoFSEnv.generate_config_file( - config_template="http.yaml", + config_template=http_config_template, config_path=self.config_path, address=self.address, wallet=self.wallet, @@ -758,8 +803,12 @@ def start(self): logger.info(f"Launched REST GW: {self}") def _generate_config(self): + rest_config_template = "rest.yaml" + if self.neofs_env.neofs_env_config["configs_templates"]["rest"] != "default": + rest_config_template = self.neofs_env.neofs_env_config["configs_templates"]["rest"] + NeoFSEnv.generate_config_file( - config_template="rest.yaml", + config_template=rest_config_template, config_path=self.config_path, address=self.address, wallet=self.wallet, diff --git a/src/neofs_testlib/env/templates/neofs_env_config.yaml b/src/neofs_testlib/env/templates/neofs_env_config.yaml index 6d8b8c7..055e53c 100644 --- a/src/neofs_testlib/env/templates/neofs_env_config.yaml +++ b/src/neofs_testlib/env/templates/neofs_env_config.yaml @@ -29,7 +29,7 @@ binaries: version: 'v0.29.0' file: 'neofs-s3-gw-linux-amd64' - neofs_s3_gw_authmate: + neofs_s3_authmate: repo: 'nspcc-dev/neofs-s3-gw' version: 'v0.29.0' file: 'neofs-s3-authmate-linux-amd64' @@ -48,3 +48,11 @@ binaries: repo: 'nspcc-dev/neo-go' version: 'v0.104.0' file: 'neo-go-linux-amd64' + +configs_templates: + network: 'default' + ir: 'default' + sn: 'default' + http: 'default' + rest: 'default' + s3: 'default'