Add functionality to deploy custom neofs environments

pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "neofs-testlib"
-version = "1.1.12"
+version = "1.1.13"
 description = "Building blocks and utilities to facilitate development of automated tests for NeoFS system"
 readme = "README.md"
 authors = [{ name = "NSPCC", email = "[email protected]" }]
@@ -23,6 +23,8 @@ dependencies = [
     "paramiko>=2.10.3",
     "pexpect>=4.8.0",
     "requests>=2.31.0",
+    "jinja2>=3.1.2",
+    "tenacity>=8.2.3",
 ]
 requires-python = ">=3.10"
 
@@ -48,7 +50,7 @@ line-length = 100
 target-version = ["py310"]
 
 [tool.bumpver]
-current_version = "1.1.12"
+current_version = "1.1.13"
 version_pattern = "MAJOR.MINOR.PATCH"
 commit_message  = "Bump version {old_version} -> {new_version}"
 commit          = true

pytest_tests/conftest.py

@@ -0,0 +1,5 @@
+def pytest_addoption(parser):
+    parser.addoption(
+        "--persist-env", action="store_true", default=False, help="persist deployed env"
+    )
+    parser.addoption("--load-env", action="store", help="load persisted env from file")

pytest_tests/container_policy.json

@@ -0,0 +1,3 @@
+{
+    "rep-1": "REP 1"
+}

pytest_tests/pytest.ini

@@ -0,0 +1,7 @@
+[pytest]
+log_cli = 1
+log_cli_level = DEBUG
+log_cli_format = %(asctime)s [%(levelname)4s] %(message)s
+log_format = %(asctime)s [%(levelname)4s] %(message)s
+log_cli_date_format = %Y-%m-%d %H:%M:%S
+log_date_format = %H:%M:%S

pytest_tests/s3_bearer_rules.json

@@ -0,0 +1,89 @@
+{
+    "records":
+    [
+        {
+           "operation":"PUT",
+           "action":"ALLOW",
+           "filters":[],
+           "targets":
+               [
+                   {
+                       "role":"OTHERS",
+                       "keys":[]
+                   }
+               ]
+        },
+	{
+           "operation":"HEAD",
+           "action":"ALLOW",
+           "filters":[],
+           "targets":
+               [
+                   {
+                       "role":"OTHERS",
+                       "keys":[]
+                   }
+               ]
+        },
+	{
+           "operation":"DELETE",
+           "action":"ALLOW",
+           "filters":[],
+           "targets":
+               [
+                   {
+                       "role":"OTHERS",
+                       "keys":[]
+                   }
+               ]
+        },
+        {
+           "operation":"SEARCH",
+           "action":"ALLOW",
+           "filters":[],
+           "targets":
+               [
+                   {
+                       "role":"OTHERS",
+                       "keys":[]
+                   }
+               ]
+         },
+         {
+           "operation":"GET",
+           "action":"ALLOW",
+           "filters":[],
+           "targets":
+               [
+                   {
+                       "role":"OTHERS",
+                       "keys":[]
+                   }
+               ]
+         },
+	{
+           "operation":"GETRANGE",
+           "action":"ALLOW",
+           "filters":[],
+           "targets":
+               [
+                   {
+                       "role":"OTHERS",
+                       "keys":[]
+                   }
+               ]
+        },
+	{
+           "operation":"GETRANGEHASH",
+           "action":"ALLOW",
+           "filters":[],
+           "targets":
+               [
+                   {
+                       "role":"OTHERS",
+                       "keys":[]
+                   }
+               ]
+        }
+    ]
+}

pytest_tests/test_env.py

@@ -0,0 +1,140 @@
+import json
+import os
+import re
+import uuid
+from time import sleep
+
+import boto3
+import pexpect
+import pytest
+from botocore.config import Config
+
+from neofs_testlib.env.env import NeoFSEnv, NodeWallet
+from neofs_testlib.utils.wallet import get_last_public_key_from_wallet, init_wallet
+
+
+def _run_with_passwd(cmd: str, password: str) -> str:
+    child = pexpect.spawn(cmd)
+    child.delaybeforesend = 1
+    child.expect(".*")
+    child.sendline(f"{password}\r")
+    child.wait()
+    cmd = child.read()
+    return cmd.decode()
+
+
+@pytest.fixture
+def neofs_env(request):
+    if request.config.getoption("--load-env"):
+        neofs_env = NeoFSEnv.load(request.config.getoption("--load-env"))
+    else:
+        neofs_env = NeoFSEnv.simple()
+
+    yield neofs_env
+
+    if request.config.getoption("--persist-env"):
+        neofs_env.persist()
+    else:
+        if not request.config.getoption("--load-env"):
+            neofs_env.kill()
+
+
+@pytest.fixture
+def wallet() -> NodeWallet:
+    wallet_name = f"{str(uuid.uuid4())}.json"
+    wallet_path = os.path.join(os.getcwd(), wallet_name)
+    wallet_password = "password"
+    wallet_address = init_wallet(wallet_path, wallet_password)
+    return NodeWallet(path=wallet_path, address=wallet_address, password=wallet_password)
+
+
+@pytest.fixture
+def s3_creds(neofs_env: NeoFSEnv, zero_fee, wallet: NodeWallet) -> tuple:
+    bucket = str(uuid.uuid4())
+    s3_bearer_rules = "pytest_tests/s3_bearer_rules.json"
+
+    gate_public_key = get_last_public_key_from_wallet(
+        neofs_env.s3_gw.wallet.path, neofs_env.s3_gw.wallet.password
+    )
+    cmd = (
+        f"{neofs_env.neofs_s3_authmate_path} --debug --with-log --timeout 1m "
+        f"issue-secret --wallet {wallet.path} --gate-public-key={gate_public_key} "
+        f"--peer {neofs_env.storage_nodes[0].endpoint} --container-friendly-name {bucket} "
+        f"--bearer-rules {s3_bearer_rules} --container-placement-policy 'REP 1' "
+        f"--container-policy container_policy.json"
+    )
+    output = _run_with_passwd(cmd, wallet.password)
+
+    # output contains some debug info and then several JSON structures, so we find each
+    # JSON structure by curly brackets (naive approach, but works while JSON is not nested)
+    # and then we take JSON containing secret_access_key
+    json_blocks = re.findall(r"\{.*?\}", output, re.DOTALL)
+    for json_block in json_blocks:
+        parsed_json_block = json.loads(json_block)
+        if "secret_access_key" in parsed_json_block:
+            return (
+                parsed_json_block["container_id"],
+                bucket,
+                parsed_json_block["access_key_id"],
+                parsed_json_block["secret_access_key"],
+                parsed_json_block["owner_private_key"],
+            )
+    raise AssertionError("Can't get s3 creds")
+
+
+@pytest.fixture
+def zero_fee(neofs_env: NeoFSEnv):
+    neofs_env.neofs_adm().morph.set_config(
+        rpc_endpoint=f"http://{neofs_env.morph_rpc}",
+        alphabet_wallets=neofs_env.alphabet_wallets_dir,
+        post_data=f"ContainerFee=0 ContainerAliasFee=0",
+    )
+
+
+def test_s3_gw_put_get(neofs_env: NeoFSEnv, s3_creds, wallet: NodeWallet):
+    (
+        cid,
+        bucket,
+        access_key_id,
+        secret_access_key,
+        _,
+    ) = s3_creds
+
+    cli = neofs_env.neofs_cli(neofs_env.generate_cli_config(wallet))
+    result = cli.container.list(rpc_endpoint=neofs_env.sn_rpc, wallet=wallet.path)
+    containers_list = result.stdout.split()
+    assert cid in containers_list, f"Expected cid {cid} in {containers_list}"
+
+    session = boto3.Session()
+    config = Config(
+        retries={
+            "max_attempts": 1,
+            "mode": "standard",
+        }
+    )
+
+    s3_client = session.client(
+        service_name="s3",
+        aws_access_key_id=access_key_id,
+        aws_secret_access_key=secret_access_key,
+        config=config,
+        endpoint_url=f"https://{neofs_env.s3_gw.address}",
+        verify=False,
+    )
+
+    bucket_name = str(uuid.uuid4())
+    params = {"Bucket": bucket_name, "CreateBucketConfiguration": {"LocationConstraint": "rep-1"}}
+    s3_client.create_bucket(**params)
+    sleep(5)
+
+    filename = neofs_env._generate_temp_file()
+
+    with open(filename, "w") as file:
+        file.write("123456789")
+
+    with open(filename, "rb") as file:
+        file_content = file.read()
+
+    filekey = os.path.basename(filename)
+    s3_client.put_object(**{"Body": file_content, "Bucket": bucket_name, "Key": filekey})
+    s3_client.get_object(**{"Bucket": bucket_name, "Key": filekey})

requirements.txt

@@ -5,6 +5,11 @@ neo-mamba==2.3.0
 paramiko==2.10.3
 pexpect==4.8.0
 requests==2.31.0
+tenacity
+jinja2
+pyyaml
+pytest
+boto3
 
 # Dev dependencies
 black==22.8.0

src/neofs_testlib/__init__.py

@@ -1 +1 @@
-__version__ = "1.1.12"
+__version__ = "1.1.13"

src/neofs_testlib/cli/neofs_adm/morph.py

@@ -150,7 +150,6 @@ def force_new_epoch(
 
     def generate_alphabet(
         self,
-        rpc_endpoint: str,
         alphabet_wallets: str,
         size: int = 7,
     ) -> CommandResult:
@@ -159,7 +158,6 @@ def generate_alphabet(
         Args:
             alphabet_wallets: Path to alphabet wallets dir.
             size: Amount of alphabet wallets to generate (default 7).
-            rpc_endpoint: N3 RPC node endpoint.
 
         Returns:
             Command's result.
@@ -175,18 +173,18 @@ def generate_alphabet(
 
     def generate_storage_wallet(
         self,
-        rpc_endpoint: str,
         alphabet_wallets: str,
         storage_wallet: str,
+        label: str,
         initial_gas: Optional[str] = None,
     ) -> CommandResult:
         """Generate storage node wallet for the morph network.
 
         Args:
             alphabet_wallets: Path to alphabet wallets dir.
             initial_gas: Initial amount of GAS to transfer.
-            rpc_endpoint: N3 RPC node endpoint.
             storage_wallet: Path to new storage node wallet.
+            label: Wallet label.
 
         Returns:
             Command's result.

src/neofs_testlib/cli/neofs_cli/cli.py

@@ -4,6 +4,7 @@
 from neofs_testlib.cli.neofs_cli.acl import NeofsCliACL
 from neofs_testlib.cli.neofs_cli.bearer import NeofsCliBearer
 from neofs_testlib.cli.neofs_cli.container import NeofsCliContainer
+from neofs_testlib.cli.neofs_cli.control import NeofsCliControl
 from neofs_testlib.cli.neofs_cli.netmap import NeofsCliNetmap
 from neofs_testlib.cli.neofs_cli.object import NeofsCliObject
 from neofs_testlib.cli.neofs_cli.session import NeofsCliSession
@@ -26,6 +27,7 @@ class NeofsCli:
     storagegroup: Optional[NeofsCliStorageGroup] = None
     util: Optional[NeofsCliUtil] = None
     version: Optional[NeofsCliVersion] = None
+    control: Optional[NeofsCliControl] = None
 
     def __init__(self, shell: Shell, neofs_cli_exec_path: str, config_file: Optional[str] = None):
         self.accounting = NeofsCliAccounting(shell, neofs_cli_exec_path, config=config_file)
@@ -39,3 +41,4 @@ def __init__(self, shell: Shell, neofs_cli_exec_path: str, config_file: Optional
         self.storagegroup = NeofsCliStorageGroup(shell, neofs_cli_exec_path, config=config_file)
         self.util = NeofsCliUtil(shell, neofs_cli_exec_path, config=config_file)
         self.version = NeofsCliVersion(shell, neofs_cli_exec_path, config=config_file)
+        self.control = NeofsCliControl(shell, neofs_cli_exec_path, config=config_file)

src/neofs_testlib/cli/neofs_cli/control.py

@@ -0,0 +1,30 @@
+from typing import Optional
+
+from neofs_testlib.cli.cli_command import CliCommand
+from neofs_testlib.shell import CommandResult
+
+
+class NeofsCliControl(CliCommand):
+    def healthcheck(
+        self,
+        endpoint: str,
+        post_data="",
+    ) -> CommandResult:
+        """
+        Get current epoch number.
+
+        Args:
+            address: Address of wallet account.
+            generate_key: Generate new private key.
+            rpc_endpoint: Remote node address (as 'multiaddr' or '<host>:<port>').
+            ttl: TTL value in request meta header (default 2).
+            wallet: Path to the wallet or binary key.
+            xhdr: Dict with request X-Headers.
+
+        Returns:
+            Command's result.
+        """
+        return self._execute(
+            "control healthcheck",
+            **{param: value for param, value in locals().items() if param not in ["self"]},
+        )