From fd922b624a1bbf4799f20438c053d4887708b1fd Mon Sep 17 00:00:00 2001 From: Evgeniy Zayats Date: Thu, 26 Dec 2024 20:48:20 +0300 Subject: [PATCH 1/2] tests: test weird filename for presigned urls closes #913 Signed-off-by: Evgeniy Zayats --- pytest_tests/lib/s3/s3_object.py | 5 +- pytest_tests/tests/s3/test_s3_presigned.py | 74 ++++++++++------------ 2 files changed, 38 insertions(+), 41 deletions(-) diff --git a/pytest_tests/lib/s3/s3_object.py b/pytest_tests/lib/s3/s3_object.py index d2ecbffbb..77d70bf1a 100644 --- a/pytest_tests/lib/s3/s3_object.py +++ b/pytest_tests/lib/s3/s3_object.py @@ -99,8 +99,9 @@ def list_objects_delete_markers_s3(s3_client, bucket: str, full_output: bool = F @allure.step("Put object S3") -def put_object_s3(s3_client, bucket: str, filepath: str, **kwargs): - filename = os.path.basename(filepath) +def put_object_s3(s3_client, bucket: str, filepath: str, filename: str = "", **kwargs): + if not filename: + filename = os.path.basename(filepath) if isinstance(s3_client, AwsCliClient): file_content = filepath diff --git a/pytest_tests/tests/s3/test_s3_presigned.py b/pytest_tests/tests/s3/test_s3_presigned.py index 65e57f071..93c2d7c55 100644 --- a/pytest_tests/tests/s3/test_s3_presigned.py +++ b/pytest_tests/tests/s3/test_s3_presigned.py @@ -1,4 +1,3 @@ -import json import logging import os import shutil @@ -9,8 +8,6 @@ import requests from helpers.common import TEST_FILES_DIR, get_assets_dir_path from helpers.file_helper import generate_file, get_file_hash -from helpers.s3_helper import object_key_from_file_path -from neofs_testlib.cli import NeofsAuthmate from s3 import s3_object from s3.s3_base import TestNeofsS3Base @@ -19,37 +16,36 @@ def pytest_generate_tests(metafunc): if "s3_client" in metafunc.fixturenames: - metafunc.parametrize("s3_client", ["aws cli", "boto3"], indirect=True) + metafunc.parametrize("s3_client", ["boto3"], indirect=True) class TestS3Presigned(TestNeofsS3Base): @allure.title("Test S3: Get Object With Presigned Url") - @pytest.mark.parametrize("url_from", ["neofs_authmate", "s3"]) + @pytest.mark.parametrize("url_from", ["s3"]) def test_s3_get_object_with_presigned_url(self, bucket, simple_object_size, url_from: str): file_path = generate_file(simple_object_size) - file_name = object_key_from_file_path(file_path) + file_names_to_check = [ + "temp_file_12345", + "%40hashed/4e/07/temp_file_12345", + "\\inter\\stingfile", + ">cool<>name<", + "&cool&&name&", + "cool\\/name\\", + "||cool||name||", + ":cool::name", + "@cool@name@", + ";cool;name;", + ";cool,name!", + ] - with allure.step("Put object into Bucket"): - s3_object.put_object_s3(self.s3_client, bucket, file_path) + if self.neofs_env.s3_gw._get_version() <= "0.33.0": + file_names_to_check = ["temp_file_12345"] - with allure.step(f"Get presigned URL from {url_from}"): - if url_from == "neofs_authmate": - neofs_authmate = NeofsAuthmate( - shell=self.shell, neofs_authmate_exec_path=self.neofs_env.neofs_s3_authmate_path - ) - raw_url = json.loads( - neofs_authmate.presigned.generate_presigned_url( - endpoint=f"https://{self.neofs_env.s3_gw.address}", - method="GET", - bucket=bucket, - object=file_name, - lifetime="30s", - aws_secret_access_key=self.secret_access_key, - aws_access_key_id=self.access_key_id, - ).stdout - ) - presigned_url = raw_url["URL"] - else: + for file_name in file_names_to_check: + with allure.step("Put object into Bucket"): + s3_object.put_object_s3(self.s3_client, bucket, file_path, file_name) + + with allure.step(f"Get presigned URL from {url_from}"): presigned_url = self.s3_client.generate_presigned_url( ClientMethod="get_object", Params={"Bucket": bucket, "Key": file_name}, @@ -58,19 +54,19 @@ def test_s3_get_object_with_presigned_url(self, bucket, simple_object_size, url_ ).strip() logger.info(f"Presigned URL: {presigned_url}") - with allure.step("Get object with generated presigned url"): - resp = requests.get(presigned_url, stream=True, timeout=30, verify=False) + with allure.step("Get object with generated presigned url"): + resp = requests.get(presigned_url, stream=True, timeout=30, verify=False) - if not resp.ok: - raise Exception( - f"""Failed to get object via presigned url: - request: {resp.request.path_url}, - response: {resp.text}, - status code: {resp.status_code} {resp.reason}""" - ) + if not resp.ok: + raise Exception( + f"""Failed to get object via presigned url: + request: {resp.request.path_url}, + response: {resp.text}, + status code: {resp.status_code} {resp.reason}""" + ) - new_file_path = os.path.join(get_assets_dir_path(), TEST_FILES_DIR, f"temp_file_{uuid.uuid4()}") - with open(new_file_path, "wb") as file: - shutil.copyfileobj(resp.raw, file) + new_file_path = os.path.join(get_assets_dir_path(), TEST_FILES_DIR, f"temp_file_{uuid.uuid4()}") + with open(new_file_path, "wb") as file: + shutil.copyfileobj(resp.raw, file) - assert get_file_hash(file_path) == get_file_hash(new_file_path), "Files hashes are different" + assert get_file_hash(file_path) == get_file_hash(new_file_path), "Files hashes are different" From e1602105e0c78f98d0787e49cff9aea2c2eb3632 Mon Sep 17 00:00:00 2001 From: Evgeniy Zayats Date: Thu, 26 Dec 2024 20:50:15 +0300 Subject: [PATCH 2/2] lib: drop authmate presigned URL support closes #914 Signed-off-by: Evgeniy Zayats --- .../cli/neofs_authmate/authmate.py | 3 -- .../cli/neofs_authmate/presigned.py | 33 ------------------- 2 files changed, 36 deletions(-) delete mode 100644 neofs-testlib/neofs_testlib/cli/neofs_authmate/presigned.py diff --git a/neofs-testlib/neofs_testlib/cli/neofs_authmate/authmate.py b/neofs-testlib/neofs_testlib/cli/neofs_authmate/authmate.py index 2a45efe9f..5f86a74e7 100644 --- a/neofs-testlib/neofs_testlib/cli/neofs_authmate/authmate.py +++ b/neofs-testlib/neofs_testlib/cli/neofs_authmate/authmate.py @@ -2,16 +2,13 @@ from neofs_testlib.cli.neofs_authmate.secret import NeofsAuthmateSecret from neofs_testlib.cli.neofs_authmate.version import NeofsAuthmateVersion -from neofs_testlib.cli.neofs_authmate.presigned import NeofsAuthmatePresigned from neofs_testlib.shell import Shell class NeofsAuthmate: secret: Optional[NeofsAuthmateSecret] = None version: Optional[NeofsAuthmateVersion] = None - presigned: Optional[NeofsAuthmatePresigned] = None def __init__(self, shell: Shell, neofs_authmate_exec_path: str): self.secret = NeofsAuthmateSecret(shell, neofs_authmate_exec_path) self.version = NeofsAuthmateVersion(shell, neofs_authmate_exec_path) - self.presigned = NeofsAuthmatePresigned(shell, neofs_authmate_exec_path) diff --git a/neofs-testlib/neofs_testlib/cli/neofs_authmate/presigned.py b/neofs-testlib/neofs_testlib/cli/neofs_authmate/presigned.py deleted file mode 100644 index 0bf7fc1c7..000000000 --- a/neofs-testlib/neofs_testlib/cli/neofs_authmate/presigned.py +++ /dev/null @@ -1,33 +0,0 @@ -from neofs_testlib.cli.cli_command import CliCommand -from neofs_testlib.shell import CommandResult - - -class NeofsAuthmatePresigned(CliCommand): - def generate_presigned_url( - self, - endpoint: str, - method: str, - bucket: str, - object: str, - lifetime: str, - aws_secret_access_key: str, - aws_access_key_id: str, - ) -> CommandResult: - """Generate presigned URL - - Args: - endpoint: Endpoint of s3-gw - method: HTTP method to perform action - bucket: Bucket name to perform action - object: Object name to perform action - lifetime: Lifetime of presigned URL - aws-access-key-id: AWS access key id to sign the URL - aws-secret-access-key: AWS access secret access key to sign the URL - - Returns: - Command's result. - """ - return self._execute( - "generate-presigned-url", - **{param: param_value for param, param_value in locals().items() if param not in ["self"]}, - )