From 88ce0bb1605b02ef5c5fba29de2630d8e8f71429 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=BDiga=20Luk=C5=A1i=C4=8D?= <31988337+zigaLuksic@users.noreply.github.com> Date: Mon, 29 Jan 2024 13:07:57 +0100 Subject: [PATCH] Adjust moto and lazy fixture after recent updates (#786) * general changes * fix remaining problems --- pyproject.toml | 3 +- tests/core/conftest.py | 27 ----------- tests/core/test_eodata.py | 7 ++- tests/core/test_eodata_io.py | 80 ++++++++++++++++++++------------ tests/core/test_utils/test_fs.py | 26 +++++++++-- 5 files changed, 79 insertions(+), 64 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index eb3e28cd..53e8c346 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -79,12 +79,11 @@ dev = [ "build", "eo-learn[FULL]", "hypothesis", - "moto", + "moto[s3]>=5.0.0", "mypy>=0.990", "pylint>=2.14.0", "pytest>=7.0.0", "pytest-cov", - "pytest-lazy-fixture", "pytest-mock", "twine", "types-python-dateutil", diff --git a/tests/core/conftest.py b/tests/core/conftest.py index 62179c9b..a382904c 100644 --- a/tests/core/conftest.py +++ b/tests/core/conftest.py @@ -10,12 +10,8 @@ from __future__ import annotations import os -from typing import Callable -import boto3 import pytest -from fs_s3fs import S3FS -from moto import mock_s3 from eolearn.core import EOPatch @@ -28,26 +24,3 @@ def test_eopatch_path_fixture() -> str: @pytest.fixture(name="test_eopatch") def test_eopatch_fixture(test_eopatch_path) -> EOPatch: return EOPatch.load(test_eopatch_path) - - -@pytest.fixture(name="create_mocked_s3fs", scope="session") -def s3_mocking_fixture() -> Callable[[str], S3FS]: - """Provides a function for mocking S3 buckets""" - - @mock_s3 - def create_mocked_s3fs(bucket_name: str = "mocked-test-bucket") -> S3FS: - """Creates a new empty mocked s3 bucket. If one such bucket already exists it deletes it first.""" - s3resource = boto3.resource("s3", region_name="eu-central-1") - - bucket = s3resource.Bucket(bucket_name) - - if bucket.creation_date: # If bucket already exists - for key in bucket.objects.all(): - key.delete() - bucket.delete() - - s3resource.create_bucket(Bucket=bucket_name, CreateBucketConfiguration={"LocationConstraint": "eu-central-1"}) - - return S3FS(bucket_name=bucket_name) - - return create_mocked_s3fs diff --git a/tests/core/test_eodata.py b/tests/core/test_eodata.py index c91cbe08..6a3321a3 100644 --- a/tests/core/test_eodata.py +++ b/tests/core/test_eodata.py @@ -382,7 +382,12 @@ def test_get_spatial_dimension(feature: Feature, expected_dim: tuple[int, int], ("patch", "expected_features"), [ ( - pytest.lazy_fixture("mini_eopatch"), + generate_eopatch({ + FeatureType.DATA: ["A", "B"], + FeatureType.MASK: ["C", "D"], + FeatureType.MASK_TIMELESS: ["E"], + FeatureType.META_INFO: ["beep"], + }), [ (FeatureType.DATA, "A"), (FeatureType.DATA, "B"), diff --git a/tests/core/test_eodata_io.py b/tests/core/test_eodata_io.py index 61a3b7cc..be5e4671 100644 --- a/tests/core/test_eodata_io.py +++ b/tests/core/test_eodata_io.py @@ -13,6 +13,7 @@ import warnings from typing import Any +import boto3 import fs import geopandas as gpd import numpy as np @@ -20,8 +21,9 @@ from fs.base import FS from fs.errors import CreateFailed, ResourceNotFound from fs.tempfs import TempFS +from fs_s3fs import S3FS from geopandas import GeoDataFrame -from moto import mock_s3 +from moto import mock_aws from numpy.testing import assert_array_equal from shapely.geometry import Point @@ -42,7 +44,25 @@ from eolearn.core.utils.parsing import FeatureParser from eolearn.core.utils.testing import assert_feature_data_equal, generate_eopatch -FS_LOADERS = [TempFS, pytest.lazy_fixture("create_mocked_s3fs")] + +@mock_aws +def create_mocked_s3fs(bucket_name: str = "mocked-test-bucket") -> S3FS: + """Creates a new empty mocked s3 bucket. If one such bucket already exists it deletes it first.""" + s3resource = boto3.resource("s3", region_name="eu-central-1") + + bucket = s3resource.Bucket(bucket_name) + + if bucket.creation_date: # If bucket already exists + for key in bucket.objects.all(): + key.delete() + bucket.delete() + + s3resource.create_bucket(Bucket=bucket_name, CreateBucketConfiguration={"LocationConstraint": "eu-central-1"}) + + return S3FS(bucket_name=bucket_name) + + +FS_LOADERS = [TempFS, create_mocked_s3fs] DUMMY_BBOX = BBox((0, 0, 1, 1), CRS.WGS84) @@ -88,7 +108,7 @@ def eopatch_fixture(): return eopatch -@mock_s3 +@mock_aws @pytest.mark.parametrize("fs_loader", FS_LOADERS) @pytest.mark.parametrize("use_zarr", [True, False]) def test_saving_in_empty_folder(eopatch, fs_loader, use_zarr: bool): @@ -105,7 +125,7 @@ def test_saving_in_empty_folder(eopatch, fs_loader, use_zarr: bool): assert temp_fs.exists(f"/{subfolder}/bbox.geojson") -@mock_s3 +@mock_aws @pytest.mark.parametrize("fs_loader", FS_LOADERS) @pytest.mark.parametrize("use_zarr", [True, False]) @pytest.mark.usefixtures("_silence_warnings") @@ -126,7 +146,7 @@ def test_saving_in_non_empty_folder(eopatch, fs_loader, use_zarr: bool): assert not temp_fs.exists(empty_file) -@mock_s3 +@mock_aws @pytest.mark.parametrize("fs_loader", FS_LOADERS) @pytest.mark.parametrize("use_zarr", [True, False]) @pytest.mark.usefixtures("_silence_warnings") @@ -153,7 +173,7 @@ def test_overwriting_non_empty_folder(eopatch, fs_loader, use_zarr: bool): assert new_eopatch == merge_eopatches(eopatch, add_eopatch) -@mock_s3 +@mock_aws @pytest.mark.parametrize("fs_loader", FS_LOADERS) @pytest.mark.parametrize("use_zarr", [True, False]) @pytest.mark.parametrize( @@ -188,7 +208,7 @@ def test_save_load_partial( assert feature not in loaded_eopatch -@mock_s3 +@mock_aws @pytest.mark.parametrize("fs_loader", FS_LOADERS) @pytest.mark.parametrize("use_zarr", [True, False]) def test_save_add_only_features(eopatch, fs_loader, use_zarr: bool): @@ -211,7 +231,7 @@ def test_save_add_only_features(eopatch, fs_loader, use_zarr: bool): ) -@mock_s3 +@mock_aws @pytest.mark.parametrize("fs_loader", FS_LOADERS) def test_bbox_always_saved(eopatch, fs_loader): with fs_loader() as temp_fs: @@ -219,7 +239,7 @@ def test_bbox_always_saved(eopatch, fs_loader): assert temp_fs.exists("/bbox.geojson") -@mock_s3 +@mock_aws @pytest.mark.parametrize("fs_loader", FS_LOADERS) @pytest.mark.parametrize( ("save_timestamps", "features", "should_save"), @@ -249,7 +269,7 @@ def test_auto_save_load_timestamps(eopatch): assert EOPatch.load("/", filesystem=temp_fs).timestamps is not None -@mock_s3 +@mock_aws @pytest.mark.parametrize("fs_loader", FS_LOADERS) @pytest.mark.parametrize( ("load_timestamps", "features", "should_load"), @@ -282,7 +302,7 @@ def test_load_timestamps_when_nonexistant(eopatch, features): loaded_patch = EOPatch.load("/", filesystem=temp_fs, features=features, load_timestamps=True) -@mock_s3 +@mock_aws @pytest.mark.parametrize("fs_loader", FS_LOADERS) @pytest.mark.parametrize("use_zarr", [True, False]) def test_temporally_empty_patch_io(fs_loader, use_zarr: bool): @@ -293,7 +313,7 @@ def test_temporally_empty_patch_io(fs_loader, use_zarr: bool): assert eopatch == EOPatch.load("/", filesystem=temp_fs) -@mock_s3 +@mock_aws @pytest.mark.parametrize("fs_loader", FS_LOADERS) @pytest.mark.parametrize("use_zarr", [True, False]) @pytest.mark.usefixtures("_silence_warnings") @@ -326,7 +346,7 @@ def test_overwrite_failure(fs_loader, use_zarr: bool): ) -@mock_s3 +@mock_aws @pytest.mark.parametrize("fs_loader", FS_LOADERS) @pytest.mark.parametrize("compress_level", [0, 1]) def test_compression_deprecation(eopatch, fs_loader, compress_level: int | None): @@ -338,14 +358,14 @@ def test_compression_deprecation(eopatch, fs_loader, compress_level: int | None) with fs_loader() as temp_fs: SaveTask(folder, filesystem=temp_fs) - with fs_loader() as temp_fs, pytest.warns(EODeprecationWarning): - SaveTask(folder, filesystem=temp_fs, compress_level=compress_level) + with fs_loader() as temp_fs, pytest.warns(EODeprecationWarning): + SaveTask(folder, filesystem=temp_fs, compress_level=compress_level) - with fs_loader() as temp_fs, pytest.warns(EODeprecationWarning): - eopatch.save(folder, filesystem=temp_fs, compress_level=compress_level) + with fs_loader() as temp_fs, pytest.warns(EODeprecationWarning): + eopatch.save(folder, filesystem=temp_fs, compress_level=compress_level) -@mock_s3 +@mock_aws @pytest.mark.parametrize("fs_loader", FS_LOADERS) @pytest.mark.parametrize("use_zarr", [True, False]) def test_save_and_load_tasks(eopatch, fs_loader, use_zarr: bool): @@ -367,7 +387,7 @@ def test_save_and_load_tasks(eopatch, fs_loader, use_zarr: bool): assert eop == eopatch -@mock_s3 +@mock_aws @pytest.mark.parametrize("fs_loader", FS_LOADERS) def test_fail_saving_nonexistent_feature(eopatch, fs_loader): features = [(FeatureType.DATA, "nonexistent")] @@ -375,7 +395,7 @@ def test_fail_saving_nonexistent_feature(eopatch, fs_loader): eopatch.save("/", filesystem=temp_fs, features=features) -@mock_s3 +@mock_aws @pytest.mark.parametrize("fs_loader", FS_LOADERS) def test_fail_loading_nonexistent_feature(fs_loader): for features in [[(FeatureType.DATA, "nonexistent")], [(FeatureType.META_INFO, "nonexistent")]]: @@ -383,7 +403,7 @@ def test_fail_loading_nonexistent_feature(fs_loader): EOPatch.load("/", filesystem=temp_fs, features=features) -@mock_s3 +@mock_aws @pytest.mark.parametrize("fs_loader", FS_LOADERS) @pytest.mark.parametrize("use_zarr", [True, False]) def test_nonexistent_location(fs_loader, use_zarr: bool): @@ -419,7 +439,7 @@ def test_nonexistent_location(fs_loader, use_zarr: bool): assert os.path.exists(full_path) -@mock_s3 +@mock_aws @pytest.mark.parametrize("fs_loader", FS_LOADERS) def test_cleanup_different_compression(fs_loader, eopatch): folder = "foo-folder" @@ -453,7 +473,7 @@ def test_cleanup_different_compression(fs_loader, eopatch): assert not temp_fs.exists(mask_timeless_path) -@mock_s3 +@mock_aws @pytest.mark.parametrize("fs_loader", FS_LOADERS) @pytest.mark.parametrize("use_zarr", [True, False]) @pytest.mark.parametrize("folder_name", ["/", "foo", "foo/bar"]) @@ -563,7 +583,7 @@ def test_zarr_and_numpy_combined_loading(eopatch): assert EOPatch.load("/", filesystem=temp_fs) == eopatch -@mock_s3 +@mock_aws @pytest.mark.parametrize("fs_loader", FS_LOADERS) @pytest.mark.parametrize( "temporal_selection", @@ -591,7 +611,7 @@ def test_partial_temporal_loading(fs_loader: type[FS], eopatch: EOPatch, tempora assert_array_equal(np.array(full_patch.timestamps)[adjusted_selection], partial_patch.timestamps) -@mock_s3 +@mock_aws @pytest.mark.parametrize("fs_loader", FS_LOADERS) def test_partial_temporal_loading_fails_for_numpy(fs_loader: type[FS], eopatch: EOPatch): _skip_when_appropriate(fs_loader, True) @@ -602,7 +622,7 @@ def test_partial_temporal_loading_fails_for_numpy(fs_loader: type[FS], eopatch: EOPatch.load(path="patch-folder", filesystem=temp_fs, temporal_selection=[0]) -@mock_s3 +@mock_aws @pytest.mark.parametrize("fs_loader", FS_LOADERS) @pytest.mark.parametrize("temporal_selection", [[3, 4, 10]]) def test_partial_temporal_loading_fails_bad_selection(fs_loader: type[FS], eopatch: EOPatch, temporal_selection): @@ -614,7 +634,7 @@ def test_partial_temporal_loading_fails_bad_selection(fs_loader: type[FS], eopat EOPatch.load(path="patch-folder", filesystem=temp_fs, temporal_selection=temporal_selection) -@mock_s3 +@mock_aws @pytest.mark.parametrize("temporal_selection", [None, slice(None, 3), slice(2, 4, 2), [3, 4]]) def test_partial_temporal_saving_into_existing(eopatch: EOPatch, temporal_selection): _skip_when_appropriate(TempFS, True) @@ -635,7 +655,7 @@ def test_partial_temporal_saving_into_existing(eopatch: EOPatch, temporal_select assert_array_equal(loaded_patch.timestamps, eopatch.timestamps) -@mock_s3 +@mock_aws @pytest.mark.parametrize("dtype", [float, np.float32, np.int8]) def test_partial_temporal_saving_just_timestamps(dtype): _skip_when_appropriate(TempFS, True) @@ -659,7 +679,7 @@ def test_partial_temporal_saving_just_timestamps(dtype): assert_array_equal(loaded_patch.timestamps, patch_skeleton.timestamps) -@mock_s3 +@mock_aws def test_partial_temporal_saving_infer(eopatch: EOPatch): _skip_when_appropriate(TempFS, True) with TempFS() as temp_fs: @@ -679,7 +699,7 @@ def test_partial_temporal_saving_infer(eopatch: EOPatch): assert_array_equal(loaded_patch.timestamps, eopatch.timestamps) -@mock_s3 +@mock_aws def test_partial_temporal_saving_fails(eopatch: EOPatch): _skip_when_appropriate(TempFS, True) with TempFS() as temp_fs: diff --git a/tests/core/test_utils/test_fs.py b/tests/core/test_utils/test_fs.py index a2622462..f2f80b61 100644 --- a/tests/core/test_utils/test_fs.py +++ b/tests/core/test_utils/test_fs.py @@ -13,6 +13,7 @@ from _thread import RLock from pathlib import Path +import boto3 import pytest from botocore.credentials import Credentials from fs.base import FS @@ -21,7 +22,7 @@ from fs.osfs import OSFS from fs.tempfs import TempFS from fs_s3fs import S3FS -from moto import mock_s3 +from moto import mock_aws from sentinelhub import SHConfig @@ -29,6 +30,23 @@ from eolearn.core.utils.fs import get_aws_credentials, get_full_path, join_path, split_all_extensions +@mock_aws +def create_mocked_s3fs(bucket_name: str = "mocked-test-bucket") -> S3FS: + """Creates a new empty mocked s3 bucket. If one such bucket already exists it deletes it first.""" + s3resource = boto3.resource("s3", region_name="eu-central-1") + + bucket = s3resource.Bucket(bucket_name) + + if bucket.creation_date: # If bucket already exists + for key in bucket.objects.all(): + key.delete() + bucket.delete() + + s3resource.create_bucket(Bucket=bucket_name, CreateBucketConfiguration={"LocationConstraint": "eu-central-1"}) + + return S3FS(bucket_name=bucket_name) + + def test_get_local_filesystem(tmp_path): filesystem = get_filesystem(tmp_path) assert isinstance(filesystem, OSFS) @@ -48,7 +66,7 @@ def test_pathlib_support(tmp_path): assert isinstance(filesystem, OSFS) -@mock_s3 +@mock_aws @pytest.mark.parametrize("aws_session_token", [None, "fake-session-token"]) def test_s3_filesystem(aws_session_token): folder_name = "my_folder" @@ -137,8 +155,8 @@ def test_tempfs_serialization(): assert not unpickled_filesystem.exists("/") -@mock_s3 -def test_s3fs_serialization(create_mocked_s3fs): +@mock_aws +def test_s3fs_serialization(): """Makes sure that after serialization and deserialization filesystem object can still be used for reading, writing, and listing objects.""" filesystem = create_mocked_s3fs()