Skip to content

Commit

Permalink
Tiff tasks parameter folder renamed to path (#768)
Browse files Browse the repository at this point in the history 
* adjust parameter name

* another simplification

* Apply suggestions from code review

Co-authored-by: Matic Lubej <[email protected]>

* break lines

---------

Co-authored-by: Matic Lubej <[email protected]>
  • Loading branch information
@zigaLuksic @mlubej
zigaLuksic and mlubej authored Nov 6, 2023
1 parent 07affc1 commit 7eb980e
Show file tree
Hide file tree
Showing 2 changed files with 45 additions and 39 deletions.
54 changes: 30 additions & 24 deletions eolearn/io/raster_io.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -42,39 +42,43 @@ class BaseRasterIoTask(IOTask, metaclass=ABCMeta):
def __init__(
self,
feature: Feature,
folder: str,
path: str = ..., # type: ignore[assignment]
*,
filesystem: FS | None = None,
image_dtype: np.dtype | type | None = None,
no_data_value: float | None = None,
create: bool = False,
config: SHConfig | None = None,
folder: str | None = None,
):
"""
:param feature: Feature which will be exported or imported
:param folder: A path to a main folder containing all image, potentially in its subfolders. If `filesystem`
parameter is defined, then `folder` should be a path relative to filesystem object. Otherwise, it should be
an absolute path.
:param path: Path pointing to an image file or to a directory of image files. If `filesystem` is not defined,
absolute paths should be provided.
:param filesystem: A filesystem object. If not given it will be initialized according to `folder` parameter.
:param image_dtype: A data type of data in exported images or data imported from images.
:param no_data_value: When exporting this is the NoData value of pixels in exported images.
When importing this value is assigned to the pixels with NoData.
:param no_data_value: When exporting this is the `NoData` value of pixels in exported images. When importing
this value is assigned to the pixels with `NoData`.
:param create: If the filesystem path doesn't exist this flag indicates to either create it or raise an error.
:param config: A configuration object with AWS credentials. By default, is set to None and in this case the
default configuration will be taken.
"""
ftype, fname = self.parse_feature(feature)
if fname is None:
raise ValueError(f"Feature {feature} is not eligible for this task.")
self.feature = ftype, fname
if folder is not None:
warnings.warn("The parameter `folder` has been renamed to `path`.", EODeprecationWarning, stacklevel=3)
path = folder

if path is ...: # type: ignore[comparison-overlap]
raise TypeError(f"{type(self).__name__} is missing a required positional argument 'path'.")

self.feature = self.parse_feature(feature)
self.image_dtype = image_dtype
self.no_data_value = no_data_value

if filesystem is None:
filesystem, folder = get_base_filesystem_and_path(folder, create=create, config=config)
filesystem, path = get_base_filesystem_and_path(path, create=create, config=config)

# the super-class takes care of filesystem pickling
super().__init__(folder, filesystem=filesystem, create=create, config=config)
super().__init__(path, filesystem=filesystem, create=create, config=config)

def _get_filename_paths(
self, filename_template: str | list[str], timestamps: list[dt.datetime] | None
Expand Down Expand Up @@ -114,8 +118,7 @@ def _get_filename_paths(
@classmethod
def _generate_paths(cls, path_template: str, timestamps: list[dt.datetime] | None) -> list[str]:
"""Uses a filename path template to create a list of actual filename paths."""
has_tiff_file_extensions = path_template.lower().endswith(".tif") or path_template.lower().endswith(".tiff")
if not has_tiff_file_extensions:
if not path_template.lower().endswith((".tif", ".tiff")):
path_template = f"{path_template}.tif"

if not timestamps:
Expand Down Expand Up @@ -145,7 +148,7 @@ class ExportToTiffTask(BaseRasterIoTask):
def __init__(
self,
feature: Feature,
folder: str,
path: str = ..., # type: ignore[assignment]
*,
date_indices: list[int] | tuple[int, int] | tuple[dt.datetime, dt.datetime] | tuple[str, str] | None = None,
band_indices: list[int] | tuple[int, int] | None = None,
Expand All @@ -156,12 +159,12 @@ def __init__(
image_dtype: np.dtype | type | None = None,
no_data_value: float | None = None,
config: SHConfig | None = None,
folder: str | None = None,
):
"""
:param feature: A feature to be exported.
:param folder: A path to a main folder containing all image, potentially in its subfolders. If `filesystem`
parameter is defined, then `folder` should be a path relative to filesystem object. Otherwise, it should be
an absolute path.
:param path: Path pointing to an image file or to a directory of image files to be exported. If `filesystem` is
not defined, absolute paths should be provided.
:param date_indices: Indices of those time frames from the give feature that will be exported to a tiff image.
It can be either a list of indices or a tuple of `2` indices defining an interval of indices or a tuple of
`2` datetime object also defining a time interval. By default, all time frames will be exported.
Expand All @@ -175,19 +178,19 @@ def __init__(
:param compress: A type of compression that rasterio should apply to an exported image.
:param filesystem: A filesystem object. If not given it will be initialized according to `folder` parameter.
:param image_dtype: A data type of data in exported images or data imported from images.
:param no_data_value: When exporting this is the NoData value of pixels in exported images.
When importing this value is assigned to the pixels with NoData.
:param no_data_value: The `NoData` value of pixels in exported images.
:param config: A configuration object with AWS credentials. By default, is set to None and in this case the
default configuration will be taken.
"""
super().__init__(
feature,
folder=folder,
path=path,
create=True,
filesystem=filesystem,
image_dtype=image_dtype,
no_data_value=no_data_value,
config=config,
folder=folder,
)

self.date_indices = date_indices
Expand Down Expand Up @@ -406,18 +409,20 @@ class ImportFromTiffTask(BaseRasterIoTask):
def __init__(
self,
feature: Feature,
folder: str,
path: str = ..., # type: ignore[assignment]
*,
use_vsi: bool | None = None,
timestamp_size: int | None = None,
filesystem: FS | None = None,
image_dtype: np.dtype | type | None = None,
no_data_value: float | None = None,
config: SHConfig | None = None,
folder: str | None = None,
):
"""
:param feature: EOPatch feature into which data will be imported
:param folder: A directory containing image files or a path of an image file
:param path: Path pointing to an image file or to a directory of image files to be imported. If `filesystem` is
not defined, absolute paths should be provided.
:param use_vsi: Deprecated.
:param timestamp_size: In case data will be imported into a time-dependant feature this parameter can be used to
specify time dimension. If not specified, time dimension will be the same as size of the timestamps
Expand All @@ -434,11 +439,12 @@ def __init__(
"""
super().__init__(
feature,
folder=folder,
path=path,
filesystem=filesystem,
image_dtype=image_dtype,
no_data_value=no_data_value,
config=config,
folder=folder,
)

if use_vsi is not None:
Expand Down
30 changes: 15 additions & 15 deletions tests/io/test_raster_io.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -133,13 +133,13 @@ def test_export_import(test_case, test_eopatch):
feature = test_case.feature_type, test_case.name

export_task = ExportToTiffTask(
feature, folder=tmp_dir_name, band_indices=test_case.bands, date_indices=test_case.times
feature, path=tmp_dir_name, band_indices=test_case.bands, date_indices=test_case.times
)
_execute_with_warning_control(export_task, test_case.warning, test_eopatch, filename=tmp_file_name)

export_task = ExportToTiffTask(
feature,
folder=tmp_dir_name,
path=tmp_dir_name,
band_indices=test_case.bands,
date_indices=test_case.times,
crs="EPSG:4326",
Expand All @@ -148,7 +148,7 @@ def test_export_import(test_case, test_eopatch):
_execute_with_warning_control(export_task, test_case.warning, test_eopatch, filename=tmp_file_name_reproject)

import_task = ImportFromTiffTask(
feature, folder=tmp_dir_name, timestamp_size=test_case.get_expected_timestamp_size()
feature, path=tmp_dir_name, timestamp_size=test_case.get_expected_timestamp_size()
)

expected_raster = test_case.get_expected()
Expand Down Expand Up @@ -196,7 +196,7 @@ def test_export2tiff_wrong_format(bands, times, test_eopatch):
with tempfile.TemporaryDirectory() as tmp_dir_name:
tmp_file_name = "temp_file.tiff"
task = ExportToTiffTask(
(FeatureType.DATA, "data"), folder=tmp_dir_name, band_indices=bands, date_indices=times, image_dtype=float
(FeatureType.DATA, "data"), path=tmp_dir_name, band_indices=bands, date_indices=times, image_dtype=float
)
with pytest.raises(ValueError):
task.execute(test_eopatch, filename=tmp_file_name)
Expand All @@ -209,7 +209,7 @@ def test_export2tiff_wrong_feature(mocker, test_eopatch):
tmp_file_name = "temp_file.tiff"
feature = FeatureType.MASK_TIMELESS, "feature-not-present"

export_task = ExportToTiffTask(feature, folder=tmp_dir_name, fail_on_missing=False)
export_task = ExportToTiffTask(feature, path=tmp_dir_name, fail_on_missing=False)
export_task(test_eopatch, filename=tmp_file_name)

assert logging.Logger.warning.call_count == 1
Expand All @@ -220,7 +220,7 @@ def test_export2tiff_wrong_feature(mocker, test_eopatch):
== "Feature (<FeatureType.MASK_TIMELESS: 'mask_timeless'>, 'feature-not-present') was not found in EOPatch"
)

failing_export_task = ExportToTiffTask(feature, folder=tmp_dir_name, fail_on_missing=True)
failing_export_task = ExportToTiffTask(feature, path=tmp_dir_name, fail_on_missing=True)
with pytest.raises(ValueError):
failing_export_task(test_eopatch, filename=tmp_file_name)

Expand Down Expand Up @@ -248,7 +248,7 @@ def test_export2tiff_separate_timestamps(test_eopatch):
full_path = os.path.join(tmp_dir_name, tmp_file_name_reproject)
export_task = ExportToTiffTask(
feature,
folder=full_path,
path=full_path,
band_indices=test_case.bands,
date_indices=test_case.times,
crs="EPSG:4326",
Expand Down Expand Up @@ -310,8 +310,8 @@ def test_timeless_feature(test_eopatch):
filename = "relative-path/my-filename.tiff"

with tempfile.TemporaryDirectory() as tmp_dir_name:
export_task = ExportToTiffTask(feature, folder=tmp_dir_name)
import_task = ImportFromTiffTask(feature, folder=tmp_dir_name)
export_task = ExportToTiffTask(feature, path=tmp_dir_name)
import_task = ImportFromTiffTask(feature, path=tmp_dir_name)

export_task.execute(test_eopatch, filename=filename)
new_eopatch = import_task.execute(test_eopatch, filename=filename)
Expand All @@ -327,8 +327,8 @@ def test_time_dependent_feature(test_eopatch):
]

with tempfile.TemporaryDirectory() as tmp_dir_name:
export_task = ExportToTiffTask(feature, folder=tmp_dir_name)
import_task = ImportFromTiffTask(feature, folder=tmp_dir_name, timestamp_size=68)
export_task = ExportToTiffTask(feature, path=tmp_dir_name)
import_task = ImportFromTiffTask(feature, path=tmp_dir_name, timestamp_size=68)

export_task(test_eopatch, filename=filename_export)
new_eopatch = import_task(filename=filename_import)
Expand All @@ -349,8 +349,8 @@ def test_time_dependent_feature_with_timestamps(test_eopatch):
filename = "relative-path/%Y%m%dT%H%M%S.tiff"

with tempfile.TemporaryDirectory() as tmp_dir_name:
export_task = ExportToTiffTask(feature, folder=tmp_dir_name)
import_task = ImportFromTiffTask(feature, folder=tmp_dir_name)
export_task = ExportToTiffTask(feature, path=tmp_dir_name)
import_task = ImportFromTiffTask(feature, path=tmp_dir_name)

export_task.execute(test_eopatch, filename=filename)
new_eopatch = import_task(test_eopatch, filename=filename)
Expand All @@ -374,7 +374,7 @@ def test_export_import_sequence(no_data_value, data_type):
filename = "test_seq.tiff"
file_path = os.path.join(tmp_dir_name, filename)
export_task = ExportToTiffTask(
feature=feature, folder=tmp_dir_name, band_indices=[0], no_data_value=no_data_value
feature=feature, path=tmp_dir_name, band_indices=[0], no_data_value=no_data_value
)
export_task.execute(eopatch=eopatch, filename=filename)

Expand All @@ -389,7 +389,7 @@ def test_export_import_sequence(no_data_value, data_type):

assert_array_equal(tif_array.mask, no_data_arr)

import_task = ImportFromTiffTask(feature=feature, folder=tmp_dir_name, no_data_value=no_data_value)
import_task = ImportFromTiffTask(feature=feature, path=tmp_dir_name, no_data_value=no_data_value)
new_eopatch = import_task.execute(filename=filename)

assert_array_equal(eopatch[feature], new_eopatch[feature])

0 comments on commit 7eb980e

Please sign in to comment.