From 4f318f90761fde18e7d286f5d6bddbc5bc844382 Mon Sep 17 00:00:00 2001 From: Eric Brune Date: Mon, 28 Oct 2024 09:47:43 +0100 Subject: [PATCH 1/2] Fixed MinMax normalization --- pangaea/engine/data_preprocessor.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/pangaea/engine/data_preprocessor.py b/pangaea/engine/data_preprocessor.py index 0e9bcb7..f82496f 100644 --- a/pangaea/engine/data_preprocessor.py +++ b/pangaea/engine/data_preprocessor.py @@ -381,9 +381,8 @@ def __call__( """ for k in self.data_min.keys(): - size = (-1,) + data["image"][k].shape[1:] - data["image"][k].sub_(self.data_min[k].view(size)).div_( - (self.data_max[k] - self.data_min[k]).view(size) + data["image"][k].sub_(self.data_min[k].view(-1, 1, 1, 1)).div_( + (self.data_max[k] - self.data_min[k]).view(-1, 1, 1, 1) ) return data From cf590756cb121b17118483ac08ef2eea0de61d44 Mon Sep 17 00:00:00 2001 From: Eric Brune Date: Thu, 7 Nov 2024 10:29:28 +0100 Subject: [PATCH 2/2] Fixes 4 ai4smallfarms --- configs/dataset/ai4smallfarms.yaml | 9 ++++----- pangaea/datasets/ai4smallfarms.py | 20 +++++++++++--------- 2 files changed, 15 insertions(+), 14 deletions(-) diff --git a/configs/dataset/ai4smallfarms.yaml b/configs/dataset/ai4smallfarms.yaml index f0083ff..c728329 100644 --- a/configs/dataset/ai4smallfarms.yaml +++ b/configs/dataset/ai4smallfarms.yaml @@ -1,16 +1,15 @@ _target_: pangaea.datasets.ai4smallfarms.AI4SmallFarms dataset_name: AI4SmallFarms root_path: ./data/ai4smallfarms -dataverse_api_token: 753e72e8-c746-4860-9d0c-3aec872d888a +download_url: https://phys-techsciences.datastations.nl auto_download: True -img_size: 807 +img_size: 496 multi_temporal: False multi_modal: False -limited_label: False ignore_index: -1 -num_classes: 2 +num_classes: 1 classes: - Background - Crop Field @@ -52,4 +51,4 @@ data_min: - 1.0 - 9.0 - 1.0 - - 66.0 + - 66.0 \ No newline at end of file diff --git a/pangaea/datasets/ai4smallfarms.py b/pangaea/datasets/ai4smallfarms.py index 7e8c9b1..9e4e8e9 100644 --- a/pangaea/datasets/ai4smallfarms.py +++ b/pangaea/datasets/ai4smallfarms.py @@ -149,7 +149,7 @@ def download(self, silent=False): return # Set up the Dataverse API - base_url = "https://phys-techsciences.datastations.nl" + base_url = self.download_url api = NativeApi(base_url) data_api = DataAccessApi(base_url) DOI = "doi:10.17026/dans-xy6-ngg6" @@ -205,19 +205,19 @@ def download(self, silent=False): # **Cleanup: Remove unwanted files and directories** unwanted_paths = [ - os.path.join(dataset_config["root_path"], "easy-migration.zip"), - os.path.join(dataset_config["root_path"], "readme.md"), + os.path.join(self.root_path, "easy-migration.zip"), + os.path.join(self.root_path, "readme.md"), os.path.join( - dataset_config["root_path"], "sentinel-2-asia", "benchmark.qgz" + self.root_path, "sentinel-2-asia", "benchmark.qgz" ), os.path.join( - dataset_config["root_path"], "sentinel-2-asia", "tiles_asia.gpkg" + self.root_path, "sentinel-2-asia", "tiles_asia.gpkg" ), - os.path.join(dataset_config["root_path"], "sentinel-2-asia", "reference"), + os.path.join(self.root_path, "sentinel-2-asia", "reference"), os.path.join( - dataset_config["root_path"], "sentinel-2-asia", "test", "output" + self.root_path, "sentinel-2-asia", "test", "output" ), - os.path.join(dataset_config["root_path"], "sentinel-2-nl"), + os.path.join(self.root_path, "sentinel-2-nl"), ] # Remove unwanted files and directories @@ -230,5 +230,7 @@ def download(self, silent=False): if not silent: print(f"Removed unwanted path: {path}") + os.rename(os.path.join(self.root_path, "sentinel-2-asia/validate"), os.path.join(self.root_path, "sentinel-2-asia/val")) + if not silent: - print(f"Downloaded dataset to {dataset_config['root_path']}") + print(f"Downloaded dataset to {self.root_path}")