From b4428cfbe063977fa6e5947ee27532275d63fba8 Mon Sep 17 00:00:00 2001 From: yurujaja Date: Fri, 27 Sep 2024 02:26:18 +0200 Subject: [PATCH 1/3] move the figure path --- {figures => .github}/geofmbenchmark.png | Bin 1 file changed, 0 insertions(+), 0 deletions(-) rename {figures => .github}/geofmbenchmark.png (100%) diff --git a/figures/geofmbenchmark.png b/.github/geofmbenchmark.png similarity index 100% rename from figures/geofmbenchmark.png rename to .github/geofmbenchmark.png From d8bd809429647ff46ed87403c74cf20da81bcc2f Mon Sep 17 00:00:00 2001 From: yurujaja Date: Fri, 27 Sep 2024 02:28:22 +0200 Subject: [PATCH 2/3] satlasnet config and readme, improve clarity --- README.md | 17 ++++++------- .../{satlasnet.yaml => satlasnet_mi.yaml} | 2 +- configs/encoder/satlasnet_si.yaml | 25 +++++++++++++++++++ 3 files changed, 34 insertions(+), 10 deletions(-) rename configs/encoder/{satlasnet.yaml => satlasnet_mi.yaml} (81%) create mode 100644 configs/encoder/satlasnet_si.yaml diff --git a/README.md b/README.md index 78ca5293..ace631fd 100644 --- a/README.md +++ b/README.md @@ -9,7 +9,7 @@ While geospatial foundation models (GFMs) have proliferated rapidly, their evalu To bridge this gap, we propose a standardized evaluation protocol that incorporates a wide-ranging selection of datasets, tasks, resolutions, and sensor types, establishing a robust and widely applicable benchmark for GFMs. -PANGAEA: a diverse benchmark for geospatial foundation models +PANGAEA: a diverse benchmark for geospatial foundation models In this repo, you can find the code to benchmark GFMs. For the moment we included several GFMs that present different approach. We look forward to adding new models and datasets. @@ -152,10 +152,9 @@ torchrun --nnodes=1 --nproc_per_node=1 pangaea/run.py \ ``` #### Multi-Temporal Semantic Segmentation - -Multi-temporal decoder config (e.g. `configs/decoder/seg_upernet_mt_ltae.yaml` if you want to use `ltae` as a strategy to combine multi-temporal info) should be used. e.g. Prithvi encoder on CropTypeMapping -In addition, in the dataset config, indicate the number of time frames, e.g., `multi_temporal: 6` - +- Multi-temporal decoder config (e.g. `configs/decoder/seg_upernet_mt_ltae.yaml` if you want to use `ltae` as a strategy to combine multi-temporal info) should be used. +- In addition, in the dataset config, indicate the number of time frames, e.g., `multi_temporal: 6` +- To use SatlasNet encoder, the `configs/encoder/satlasnet_mi.yaml` is required. ``` torchrun --nnodes=1 --nproc_per_node=1 pangaea/run.py \ --config-name=train \ @@ -189,7 +188,7 @@ To overwrite parameters, please check the Single Temporal Semantic Segmentation #### Single Temporal Regression The regression decoder (e.g. `configs/decoder/reg_upernet.yaml`) and the regression task (e.g. `configs/task/regression.yaml`) configs should be used. -e.g. Prithvi encoder on BioMassters +E.g. Prithvi encoder on BioMassters ``` torchrun --nnodes=1 --nproc_per_node=1 pangaea/run.py \ @@ -201,13 +200,13 @@ torchrun --nnodes=1 --nproc_per_node=1 pangaea/run.py \ criterion=cross_entropy \ task=regression ``` - +To use SatlasNet encoder, the `configs/encoder/satlasnet_si.yaml` is required. To overwrite parameters, please check the Single Temporal Semantic Segmentation example #### Multi-Temporal Regression -The multi-temporal regression decoder (e.g. `configs/decoder/reg_upernet_mt_ltae.yaml` or `configs/decoder/reg_upernet_mt_linear.yaml`) and the regression task (e.g. `configs/task/regression.yaml`) configs should be used. -e.g. Prithvi encoder on BioMassters +The multi-temporal regression decoder (e.g. `configs/decoder/reg_upernet_mt_ltae.yaml` or `configs/decoder/reg_upernet_mt_linear.yaml`) and the regression task (e.g. `configs/task/regression.yaml`) configs should be used. To use SatlasNet encoder, the `configs/encoder/satlasnet_mi.yaml` is required. +Take Prithvi encoder on BioMassters as example: ``` torchrun --nnodes=1 --nproc_per_node=1 pangaea/run.py \ diff --git a/configs/encoder/satlasnet.yaml b/configs/encoder/satlasnet_mi.yaml similarity index 81% rename from configs/encoder/satlasnet.yaml rename to configs/encoder/satlasnet_mi.yaml index 2ccf403e..2383bdf0 100644 --- a/configs/encoder/satlasnet.yaml +++ b/configs/encoder/satlasnet_mi.yaml @@ -2,7 +2,7 @@ _target_: pangaea.encoders.satlasnet_encoder.SatlasNet_Encoder encoder_weights: null download_url: null -model_identifier: Sentinel2_SwinB_MI_MS #Sentinel2_SwinB_SI_RGB # +model_identifier: Sentinel2_SwinB_MI_MS # Multi-Image Multi-Spectral fpn: False input_size: 128 output_dim: 1024 diff --git a/configs/encoder/satlasnet_si.yaml b/configs/encoder/satlasnet_si.yaml new file mode 100644 index 00000000..5759fcb4 --- /dev/null +++ b/configs/encoder/satlasnet_si.yaml @@ -0,0 +1,25 @@ + +_target_: pangaea.encoders.satlasnet_encoder.SatlasNet_Encoder +encoder_weights: null +download_url: null +model_identifier: Sentinel2_SwinB_SI_MS # Single Image Multi-Spectral +fpn: False +input_size: 128 +output_dim: 1024 + +input_bands: + optical: + # - B4 + # - B3 + # - B2 + + #MS Bands + - B2 + - B3 + - B4 + - B5 + - B6 + - B7 + - B8 + - B11 + - B12 From fbc1cca41ecad4df6a55854bdd7f98eafe4e36df Mon Sep 17 00:00:00 2001 From: yurujaja Date: Fri, 27 Sep 2024 09:18:21 +0200 Subject: [PATCH 3/3] minor fix mae_sar --- README.md | 30 ++++++++++++++++++++------ pangaea/encoders/ssl4eo_mae_encoder.py | 2 ++ 2 files changed, 25 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index ace631fd..52f5377f 100644 --- a/README.md +++ b/README.md @@ -154,19 +154,33 @@ torchrun --nnodes=1 --nproc_per_node=1 pangaea/run.py \ #### Multi-Temporal Semantic Segmentation - Multi-temporal decoder config (e.g. `configs/decoder/seg_upernet_mt_ltae.yaml` if you want to use `ltae` as a strategy to combine multi-temporal info) should be used. - In addition, in the dataset config, indicate the number of time frames, e.g., `multi_temporal: 6` -- To use SatlasNet encoder, the `configs/encoder/satlasnet_mi.yaml` is required. + +An example of using SSL4EO-DINO on CropTypeMapping is as below ``` torchrun --nnodes=1 --nproc_per_node=1 pangaea/run.py \ --config-name=train \ dataset=croptypemapping \ - encoder=prithvi \ + encoder=ssl4eo_dino \ decoder=seg_upernet_mt_ltae \ preprocessing=seg_resize \ criterion=cross_entropy \ task=segmentation ``` -To overwrite parameters, please check the Single Temporal Semantic Segmentation example +To use SatlasNet encoder, the `configs/encoder/satlasnet_mi.yaml` is required +``` +torchrun --nnodes=1 --nproc_per_node=1 pangaea/run.py \ + --config-name=train \ + dataset=croptypemapping \ + encoder=satlasnet_mi \ + decoder=seg_upernet_mt_ltae decoder.multi_temporal_strategy=null \ + preprocessing=seg_resize \ + criterion=cross_entropy \ + task=segmentation +``` + + +To overwrite parameters, please check the Single Temporal Semantic Segmentation example. #### Change Detection @@ -183,7 +197,7 @@ torchrun --nnodes=1 --nproc_per_node=1 pangaea/run.py \ task=change_detection ``` -To overwrite parameters, please check the Single Temporal Semantic Segmentation example +To overwrite parameters, please check the Single Temporal Semantic Segmentation example. #### Single Temporal Regression @@ -201,11 +215,12 @@ torchrun --nnodes=1 --nproc_per_node=1 pangaea/run.py \ task=regression ``` To use SatlasNet encoder, the `configs/encoder/satlasnet_si.yaml` is required. -To overwrite parameters, please check the Single Temporal Semantic Segmentation example +To overwrite parameters, please check the Single Temporal Semantic Segmentation example. #### Multi-Temporal Regression -The multi-temporal regression decoder (e.g. `configs/decoder/reg_upernet_mt_ltae.yaml` or `configs/decoder/reg_upernet_mt_linear.yaml`) and the regression task (e.g. `configs/task/regression.yaml`) configs should be used. To use SatlasNet encoder, the `configs/encoder/satlasnet_mi.yaml` is required. +The multi-temporal regression decoder (e.g. `configs/decoder/reg_upernet_mt_ltae.yaml` or `configs/decoder/reg_upernet_mt_linear.yaml`) and the regression task (e.g. `configs/task/regression.yaml`) configs should be used. + Take Prithvi encoder on BioMassters as example: ``` @@ -219,7 +234,8 @@ torchrun --nnodes=1 --nproc_per_node=1 pangaea/run.py \ task=regression ``` -To overwrite parameters, please check the Single Temporal Semantic Segmentation example +To use SatlasNet encoder, please refer to the multi-temporal semantic segmentation example. +To overwrite parameters, please check the Single Temporal Semantic Segmentation example. ### 💻 End-to-end Finetuning diff --git a/pangaea/encoders/ssl4eo_mae_encoder.py b/pangaea/encoders/ssl4eo_mae_encoder.py index 21abb6eb..4597e7c8 100644 --- a/pangaea/encoders/ssl4eo_mae_encoder.py +++ b/pangaea/encoders/ssl4eo_mae_encoder.py @@ -197,6 +197,7 @@ def __init__( input_size: int, input_bands: dict[str, list[str]], output_layers: int | list[int], + download_url: str, embed_dim: int = 1024, patch_size: int = 16, in_chans: int = 3, @@ -217,6 +218,7 @@ def __init__( num_heads=num_heads, mlp_ratio=mlp_ratio, norm_layer=norm_layer, + download_url=download_url, ) self.model_name = "ssl4eo_mae_sar"