From 409be4956c14f73a731b05072a703c2aba26c402 Mon Sep 17 00:00:00 2001 From: Alexandra Kirk Date: Thu, 23 May 2024 15:12:27 -0600 Subject: [PATCH 1/2] fix: houston-urbanization discovery prefix fix (#129) --- .../production/discovery-items/houston-urbanization.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ingestion-data/production/discovery-items/houston-urbanization.json b/ingestion-data/production/discovery-items/houston-urbanization.json index 78e5d6e8..ffec9017 100644 --- a/ingestion-data/production/discovery-items/houston-urbanization.json +++ b/ingestion-data/production/discovery-items/houston-urbanization.json @@ -2,7 +2,7 @@ "collection": "houston-urbanization", "bucket": "veda-data-store", "prefix": "houston-urbanization/", - "filename_regex": "^(.*)HOUS_DFAL_urbanization", + "filename_regex": "^(.*)HOUS_DFAL_urbanization.*[.]tif$", "discovery": "s3", "upload": false } \ No newline at end of file From eed1b3c271e1a6a79c92acf708da246ca7886531 Mon Sep 17 00:00:00 2001 From: Jennifer Tran <12633533+botanical@users.noreply.github.com> Date: Thu, 23 May 2024 14:17:09 -0700 Subject: [PATCH 2/2] fix: collection configs (#130) * fix: campire-data transfer config * fix: lis-global-da-gpp-trend transfer-config and discovery-items * fix: update snow-projection configs and lis-global-da-tws-trend --------- Co-authored-by: Jennifer Tran --- .../lis-global-da-gpp-trend.json | 36 ++++++++++++++----- .../lis-global-da-tws-trend.json | 4 ++- .../snow-projections-diff-585.json | 2 +- .../snow-projections-median-245.json | 2 +- .../snow-projections-median-585.json | 2 +- .../transfer-config/campfire-nlcd.json | 2 +- .../lis-global-da-gpp-trend.json | 16 ++++++++- 7 files changed, 50 insertions(+), 14 deletions(-) diff --git a/ingestion-data/production/discovery-items/lis-global-da-gpp-trend.json b/ingestion-data/production/discovery-items/lis-global-da-gpp-trend.json index 59028f0b..83b5b67b 100644 --- a/ingestion-data/production/discovery-items/lis-global-da-gpp-trend.json +++ b/ingestion-data/production/discovery-items/lis-global-da-gpp-trend.json @@ -1,8 +1,28 @@ -{ - "collection": "lis-global-da-gpp-trend", - "bucket": "veda-data-store", - "prefix": "lis-global-da-gpp-trend/", - "filename_regex": "^(.*)DAGPP_STL_based_trend", - "discovery": "s3", - "upload": false -} \ No newline at end of file +[ + { + "collection": "lis-global-da-gpp-trend", + "bucket": "veda-data-store", + "prefix": "lis-global-da-gpp-trend/", + "filename_regex": "^(.*)DAGPP_STL_based_trend_v3_2003_2021.cog.tif", + "discovery": "s3", + "upload": false + }, + { + "collection": "lis-global-da-gpp-trend", + "bucket": "veda-data-store", + "prefix": "lis-global-da-gpp-trend/", + "filename_regex": "^(.*)DAGPP_STL_based_trendv2_2003_2021.cog.tif", + "discovery": "s3", + "upload": false + }, + { + "collection": "lis-global-da-gpp-trend", + "bucket": "veda-data-store", + "prefix": "lis-global-da-gpp-trend/", + "filename_regex": "^(.*)DAGPP_STL_based_trend.cog.tif", + "discovery": "s3", + "upload": false, + "start_datetime": "2003-01-01T00:00:00Z", + "end_datetime": "2021-12-31T23:59:59Z" + } +] \ No newline at end of file diff --git a/ingestion-data/production/discovery-items/lis-global-da-tws-trend.json b/ingestion-data/production/discovery-items/lis-global-da-tws-trend.json index 15922ef8..d6ba153c 100644 --- a/ingestion-data/production/discovery-items/lis-global-da-tws-trend.json +++ b/ingestion-data/production/discovery-items/lis-global-da-tws-trend.json @@ -21,6 +21,8 @@ "prefix": "lis-global-da-tws-trend/", "filename_regex":"^(.*)DATWS_STL_based_trend.cog.tif", "discovery": "s3", - "upload": false + "upload": false, + "start_datetime": "2003-01-01T00:00:00Z", + "end_datetime": "2021-12-31T23:59:59Z" } ] diff --git a/ingestion-data/production/discovery-items/snow-projections-diff-585.json b/ingestion-data/production/discovery-items/snow-projections-diff-585.json index 4b28aca5..b516236a 100644 --- a/ingestion-data/production/discovery-items/snow-projections-diff-585.json +++ b/ingestion-data/production/discovery-items/snow-projections-diff-585.json @@ -1,7 +1,7 @@ [ { "collection": "snow-projections-diff-585", - "prefix": "snow-projections-diff-245/", + "prefix": "snow-projections-diff-585/", "bucket": "veda-data-store", "filename_regex": "^(.*)ssp585(.*)percChange.cog.tif$", "discovery": "s3", diff --git a/ingestion-data/production/discovery-items/snow-projections-median-245.json b/ingestion-data/production/discovery-items/snow-projections-median-245.json index fa33b376..81d17522 100644 --- a/ingestion-data/production/discovery-items/snow-projections-median-245.json +++ b/ingestion-data/production/discovery-items/snow-projections-median-245.json @@ -1,7 +1,7 @@ [ { "collection": "snow-projections-median-245", - "prefix": "snow-projections-diff-245/", + "prefix": "snow-projections-median-245/", "bucket": "veda-data-store", "filename_regex": "^(.*)ssp245(.*)median.cog.tif$", "discovery": "s3", diff --git a/ingestion-data/production/discovery-items/snow-projections-median-585.json b/ingestion-data/production/discovery-items/snow-projections-median-585.json index 484291de..60de9d2d 100644 --- a/ingestion-data/production/discovery-items/snow-projections-median-585.json +++ b/ingestion-data/production/discovery-items/snow-projections-median-585.json @@ -1,7 +1,7 @@ [ { "collection": "snow-projections-median-585", - "prefix": "snow-projections-diff-245/", + "prefix": "snow-projections-median-585/", "bucket": "veda-data-store", "filename_regex": "^(.*)ssp585(.*)median.cog.tif$", "discovery": "s3", diff --git a/ingestion-data/transfer-config/campfire-nlcd.json b/ingestion-data/transfer-config/campfire-nlcd.json index e7b5d66e..4496992a 100644 --- a/ingestion-data/transfer-config/campfire-nlcd.json +++ b/ingestion-data/transfer-config/campfire-nlcd.json @@ -4,6 +4,6 @@ "origin_prefix": "campfire-data/", "target_bucket": "veda-data-store", "collection": "campfire-nlcd", - "filename_regex": "^(.*)CampFire_NLCD_2019" + "filename_regex": "^(.*)CampFire_NLCD_201\\d" } ] \ No newline at end of file diff --git a/ingestion-data/transfer-config/lis-global-da-gpp-trend.json b/ingestion-data/transfer-config/lis-global-da-gpp-trend.json index 8c6c2a45..3273cf5c 100644 --- a/ingestion-data/transfer-config/lis-global-da-gpp-trend.json +++ b/ingestion-data/transfer-config/lis-global-da-gpp-trend.json @@ -4,6 +4,20 @@ "origin_prefix": "EIS/COG/LIS_GLOBAL_DA/DA_Trends/", "target_bucket": "veda-data-store", "collection": "lis-global-da-gpp-trend", - "filename_regex": "^(.*)DAGPP_STL_based_trend" + "filename_regex": "^(.*)DAGPP_STL_based_trend.cog.tif" + }, + { + "origin_bucket": "veda-data-store-staging", + "origin_prefix": "EIS/COG/LIS_GLOBAL_DA/DA_Trends/", + "target_bucket": "veda-data-store", + "collection": "lis-global-da-gpp-trend", + "filename_regex": "^(.*)DAGPP_STL_based_trend_v3_2003_2021.cog.tif" + }, + { + "origin_bucket": "veda-data-store-staging", + "origin_prefix": "EIS/COG/LIS_GLOBAL_DA/DA_Trends/", + "target_bucket": "veda-data-store", + "collection": "lis-global-da-gpp-trend", + "filename_regex": "^(.*)DAGPP_STL_based_trendv2_2003_2021.cog.tif" } ] \ No newline at end of file