Merge branch 'main' into iss392_DataManagementItems_For_WV03_MSI_L2A

NASA-IMPACT · Nov 20, 2024 · 66247aa · 66247aa
2 parents 52e2e52 + f91e8d5
commit 66247aa
Show file tree

Hide file tree

Showing 27 changed files with 1,202 additions and 806 deletions.
diff --git a/Makefile b/Makefile
@@ -119,7 +119,7 @@ fmt: docker
 init-%: docker
 	$(TERRASPACE) init $*
 
-install: docker zip_lambdas
+install: docker
 	$(DOCKER_RUN) $(IMAGE) -ic "YARN_SILENT=1 yarn install --ignore-optional && YARN_SILENT=1 yarn --cwd scripts install"
 
 ## logs: Shows last 10 lines of all Terraspace logs
@@ -155,7 +155,7 @@ plan-%: install
 ## pre-deploy-setup: Setup resources prior to initial deployment (idempotent)
 pre-deploy-setup: all-init
 	# Ensure buckets exist, grab the name of the "internal" bucket, and copy launchpad.pfx there.
-	$(DOCKER_RUN) --interactive $(IMAGE) -ic "bin/ensure-buckets-exist.sh 2>/dev/null
+	$(DOCKER_RUN) --interactive $(IMAGE) -ic "bin/ensure-buckets-exist.sh 2>/dev/null"
 
 ## terraform-doctor-STACK: Fixes "duplicate resource" errors for specified STACK
 terraform-doctor-%: docker
@@ -190,4 +190,5 @@ validate-%: docker
 
 ## Zip any lambda functions to prepare for deployment
 zip_lambdas:
+	DOTENV=$(DOTENV) \
 	sh app/stacks/post-deploy-mods/resources/lambdas/pre-filter-DistributionApiEndpoints/zip_lambda.sh
diff --git a/app/stacks/cumulus/main.tf b/app/stacks/cumulus/main.tf
@@ -576,7 +576,7 @@ module "cumulus" {
     {
       id              = "backgroundJobQueue",
       url             = aws_sqs_queue.background_job_queue.id,
-      execution_limit = 1400
+      execution_limit = 200
     }
   ]
 }
diff --git a/app/stacks/cumulus/resources/collections/PSScene3Band___1.json b/app/stacks/cumulus/resources/collections/PSScene3Band___1.json
@@ -3,15 +3,22 @@
   "version": "1",
   "duplicateHandling": "replace",
   "granuleId": ".*",
-  "granuleIdExtraction": "^(\\d{8}_\\d{6}_.+)(?:_\\dB_.+|_thumb.*|_cmr[.]json|_metadata[.]json)$",
+  "granuleIdExtraction": "^(\\d{8}_\\d{6}_.+)(?:_\\dB_.+|-BROWSE.png|_cmr[.]json|_metadata[.]json|-thumb.png)$",
   "sampleFileName": "20160628_204526_1_0c74_3B_Analytic_DN_metadata.xml",
-  "ignoreFilesConfigForDiscovery": true,
-  "url_path": "planet/PSScene3Band",
+  "ignoreFilesConfigForDiscovery": false,
+  "url_path": "{cmrMetadata.CollectionReference.ShortName}___{cmrMetadata.CollectionReference.Version}/{dateFormat(cmrMetadata.TemporalExtent.SingleDateTime, YYYY)}/{dateFormat(cmrMetadata.TemporalExtent.SingleDateTime, MM)}/{dateFormat(cmrMetadata.TemporalExtent.SingleDateTime, DD)}/{cmrMetadata.GranuleUR}",
   "files": [
     {
-      "regex": "^(\\d{8}_\\d{6}_.+)(?:_\\dB_.+|_thumb.*|_cmr[.]json|_metadata[.]json)$",
+      "regex": "^(\\d{8}_\\d{6}_.+)(?:_\\dB_.+|-BROWSE.png|_cmr[.]json|_metadata[.]json)$",
       "sampleFileName": "20160628_204526_1_0c74_3B_Analytic_DN_metadata.xml",
+      "type": "data",
       "bucket": "protected"
+    },
+    {
+      "regex": "^(\\d{8}_\\d{6}_.+)(?:.*-thumb.png)$",
+      "sampleFileName": "20160628_204526_1_0c74-thumb.png",
+      "type": "browse",
+      "bucket": "public"
     }
   ],
   "meta": {

diff --git a/app/stacks/cumulus/resources/collections/WV03_MSI_L1B___1.json b/app/stacks/cumulus/resources/collections/WV03_MSI_L1B___1.json
@@ -1,7 +1,7 @@
 {
   "name": "WV03_MSI_L1B",
   "version": "1",
-  "duplicateHandling": "skip",
+  "duplicateHandling": "replace",
   "granuleId": ".*",
   "granuleIdExtraction": "^(WV03_.+-M1BS-.+_P\\d{3}).+(?<!rename)$",
   "sampleFileName": "WV03_20140824082814_10400100012AD900_14AUG24082814-M1BS-504548417070_01_P001-BROWSE.jpg",

diff --git a/app/stacks/cumulus/resources/collections/WV03_Pan_L1B___1.json b/app/stacks/cumulus/resources/collections/WV03_Pan_L1B___1.json
@@ -1,7 +1,7 @@
 {
   "name": "WV03_Pan_L1B",
   "version": "1",
-  "duplicateHandling": "skip",
+  "duplicateHandling": "replace",
   "granuleId": ".*",
   "granuleIdExtraction": "^(WV03_.+-P1BS-.+_P\\d{3}).+(?<!rename)$",
   "sampleFileName": "WV03_20140824052550_104001000109D200_14AUG24052550-P1BS-506481065090_01_P001-BROWSE.jpg",

diff --git a/app/stacks/cumulus/resources/rules/GE01_MSI_L1B/v1/GE01_MSI_L1B___1_2014_NovDec.json b/app/stacks/cumulus/resources/rules/GE01_MSI_L1B/v1/GE01_MSI_L1B___1_2014_NovDec.json
@@ -0,0 +1,24 @@
+{
+  "name": "GE01_MSI_L1B___1_2014_NovDec",
+  "state": "DISABLED",
+  "rule": {
+    "type": "onetime"
+  },
+  "provider": "maxar",
+  "collection": {
+    "name": "GE01_MSI_L1B",
+    "version": "1"
+  },
+  "workflow": "DiscoverAndQueueGranules",
+  "meta": {
+    "discoverOnly": false,
+    "providerPathFormat": "'css/nga/GE01/1B/'yyyy/DDD",
+    "ingestedPathFormat": "'GE01_MSI_L1B___1/'yyyy/DDD",
+    "rule": {
+      "state": "DISABLED"
+    },
+    "startDate": "2014-11-22T00:00:00Z",
+    "endDate": "2015-01-01T00:00:00Z",
+    "step": "P1D"
+  }
+}
diff --git a/app/stacks/cumulus/resources/rules/GE01_MSI_L1B/v1/GE01_MSI_L1B___1_2014_NovDec_2.json b/app/stacks/cumulus/resources/rules/GE01_MSI_L1B/v1/GE01_MSI_L1B___1_2014_NovDec_2.json
@@ -0,0 +1,24 @@
+{
+  "name": "GE01_MSI_L1B___1_2014_NovDec_2",
+  "state": "DISABLED",
+  "rule": {
+    "type": "onetime"
+  },
+  "provider": "maxar",
+  "collection": {
+    "name": "GE01_MSI_L1B",
+    "version": "1"
+  },
+  "workflow": "DiscoverAndQueueGranules",
+  "meta": {
+    "discoverOnly": false,
+    "providerPathFormat": "'css/nga/GE01/1B/'yyyy/DDD",
+    "ingestedPathFormat": "'GE01_MSI_L1B___1/'yyyy/DDD",
+    "rule": {
+      "state": "DISABLED"
+    },
+    "startDate": "2014-11-22T00:00:00Z",
+    "endDate": "2015-01-01T00:00:00Z",
+    "step": "P1D"
+  }
+}
diff --git a/app/stacks/cumulus/resources/rules/GE01_MSI_L1B/v1/GE01_MSI_L1B___1_Migration.json b/app/stacks/cumulus/resources/rules/GE01_MSI_L1B/v1/GE01_MSI_L1B___1_Migration.json
@@ -0,0 +1,21 @@
+{
+  "name": "GE01_MSI_L1B___1_Migration",
+  "state": "DISABLED",
+  "rule": {
+    "type": "onetime"
+  },
+  "provider": "cumulus",
+  "collection": {
+    "name": "GE01_MSI_L1B",
+    "version": "1"
+  },
+  "workflow": "DiscoverAndQueueGranules",
+  "meta": {
+    "discoverOnly": false,
+    "providerPathFormat": "'GE01_MSI_L1B___1/'yyyy/DDD",
+    "ingestedPathFormat": "'GE01_MSI_L1B___1/'yyyy/DDD",
+    "startDate": "2009-01-01T00:00:00Z",
+    "endDate": "2023-01-01T00:00:00Z",
+    "step": "P1D"
+  }
+}
diff --git a/app/stacks/cumulus/resources/rules/GE01_MSI_L1B/v1/GE01_MSI_L1B___1_UAT_Jan2009.json b/app/stacks/cumulus/resources/rules/GE01_MSI_L1B/v1/GE01_MSI_L1B___1_UAT_Jan2009.json
@@ -0,0 +1,24 @@
+{
+  "name": "GE01_MSI_L1B___1_UAT_Jan2009",
+  "state": "DISABLED",
+  "rule": {
+    "type": "onetime"
+  },
+  "provider": "maxar",
+  "collection": {
+    "name": "GE01_MSI_L1B",
+    "version": "1"
+  },
+  "workflow": "DiscoverAndQueueGranules",
+  "meta": {
+    "discoverOnly": false,
+    "providerPathFormat": "'css/nga/GE01/1B/'yyyy/DDD",
+    "ingestedPathFormat": "'GE01_MSI_L1B___1/'yyyy/DDD",
+    "rule": {
+      "state": "DISABLED"
+    },
+    "startDate": "2009-01-15T00:00:00Z",
+    "endDate": "2009-01-31T00:00:00Z",
+    "step": "P1D"
+  }
+}
diff --git a/app/stacks/cumulus/resources/rules/PSScene3Band/v1/PSScene3Band___1.json b/app/stacks/cumulus/resources/rules/PSScene3Band/v1/PSScene3Band___1.json
@@ -0,0 +1,21 @@
+{
+  "name": "PSScene3Band___1",
+  "state": "DISABLED",
+  "provider": "cumulus",
+  "collection": {
+    "name": "PSScene3Band",
+    "version": "1"
+  },
+  "workflow": "DiscoverAndQueueGranules",
+  "rule": {
+    "type": "onetime"
+  },
+  "meta": {
+    "discoverOnly": false,
+    "providerPathFormat": "'planet/PSScene3Band/'yyyyMMdd",
+    "ingestedPathFormat": "'PSScene3Band___1/'yyyy/MM/dd",
+    "startDate": "2020-12-01T00:00:00Z",
+    "endDate": "2020-12-03T00:00:00Z",
+    "step": "P1D"
+  }
+}
diff --git a/app/stacks/cumulus/resources/rules/PSScene3Band/v1/PSScene3Band___1_Migration.json b/app/stacks/cumulus/resources/rules/PSScene3Band/v1/PSScene3Band___1_Migration.json
@@ -0,0 +1,21 @@
+{
+  "name": "PSScene3Band___1_Migration",
+  "state": "DISABLED",
+  "provider": "cumulus",
+  "collection": {
+    "name": "PSScene3Band",
+    "version": "1"
+  },
+  "workflow": "DiscoverAndQueueGranules",
+  "rule": {
+    "type": "onetime"
+  },
+  "meta": {
+    "discoverOnly": false,
+    "providerPathFormat": "'planet/PSScene3Band/'yyyyMMdd",
+    "ingestedPathFormat": "'PSScene3Band___1/'yyyy/MM/dd",
+    "startDate": "2015-01-01T00:00:00Z",
+    "endDate": "2021-01-01T00:00:00Z",
+    "step": "P1D"
+  }
+}
diff --git a/app/stacks/cumulus/resources/rules/WV01_Pan_L1B/v1/WV01_Pan_L1B___1_2008__StartFeb16.json b/app/stacks/cumulus/resources/rules/WV01_Pan_L1B/v1/WV01_Pan_L1B___1_2008__StartFeb16.json
@@ -0,0 +1,24 @@
+{
+  "name": "WV01_Pan_L1B___1_2008__StartFeb16",
+  "state": "DISABLED",
+  "rule": {
+    "type": "onetime"
+  },
+  "provider": "maxar",
+  "collection": {
+    "name": "WV01_Pan_L1B",
+    "version": "1"
+  },
+  "workflow": "DiscoverAndQueueGranules",
+  "meta": {
+    "discoverOnly": false,
+    "providerPathFormat": "'css/nga/WV01/1B/'yyyy/DDD",
+    "ingestedPathFormat": "'WV01_Pan_L1B___1/'yyyy/DDD",
+    "rule": {
+      "state": "DISABLED"
+    },
+    "startDate": "2008-02-16T00:00:00Z",
+    "endDate": "2009-01-01T00:00:00Z",
+    "step": "P1D"
+  }
+}
diff --git a/app/stacks/cumulus/resources/rules/WV01_Pan_L1B/v1/WV01_Pan_L1B___1_Migration.json b/app/stacks/cumulus/resources/rules/WV01_Pan_L1B/v1/WV01_Pan_L1B___1_Migration.json
@@ -0,0 +1,21 @@
+{
+  "name": "WV01_Pan_L1B___1_Migration",
+  "state": "DISABLED",
+  "rule": {
+    "type": "onetime"
+  },
+  "provider": "cumulus",
+  "collection": {
+    "name": "WV01_Pan_L1B",
+    "version": "1"
+  },
+  "workflow": "DiscoverAndQueueGranules",
+  "meta": {
+    "discoverOnly": false,
+    "providerPathFormat": "'WV01_Pan_L1B___1/'yyyy/DDD",
+    "ingestedPathFormat": "'WV01_Pan_L1B___1/'yyyy/DDD",
+    "startDate": "2007-01-01T00:00:00Z",
+    "endDate": "2023-01-01T00:00:00Z",
+    "step": "P1D"
+  }
+}
diff --git a/app/stacks/cumulus/resources/rules/WV02_MSI_L1B/v1/WV02_MSI_L1B___1_Migration.json b/app/stacks/cumulus/resources/rules/WV02_MSI_L1B/v1/WV02_MSI_L1B___1_Migration.json
@@ -0,0 +1,21 @@
+{
+  "name": "WV02_MSI_L1B___1_Migration",
+  "state": "DISABLED",
+  "rule": {
+    "type": "onetime"
+  },
+  "provider": "cumulus",
+  "collection": {
+    "name": "WV02_MSI_L1B",
+    "version": "1"
+  },
+  "workflow": "DiscoverAndQueueGranules",
+  "meta": {
+    "discoverOnly": false,
+    "providerPathFormat": "'WV02_MSI_L1B___1/'yyyy/DDD",
+    "ingestedPathFormat": "'WV02_MSI_L1B___1/'yyyy/DDD",
+    "startDate": "2009-01-01T00:00:00Z",
+    "endDate": "2023-01-01T00:00:00Z",
+    "step": "P1D"
+  }
+}
diff --git a/app/stacks/cumulus/resources/rules/WV03_MSI_L1B/v1/WV03_MSI_L1B___1_Migration.json b/app/stacks/cumulus/resources/rules/WV03_MSI_L1B/v1/WV03_MSI_L1B___1_Migration.json
@@ -0,0 +1,21 @@
+{
+  "name": "WV03_MSI_L1B___1_Migration",
+  "state": "DISABLED",
+  "rule": {
+    "type": "onetime"
+  },
+  "provider": "cumulus",
+  "collection": {
+    "name": "WV03_MSI_L1B",
+    "version": "1"
+  },
+  "workflow": "DiscoverAndQueueGranules",
+  "meta": {
+    "discoverOnly": false,
+    "providerPathFormat": "'WV03_MSI_L1B___1/'yyyy/DDD",
+    "ingestedPathFormat": "'WV03_MSI_L1B___1/'yyyy/DDD",
+    "startDate": "2014-01-01T00:00:00Z",
+    "endDate": "2023-01-01T00:00:00Z",
+    "step": "P1D"
+  }
+}
diff --git a/app/stacks/cumulus/resources/rules/WV03_Pan_L1B/v1/WV03_Pan_L1B___1_2018__Finish.json b/app/stacks/cumulus/resources/rules/WV03_Pan_L1B/v1/WV03_Pan_L1B___1_2018__Finish.json
@@ -0,0 +1,24 @@
+{
+  "name": "WV03_Pan_L1B___1_2018__Finish",
+  "state": "DISABLED",
+  "rule": {
+    "type": "onetime"
+  },
+  "provider": "maxar",
+  "collection": {
+    "name": "WV03_Pan_L1B",
+    "version": "1"
+  },
+  "workflow": "DiscoverAndQueueGranules",
+  "meta": {
+    "discoverOnly": false,
+    "providerPathFormat": "'css/nga/WV03/1B/'yyyy/DDD",
+    "ingestedPathFormat": "'WV03_Pan_L1B___1/'yyyy/DDD",
+    "rule": {
+      "state": "DISABLED"
+    },
+    "startDate": "2018-11-14T00:00:00Z",
+    "endDate": "2019-01-01T00:00:00Z",
+    "step": "P1D"
+  }
+}
diff --git a/app/stacks/cumulus/resources/rules/WV03_Pan_L1B/v1/WV03_Pan_L1B___1_2021_Jan13_and_on.json b/app/stacks/cumulus/resources/rules/WV03_Pan_L1B/v1/WV03_Pan_L1B___1_2021_Jan13_and_on.json
@@ -0,0 +1,24 @@
+{
+  "name": "WV03_Pan_L1B___1_2021_Jan13_and_on",
+  "state": "DISABLED",
+  "rule": {
+    "type": "onetime"
+  },
+  "provider": "maxar",
+  "collection": {
+    "name": "WV03_Pan_L1B",
+    "version": "1"
+  },
+  "workflow": "DiscoverAndQueueGranules",
+  "meta": {
+    "discoverOnly": false,
+    "providerPathFormat": "'css/nga/WV03/1B/'yyyy/DDD",
+    "ingestedPathFormat": "'WV03_Pan_L1B___1/'yyyy/DDD",
+    "rule": {
+      "state": "DISABLED"
+    },
+    "startDate": "2021-01-13T00:00:00Z",
+    "endDate": "2022-01-01T00:00:00Z",
+    "step": "P1D"
+  }
+}
diff --git a/app/stacks/cumulus/resources/rules/WV03_Pan_L1B/v1/WV03_Pan_L1B___1_2021_Jan14_and_on.json b/app/stacks/cumulus/resources/rules/WV03_Pan_L1B/v1/WV03_Pan_L1B___1_2021_Jan14_and_on.json
@@ -0,0 +1,24 @@
+{
+  "name": "WV03_Pan_L1B___1_2021_Jan14_and_on",
+  "state": "DISABLED",
+  "rule": {
+    "type": "onetime"
+  },
+  "provider": "maxar",
+  "collection": {
+    "name": "WV03_Pan_L1B",
+    "version": "1"
+  },
+  "workflow": "DiscoverAndQueueGranules",
+  "meta": {
+    "discoverOnly": false,
+    "providerPathFormat": "'css/nga/WV03/1B/'yyyy/DDD",
+    "ingestedPathFormat": "'WV03_Pan_L1B___1/'yyyy/DDD",
+    "rule": {
+      "state": "DISABLED"
+    },
+    "startDate": "2021-01-14T00:00:00Z",
+    "endDate": "2022-01-01T00:00:00Z",
+    "step": "P1D"
+  }
+}