diff --git a/_templates/rule/new/rule.ejs.t b/_templates/rule/new/rule.ejs.t index 2ad3fd6..37da6f6 100644 --- a/_templates/rule/new/rule.ejs.t +++ b/_templates/rule/new/rule.ejs.t @@ -28,6 +28,7 @@ message: > "ingestedPathFormat": "<%- ingestedPathFormat %>", "startDate": "<%= startYear %>-01-01T00:00:00Z", "endDate": "<%= parseInt(endYear) + 1 %>-01-01T00:00:00Z", - "step": "P1D" + "step": "P1D", + "maxBatchSize": 200, } } diff --git a/app/stacks/cumulus/main.tf b/app/stacks/cumulus/main.tf index e417b8b..f686d17 100644 --- a/app/stacks/cumulus/main.tf +++ b/app/stacks/cumulus/main.tf @@ -161,7 +161,7 @@ resource "aws_cloudwatch_event_target" "background_job_queue_watcher" { arn = module.cumulus.sqs2sfThrottle_lambda_function_arn input = jsonencode( { - messageLimit = 500 + messageLimit = 1500 queueUrl = aws_sqs_queue.background_job_queue.id timeLimit = 60 } @@ -576,7 +576,7 @@ module "cumulus" { { id = "backgroundJobQueue", url = aws_sqs_queue.background_job_queue.id, - execution_limit = 300 + execution_limit = 500 } ] } diff --git a/app/stacks/cumulus/templates/discover-granules-workflow.asl.json b/app/stacks/cumulus/templates/discover-granules-workflow.asl.json index 6d92380..a2ad21f 100644 --- a/app/stacks/cumulus/templates/discover-granules-workflow.asl.json +++ b/app/stacks/cumulus/templates/discover-granules-workflow.asl.json @@ -287,6 +287,7 @@ "TargetPath": "$.payload" }, "task_config": { + "concurrency": 1, "queueUrl": "${background_job_queue_url}", "preferredQueueBatchSize": "{$.meta.collection.meta.preferredQueueBatchSize}", "provider": "{$.meta.provider}", diff --git a/src/lib/discovery.ts b/src/lib/discovery.ts index 03a4794..6058e57 100644 --- a/src/lib/discovery.ts +++ b/src/lib/discovery.ts @@ -61,13 +61,13 @@ type BucketKey = { readonly key: string; }; -// Default maximum batch size for batching granules after discovery is 1000, but this +// Default maximum batch size for batching granules after discovery is 200, but this // can be set on a per rule basis by setting `meta.maxBatchSize` in a rule definition. export const BatchGranulesInput = t.readonly( t.type({ config: t.type({ providerPath: t.string, - maxBatchSize: tt.fromNullable(t.number, 1000), + maxBatchSize: tt.fromNullable(t.number, 200), }), input: DiscoverGranulesOutput, })