Route SNS notifications through a Lambda function (#5246)

DataBiosphere · Aug 30, 2023 · 497b492 · 497b492
1 parent 7fb38a6
commit 497b492
Show file tree

Hide file tree

Showing 6 changed files with 170 additions and 4 deletions.
diff --git a/lambdas/indexer/app.py b/lambdas/indexer/app.py
@@ -34,6 +34,9 @@
 from azul.indexer.log_forwarding_controller import (
     LogForwardingController,
 )
+from azul.indexer.notify_controller import (
+    NotifyController,
+)
 from azul.logging import (
     configure_app_logging,
 )
@@ -74,6 +77,10 @@ class IndexerApp(AzulChaliceApp, SignatureHelper):
     def health_controller(self):
         return self._controller(HealthController, lambda_name='indexer')
 
+    @cached_property
+    def notify_controller(self):
+        return self._controller(NotifyController)
+
     @cached_property
     def index_controller(self) -> IndexController:
         return self._controller(IndexController)
@@ -97,6 +104,13 @@ def log_forwarder(self, prefix: str):
         else:
             return lambda func: func
 
+    @property
+    def monitoring(self):
+        if config.enable_monitoring:
+            return self.on_sns_message(topic=config.qualified_resource_name('monitoring'))
+        else:
+            return lambda func: func
+
     def _authenticate(self) -> Optional[HMACAuthentication]:
         return self.auth_from_request(self.current_request)
 
@@ -291,3 +305,8 @@ def forward_alb_logs(event: chalice.app.S3Event):
 @app.log_forwarder(config.s3_access_log_path_prefix(deployment=None))
 def forward_s3_logs(event: chalice.app.S3Event):
     app.log_controller.forward_s3_access_logs(event)
+
+
+@app.monitoring
+def notify(event: chalice.app.SNSEvent):
+    app.notify_controller.digest(event)
diff --git a/src/azul/deployment.py b/src/azul/deployment.py
@@ -139,6 +139,10 @@ def securityhub(self):
     def sns(self):
         return self.client('sns')
 
+    @property
+    def ses(self):
+        return self.client('ses')
+
     @property
     def sts(self):
         return self.client('sts')

diff --git a/src/azul/indexer/notify_controller.py b/src/azul/indexer/notify_controller.py
@@ -0,0 +1,53 @@
+import json
+
+import chalice.app
+from more_itertools import (
+    one,
+)
+
+from azul import (
+    JSON,
+    cached_property,
+)
+from azul.chalice import (
+    AppController,
+)
+from azul.indexer.notify_service import (
+    AzulEmailNotificationService,
+)
+
+
+class NotifyController(AppController):
+
+    @cached_property
+    def email(self):
+        return AzulEmailNotificationService()
+
+    def digest(self, event: chalice.app.SNSEvent) -> None:
+        email_body = self._format_to_html_body(json.loads(event.message))
+        self.email.notify_group(subject=event.subject, html=email_body)
+
+    def _format_to_html_body(self, body: dict) -> str:
+        trigger: dict = body['Trigger']
+        metric: dict = one([m for m in trigger.pop('Metrics')
+                            if 'MetricStat' in m])['MetricStat']['Metric']
+        alarm_name = body['AlarmName']
+        trigger = {
+            k: v if type(v) is str else str(v)
+            for k, v in trigger.items()
+        }
+        return f'''<html>
+        <body>
+            <h3>
+                The alarm {alarm_name!r} is in {body['NewStateValue']!r} state.<br />{body['NewStateReason']}
+            </h3>
+            {''.join(['<p><strong>' + h[0] + '</strong><br />' + h[1] + '</p>'
+                      for h in (('Alarm State', body['NewStateValue']),
+                                ('Alarm Name', alarm_name),
+                                ('Timestamp', body['StateChangeTime']),
+                                ('Namespace', metric.get('Namespace')),
+                                ('Metric Name', metric.get('MetricName')),
+                                *(trigger.items()),
+                                )])}
+        </body>
+        </html>'''
diff --git a/src/azul/indexer/notify_service.py b/src/azul/indexer/notify_service.py
@@ -0,0 +1,31 @@
+from azul import (
+    JSON,
+    config,
+)
+from azul.deployment import (
+    aws,
+)
+
+
+class AzulEmailNotificationService:
+
+    def notify_group(self, subject: str, html: str) -> None:
+        aws.ses.send_email(
+            Source=f'NOTIFY@{config.indexer_endpoint.host}',
+            Destination={
+                'ToAddresses': [config.monitoring_email]
+            },
+            Message=self._format_message(subject, html)
+        )
+
+    def _format_message(self, subject: str, html: str) -> JSON:
+        return {
+            'Subject': {
+                'Data': subject
+            },
+            'Body': {
+                'Html': {
+                    'Data': html
+                }
+            }
+        }
diff --git a/terraform/api_gateway.tf.json.template.py b/terraform/api_gateway.tf.json.template.py
@@ -317,7 +317,52 @@ def for_domain(cls, domain):
                         'sampled_requests_enabled': True,
                     }
                 }
-            }
+            },
+            **(
+                (
+                    {
+                        'aws_route53_record': {
+                            'notify_amazonses_record': {
+                                'zone_id': '${data.aws_route53_zone.dev_singlecell_gi_ucsc_edu.id}',
+                                'name': f'_amazonses.{config.api_lambda_domain("indexer")}',
+                                'type': 'TXT',
+                                'ttl': '600',
+                                'records': [
+                                    '${aws_ses_domain_identity.notify.verification_token}'
+                                ]
+                            }
+                        },
+                        'aws_ses_domain_identity': {
+                            'notify': {
+                                'domain': config.api_lambda_domain('indexer')
+                            }
+                        },
+                        'aws_ses_identity_policy': {
+                            'notify': {
+                                'identity': '${aws_ses_domain_identity.notify.arn}',
+                                'name': config.qualified_resource_name('notify'),
+                                'policy': json.dumps({
+                                    "Version": "2012-10-17",
+                                    "Statement": [
+                                        {
+                                            'Effect': 'Allow',
+                                            'Principal': {
+                                                'AWS': f'arn:aws:sts::{aws.account}:assumed-role/{config.indexer_name}/'
+                                                       + config.indexer_function_name('notify')
+                                            },
+                                            'Action': [
+                                                'ses:SendEmail',
+                                                'ses:SendRawEmail'
+                                            ],
+                                            'Resource': '${aws_ses_domain_identity.notify.arn}',
+                                        }
+                                    ]
+                                })
+                            }
+                        }
+                    }
+                ) if config.enable_monitoring else {}
+            )
         },
         *([
               {
@@ -327,13 +372,21 @@ def for_domain(cls, domain):
                           'maximum_retry_attempts': 0
                       }
                       for function_name in (
-                          'forward_alb_logs',
-                          'forward_s3_logs',
+                          *(
+                              ('forward_alb_logs', 'forward_s3_logs')
+                              if config.enable_log_forwarding else
+                              ()
+                          ),
+                          *(
+                              ('notify',)
+                              if config.enable_monitoring else
+                              ()
+                          )
                       )
                   }
               }
           ]
-          if config.enable_log_forwarding else
+          if config.enable_log_forwarding or config.enable_monitoring else
           []),
         *(
             {

diff --git a/terraform/shared/shared.tf.json.template.py b/terraform/shared/shared.tf.json.template.py
@@ -160,6 +160,12 @@ def paren(s: str) -> str:
             vpc.default_vpc_name: {
                 'default': True
             }
+        },
+        'aws_route53_zone': {
+            'gitlab': {
+                'name': config.domain_name + '.',
+                'private_zone': False
+            }
         }
     },
     'resource': {