[#3725] Add problems to the email digest

src/openforms/emails/tasks.py

@@ -5,13 +5,11 @@
 from django.utils import timezone
 from django.utils.translation import gettext_lazy as _
 
-from django_yubin.models import Message
-
 from openforms.celery import app
 from openforms.config.models import GlobalConfiguration
-from openforms.logging.models import TimelineLogProxy
+from openforms.registrations.utils import collect_registrations_failures
 
-from .utils import send_mail_html
+from .utils import collect_failed_emails, send_mail_html
 
 
 @app.task
@@ -20,32 +18,24 @@ def send_email_digest() -> None:
     if not (recipients := config.recipients_email_digest):
         return
 
-    period_start = timezone.now() - timedelta(days=1)
+    yesterday = timezone.now() - timedelta(days=1)
 
-    logs = TimelineLogProxy.objects.filter(
-        timestamp__gt=period_start,
-        extra_data__status=Message.STATUS_FAILED,
-        extra_data__include_in_daily_digest=True,
-    ).distinct("content_type", "extra_data__status", "extra_data__event")
+    failed_emails = collect_failed_emails(yesterday)
+    failed_registrations = collect_registrations_failures(yesterday)
 
-    if not logs:
+    if not (failed_emails or failed_registrations):
         return
 
     content = render_to_string(
         "emails/admin_digest.html",
         {
-            "logs": [
-                {
-                    "submission_uuid": log.content_object.uuid,
-                    "event": log.extra_data["event"],
-                }
-                for log in logs
-            ],
+            "failed_emails": failed_emails,
+            "failed_registrations": failed_registrations,
         },
     )
 
     send_mail_html(
-        _("[Open Forms] Daily summary of failed emails"),
+        _("[Open Forms] Daily summary of failed procedures"),
         content,
         settings.DEFAULT_FROM_EMAIL,
         recipients,

src/openforms/emails/templates/emails/admin_digest.html

@@ -1,10 +1,24 @@
 {% load static i18n %}
 
 <p>
-    {% blocktranslate %}Here is a summary of the emails that failed to send yesterday:{% endblocktranslate %}
+    {% blocktranslate %}Here is a summary of the failed procedures in the past 24 hours:{% endblocktranslate %}
 </p>
-<ul>
-    {% for log in logs %}<li>- Email for the event "{{ log.event }}" for submission {{ log.submission_uuid }}.</li>
-    {% endfor %}
-</ul>
 
+{% if failed_emails %}
+    <h5>{% trans "Emails" %}</h5>
+    <ul>
+        {% for email in failed_emails %}
+            <li>- Email for the event "{{ email.event }}" for submission {{ email.submission_uuid }}.</li>
+        {% endfor %}
+    </ul>
+{% endif %}
+
+{% if failed_registrations %}
+    <h5>{% trans "Registrations" %}</h5>
+    <ul>
+        {% for registration in failed_registrations %}
+            <li>- {{ registration.form_name }} failed {{ registration.failed_submissions_counter }} times between {{ registration.initial_failure_at|date:"H:i" }} and {{ registration.last_failure_at|date:"H:i" }}.</li>
+            <a href="{{ registration.admin_link }}">Click here to see all the failed submissions for form {{ registration.form_name }} in the admin</a>
+        {% endfor %}
+    </ul>
+{% endif %}

src/openforms/emails/tests/test_tasks.py

@@ -9,7 +9,9 @@
 from freezegun import freeze_time
 
 from openforms.config.models import GlobalConfiguration
+from openforms.forms.tests.factories import FormFactory
 from openforms.logging.tests.factories import TimelineLogProxyFactory
+from openforms.submissions.constants import RegistrationStatuses
 from openforms.submissions.tests.factories import SubmissionFactory
 
 from ..tasks import send_email_digest
@@ -72,8 +74,9 @@ def test_create_digest_email(self):
         expected_content = dedent(
             f"""
             <p>
-                Here is a summary of the emails that failed to send yesterday:
+                Here is a summary of the failed procedures in the past 24 hours:
             </p>
+            <h5>Emails</h5>
             <ul>
                 <li>- Email for the event "registration" for submission {submission.uuid}.</li>
 
@@ -128,8 +131,9 @@ def test_that_repeated_failures_are_not_mentioned_multiple_times(self):
         expected_content = dedent(
             f"""
             <p>
-                Here is a summary of the emails that failed to send yesterday:
+                Here is a summary of the failed procedures in the past 24 hours:
             </p>
+            <h5>Emails</h5>
             <ul>
                 <li>- Email for the event "registration" for submission {submission.uuid}.</li>
 
@@ -177,3 +181,92 @@ def test_no_recipients(self):
             send_email_digest()
 
         self.assertEqual(0, len(mail.outbox))
+
+    def test_failed_submissions_are_correctly_sent(self):
+        # 1st form with 2 failures in the past 24 hours
+        form_1 = FormFactory.create()
+        failed_submission_1 = SubmissionFactory.create(
+            form=form_1, registration_status=RegistrationStatuses.failed
+        )
+        failed_submission_2 = SubmissionFactory.create(
+            form=form_1, registration_status=RegistrationStatuses.failed
+        )
+
+        # 1st failure
+        with freeze_time("2023-01-02T12:30:00+01:00"):
+            TimelineLogProxyFactory.create(
+                template="logging/events/registration_failure.txt",
+                content_object=failed_submission_1,
+                extra_data={
+                    "log_event": "registration_failure",
+                    "include_in_daily_digest": True,
+                },
+            )
+
+        # 2nd failure
+        with freeze_time("2023-01-02T18:30:00+01:00"):
+            TimelineLogProxyFactory.create(
+                template="logging/events/registration_failure.txt",
+                content_object=failed_submission_2,
+                extra_data={
+                    "log_event": "registration_failure",
+                    "include_in_daily_digest": True,
+                },
+            )
+
+        # 2nd form with 1 failure in the past 24 hours
+        form_2 = FormFactory.create()
+        failed_submission = SubmissionFactory.create(
+            form=form_2, registration_status=RegistrationStatuses.failed
+        )
+
+        # failure
+        with freeze_time("2023-01-02T12:30:00+01:00"):
+            TimelineLogProxyFactory.create(
+                template="logging/events/registration_failure.txt",
+                content_object=failed_submission,
+                extra_data={
+                    "log_event": "registration_failure",
+                    "include_in_daily_digest": True,
+                },
+            )
+
+        with (
+            freeze_time("2023-01-03T01:00:00+01:00"),
+            patch(
+                "openforms.emails.tasks.GlobalConfiguration.get_solo",
+                return_value=GlobalConfiguration(
+                    recipients_email_digest=["[email protected]"]
+                ),
+            ),
+            patch("openforms.emails.tasks.send_mail_html") as patch_email,
+        ):
+            send_email_digest()
+
+            patch_email.assert_called_once()
+
+            args = patch_email.call_args.args
+            expected_content = dedent(
+                f"""
+                <p>
+                    Here is a summary of the failed procedures in the past 24 hours:
+                </p>
+                <h5>Registrations</h5>
+                <ul>
+                    <li>- {form_1.name} failed 2 times between 12:30 and 18:30.</li>
+                    <a href="/admin/submissions/submission/?form__id__exact={form_1.id}&needs_on_completion_retry__exact=1&from_time=24hAgo">
+                    Click here to see all the failed submissions for form {form_1.name} in the admin
+                    </a>
+                    <li>- {form_2.name} failed 1 times between 12:30 and 12:30.</li>
+                    <a href="/admin/submissions/submission/?form__id__exact={form_2.id}&needs_on_completion_retry__exact=1&from_time=24hAgo">
+                    Click here to see all the failed submissions for form {form_2.name} in the admin
+                    </a>
+                </ul>
+
+            """
+            ).strip()
+
+        self.assertHTMLEqual(
+            expected_content,
+            args[1].strip(),
+        )

src/openforms/emails/utils.py

@@ -1,16 +1,19 @@
 import logging
 import re
+from datetime import datetime
 from typing import Any, Sequence
 from urllib.parse import urlsplit
 
 from django.conf import settings
 from django.template.loader import get_template
 
+from django_yubin.models import Message
 from mail_cleaner.mail import send_mail_plus
 from mail_cleaner.sanitizer import sanitize_content as _sanitize_content
 from mail_cleaner.text import strip_tags_plus
 
 from openforms.config.models import GlobalConfiguration, Theme
+from openforms.logging.models import TimelineLogProxy
 from openforms.template import openforms_backend, render_from_string
 
 from .context import get_wrapper_context
@@ -100,3 +103,23 @@ def render_email_template(
         backend=openforms_backend,
         disable_autoescape=disable_autoescape,
     )
+
+
+def collect_failed_emails(since: datetime) -> list[dict[str, str] | None]:
+    logs = TimelineLogProxy.objects.filter(
+        timestamp__gt=since,
+        extra_data__status=Message.STATUS_FAILED,
+        extra_data__include_in_daily_digest=True,
+    ).distinct("content_type", "extra_data__status", "extra_data__event")
+
+    if not logs:
+        return
+    failed_emails = [
+        {
+            "submission_uuid": log.content_object.uuid,
+            "event": log.extra_data["event"],
+        }
+        for log in logs
+    ]
+
+    return failed_emails

src/openforms/logging/logevent.py

@@ -208,12 +208,17 @@ def registration_success(submission: Submission, plugin):
     )
 
 
-def registration_failure(submission: Submission, error: Exception, plugin=None):
+def registration_failure(
+    submission: Submission, error: Exception, include_in_daily_digest: bool, plugin=None
+):
     _create_log(
         submission,
         "registration_failure",
         plugin=plugin,
         error=error,
+        extra_data={
+            "include_in_daily_digest": include_in_daily_digest,
+        },
     )
 
 

src/openforms/registrations/tasks.py

@@ -224,7 +224,7 @@ def register_submission(submission_id: int, event: PostSubmissionEvents | str) -
             RegistrationStatuses.failed,
             {"traceback": "".join(traceback.format_exception(exc))},
         )
-        logevent.registration_failure(submission, exc)
+        logevent.registration_failure(submission, exc, True)
         if event == PostSubmissionEvents.on_retry:
             raise exc
         return
@@ -238,7 +238,7 @@ def register_submission(submission_id: int, event: PostSubmissionEvents | str) -
     try:
         options_serializer.is_valid(raise_exception=True)
     except ValidationError as exc:
-        logevent.registration_failure(submission, exc, plugin)
+        logevent.registration_failure(submission, exc, True, plugin)
         logger.warning(
             "Registration using plugin '%r' for submission '%s' failed",
             plugin,
@@ -268,7 +268,7 @@ def register_submission(submission_id: int, event: PostSubmissionEvents | str) -
         submission.save_registration_status(
             RegistrationStatuses.failed, {"traceback": traceback.format_exc()}
         )
-        logevent.registration_failure(submission, exc, plugin)
+        logevent.registration_failure(submission, exc, True, plugin)
         submission.save_registration_status(
             RegistrationStatuses.failed, {"traceback": traceback.format_exc()}
         )
@@ -285,7 +285,7 @@ def register_submission(submission_id: int, event: PostSubmissionEvents | str) -
         submission.save_registration_status(
             RegistrationStatuses.failed, {"traceback": traceback.format_exc()}
         )
-        logevent.registration_failure(submission, exc, plugin)
+        logevent.registration_failure(submission, exc, True, plugin)
         if event == PostSubmissionEvents.on_retry:
             raise exc
         return

src/openforms/registrations/utils.py

@@ -1,8 +1,11 @@
+from datetime import datetime
 from typing import Any, Callable
 
 from glom import assign, glom
 
+from openforms.logging.models import TimelineLogProxy
 from openforms.submissions.models import Submission
+from openforms.submissions.utils import get_filtered_submission_admin_url
 
 unset = object()
 
@@ -30,3 +33,34 @@ def execute_unless_result_exists(
     assign(submission.registration_result, spec, result, missing=dict)
     submission.save(update_fields=["registration_result"])
     return callback_result
+
+
+def collect_registrations_failures(since: datetime) -> list[dict[str, str] | None]:
+    logs = TimelineLogProxy.objects.filter(
+        timestamp__gt=since,
+        extra_data__log_event="registration_failure",
+        extra_data__include_in_daily_digest=True,
+    ).order_by("timestamp")
+
+    if not logs:
+        return
+
+    connected_forms = {}
+    for log in logs:
+        form = log.content_object.form
+
+        if form.uuid in connected_forms:
+            connected_forms[form.uuid]["failed_submissions_counter"] += 1
+            connected_forms[form.uuid]["last_failure_at"] = log.timestamp
+        else:
+            connected_forms[form.uuid] = {
+                "form_name": form.name,
+                "failed_submissions_counter": 1,
+                "initial_failure_at": log.timestamp,
+                "last_failure_at": log.timestamp,
+                "admin_link": get_filtered_submission_admin_url(form.id, 1, "24hAgo"),
+            }
+
+    failed_registrations = [form_detail for form_detail in connected_forms.values()]
+
+    return failed_registrations

src/openforms/submissions/admin.py

@@ -1,9 +1,12 @@
+from datetime import timedelta
+
 from django import forms
 from django.contrib import admin, messages
 from django.contrib.contenttypes.admin import GenericTabularInline
 from django.db.models import Q
 from django.http import Http404
 from django.template.defaultfilters import filesizeformat
+from django.utils import timezone
 from django.utils.translation import gettext_lazy as _, ngettext
 
 from privates.admin import PrivateMediaMixin
@@ -82,6 +85,21 @@
         return [self.parameter_name]
 
 
+class SubmissionTimeListFilter(admin.SimpleListFilter):
+    title = _("time")
+    parameter_name = "from_time"
+
+    def lookups(self, request, model_admin):
+        return [
+            ("24hAgo", _("In the past 24 hours")),
+        ]
+
+    def queryset(self, request, queryset):
+        yesterday = timezone.now() - timedelta(days=1)
+        if self.value() == "24hAgo":
+            return queryset.filter(last_register_date__gt=yesterday)
+
+
 class SubmissionStepInline(admin.StackedInline):
     model = SubmissionStep
     extra = 0
@@ -170,6 +188,7 @@
     )
     list_filter = (
         SubmissionTypeListFilter,
+        SubmissionTimeListFilter,
         "registration_status",
         "needs_on_completion_retry",
         "form",