diff --git a/src/openforms/emails/tasks.py b/src/openforms/emails/tasks.py
index 248e50ad53..5e96a0a421 100644
--- a/src/openforms/emails/tasks.py
+++ b/src/openforms/emails/tasks.py
@@ -1,51 +1,59 @@
-from datetime import timedelta
+from datetime import datetime, timedelta
+from typing import Any
from django.conf import settings
from django.template.loader import render_to_string
from django.utils import timezone
from django.utils.translation import gettext_lazy as _
-from django_yubin.models import Message
-
from openforms.celery import app
from openforms.config.models import GlobalConfiguration
-from openforms.logging.models import TimelineLogProxy
+from openforms.logging.service import (
+ collect_failed_emails,
+ collect_failed_registrations,
+)
from .utils import send_mail_html
+class Digest:
+ def __init__(self, since: datetime) -> None:
+ self.since = since
+
+ def get_context_data(self) -> dict[str, Any]:
+ failed_emails = collect_failed_emails(self.since)
+ failed_registrations = collect_failed_registrations(self.since)
+
+ if not (failed_emails or failed_registrations):
+ return {}
+
+ return {
+ "failed_emails": failed_emails,
+ "failed_registrations": failed_registrations,
+ }
+
+ def render(self) -> str:
+ if not (context := self.get_context_data()):
+ return ""
+
+ return render_to_string("emails/admin_digest.html", context)
+
+
@app.task
def send_email_digest() -> None:
config = GlobalConfiguration.get_solo()
if not (recipients := config.recipients_email_digest):
return
- period_start = timezone.now() - timedelta(days=1)
-
- logs = TimelineLogProxy.objects.filter(
- timestamp__gt=period_start,
- extra_data__status=Message.STATUS_FAILED,
- extra_data__include_in_daily_digest=True,
- ).distinct("content_type", "extra_data__status", "extra_data__event")
+ yesterday = timezone.now() - timedelta(days=1)
+ digest = Digest(since=yesterday)
+ content = digest.render()
- if not logs:
+ if not content:
return
- content = render_to_string(
- "emails/admin_digest.html",
- {
- "logs": [
- {
- "submission_uuid": log.content_object.uuid,
- "event": log.extra_data["event"],
- }
- for log in logs
- ],
- },
- )
-
send_mail_html(
- _("[Open Forms] Daily summary of failed emails"),
+ _("[Open Forms] Daily summary of detected problems"),
content,
settings.DEFAULT_FROM_EMAIL,
recipients,
diff --git a/src/openforms/emails/templates/emails/admin_digest.html b/src/openforms/emails/templates/emails/admin_digest.html
index a644f27753..f3a70dcc97 100644
--- a/src/openforms/emails/templates/emails/admin_digest.html
+++ b/src/openforms/emails/templates/emails/admin_digest.html
@@ -1,10 +1,37 @@
{% load static i18n %}
-
- {% blocktranslate %}Here is a summary of the emails that failed to send yesterday:{% endblocktranslate %}
-
-
- {% for log in logs %}- - Email for the event "{{ log.event }}" for submission {{ log.submission_uuid }}.
- {% endfor %}
-
+
+ {% blocktranslate %}Here is a summary of the problems detected in the past 24 hours:{% endblocktranslate %}
+
+{% if failed_emails %}
+ {% trans "Emails that failed to send" %}
+
+ {% for email in failed_emails %}
+ -
+ {% blocktranslate with event=email.event submission_uuid=email.submission_uuid trimmed %}
+ Email for the event "{{ event }}" for submission {{ submission_uuid }}.
+ {% endblocktranslate %}
+
+ {% endfor %}
+
+{% endif %}
+
+{% if failed_registrations %}
+ {% trans "Registrations" %}
+
+{% endif %}
diff --git a/src/openforms/emails/tests/test_tasks.py b/src/openforms/emails/tests/test_tasks.py
index ab5b277b8f..63f5d59b0c 100644
--- a/src/openforms/emails/tests/test_tasks.py
+++ b/src/openforms/emails/tests/test_tasks.py
@@ -1,5 +1,3 @@
-from datetime import datetime
-from textwrap import dedent
from unittest.mock import patch
from django.core import mail
@@ -9,7 +7,10 @@
from freezegun import freeze_time
from openforms.config.models import GlobalConfiguration
-from openforms.logging.tests.factories import TimelineLogProxyFactory
+from openforms.forms.tests.factories import FormFactory
+from openforms.logging import logevent
+from openforms.registrations.exceptions import RegistrationFailed
+from openforms.submissions.constants import RegistrationStatuses
from openforms.submissions.tests.factories import SubmissionFactory
from ..tasks import send_email_digest
@@ -22,35 +23,31 @@ def test_create_digest_email(self):
with freeze_time("2023-01-02T12:30:00+01:00"):
# Log email failed within last day
- TimelineLogProxyFactory.create(
- template="logging/events/email_status_change.txt",
- content_object=submission,
- extra_data={
- "status": Message.STATUS_FAILED,
- "event": "registration",
- "include_in_daily_digest": True,
- },
+ logevent.email_status_change(
+ submission,
+ event="registration",
+ status=Message.STATUS_FAILED,
+ status_label="Failed",
+ include_in_daily_digest=True,
)
+
# Successfully sent email
- TimelineLogProxyFactory.create(
- template="logging/events/email_status_change.txt",
- content_object=submission,
- extra_data={
- "status": Message.STATUS_SENT,
- "include_in_daily_digest": True,
- },
+ logevent.email_status_change(
+ submission,
+ event="registration",
+ status=Message.STATUS_SENT,
+ status_label="Sent",
+ include_in_daily_digest=True,
)
with freeze_time("2023-01-01T12:30:00+01:00"):
# Log email failed more than 24h ago
- TimelineLogProxyFactory.create(
- template="logging/events/email_status_change.txt",
- content_object=submission,
- timestamp=datetime(2023, 1, 1, 12, 30),
- extra_data={
- "status": Message.STATUS_FAILED,
- "include_in_daily_digest": True,
- },
+ logevent.email_status_change(
+ submission,
+ event="registration",
+ status=Message.STATUS_FAILED,
+ status_label="Failed",
+ include_in_daily_digest=True,
)
with (
@@ -61,54 +58,38 @@ def test_create_digest_email(self):
recipients_email_digest=["tralala@test.nl", "trblblb@test.nl"]
),
),
- patch("openforms.emails.tasks.send_mail_html") as patch_email,
):
send_email_digest()
- patch_email.assert_called_once()
-
- args = patch_email.call_args.args
-
- expected_content = dedent(
- f"""
-
- Here is a summary of the emails that failed to send yesterday:
-
-
- - - Email for the event "registration" for submission {submission.uuid}.
+ sent_email = mail.outbox[0]
+ submission_occurences = sent_email.body.count(str(submission.uuid))
-
- """
- ).strip()
-
- self.assertEqual(args[3], ["tralala@test.nl", "trblblb@test.nl"])
- self.assertHTMLEqual(
- expected_content,
- args[1].strip(),
+ self.assertEqual(
+ sent_email.subject, "[Open Forms] Daily summary of detected problems"
+ )
+ self.assertEqual(
+ sent_email.recipients(), ["tralala@test.nl", "trblblb@test.nl"]
)
+ self.assertEqual(submission_occurences, 1)
def test_that_repeated_failures_are_not_mentioned_multiple_times(self):
submission = SubmissionFactory.create()
with freeze_time("2023-01-02T12:30:00+01:00"):
- TimelineLogProxyFactory.create(
- template="logging/events/email_status_change.txt",
- content_object=submission,
- extra_data={
- "status": Message.STATUS_FAILED,
- "event": "registration",
- "include_in_daily_digest": True,
- },
+ logevent.email_status_change(
+ submission,
+ event="registration",
+ status=Message.STATUS_FAILED,
+ status_label="Failed",
+ include_in_daily_digest=True,
)
with freeze_time("2023-01-02T13:30:00+01:00"):
- TimelineLogProxyFactory.create(
- template="logging/events/email_status_change.txt",
- content_object=submission,
- extra_data={
- "status": Message.STATUS_FAILED,
- "event": "registration",
- "include_in_daily_digest": True,
- },
+ logevent.email_status_change(
+ submission,
+ event="registration",
+ status=Message.STATUS_FAILED,
+ status_label="Failed",
+ include_in_daily_digest=True,
)
with (
@@ -119,30 +100,21 @@ def test_that_repeated_failures_are_not_mentioned_multiple_times(self):
recipients_email_digest=["tralala@test.nl", "trblblb@test.nl"]
),
),
- patch("openforms.emails.tasks.send_mail_html") as patch_email,
):
send_email_digest()
- args = patch_email.call_args.args
+ sent_email = mail.outbox[0]
+ submission_occurencies = sent_email.body.count(str(submission.uuid))
- expected_content = dedent(
- f"""
-
- Here is a summary of the emails that failed to send yesterday:
-
-
- - - Email for the event "registration" for submission {submission.uuid}.
-
-
- """
- ).strip()
-
- self.assertHTMLEqual(
- expected_content,
- args[1].strip(),
+ self.assertEqual(
+ sent_email.subject, "[Open Forms] Daily summary of detected problems"
)
+ self.assertEqual(
+ sent_email.recipients(), ["tralala@test.nl", "trblblb@test.nl"]
+ )
+ self.assertEqual(submission_occurencies, 1)
- def test_no_email_send_if_no_logs(self):
+ def test_no_email_sent_if_no_logs(self):
with patch(
"openforms.emails.tasks.GlobalConfiguration.get_solo",
return_value=GlobalConfiguration(
@@ -153,18 +125,16 @@ def test_no_email_send_if_no_logs(self):
self.assertEqual(0, len(mail.outbox))
- def test_no_recipients(self):
+ def test_no_email_sent_if_no_recipients(self):
submission = SubmissionFactory.create()
with freeze_time("2023-01-02T12:30:00+01:00"):
- TimelineLogProxyFactory.create(
- template="logging/events/email_status_change.txt",
- content_object=submission,
- extra_data={
- "status": Message.STATUS_FAILED,
- "event": "registration",
- "include_in_daily_digest": True,
- },
+ logevent.email_status_change(
+ submission,
+ event="registration",
+ status=Message.STATUS_FAILED,
+ status_label="Failed",
+ include_in_daily_digest=True,
)
with (
@@ -177,3 +147,60 @@ def test_no_recipients(self):
send_email_digest()
self.assertEqual(0, len(mail.outbox))
+
+ def test_email_sent_if_failed_submissions_exist(self):
+ # 1st form with 2 failures in the past 24 hours
+ form_1 = FormFactory.create()
+ failed_submission_1 = SubmissionFactory.create(
+ form=form_1, registration_status=RegistrationStatuses.failed
+ )
+ failed_submission_2 = SubmissionFactory.create(
+ form=form_1, registration_status=RegistrationStatuses.failed
+ )
+
+ # 1st failure
+ with freeze_time("2023-01-02T12:30:00+01:00"):
+ logevent.registration_failure(
+ failed_submission_1,
+ RegistrationFailed("Registration plugin is not enabled"),
+ )
+
+ # 2nd failure
+ with freeze_time("2023-01-02T18:30:00+01:00"):
+ logevent.registration_failure(
+ failed_submission_2,
+ RegistrationFailed("Registration plugin is not enabled"),
+ )
+
+ # 2nd form with 1 failure in the past 24 hours
+ form_2 = FormFactory.create()
+ failed_submission = SubmissionFactory.create(
+ form=form_2, registration_status=RegistrationStatuses.failed
+ )
+
+ # failure
+ with freeze_time("2023-01-02T12:30:00+01:00"):
+ logevent.registration_failure(
+ failed_submission,
+ RegistrationFailed("Registration plugin is not enabled"),
+ )
+
+ with (
+ freeze_time("2023-01-03T01:00:00+01:00"),
+ patch(
+ "openforms.emails.tasks.GlobalConfiguration.get_solo",
+ return_value=GlobalConfiguration(
+ recipients_email_digest=["user@example.com"]
+ ),
+ ),
+ ):
+ send_email_digest()
+
+ sent_email = mail.outbox[0]
+
+ self.assertEqual(
+ sent_email.subject, "[Open Forms] Daily summary of detected problems"
+ )
+ self.assertEqual(sent_email.recipients(), ["user@example.com"])
+ self.assertIn(form_1.name, sent_email.body)
+ self.assertIn(form_2.name, sent_email.body)
diff --git a/src/openforms/emails/views.py b/src/openforms/emails/views.py
index fa6c065829..be12b8041c 100644
--- a/src/openforms/emails/views.py
+++ b/src/openforms/emails/views.py
@@ -1,6 +1,10 @@
+from datetime import timedelta
+
from django.shortcuts import get_object_or_404
+from django.utils import timezone
from django.views.generic import TemplateView
+from openforms.emails.tasks import Digest
from openforms.submissions.models import Submission
from openforms.utils.views import DevViewMixin, EmailDebugViewMixin
@@ -17,16 +21,28 @@ class EmailWrapperTestView(
def get_email_content(self):
content = "content goes here"
- if self.kwargs.get("submission_id"):
- submission = get_object_or_404(Submission, id=self.kwargs["submission_id"])
-
- content_template = get_confirmation_email_templates(submission)[1]
- context = get_confirmation_email_context_data(submission)
- mode = self._get_mode()
- if mode == "text":
- context["rendering_text"] = True
- content = render_email_template(content_template, context)
- if mode == "text":
- content = strip_tags_plus(content, keep_leading_whitespace=True)
+ match self.kwargs:
+ case {"submission_id": int()}:
+ submission = get_object_or_404(
+ Submission, id=self.kwargs["submission_id"]
+ )
+
+ content_template = get_confirmation_email_templates(submission)[1]
+ context = get_confirmation_email_context_data(submission)
+ mode = self._get_mode()
+ if mode == "text":
+ context["rendering_text"] = True
+ content = render_email_template(content_template, context)
+ if mode == "text":
+ content = strip_tags_plus(content, keep_leading_whitespace=True)
+
+ case {"email_digest": True}:
+ days_before = int(self.request.GET.get("days_before", 1))
+ interval = timezone.now() - timedelta(days=days_before)
+ digest = Digest(since=interval)
+ return digest.render()
+
+ case _:
+ pass # render wrapper
return content
diff --git a/src/openforms/logging/service.py b/src/openforms/logging/service.py
new file mode 100644
index 0000000000..b0642fbf81
--- /dev/null
+++ b/src/openforms/logging/service.py
@@ -0,0 +1,77 @@
+import uuid
+from dataclasses import dataclass
+from datetime import datetime
+from itertools import groupby
+from typing import Iterable
+
+from django_yubin.models import Message
+
+from openforms.logging.models import TimelineLogProxy
+from openforms.submissions.utils import get_filtered_submission_admin_url
+
+
+@dataclass
+class FailedEmail:
+ submission_uuid: uuid.UUID
+ event: str
+
+
+@dataclass
+class FailedRegistration:
+ form_name: str
+ failed_submissions_counter: int
+ initial_failure_at: datetime
+ last_failure_at: datetime
+ admin_link: str
+
+
+def collect_failed_emails(since: datetime) -> Iterable[FailedEmail]:
+ logs = TimelineLogProxy.objects.filter(
+ timestamp__gt=since,
+ extra_data__status=Message.STATUS_FAILED,
+ extra_data__include_in_daily_digest=True,
+ ).distinct("content_type", "extra_data__status", "extra_data__event")
+
+ if not logs:
+ return []
+
+ failed_emails = [
+ FailedEmail(
+ submission_uuid=log.content_object.uuid, event=log.extra_data["event"]
+ )
+ for log in logs
+ ]
+
+ return failed_emails
+
+
+def collect_failed_registrations(
+ since: datetime,
+) -> Iterable[dict[str, FailedRegistration]]:
+ logs = TimelineLogProxy.objects.filter(
+ timestamp__gt=since,
+ extra_data__log_event="registration_failure",
+ ).order_by("timestamp")
+
+ if not logs:
+ return []
+
+ form_sorted_logs = sorted(logs, key=lambda x: x.content_object.form.admin_name)
+
+ grouped_logs = groupby(form_sorted_logs, key=lambda log: log.content_object.form)
+
+ failed_registrations = {}
+ for form, submission_logs in grouped_logs:
+ logs = list(submission_logs)
+ timestamps = [log.timestamp for log in logs]
+ failed_registrations[str(form.uuid)] = {
+ "form_name": form.name,
+ "failed_submissions_counter": len(logs),
+ "initial_failure_at": min(timestamps),
+ "last_failure_at": max(timestamps),
+ "admin_link": get_filtered_submission_admin_url(
+ form.id, filter_retry=True, registration_time="24hAgo"
+ ),
+ }
+
+ return list(failed_registrations.values())
diff --git a/src/openforms/submissions/admin.py b/src/openforms/submissions/admin.py
index f1113e1911..688e9875d6 100644
--- a/src/openforms/submissions/admin.py
+++ b/src/openforms/submissions/admin.py
@@ -1,9 +1,12 @@
+from datetime import timedelta
+
from django import forms
from django.contrib import admin, messages
from django.contrib.contenttypes.admin import GenericTabularInline
from django.db.models import Q
from django.http import Http404
from django.template.defaultfilters import filesizeformat
+from django.utils import timezone
from django.utils.translation import gettext_lazy as _, ngettext
from privates.admin import PrivateMediaMixin
@@ -82,6 +85,21 @@ def expected_parameters(self):
return [self.parameter_name]
+class SubmissionTimeListFilter(admin.SimpleListFilter):
+ title = _("registration time")
+ parameter_name = "registration_time"
+
+ def lookups(self, request, model_admin):
+ return [
+ ("24hAgo", _("In the past 24 hours")),
+ ]
+
+ def queryset(self, request, queryset):
+ yesterday = timezone.now() - timedelta(days=1)
+ if self.value() == "24hAgo":
+ return queryset.filter(last_register_date__gt=yesterday)
+
+
class SubmissionStepInline(admin.StackedInline):
model = SubmissionStep
extra = 0
@@ -170,6 +188,7 @@ class SubmissionAdmin(admin.ModelAdmin):
)
list_filter = (
SubmissionTypeListFilter,
+ SubmissionTimeListFilter,
"registration_status",
"needs_on_completion_retry",
"form",
diff --git a/src/openforms/submissions/tests/test_admin.py b/src/openforms/submissions/tests/test_admin.py
index 9ce7238019..7a2e051578 100644
--- a/src/openforms/submissions/tests/test_admin.py
+++ b/src/openforms/submissions/tests/test_admin.py
@@ -1,16 +1,22 @@
from unittest.mock import patch
+from django.contrib.admin import AdminSite
+from django.test import TestCase
from django.urls import reverse
+from django.utils import timezone
from django_webtest import WebTest
+from freezegun import freeze_time
from furl import furl
from maykin_2fa.test import disable_admin_mfa
from openforms.accounts.tests.factories import UserFactory
from openforms.logging.logevent import submission_start
from openforms.logging.models import TimelineLogProxy
+from openforms.submissions.models.submission import Submission
from ...config.models import GlobalConfiguration
+from ..admin import SubmissionAdmin, SubmissionTimeListFilter
from ..constants import PostSubmissionEvents, RegistrationStatuses
from .factories import SubmissionFactory
@@ -213,3 +219,36 @@ def test_change_view(self):
"admin:submissions_submission_change", kwargs={"object_id": "0"}
)
self.app.get(change_url, user=self.user, status=404)
+
+
+class TestSubmissionTimeListFilterAdmin(TestCase):
+ def test_time_filtering(self):
+ with freeze_time("2023-04-02T12:30:00+01:00"):
+ # registered in the past 24 hours
+ submission_1 = SubmissionFactory.create(
+ last_register_date=timezone.now(),
+ registration_status=RegistrationStatuses.failed,
+ )
+
+ with freeze_time("2023-01-02T12:30:00+01:00"):
+ # registered out of filtering bounds
+ submission_2 = SubmissionFactory.create(
+ last_register_date=timezone.now(),
+ registration_status=RegistrationStatuses.failed,
+ )
+
+ with freeze_time("2023-04-02T18:30:00+01:00"):
+ site = AdminSite()
+ model_admin = SubmissionAdmin(Submission, site)
+ filter_instance = SubmissionTimeListFilter(
+ request=None,
+ params={"registration_time": "24hAgo"},
+ model=Submission,
+ model_admin=model_admin,
+ )
+
+ queryset = Submission.objects.all()
+ filtered_queryset = filter_instance.queryset(None, queryset)
+
+ self.assertQuerySetEqual(queryset, [submission_1, submission_2], ordered=False)
+ self.assertQuerySetEqual(filtered_queryset, [submission_1], ordered=False)
diff --git a/src/openforms/submissions/utils.py b/src/openforms/submissions/utils.py
index 519b3f603b..16294f1c60 100644
--- a/src/openforms/submissions/utils.py
+++ b/src/openforms/submissions/utils.py
@@ -8,6 +8,7 @@
from django.http import HttpRequest
from django.utils import translation
+from furl import furl
from rest_framework.permissions import SAFE_METHODS
from rest_framework.request import Request
from rest_framework.reverse import reverse
@@ -335,3 +336,15 @@ def get_report_download_url(request: Request, report: SubmissionReport) -> str:
kwargs={"report_id": report.id, "token": token},
)
return request.build_absolute_uri(download_url)
+
+
+def get_filtered_submission_admin_url(
+ form_id: int, *, filter_retry: bool, registration_time: str
+) -> str:
+ query_params = {
+ "form__id__exact": form_id,
+ "needs_on_completion_retry__exact": 1 if filter_retry else 0,
+ "registration_time": registration_time,
+ }
+ submissions_admin_url = furl(reverse("admin:submissions_submission_changelist"))
+ return submissions_admin_url.add(query_params).url
diff --git a/src/openforms/urls.py b/src/openforms/urls.py
index b33ae47dcf..46e31d89cb 100644
--- a/src/openforms/urls.py
+++ b/src/openforms/urls.py
@@ -112,6 +112,12 @@
EmailWrapperTestView.as_view(),
name="dev-email-wrapper",
),
+ path(
+ "dev/email/digest",
+ EmailWrapperTestView.as_view(),
+ {"email_digest": True},
+ name="dev-email-digest",
+ ),
path(
"dev/email/confirmation/",
EmailWrapperTestView.as_view(),