From 33b6e2efa4cfb1a85c445a5282789a2f191f39ef Mon Sep 17 00:00:00 2001 From: Sergei Maertens Date: Fri, 20 Dec 2024 16:31:18 +0100 Subject: [PATCH] :sparkles: [#4930] Implement the actual export Export submission statistics based on the timeline logs. Note: if the timeline logs are pruned, this affects the exports. It's up to the users to periodically create these exports and save them somewhere if they periodically prune log records. Note 2: filtering on forms only works on new log records, as existing log records don't have the form ID stored in the structured data. Submissions that were deleted for which existing log records are present will display 'unknown' for some columns because the relevant information has been deleted. Only from 3.0 onwards are we snapshotting the data required for the exports. --- src/openforms/forms/admin/views.py | 22 ++++- src/openforms/forms/forms/form_statistics.py | 11 +++ src/openforms/forms/statistics.py | 87 ++++++++++++++++++++ src/openforms/logging/logevent.py | 14 +++- 4 files changed, 132 insertions(+), 2 deletions(-) create mode 100644 src/openforms/forms/statistics.py diff --git a/src/openforms/forms/admin/views.py b/src/openforms/forms/admin/views.py index ae9476df63..fdfbdd5c3e 100644 --- a/src/openforms/forms/admin/views.py +++ b/src/openforms/forms/admin/views.py @@ -1,4 +1,5 @@ import zipfile +from datetime import date from uuid import uuid4 from django import forms @@ -8,14 +9,16 @@ from django.contrib.auth.mixins import LoginRequiredMixin, PermissionRequiredMixin from django.contrib.messages.views import SuccessMessageMixin from django.contrib.postgres.forms import SimpleArrayField -from django.http import FileResponse +from django.http import FileResponse, HttpResponse from django.shortcuts import get_object_or_404 from django.urls import reverse_lazy from django.utils.decorators import method_decorator +from django.utils.http import content_disposition_header from django.utils.translation import gettext_lazy as _ from django.views import View from django.views.generic.edit import FormView +from import_export.formats.base_formats import XLSX from privates.storages import private_media_storage from rest_framework.exceptions import ValidationError @@ -126,6 +129,23 @@ class ExportSubmissionStatisticsView( # must be set by the ModelAdmin media: forms.Media | None = None + def form_valid(self, form: ExportStatisticsForm) -> HttpResponse: + start_date: date = form.cleaned_data["start_date"] + end_date: date = form.cleaned_data["end_date"] + dataset = form.export() + format = XLSX() + filename = f"submissions_{start_date.isoformat()}_{end_date.isoformat()}.xlsx" + return HttpResponse( + format.export_data(dataset), + content_type=format.get_content_type(), + headers={ + "Content-Disposition": content_disposition_header( + as_attachment=True, + filename=filename, + ), + }, + ) + def get_context_data(self, **kwargs): assert ( self.media is not None diff --git a/src/openforms/forms/forms/form_statistics.py b/src/openforms/forms/forms/form_statistics.py index 5e9a5e6cea..a367e2bbd0 100644 --- a/src/openforms/forms/forms/form_statistics.py +++ b/src/openforms/forms/forms/form_statistics.py @@ -8,8 +8,10 @@ from django.utils.translation import gettext_lazy as _ from dateutil.relativedelta import relativedelta +from tablib import Dataset from ..models import Form +from ..statistics import export_registration_statistics def get_first_of_current_month() -> date: @@ -50,3 +52,12 @@ class ExportStatisticsForm(forms.Form): "multiple options." ), ) + + def export(self) -> Dataset: + start_date: date = self.cleaned_data["start_date"] + end_date: date = self.cleaned_data["end_date"] + return export_registration_statistics( + start_date, + end_date, + self.cleaned_data["limit_to_forms"], + ) diff --git a/src/openforms/forms/statistics.py b/src/openforms/forms/statistics.py new file mode 100644 index 0000000000..2df0338fbe --- /dev/null +++ b/src/openforms/forms/statistics.py @@ -0,0 +1,87 @@ +from datetime import date, datetime, time + +from django.contrib.contenttypes.models import ContentType +from django.db import models +from django.utils.timezone import make_aware +from django.utils.translation import gettext_lazy as _ + +from tablib import Dataset + +from openforms.logging import logevent +from openforms.logging.models import TimelineLogProxy +from openforms.submissions.models import Submission + +from .models import Form + + +def export_registration_statistics( + start_date: date, + end_date: date, + limit_to_forms: models.QuerySet[Form] | None = None, +) -> Dataset: + dataset = Dataset( + headers=( + _("Public reference"), + _("Form name (public)"), + _("Form name (internal)"), + _("Submitted on"), + _("Registered on"), + ), + title=_("Successfully registered submissions between {start} and {end}").format( + start=start_date.isoformat(), + end=end_date.isoformat(), + ), + ) + + _start_date = make_aware(datetime.combine(start_date, time.min)) + _end_date = make_aware(datetime.combine(end_date, time.max)) + + log_records = TimelineLogProxy.objects.filter( + content_type=ContentType.objects.get_for_model(Submission), + timestamp__gte=_start_date, + timestamp__lt=_end_date, + # see openforms.logging.logevent for the data structure of the extra_data + # JSONField + extra_data__log_event=logevent.REGISTRATION_SUCCESS_EVENT, + ).order_by("timestamp") + + if limit_to_forms: + form_ids = list(limit_to_forms.values_list("pk", flat=True)) + log_records = log_records.filter(extra_data__form_id__in=form_ids) + + for record in log_records.iterator(): + extra_data = record.extra_data + # GFKs will be broken when the submissions are pruned, so prefer extracting + # information from the extra_data snapshot + submission: Submission | None = record.content_object + dataset.append( + ( + # public reference + extra_data.get( + "public_reference", + ( + submission.public_registration_reference + if submission + else "-unknown-" + ), + ), + # public form name + extra_data.get( + "form_name", submission.form.name if submission else "-unknown-" + ), + # internal form name + extra_data.get( + "internal_form_name", + submission.form.internal_name if submission else "-unknown-", + ), + # when the user submitted the form + extra_data.get( + "submitted_on", + submission.completed_on.isoformat() if submission else None, + ), + # when the registration succeeeded - this must be close to when it was logged + record.timestamp.isoformat(), + ) + ) + + return dataset diff --git a/src/openforms/logging/logevent.py b/src/openforms/logging/logevent.py index c50634f453..e2a75cc609 100644 --- a/src/openforms/logging/logevent.py +++ b/src/openforms/logging/logevent.py @@ -216,10 +216,22 @@ def registration_start(submission: Submission): ) +REGISTRATION_SUCCESS_EVENT = "registration_success" + + def registration_success(submission: Submission, plugin): + extra_data = { + # note: these keys are used in form statistics exports! + "public_reference": submission.public_registration_reference, + "form_id": submission.form.pk, + "form_name": submission.form.name, + "internal_form_name": submission.form.internal_name, + "submitted_on": submission.completed_on, + } _create_log( submission, - "registration_success", + REGISTRATION_SUCCESS_EVENT, + extra_data=extra_data, plugin=plugin, )