Merge pull request #4962 from open-formulieren/feature/4930-bulk-expo…

…rt-submission-stats

Enable exporting (registered) submission metadata

src/openforms/forms/admin/form_statistics.py

@@ -1,6 +1,8 @@
 from django.contrib import admin
+from django.urls import path
 
 from ..models import FormStatistics
+from .views import ExportSubmissionStatisticsView
 
 
 @admin.register(FormStatistics)
@@ -31,3 +33,15 @@ def has_delete_permission(self, request, obj=None):
 
     def has_change_permission(self, request, obj=None):
         return False
+
+    def get_urls(self):
+        urls = super().get_urls()
+        export_view = self.admin_site.admin_view(
+            ExportSubmissionStatisticsView.as_view(
+                media=self.media,
+            )  # pyright: ignore[reportArgumentType]
+        )
+        custom_urls = [
+            path("export/", export_view, name="formstatistics_export"),
+        ]
+        return custom_urls + urls

src/openforms/forms/admin/views.py

@@ -1,25 +1,32 @@
 import zipfile
+from datetime import date
 from uuid import uuid4
 
 from django import forms
 from django.contrib import messages
+from django.contrib.admin.helpers import AdminField
+from django.contrib.admin.views.decorators import staff_member_required
 from django.contrib.auth.mixins import LoginRequiredMixin, PermissionRequiredMixin
 from django.contrib.messages.views import SuccessMessageMixin
 from django.contrib.postgres.forms import SimpleArrayField
-from django.http import FileResponse
+from django.http import FileResponse, HttpResponse
 from django.shortcuts import get_object_or_404
 from django.urls import reverse_lazy
+from django.utils.decorators import method_decorator
+from django.utils.http import content_disposition_header
 from django.utils.translation import gettext_lazy as _
 from django.views import View
 from django.views.generic.edit import FormView
 
+from import_export.formats.base_formats import XLSX
 from privates.storages import private_media_storage
 from rest_framework.exceptions import ValidationError
 
 from openforms.logging import logevent
 
+from ..forms import ExportStatisticsForm
 from ..forms.form import FormImportForm
-from ..models.form import Form, FormsExport
+from ..models import Form, FormsExport, FormStatistics
 from ..utils import import_form
 from .tasks import process_forms_export, process_forms_import
 
@@ -109,3 +116,53 @@ def _bulk_import_forms(self, import_file):
         filename = private_media_storage.save(name, import_file)
 
         process_forms_import.delay(filename, self.request.user.id)
+
+
+@method_decorator(staff_member_required, name="dispatch")
+class ExportSubmissionStatisticsView(
+    LoginRequiredMixin, PermissionRequiredMixin, FormView
+):
+    permission_required = "forms.view_formstatistics"
+    template_name = "admin/forms/formstatistics/export_form.html"
+    form_class = ExportStatisticsForm
+
+    # must be set by the ModelAdmin
+    media: forms.Media | None = None
+
+    def form_valid(self, form: ExportStatisticsForm) -> HttpResponse:
+        start_date: date = form.cleaned_data["start_date"]
+        end_date: date = form.cleaned_data["end_date"]
+        dataset = form.export()
+        format = XLSX()
+        filename = f"submissions_{start_date.isoformat()}_{end_date.isoformat()}.xlsx"
+        return HttpResponse(
+            format.export_data(dataset),
+            content_type=format.get_content_type(),
+            headers={
+                "Content-Disposition": content_disposition_header(
+                    as_attachment=True,
+                    filename=filename,
+                ),
+            },
+        )
+
+    def get_context_data(self, **kwargs):
+        assert (
+            self.media is not None
+        ), "You must pass media=self.media in the model admin"
+        context = super().get_context_data(**kwargs)
+
+        form = context["form"]
+
+        def form_fields():
+            for name in form.fields:
+                yield AdminField(form, name, is_first=False)
+
+        context.update(
+            {
+                "opts": FormStatistics._meta,
+                "media": self.media + form.media,
+                "form_fields": form_fields,
+            }
+        )
+        return context

src/openforms/forms/forms/__init__.py

@@ -1 +1,4 @@
-from .form_definition import FormDefinitionForm  # noqa
+from .form_definition import FormDefinitionForm
+from .form_statistics import ExportStatisticsForm
+
+__all__ = ["FormDefinitionForm", "ExportStatisticsForm"]

src/openforms/forms/forms/form_statistics.py

@@ -0,0 +1,68 @@
+from __future__ import annotations
+
+from datetime import date
+
+from django import forms
+from django.contrib.admin.widgets import AdminDateWidget
+from django.utils import timezone
+from django.utils.translation import gettext_lazy as _
+
+from dateutil.relativedelta import relativedelta
+from tablib import Dataset
+
+from ..models import Form
+from ..statistics import export_registration_statistics
+
+
+def get_first_of_previous_month() -> date:
+    now = timezone.localtime()
+    one_month_ago = now - relativedelta(months=1)
+    first_of_previous_month = one_month_ago.replace(day=1)
+    return first_of_previous_month.date()
+
+
+def get_last_of_previous_month() -> date:
+    now = timezone.localtime()
+    first_of_current_month = now.replace(day=1)
+    get_last_of_previous_month = first_of_current_month - relativedelta(days=1)
+    return get_last_of_previous_month.date()
+
+
+class ExportStatisticsForm(forms.Form):
+    start_date = forms.DateField(
+        label=_("From"),
+        required=True,
+        initial=get_first_of_previous_month,
+        help_text=_(
+            "Export form submission that were submitted on or after this date."
+        ),
+        widget=AdminDateWidget,
+    )
+    end_date = forms.DateField(
+        label=_("Until"),
+        required=True,
+        initial=get_last_of_previous_month,
+        help_text=_(
+            "Export form submission that were submitted before or on this date."
+        ),
+        widget=AdminDateWidget,
+    )
+    limit_to_forms = forms.ModelMultipleChoiceField(
+        label=_("Forms"),
+        required=False,
+        queryset=Form.objects.filter(_is_deleted=False),
+        help_text=_(
+            "Limit the export to the selected forms, if specified. Leave the field "
+            "empty to export all forms. Hold CTRL (or COMMAND on Mac) to select "
+            "multiple options."
+        ),
+    )
+
+    def export(self) -> Dataset:
+        start_date: date = self.cleaned_data["start_date"]
+        end_date: date = self.cleaned_data["end_date"]
+        return export_registration_statistics(
+            start_date,
+            end_date,
+            self.cleaned_data["limit_to_forms"],
+        )

src/openforms/forms/statistics.py

@@ -0,0 +1,102 @@
+from datetime import date, datetime, time
+
+from django.contrib.contenttypes.models import ContentType
+from django.db import models
+from django.utils.timezone import make_aware
+from django.utils.translation import gettext_lazy as _
+
+from tablib import Dataset
+
+from openforms.logging import logevent
+from openforms.logging.models import TimelineLogProxy
+from openforms.submissions.models import Submission
+
+from .models import Form
+
+
+def export_registration_statistics(
+    start_date: date,
+    end_date: date,
+    limit_to_forms: models.QuerySet[Form] | None = None,
+) -> Dataset:
+    """
+    Export the form registration statistics to a tablib Dataset.
+
+    The export retrieves log records within the specified date range (closed interval,
+    [start_date, end_date]), optionally filtering them down to a set of forms. Only log
+    records for successful registration are considered.
+
+    :arg start_date: include log records starting from this date (midnight, in the local
+      timezone).
+    :arg end_date: include log records until (and including) this date. Log record up to
+      midnight (in the local timezone) the next day are included, i.e.
+      ``$date, 23:59:59 999999us``.
+    :arg limit_to_forms: A queryset of forms to limit the export to. If not provided or
+      ``None`` is given, all forms are included.
+    """
+    dataset = Dataset(
+        headers=(
+            _("Public reference"),
+            _("Form name (public)"),
+            _("Form name (internal)"),
+            _("Submitted on"),
+            _("Registered on"),
+        ),
+        title=_("Successfully registered submissions between {start} and {end}").format(
+            start=start_date.isoformat(),
+            end=end_date.isoformat(),
+        ),
+    )
+
+    _start_date = make_aware(datetime.combine(start_date, time.min))
+    _end_date = make_aware(datetime.combine(end_date, time.max))
+
+    log_records = TimelineLogProxy.objects.filter(
+        content_type=ContentType.objects.get_for_model(Submission),
+        timestamp__gte=_start_date,
+        timestamp__lt=_end_date,
+        # see openforms.logging.logevent for the data structure of the extra_data
+        # JSONField
+        extra_data__log_event=logevent.REGISTRATION_SUCCESS_EVENT,
+    ).order_by("timestamp")
+
+    if limit_to_forms:
+        form_ids = list(limit_to_forms.values_list("pk", flat=True))
+        log_records = log_records.filter(extra_data__form_id__in=form_ids)
+
+    for record in log_records.iterator():
+        extra_data = record.extra_data
+        # GFKs will be broken when the submissions are pruned, so prefer extracting
+        # information from the extra_data snapshot
+        submission: Submission | None = record.content_object
+        dataset.append(
+            (
+                # public reference
+                extra_data.get(
+                    "public_reference",
+                    (
+                        submission.public_registration_reference
+                        if submission
+                        else "-unknown-"
+                    ),
+                ),
+                # public form name
+                extra_data.get(
+                    "form_name", submission.form.name if submission else "-unknown-"
+                ),
+                # internal form name
+                extra_data.get(
+                    "internal_form_name",
+                    submission.form.internal_name if submission else "-unknown-",
+                ),
+                # when the user submitted the form
+                extra_data.get(
+                    "submitted_on",
+                    submission.completed_on.isoformat() if submission else None,
+                ),
+                # when the registration succeeeded - this must be close to when it was logged
+                record.timestamp.isoformat(),
+            )
+        )
+
+    return dataset

src/openforms/forms/templates/admin/forms/formstatistics/change_list_object_tools.html

@@ -0,0 +1,11 @@
+{% extends "admin/change_list_object_tools.html" %}
+{% load i18n %}
+
+{% block object-tools-items %}
+    <li>
+        <a href="{% url 'admin:formstatistics_export' %}" class="viewlink">
+            {% trans "Export submission statistics" %}
+        </a>
+    </li>
+    {{ block.super }}
+{% endblock %}

src/openforms/forms/templates/admin/forms/formstatistics/export_form.html

@@ -0,0 +1,66 @@
+{% extends "admin/base_site.html" %}
+{% load static i18n django_admin_index %}
+
+{% block extrahead %}{{ block.super }}
+<script src="{% url 'admin:jsi18n' %}"></script>
+{{ media }}
+{% endblock %}
+
+{% block extrastyle %}{{ block.super }}
+<link rel="stylesheet" href="{% static "admin/css/forms.css" %}">
+<link rel="stylesheet" href="{% static "admin/css/admin-index.css" %}">{% endblock %}
+
+{% block nav-global %}{% include "django_admin_index/includes/app_list.html" %}{% endblock nav-global %}
+
+{% block title %} {% trans "Export submission statistics" %} {{ block.super }} {% endblock %}
+
+{% block breadcrumbs %}
+<div class="breadcrumbs">
+<a href="{% url 'admin:index' %}">{% trans 'Home' %}</a>
+&rsaquo; <a href="{% url 'admin:forms_formstatistics_changelist' %}">{{ opts.verbose_name_plural|capfirst }}</a>
+&rsaquo; {% trans 'Export submission statistics' %}
+</div>
+{% endblock %}
+
+{% block content %}
+<h1>{% trans 'Export submission statistics' %}</h1>
+
+<div id="content-main">
+    <form action="." method="post" id="export-statistics">
+        {% csrf_token %}
+
+        <fieldset class="module aligned">
+            <div class="description">{% blocktrans trimmed %}
+              <p>Here you can create an export of successfully registered form submissions. The
+              export file contains the following columns: public reference, form name,
+              form internal name, the submission datetime and the timestamp of registration.</p>
+
+              <p>You can use the filters below to limit the result set in the export.</p>
+            {% endblocktrans %}</div>
+
+            {# TODO: doesn't handle checkboxes, see admin/includes/fieldset.html for when this is necessary #}
+            {% for field in form_fields %}
+                <div class="form-row {% if field.errors %}errors{% endif %}">
+                    {{ field.errors }}
+                    <div>
+                        <div class="flex-container {% if field.errors %}errors{% endif %}">
+                            {{ field.label_tag }}
+                            {{ field.field }}
+                        </div>
+                    </div>
+
+                    {% if field.field.help_text %}
+                        <div class="help" {% if field.field.id_for_label %} id="{{ field.field.id_for_label }}_helptext"{% endif %}>
+                            <div>{{ field.field.help_text|safe }}</div>
+                        </div>
+                    {% endif %}
+                </div>
+            {% endfor %}
+        </fieldset>
+
+        <div class="submit-row">
+            <input type="submit" class="default" value="{% trans 'Export' %}">
+        </div>
+    </form>
+</div>
+{% endblock %}