From 57d2b300d720000463c95124e5f03dac0c2557a5 Mon Sep 17 00:00:00 2001 From: Struan Donald Date: Wed, 8 Jan 2025 17:55:19 +0000 Subject: [PATCH] bulk reponse editing interface Allows uploading a CSV with updated responses to a question for bulk editing --- ceuk-marking/urls.py | 5 + crowdsourcer/fixtures/responses.json | 2 +- crowdsourcer/forms.py | 72 +++++++++ crowdsourcer/scoring.py | 2 +- .../questions/question_bulk_upload.html | 21 +++ .../tests/data/test_question_upload.csv | 4 + .../test_question_upload_one_unchanged.csv | 4 + crowdsourcer/tests/test_question_views.py | 125 ++++++++++++++ crowdsourcer/views/questions.py | 153 +++++++++++++++++- 9 files changed, 383 insertions(+), 5 deletions(-) create mode 100644 crowdsourcer/templates/crowdsourcer/questions/question_bulk_upload.html create mode 100644 crowdsourcer/tests/data/test_question_upload.csv create mode 100644 crowdsourcer/tests/data/test_question_upload_one_unchanged.csv create mode 100644 crowdsourcer/tests/test_question_views.py diff --git a/ceuk-marking/urls.py b/ceuk-marking/urls.py index ea4e1f4e..c889df89 100644 --- a/ceuk-marking/urls.py +++ b/ceuk-marking/urls.py @@ -328,6 +328,11 @@ questions.SectionList.as_view(), name="question_sections", ), + path( + "questions/update///", + questions.QuestionBulkUpdateView.as_view(), + name="question_bulk_update", + ), ] urlpatterns = [ diff --git a/crowdsourcer/fixtures/responses.json b/crowdsourcer/fixtures/responses.json index d43a22a4..012d3a29 100644 --- a/crowdsourcer/fixtures/responses.json +++ b/crowdsourcer/fixtures/responses.json @@ -10,7 +10,7 @@ "response_type": 1, "public_notes": "public notrs", "page_number": "0", - "evidence": "", + "evidence": null, "private_notes": "private notes", "revision_type": null, "revision_notes": null, diff --git a/crowdsourcer/forms.py b/crowdsourcer/forms.py index 935b1931..19e74a37 100644 --- a/crowdsourcer/forms.py +++ b/crowdsourcer/forms.py @@ -584,3 +584,75 @@ def __init__(self, properties: {}, **kwargs): QuestionFormset = modelformset_factory( Question, fields=["question_type", "weighting"], extra=0, can_delete=False ) + + +class QuestionBulkUploadForm(Form): + question = CharField(widget=HiddenInput) + stage = ChoiceField(required=True, choices=[]) + updated_responses = FileField() + + def clean(self): + data = self.cleaned_data.get("updated_responses") + + try: + df = pd.read_csv( + data, + usecols=[ + "authority", + "answer", + "score", + "public_notes", + "page_number", + "evidence", + "private_notes", + ], + ) + except ValueError as v: + raise ValidationError(f"Problem processing csv file: {v}") + + self.responses_df = df + + try: + question = Question.objects.get(id=self.cleaned_data["question"]) + except Question.DoesNotExist: + raise ValidationError(f"Bad question id: {self.cleaned_data['question']}") + + is_multi = question.question_type == "multiple_choice" + + file_errors = [] + for _, row in self.responses_df.iterrows(): + desc = row["answer"].strip() + try: + PublicAuthority.objects.get( + name=row["authority"], + marking_session=self.session, + ) + except PublicAuthority.DoesNotExist: + file_errors.append(f"No such authority: {row['authority']}") + continue + + if desc == "-": + continue + + if not is_multi: + answers = [desc] + else: + answers = desc.split("|") + + for answer in answers: + try: + Option.objects.get(question=question, description=answer) + except Option.DoesNotExist: + file_errors.append( + f"No such answer for {row['authority']}: {answer}" + ) + continue + + if len(file_errors) > 0: + raise ValidationError(file_errors) + + def __init__(self, question_id, stage_choices, session, **kwargs): + super().__init__(**kwargs) + self.session = session + self.initial["question"] = question_id + self.fields["stage"].choices = stage_choices diff --git a/crowdsourcer/scoring.py b/crowdsourcer/scoring.py index 70026b6c..2ecac1f4 100644 --- a/crowdsourcer/scoring.py +++ b/crowdsourcer/scoring.py @@ -614,7 +614,7 @@ def get_response_data( for opt in response.multi_option.all(): descs.append(opt.description) score += opt.score - answer = ",".join(descs) + answer = "|".join(descs) elif response.option is not None: score = response.option.score answer = response.option.description diff --git a/crowdsourcer/templates/crowdsourcer/questions/question_bulk_upload.html b/crowdsourcer/templates/crowdsourcer/questions/question_bulk_upload.html new file mode 100644 index 00000000..81f5df8c --- /dev/null +++ b/crowdsourcer/templates/crowdsourcer/questions/question_bulk_upload.html @@ -0,0 +1,21 @@ +{% extends 'crowdsourcer/base.html' %} + +{% load crowdsourcer_tags django_bootstrap5 %} + +{% block content %} +{% if show_login %} +

Sign in

+ Sign in +{% else %} +

Update Responses for {{ section.title }} {{ question.number_and_part }}

+

{{ question.description }}

+ + +
+ {% csrf_token %} + {% bootstrap_form form %} + +
+ +{% endif %} +{% endblock %} diff --git a/crowdsourcer/tests/data/test_question_upload.csv b/crowdsourcer/tests/data/test_question_upload.csv new file mode 100644 index 00000000..3e26ea87 --- /dev/null +++ b/crowdsourcer/tests/data/test_question_upload.csv @@ -0,0 +1,4 @@ +authority,answer,score,public_notes,page_number,evidence,private_notes +Aberdeen City Council,Yes,1,uploaded public notes,99,uploaded evidence,uploaded private notes +Aberdeenshire Council,No,0,,,, +Adur District Council,-,-,-,-,-,- diff --git a/crowdsourcer/tests/data/test_question_upload_one_unchanged.csv b/crowdsourcer/tests/data/test_question_upload_one_unchanged.csv new file mode 100644 index 00000000..f81dc4bb --- /dev/null +++ b/crowdsourcer/tests/data/test_question_upload_one_unchanged.csv @@ -0,0 +1,4 @@ +authority,answer,score,public_notes,page_number,evidence,private_notes +Aberdeen City Council,Yes,1,uploaded public notes,99,uploaded evidence,uploaded private notes +Aberdeenshire Council,Yes,0,public notrs,0,,private notes +Adur District Council,-,-,-,-,-,- diff --git a/crowdsourcer/tests/test_question_views.py b/crowdsourcer/tests/test_question_views.py new file mode 100644 index 00000000..6706dd92 --- /dev/null +++ b/crowdsourcer/tests/test_question_views.py @@ -0,0 +1,125 @@ +import pathlib + +from django.contrib.auth.models import Permission, User +from django.test import TestCase +from django.urls import reverse + +from crowdsourcer.models import Question, Response + + +class BaseTestCase(TestCase): + fixtures = [ + "authorities.json", + "basics.json", + "users.json", + "questions.json", + "options.json", + "assignments.json", + "responses.json", + ] + + def setUp(self): + p = Permission.objects.get(codename="can_manage_users") + u = User.objects.get(username="volunteer_admin") + u.user_permissions.add(p) + + self.client.force_login(u) + self.user = u + + +class TestBulkUpload(BaseTestCase): + def test_one_update_one_new(self): + url = reverse("question_bulk_update", args=("Transport", "1")) + response = self.client.get(url) + + q = Question.objects.get( + section__title="Transport", + section__marking_session__label="Default", + number=1, + ) + + all_r = Response.objects.filter(question=q, response_type__type="First Mark") + self.assertEqual(all_r.count(), 1) + + r = Response.objects.get(question=q, authority__name="Aberdeenshire Council") + + self.assertEqual(r.option.description, "Yes") + self.assertEqual(r.page_number, "0") + + upload_file = ( + pathlib.Path(__file__).parent.resolve() + / "data" + / "test_question_upload.csv" + ) + + with open(upload_file, "rb") as fp: + response = self.client.post( + url, + data={ + "question": 281, + "updated_responses": fp, + "stage": "First Mark", + }, + ) + + self.assertRedirects(response, "/Default" + url) + self.assertEqual(all_r.count(), 2) + + r = Response.objects.get(question=q, authority__name="Aberdeen City Council") + + self.assertEqual(r.option.description, "Yes") + self.assertEqual(r.page_number, "99") + + r = Response.objects.get(question=q, authority__name="Aberdeenshire Council") + + self.assertEqual(r.option.description, "No") + self.assertEqual(r.page_number, None) + + def test_one_new_one_unchanged(self): + url = reverse("question_bulk_update", args=("Transport", "1")) + response = self.client.get(url) + + q = Question.objects.get( + section__title="Transport", + section__marking_session__label="Default", + number=1, + ) + + all_r = Response.objects.filter(question=q, response_type__type="First Mark") + self.assertEqual(all_r.count(), 1) + + r = Response.objects.get(question=q, authority__name="Aberdeenshire Council") + + last_update = r.last_update + self.assertEqual(r.option.description, "Yes") + self.assertEqual(r.page_number, "0") + + upload_file = ( + pathlib.Path(__file__).parent.resolve() + / "data" + / "test_question_upload_one_unchanged.csv" + ) + + with open(upload_file, "rb") as fp: + response = self.client.post( + url, + data={ + "question": 281, + "updated_responses": fp, + "stage": "First Mark", + }, + ) + + self.assertRedirects(response, "/Default" + url) + self.assertEqual(all_r.count(), 2) + + r = Response.objects.get(question=q, authority__name="Aberdeen City Council") + + self.assertEqual(r.option.description, "Yes") + self.assertEqual(r.page_number, "99") + + r = Response.objects.get(question=q, authority__name="Aberdeenshire Council") + + self.assertEqual(r.option.description, "Yes") + self.assertEqual(r.page_number, "0") + self.assertEqual(last_update, r.last_update) diff --git a/crowdsourcer/views/questions.py b/crowdsourcer/views/questions.py index 3fc006f3..f014a1e5 100644 --- a/crowdsourcer/views/questions.py +++ b/crowdsourcer/views/questions.py @@ -1,10 +1,22 @@ +import re + from django.contrib.auth.mixins import UserPassesTestMixin +from django.db import transaction from django.shortcuts import get_object_or_404 from django.urls import reverse from django.views.generic import FormView, ListView -from crowdsourcer.forms import OptionFormset, QuestionFormset -from crowdsourcer.models import Option, Question, Section +import pandas as pd + +from crowdsourcer.forms import OptionFormset, QuestionBulkUploadForm, QuestionFormset +from crowdsourcer.models import ( + Option, + PublicAuthority, + Question, + Response, + ResponseType, + Section, +) class SectionList(ListView): @@ -27,7 +39,7 @@ def get_success_url(self): "session_urls:edit_options", kwargs={ "marking_session": self.request.current_session.label, - "section_name": "Buildings & Heating", + "section_name": self.kwargs["section_name"], }, ) @@ -97,3 +109,138 @@ def get_context_data(self, **kwargs): def form_valid(self, form): form.save() return super().form_valid(form) + + +class QuestionBulkUpdateView(UserPassesTestMixin, FormView): + template_name = "crowdsourcer/questions/question_bulk_upload.html" + form_class = QuestionBulkUploadForm + + def test_func(self): + return self.request.user.has_perm("crowdsourcer.can_manage_users") + + def get_success_url(self): + return reverse( + "session_urls:question_bulk_update", + kwargs={ + "marking_session": self.request.current_session.label, + "section_name": self.kwargs["section_name"], + "question": self.kwargs["question"], + }, + ) + + def get_form(self): + self.section = get_object_or_404( + Section, + title=self.kwargs["section_name"], + marking_session=self.request.current_session, + ) + + q_parts = re.match(r"(\d+)([a-z]?)", self.kwargs["question"]) + q_kwargs = { + "section": self.section, + "number": q_parts.groups()[0], + } + + if len(q_parts.groups()) == 2 and q_parts.groups()[1] != "": + q_kwargs["number_part"] = q_parts.groups()[1] + + self.question = get_object_or_404(Question, **q_kwargs) + return self.form_class( + self.question.pk, + [(rt.type, rt.type) for rt in ResponseType.objects.all()], + self.request.current_session, + **self.get_form_kwargs(), + ) + + def get_context_data(self, **kwargs): + context = super().get_context_data(**kwargs) + + context["section"] = self.section + context["question"] = self.question + + return context + + def form_valid(self, form): + data = form.cleaned_data + + question = get_object_or_404(Question, id=data["question"]) + stage = get_object_or_404(ResponseType, type=data["stage"]) + is_multi = question.question_type == "multiple_choice" + + with transaction.atomic(): + for index, row in form.responses_df.iterrows(): + answer = row["answer"].strip() + if answer == "-": + continue + + if is_multi: + answers = answer.split("|") + + authority = PublicAuthority.objects.get( + name=row["authority"], + marking_session=self.request.current_session, + ) + if not is_multi: + option = Option.objects.get(question=question, description=answer) + + try: + response = Response.objects.get( + question=question, response_type=stage, authority=authority + ) + changed = False + opts = {} + for col in [ + "page_number", + "evidence", + "public_notes", + "private_notes", + ]: + val = row[col] + if pd.isna(val): + val = None + if col == "private_notes": + val = "" + if val != getattr(response, col): + opts[col] = val + changed = True + if not is_multi and response.option != option: + changed = True + opts["option"] = option + + if changed: + response.user = self.request.user + for k, v in opts.items(): + setattr(response, k, v) + response.save() + + if is_multi: + response.multi_option.clear() + for a in answers: + option = Option.objects.get( + question=question, description=a + ) + response.multi_option.add(option.id) + + except Response.DoesNotExist: + opts = { + "question": question, + "response_type": stage, + "authority": authority, + "user": self.request.user, + } + for col in ["page_number", "evidence", "public_notes"]: + if pd.isna(row[col]) is False: + opts[col] = row[col] + if not is_multi: + opts["option"] = option + + response = Response.objects.create(**opts) + + if is_multi: + for a in answers: + option = Option.objects.get( + question=question, description=a + ) + response.multi_option.add(option.id) + + return super().form_valid(form)