From 656ae28374e6394370ca623ac8757bcccb6bcd5c Mon Sep 17 00:00:00 2001 From: rwa Date: Mon, 26 Jul 2021 23:47:45 +0200 Subject: [PATCH] Extract Letter.reference_number to a model. (#1015) Mostly for autocomplete purposes. Fixes #1010 --- .../small_eod/collections/tests/test_views.py | 2 +- .../small_eod/collections/views.py | 2 +- backend-project/small_eod/events/models.py | 7 ++ backend-project/small_eod/letters/admin.py | 3 +- .../small_eod/letters/factories.py | 11 +++- .../small_eod/letters/filterset.py | 12 +++- .../0016_extract_reference_number_model.py | 65 +++++++++++++++++++ backend-project/small_eod/letters/models.py | 26 +++++++- .../small_eod/letters/searchset.py | 7 ++ .../small_eod/letters/serializers.py | 32 ++++++++- backend-project/small_eod/letters/views.py | 19 ++++-- .../small_eod/migration_v1/migrator.py | 9 ++- backend-project/small_eod/notes/models.py | 7 ++ 13 files changed, 185 insertions(+), 17 deletions(-) create mode 100644 backend-project/small_eod/letters/migrations/0016_extract_reference_number_model.py diff --git a/backend-project/small_eod/collections/tests/test_views.py b/backend-project/small_eod/collections/tests/test_views.py index 5d90f3c40..24c29511d 100644 --- a/backend-project/small_eod/collections/tests/test_views.py +++ b/backend-project/small_eod/collections/tests/test_views.py @@ -172,7 +172,7 @@ def get_extra_kwargs(self): return dict(collection_pk=self.collection.pk, case_pk=self.obj.case.pk) def validate_item(self, item): - self.assertEqual(self.obj.reference_number, item["referenceNumber"]) + self.assertEqual(self.obj.reference_number.name, item["referenceNumber"]) def increase_list(self): children = self.factory_class.create_batch(case=self.obj.case, size=5) diff --git a/backend-project/small_eod/collections/views.py b/backend-project/small_eod/collections/views.py index 93fd939d5..dfd5f1a3e 100644 --- a/backend-project/small_eod/collections/views.py +++ b/backend-project/small_eod/collections/views.py @@ -92,7 +92,7 @@ def get_queryset(self): case = Case.objects.filter(**parse_query(collection.query)).get( pk=self.kwargs["case_pk"] ) - return self.model.objects.filter(case=case).all() + return self.model.objects.filter(case=case).with_nested_resources().all() class EventCollectionViewSet(BaseSubCollection): diff --git a/backend-project/small_eod/events/models.py b/backend-project/small_eod/events/models.py index ee28cd9a0..1eb7bc95f 100644 --- a/backend-project/small_eod/events/models.py +++ b/backend-project/small_eod/events/models.py @@ -5,7 +5,14 @@ from ..generic.models import TimestampUserLogModel +class EventQuerySet(models.QuerySet): + def with_nested_resources(self): + return self + + class Event(TimestampUserLogModel): + objects = EventQuerySet.as_manager() + date = models.DateTimeField(verbose_name=_("Date"), help_text=_("Date of event.")) name = models.CharField( max_length=256, verbose_name=_("Name"), help_text=_("Name of event.") diff --git a/backend-project/small_eod/letters/admin.py b/backend-project/small_eod/letters/admin.py index 4c8e58052..d735afc52 100644 --- a/backend-project/small_eod/letters/admin.py +++ b/backend-project/small_eod/letters/admin.py @@ -10,7 +10,7 @@ from django.utils.translation import ugettext_lazy as _ from ..files.models import File -from .models import DocumentType, Letter +from .models import DocumentType, Letter, ReferenceNumber def link_to_case(obj): @@ -114,3 +114,4 @@ def get_queryset(self, request): admin.site.register(Letter, LetterAdmin) admin.site.register(DocumentType) +admin.site.register(ReferenceNumber) diff --git a/backend-project/small_eod/letters/factories.py b/backend-project/small_eod/letters/factories.py index 90f1bc9d1..a821cde4e 100644 --- a/backend-project/small_eod/letters/factories.py +++ b/backend-project/small_eod/letters/factories.py @@ -9,7 +9,7 @@ FuzzyTrueOrFalse, ) from ..institutions.factories import InstitutionFactory -from .models import DocumentType, Letter +from .models import DocumentType, Letter, ReferenceNumber class DocumentTypeFactory(DjangoModelFactory): @@ -19,6 +19,13 @@ class Meta: model = DocumentType +class ReferenceNumberFactory(DjangoModelFactory): + name = factory.Sequence(lambda n: "letter-reference_number-%04d" % n) + + class Meta: + model = ReferenceNumber + + class LetterFactory(AbstractTimestampUserFactory, DjangoModelFactory): final = FuzzyTrueOrFalse() @@ -27,12 +34,12 @@ class LetterFactory(AbstractTimestampUserFactory, DjangoModelFactory): comment = factory.Sequence(lambda n: "letter-comment-%04d" % n) excerpt = factory.Sequence(lambda n: "letter-excerpt-%04d" % n) - reference_number = factory.Sequence(lambda n: "letter-reference_number-%04d" % n) case = factory.SubFactory(CaseFactory) channel = factory.SubFactory(ChannelFactory) institution = factory.SubFactory(InstitutionFactory) document_type = factory.SubFactory(DocumentTypeFactory) + reference_number = factory.SubFactory(ReferenceNumberFactory) class Meta: model = Letter diff --git a/backend-project/small_eod/letters/filterset.py b/backend-project/small_eod/letters/filterset.py index 85f14b131..8c0d4d159 100644 --- a/backend-project/small_eod/letters/filterset.py +++ b/backend-project/small_eod/letters/filterset.py @@ -1,8 +1,8 @@ from django_filters.filterset import FilterSet from ..search.filter import SearchFilter -from .models import DocumentType, Letter -from .searchset import DocumentTypeSearchSet, LetterSearchSet +from .models import DocumentType, Letter, ReferenceNumber +from .searchset import DocumentTypeSearchSet, LetterSearchSet, ReferenceNumberSearchSet class DocumentTypeFilterSet(FilterSet): @@ -13,6 +13,14 @@ class Meta: fields = ["query"] +class ReferenceNumberFilterSet(FilterSet): + query = SearchFilter(searchset=ReferenceNumberSearchSet()) + + class Meta: + model = ReferenceNumber + fields = ["query"] + + class LetterFilterSet(FilterSet): query = SearchFilter(searchset=LetterSearchSet()) diff --git a/backend-project/small_eod/letters/migrations/0016_extract_reference_number_model.py b/backend-project/small_eod/letters/migrations/0016_extract_reference_number_model.py new file mode 100644 index 000000000..bcfae4090 --- /dev/null +++ b/backend-project/small_eod/letters/migrations/0016_extract_reference_number_model.py @@ -0,0 +1,65 @@ +from django.db import migrations, models +from django.db.models import F +import django.db.models.deletion + +def create_reference_numbers(apps, schema_editor): + Letter = apps.get_model("letters", "Letter") + ReferenceNumber = apps.get_model("letters", "ReferenceNumber") + db_alias = schema_editor.connection.alias + + # Gather existing reference numbers. + reference_numbers_names = Letter.objects.using(db_alias).values_list("reference_number", flat=True).distinct().order_by("reference_number") + + # Create a new model for each known reference number. + reference_number_objs = ReferenceNumber.objects.using(db_alias).bulk_create([ReferenceNumber(name=reference_number) for reference_number in reference_numbers_names]) + reference_numbers_name_to_obj = { rn.name: rn for rn in reference_number_objs } + + # Reference new models in each Letter. + letters = Letter.objects.using(db_alias).only("reference_number", "reference_number_temp").all() + for l in letters: + # We've iterated over all letters. Lookup should never fail. + l.reference_number_temp = reference_numbers_name_to_obj[l.reference_number] + Letter.objects.using(db_alias).bulk_update(letters, ['reference_number_temp']) + +def reverse_create_reference_numbers(apps, schema_editor): + Letter = apps.get_model("letters", "Letter") + db_alias = schema_editor.connection.alias + + # Copy the related object's name into Letter's own field. + letters = Letter.objects.using(db_alias).only("reference_number", "reference_number_temp").select_related("reference_number_temp").all() + for l in letters: + l.reference_number = l.reference_number_temp.name + Letter.objects.using(db_alias).bulk_update(letters, ['reference_number']) + + +class Migration(migrations.Migration): + + dependencies = [ + ('letters', '0015_alters_for_v1_data_migration'), + ] + + operations = [ + migrations.CreateModel( + name='ReferenceNumber', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('name', models.CharField(help_text='Reference number of letter.', max_length=256, unique=True, verbose_name='Reference number')), + ], + ), + # Add a temporary field to save new data to. + # It will be renamed in a separate step. + migrations.AddField( + model_name='letter', + name='reference_number_temp', + field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.DO_NOTHING, to='letters.referencenumber', verbose_name='Reference number'), + ), + # Copy values to a new field. + migrations.RunPython(create_reference_numbers, reverse_create_reference_numbers), + # Clean up old data. + migrations.RemoveField(model_name='letter', name='reference_number'), + migrations.RenameField( + model_name='letter', + old_name='reference_number_temp', + new_name='reference_number', + ), + ] diff --git a/backend-project/small_eod/letters/models.py b/backend-project/small_eod/letters/models.py index dff2faf0d..906ca16b8 100644 --- a/backend-project/small_eod/letters/models.py +++ b/backend-project/small_eod/letters/models.py @@ -17,7 +17,26 @@ class DocumentType(models.Model): ) +class ReferenceNumber(models.Model): + name = models.CharField( + max_length=256, + verbose_name=_("Reference number"), + help_text=_("Reference number of letter."), + unique=True, + ) + + def __str__(self): + return self.name + + +class LetterQuerySet(models.QuerySet): + def with_nested_resources(self): + return self.select_related("reference_number") + + class Letter(TimestampUserLogModel): + objects = LetterQuerySet.as_manager() + class Direction(models.TextChoices): IN = "IN", "Received" OUT = "OUT", "Sent" @@ -53,10 +72,11 @@ class Direction(models.TextChoices): help_text=_("Excerpt of letter."), blank=True, ) - reference_number = models.CharField( - max_length=256, + reference_number = models.ForeignKey( + to=ReferenceNumber, + on_delete=models.DO_NOTHING, verbose_name=_("Reference number"), - help_text=_("Reference number of letter."), + null=True, blank=True, ) case = models.ForeignKey( diff --git a/backend-project/small_eod/letters/searchset.py b/backend-project/small_eod/letters/searchset.py index ad95b466f..9ac371a1d 100644 --- a/backend-project/small_eod/letters/searchset.py +++ b/backend-project/small_eod/letters/searchset.py @@ -10,6 +10,13 @@ class DocumentTypeSearchSet(BaseSearchSet): } +class ReferenceNumberSearchSet(BaseSearchSet): + search_fields = ["name"] + filters = { + "id": lambda value: Q(pk=value), + } + + class LetterSearchSet(BaseSearchSet): search_fields = ["comment"] filters = { diff --git a/backend-project/small_eod/letters/serializers.py b/backend-project/small_eod/letters/serializers.py index c27809f7e..de3947368 100644 --- a/backend-project/small_eod/letters/serializers.py +++ b/backend-project/small_eod/letters/serializers.py @@ -9,7 +9,7 @@ from ..files.serializers import FileSerializer from ..generic.serializers import UserLogModelSerializer from ..institutions.models import Institution -from .models import DocumentType, Letter +from .models import DocumentType, Letter, ReferenceNumber class DocumentTypeSerializer(serializers.ModelSerializer): @@ -18,7 +18,14 @@ class Meta: fields = ["id", "name"] +class ReferenceNumberSerializer(serializers.ModelSerializer): + class Meta: + model = ReferenceNumber + fields = ["id", "name"] + + class LetterSerializer(UserLogModelSerializer): + reference_number = serializers.CharField(default=None) document_type = serializers.PrimaryKeyRelatedField( many=False, default=None, queryset=DocumentType.objects.all() ) @@ -54,6 +61,15 @@ class Meta: ] def create(self, validated_data): + # Reference numbers use the "tag" mode - they're provided by value and + # created if not matching any known objects. + reference_number_value = validated_data.pop("reference_number") + reference_number = ( + ReferenceNumber.objects.get_or_create(name=reference_number_value)[0] + if reference_number_value is not None + else None + ) + channel = validated_data.pop("channel") document_type = validated_data.pop("document_type") institution = validated_data.pop("institution") @@ -62,6 +78,7 @@ def create(self, validated_data): letter = super().create(validated_data) letter.channel = channel letter.document_type = document_type + letter.reference_number = reference_number letter.institution = institution letter.case = case letter.save() @@ -74,11 +91,24 @@ def update(self, instance, validated_data): Iterating over those 3 and updating fields of the related objects, using key-value pairs from PATCH request. """ + # NOTE(rwa_kulszowa): the section below doesn't seem to do much. nested = [] for nested_object in nested: for attr, value in nested_object["data"].items(): setattr(nested_object["instance"], attr, value) nested_object["instance"].save() + + # Create a new reference number if necessary. + # See comment in `create`. + if "reference_number" in validated_data: + reference_number_value = validated_data.pop("reference_number") + reference_number = ( + ReferenceNumber.objects.get_or_create(name=reference_number_value)[0] + if reference_number_value is not None + else None + ) + validated_data["reference_number"] = reference_number + return super().update(instance, validated_data) diff --git a/backend-project/small_eod/letters/views.py b/backend-project/small_eod/letters/views.py index b283e6088..9b40dd74d 100644 --- a/backend-project/small_eod/letters/views.py +++ b/backend-project/small_eod/letters/views.py @@ -8,13 +8,18 @@ from ..files.models import File from ..files.serializers import FileSerializer -from .filterset import DocumentTypeFilterSet, LetterFilterSet -from .models import DocumentType, Letter -from .serializers import DocumentTypeSerializer, LetterSerializer, SignRequestSerializer +from .filterset import DocumentTypeFilterSet, LetterFilterSet, ReferenceNumberFilterSet +from .models import DocumentType, Letter, ReferenceNumber +from .serializers import ( + DocumentTypeSerializer, + LetterSerializer, + ReferenceNumberSerializer, + SignRequestSerializer, +) class LetterViewSet(viewsets.ModelViewSet): - queryset = Letter.objects.prefetch_related("attachments").all() + queryset = Letter.objects.prefetch_related("attachments", "reference_number").all() serializer_class = LetterSerializer filter_backends = (DjangoFilterBackend, OrderingFilter) filterset_class = LetterFilterSet @@ -44,6 +49,12 @@ class DocumentTypeViewSet(viewsets.ModelViewSet): filterset_class = DocumentTypeFilterSet +class ReferenceNumberViewSet(viewsets.ModelViewSet): + queryset = ReferenceNumber.objects.all() + serializer_class = ReferenceNumberSerializer + filterset_class = ReferenceNumberFilterSet + + class FileViewSet( viewsets.ModelViewSet, viewsets.GenericViewSet, diff --git a/backend-project/small_eod/migration_v1/migrator.py b/backend-project/small_eod/migration_v1/migrator.py index 82b2349e3..b0106802c 100644 --- a/backend-project/small_eod/migration_v1/migrator.py +++ b/backend-project/small_eod/migration_v1/migrator.py @@ -11,7 +11,7 @@ from ..features.models import Feature, FeatureOption from ..files.models import File from ..institutions.models import Institution -from ..letters.models import DocumentType, Letter +from ..letters.models import DocumentType, Letter, ReferenceNumber from ..tags.models import Tag from . import models as models_v1 @@ -270,13 +270,17 @@ def migrate_letter(old_letter): ) ) + new_reference_number, _ = ReferenceNumber.objects.get_or_create( + name=old_letter.identifier + ) + new_letter = Letter( direction=new_direction, date=new_date, comment=old_letter.comment, document_type=migrate_lettername(old_letter.name), # excerpt=??? - reference_number=old_letter.identifier, + reference_number=new_reference_number, ) if old_letter.institution: new_letter.institution = migrate_institution(old_letter.institution) @@ -311,6 +315,7 @@ def run(clean=False): FeatureOption.objects.all().delete() Channel.objects.all().delete() DocumentType.objects.all().delete() + ReferenceNumber.objects.all().delete() get_user_model().objects.all().delete() logger.info("Running v1 -> v2 data migration") diff --git a/backend-project/small_eod/notes/models.py b/backend-project/small_eod/notes/models.py index a041c4a6b..4e026c176 100644 --- a/backend-project/small_eod/notes/models.py +++ b/backend-project/small_eod/notes/models.py @@ -5,7 +5,14 @@ from ..generic.models import TimestampUserLogModel +class NoteQuerySet(models.QuerySet): + def with_nested_resources(self): + return self + + class Note(TimestampUserLogModel): + objects = NoteQuerySet.as_manager() + comment = models.CharField(max_length=256, verbose_name=_("Comment")) case = models.ForeignKey( to=Case,