Extract Letter.reference_number to a model. (#1015)

Mostly for autocomplete purposes.

Fixes #1010

backend-project/small_eod/collections/tests/test_views.py

@@ -172,7 +172,7 @@ def get_extra_kwargs(self):
         return dict(collection_pk=self.collection.pk, case_pk=self.obj.case.pk)
 
     def validate_item(self, item):
-        self.assertEqual(self.obj.reference_number, item["referenceNumber"])
+        self.assertEqual(self.obj.reference_number.name, item["referenceNumber"])
 
     def increase_list(self):
         children = self.factory_class.create_batch(case=self.obj.case, size=5)

backend-project/small_eod/collections/views.py

@@ -92,7 +92,7 @@ def get_queryset(self):
         case = Case.objects.filter(**parse_query(collection.query)).get(
             pk=self.kwargs["case_pk"]
         )
-        return self.model.objects.filter(case=case).all()
+        return self.model.objects.filter(case=case).with_nested_resources().all()
 
 
 class EventCollectionViewSet(BaseSubCollection):

backend-project/small_eod/events/models.py

@@ -5,7 +5,14 @@
 from ..generic.models import TimestampUserLogModel
 
 
+class EventQuerySet(models.QuerySet):
+    def with_nested_resources(self):
+        return self
+
+
 class Event(TimestampUserLogModel):
+    objects = EventQuerySet.as_manager()
+
     date = models.DateTimeField(verbose_name=_("Date"), help_text=_("Date of event."))
     name = models.CharField(
         max_length=256, verbose_name=_("Name"), help_text=_("Name of event.")

backend-project/small_eod/letters/admin.py

@@ -10,7 +10,7 @@
 from django.utils.translation import ugettext_lazy as _
 
 from ..files.models import File
-from .models import DocumentType, Letter
+from .models import DocumentType, Letter, ReferenceNumber
 
 
 def link_to_case(obj):
@@ -114,3 +114,4 @@ def get_queryset(self, request):
 
 admin.site.register(Letter, LetterAdmin)
 admin.site.register(DocumentType)
+admin.site.register(ReferenceNumber)

backend-project/small_eod/letters/factories.py

@@ -9,7 +9,7 @@
     FuzzyTrueOrFalse,
 )
 from ..institutions.factories import InstitutionFactory
-from .models import DocumentType, Letter
+from .models import DocumentType, Letter, ReferenceNumber
 
 
 class DocumentTypeFactory(DjangoModelFactory):
@@ -19,6 +19,13 @@ class Meta:
         model = DocumentType
 
 
+class ReferenceNumberFactory(DjangoModelFactory):
+    name = factory.Sequence(lambda n: "letter-reference_number-%04d" % n)
+
+    class Meta:
+        model = ReferenceNumber
+
+
 class LetterFactory(AbstractTimestampUserFactory, DjangoModelFactory):
 
     final = FuzzyTrueOrFalse()
@@ -27,12 +34,12 @@ class LetterFactory(AbstractTimestampUserFactory, DjangoModelFactory):
 
     comment = factory.Sequence(lambda n: "letter-comment-%04d" % n)
     excerpt = factory.Sequence(lambda n: "letter-excerpt-%04d" % n)
-    reference_number = factory.Sequence(lambda n: "letter-reference_number-%04d" % n)
 
     case = factory.SubFactory(CaseFactory)
     channel = factory.SubFactory(ChannelFactory)
     institution = factory.SubFactory(InstitutionFactory)
     document_type = factory.SubFactory(DocumentTypeFactory)
+    reference_number = factory.SubFactory(ReferenceNumberFactory)
 
     class Meta:
         model = Letter

backend-project/small_eod/letters/filterset.py

@@ -1,8 +1,8 @@
 from django_filters.filterset import FilterSet
 
 from ..search.filter import SearchFilter
-from .models import DocumentType, Letter
-from .searchset import DocumentTypeSearchSet, LetterSearchSet
+from .models import DocumentType, Letter, ReferenceNumber
+from .searchset import DocumentTypeSearchSet, LetterSearchSet, ReferenceNumberSearchSet
 
 
 class DocumentTypeFilterSet(FilterSet):
@@ -13,6 +13,14 @@ class Meta:
         fields = ["query"]
 
 
+class ReferenceNumberFilterSet(FilterSet):
+    query = SearchFilter(searchset=ReferenceNumberSearchSet())
+
+    class Meta:
+        model = ReferenceNumber
+        fields = ["query"]
+
+
 class LetterFilterSet(FilterSet):
     query = SearchFilter(searchset=LetterSearchSet())
 

backend-project/small_eod/letters/migrations/0016_extract_reference_number_model.py

@@ -0,0 +1,65 @@
+from django.db import migrations, models
+from django.db.models import F
+import django.db.models.deletion
+
+def create_reference_numbers(apps, schema_editor):
+    Letter = apps.get_model("letters", "Letter")
+    ReferenceNumber = apps.get_model("letters", "ReferenceNumber")
+    db_alias = schema_editor.connection.alias
+
+    # Gather existing reference numbers.
+    reference_numbers_names = Letter.objects.using(db_alias).values_list("reference_number", flat=True).distinct().order_by("reference_number")
+
+    # Create a new model for each known reference number.
+    reference_number_objs = ReferenceNumber.objects.using(db_alias).bulk_create([ReferenceNumber(name=reference_number) for reference_number in reference_numbers_names])
+    reference_numbers_name_to_obj = { rn.name: rn for rn in reference_number_objs }
+
+    # Reference new models in each Letter.
+    letters = Letter.objects.using(db_alias).only("reference_number", "reference_number_temp").all()
+    for l in letters:
+        # We've iterated over all letters. Lookup should never fail.
+        l.reference_number_temp = reference_numbers_name_to_obj[l.reference_number]
+    Letter.objects.using(db_alias).bulk_update(letters, ['reference_number_temp'])
+
+def reverse_create_reference_numbers(apps, schema_editor):
+    Letter = apps.get_model("letters", "Letter")
+    db_alias = schema_editor.connection.alias
+
+    # Copy the related object's name into Letter's own field.
+    letters = Letter.objects.using(db_alias).only("reference_number", "reference_number_temp").select_related("reference_number_temp").all()
+    for l in letters:
+        l.reference_number = l.reference_number_temp.name
+    Letter.objects.using(db_alias).bulk_update(letters, ['reference_number'])
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('letters', '0015_alters_for_v1_data_migration'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='ReferenceNumber',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('name', models.CharField(help_text='Reference number of letter.', max_length=256, unique=True, verbose_name='Reference number')),
+            ],
+        ),
+        # Add a temporary field to save new data to.
+        # It will be renamed in a separate step.
+        migrations.AddField(
+            model_name='letter',
+            name='reference_number_temp',
+            field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.DO_NOTHING, to='letters.referencenumber', verbose_name='Reference number'),
+        ),
+        # Copy values to a new field.
+        migrations.RunPython(create_reference_numbers, reverse_create_reference_numbers),
+        # Clean up old data.
+        migrations.RemoveField(model_name='letter', name='reference_number'),
+        migrations.RenameField(
+            model_name='letter',
+            old_name='reference_number_temp',
+            new_name='reference_number',
+        ),
+    ]

backend-project/small_eod/letters/models.py

@@ -17,7 +17,26 @@ class DocumentType(models.Model):
     )
 
 
+class ReferenceNumber(models.Model):
+    name = models.CharField(
+        max_length=256,
+        verbose_name=_("Reference number"),
+        help_text=_("Reference number of letter."),
+        unique=True,
+    )
+
+    def __str__(self):
+        return self.name
+
+
+class LetterQuerySet(models.QuerySet):
+    def with_nested_resources(self):
+        return self.select_related("reference_number")
+
+
 class Letter(TimestampUserLogModel):
+    objects = LetterQuerySet.as_manager()
+
     class Direction(models.TextChoices):
         IN = "IN", "Received"
         OUT = "OUT", "Sent"
@@ -53,10 +72,11 @@ class Direction(models.TextChoices):
         help_text=_("Excerpt of letter."),
         blank=True,
     )
-    reference_number = models.CharField(
-        max_length=256,
+    reference_number = models.ForeignKey(
+        to=ReferenceNumber,
+        on_delete=models.DO_NOTHING,
         verbose_name=_("Reference number"),
-        help_text=_("Reference number of letter."),
+        null=True,
         blank=True,
     )
     case = models.ForeignKey(

backend-project/small_eod/letters/searchset.py

@@ -10,6 +10,13 @@ class DocumentTypeSearchSet(BaseSearchSet):
     }
 
 
+class ReferenceNumberSearchSet(BaseSearchSet):
+    search_fields = ["name"]
+    filters = {
+        "id": lambda value: Q(pk=value),
+    }
+
+
 class LetterSearchSet(BaseSearchSet):
     search_fields = ["comment"]
     filters = {

backend-project/small_eod/letters/serializers.py

@@ -9,7 +9,7 @@
 from ..files.serializers import FileSerializer
 from ..generic.serializers import UserLogModelSerializer
 from ..institutions.models import Institution
-from .models import DocumentType, Letter
+from .models import DocumentType, Letter, ReferenceNumber
 
 
 class DocumentTypeSerializer(serializers.ModelSerializer):
@@ -18,7 +18,14 @@ class Meta:
         fields = ["id", "name"]
 
 
+class ReferenceNumberSerializer(serializers.ModelSerializer):
+    class Meta:
+        model = ReferenceNumber
+        fields = ["id", "name"]
+
+
 class LetterSerializer(UserLogModelSerializer):
+    reference_number = serializers.CharField(default=None)
     document_type = serializers.PrimaryKeyRelatedField(
         many=False, default=None, queryset=DocumentType.objects.all()
     )
@@ -54,6 +61,15 @@ class Meta:
         ]
 
     def create(self, validated_data):
+        # Reference numbers use the "tag" mode - they're provided by value and
+        # created if not matching any known objects.
+        reference_number_value = validated_data.pop("reference_number")
+        reference_number = (
+            ReferenceNumber.objects.get_or_create(name=reference_number_value)[0]
+            if reference_number_value is not None
+            else None
+        )
+
         channel = validated_data.pop("channel")
         document_type = validated_data.pop("document_type")
         institution = validated_data.pop("institution")
@@ -62,6 +78,7 @@ def create(self, validated_data):
         letter = super().create(validated_data)
         letter.channel = channel
         letter.document_type = document_type
+        letter.reference_number = reference_number
         letter.institution = institution
         letter.case = case
         letter.save()
@@ -74,11 +91,24 @@ def update(self, instance, validated_data):
         Iterating over those 3 and updating fields of the related objects,
         using key-value pairs from PATCH request.
         """
+        # NOTE(rwa_kulszowa): the section below doesn't seem to do much.
         nested = []
         for nested_object in nested:
             for attr, value in nested_object["data"].items():
                 setattr(nested_object["instance"], attr, value)
             nested_object["instance"].save()
+
+        # Create a new reference number if necessary.
+        # See comment in `create`.
+        if "reference_number" in validated_data:
+            reference_number_value = validated_data.pop("reference_number")
+            reference_number = (
+                ReferenceNumber.objects.get_or_create(name=reference_number_value)[0]
+                if reference_number_value is not None
+                else None
+            )
+            validated_data["reference_number"] = reference_number
+
         return super().update(instance, validated_data)
 
 

backend-project/small_eod/letters/views.py

@@ -8,13 +8,18 @@
 
 from ..files.models import File
 from ..files.serializers import FileSerializer
-from .filterset import DocumentTypeFilterSet, LetterFilterSet
-from .models import DocumentType, Letter
-from .serializers import DocumentTypeSerializer, LetterSerializer, SignRequestSerializer
+from .filterset import DocumentTypeFilterSet, LetterFilterSet, ReferenceNumberFilterSet
+from .models import DocumentType, Letter, ReferenceNumber
+from .serializers import (
+    DocumentTypeSerializer,
+    LetterSerializer,
+    ReferenceNumberSerializer,
+    SignRequestSerializer,
+)
 
 
 class LetterViewSet(viewsets.ModelViewSet):
-    queryset = Letter.objects.prefetch_related("attachments").all()
+    queryset = Letter.objects.prefetch_related("attachments", "reference_number").all()
     serializer_class = LetterSerializer
     filter_backends = (DjangoFilterBackend, OrderingFilter)
     filterset_class = LetterFilterSet
@@ -44,6 +49,12 @@ class DocumentTypeViewSet(viewsets.ModelViewSet):
     filterset_class = DocumentTypeFilterSet
 
 
+class ReferenceNumberViewSet(viewsets.ModelViewSet):
+    queryset = ReferenceNumber.objects.all()
+    serializer_class = ReferenceNumberSerializer
+    filterset_class = ReferenceNumberFilterSet
+
+
 class FileViewSet(
     viewsets.ModelViewSet,
     viewsets.GenericViewSet,

backend-project/small_eod/migration_v1/migrator.py

@@ -11,7 +11,7 @@
 from ..features.models import Feature, FeatureOption
 from ..files.models import File
 from ..institutions.models import Institution
-from ..letters.models import DocumentType, Letter
+from ..letters.models import DocumentType, Letter, ReferenceNumber
 from ..tags.models import Tag
 from . import models as models_v1
 
@@ -270,13 +270,17 @@ def migrate_letter(old_letter):
         )
     )
 
+    new_reference_number, _ = ReferenceNumber.objects.get_or_create(
+        name=old_letter.identifier
+    )
+
     new_letter = Letter(
         direction=new_direction,
         date=new_date,
         comment=old_letter.comment,
         document_type=migrate_lettername(old_letter.name),
         # excerpt=???
-        reference_number=old_letter.identifier,
+        reference_number=new_reference_number,
     )
     if old_letter.institution:
         new_letter.institution = migrate_institution(old_letter.institution)
@@ -311,6 +315,7 @@ def run(clean=False):
         FeatureOption.objects.all().delete()
         Channel.objects.all().delete()
         DocumentType.objects.all().delete()
+        ReferenceNumber.objects.all().delete()
         get_user_model().objects.all().delete()
 
     logger.info("Running v1 -> v2 data migration")

backend-project/small_eod/notes/models.py

@@ -5,7 +5,14 @@
 from ..generic.models import TimestampUserLogModel
 
 
+class NoteQuerySet(models.QuerySet):
+    def with_nested_resources(self):
+        return self
+
+
 class Note(TimestampUserLogModel):
+    objects = NoteQuerySet.as_manager()
+
     comment = models.CharField(max_length=256, verbose_name=_("Comment"))
     case = models.ForeignKey(
         to=Case,