diff --git a/boranga/components/main/api.py b/boranga/components/main/api.py index 4590b281..68d54de2 100755 --- a/boranga/components/main/api.py +++ b/boranga/components/main/api.py @@ -54,13 +54,29 @@ class ContentTypeViewSet(viewsets.ReadOnlyModelViewSet): ) def ocr_bulk_import_content_types(self, request): """Returns a list of content types that are allowed to be imported in the ocr bulk importer""" - content_types = ContentType.objects.filter( - app_label="boranga", - ).filter( - Q(model__startswith="occurrencereport") - | Q(model__startswith="ocr") - | Q(model__iexact="occurrence") - | Q(model__iexact="submitterinformation") + content_types = ( + ContentType.objects.filter( + app_label="boranga", + ) + .filter( + Q(model__startswith="occurrencereport") + | Q(model__startswith="ocr") + | Q(model__iexact="occurrence") + | Q(model__iexact="submitterinformation") + ) + .exclude( + model__in=[ + "occurrencereportproposalrequest", + "occurrencereportdeclineddetails", + "occurrencereportshapefiledocument", + ] + ) + .exclude(model__icontains="amendment") + .exclude(model__icontains="bulkimport") + .exclude(model__icontains="referral") + .exclude(model__icontains="referee") + .exclude(model__icontains="occurrencereportlog") + .exclude(model__icontains="useraction") ) serializer = self.get_serializer(content_types, many=True) return Response(serializer.data) diff --git a/boranga/components/main/serializers.py b/boranga/components/main/serializers.py index 7ff141af..90841054 100755 --- a/boranga/components/main/serializers.py +++ b/boranga/components/main/serializers.py @@ -117,6 +117,7 @@ def get_user_can_administer(self, obj): class ContentTypeSerializer(serializers.ModelSerializer): model_fields = serializers.SerializerMethodField() model_verbose_name = serializers.SerializerMethodField() + model_abbreviation = serializers.SerializerMethodField() class Meta: model = ContentType @@ -127,6 +128,11 @@ def get_model_verbose_name(self, obj): return None return obj.model_class()._meta.verbose_name.title() + def get_model_abbreviation(self, obj): + if not obj.model_class(): + return None + return obj.model_class().BULK_IMPORT_ABBREVIATION + def get_model_fields(self, obj): if not obj.model_class(): return [] diff --git a/boranga/components/occurrence/models.py b/boranga/components/occurrence/models.py index 2d1e7411..eee0c1df 100644 --- a/boranga/components/occurrence/models.py +++ b/boranga/components/occurrence/models.py @@ -31,7 +31,16 @@ from django.core.files.uploadedfile import InMemoryUploadedFile from django.core.validators import MaxValueValidator, MinValueValidator from django.db import IntegrityError, models, transaction -from django.db.models import CharField, Count, Func, ManyToManyField, Max, Q +from django.db.models import ( + CharField, + Count, + Func, + ManyToManyField, + Max, + OuterRef, + Q, + Subquery, +) from django.db.models.functions import Cast, Length from django.utils import timezone from django.utils.functional import cached_property @@ -150,6 +159,7 @@ class OccurrenceReport(SubmitterInformationModelMixin, RevisionedMixin): objects = OccurrenceReportManager() + BULK_IMPORT_ABBREVIATION = "ocr" BULK_IMPORT_EXCLUDE_FIELDS = ["occurrence_report_number", "import_hash"] CUSTOMER_STATUS_DRAFT = "draft" @@ -1331,6 +1341,8 @@ class Meta: class OccurrenceReportApprovalDetails(models.Model): + BULK_IMPORT_ABBREVIATION = "ocrapp" + occurrence_report = models.OneToOneField( OccurrenceReport, on_delete=models.CASCADE, related_name="approval_details" ) @@ -1904,6 +1916,8 @@ def __str__(self): # NOTE: this and OCCLocation have a number of unused fields that should be removed class OCRLocation(models.Model): + BULK_IMPORT_ABBREVIATION = "ocrloc" + """ Location data for occurrence report @@ -2125,6 +2139,8 @@ class Meta: class OccurrenceReportGeometry(GeometryBase, DrawnByGeometry): + BULK_IMPORT_ABBREVIATION = "ocrgeo" + occurrence_report = models.ForeignKey( OccurrenceReport, on_delete=models.CASCADE, @@ -2154,6 +2170,8 @@ def save(self, *args, **kwargs): class OCRObserverDetail(RevisionedMixin): + BULK_IMPORT_ABBREVIATION = "ocrcon" + """ Observer data for occurrence report @@ -2347,6 +2365,7 @@ def __str__(self): class OCRHabitatComposition(models.Model): + BULK_IMPORT_ABBREVIATION = "ocrhab" """ Habitat data for occurrence report @@ -2400,6 +2419,7 @@ def __init__(self, *args, **kwargs): class OCRHabitatCondition(models.Model): + BULK_IMPORT_ABBREVIATION = "ocrhq" """ Habitat Condition data for occurrence report @@ -2461,6 +2481,8 @@ def __str__(self): class OCRVegetationStructure(models.Model): + BULK_IMPORT_ABBREVIATION = "ocrveg" + """ Vegetation Structure data for occurrence report @@ -2517,6 +2539,8 @@ def __str__(self): class OCRFireHistory(models.Model): + BULK_IMPORT_ABBREVIATION = "ocrfh" + """ Fire History data for occurrence report @@ -2546,6 +2570,8 @@ def __str__(self): class OCRAssociatedSpecies(models.Model): + BULK_IMPORT_ABBREVIATION = "ocrspe" + """ Associated Species data for occurrence report @@ -2600,6 +2626,8 @@ def __str__(self): class OCRObservationDetail(models.Model): + BULK_IMPORT_ABBREVIATION = "ocrobs" + """ Observation Details data for occurrence report @@ -2737,6 +2765,8 @@ def __str__(self): class OCRPlantCount(models.Model): + BULK_IMPORT_ABBREVIATION = "ocrnum" + """ Plant Count data for occurrence report @@ -2943,6 +2973,8 @@ def __str__(self): class OCRAnimalObservation(models.Model): + BULK_IMPORT_ABBREVIATION = "ocrnum" + """ Animal Observation data for occurrence report @@ -3130,6 +3162,8 @@ def __str__(self): class OCRIdentification(models.Model): + BULK_IMPORT_ABBREVIATION = "ocrid" + """ Identification data for occurrence report @@ -3171,6 +3205,8 @@ def __str__(self): class OccurrenceReportDocument(Document): + BULK_IMPORT_ABBREVIATION = "ocrdoc" + document_number = models.CharField(max_length=9, blank=True, default="") occurrence_report = models.ForeignKey( "OccurrenceReport", related_name="documents", on_delete=models.CASCADE @@ -3284,6 +3320,7 @@ class Meta: class OCRConservationThreat(RevisionedMixin): + BULK_IMPORT_ABBREVIATION = "ocrthr" """ Threat for a occurrence_report in a particular location. @@ -3381,6 +3418,7 @@ def get_queryset(self): class Occurrence(RevisionedMixin): + BULK_IMPORT_ABBREVIATION = "occ" REVIEW_STATUS_CHOICES = ( ("not_reviewed", "Not Reviewed"), @@ -6737,11 +6775,6 @@ def related_model_qs(self): if issubclass(self.related_model, ArchivableModel): related_model_qs = self.related_model.objects.exclude(archived=True) - if hasattr(self.related_model, "group_type"): - related_model_qs = related_model_qs.only(display_field, "group_type") - else: - related_model_qs = related_model_qs.only(display_field) - return related_model_qs.order_by(display_field) @cached_property @@ -6872,11 +6905,34 @@ def get_sample_value(self, errors, species_or_community_identifier=None): if isinstance(field, models.ForeignKey): related_model_qs = self.filtered_related_model_qs + # Special case for species or community + # Ensure only species or communities that have occurrences are selected + # in case the schema includes the occurrence model (quite likely) + if field.name == "species": + related_model_qs = related_model_qs.annotate( + occurrence_count=Subquery( + Occurrence.objects.filter(species__pk=OuterRef("pk")) + .values("species") + .annotate(count=Count("id")) + .values("count") + ) + ).filter(occurrence_count__gt=0) + if field.name == "community": + related_model_qs = related_model_qs.annotate( + occurrence_count=Subquery( + Occurrence.objects.filter(community__pk=OuterRef("pk")) + .values("community") + .annotate(count=Count("id")) + .values("count") + ) + ).filter(occurrence_count__gt=0) + if not related_model_qs.exists(): + error_message = f"No records found for foreign key field {field.related_model._meta.model_name}" errors.append( { "error_type": "no_records", - "error_message": f"No records found for foreign key {field.related_model._meta.model_name}", + "error_message": error_message, } ) @@ -6987,6 +7043,18 @@ def get_sample_value(self, errors, species_or_community_identifier=None): filter_field = { "community__taxonomy__community_migrated_id": species_or_community_identifier } + if not random_occurrence.filter(**filter_field).exists(): + error_message = ( + f"No occurrences found where species or community identifier = " + f"{species_or_community_identifier}" + ) + errors.append( + { + "error_type": "no_occurrences", + "error_message": error_message, + } + ) + return None return ( random_occurrence.filter(**filter_field) .order_by("?") diff --git a/boranga/components/users/models.py b/boranga/components/users/models.py index abc7e05d..e3203908 100755 --- a/boranga/components/users/models.py +++ b/boranga/components/users/models.py @@ -52,6 +52,8 @@ def get_filter_list(cls): class SubmitterInformation(models.Model): + BULK_IMPORT_ABBREVIATION = "ocrsub" + email_user = models.IntegerField(blank=True, null=True) name = models.CharField(max_length=100, blank=True, null=True) contact_details = models.TextField(blank=True, null=True) diff --git a/boranga/frontend/boranga/src/components/internal/occurrence/bulk_import_schema.vue b/boranga/frontend/boranga/src/components/internal/occurrence/bulk_import_schema.vue index 2734e755..13bd4fdb 100644 --- a/boranga/frontend/boranga/src/components/internal/occurrence/bulk_import_schema.vue +++ b/boranga/frontend/boranga/src/components/internal/occurrence/bulk_import_schema.vue @@ -39,9 +39,13 @@ class="bi bi-copy me-2"> Copy + Validat Preview @@ -450,21 +454,21 @@ @@ -580,6 +593,7 @@ export default { showDjangoImportFieldSelect: false, newColumn: null, saving: false, + validatingSchema: false, errors: null } }, @@ -714,7 +728,7 @@ export default { )[0] this.$nextTick(() => { this.enablePopovers(); - this.selectedColumn.xlsx_column_header_name = this.selectedField.display_name + this.selectedColumn.xlsx_column_header_name = `${this.selectedContentType.model_abbreviation.toUpperCase()} ${this.selectedField.display_name}` if (!this.selectedColumn.id) { this.selectedColumn.xlsx_data_validation_allow_blank = this.selectedField.allow_null } @@ -750,7 +764,8 @@ export default { default_value: null, import_validations: [], lookup_filters: [], - is_editable_by_user: true + is_editable_by_user: true, + is_emailuser_column: false } }, addSingleColumn() { @@ -804,7 +819,7 @@ export default { // Remove columns that are already in the schema newColumns = newColumns.filter(newColumn => !this.schema.columns.some(column => column.django_import_field_name == newColumn.django_import_field_name)) - if(newColumns.length == 0) { + if (newColumns.length == 0) { swal.fire({ title: 'No New Columns Added', text: 'All fields from the selected model are already in the schema', @@ -821,7 +836,7 @@ export default { newColumns = newColumns.filter(column => !column.xlsx_data_validation_allow_blank) } - if(newColumns.length == 0) { + if (newColumns.length == 0) { swal.fire({ title: 'No New Columns Added', text: 'There are no mandatory fields from the selected model that are not already in the schema', @@ -841,7 +856,7 @@ export default { this.addEditMode = false let lastColumnIndex = this.schema.columns.length; - if(this.schema.columns.filter(column => column.django_import_content_type == selectedContentType).length > 0) { + if (this.schema.columns.filter(column => column.django_import_content_type == selectedContentType).length > 0) { let lastColumn = this.schema.columns.findLast(column => column.django_import_content_type == selectedContentType) lastColumnIndex = this.schema.columns.indexOf(lastColumn) + 1 } @@ -947,6 +962,7 @@ export default { this.save(); }, validate() { + this.validatingSchema = true; this.$http.get(`${api_endpoints.occurrence_report_bulk_import_schemas}${this.schema.id}/validate/`) .then(response => { swal.fire({ @@ -966,16 +982,21 @@ export default { } else if (Object.hasOwn(error, 'body')) { errors = error.body } - let error_message = 'Something went wrong :-(' - if (errors instanceof Array) { - error_message = '' + let error_message_string = 'Something went wrong :-(' + if (errors instanceof Object) { + error_message_string = '' for (let i = 0; i < errors.length; i++) { - error_message += `
  • ${errors[i].error_message}
    • ` + let error_message = errors[i].error_message ? errors[i].error_message : errors[i] + error_message_string += `
  • ${error_message}
    • ` } + console.log(error_message_string) + } else if (typeof errors === 'string') { + error_message_string = errors } + console.error(error_message_string) swal.fire({ title: 'Schema Validation Failed', - html: error_message, + html: error_message_string, icon: 'error', confirmButtonText: 'OK', customClass: { @@ -983,6 +1004,9 @@ export default { } }) }) + .finally(() => { + this.validatingSchema = false; + }) }, save() { // If there is a column with no django_import_content_type or django_import_field_name, remove it diff --git a/boranga/settings.py b/boranga/settings.py index 47cd87c2..1b722baf 100755 --- a/boranga/settings.py +++ b/boranga/settings.py @@ -235,6 +235,8 @@ def show_toolbar(request): CRON_CLASSES = [ "appmonitor_client.cron.CronJobAppMonitorClient", "boranga.cron.CronJobFetchNomosTaxonDataDaily", + "boranga.cron.CronJobOCRPreProcessBulkImportTasks", + "boranga.cron.CronJobOCRProcessBulkImportQueue", ]