From 5922d01a018ea63f07eef91c89bcd8211064f828 Mon Sep 17 00:00:00 2001 From: Kiran Dawadi Date: Sun, 15 Dec 2024 19:52:47 -0600 Subject: [PATCH 1/5] Add documentation for PairedFieldDescriptor implementation --- .../utils/README_PAIRED_FIELD_DESCRIPTOR.md | 90 +++++++++++++++++++ 1 file changed, 90 insertions(+) create mode 100644 sde_collections/utils/README_PAIRED_FIELD_DESCRIPTOR.md diff --git a/sde_collections/utils/README_PAIRED_FIELD_DESCRIPTOR.md b/sde_collections/utils/README_PAIRED_FIELD_DESCRIPTOR.md new file mode 100644 index 00000000..3003cd58 --- /dev/null +++ b/sde_collections/utils/README_PAIRED_FIELD_DESCRIPTOR.md @@ -0,0 +1,90 @@ +# Paired Field Descriptor System + +## Overview + +The Paired Field Descriptor is a Django model descriptor designed to manage fields with both manual and machine learning (ML) generated variants. This system provides a flexible approach to handling metadata fields, with a focus on tag management and priority handling. + +## Core Concepts + +### Field Pairing Mechanism +The descriptor automatically creates two associated fields for each defined descriptor: +- **Manual Field**: Manually entered or curated metadata +- **ML Field**: Machine learning generated metadata + +### Key Characteristics +- Manual field takes precedence over ML field +- Flexible field type support +- Handles empty arrays and None values +- Requires explicit setting of ML fields + +## Implementation + +### Creating a Paired Field Descriptor + +```python +tdamm_tag = PairedFieldDescriptor( + field_name="tdamm_tag", + field_type=ArrayField(models.CharField(max_length=255, choices=TDAMMTags.choices), blank=True, null=True), + verbose_name="TDAMM Tags", +) +``` + +#### Parameters +- `field_name`: Base name for the descriptor +- `field_type`: Django field type (supports various field types) +- `verbose_name`: Optional human-readable name + +### Field Naming Convention +When you define a descriptor, two additional fields are automatically created: +- `{field_name}_manual`: For manually entered values +- `{field_name}_ml`: For machine learning generated values + +## Characteristics + +### Field Priority +1. Manual field always takes precedence +2. ML field serves as a fallback +3. Empty manual fields or None values defer to ML field + +### Field Retrieval +```python +# Retrieval automatically prioritizes manual field +tags = url.tdamm_tag # Returns manual tags if exist, otherwise ML tags +``` + +### Field Setting +```python +# Sets only the manual field +url.tdamm_tag = ["MMA_M_EM", "MMA_M_G"] + +# ML field must be set explicitly +url.tdamm_tag_ml = ["MMA_O_BH"] +``` + +### Field Deletion +```python +# Deletes both manual and ML fields +del url.tdamm_tag +``` + +### Data Preservation +- Paired fields maintain their state during: + - Dump to Delta migration + - Delta to Curated promotion +- Manual entries take precedence in all migration stages + +## Serializer Integration + +Here's the way to configure the serializer to retrieve the paired field, seamlessly extracting either manual or ML tags based on the descriptor's priority rules. +```python +class DeltaUrlSerializer(serializers.ModelSerializer): + tdamm_tag = serializers.SerializerMethodField() + + class Meta: + model = DeltaUrl + fields = ("url", "tdamm_tag") + + def get_tdamm_tag(self, obj): + tags = obj.tdamm_tag + return tags if tags is not None else [] +``` \ No newline at end of file From 3c8b985c4e517d1698b28e30047a9b871edd079f Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 16 Dec 2024 01:56:34 +0000 Subject: [PATCH 2/5] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- sde_collections/utils/README_PAIRED_FIELD_DESCRIPTOR.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sde_collections/utils/README_PAIRED_FIELD_DESCRIPTOR.md b/sde_collections/utils/README_PAIRED_FIELD_DESCRIPTOR.md index 3003cd58..cd6cc4fc 100644 --- a/sde_collections/utils/README_PAIRED_FIELD_DESCRIPTOR.md +++ b/sde_collections/utils/README_PAIRED_FIELD_DESCRIPTOR.md @@ -87,4 +87,4 @@ class DeltaUrlSerializer(serializers.ModelSerializer): def get_tdamm_tag(self, obj): tags = obj.tdamm_tag return tags if tags is not None else [] -``` \ No newline at end of file +``` From 3e828b28d2c6045eed81414e257a94a2b1123163 Mon Sep 17 00:00:00 2001 From: Dhanur Sharma Date: Wed, 18 Dec 2024 17:46:41 +0530 Subject: [PATCH 3/5] Javascript updated --- sde_indexing_helper/static/js/delta_url_list.js | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/sde_indexing_helper/static/js/delta_url_list.js b/sde_indexing_helper/static/js/delta_url_list.js index 060eff59..8caee0d6 100644 --- a/sde_indexing_helper/static/js/delta_url_list.js +++ b/sde_indexing_helper/static/js/delta_url_list.js @@ -2199,6 +2199,7 @@ function handleWorkflowStatusSelect() { function handleReindexingStatusSelect() { $("body").on("click", ".reindexing_status_select", function () { + console.log("Reindexing status select clicked"); $("#reindexingStatusChangeModal").modal(); var collectionName = $(".urlStyle").text(); var collection_id = $(this).data("collection-id"); @@ -2219,13 +2220,13 @@ function handleReindexingStatusSelect() { case "changeReindexingStatus": var color_choices = { 1: "btn-light", // REINDEXING_NOT_NEEDED - 2: "btn-warning", // REINDEXING_NEEDED_ON_DEV - 3: "btn-secondary", // REINDEXING_FINISHED_ON_DEV - 4: "btn-info", // REINDEXING_READY_FOR_CURATION - 5: "btn-warning", // REINDEXING_CURATION_IN_PROGRESS - 6: "btn-primary", // REINDEXING_CURATED - 7: "btn-success" // REINDEXING_INDEXED_ON_PROD - }; + 2: "btn-danger", // REINDEXING_NEEDED_ON_DEV (matching Ready For Engineering) + 3: "btn-info", // REINDEXING_FINISHED_ON_DEV (matching Indexing Finished on LRM Dev) + 4: "btn-info", // REINDEXING_READY_FOR_CURATION (matching Ready for Curation) + 5: "btn-success", // REINDEXING_CURATION_IN_PROGRESS (matching Curation in Progress) + 6: "btn-primary", // REINDEXING_CURATED (matching Curated) + 7: "btn-primary" // REINDEXING_INDEXED_ON_PROD (matching Prod: Perfect) + }; $button = $(`#reindexing-status-button-${collection_id}`); @@ -2233,7 +2234,9 @@ function handleReindexingStatusSelect() { $button.removeClass( "btn-light btn-danger btn-warning btn-info btn-success btn-primary btn-secondary" ); + console.log("After remove class:", $button.attr('class')); $button.addClass(color_choices[parseInt(reindexing_status)]); + console.log("After add class:", $button.attr('class')); postReindexingStatus(collection_id, reindexing_status); $("#reindexingStatusChangeModal").modal("hide"); break; From 006440381cd781954759b2193e5cbd197f5c082f Mon Sep 17 00:00:00 2001 From: Dhanur Sharma Date: Wed, 18 Dec 2024 17:50:39 +0530 Subject: [PATCH 4/5] Updated collection.py --- sde_collections/models/collection.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/sde_collections/models/collection.py b/sde_collections/models/collection.py index e9d4295e..0f1162e1 100644 --- a/sde_collections/models/collection.py +++ b/sde_collections/models/collection.py @@ -346,13 +346,13 @@ def workflow_status_button_color(self) -> str: @property def reindexing_status_button_color(self) -> str: color_choices = { - 1: "btn-light", # NOT_NEEDED - 2: "btn-warning", # NEEDED - 3: "btn-secondary", # FINISHED - 4: "btn-info", # READY_FOR_CURATION - 5: "btn-warning", # CURATION_IN_PROGRESS - 6: "btn-primary", # CURATED - 7: "btn-success", # INDEXED_ON_PROD + 1: "btn-light", # REINDEXING_NOT_NEEDED + 2: "btn-danger", # REINDEXING_NEEDED_ON_DEV (matching Ready For Engineering) + 3: "btn-info", # REINDEXING_FINISHED_ON_DEV (matching Indexing Finished on LRM Dev) + 4: "btn-info", # REINDEXING_READY_FOR_CURATION (matching Ready for Curation) + 5: "btn-success", # REINDEXING_CURATION_IN_PROGRESS (matching Curation in Progress) + 6: "btn-primary", # REINDEXING_CURATED (matching Curated) + 7: "btn-primary", # REINDEXING_INDEXED_ON_PROD (matching Prod: Perfect) } return color_choices[self.reindexing_status] From a081574298b0dbd28a1959ddfa1928b3bf7aa0d1 Mon Sep 17 00:00:00 2001 From: Dhanur Sharma Date: Wed, 18 Dec 2024 18:04:53 +0530 Subject: [PATCH 5/5] Removed console log statements --- sde_indexing_helper/static/js/delta_url_list.js | 3 --- 1 file changed, 3 deletions(-) diff --git a/sde_indexing_helper/static/js/delta_url_list.js b/sde_indexing_helper/static/js/delta_url_list.js index 8caee0d6..538c6e6a 100644 --- a/sde_indexing_helper/static/js/delta_url_list.js +++ b/sde_indexing_helper/static/js/delta_url_list.js @@ -2199,7 +2199,6 @@ function handleWorkflowStatusSelect() { function handleReindexingStatusSelect() { $("body").on("click", ".reindexing_status_select", function () { - console.log("Reindexing status select clicked"); $("#reindexingStatusChangeModal").modal(); var collectionName = $(".urlStyle").text(); var collection_id = $(this).data("collection-id"); @@ -2234,9 +2233,7 @@ function handleReindexingStatusSelect() { $button.removeClass( "btn-light btn-danger btn-warning btn-info btn-success btn-primary btn-secondary" ); - console.log("After remove class:", $button.attr('class')); $button.addClass(color_choices[parseInt(reindexing_status)]); - console.log("After add class:", $button.attr('class')); postReindexingStatus(collection_id, reindexing_status); $("#reindexingStatusChangeModal").modal("hide"); break;