task sync manager

apps/taskman/constants.py

@@ -11,6 +11,9 @@
 JIRA_TASKMAN_AUTO_SYNC_FLAW = get_env(
     "JIRA_TASKMAN_AUTO_SYNC_FLAW", default="0", is_bool=True
 )
+JIRA_TASKMAN_ASYNCHRONOUS_SYNC = get_env(
+    "JIRA_TASKMAN_ASYNCHRONOUS_SYNC", default="False", is_bool=True
+)
 
 SYNC_REQUIRED_FIELDS = [
     "cve_id",

apps/taskman/service.py

@@ -21,11 +21,7 @@
     JIRA_TASKMAN_PROJECT_KEY,
     JIRA_TASKMAN_URL,
 )
-from .exceptions import (
-    JiraTaskErrorException,
-    MissingJiraTokenException,
-    TaskWritePermissionsException,
-)
+from .exceptions import JiraTaskErrorException, TaskWritePermissionsException
 
 logger = logging.getLogger(__name__)
 
@@ -80,14 +76,12 @@ def __init__(self, token) -> None:
 
         Keyword arguments:
         token -- user token used in every request to Jira
+                 the service one is used if not provided
         """
         super().__init__()
-        if not token:
-            raise MissingJiraTokenException(
-                "User's Jira Token is required to perform this action."
-            )
         self._jira_server = JIRA_TASKMAN_URL
-        self._jira_token = token
+        if token:
+            self._jira_token = token
 
     def _check_token(self) -> None:
         """

conftest.py

@@ -262,10 +262,8 @@ def enable_jira_task_sync(monkeypatch) -> None:
     """
     import apps.taskman.mixins as mixins
     import apps.taskman.service as service
-    import osidb.models.flaw.flaw as flaw_module
     import osidb.serializer as serializer
 
-    monkeypatch.setattr(flaw_module, "JIRA_TASKMAN_AUTO_SYNC_FLAW", True)
     monkeypatch.setattr(mixins, "JIRA_TASKMAN_AUTO_SYNC_FLAW", True)
     monkeypatch.setattr(serializer, "JIRA_TASKMAN_AUTO_SYNC_FLAW", True)
     monkeypatch.setattr(service, "JIRA_STORY_ISSUE_TYPE_ID", "17")

docker-compose.yml

@@ -81,6 +81,7 @@ services:
         JIRA_AUTH_TOKEN: ${JIRA_AUTH_TOKEN:?Variable JIRA_AUTH_TOKEN must be set.}
         JIRA_MAX_CONNECTION_AGE: ${JIRA_MAX_CONNECTION_AGE}
         JIRA_STORY_ISSUE_TYPE_ID:  ${JIRA_STORY_ISSUE_TYPE_ID}
+        JIRA_TASKMAN_ASYNCHRONOUS_SYNC: ${JIRA_TASKMAN_ASYNCHRONOUS_SYNC}
         JIRA_TASKMAN_AUTO_SYNC_FLAW: ${JIRA_TASKMAN_AUTO_SYNC_FLAW}
         JIRA_TASKMAN_PROJECT_ID: ${JIRA_TASKMAN_PROJECT_ID}
         JIRA_TASKMAN_PROJECT_KEY: ${JIRA_TASKMAN_PROJECT_KEY}

osidb/migrations/0178_jiratasksyncmanager.py

@@ -0,0 +1,122 @@
+# Generated by Django 4.2.16 on 2024-12-17 09:34
+
+from django.db import migrations, models
+import django.db.models.deletion
+import pgtrigger.compiler
+import pgtrigger.migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("osidb", "0177_remove_affect_insert_insert_and_more"),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name="JiraTaskSyncManager",
+            fields=[
+                (
+                    "id",
+                    models.BigAutoField(
+                        auto_created=True,
+                        primary_key=True,
+                        serialize=False,
+                        verbose_name="ID",
+                    ),
+                ),
+                ("sync_id", models.CharField(max_length=100, unique=True)),
+                ("last_scheduled_dt", models.DateTimeField(blank=True, null=True)),
+                ("last_started_dt", models.DateTimeField(blank=True, null=True)),
+                ("last_finished_dt", models.DateTimeField(blank=True, null=True)),
+                ("last_failed_dt", models.DateTimeField(blank=True, null=True)),
+                ("last_failed_reason", models.TextField(blank=True, null=True)),
+                ("last_consecutive_failures", models.IntegerField(default=0)),
+                ("permanently_failed", models.BooleanField(default=False)),
+                ("last_rescheduled_dt", models.DateTimeField(blank=True, null=True)),
+                ("last_rescheduled_reason", models.TextField(blank=True, null=True)),
+                ("last_consecutive_reschedules", models.IntegerField(default=0)),
+            ],
+            options={
+                "abstract": False,
+            },
+        ),
+        pgtrigger.migrations.RemoveTrigger(
+            model_name="flaw",
+            name="insert_insert",
+        ),
+        pgtrigger.migrations.RemoveTrigger(
+            model_name="flaw",
+            name="update_update",
+        ),
+        pgtrigger.migrations.RemoveTrigger(
+            model_name="flaw",
+            name="delete_delete",
+        ),
+        migrations.AddField(
+            model_name="flaw",
+            name="task_sync_manager",
+            field=models.ForeignKey(
+                blank=True,
+                null=True,
+                on_delete=django.db.models.deletion.CASCADE,
+                to="osidb.jiratasksyncmanager",
+            ),
+        ),
+        migrations.AddField(
+            model_name="flawaudit",
+            name="task_sync_manager",
+            field=models.ForeignKey(
+                blank=True,
+                db_constraint=False,
+                null=True,
+                on_delete=django.db.models.deletion.DO_NOTHING,
+                related_name="+",
+                related_query_name="+",
+                to="osidb.jiratasksyncmanager",
+            ),
+        ),
+        pgtrigger.migrations.AddTrigger(
+            model_name="flaw",
+            trigger=pgtrigger.compiler.Trigger(
+                name="insert_insert",
+                sql=pgtrigger.compiler.UpsertTriggerSql(
+                    func='INSERT INTO "osidb_flawaudit" ("acl_read", "acl_write", "bzsync_manager_id", "comment_zero", "components", "created_dt", "cve_description", "cve_id", "cwe_id", "download_manager_id", "group_key", "impact", "last_validated_dt", "major_incident_start_dt", "major_incident_state", "mitigation", "nist_cvss_validation", "owner", "pgh_context_id", "pgh_created_at", "pgh_label", "pgh_obj_id", "reported_dt", "requires_cve_description", "source", "statement", "task_download_manager_id", "task_key", "task_sync_manager_id", "task_updated_dt", "team_id", "title", "unembargo_dt", "uuid", "workflow_name", "workflow_state") VALUES (NEW."acl_read", NEW."acl_write", NEW."bzsync_manager_id", NEW."comment_zero", NEW."components", NEW."created_dt", NEW."cve_description", NEW."cve_id", NEW."cwe_id", NEW."download_manager_id", NEW."group_key", NEW."impact", NEW."last_validated_dt", NEW."major_incident_start_dt", NEW."major_incident_state", NEW."mitigation", NEW."nist_cvss_validation", NEW."owner", _pgh_attach_context(), NOW(), \'insert\', NEW."uuid", NEW."reported_dt", NEW."requires_cve_description", NEW."source", NEW."statement", NEW."task_download_manager_id", NEW."task_key", NEW."task_sync_manager_id", NEW."task_updated_dt", NEW."team_id", NEW."title", NEW."unembargo_dt", NEW."uuid", NEW."workflow_name", NEW."workflow_state"); RETURN NULL;',
+                    hash="847485908d4ffb2aabc2b753d9530b854f5c21a5",
+                    operation="INSERT",
+                    pgid="pgtrigger_insert_insert_4e668",
+                    table="osidb_flaw",
+                    when="AFTER",
+                ),
+            ),
+        ),
+        pgtrigger.migrations.AddTrigger(
+            model_name="flaw",
+            trigger=pgtrigger.compiler.Trigger(
+                name="update_update",
+                sql=pgtrigger.compiler.UpsertTriggerSql(
+                    condition='WHEN (OLD."acl_read" IS DISTINCT FROM (NEW."acl_read") OR OLD."acl_write" IS DISTINCT FROM (NEW."acl_write") OR OLD."bzsync_manager_id" IS DISTINCT FROM (NEW."bzsync_manager_id") OR OLD."comment_zero" IS DISTINCT FROM (NEW."comment_zero") OR OLD."components" IS DISTINCT FROM (NEW."components") OR OLD."created_dt" IS DISTINCT FROM (NEW."created_dt") OR OLD."cve_description" IS DISTINCT FROM (NEW."cve_description") OR OLD."cve_id" IS DISTINCT FROM (NEW."cve_id") OR OLD."cwe_id" IS DISTINCT FROM (NEW."cwe_id") OR OLD."download_manager_id" IS DISTINCT FROM (NEW."download_manager_id") OR OLD."group_key" IS DISTINCT FROM (NEW."group_key") OR OLD."impact" IS DISTINCT FROM (NEW."impact") OR OLD."last_validated_dt" IS DISTINCT FROM (NEW."last_validated_dt") OR OLD."major_incident_start_dt" IS DISTINCT FROM (NEW."major_incident_start_dt") OR OLD."major_incident_state" IS DISTINCT FROM (NEW."major_incident_state") OR OLD."mitigation" IS DISTINCT FROM (NEW."mitigation") OR OLD."nist_cvss_validation" IS DISTINCT FROM (NEW."nist_cvss_validation") OR OLD."owner" IS DISTINCT FROM (NEW."owner") OR OLD."reported_dt" IS DISTINCT FROM (NEW."reported_dt") OR OLD."requires_cve_description" IS DISTINCT FROM (NEW."requires_cve_description") OR OLD."source" IS DISTINCT FROM (NEW."source") OR OLD."statement" IS DISTINCT FROM (NEW."statement") OR OLD."task_download_manager_id" IS DISTINCT FROM (NEW."task_download_manager_id") OR OLD."task_key" IS DISTINCT FROM (NEW."task_key") OR OLD."task_sync_manager_id" IS DISTINCT FROM (NEW."task_sync_manager_id") OR OLD."task_updated_dt" IS DISTINCT FROM (NEW."task_updated_dt") OR OLD."team_id" IS DISTINCT FROM (NEW."team_id") OR OLD."title" IS DISTINCT FROM (NEW."title") OR OLD."unembargo_dt" IS DISTINCT FROM (NEW."unembargo_dt") OR OLD."uuid" IS DISTINCT FROM (NEW."uuid") OR OLD."workflow_name" IS DISTINCT FROM (NEW."workflow_name") OR OLD."workflow_state" IS DISTINCT FROM (NEW."workflow_state"))',
+                    func='INSERT INTO "osidb_flawaudit" ("acl_read", "acl_write", "bzsync_manager_id", "comment_zero", "components", "created_dt", "cve_description", "cve_id", "cwe_id", "download_manager_id", "group_key", "impact", "last_validated_dt", "major_incident_start_dt", "major_incident_state", "mitigation", "nist_cvss_validation", "owner", "pgh_context_id", "pgh_created_at", "pgh_label", "pgh_obj_id", "reported_dt", "requires_cve_description", "source", "statement", "task_download_manager_id", "task_key", "task_sync_manager_id", "task_updated_dt", "team_id", "title", "unembargo_dt", "uuid", "workflow_name", "workflow_state") VALUES (NEW."acl_read", NEW."acl_write", NEW."bzsync_manager_id", NEW."comment_zero", NEW."components", NEW."created_dt", NEW."cve_description", NEW."cve_id", NEW."cwe_id", NEW."download_manager_id", NEW."group_key", NEW."impact", NEW."last_validated_dt", NEW."major_incident_start_dt", NEW."major_incident_state", NEW."mitigation", NEW."nist_cvss_validation", NEW."owner", _pgh_attach_context(), NOW(), \'update\', NEW."uuid", NEW."reported_dt", NEW."requires_cve_description", NEW."source", NEW."statement", NEW."task_download_manager_id", NEW."task_key", NEW."task_sync_manager_id", NEW."task_updated_dt", NEW."team_id", NEW."title", NEW."unembargo_dt", NEW."uuid", NEW."workflow_name", NEW."workflow_state"); RETURN NULL;',
+                    hash="1c9465368a05ed2227d818e1dd9296ebbf054583",
+                    operation="UPDATE",
+                    pgid="pgtrigger_update_update_96595",
+                    table="osidb_flaw",
+                    when="AFTER",
+                ),
+            ),
+        ),
+        pgtrigger.migrations.AddTrigger(
+            model_name="flaw",
+            trigger=pgtrigger.compiler.Trigger(
+                name="delete_delete",
+                sql=pgtrigger.compiler.UpsertTriggerSql(
+                    func='INSERT INTO "osidb_flawaudit" ("acl_read", "acl_write", "bzsync_manager_id", "comment_zero", "components", "created_dt", "cve_description", "cve_id", "cwe_id", "download_manager_id", "group_key", "impact", "last_validated_dt", "major_incident_start_dt", "major_incident_state", "mitigation", "nist_cvss_validation", "owner", "pgh_context_id", "pgh_created_at", "pgh_label", "pgh_obj_id", "reported_dt", "requires_cve_description", "source", "statement", "task_download_manager_id", "task_key", "task_sync_manager_id", "task_updated_dt", "team_id", "title", "unembargo_dt", "uuid", "workflow_name", "workflow_state") VALUES (OLD."acl_read", OLD."acl_write", OLD."bzsync_manager_id", OLD."comment_zero", OLD."components", OLD."created_dt", OLD."cve_description", OLD."cve_id", OLD."cwe_id", OLD."download_manager_id", OLD."group_key", OLD."impact", OLD."last_validated_dt", OLD."major_incident_start_dt", OLD."major_incident_state", OLD."mitigation", OLD."nist_cvss_validation", OLD."owner", _pgh_attach_context(), NOW(), \'delete\', OLD."uuid", OLD."reported_dt", OLD."requires_cve_description", OLD."source", OLD."statement", OLD."task_download_manager_id", OLD."task_key", OLD."task_sync_manager_id", OLD."task_updated_dt", OLD."team_id", OLD."title", OLD."unembargo_dt", OLD."uuid", OLD."workflow_name", OLD."workflow_state"); RETURN NULL;',
+                    hash="ddf74d3210cd3db959e98af720e1965332a0c84b",
+                    operation="DELETE",
+                    pgid="pgtrigger_delete_delete_f2e13",
+                    table="osidb_flaw",
+                    when="AFTER",
+                ),
+            ),
+        ),
+    ]

osidb/models/flaw/flaw.py

@@ -16,7 +16,7 @@
 from apps.bbsync.constants import SYNC_FLAWS_TO_BZ, SYNC_FLAWS_TO_BZ_ASYNCHRONOUSLY
 from apps.bbsync.mixins import BugzillaSyncMixin
 from apps.taskman.constants import (
-    JIRA_TASKMAN_AUTO_SYNC_FLAW,
+    JIRA_TASKMAN_ASYNCHRONOUS_SYNC,
     SYNC_REQUIRED_FIELDS,
     TRANSITION_REQUIRED_FIELDS,
 )
@@ -39,6 +39,7 @@
     BZSyncManager,
     FlawDownloadManager,
     JiraTaskDownloadManager,
+    JiraTaskSyncManager,
 )
 from osidb.validators import no_future_date, validate_cve_id, validate_cwe_id
 
@@ -1037,22 +1038,38 @@ def tasksync(
         based on the task existence it is either created or updated and/or transitioned
         old pre-OSIDB flaws without tasks are ignored unless force_creation is set
         """
-        if not JIRA_TASKMAN_AUTO_SYNC_FLAW or not jira_token:
-            return
+        update_task = False
+        transition_task = False
 
         if not self.task_key:
             # old pre-OSIDB flaws without tasks are ignored by default
             if force_creation or not self.meta_attr.get("bz_id"):
-                self._create_or_update_task(jira_token)
+                update_task = True
 
         elif diff is not None:
             if any(field in diff.keys() for field in SYNC_REQUIRED_FIELDS):
-                self._create_or_update_task(jira_token)
+                update_task = True
 
             if any(field in diff.keys() for field in TRANSITION_REQUIRED_FIELDS):
+                transition_task = True
+
+        if not update_task and not transition_task:
+            return
+
+        # switch of sync/async processing
+        if JIRA_TASKMAN_ASYNCHRONOUS_SYNC:
+            JiraTaskSyncManager.check_for_reschedules()
+            # TODO parameters will vanish if the sync manager task run fails and is rescheduled
+            JiraTaskSyncManager.schedule(str(self.uuid), update_task, transition_task)
+        else:
+
+            if update_task:
+                self._create_or_update_task(jira_token)
+
+            if transition_task:
                 self._transition_task(jira_token)
 
-    def _create_or_update_task(self, jira_token):
+    def _create_or_update_task(self, jira_token=None):
         """
         create or update the Jira task of this flaw based on its existence
         """
@@ -1079,7 +1096,7 @@ def _create_or_update_task(self, jira_token):
                 task_updated_dt=self.task_updated_dt
             )
 
-    def _transition_task(self, jira_token):
+    def _transition_task(self, jira_token=None):
         """
         transition the Jira task of this flaw
         """
@@ -1106,6 +1123,9 @@ def _transition_task(self, jira_token):
     task_download_manager = models.ForeignKey(
         JiraTaskDownloadManager, null=True, blank=True, on_delete=models.CASCADE
     )
+    task_sync_manager = models.ForeignKey(
+        JiraTaskSyncManager, null=True, blank=True, on_delete=models.CASCADE
+    )
     bzsync_manager = models.ForeignKey(
         BZSyncManager, null=True, blank=True, on_delete=models.CASCADE
     )

osidb/sync_manager.py

@@ -679,6 +679,59 @@ def __str__(self):
         return result
 
 
+class JiraTaskSyncManager(SyncManager):
+    """
+    Sync manager class for OSIDB => Jira Task synchronization.
+    """
+
+    @staticmethod
+    @app.task(name="sync_manager.jira_task_sync", bind=True)
+    def sync_task(self, flaw_id, update_task=False, transition_task=False):
+        """
+        perform the sync of the task of the given flaw to Jira
+
+        the task may not be exising yet when performing the first
+        sync therefore we use the flaw UUID as the identifier
+
+        the task update and task transition use two different Jira
+        endpoints so the call parameters specify what to perform
+        """
+        from osidb.models import Flaw
+
+        JiraTaskSyncManager.started(flaw_id, self)
+
+        set_user_acls(settings.ALL_GROUPS)
+
+        try:
+            flaw = Flaw.objects.get(uuid=flaw_id)
+
+            if update_task:
+                flaw._create_or_update_task()
+
+            if transition_task:
+                flaw._transition_task()
+
+        except Exception as e:
+            JiraTaskSyncManager.failed(flaw_id, e)
+        else:
+            JiraTaskSyncManager.finished(flaw_id)
+
+    def update_synced_links(self):
+        from osidb.models import Flaw
+
+        Flaw.objects.filter(uuid=self.sync_id).update(bzsync_manager=self)
+
+    def __str__(self):
+        from osidb.models import Flaw
+
+        result = super().__str__()
+        flaws = Flaw.objects.filter(uuid=self.sync_id)
+        cves = [f.cve_id or f.uuid for f in flaws]
+        result += f"Flaws: {cves}\n"
+
+        return result
+
+
 class JiraTrackerDownloadManager(SyncManager):
     """
     Sync manager class for Jira => OSIDB Tracker synchronization.