From 1ed8cac8711b3c93bb031fc2c532cf68526e7ce0 Mon Sep 17 00:00:00 2001 From: osoukup Date: Mon, 16 Dec 2024 15:52:43 +0100 Subject: [PATCH 1/3] remove duplicite task sync enablement condition as this is already checked in the JiraTaskSyncMixin where is the right place and also in the serializer where it should eventually go away --- osidb/models/flaw/flaw.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/osidb/models/flaw/flaw.py b/osidb/models/flaw/flaw.py index 9ad106d97..28ca7e2d3 100644 --- a/osidb/models/flaw/flaw.py +++ b/osidb/models/flaw/flaw.py @@ -16,7 +16,7 @@ from apps.bbsync.constants import SYNC_FLAWS_TO_BZ, SYNC_FLAWS_TO_BZ_ASYNCHRONOUSLY from apps.bbsync.mixins import BugzillaSyncMixin from apps.taskman.constants import ( - JIRA_TASKMAN_AUTO_SYNC_FLAW, + JIRA_TASKMAN_ASYNCHRONOUS_SYNC, SYNC_REQUIRED_FIELDS, TRANSITION_REQUIRED_FIELDS, ) @@ -1037,9 +1037,6 @@ def tasksync( based on the task existence it is either created or updated and/or transitioned old pre-OSIDB flaws without tasks are ignored unless force_creation is set """ - if not JIRA_TASKMAN_AUTO_SYNC_FLAW or not jira_token: - return - if not self.task_key: # old pre-OSIDB flaws without tasks are ignored by default if force_creation or not self.meta_attr.get("bz_id"): From d57e7ecb939272e5d148d4a8edb35b7324b18435 Mon Sep 17 00:00:00 2001 From: osoukup Date: Mon, 16 Dec 2024 16:51:11 +0100 Subject: [PATCH 2/3] leave the user token requirement check to the upper layers so it is possible to easily switch between user and service accounts and also because the user token requirement checks are already in place in the serializer and taskman mixin levels --- apps/taskman/service.py | 14 ++++---------- 1 file changed, 4 insertions(+), 10 deletions(-) diff --git a/apps/taskman/service.py b/apps/taskman/service.py index 94b3d3015..cfb766888 100644 --- a/apps/taskman/service.py +++ b/apps/taskman/service.py @@ -21,11 +21,7 @@ JIRA_TASKMAN_PROJECT_KEY, JIRA_TASKMAN_URL, ) -from .exceptions import ( - JiraTaskErrorException, - MissingJiraTokenException, - TaskWritePermissionsException, -) +from .exceptions import JiraTaskErrorException, TaskWritePermissionsException logger = logging.getLogger(__name__) @@ -80,14 +76,12 @@ def __init__(self, token) -> None: Keyword arguments: token -- user token used in every request to Jira + the service one is used if not provided """ super().__init__() - if not token: - raise MissingJiraTokenException( - "User's Jira Token is required to perform this action." - ) self._jira_server = JIRA_TASKMAN_URL - self._jira_token = token + if token: + self._jira_token = token def _check_token(self) -> None: """ From a636ced083ee274a02587c10f25e13d07bbfa341 Mon Sep 17 00:00:00 2001 From: osoukup Date: Tue, 17 Dec 2024 16:56:32 +0100 Subject: [PATCH 3/3] tmp --- apps/taskman/constants.py | 3 + conftest.py | 2 - docker-compose.yml | 1 + osidb/migrations/0178_jiratasksyncmanager.py | 122 +++++++++++++++++++ osidb/models/flaw/flaw.py | 31 ++++- osidb/sync_manager.py | 53 ++++++++ 6 files changed, 206 insertions(+), 6 deletions(-) create mode 100644 osidb/migrations/0178_jiratasksyncmanager.py diff --git a/apps/taskman/constants.py b/apps/taskman/constants.py index 39bb85256..24ecc63ac 100644 --- a/apps/taskman/constants.py +++ b/apps/taskman/constants.py @@ -11,6 +11,9 @@ JIRA_TASKMAN_AUTO_SYNC_FLAW = get_env( "JIRA_TASKMAN_AUTO_SYNC_FLAW", default="0", is_bool=True ) +JIRA_TASKMAN_ASYNCHRONOUS_SYNC = get_env( + "JIRA_TASKMAN_ASYNCHRONOUS_SYNC", default="False", is_bool=True +) SYNC_REQUIRED_FIELDS = [ "cve_id", diff --git a/conftest.py b/conftest.py index b22e86d51..e82ea9138 100644 --- a/conftest.py +++ b/conftest.py @@ -262,10 +262,8 @@ def enable_jira_task_sync(monkeypatch) -> None: """ import apps.taskman.mixins as mixins import apps.taskman.service as service - import osidb.models.flaw.flaw as flaw_module import osidb.serializer as serializer - monkeypatch.setattr(flaw_module, "JIRA_TASKMAN_AUTO_SYNC_FLAW", True) monkeypatch.setattr(mixins, "JIRA_TASKMAN_AUTO_SYNC_FLAW", True) monkeypatch.setattr(serializer, "JIRA_TASKMAN_AUTO_SYNC_FLAW", True) monkeypatch.setattr(service, "JIRA_STORY_ISSUE_TYPE_ID", "17") diff --git a/docker-compose.yml b/docker-compose.yml index c83119fe4..f646bcd31 100755 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -81,6 +81,7 @@ services: JIRA_AUTH_TOKEN: ${JIRA_AUTH_TOKEN:?Variable JIRA_AUTH_TOKEN must be set.} JIRA_MAX_CONNECTION_AGE: ${JIRA_MAX_CONNECTION_AGE} JIRA_STORY_ISSUE_TYPE_ID: ${JIRA_STORY_ISSUE_TYPE_ID} + JIRA_TASKMAN_ASYNCHRONOUS_SYNC: ${JIRA_TASKMAN_ASYNCHRONOUS_SYNC} JIRA_TASKMAN_AUTO_SYNC_FLAW: ${JIRA_TASKMAN_AUTO_SYNC_FLAW} JIRA_TASKMAN_PROJECT_ID: ${JIRA_TASKMAN_PROJECT_ID} JIRA_TASKMAN_PROJECT_KEY: ${JIRA_TASKMAN_PROJECT_KEY} diff --git a/osidb/migrations/0178_jiratasksyncmanager.py b/osidb/migrations/0178_jiratasksyncmanager.py new file mode 100644 index 000000000..6bf4144cf --- /dev/null +++ b/osidb/migrations/0178_jiratasksyncmanager.py @@ -0,0 +1,122 @@ +# Generated by Django 4.2.16 on 2024-12-17 09:34 + +from django.db import migrations, models +import django.db.models.deletion +import pgtrigger.compiler +import pgtrigger.migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ("osidb", "0177_remove_affect_insert_insert_and_more"), + ] + + operations = [ + migrations.CreateModel( + name="JiraTaskSyncManager", + fields=[ + ( + "id", + models.BigAutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ("sync_id", models.CharField(max_length=100, unique=True)), + ("last_scheduled_dt", models.DateTimeField(blank=True, null=True)), + ("last_started_dt", models.DateTimeField(blank=True, null=True)), + ("last_finished_dt", models.DateTimeField(blank=True, null=True)), + ("last_failed_dt", models.DateTimeField(blank=True, null=True)), + ("last_failed_reason", models.TextField(blank=True, null=True)), + ("last_consecutive_failures", models.IntegerField(default=0)), + ("permanently_failed", models.BooleanField(default=False)), + ("last_rescheduled_dt", models.DateTimeField(blank=True, null=True)), + ("last_rescheduled_reason", models.TextField(blank=True, null=True)), + ("last_consecutive_reschedules", models.IntegerField(default=0)), + ], + options={ + "abstract": False, + }, + ), + pgtrigger.migrations.RemoveTrigger( + model_name="flaw", + name="insert_insert", + ), + pgtrigger.migrations.RemoveTrigger( + model_name="flaw", + name="update_update", + ), + pgtrigger.migrations.RemoveTrigger( + model_name="flaw", + name="delete_delete", + ), + migrations.AddField( + model_name="flaw", + name="task_sync_manager", + field=models.ForeignKey( + blank=True, + null=True, + on_delete=django.db.models.deletion.CASCADE, + to="osidb.jiratasksyncmanager", + ), + ), + migrations.AddField( + model_name="flawaudit", + name="task_sync_manager", + field=models.ForeignKey( + blank=True, + db_constraint=False, + null=True, + on_delete=django.db.models.deletion.DO_NOTHING, + related_name="+", + related_query_name="+", + to="osidb.jiratasksyncmanager", + ), + ), + pgtrigger.migrations.AddTrigger( + model_name="flaw", + trigger=pgtrigger.compiler.Trigger( + name="insert_insert", + sql=pgtrigger.compiler.UpsertTriggerSql( + func='INSERT INTO "osidb_flawaudit" ("acl_read", "acl_write", "bzsync_manager_id", "comment_zero", "components", "created_dt", "cve_description", "cve_id", "cwe_id", "download_manager_id", "group_key", "impact", "last_validated_dt", "major_incident_start_dt", "major_incident_state", "mitigation", "nist_cvss_validation", "owner", "pgh_context_id", "pgh_created_at", "pgh_label", "pgh_obj_id", "reported_dt", "requires_cve_description", "source", "statement", "task_download_manager_id", "task_key", "task_sync_manager_id", "task_updated_dt", "team_id", "title", "unembargo_dt", "uuid", "workflow_name", "workflow_state") VALUES (NEW."acl_read", NEW."acl_write", NEW."bzsync_manager_id", NEW."comment_zero", NEW."components", NEW."created_dt", NEW."cve_description", NEW."cve_id", NEW."cwe_id", NEW."download_manager_id", NEW."group_key", NEW."impact", NEW."last_validated_dt", NEW."major_incident_start_dt", NEW."major_incident_state", NEW."mitigation", NEW."nist_cvss_validation", NEW."owner", _pgh_attach_context(), NOW(), \'insert\', NEW."uuid", NEW."reported_dt", NEW."requires_cve_description", NEW."source", NEW."statement", NEW."task_download_manager_id", NEW."task_key", NEW."task_sync_manager_id", NEW."task_updated_dt", NEW."team_id", NEW."title", NEW."unembargo_dt", NEW."uuid", NEW."workflow_name", NEW."workflow_state"); RETURN NULL;', + hash="847485908d4ffb2aabc2b753d9530b854f5c21a5", + operation="INSERT", + pgid="pgtrigger_insert_insert_4e668", + table="osidb_flaw", + when="AFTER", + ), + ), + ), + pgtrigger.migrations.AddTrigger( + model_name="flaw", + trigger=pgtrigger.compiler.Trigger( + name="update_update", + sql=pgtrigger.compiler.UpsertTriggerSql( + condition='WHEN (OLD."acl_read" IS DISTINCT FROM (NEW."acl_read") OR OLD."acl_write" IS DISTINCT FROM (NEW."acl_write") OR OLD."bzsync_manager_id" IS DISTINCT FROM (NEW."bzsync_manager_id") OR OLD."comment_zero" IS DISTINCT FROM (NEW."comment_zero") OR OLD."components" IS DISTINCT FROM (NEW."components") OR OLD."created_dt" IS DISTINCT FROM (NEW."created_dt") OR OLD."cve_description" IS DISTINCT FROM (NEW."cve_description") OR OLD."cve_id" IS DISTINCT FROM (NEW."cve_id") OR OLD."cwe_id" IS DISTINCT FROM (NEW."cwe_id") OR OLD."download_manager_id" IS DISTINCT FROM (NEW."download_manager_id") OR OLD."group_key" IS DISTINCT FROM (NEW."group_key") OR OLD."impact" IS DISTINCT FROM (NEW."impact") OR OLD."last_validated_dt" IS DISTINCT FROM (NEW."last_validated_dt") OR OLD."major_incident_start_dt" IS DISTINCT FROM (NEW."major_incident_start_dt") OR OLD."major_incident_state" IS DISTINCT FROM (NEW."major_incident_state") OR OLD."mitigation" IS DISTINCT FROM (NEW."mitigation") OR OLD."nist_cvss_validation" IS DISTINCT FROM (NEW."nist_cvss_validation") OR OLD."owner" IS DISTINCT FROM (NEW."owner") OR OLD."reported_dt" IS DISTINCT FROM (NEW."reported_dt") OR OLD."requires_cve_description" IS DISTINCT FROM (NEW."requires_cve_description") OR OLD."source" IS DISTINCT FROM (NEW."source") OR OLD."statement" IS DISTINCT FROM (NEW."statement") OR OLD."task_download_manager_id" IS DISTINCT FROM (NEW."task_download_manager_id") OR OLD."task_key" IS DISTINCT FROM (NEW."task_key") OR OLD."task_sync_manager_id" IS DISTINCT FROM (NEW."task_sync_manager_id") OR OLD."task_updated_dt" IS DISTINCT FROM (NEW."task_updated_dt") OR OLD."team_id" IS DISTINCT FROM (NEW."team_id") OR OLD."title" IS DISTINCT FROM (NEW."title") OR OLD."unembargo_dt" IS DISTINCT FROM (NEW."unembargo_dt") OR OLD."uuid" IS DISTINCT FROM (NEW."uuid") OR OLD."workflow_name" IS DISTINCT FROM (NEW."workflow_name") OR OLD."workflow_state" IS DISTINCT FROM (NEW."workflow_state"))', + func='INSERT INTO "osidb_flawaudit" ("acl_read", "acl_write", "bzsync_manager_id", "comment_zero", "components", "created_dt", "cve_description", "cve_id", "cwe_id", "download_manager_id", "group_key", "impact", "last_validated_dt", "major_incident_start_dt", "major_incident_state", "mitigation", "nist_cvss_validation", "owner", "pgh_context_id", "pgh_created_at", "pgh_label", "pgh_obj_id", "reported_dt", "requires_cve_description", "source", "statement", "task_download_manager_id", "task_key", "task_sync_manager_id", "task_updated_dt", "team_id", "title", "unembargo_dt", "uuid", "workflow_name", "workflow_state") VALUES (NEW."acl_read", NEW."acl_write", NEW."bzsync_manager_id", NEW."comment_zero", NEW."components", NEW."created_dt", NEW."cve_description", NEW."cve_id", NEW."cwe_id", NEW."download_manager_id", NEW."group_key", NEW."impact", NEW."last_validated_dt", NEW."major_incident_start_dt", NEW."major_incident_state", NEW."mitigation", NEW."nist_cvss_validation", NEW."owner", _pgh_attach_context(), NOW(), \'update\', NEW."uuid", NEW."reported_dt", NEW."requires_cve_description", NEW."source", NEW."statement", NEW."task_download_manager_id", NEW."task_key", NEW."task_sync_manager_id", NEW."task_updated_dt", NEW."team_id", NEW."title", NEW."unembargo_dt", NEW."uuid", NEW."workflow_name", NEW."workflow_state"); RETURN NULL;', + hash="1c9465368a05ed2227d818e1dd9296ebbf054583", + operation="UPDATE", + pgid="pgtrigger_update_update_96595", + table="osidb_flaw", + when="AFTER", + ), + ), + ), + pgtrigger.migrations.AddTrigger( + model_name="flaw", + trigger=pgtrigger.compiler.Trigger( + name="delete_delete", + sql=pgtrigger.compiler.UpsertTriggerSql( + func='INSERT INTO "osidb_flawaudit" ("acl_read", "acl_write", "bzsync_manager_id", "comment_zero", "components", "created_dt", "cve_description", "cve_id", "cwe_id", "download_manager_id", "group_key", "impact", "last_validated_dt", "major_incident_start_dt", "major_incident_state", "mitigation", "nist_cvss_validation", "owner", "pgh_context_id", "pgh_created_at", "pgh_label", "pgh_obj_id", "reported_dt", "requires_cve_description", "source", "statement", "task_download_manager_id", "task_key", "task_sync_manager_id", "task_updated_dt", "team_id", "title", "unembargo_dt", "uuid", "workflow_name", "workflow_state") VALUES (OLD."acl_read", OLD."acl_write", OLD."bzsync_manager_id", OLD."comment_zero", OLD."components", OLD."created_dt", OLD."cve_description", OLD."cve_id", OLD."cwe_id", OLD."download_manager_id", OLD."group_key", OLD."impact", OLD."last_validated_dt", OLD."major_incident_start_dt", OLD."major_incident_state", OLD."mitigation", OLD."nist_cvss_validation", OLD."owner", _pgh_attach_context(), NOW(), \'delete\', OLD."uuid", OLD."reported_dt", OLD."requires_cve_description", OLD."source", OLD."statement", OLD."task_download_manager_id", OLD."task_key", OLD."task_sync_manager_id", OLD."task_updated_dt", OLD."team_id", OLD."title", OLD."unembargo_dt", OLD."uuid", OLD."workflow_name", OLD."workflow_state"); RETURN NULL;', + hash="ddf74d3210cd3db959e98af720e1965332a0c84b", + operation="DELETE", + pgid="pgtrigger_delete_delete_f2e13", + table="osidb_flaw", + when="AFTER", + ), + ), + ), + ] diff --git a/osidb/models/flaw/flaw.py b/osidb/models/flaw/flaw.py index 28ca7e2d3..b05aab77e 100644 --- a/osidb/models/flaw/flaw.py +++ b/osidb/models/flaw/flaw.py @@ -39,6 +39,7 @@ BZSyncManager, FlawDownloadManager, JiraTaskDownloadManager, + JiraTaskSyncManager, ) from osidb.validators import no_future_date, validate_cve_id, validate_cwe_id @@ -1037,19 +1038,38 @@ def tasksync( based on the task existence it is either created or updated and/or transitioned old pre-OSIDB flaws without tasks are ignored unless force_creation is set """ + update_task = False + transition_task = False + if not self.task_key: # old pre-OSIDB flaws without tasks are ignored by default if force_creation or not self.meta_attr.get("bz_id"): - self._create_or_update_task(jira_token) + update_task = True elif diff is not None: if any(field in diff.keys() for field in SYNC_REQUIRED_FIELDS): - self._create_or_update_task(jira_token) + update_task = True if any(field in diff.keys() for field in TRANSITION_REQUIRED_FIELDS): + transition_task = True + + if not update_task and not transition_task: + return + + # switch of sync/async processing + if JIRA_TASKMAN_ASYNCHRONOUS_SYNC: + JiraTaskSyncManager.check_for_reschedules() + # TODO parameters will vanish if the sync manager task run fails and is rescheduled + JiraTaskSyncManager.schedule(str(self.uuid), update_task, transition_task) + else: + + if update_task: + self._create_or_update_task(jira_token) + + if transition_task: self._transition_task(jira_token) - def _create_or_update_task(self, jira_token): + def _create_or_update_task(self, jira_token=None): """ create or update the Jira task of this flaw based on its existence """ @@ -1076,7 +1096,7 @@ def _create_or_update_task(self, jira_token): task_updated_dt=self.task_updated_dt ) - def _transition_task(self, jira_token): + def _transition_task(self, jira_token=None): """ transition the Jira task of this flaw """ @@ -1103,6 +1123,9 @@ def _transition_task(self, jira_token): task_download_manager = models.ForeignKey( JiraTaskDownloadManager, null=True, blank=True, on_delete=models.CASCADE ) + task_sync_manager = models.ForeignKey( + JiraTaskSyncManager, null=True, blank=True, on_delete=models.CASCADE + ) bzsync_manager = models.ForeignKey( BZSyncManager, null=True, blank=True, on_delete=models.CASCADE ) diff --git a/osidb/sync_manager.py b/osidb/sync_manager.py index baa150b00..c1130759d 100644 --- a/osidb/sync_manager.py +++ b/osidb/sync_manager.py @@ -679,6 +679,59 @@ def __str__(self): return result +class JiraTaskSyncManager(SyncManager): + """ + Sync manager class for OSIDB => Jira Task synchronization. + """ + + @staticmethod + @app.task(name="sync_manager.jira_task_sync", bind=True) + def sync_task(self, flaw_id, update_task=False, transition_task=False): + """ + perform the sync of the task of the given flaw to Jira + + the task may not be exising yet when performing the first + sync therefore we use the flaw UUID as the identifier + + the task update and task transition use two different Jira + endpoints so the call parameters specify what to perform + """ + from osidb.models import Flaw + + JiraTaskSyncManager.started(flaw_id, self) + + set_user_acls(settings.ALL_GROUPS) + + try: + flaw = Flaw.objects.get(uuid=flaw_id) + + if update_task: + flaw._create_or_update_task() + + if transition_task: + flaw._transition_task() + + except Exception as e: + JiraTaskSyncManager.failed(flaw_id, e) + else: + JiraTaskSyncManager.finished(flaw_id) + + def update_synced_links(self): + from osidb.models import Flaw + + Flaw.objects.filter(uuid=self.sync_id).update(bzsync_manager=self) + + def __str__(self): + from osidb.models import Flaw + + result = super().__str__() + flaws = Flaw.objects.filter(uuid=self.sync_id) + cves = [f.cve_id or f.uuid for f in flaws] + result += f"Flaws: {cves}\n" + + return result + + class JiraTrackerDownloadManager(SyncManager): """ Sync manager class for Jira => OSIDB Tracker synchronization.