Implement the core functionality

README.md

@@ -10,26 +10,30 @@ Product Pages (PP).
 ReTaSC is meant to run as a batch job regularly to monitor schedules in PP and
 create or update Jira issues according to custom rules and Jira templates.
 
-ReTaSC manages the Jira issues it creates until resolved. This means that if a
-PP schedule or a rule changes, the related unresolved Jira issues are also
-updated or even closed.
+ReTaSC creates and manages Jira issues until resolved. This means that if a PP
+schedule or a rule changes, the related unresolved Jira issues are also updated
+and even closed.
 
-## Tasks and Rules
+## Tasks, Rules and Prerequisites
 
 Tasks are objects managed by ReTaSC. Each Task is related to a specific Rule
 and a Product Release (an identifier in PP).
 
-Rules describe how to manage related tasks using:
+Rules describe prerequisites to manage a Product Release.
 
-- Prerequisites - PP schedule item name with number of days before/after the
-  date, and list of other dependent Rules
+Prerequisites are requirements that enhance template parameters and can block
+Task completion if some conditions are not met. Here are some pre-defined
+prerequisites:
+
+- PP schedule item name, for example:
+  `schedule_task: "GA for rhel {{ major }}.{{ minor }}"`
+- a target date, for example: `target_date: "start_date - 3|weeks"`
+- a condition, for example: `condition: "major >= 10"`
+- reference to other Rule that must be completed
 - Jira issue templates
-- Definition of Done (DoD) - currently, the only supported DoD is: "related
-  Jira issues have been resolved"
 
 Task state can be one of:
 
-- Missing (PP schedule is not defined yet)
 - Pending (some prerequisites and not satisfied)
 - In-progress (prerequisites are satisfied and DoD is not)
 - Completed (prerequisites and DoD is satisfied)
@@ -38,6 +42,10 @@ Task state can be one of:
 
 Below is list of environment variables supported in the container image:
 
+- `RETASC_JIRA_URL` - Jira URL
+- `RETASC_JIRA_TOKEN` - Jira access token
+- `RETASC_RULES_PATH` - Path to rules
+- `RETASC_PP_URL` - Product Pages URL
 - `RETASC_LOGGING_CONFIG` - Path to JSON file with the logging configuration;
   see details in [Configuration dictionary
   schema](https://docs.python.org/3/library/logging.config.html#logging-config-dictschema)

examples/rules/rules.yaml

@@ -1,28 +1,29 @@
 - version: 1
   name: "Example Rule"
   prerequisites:
-    pp_schedule_item_name: "Release Date"
-    days_before_or_after: -7
-    dependent_rules:
-      - "Dependent Rule 1"
-      - "Dependent Rule 2"
-  jira_issues:
-    - template: "examples/jira/main.yaml"
+    - condition: "major >= 10"
+    - schedule_task: "GA for rhel {{ major }}.{{ minor }}"
+    - target_date: "start_date - 7|days"
+    - rule: "Dependent Rule 1"
+    - rule: "Dependent Rule 2"
+    - jira_issue_id: main
+      template: "examples/jira/main.yaml"
       subtasks:
-        - template: "examples/jira/add_beta_repos.yaml"
-        - template: "examples/jira/notify_team.yaml"
-    - template: "examples/jira/secondary.yaml"
+        - id: add_beta_repos
+          template: "examples/jira/add_beta_repos.yaml"
+        - id: notify_team
+          template: "examples/jira/notify_team.yaml"
+    - jira_issue_id: secondary
+      template: "examples/jira/secondary.yaml"
 
 - version: 1
   name: "Dependent Rule 1"
   prerequisites:
-    pp_schedule_item_name: "Release Date"
-    days_before_or_after: -21
-  jira_issues: []
+    - schedule_task: "TASK"
+    - target_date: "start_date - 3|weeks"
 
 - version: 1
   name: "Dependent Rule 2"
   prerequisites:
-    pp_schedule_item_name: "Release Date"
-    days_before_or_after: -14
-  jira_issues: []
+    - schedule_task: "TASK"
+    - target_date: "start_date - 2|weeks"

pyproject.toml

@@ -23,6 +23,7 @@ urllib3 = "^2.2.3"
 pydantic = "^2.9.2"
 pyyaml = "^6.0.1"
 atlassian-python-api = "^3.41.16"
+Jinja2 = "^3.1.4"
 
 [tool.poetry.group.dev.dependencies]
 requests-mock = "^1.12.1"

src/retasc/__main__.py

@@ -6,10 +6,11 @@
 
 from retasc import __doc__ as doc
 from retasc import __version__
+from retasc.models.generate_schema import generate_schema
+from retasc.models.parse_rules import RuleParsingError, parse_rules
 from retasc.retasc_logging import init_logging
+from retasc.run import run
 from retasc.tracing import init_tracing
-from retasc.validator.generate_schema import generate_schema
-from retasc.validator.parse_rules import RuleParsingError, parse_rules
 
 logger = logging.getLogger(__name__)
 
@@ -45,6 +46,14 @@ def parse_args():
         action="store_true",
     )
 
+    subparsers.add_parser(
+        "run", help="Process rules, data from Product Pages and apply changes to Jira"
+    )
+    subparsers.add_parser(
+        "dry-run",
+        help='Same as "run" but without creating, deleting or modifying any Jira issues',
+    )
+
     return parser.parse_args()
 
 
@@ -62,4 +71,7 @@ def main():
         print("Validation succeeded: The rule files are valid")
     elif args.command == "generate-schema":
         generate_schema(args.schema_file, output_json=args.json)
+    elif args.command in ("run", "dry-run"):
+        dry_run = args.command == "dry-run"
+        run(dry_run=dry_run)
     sys.exit(0)

src/retasc/jira.py

@@ -0,0 +1,11 @@
+# SPDX-License-Identifier: GPL-3.0-or-later
+JIRA_LABEL = "retasc-managed"
+JIRA_ISSUE_ID_LABEL_PREFIX = "retasc-id-"
+JIRA_MANAGED_FIELDS = [
+    "assignee",
+    "description",
+    "duedate",
+    "labels",
+    "resolution",
+    "summary",
+]

src/retasc/jira_client.py

@@ -1,22 +1,27 @@
 import logging
 
 from atlassian import Jira
+from opentelemetry import trace
+from requests import Session
 
 logger = logging.getLogger(__name__)
+tracer = trace.get_tracer(__name__)
 
 
 class JiraClient:
     """
     Jira Client Wrapper
     """
 
-    def __init__(self, api_url: str, token: str | None = None):
+    def __init__(self, api_url: str, *, token: str, session: Session):
         self.api_url = api_url
         self.jira = Jira(
             url=api_url,
             token=token,
+            session=session,
         )
 
+    @tracer.start_as_current_span("JiraClient.edit_issue")
     def edit_issue(
         self, issue_key: str, fields: dict, notify_users: bool = True
     ) -> None:
@@ -40,46 +45,51 @@ def edit_issue(
         logger.info("Updating Jira issue %r with fields: %r", issue_key, fields)
         self.jira.edit_issue(issue_key, fields, notify_users=notify_users)
 
-    def create_issue(
-        self,
-        project_key: str,
-        summary: str,
-        description: str,
-        issue_type: str,
-        fields: dict = {},
-    ) -> dict:
+    @tracer.start_as_current_span("JiraClient.create_issue")
+    def create_issue(self, fields: dict) -> dict:
         """
         Create a new Jira issue
         """
+        logger.info("Creating new Jira issue with fields: %r", fields)
 
-        issue_dict = {
-            "project": {"key": project_key},
-            "summary": summary,
-            "description": description,
-            "issuetype": {"name": issue_type},
-        }
+        data = self.jira.create_issue(fields)
+        if isinstance(data, dict):
+            return data
 
-        issue_dict.update(fields)
+        raise RuntimeError(f"Unexpected response: {data!r}")
 
-        logger.info("Creating new Jira issue with fields: %r", issue_dict)
-
-        issue = self.jira.create_issue(issue_dict)
-        return issue
-
-    def search_issues(self, jql: str) -> list:
+    @tracer.start_as_current_span("JiraClient.search_issues")
+    def search_issues(self, jql: str, fields: list[str] | None = None) -> list:
         """
         Search Issues by JQL
 
         :param jql: string: like "project = DEMO AND status NOT IN (Closed, Resolved) ORDER BY issuekey"
         """
 
-        issue_list = self.jira.jql_get_list_of_tickets(jql)
-        return issue_list
+        if fields:
+            return self.jira.jql_get_list_of_tickets(jql, fields=fields)
+        return self.jira.jql_get_list_of_tickets(jql)
 
+    @tracer.start_as_current_span("JiraClient.get_issues")
     def get_issue(self, issue_key: str) -> dict:
         """
         Get a Jira issue.
         """
 
-        issue = self.jira.issue(issue_key)
-        return issue
+        data = self.jira.issue(issue_key)
+        if isinstance(data, dict):
+            return data
+
+        raise RuntimeError(f"Unexpected response: {data}")
+
+
+class DryRunJiraClient(JiraClient):
+    def edit_issue(
+        self, issue_key: str, fields: dict, notify_users: bool = True
+    ) -> None:
+        # Skip modifying issues in dry-run mode.
+        pass
+
+    def create_issue(self, fields: dict) -> dict:
+        # Skip creating issues in dry-run mode and return dummy data.
+        return {"key": "DRYRUN", "fields": {"resolution": None, **fields}}

src/retasc/validator/generate_schema.py → src/retasc/models/generate_schema.py

@@ -5,7 +5,7 @@
 
 import yaml
 
-from retasc.validator.models import Rule
+from retasc.models.rule import Rule
 
 
 def _generate_schema(file: TextIO, generator) -> None:

src/retasc/validator/parse_rules.py → src/retasc/models/parse_rules.py

@@ -2,15 +2,15 @@
 import logging
 import os
 from collections import defaultdict
-from collections.abc import Iterator
 from dataclasses import dataclass, field
 from glob import iglob
 from itertools import chain
 
 import yaml
 from pydantic import ValidationError
 
-from retasc.validator.models import Rule
+from retasc.models.rule import Rule
+from retasc.utils import to_comma_separated
 
 logger = logging.getLogger(__name__)
 
@@ -36,21 +36,6 @@ def parse_yaml_objects(rule_file: str) -> list[dict]:
         return [data]
 
 
-def template_filenames(rule: Rule) -> Iterator[str]:
-    for issue in rule.jira_issues:
-        yield issue.template
-        yield from (x.template for x in issue.subtasks)
-
-
-def template_paths(rule: Rule, templates_path: str) -> Iterator[str]:
-    for file in template_filenames(rule):
-        yield f"{templates_path}/{file}"
-
-
-def to_comma_separated(items: list) -> str:
-    return ", ".join(sorted(repr(str(x)) for x in items))
-
-
 @dataclass
 class ParseState:
     """Keeps state for parsing and validation."""
@@ -90,37 +75,22 @@ def validate_unique_rule_names(self) -> None:
 
     def validate_existing_dependent_rules(self) -> None:
         for rule in self.rules.values():
-            missing_rules = [
-                name
-                for name in rule.prerequisites.dependent_rules
-                if name not in self.rules
+            errors = [
+                error
+                for prereq in rule.prerequisites
+                for error in prereq.validation_errors(self.rules.values())
             ]
-            if missing_rules:
-                rules_list = to_comma_separated(missing_rules)
-                self._add_invalid_rule_error(
-                    rule, f"Dependent rules do not exist: {rules_list}"
-                )
-
-    def validate_existing_jira_templates(self, templates_path: str) -> None:
-        for rule in self.rules.values():
-            missing_files = [
-                file
-                for file in template_paths(rule, templates_path)
-                if not os.path.isfile(file)
-            ]
-            if missing_files:
-                file_list = to_comma_separated(missing_files)
-                self._add_invalid_rule_error(
-                    rule,
-                    f"Jira issue template files not found: {file_list}",
-                )
+            if errors:
+                self._add_invalid_rule_error(rule, "\n  ".join(errors))
 
     def _add_invalid_rule_error(self, rule: Rule, error: str) -> None:
         filename = self.rule_files[rule.name][0]
-        self.errors.append(f"Invalid rule {rule.name!r} (file {filename!r}): {error}")
+        self.errors.append(
+            f"Invalid rule {rule.name!r} (file {filename!r}):\n  {error}"
+        )
 
 
-def parse_rules(path: str, templates_path: str = ".") -> dict[str, Rule]:
+def parse_rules(path: str) -> dict[str, Rule]:
     """
     Parses rules in path recursively to dict with rule name as key and the rule
     as value.
@@ -132,7 +102,6 @@ def parse_rules(path: str, templates_path: str = ".") -> dict[str, Rule]:
 
     state.validate_unique_rule_names()
     state.validate_existing_dependent_rules()
-    state.validate_existing_jira_templates(templates_path)
 
     if state.errors:
         error_list = "\n".join(state.errors)

src/retasc/models/prerequisites/__init__.py

@@ -0,0 +1,14 @@
+# SPDX-License-Identifier: GPL-3.0-or-later
+from .condition import PrerequisiteCondition
+from .jira_issue import PrerequisiteJiraIssue
+from .rule import PrerequisiteRule
+from .schedule import PrerequisiteSchedule
+from .target_date import PrerequisiteTargetDate
+
+type Prerequisite = (
+    PrerequisiteCondition
+    | PrerequisiteJiraIssue
+    | PrerequisiteRule
+    | PrerequisiteSchedule
+    | PrerequisiteTargetDate
+)