diff --git a/packit_service/config.py b/packit_service/config.py index bf6231460..e8d89b703 100644 --- a/packit_service/config.py +++ b/packit_service/config.py @@ -113,6 +113,7 @@ def __init__( package_config_path_override: Optional[str] = None, command_handler_storage_class: Optional[str] = None, appcode: Optional[str] = None, + enabled_projects_for_fedora_ci: Optional[Union[set[str], list[str]]] = None, **kwargs, ): super().__init__(**kwargs) @@ -157,6 +158,10 @@ def __init__( enabled_projects_for_internal_tf or [], ) + # e.g.: + # - https://src.fedoraproject.org/rpms/packit + self.enabled_projects_for_fedora_ci: set[str] = set(enabled_projects_for_fedora_ci or []) + self.projects_to_sync = projects_to_sync or [] # Full URL to the dashboard, e.g. https://dashboard.packit.dev @@ -211,7 +216,8 @@ def hide(token: str) -> str: f"enabled_projects_for_srpm_in_copr= '{self.enabled_projects_for_srpm_in_copr}', " f"comment_command_prefix='{self.comment_command_prefix}', " f"redhat_api_refresh_token='{hide(self.redhat_api_refresh_token)}', " - f"package_config_path_override='{self.package_config_path_override}')" + f"package_config_path_override='{self.package_config_path_override}', " + f"enabled_projects_for_fedora_ci='{self.enabled_projects_for_fedora_ci}')" ) @classmethod diff --git a/packit_service/schema.py b/packit_service/schema.py index ddca8df2f..0690e39ac 100644 --- a/packit_service/schema.py +++ b/packit_service/schema.py @@ -83,6 +83,7 @@ class ServiceConfigSchema(UserConfigSchema): package_config_path_override = fields.String() command_handler_storage_class = fields.String(missing="gp2") appcode = fields.String() + enabled_projects_for_fedora_ci = fields.List(fields.String()) @post_load def make_instance(self, data, **kwargs): diff --git a/packit_service/utils.py b/packit_service/utils.py index 996a7a2cf..33e27127a 100644 --- a/packit_service/utils.py +++ b/packit_service/utils.py @@ -217,7 +217,7 @@ def get_packit_commands_from_comment( return [] -def get_koji_task_id_and_url_from_stdout(stdout: str): +def get_koji_task_id_and_url_from_stdout(stdout: str) -> tuple[Optional[int], Optional[str]]: task_id, task_url = None, None task_id_match = search(pattern=r"Created task: (\d+)", string=stdout) diff --git a/packit_service/worker/checker/koji.py b/packit_service/worker/checker/koji.py index e10a0a570..0a106bb3c 100644 --- a/packit_service/worker/checker/koji.py +++ b/packit_service/worker/checker/koji.py @@ -3,6 +3,8 @@ import logging +from ogr.services.pagure import PagureProject + from packit_service.constants import ( KOJI_PRODUCTION_BUILDS_ISSUE, PERMISSIONS_ERROR_WRITE_OR_ADMIN, @@ -25,6 +27,11 @@ def pre_check(self) -> bool: return self.koji_build_helper.is_job_config_trigger_matching(self.job_config) +class IsUpstreamKojiScratchBuild(Checker, GetKojiBuildJobHelperMixin): + def pre_check(self) -> bool: + return not isinstance(self.koji_build_helper.project, PagureProject) + + class PermissionOnKoji(Checker, GetKojiBuildJobHelperMixin): def pre_check(self) -> bool: if ( diff --git a/packit_service/worker/events/pagure.py b/packit_service/worker/events/pagure.py index a733b4e67..b86e460b4 100644 --- a/packit_service/worker/events/pagure.py +++ b/packit_service/worker/events/pagure.py @@ -199,6 +199,7 @@ def __init__( project_url: str, commit_sha: str, user_login: str, + target_branch: str, ): super().__init__(project_url=project_url, pr_id=pr_id) self.action = action @@ -212,6 +213,7 @@ def __init__( self.identifier = str(pr_id) self.git_ref = None # pr_id will be used for checkout self.project_url = project_url + self.target_branch = target_branch def get_dict(self, default_dict: Optional[dict] = None) -> dict: result = super().get_dict() diff --git a/packit_service/worker/handlers/abstract.py b/packit_service/worker/handlers/abstract.py index bd7f99a88..d47ddf8ed 100644 --- a/packit_service/worker/handlers/abstract.py +++ b/packit_service/worker/handlers/abstract.py @@ -43,6 +43,9 @@ set, ) SUPPORTED_EVENTS_FOR_HANDLER: dict[type["JobHandler"], set[type["Event"]]] = defaultdict(set) +SUPPORTED_EVENTS_FOR_HANDLER_FEDORA_CI: dict[type["JobHandler"], set[type["Event"]]] = defaultdict( + set +) MAP_COMMENT_TO_HANDLER: dict[str, set[type["JobHandler"]]] = defaultdict(set) MAP_CHECK_PREFIX_TO_HANDLER: dict[str, set[type["JobHandler"]]] = defaultdict(set) @@ -84,7 +87,7 @@ def reacts_to(event: type["Event"]): """ [class decorator] Specify an event for which we want to use this handler. - Matching is done via isinstance so you can use some abstract class as well. + Matching is done via `isinstance` so you can use some abstract class as well. Multiple decorators are allowed. @@ -104,6 +107,30 @@ def _add_to_mapping(kls: type["JobHandler"]): return _add_to_mapping +def reacts_to_as_fedora_ci(event: type["Event"]): + """ + [class decorator] + Specify an event for which we want to use this handler as a Fedora CI. + Matching is done via `isinstance` so you can use some abstract class as well. + + Multiple decorators are allowed. + + Example: + ``` + @reacts_to(ReleaseEvent) + @reacts_to(PullRequestGithubEvent) + @reacts_to(PushGitHubEvent) + class CoprBuildHandler(JobHandler): + ``` + """ + + def _add_to_mapping(kls: type["JobHandler"]): + SUPPORTED_EVENTS_FOR_HANDLER_FEDORA_CI[kls].add(event) + return kls + + return _add_to_mapping + + def run_for_comment(command: str): """ [class decorator] @@ -190,6 +217,8 @@ class TaskName(str, enum.Enum): tag_into_sidetag = "task.tag_into_sidetag" openscanhub_task_finished = "task.openscanhub_task_finished" openscanhub_task_started = "task.openscanhub_task_started" + downstream_koji_scratch_build = "task.run_downstream_koji_scratch_build_handler" + downstream_koji_scratch_build_report = "task.run_downstream_koji_scratch_build_report_handler" class Handler(PackitAPIProtocol, Config): diff --git a/packit_service/worker/handlers/distgit.py b/packit_service/worker/handlers/distgit.py index b3b7e836b..5d18ddcd9 100644 --- a/packit_service/worker/handlers/distgit.py +++ b/packit_service/worker/handlers/distgit.py @@ -25,6 +25,7 @@ PackitException, ReleaseSkippedPackitException, ) +from packit.utils import commands from packit.utils.koji_helper import KojiHelper from packit_service import sentry_integration @@ -53,6 +54,7 @@ SyncReleaseTargetStatus, ) from packit_service.service.urls import ( + get_koji_build_info_url, get_propose_downstream_info_url, get_pull_from_upstream_info_url, ) @@ -79,6 +81,7 @@ IssueCommentGitlabEvent, KojiTaskEvent, PullRequestCommentPagureEvent, + PullRequestPagureEvent, PushPagureEvent, ReleaseEvent, ReleaseGitlabEvent, @@ -91,10 +94,12 @@ TaskName, configured_as, reacts_to, + reacts_to_as_fedora_ci, run_for_check_rerun, run_for_comment, ) from packit_service.worker.handlers.mixin import GetProjectToSyncMixin +from packit_service.worker.helpers.fedora_ci import FedoraCIHelper from packit_service.worker.helpers.sidetag import SidetagHelper from packit_service.worker.helpers.sync_release.propose_downstream import ( ProposeDownstreamJobHelper, @@ -740,6 +745,146 @@ def run(self) -> TaskResults: return super().run() +@reacts_to_as_fedora_ci(event=PullRequestPagureEvent) +class DownstreamKojiScratchBuildHandler( + RetriableJobHandler, ConfigFromUrlMixin, LocalProjectMixin, PackitAPIWithDownstreamMixin +): + """ + This handler can submit a scratch build in Koji from a dist-git (Fedora CI). + """ + + task_name = TaskName.downstream_koji_scratch_build + + def __init__( + self, + package_config: PackageConfig, + job_config: JobConfig, + event: dict, + celery_task: Task, + koji_group_model_id: Optional[int] = None, + ): + super().__init__( + package_config=package_config, + job_config=job_config, + event=event, + celery_task=celery_task, + ) + self._project_url = self.data.project_url + self._packit_api = None + self._koji_group_model_id = koji_group_model_id + self._ci_helper: Optional[FedoraCIHelper] = None + + @property + def ci_helper(self) -> FedoraCIHelper: + if not self._ci_helper: + self._ci_helper = FedoraCIHelper( + project=self.project, + metadata=self.data, + ) + return self._ci_helper + + @property + def dist_git_branch(self) -> str: + return self.data.event_dict.get("target_branch") + + @property + def repo_url(self) -> str: + event_dict = self.data.event_dict + full_repo_name = ( + f"forks/{event_dict.get('base_repo_owner')}/" + f"{event_dict.get('base_repo_namespace')}/{event_dict.get('base_repo_name')}" + ) + return f"git+https://src.fedoraproject.org/{full_repo_name}.git#{self.data.commit_sha}" + + def run(self) -> TaskResults: + try: + self.packit_api.init_kerberos_ticket() + except PackitCommandFailedError as ex: + msg = f"Kerberos authentication error: {ex.stderr_output}" + logger.error(msg) + self.ci_helper.report( + state=BaseCommitStatus.error, + description=msg, + url=None, + ) + return TaskResults(success=False, details={"msg": msg}) + + build_group = KojiBuildGroupModel.create( + run_model=PipelineModel.create( + project_event=self.data.db_project_event, + ) + ) + + koji_build = KojiBuildTargetModel.create( + task_id=None, + web_url=None, + target=self.dist_git_branch, + status="pending", + scratch=True, + koji_build_group=build_group, + ) + try: + stdout = self.run_koji_build() + if stdout: + task_id, web_url = get_koji_task_id_and_url_from_stdout(stdout) + koji_build.set_task_id(str(task_id)) + koji_build.set_web_url(web_url) + koji_build.set_build_submission_stdout(stdout) + url = get_koji_build_info_url(koji_build.id) + self.ci_helper.report( + state=BaseCommitStatus.running, + description="RPM build was submitted ...", + url=url, + ) + except Exception as ex: + sentry_integration.send_to_sentry(ex) + self.ci_helper.report( + state=BaseCommitStatus.error, + description=f"Submit of the build failed: {ex}", + url=None, + ) + if isinstance(ex, PackitCommandFailedError): + error = f"{ex!s}\n{ex.stderr_output}" + koji_build.set_build_submission_stdout(ex.stdout_output) + koji_build.set_data({"error": error}) + + koji_build.set_status("error") + return TaskResults( + success=False, + details={ + "msg": "Koji scratch build submit was not successful.", + "error": str(ex), + }, + ) + + return TaskResults(success=True, details={}) + + def run_koji_build( + self, + ): + """ + Perform a `koji build` from SCM. + + Returns: + str output + """ + cmd = [ + "koji", + "build", + "--scratch", + "--nowait", + self.dist_git_branch, + self.repo_url, + ] + logger.info("Starting a Koji scratch build.") + return commands.run_command_remote( + cmd=cmd, + cwd=self.local_project.working_dir, + output=True, + print_live=True, + ).stdout + + class AbstractDownstreamKojiBuildHandler( abc.ABC, RetriableJobHandler, diff --git a/packit_service/worker/handlers/koji.py b/packit_service/worker/handlers/koji.py index fd95fb24f..84cd6d878 100644 --- a/packit_service/worker/handlers/koji.py +++ b/packit_service/worker/handlers/koji.py @@ -6,6 +6,7 @@ """ import logging +from abc import ABC, abstractmethod from datetime import datetime from os import getenv from typing import Optional @@ -40,6 +41,7 @@ from packit_service.worker.checker.abstract import Checker from packit_service.worker.checker.koji import ( IsJobConfigTriggerMatching, + IsUpstreamKojiScratchBuild, PermissionOnKoji, SidetagExists, ) @@ -62,6 +64,7 @@ TaskName, configured_as, reacts_to, + reacts_to_as_fedora_ci, run_for_check_rerun, run_for_comment, ) @@ -69,6 +72,7 @@ from packit_service.worker.handlers.distgit import DownstreamKojiBuildHandler from packit_service.worker.handlers.mixin import GetKojiBuildJobHelperMixin from packit_service.worker.helpers.build.koji_build import KojiBuildJobHelper +from packit_service.worker.helpers.fedora_ci import FedoraCIHelper from packit_service.worker.helpers.sidetag import SidetagHelper from packit_service.worker.mixin import ( ConfigFromEventMixin, @@ -130,16 +134,9 @@ def run(self) -> TaskResults: return self.koji_build_helper.run_koji_build() -@configured_as(job_type=JobType.production_build) -@configured_as(job_type=JobType.upstream_koji_build) -@reacts_to(event=KojiTaskEvent) -class KojiTaskReportHandler( - JobHandler, - PackitAPIWithDownstreamMixin, - ConfigFromEventMixin, +class AbstractKojiTaskReportHandler( + ABC, JobHandler, PackitAPIWithDownstreamMixin, ConfigFromEventMixin ): - task_name = TaskName.upstream_koji_build_report - def __init__( self, package_config: PackageConfig, @@ -156,6 +153,14 @@ def __init__( self._db_project_event: Optional[ProjectEventModel] = None self._build: Optional[KojiBuildTargetModel] = None + @abstractmethod + def report(self, description: str, commit_status: BaseCommitStatus, url: str): ... + + @abstractmethod + def notify_about_failure_if_configured( + self, packit_dashboard_url: str, external_dashboard_url: str, logs_url: str + ): ... + @property def build(self) -> Optional[KojiBuildTargetModel]: if not self._build: @@ -171,21 +176,17 @@ def db_project_event(self) -> Optional[ProjectEventModel]: return self._db_project_event def run(self): - build = KojiBuildTargetModel.get_by_task_id( - task_id=str(self.koji_task_event.task_id), - ) - - if not build: + if not self.build: msg = f"Koji task {self.koji_task_event.task_id} not found in the database." logger.warning(msg) return TaskResults(success=False, details={"msg": msg}) logger.debug( - f"Build on {build.target} in koji changed state " + f"Build on {self.build.target} in Koji changed state " f"from {self.koji_task_event.old_state} to {self.koji_task_event.state}.", ) - build.set_build_start_time( + self.build.set_build_start_time( ( datetime.utcfromtimestamp(self.koji_task_event.start_time) if self.koji_task_event.start_time @@ -193,7 +194,7 @@ def run(self): ), ) - build.set_build_finished_time( + self.build.set_build_finished_time( ( datetime.utcfromtimestamp(self.koji_task_event.completion_time) if self.koji_task_event.completion_time @@ -201,15 +202,7 @@ def run(self): ), ) - url = get_koji_build_info_url(build.id) - build_job_helper = KojiBuildJobHelper( - service_config=self.service_config, - package_config=self.package_config, - project=self.project, - metadata=self.data, - db_project_event=self.db_project_event, - job_config=self.job_config, - ) + url = get_koji_build_info_url(self.build.id) new_commit_status = { KojiTaskState.free: BaseCommitStatus.pending, @@ -233,7 +226,7 @@ def run(self): logger.debug( f"We don't react to this koji build state change: {self.koji_task_event.state}", ) - elif new_commit_status.value == build.status: + elif new_commit_status.value == self.build.status: logger.debug( "Status was already processed (status in the DB is the " "same as the one about to report)", @@ -244,38 +237,102 @@ def run(self): ) else: - build.set_status(new_commit_status.value) - build_job_helper.report_status_to_all_for_chroot( - description=description, - state=new_commit_status, - url=url, - chroot=build.target, - ) + self.build.set_status(new_commit_status.value) + self.report(description, new_commit_status, url) koji_build_logs = self.koji_task_event.get_koji_build_rpm_tasks_logs_urls( self.service_config.koji_logs_url, ) - build.set_build_logs_urls(koji_build_logs) + self.build.set_build_logs_urls(koji_build_logs) koji_rpm_task_web_url = KojiTaskEvent.get_koji_rpm_build_web_url( - rpm_build_task_id=int(build.task_id), + rpm_build_task_id=int(self.build.task_id), koji_web_url=self.service_config.koji_web_url, ) - build.set_web_url(koji_rpm_task_web_url) + self.build.set_web_url(koji_rpm_task_web_url) if self.koji_task_event.state == KojiTaskState.failed: - build_job_helper.notify_about_failure_if_configured( + self.notify_about_failure_if_configured( packit_dashboard_url=url, external_dashboard_url=koji_rpm_task_web_url, logs_url=koji_build_logs, ) msg = ( - f"Build on {build.target} in koji changed state " + f"Build on {self.build.target} in koji changed state " f"from {self.koji_task_event.old_state} to {self.koji_task_event.state}." ) return TaskResults(success=True, details={"msg": msg}) +@configured_as(job_type=JobType.production_build) +@configured_as(job_type=JobType.upstream_koji_build) +@reacts_to(event=KojiTaskEvent) +class KojiTaskReportHandler(AbstractKojiTaskReportHandler): + task_name = TaskName.upstream_koji_build_report + _helper: Optional[KojiBuildJobHelper] = None + + @property + def helper(self): + if not self._helper: + self._helper = KojiBuildJobHelper( + service_config=self.service_config, + package_config=self.package_config, + project=self.project, + metadata=self.data, + db_project_event=self.db_project_event, + job_config=self.job_config, + ) + return self._helper + + @staticmethod + def get_checkers() -> tuple[type[Checker], ...]: + return (IsUpstreamKojiScratchBuild,) + + def report(self, description: str, commit_status: BaseCommitStatus, url: str): + self.helper.report_status_to_all_for_chroot( + description=description, + state=commit_status, + url=url, + chroot=self.build.target, + ) + + def notify_about_failure_if_configured( + self, packit_dashboard_url: str, external_dashboard_url: str, logs_url: str + ): + self.helper.notify_about_failure_if_configured( + packit_dashboard_url=packit_dashboard_url, + external_dashboard_url=external_dashboard_url, + logs_url=logs_url, + ) + + +@reacts_to_as_fedora_ci(event=KojiTaskEvent) +class KojiTaskReportDownstreamHandler(AbstractKojiTaskReportHandler): + task_name = TaskName.downstream_koji_build_report + _helper: Optional[FedoraCIHelper] = None + + @property + def helper(self): + if not self._helper: + self._helper = FedoraCIHelper( + project=self.project, + metadata=self.data, + ) + return self._helper + + def report(self, description: str, commit_status: BaseCommitStatus, url: str): + self.helper.report( + state=commit_status, + description=description, + url=url, + ) + + def notify_about_failure_if_configured( + self, packit_dashboard_url: str, external_dashboard_url: str, logs_url: str + ): + pass + + @configured_as(job_type=JobType.koji_build) @configured_as(job_type=JobType.bodhi_update) @reacts_to(event=KojiBuildEvent) diff --git a/packit_service/worker/helpers/fedora_ci.py b/packit_service/worker/helpers/fedora_ci.py new file mode 100644 index 000000000..49a4a4e23 --- /dev/null +++ b/packit_service/worker/helpers/fedora_ci.py @@ -0,0 +1,45 @@ +# Copyright Contributors to the Packit project. +# SPDX-License-Identifier: MIT + +import logging + +from ogr.abstract import GitProject + +from packit_service.worker.events import EventData +from packit_service.worker.reporting import BaseCommitStatus +from packit_service.worker.reporting.reporters.base import StatusReporter + +logger = logging.getLogger(__name__) + + +class FedoraCIHelper: + status_name: str = "Packit - scratch build" + + def __init__( + self, + project: GitProject, + metadata: EventData, + ): + self.project = project + self.metadata = metadata + + self._status_reporter = None + + @property + def status_reporter(self) -> StatusReporter: + if not self._status_reporter: + self._status_reporter = StatusReporter.get_instance( + project=self.project, + commit_sha=self.metadata.commit_sha, + pr_id=self.metadata.pr_id, + packit_user=None, + ) + return self._status_reporter + + def report(self, state: BaseCommitStatus, description: str, url: str): + self.status_reporter.set_status( + state=state, + description=description, + url=url, + check_name=self.status_name, + ) diff --git a/packit_service/worker/jobs.py b/packit_service/worker/jobs.py index 3ad461152..36d5e979b 100644 --- a/packit_service/worker/jobs.py +++ b/packit_service/worker/jobs.py @@ -37,6 +37,7 @@ InstallationEvent, IssueCommentEvent, PullRequestCommentPagureEvent, + PullRequestPagureEvent, ) from packit_service.worker.events.comment import ( AbstractCommentEvent, @@ -48,7 +49,7 @@ AbstractForgeIndependentEvent, AbstractResultEvent, ) -from packit_service.worker.events.koji import KojiBuildTagEvent +from packit_service.worker.events.koji import KojiBuildTagEvent, KojiTaskEvent from packit_service.worker.handlers import ( CoprBuildHandler, GithubAppInstallationHandler, @@ -63,6 +64,7 @@ MAP_JOB_TYPE_TO_HANDLER, MAP_REQUIRED_JOB_TYPE_TO_HANDLER, SUPPORTED_EVENTS_FOR_HANDLER, + SUPPORTED_EVENTS_FOR_HANDLER_FEDORA_CI, JobHandler, ) from packit_service.worker.handlers.bodhi import ( @@ -242,6 +244,13 @@ def process(self) -> list[TaskResults]: ).apply_async() # should we comment about not processing if the comment is not # on the issue created by us or not in packit/notifications? + elif ( + isinstance(self.event, (PullRequestPagureEvent, KojiTaskEvent)) + and self.event.db_project_object + and (url := self.event.db_project_object.project.project_url) + and url in self.service_config.enabled_projects_for_fedora_ci + ): + processing_results = self.process_fedora_ci_jobs() else: # Processing the jobs from the config. processing_results = self.process_jobs() @@ -431,6 +440,55 @@ def is_packit_config_present(self) -> bool: # success=True - it's not an error that people don't have packit.yaml in their repo return self.event.packages_config + def process_fedora_ci_jobs(self) -> list[TaskResults]: + """ + Create Celery tasks for a job handler (if the trigger matches) for Fedora CI. + + Returns: + A list of task results for each task created. + """ + matching_handlers = { + handler + for handler, supported_events in SUPPORTED_EVENTS_FOR_HANDLER_FEDORA_CI.items() + if isinstance(self.event, tuple(supported_events)) + } + + if not matching_handlers: + logger.debug(f"No handler found for event {self.event} for Fedora CI.") + return [] + + # TODO: add allowlist checks here + + processing_results: list[TaskResults] = [] + + for handler_kls in matching_handlers: + # TODO: pre-checks + # TODO: report task accepted + + celery_signature = celery.signature( + handler_kls.task_name.value, + kwargs={ + "package_config": None, + "job_config": None, + "event": self.event.get_dict(), + }, + ) + + celery_signature.apply_async() + logger.debug(f"Celery signature sent for handler {handler_kls}.") + + processing_results.append( + TaskResults( + success=True, + details={ + "msg": "Job created.", + "event": self.event.get_dict(), + }, + ) + ) + + return processing_results + def process_jobs(self) -> list[TaskResults]: """ Create Celery tasks for a job handler (if trigger matches) for every diff --git a/packit_service/worker/parser.py b/packit_service/worker/parser.py index 9d77676db..8255957b8 100644 --- a/packit_service/worker/parser.py +++ b/packit_service/worker/parser.py @@ -1736,6 +1736,7 @@ def parse_pagure_pull_request_event( target_repo = repo_from["name"] if repo_from else base_repo_name https_url = event["pullrequest"]["project"]["full_url"] commit_sha = event["pullrequest"]["commit_stop"] + target_branch = event["pullrequest"]["branch"] return PullRequestPagureEvent( action=PullRequestAction[action], @@ -1748,6 +1749,7 @@ def parse_pagure_pull_request_event( project_url=https_url, commit_sha=commit_sha, user_login=pagure_login, + target_branch=target_branch, ) @staticmethod diff --git a/packit_service/worker/tasks.py b/packit_service/worker/tasks.py index 897f9a0ce..8411d0450 100644 --- a/packit_service/worker/tasks.py +++ b/packit_service/worker/tasks.py @@ -71,6 +71,7 @@ ) from packit_service.worker.handlers.distgit import ( DownstreamKojiBuildHandler, + DownstreamKojiScratchBuildHandler, PullFromUpstreamHandler, RetriggerDownstreamKojiBuildHandler, TagIntoSidetagHandler, @@ -79,6 +80,7 @@ from packit_service.worker.handlers.koji import ( KojiBuildReportHandler, KojiBuildTagHandler, + KojiTaskReportDownstreamHandler, ) from packit_service.worker.handlers.usage import check_onboarded_projects from packit_service.worker.helpers.build.babysit import ( @@ -395,6 +397,31 @@ def run_koji_build_report_handler(event: dict, package_config: dict, job_config: return get_handlers_task_results(handler.run_job(), event) +@celery_app.task(name=TaskName.downstream_koji_scratch_build_report, base=TaskWithRetry) +def run_downstream_koji_scratch_build_report_handler( + event: dict, package_config: dict, job_config: dict +): + handler = KojiTaskReportDownstreamHandler( + package_config=load_package_config(package_config), + job_config=load_job_config(job_config), + event=event, + ) + return get_handlers_task_results(handler.run_job(), event) + + +@celery_app.task(bind=True, name=TaskName.downstream_koji_scratch_build, base=TaskWithRetry) +def run_downstream_koji_scratch_build_handler( + self, event: dict, package_config: dict, job_config: dict +): + handler = DownstreamKojiScratchBuildHandler( + package_config=load_package_config(package_config), + job_config=load_job_config(job_config), + event=event, + celery_task=self, + ) + return get_handlers_task_results(handler.run_job(), event) + + @celery_app.task( name=TaskName.sync_from_downstream, base=TaskWithRetry, diff --git a/tests/conftest.py b/tests/conftest.py index c6a041586..878afb60a 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -332,6 +332,64 @@ def koji_build_pr(): return koji_build_model +@pytest.fixture() +def koji_build_pr_downstream(): + project_model = flexmock( + repo_name="packit", + namespace="rpms", + project_url="https://src.fedoraproject.org/rpms/packit", + ) + pr_model = flexmock( + id=1, + pr_id=123, + project=project_model, + job_config_trigger_type=JobConfigTriggerType.pull_request, + project_event_model_type=ProjectEventModelType.pull_request, + commit_sha="0011223344", + ) + project_event_model = flexmock( + id=2, + type=ProjectEventModelType.pull_request, + event_id=1, + get_project_event_object=lambda: pr_model, + ) + runs = [] + srpm_build = flexmock(logs="asdsdf", url=None, runs=runs) + koji_build_model = flexmock( + id=1, + task_id="1", + commit_sha="0011223344", + project_name="some-project", + owner="some-owner", + web_url="https://some-url", + target="some-target", + status="some-status", + runs=runs, + ) + koji_build_model._srpm_build_for_mocking = srpm_build + koji_build_model.get_project_event_object = lambda: pr_model + koji_build_model.get_srpm_build = lambda: srpm_build + koji_build_model.should_receive("get_project_event_model").and_return( + project_event_model, + ) + + flexmock(ProjectEventModel).should_receive("get_or_create").with_args( + type=pr_model.project_event_model_type, + event_id=pr_model.id, + commit_sha="0011223344", + ).and_return(project_event_model) + + run_model = flexmock( + id=3, + job_project_event=project_event_model, + srpm_build=srpm_build, + copr_build=koji_build_model, + ) + runs.append(run_model) + + return koji_build_model + + @pytest.fixture() def add_pull_request_event_with_sha_123456(): db_project_object = flexmock( diff --git a/tests/integration/test_dg_pr.py b/tests/integration/test_dg_pr.py new file mode 100644 index 000000000..277a8d0f6 --- /dev/null +++ b/tests/integration/test_dg_pr.py @@ -0,0 +1,144 @@ +# Copyright Contributors to the Packit project. +# SPDX-License-Identifier: MIT + +import json +from pathlib import Path + +import pytest +from celery.canvas import Signature +from flexmock import flexmock +from packit.api import PackitAPI +from packit.config import ( + JobConfigTriggerType, +) +from packit.local_project import LocalProjectBuilder +from packit.utils import commands + +from packit_service.config import ServiceConfig +from packit_service.constants import SANDCASTLE_WORK_DIR +from packit_service.models import ( + KojiBuildGroupModel, + KojiBuildTargetModel, + PipelineModel, + ProjectEventModel, + ProjectEventModelType, + PullRequestModel, +) +from packit_service.worker.handlers import distgit +from packit_service.worker.jobs import SteveJobs +from packit_service.worker.monitoring import Pushgateway +from packit_service.worker.reporting import BaseCommitStatus +from packit_service.worker.reporting.reporters.base import StatusReporter +from packit_service.worker.tasks import ( + run_downstream_koji_scratch_build_handler, +) +from tests.spellbook import DATA_DIR, first_dict_value, get_parameters_from_results + + +@pytest.fixture() +def distgit_pr_event(): + return json.loads((DATA_DIR / "fedmsg" / "pagure_pr_new.json").read_text()) + + +def test_downstream_koji_scratch_build(distgit_pr_event): + dg_project = ( + flexmock(namespace="rpms", repo="optee_os") + .should_receive("is_private") + .and_return(False) + .mock() + ) + service_config = ( + flexmock( + enabled_projects_for_fedora_ci="https://src.fedoraproject.org/rpms/optee_os", + command_handler_work_dir=SANDCASTLE_WORK_DIR, + repository_cache="/tmp/repository-cache", + add_repositories_to_repository_cache=False, + ) + .should_receive("get_project") + .and_return(dg_project) + .mock() + ) + flexmock(ServiceConfig).should_receive("get_service_config").and_return(service_config) + db_project_object = flexmock( + id=9, + job_config_trigger_type=JobConfigTriggerType.pull_request, + project_event_model_type=ProjectEventModelType.pull_request, + project=flexmock(project_url="https://src.fedoraproject.org/rpms/optee_os"), + ) + db_project_event = ( + flexmock().should_receive("get_project_event_object").and_return(db_project_object).mock() + ) + flexmock(ProjectEventModel).should_receive("get_or_create").with_args( + type=ProjectEventModelType.pull_request, + event_id=9, + commit_sha="abcd", + ).and_return(flexmock()) + flexmock(PullRequestModel).should_receive("get_or_create").with_args( + pr_id=2, + namespace="rpms", + repo_name="optee_os", + project_url="https://src.fedoraproject.org/rpms/optee_os", + ).and_return(db_project_object) + flexmock(ProjectEventModel).should_receive("get_or_create").and_return( + db_project_event, + ) + flexmock(PipelineModel).should_receive("create") + + koji_build = flexmock( + id=123, + target="main", + status="queued", + set_status=lambda x: None, + set_task_id=lambda x: None, + set_web_url=lambda x: None, + set_build_logs_urls=lambda x: None, + set_data=lambda x: None, + set_build_submission_stdout=lambda x: None, + ) + + flexmock(KojiBuildTargetModel).should_receive("create").and_return(koji_build) + flexmock(KojiBuildGroupModel).should_receive("create").and_return( + flexmock(grouped_targets=[koji_build]), + ) + + flexmock(LocalProjectBuilder, _refresh_the_state=lambda *args: None) + flexmock(Signature).should_receive("apply_async").once() + flexmock(Pushgateway).should_receive("push").times(2).and_return() + flexmock(commands).should_receive("run_command_remote").with_args( + cmd=[ + "koji", + "build", + "--scratch", + "--nowait", + "rawhide", + "git+https://src.fedoraproject.org/forks/zbyszek/rpms/optee_os.git#889f07af35d27bbcaf9c535c17a63b974aa42ee3", + ], + cwd=Path, + output=True, + print_live=True, + ).and_return(flexmock(stdout="some output")) + flexmock(PackitAPI).should_receive("init_kerberos_ticket") + + flexmock(distgit).should_receive("get_koji_task_id_and_url_from_stdout").and_return( + (123, "koji-web-url") + ).once() + + flexmock(StatusReporter).should_receive("set_status").with_args( + state=BaseCommitStatus.running, + description="RPM build was submitted ...", + url="https://localhost/jobs/koji/123", + check_name="Packit - scratch build", + ).once() + + processing_results = SteveJobs().process_message(distgit_pr_event) + event_dict, job, job_config, package_config = get_parameters_from_results( + processing_results, + ) + assert json.dumps(event_dict) + results = run_downstream_koji_scratch_build_handler( + package_config=package_config, + event=event_dict, + job_config=job_config, + ) + + assert first_dict_value(results["job"])["success"] diff --git a/tests/integration/test_listen_to_fedmsg.py b/tests/integration/test_listen_to_fedmsg.py index a773cfc2b..09fe42d2e 100644 --- a/tests/integration/test_listen_to_fedmsg.py +++ b/tests/integration/test_listen_to_fedmsg.py @@ -59,6 +59,7 @@ from packit_service.worker.tasks import ( run_copr_build_end_handler, run_copr_build_start_handler, + run_downstream_koji_scratch_build_report_handler, run_koji_build_report_handler, run_koji_build_tag_handler, run_testing_farm_handler, @@ -2567,6 +2568,64 @@ def test_koji_build_end(koji_build_scratch_end, pc_koji_build_pr, koji_build_pr) assert first_dict_value(results["job"])["success"] +def test_koji_build_end_downstream( + koji_build_scratch_end, pc_koji_build_pr, koji_build_pr_downstream +): + service_config = ( + flexmock( + enabled_projects_for_fedora_ci="https://src.fedoraproject.org/rpms/packit", + koji_logs_url="", + koji_web_url="", + ) + .should_receive("get_project") + .and_return(flexmock(namespace="rpms", repo="packit")) + .mock() + ) + flexmock(ServiceConfig).should_receive("get_service_config").and_return(service_config) + koji_build_pr_downstream.target = "rawhide" + flexmock(GithubProject).should_receive("is_private").and_return(False) + flexmock(KojiTaskEvent).should_receive("get_packages_config").and_return( + pc_koji_build_pr, + ) + + flexmock(KojiBuildTargetModel).should_receive("get_by_task_id").and_return( + koji_build_pr_downstream, + ) + url = get_koji_build_info_url(1) + flexmock(requests).should_receive("get").and_return(requests.Response()) + flexmock(requests.Response).should_receive("raise_for_status").and_return(None) + + koji_build_pr_downstream.should_receive("set_build_start_time").once() + koji_build_pr_downstream.should_receive("set_build_finished_time").once() + koji_build_pr_downstream.should_receive("set_status").with_args("success").once() + koji_build_pr_downstream.should_receive("set_build_logs_urls") + koji_build_pr_downstream.should_receive("set_web_url") + + # check if packit-service set correct PR status + flexmock(StatusReporter).should_receive("set_status").with_args( + state=BaseCommitStatus.success, + description="RPM build succeeded.", + url=url, + check_name="Packit - scratch build", + ).once() + flexmock(Signature).should_receive("apply_async").once() + flexmock(Pushgateway).should_receive("push").times(2).and_return() + + processing_results = SteveJobs().process_message(koji_build_scratch_end) + event_dict, job, job_config, package_config = get_parameters_from_results( + processing_results, + ) + assert json.dumps(event_dict) + + results = run_downstream_koji_scratch_build_report_handler( + package_config=package_config, + event=event_dict, + job_config=job_config, + ) + + assert first_dict_value(results["job"])["success"] + + def test_koji_build_tag( koji_build_tagged, pc_koji_build_tag_specfile, diff --git a/tests/spellbook.py b/tests/spellbook.py index 01d0f9ee9..42a2d82b5 100644 --- a/tests/spellbook.py +++ b/tests/spellbook.py @@ -27,9 +27,9 @@ def get_parameters_from_results( assert len(results) == 1 event_dict = results[0]["details"]["event"] - job = results[0]["details"]["job"] - job_config = results[0]["details"]["job_config"] - package_config = results[0]["details"]["package_config"] + job = results[0]["details"].get("job") + job_config = results[0]["details"].get("job_config") + package_config = results[0]["details"].get("package_config") return event_dict, job, job_config, package_config