diff --git a/integrations/sonarqube/.port/resources/blueprints.json b/integrations/sonarqube/.port/resources/blueprints.json index 871e945619..db84f6abf5 100644 --- a/integrations/sonarqube/.port/resources/blueprints.json +++ b/integrations/sonarqube/.port/resources/blueprints.json @@ -65,9 +65,19 @@ "icon": "Git", "title": "Main Branch" }, - "tags": { - "type": "array", - "title": "Tags" + "mainBranchLastAnalysisDate": { + "type": "string", + "format": "date-time", + "icon": "Clock", + "title": "Main Branch Last Analysis Date" + }, + "revision": { + "type": "string", + "title": "Revision" + }, + "managed": { + "type": "boolean", + "title": "Managed" } }, "required": [] diff --git a/integrations/sonarqube/.port/resources/port-app-config.yaml b/integrations/sonarqube/.port/resources/port-app-config.yaml index a1e7751f1c..35c2b3c2ce 100644 --- a/integrations/sonarqube/.port/resources/port-app-config.yaml +++ b/integrations/sonarqube/.port/resources/port-app-config.yaml @@ -1,12 +1,12 @@ createMissingRelatedEntities: true deleteDependentEntities: true resources: - - kind: projects + - kind: projects_ga selector: query: 'true' apiFilters: - filter: - qualifier: TRK + qualifier: + - TRK metrics: - code_smells - coverage @@ -27,7 +27,7 @@ resources: organization: .organization link: .__link qualityGateStatus: .__branch.status.qualityGateStatus - lastAnalysisDate: .__branch.analysisDate + lastAnalysisDate: .analysisDate numberOfBugs: .__measures[]? | select(.metric == "bugs") | .value numberOfCodeSmells: .__measures[]? | select(.metric == "code_smells") | .value numberOfVulnerabilities: .__measures[]? | select(.metric == "vulnerabilities") | .value @@ -35,7 +35,9 @@ resources: numberOfDuplications: .__measures[]? | select(.metric == "duplicated_files") | .value coverage: .__measures[]? | select(.metric == "coverage") | .value mainBranch: .__branch.name - tags: .tags + mainBranchLastAnalysisDate: .__branch.analysisDate + revision: .revision + managed: .managed - kind: analysis selector: query: 'true' diff --git a/integrations/sonarqube/.port/spec.yaml b/integrations/sonarqube/.port/spec.yaml index 7f4d55648c..a4ff88cf0b 100644 --- a/integrations/sonarqube/.port/spec.yaml +++ b/integrations/sonarqube/.port/spec.yaml @@ -6,6 +6,7 @@ features: section: Code Quality & Security resources: - kind: projects + - kind: projects_ga - kind: saas_analysis - kind: onprem_analysis - kind: issues diff --git a/integrations/sonarqube/CHANGELOG.md b/integrations/sonarqube/CHANGELOG.md index 6a01525aae..cf1fed0e94 100644 --- a/integrations/sonarqube/CHANGELOG.md +++ b/integrations/sonarqube/CHANGELOG.md @@ -7,6 +7,21 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 + +## 0.1.122 (2024-12-23) + + +### Improvements + +- Increased logs presence in integration +- Replaced calls to internal API for projects to GA version, making the use of internal APIs optional + + +### Bug Fixes + +- Fixed a bug in the pagination logic to use total record count instead of response size, preventing early termination (0.1.121) + + ## 0.1.121 (2024-12-22) diff --git a/integrations/sonarqube/client.py b/integrations/sonarqube/client.py index afc6acd1a7..f84d4d9b82 100644 --- a/integrations/sonarqube/client.py +++ b/integrations/sonarqube/client.py @@ -4,19 +4,14 @@ import httpx from loguru import logger -from port_ocean.context.event import event from port_ocean.utils import http_async_client - -from integration import ( - CustomSelector, - SonarQubeIssueResourceConfig, - SonarQubeProjectResourceConfig, -) +from port_ocean.utils.async_iterators import stream_async_iterators_tasks +from port_ocean.utils.cache import cache_iterator_result def turn_sequence_to_chunks( - sequence: list[str], chunk_size: int -) -> Generator[list[str], None, None]: + sequence: list[Any], chunk_size: int +) -> Generator[list[Any], None, None]: if chunk_size >= len(sequence): yield sequence return @@ -34,7 +29,9 @@ def turn_sequence_to_chunks( class Endpoints: - PROJECTS = "components/search_projects" + COMPONENTS = "components/search_projects" + COMPONENT_SHOW = "components/show" + PROJECTS = "projects/search" WEBHOOKS = "webhooks" MEASURES = "measures/component" BRANCHES = "project_branches/list" @@ -45,11 +42,19 @@ class Endpoints: PAGE_SIZE = 100 +PROJECTS_RESYNC_BATCH_SIZE = 20 PORTFOLIO_VIEW_QUALIFIERS = ["VW", "SVW"] class SonarQubeClient: + """ + This client has no rate limiting logic implemented. This is + because [SonarQube API does not have rate limiting) + [https://community.sonarsource.com/t/need-api-limit-documentation/116582]. + The client is used to interact with the SonarQube API to fetch data. + """ + def __init__( self, base_url: str, @@ -66,6 +71,7 @@ def __init__( self.http_client = http_async_client self.http_client.headers.update(self.api_auth_params["headers"]) self.metrics: list[str] = [] + self.webhook_invoke_url = f"{self.app_host}/integration/webhook" @property def api_auth_params(self) -> dict[str, Any]: @@ -88,7 +94,7 @@ def api_auth_params(self) -> dict[str, Any]: }, } - async def send_api_request( + async def _send_api_request( self, endpoint: str, method: str = "GET", @@ -113,46 +119,47 @@ async def send_api_request( ) raise - async def send_paginated_api_request( + async def _send_paginated_request( self, endpoint: str, data_key: str, method: str = "GET", query_params: Optional[dict[str, Any]] = None, json_data: Optional[dict[str, Any]] = None, - ) -> list[dict[str, Any]]: - + ) -> AsyncGenerator[list[dict[str, Any]], None]: query_params = query_params or {} query_params["ps"] = PAGE_SIZE - all_resources = [] # List to hold all fetched resources + logger.info(f"Starting paginated request to {endpoint}") try: while True: - logger.info( - f"Sending API request to {method} {endpoint} with query params: {query_params}" - ) - response = await self.http_client.request( + response = await self._send_api_request( + endpoint=endpoint, method=method, - url=f"{self.base_url}/api/{endpoint}", - params=query_params, - json=json_data, + query_params=query_params, + json_data=json_data, ) - response.raise_for_status() - response_json = response.json() - resource = response_json.get(data_key, []) - if not resource: - logger.warning(f"No {data_key} found in response: {response_json}") + resources = response.get(data_key, []) + if not resources: + logger.warning(f"No {data_key} found in response: {response}") - all_resources.extend(resource) + if resources: + logger.info(f"Fetched {len(resources)} {data_key} from API") + yield resources - # Check for paging information and decide whether to fetch more pages - paging_info = response_json.get("paging") - if paging_info is None or len(resource) < PAGE_SIZE: + paging_info = response.get("paging") + if not paging_info: break - query_params["p"] = paging_info["pageIndex"] + 1 - - return all_resources - + page_index = paging_info.get( + "pageIndex", 1 + ) # SonarQube pageIndex starts at 1 + page_size = paging_info.get("pageSize", PAGE_SIZE) + total_records = paging_info.get("total", 0) + logger.error("Fetching paginated data") + # Check if we have fetched all records + if page_index * page_size >= total_records: + break + query_params["p"] = page_index + 1 except httpx.HTTPStatusError as e: logger.error( f"HTTP error with status code: {e.response.status_code} and response text: {e.response.text}" @@ -165,56 +172,48 @@ async def send_paginated_api_request( logger.error( "The request exceeded the maximum number of issues that can be returned (10,000) from SonarQube API. Consider using apiFilters in the config mapping to narrow the scope of your search. Returning accumulated issues and skipping further results." ) - return all_resources if e.response.status_code == 404: logger.error(f"Resource not found: {e.response.text}") - return all_resources + raise except httpx.HTTPError as e: logger.error(f"HTTP occurred while fetching paginated data: {e}") raise + @cache_iterator_result() async def get_components( - self, api_query_params: Optional[dict[str, Any]] = None - ) -> list[dict[str, Any]]: + self, + query_params: Optional[dict[str, Any]] = None, + ) -> AsyncGenerator[list[dict[str, Any]], None]: """ Retrieve all components from SonarQube organization. :return: A list of components associated with the specified organization. """ - query_params = {} if self.organization_id: - query_params["organization"] = self.organization_id logger.info( f"Fetching all components in organization: {self.organization_id}" ) - ## Handle api_query_params based on environment if not self.is_onpremise: logger.warning( - f"Received request to fetch SonarQube components with api_query_params {api_query_params}. Skipping because api_query_params is only supported on on-premise environments" + f"Received request to fetch SonarQube components with query_params {query_params}. Skipping because api_query_params is only supported on on-premise environments" ) - else: - if api_query_params: - query_params.update(api_query_params) - elif event.resource_config: - # This might be called from places where event.resource_config is not set - # like on_start() when creating webhooks - - selector = cast(CustomSelector, event.resource_config.selector) - query_params.update(selector.generate_request_params()) try: - response = await self.send_paginated_api_request( - endpoint=Endpoints.PROJECTS, + async for components in self._send_paginated_request( + endpoint=Endpoints.COMPONENTS, data_key="components", + method="GET", query_params=query_params, - ) - logger.info( - f"Fetched {len(response)} components {[item.get("key") for item in response]} from SonarQube" - ) - return response + ): + logger.info( + f"Fetched {len(components)} components {[item.get('key') for item in components]} from SonarQube" + ) + yield await asyncio.gather( + *[self.get_single_project(project) for project in components] + ) except Exception as e: logger.error(f"Error occurred while fetching components: {e}") raise @@ -228,8 +227,8 @@ async def get_single_component(self, project: dict[str, Any]) -> dict[str, Any]: """ project_key = project.get("key") logger.info(f"Fetching component data in : {project_key}") - response = await self.send_api_request( - endpoint="components/show", + response = await self._send_api_request( + endpoint=Endpoints.COMPONENT_SHOW, query_params={"component": project_key}, ) return response.get("component", {}) @@ -243,7 +242,7 @@ async def get_measures(self, project_key: str) -> list[Any]: :return: A list of measures associated with the specified component. """ logger.info(f"Fetching all measures in : {project_key}") - response = await self.send_api_request( + response = await self._send_api_request( endpoint=Endpoints.MEASURES, query_params={ "component": project_key, @@ -255,7 +254,7 @@ async def get_measures(self, project_key: str) -> list[Any]: async def get_branches(self, project_key: str) -> list[Any]: """A function to make API request to SonarQube and retrieve branches within an organization""" logger.info(f"Fetching all branches in : {project_key}") - response = await self.send_api_request( + response = await self._send_api_request( endpoint=Endpoints.BRANCHES, query_params={"project": project_key} ) return response.get("branches", []) @@ -284,51 +283,53 @@ async def get_single_project(self, project: dict[str, Any]) -> dict[str, Any]: return project - async def get_all_projects(self) -> AsyncGenerator[list[dict[str, Any]], None]: - """ - Retrieve all projects from SonarQube API. + @cache_iterator_result() + async def get_projects( + self, params: dict[str, Any] = {}, enrich_project: bool = False + ) -> AsyncGenerator[list[dict[str, Any]], None]: + if self.organization_id: + params["organization"] = self.organization_id + + async for projects in self._send_paginated_request( + endpoint=Endpoints.PROJECTS, + data_key="components", + method="GET", + query_params=params, + ): + # if enrich_project is True, fetch the project details + # including measures, branches and link + if enrich_project: + yield await asyncio.gather( + *[self.get_single_project(project) for project in projects] + ) + else: + yield projects - :return (list[Any]): A list containing projects data for your organization. - """ - logger.info(f"Fetching all projects in organization: {self.organization_id}") - self.metrics = cast( - SonarQubeProjectResourceConfig, event.resource_config - ).selector.metrics - components = await self.get_components() - for component in components: - project_data = await self.get_single_project(project=component) - yield [project_data] - - async def get_all_issues(self) -> AsyncGenerator[list[dict[str, Any]], None]: + async def get_all_issues( + self, + query_params: dict[str, Any], + project_query_params: dict[str, Any], + ) -> AsyncGenerator[list[dict[str, Any]], None]: """ Retrieve issues data across all components from SonarQube API as an asynchronous generator. :return (list[Any]): A list containing issues data for all projects. """ - selector = cast(SonarQubeIssueResourceConfig, event.resource_config).selector - api_query_params = selector.generate_request_params() - - project_api_query_params = ( - selector.project_api_filters.generate_request_params() - if selector.project_api_filters - else None - ) - - components = await self.get_components( - api_query_params=project_api_query_params - ) - for component in components: - response = await self.get_issues_by_component( - component=component, api_query_params=api_query_params - ) - yield response + async for components in self.get_projects( + params=project_query_params, enrich_project=False + ): + for component in components: + async for responses in self.get_issues_by_component( + component=component, query_params=query_params + ): + yield responses async def get_issues_by_component( self, component: dict[str, Any], - api_query_params: Optional[dict[str, Any]] = None, - ) -> list[dict[str, Any]]: + query_params: dict[str, Any] = {}, + ) -> AsyncGenerator[list[dict[str, Any]], None]: """ Retrieve issues data across a single component (in this case, project) from SonarQube API. @@ -336,30 +337,25 @@ async def get_issues_by_component( :return (list[Any]): A list containing issues data for the specified component. """ - component_issues = [] component_key = component.get("key") if self.is_onpremise: - query_params = {"components": component_key} + query_params["components"] = component_key else: - query_params = {"componentKeys": component_key} - - if api_query_params: - query_params.update(api_query_params) + query_params["componentKeys"] = component_key - response = await self.send_paginated_api_request( + async for responses in self._send_paginated_request( endpoint=Endpoints.ISSUES_SEARCH, data_key="issues", query_params=query_params, - ) - - for issue in response: - issue["__link"] = ( - f"{self.base_url}/project/issues?open={issue.get('key')}&id={component_key}" - ) - component_issues.append(issue) - - return component_issues + ): + yield [ + { + **issue, + "__link": f"{self.base_url}/project/issues?open={issue.get('key')}&id={component_key}", + } + for issue in responses + ] async def get_all_sonarcloud_analyses( self, @@ -369,15 +365,17 @@ async def get_all_sonarcloud_analyses( :return (list[Any]): A list containing analysis data for all components. """ - components = await self.get_components() - - for component in components: - response = await self.get_analysis_by_project(component=component) - yield response + async for components in self.get_projects(enrich_project=False): + tasks = [ + self.get_analysis_by_project(component=component) + for component in components + ] + async for project_analysis in stream_async_iterators_tasks(*tasks): + yield project_analysis async def get_analysis_by_project( self, component: dict[str, Any] - ) -> list[dict[str, Any]]: + ) -> AsyncGenerator[list[dict[str, Any]], None]: """ Retrieve analysis data for the given component from SonarQube API. @@ -386,37 +384,35 @@ async def get_analysis_by_project( :return (list[dict[str, Any]]): A list containing analysis data for all components. """ component_key = component.get("key") - component_analysis_data = [] logger.info(f"Fetching all analysis data in : {component_key}") - response = await self.send_paginated_api_request( + async for response in self._send_paginated_request( endpoint=Endpoints.ANALYSIS, data_key="activityFeed", query_params={"project": component_key}, - ) - - for activity in response: - if activity["type"] == "analysis": - analysis_data = activity["data"] - branch_data = analysis_data.get("branch", {}) - pr_data = analysis_data.get("pullRequest", {}) - - analysis_data["__branchName"] = branch_data.get( - "name", pr_data.get("branch") - ) - analysis_data["__analysisDate"] = branch_data.get( - "analysisDate", pr_data.get("analysisDate") - ) - analysis_data["__commit"] = branch_data.get( - "commit", pr_data.get("commit") - ) - analysis_data["__component"] = component - analysis_data["__project"] = component_key - - component_analysis_data.append(analysis_data) - - return component_analysis_data + ): + component_analysis_data = [] + for activity in response: + if activity["type"] == "analysis": + analysis_data = activity["data"] + branch_data = analysis_data.get("branch", {}) + pr_data = analysis_data.get("pullRequest", {}) + + analysis_data["__branchName"] = branch_data.get( + "name", pr_data.get("branch") + ) + analysis_data["__analysisDate"] = branch_data.get( + "analysisDate", pr_data.get("analysisDate") + ) + analysis_data["__commit"] = branch_data.get( + "commit", pr_data.get("commit") + ) + analysis_data["__component"] = component + analysis_data["__project"] = component_key + + component_analysis_data.append(analysis_data) + yield component_analysis_data async def get_analysis_for_task( self, @@ -430,25 +426,26 @@ async def get_analysis_for_task( """ ## Get the compute engine task that runs the analysis task_id = webhook_data.get("taskId") - task_response = await self.send_api_request( + task_response = await self._send_api_request( endpoint="ce/task", query_params={"id": task_id} ) analysis_identifier = task_response.get("task", {}).get("analysisId") ## Now get all the analysis data for the given project and and filter by the analysisId project = cast(dict[str, Any], webhook_data.get("project")) - project_analysis_data = await self.get_analysis_by_project(component=project) - - for analysis_object in project_analysis_data: - if analysis_object.get("analysisId") == analysis_identifier: - return analysis_object + async for project_analysis_data in self.get_analysis_by_project( + component=project + ): + for analysis_object in project_analysis_data: + if analysis_object.get("analysisId") == analysis_identifier: + return analysis_object return {} ## when no data is found async def get_pull_requests_for_project( self, project_key: str ) -> list[dict[str, Any]]: logger.info(f"Fetching all pull requests in project : {project_key}") - response = await self.send_api_request( + response = await self._send_api_request( endpoint="project_pull_requests/list", query_params={"project": project_key}, ) @@ -458,7 +455,7 @@ async def get_pull_request_measures( self, project_key: str, pull_request_key: str ) -> list[dict[str, Any]]: logger.info(f"Fetching measures for pull request: {pull_request_key}") - response = await self.send_api_request( + response = await self._send_api_request( endpoint=Endpoints.MEASURES, query_params={ "component": project_key, @@ -495,23 +492,27 @@ async def get_measures_for_all_pull_requests( async def get_all_sonarqube_analyses( self, ) -> AsyncGenerator[list[dict[str, Any]], None]: - components = await self.get_components() - for component in components: - analysis_data = await self.get_measures_for_all_pull_requests( - project_key=component["key"] - ) - yield analysis_data + async for components in self.get_projects(enrich_project=False): + for analysis in await asyncio.gather( + *[ + self.get_measures_for_all_pull_requests( + project_key=component["key"] + ) + for component in components + ] + ): + yield analysis async def _get_all_portfolios(self) -> list[dict[str, Any]]: logger.info( f"Fetching all root portfolios in organization: {self.organization_id}" ) - response = await self.send_api_request(endpoint=Endpoints.PORTFOLIOS) + response = await self._send_api_request(endpoint=Endpoints.PORTFOLIOS) return response.get("views", []) async def _get_portfolio_details(self, portfolio_key: str) -> dict[str, Any]: logger.info(f"Fetching portfolio details for: {portfolio_key}") - response = await self.send_api_request( + response = await self._send_api_request( endpoint=Endpoints.PORTFOLIO_DETAILS, query_params={"key": portfolio_key}, ) @@ -528,25 +529,32 @@ def _extract_subportfolios(self, portfolio: dict[str, Any]) -> list[dict[str, An return all_portfolios async def get_all_portfolios(self) -> AsyncGenerator[list[dict[str, Any]], None]: - logger.info(f"Fetching all portfolios in organization: {self.organization_id}") - portfolios = await self._get_all_portfolios() - portfolio_keys_chunks = turn_sequence_to_chunks( - [portfolio["key"] for portfolio in portfolios], MAX_PORTFOLIO_REQUESTS - ) + if self.organization_id: + logger.info("Skipping portfolio ingestion since organization ID is absent") + else: + logger.info( + f"Fetching all portfolios in organization: {self.organization_id}" + ) + portfolios = await self._get_all_portfolios() + portfolio_keys_chunks = turn_sequence_to_chunks( + [portfolio["key"] for portfolio in portfolios], MAX_PORTFOLIO_REQUESTS + ) - for portfolio_keys in portfolio_keys_chunks: - try: - portfolios_data = await asyncio.gather( - *[ - self._get_portfolio_details(portfolio_key) - for portfolio_key in portfolio_keys - ] - ) - for portfolio_data in portfolios_data: - yield [portfolio_data] - yield self._extract_subportfolios(portfolio_data) - except (httpx.HTTPStatusError, httpx.HTTPError) as e: - logger.error(f"Error occurred while fetching portfolio details: {e}") + for portfolio_keys in portfolio_keys_chunks: + try: + portfolios_data = await asyncio.gather( + *[ + self._get_portfolio_details(portfolio_key) + for portfolio_key in portfolio_keys + ] + ) + for portfolio_data in portfolios_data: + yield [portfolio_data] + yield self._extract_subportfolios(portfolio_data) + except (httpx.HTTPStatusError, httpx.HTTPError) as e: + logger.error( + f"Error occurred while fetching portfolio details: {e}" + ) def sanity_check(self) -> None: try: @@ -569,55 +577,70 @@ def sanity_check(self) -> None: ) raise - async def get_or_create_webhook_url(self) -> None: + async def _create_webhook_payload_for_project( + self, project_key: str + ) -> dict[str, Any]: """ - Get or create webhook URL for projects + Create webhook for a project - :return: None + :param project_key: Project key + + :return: dict[str, Any] """ - logger.info(f"Subscribing to webhooks in organization: {self.organization_id}") - webhook_endpoint = Endpoints.WEBHOOKS - invoke_url = f"{self.app_host}/integration/webhook" - projects = await self.get_components() - - # Iterate over projects and add webhook - webhooks_to_create = [] - for project in projects: - project_key = project["key"] - logger.info(f"Fetching existing webhooks in project: {project_key}") - params = {} - if self.organization_id: - params["organization"] = self.organization_id - webhooks_response = await self.send_api_request( - endpoint=f"{webhook_endpoint}/list", - query_params={ - "project": project_key, - **params, - }, - ) + logger.info(f"Fetching existing webhooks in project: {project_key}") + params = {} + if self.organization_id: + params["organization"] = self.organization_id + + webhooks_response = await self._send_api_request( + endpoint=f"{Endpoints.WEBHOOKS}/list", + query_params={ + "project": project_key, + **params, + }, + ) - webhooks = webhooks_response.get("webhooks", []) - logger.info(webhooks) + webhooks = webhooks_response.get("webhooks", []) + logger.info(webhooks) - if any(webhook["url"] == invoke_url for webhook in webhooks): - logger.info(f"Webhook already exists in project: {project_key}") - continue + if any(webhook["url"] == self.webhook_invoke_url for webhook in webhooks): + logger.info(f"Webhook already exists in project: {project_key}") + return {} - params = {} - if self.organization_id: - params["organization"] = self.organization_id - webhooks_to_create.append( - { - "name": "Port Ocean Webhook", - "project": project_key, - **params, - } - ) + params = {} + if self.organization_id: + params["organization"] = self.organization_id + return { + "name": "Port Ocean Webhook", + "project": project_key, + **params, + } - for webhook in webhooks_to_create: - await self.send_api_request( - endpoint=f"{webhook_endpoint}/create", + async def _create_webhooks_for_projects( + self, webhook_payloads: list[dict[str, Any]] + ) -> None: + for webhook in webhook_payloads: + await self._send_api_request( + endpoint=f"{Endpoints.WEBHOOKS}/create", method="POST", - query_params={**webhook, "url": invoke_url}, + query_params={**webhook, "url": self.webhook_invoke_url}, ) logger.info(f"Webhook added to project: {webhook['project']}") + + async def get_or_create_webhook_url(self) -> None: + """ + Get or create webhook URL for projects + + :return: None + """ + logger.info(f"Subscribing to webhooks in organization: {self.organization_id}") + async for projects in self.get_projects(enrich_project=False): + webhooks_to_create = [] + for project in projects: + project_webhook_payload = ( + await self._create_webhook_payload_for_project(project["key"]) + ) + if project_webhook_payload: + webhooks_to_create.append(project_webhook_payload) + + await self._create_webhooks_for_projects(webhooks_to_create) diff --git a/integrations/sonarqube/examples/blueprints.json b/integrations/sonarqube/examples/blueprints.json index b71c1ddff5..69344bae91 100644 --- a/integrations/sonarqube/examples/blueprints.json +++ b/integrations/sonarqube/examples/blueprints.json @@ -1,57 +1,440 @@ -{ - "identifier": "sonarQubePortfolio", - "title": "SonarQube Portfolio", - "icon": "sonarqube", - "schema": { - "properties": { - "description": { - "type": "string", - "title": "Description" +[ + { + "identifier": "sonarQubeProject", + "title": "SonarQube Project", + "icon": "sonarqube", + "schema": { + "properties": { + "organization": { + "type": "string", + "title": "Organization", + "icon": "TwoUsers" + }, + "link": { + "type": "string", + "format": "url", + "title": "Link", + "icon": "Link" + }, + "lastAnalysisDate": { + "type": "string", + "format": "date-time", + "icon": "Clock", + "title": "Last Analysis Date" + }, + "qualityGateStatus": { + "title": "Quality Gate Status", + "type": "string", + "enum": [ + "OK", + "WARN", + "ERROR" + ], + "enumColors": { + "OK": "green", + "WARN": "yellow", + "ERROR": "red" + } + }, + "numberOfBugs": { + "type": "number", + "title": "Number Of Bugs" + }, + "numberOfCodeSmells": { + "type": "number", + "title": "Number Of CodeSmells" + }, + "numberOfVulnerabilities": { + "type": "number", + "title": "Number Of Vulnerabilities" + }, + "numberOfHotSpots": { + "type": "number", + "title": "Number Of HotSpots" + }, + "numberOfDuplications": { + "type": "number", + "title": "Number Of Duplications" + }, + "coverage": { + "type": "number", + "title": "Coverage" + }, + "mainBranch": { + "type": "string", + "icon": "Git", + "title": "Main Branch" + }, + "mainBranchLastAnalysisDate": { + "type": "string", + "format": "date-time", + "icon": "Clock", + "title": "Main Branch Last Analysis Date" + }, + "revision": { + "type": "string", + "title": "Revision" + }, + "managed": { + "type": "boolean", + "title": "Managed" + } }, - "originalKey": { - "type": "string", - "title": "Original Key" + "required": [] + }, + "mirrorProperties": {}, + "calculationProperties": {}, + "aggregationProperties": { + "criticalOpenIssues": { + "title": "Number Of Open Critical Issues", + "type": "number", + "target": "sonarQubeIssue", + "query": { + "combinator": "and", + "rules": [ + { + "property": "status", + "operator": "in", + "value": ["OPEN", "REOPENED"] + }, + { + "property": "severity", + "operator": "=", + "value": "CRITICAL" + } + ] + }, + "calculationSpec": { + "calculationBy": "entities", + "func": "count" + } }, - "visibility": { - "type": "string", - "title": "Visibility", - "enum": ["PUBLIC", "PRIVATE"], - "enumColors": { - "PUBLIC": "green", - "PRIVATE": "lightGray" + "numberOfOpenIssues": { + "title": "Number Of Open Issues", + "type": "number", + "target": "sonarQubeIssue", + "query": { + "combinator": "and", + "rules": [ + { + "property": "status", + "operator": "in", + "value": [ + "OPEN", + "REOPENED" + ] + } + ] + }, + "calculationSpec": { + "calculationBy": "entities", + "func": "count" + } + } + }, + "relations": {} + }, + { + "identifier": "sonarQubeProject", + "title": "SonarQube Project", + "icon": "sonarqube", + "schema": { + "properties": { + "organization": { + "type": "string", + "title": "Organization", + "icon": "TwoUsers" + }, + "link": { + "type": "string", + "format": "url", + "title": "Link", + "icon": "Link" + }, + "lastAnalysisDate": { + "type": "string", + "format": "date-time", + "icon": "Clock", + "title": "Last Analysis Date" + }, + "qualityGateStatus": { + "title": "Quality Gate Status", + "type": "string", + "enum": [ + "OK", + "WARN", + "ERROR" + ], + "enumColors": { + "OK": "green", + "WARN": "yellow", + "ERROR": "red" + } + }, + "numberOfBugs": { + "type": "number", + "title": "Number Of Bugs" + }, + "numberOfCodeSmells": { + "type": "number", + "title": "Number Of CodeSmells" + }, + "numberOfVulnerabilities": { + "type": "number", + "title": "Number Of Vulnerabilities" + }, + "numberOfHotSpots": { + "type": "number", + "title": "Number Of HotSpots" + }, + "numberOfDuplications": { + "type": "number", + "title": "Number Of Duplications" + }, + "coverage": { + "type": "number", + "title": "Coverage" + }, + "mainBranch": { + "type": "string", + "icon": "Git", + "title": "Main Branch" + }, + "tags": { + "type": "array", + "title": "Tags" } }, - "selectionMode": { - "type": "string", - "title": "Selection Mode", - "enum": ["AUTO", "MANUAL", "NONE"], - "enumColors": { - "AUTO": "blue", - "MANUAL": "green", - "NONE": "lightGray" + "required": [] + }, + "mirrorProperties": {}, + "calculationProperties": {}, + "aggregationProperties": { + "criticalOpenIssues": { + "title": "Number Of Open Critical Issues", + "type": "number", + "target": "sonarQubeIssue", + "query": { + "combinator": "and", + "rules": [ + { + "property": "status", + "operator": "in", + "value": ["OPEN", "REOPENED"] + }, + { + "property": "severity", + "operator": "=", + "value": "CRITICAL" + } + ] + }, + "calculationSpec": { + "calculationBy": "entities", + "func": "count" } }, - "disabled": { - "type": "boolean", - "title": "Disabled" + "numberOfOpenIssues": { + "title": "Number Of Open Issues", + "type": "number", + "target": "sonarQubeIssue", + "query": { + "combinator": "and", + "rules": [ + { + "property": "status", + "operator": "in", + "value": [ + "OPEN", + "REOPENED" + ] + } + ] + }, + "calculationSpec": { + "calculationBy": "entities", + "func": "count" + } + } + }, + "relations": {} + }, + { + "identifier": "sonarQubeAnalysis", + "title": "SonarQube Analysis", + "icon": "sonarqube", + "schema": { + "properties": { + "branch": { + "type": "string", + "title": "Branch", + "icon": "GitVersion" + }, + "fixedIssues": { + "type": "number", + "title": "Fixed Issues" + }, + "newIssues": { + "type": "number", + "title": "New Issues" + }, + "coverage": { + "title": "Coverage", + "type": "number" + }, + "duplications": { + "type": "number", + "title": "Duplications" + }, + "createdAt": { + "type": "string", + "format": "date-time", + "title": "Created At" + } + } + }, + "relations": { + "sonarQubeProject": { + "target": "sonarQubeProject", + "required": false, + "title": "SonarQube Project", + "many": false + } + } + }, + { + "identifier": "sonarQubeIssue", + "title": "SonarQube Issue", + "icon": "sonarqube", + "schema": { + "properties": { + "type": { + "type": "string", + "title": "Type", + "enum": [ + "CODE_SMELL", + "BUG", + "VULNERABILITY" + ] + }, + "severity": { + "type": "string", + "title": "Severity", + "enum": [ + "MAJOR", + "INFO", + "MINOR", + "CRITICAL", + "BLOCKER" + ], + "enumColors": { + "MAJOR": "orange", + "INFO": "green", + "CRITICAL": "red", + "BLOCKER": "red", + "MINOR": "yellow" + } + }, + "link": { + "type": "string", + "format": "url", + "icon": "Link", + "title": "Link" + }, + "status": { + "type": "string", + "title": "Status", + "enum": [ + "OPEN", + "CLOSED", + "RESOLVED", + "REOPENED", + "CONFIRMED" + ] + }, + "assignees": { + "title": "Assignees", + "type": "string", + "icon": "TwoUsers" + }, + "tags": { + "type": "array", + "title": "Tags" + }, + "createdAt": { + "type": "string", + "format": "date-time", + "title": "Created At" + } + } + }, + "relations": { + "sonarQubeProject": { + "target": "sonarQubeProject", + "required": false, + "title": "SonarQube Project", + "many": false } } }, - "mirrorProperties": {}, - "calculationProperties": {}, - "aggregationProperties": {}, - "relations": { - "subPortfolios": { - "target": "sonarQubePortfolio", - "required": false, - "title": "Sub Portfolios", - "many": true + { + "identifier": "sonarQubePortfolio", + "title": "SonarQube Portfolio", + "icon": "sonarqube", + "schema": { + "properties": { + "description": { + "type": "string", + "title": "Description" + }, + "visibility": { + "type": "string", + "title": "Visibility", + "enum": [ + "PUBLIC", + "PRIVATE" + ], + "enumColors": { + "PUBLIC": "green", + "PRIVATE": "lightGray" + } + }, + "selectionMode": { + "type": "string", + "title": "Selection Mode", + "enum": [ + "AUTO", + "MANUAL", + "NONE" + ], + "enumColors": { + "AUTO": "blue", + "MANUAL": "green", + "NONE": "lightGray" + } + }, + "disabled": { + "type": "boolean", + "title": "Disabled" + } + } }, - "referencedBy": { - "target": "sonarQubePortfolio", - "required": false, - "title": "Referenced By", - "many": true + "mirrorProperties": {}, + "calculationProperties": {}, + "aggregationProperties": {}, + "relations": { + "subPortfolios": { + "target": "sonarQubePortfolio", + "required": false, + "title": "Sub Portfolios", + "many": true + }, + "referencedBy": { + "target": "sonarQubePortfolio", + "required": false, + "title": "Referenced By", + "many": true + } } } -} +] diff --git a/integrations/sonarqube/examples/mappings.yaml b/integrations/sonarqube/examples/mappings.yaml index 3959abb982..083ef188cd 100644 --- a/integrations/sonarqube/examples/mappings.yaml +++ b/integrations/sonarqube/examples/mappings.yaml @@ -1,9 +1,158 @@ createMissingRelatedEntities: true deleteDependentEntities: true resources: + - kind: projects_ga + selector: + query: 'true' + apiFilters: + qualifier: + - TRK + metrics: + - code_smells + - coverage + - bugs + - vulnerabilities + - duplicated_files + - security_hotspots + - new_violations + - new_coverage + - new_duplicated_lines_density + port: + entity: + mappings: + blueprint: '"sonarQubeProject"' + identifier: .key + title: .name + properties: + organization: .organization + link: .__link + qualityGateStatus: .__branch.status.qualityGateStatus + lastAnalysisDate: .analysisDate + numberOfBugs: .__measures[]? | select(.metric == "bugs") | .value + numberOfCodeSmells: .__measures[]? | select(.metric == "code_smells") | .value + numberOfVulnerabilities: .__measures[]? | select(.metric == "vulnerabilities") | .value + numberOfHotSpots: .__measures[]? | select(.metric == "security_hotspots") | .value + numberOfDuplications: .__measures[]? | select(.metric == "duplicated_files") | .value + coverage: .__measures[]? | select(.metric == "coverage") | .value + mainBranch: .__branch.name + mainBranchLastAnalysisDate: .__branch.analysisDate + revision: .revision + managed: .managed + - kind: projects + selector: + query: 'true' + apiFilters: + filter: + qualifier: TRK + metrics: + - code_smells + - coverage + - bugs + - vulnerabilities + - duplicated_files + - security_hotspots + - new_violations + - new_coverage + - new_duplicated_lines_density + port: + entity: + mappings: + blueprint: '"sonarQubeProject"' + identifier: .key + title: .name + properties: + organization: .organization + link: .__link + qualityGateStatus: .__branch.status.qualityGateStatus + lastAnalysisDate: .__branch.analysisDate + numberOfBugs: .__measures[]? | select(.metric == "bugs") | .value + numberOfCodeSmells: .__measures[]? | select(.metric == "code_smells") | .value + numberOfVulnerabilities: .__measures[]? | select(.metric == "vulnerabilities") | .value + numberOfHotSpots: .__measures[]? | select(.metric == "security_hotspots") | .value + numberOfDuplications: .__measures[]? | select(.metric == "duplicated_files") | .value + coverage: .__measures[]? | select(.metric == "coverage") | .value + mainBranch: .__branch.name + tags: .tags + - kind: analysis + selector: + query: 'true' + port: + entity: + mappings: + blueprint: '"sonarQubeAnalysis"' + identifier: .analysisId + title: .__commit.message // .analysisId + properties: + branch: .__branchName + fixedIssues: .measures.violations_fixed + newIssues: .measures.violations_added + coverage: .measures.coverage_change + duplications: .measures.duplicated_lines_density_change + createdAt: .__analysisDate + relations: + sonarQubeProject: .__project + - kind: onprem_analysis + selector: + query: 'true' + port: + entity: + mappings: + blueprint: '"sonarQubeAnalysis"' + identifier: .__project + "-" + .key + title: .title + properties: + branch: .branch + newIssues: .__measures[]? | select(.metric == "new_violations") | .period.value + coverage: .__measures[]? | select(.metric == "new_coverage") | .period.value + duplications: .__measures[]? | select(.metric == "new_duplicated_lines_density") | .period.value + createdAt: .analysisDate + relations: + sonarQubeProject: .__project + - kind: issues + selector: + query: 'true' + apiFilters: + resolved: 'false' + projectApiFilters: + filter: + qualifier: TRK + port: + entity: + mappings: + blueprint: '"sonarQubeIssue"' + identifier: .key + title: .message + properties: + type: .type + severity: .severity + link: .__link + status: .status + assignees: .assignee + tags: .tags + createdAt: .creationDate + relations: + sonarQubeProject: .project + - kind: saas_analysis + selector: + query: 'true' + port: + entity: + mappings: + blueprint: '"sonarQubeAnalysis"' + identifier: .analysisId + title: .__commit.message // .analysisId + properties: + branch: .__branchName + fixedIssues: .measures.violations_fixed + newIssues: .measures.violations_added + coverage: .measures.coverage_change + duplications: .measures.duplicated_lines_density_change + createdAt: .__analysisDate + relations: + sonarQubeProject: .__project - kind: portfolios selector: - query: "true" + query: 'true' port: entity: mappings: @@ -12,10 +161,9 @@ resources: title: .name properties: description: .description - originalKey: .originalKey - visibility: .visibility | ascii_upcase - selectionMode: .selectionMode | ascii_upcase + visibility: if .visibility then .visibility | ascii_upcase else null end + selectionMode: if .selectionMode then .selectionMode | ascii_upcase else null end disabled: .disabled relations: - subPortfolios: .subViews | map(select((.qualifier | IN("VW", "SVW")))) | .[].key - referencedBy: .referencedBy | map(select((.qualifier | IN("VW", "SVW")))) | .[].key + subPortfolios: .subViews | map(select(.qualifier as $qualifier | ["VW", "SVW"] | contains([$qualifier])) | .key) + referencedBy: .referencedBy | map(select(.qualifier as $qualifier | ["VW", "SVW"] | contains([$qualifier])) | .key) diff --git a/integrations/sonarqube/examples/project.entity.json b/integrations/sonarqube/examples/project.entity.json new file mode 100644 index 0000000000..52ae086867 --- /dev/null +++ b/integrations/sonarqube/examples/project.entity.json @@ -0,0 +1,19 @@ +{ + "identifier": "port-labs_Port_port-api", + "title": "Port API", + "team": [], + "properties": { + "organization": "port-labs", + "link": "https://sonarcloud.io/project/overview?id=port-labs_Port_port-api", + "qualityGateStatus": "ERROR", + "numberOfBugs": 20, + "numberOfCodeSmells": 262, + "numberOfVulnerabilities": 0, + "numberOfHotSpots": 3, + "numberOfDuplications": 18, + "coverage": 0, + "mainBranch": "main" + }, + "relations": {}, + "icon": "sonarqube" + } diff --git a/integrations/sonarqube/examples/project.response.json b/integrations/sonarqube/examples/project.response.json new file mode 100644 index 0000000000..743dbcc425 --- /dev/null +++ b/integrations/sonarqube/examples/project.response.json @@ -0,0 +1,9 @@ +{ + "key": "project-key-1", + "name": "Project Name 1", + "qualifier": "TRK", + "visibility": "public", + "lastAnalysisDate": "2017-03-01T11:39:03+0300", + "revision": "cfb82f55c6ef32e61828c4cb3db2da12795fd767", + "managed": false +} diff --git a/integrations/sonarqube/integration.py b/integrations/sonarqube/integration.py index ba1fb15e77..688cb6728e 100644 --- a/integrations/sonarqube/integration.py +++ b/integrations/sonarqube/integration.py @@ -1,5 +1,5 @@ from abc import abstractmethod -from typing import Any, Literal, Union +from typing import Annotated, Any, Literal, Union from port_ocean.core.handlers.port_app_config.api import APIPortAppConfig from port_ocean.core.handlers.port_app_config.models import ( @@ -14,6 +14,7 @@ class ObjectKind: PROJECTS = "projects" + PROJECTS_GA = "projects_ga" ISSUES = "issues" ANALYSIS = "analysis" SASS_ANALYSIS = "saas_analysis" @@ -44,7 +45,6 @@ class SonarQubeComponentSearchFilter(BaseModel): def generate_search_filters(self) -> str: params = [] for field, value in self.dict(exclude_none=True).items(): - if field == "metrics": for metric_filter in value: for metric_key, metric_value in metric_filter.items(): @@ -79,6 +79,31 @@ def generate_request_params(self) -> dict[str, Any]: return value +class SonarQubeGAProjectAPIFilter(BaseSonarQubeApiFilter): + analyzed_before: str | None = Field( + alias="analyzedBefore", + description="To retrieve projects analyzed before the given date", + ) + on_provisioned_only: bool | None = Field( + alias="onProvisionedOnly", + description="To retrieve projects on provisioned only", + ) + projects: list[str] | None = Field(description="List of projects") + qualifiers: list[Literal["TRK", "APP"]] | None = Field( + description="List of qualifiers", alias="qualifier" + ) + + def generate_request_params(self) -> dict[str, Any]: + value = self.dict(exclude_none=True) + if self.projects: + value["projects"] = ",".join(self.projects) + + if self.qualifiers: + value["qualifiers"] = ",".join(self.qualifiers) + + return value + + class SonarQubeIssueApiFilter(BaseSonarQubeApiFilter): assigned: Literal["yes", "no", "true", "false"] | None = Field( description="To retrieve assigned or unassigned issues" @@ -161,56 +186,94 @@ def generate_request_params(self) -> dict[str, Any]: class CustomResourceConfig(ResourceConfig): selector: CustomSelector + kind: Literal[ + "analysis", + "onprem_analysis", + "saas_analysis", + "portfolios", + ] + + +class SonarQubeComponentProjectSelector(SelectorWithApiFilters): + @staticmethod + def default_metrics() -> list[str]: + return [ + "code_smells", + "coverage", + "bugs", + "vulnerabilities", + "duplicated_files", + "security_hotspots", + "new_violations", + "new_coverage", + "new_duplicated_lines_density", + ] + + api_filters: SonarQubeProjectApiFilter | None = Field(alias="apiFilters") + metrics: list[str] = Field( + description="List of metric keys", default=default_metrics() + ) class SonarQubeProjectResourceConfig(CustomResourceConfig): - class SonarQubeProjectSelector(SelectorWithApiFilters): - - @staticmethod - def default_metrics() -> list[str]: - return [ - "code_smells", - "coverage", - "bugs", - "vulnerabilities", - "duplicated_files", - "security_hotspots", - "new_violations", - "new_coverage", - "new_duplicated_lines_density", - ] + kind: Literal["projects"] # type: ignore + selector: SonarQubeComponentProjectSelector + + +class SonarQubeGAProjectSelector(CustomSelector): + @staticmethod + def default_metrics() -> list[str]: + return [ + "code_smells", + "coverage", + "bugs", + "vulnerabilities", + "duplicated_files", + "security_hotspots", + "new_violations", + "new_coverage", + "new_duplicated_lines_density", + ] + + api_filters: SonarQubeGAProjectAPIFilter | None = Field(alias="apiFilters") + + metrics: list[str] = Field( + description="List of metric keys", default=default_metrics() + ) + - api_filters: SonarQubeProjectApiFilter | None = Field(alias="apiFilters") - metrics: list[str] = Field( - description="List of metric keys", default=default_metrics() - ) +class SonarQubeGAProjectResourceConfig(CustomResourceConfig): - kind: Literal["projects"] - selector: SonarQubeProjectSelector + kind: Literal["projects_ga"] # type: ignore + selector: SonarQubeGAProjectSelector + + +class SonarQubeIssueSelector(SelectorWithApiFilters): + api_filters: SonarQubeIssueApiFilter | None = Field(alias="apiFilters") + project_api_filters: SonarQubeGAProjectAPIFilter | None = Field( + alias="projectApiFilters", + description="Allows users to control which projects to query the issues for", + ) class SonarQubeIssueResourceConfig(CustomResourceConfig): - class SonarQubeIssueSelector(SelectorWithApiFilters): - api_filters: SonarQubeIssueApiFilter | None = Field(alias="apiFilters") - project_api_filters: SonarQubeProjectApiFilter | None = Field( - alias="projectApiFilters", - description="Allows users to control which projects to query the issues for", - ) - - kind: Literal["issues"] + kind: Literal["issues"] # type: ignore selector: SonarQubeIssueSelector +SonarResourcesConfig = Annotated[ + Union[ + SonarQubeProjectResourceConfig, + SonarQubeIssueResourceConfig, + SonarQubeGAProjectResourceConfig, + CustomResourceConfig, + ], + Field(discriminator="kind"), +] + + class SonarQubePortAppConfig(PortAppConfig): - resources: list[ - Union[ - SonarQubeProjectResourceConfig, - SonarQubeIssueResourceConfig, - CustomResourceConfig, - ] - ] = Field( - default_factory=list - ) # type: ignore + resources: list[SonarResourcesConfig] = Field(default_factory=list) # type: ignore class SonarQubeIntegration(BaseIntegration): diff --git a/integrations/sonarqube/main.py b/integrations/sonarqube/main.py index 0195e5d9e2..68b9fb5fcb 100644 --- a/integrations/sonarqube/main.py +++ b/integrations/sonarqube/main.py @@ -1,11 +1,18 @@ -from typing import Any +from typing import Any, cast from loguru import logger +from port_ocean.context.event import event from port_ocean.context.ocean import ocean from port_ocean.core.ocean_types import ASYNC_GENERATOR_RESYNC_TYPE from client import SonarQubeClient -from integration import ObjectKind +from integration import ( + ObjectKind, + SonarQubeGAProjectResourceConfig, + SonarQubeIssueResourceConfig, + SonarQubeProjectResourceConfig, +) +from utils import produce_component_params def init_sonar_client() -> SonarQubeClient: @@ -23,10 +30,18 @@ def init_sonar_client() -> SonarQubeClient: @ocean.on_resync(ObjectKind.PROJECTS) async def on_project_resync(kind: str) -> ASYNC_GENERATOR_RESYNC_TYPE: - logger.info(f"Listing Sonarqube resource: {kind}") + logger.warning( + "The `project` resource is deprecated. Please use `projects_ga` instead." + ) + selector = cast(SonarQubeProjectResourceConfig, event.resource_config).selector + sonar_client.metrics = selector.metrics + + component_params = produce_component_params(sonar_client, selector) + fetched_projects = False - async for project_list in sonar_client.get_all_projects(): - yield project_list + async for projects in sonar_client.get_components(query_params=component_params): + logger.info(f"Received project batch of size: {len(projects)}") + yield projects fetched_projects = True if not fetched_projects: @@ -36,46 +51,59 @@ async def on_project_resync(kind: str) -> ASYNC_GENERATOR_RESYNC_TYPE: ) +@ocean.on_resync(ObjectKind.PROJECTS_GA) +async def on_ga_project_resync(kind: str) -> ASYNC_GENERATOR_RESYNC_TYPE: + selector = cast(SonarQubeGAProjectResourceConfig, event.resource_config).selector + sonar_client.metrics = selector.metrics + params = {} + if api_filters := selector.api_filters: + params = api_filters.generate_request_params() + + async for projects in sonar_client.get_projects(params): + logger.info(f"Received project batch of size: {len(projects)}") + yield projects + + @ocean.on_resync(ObjectKind.ISSUES) async def on_issues_resync(kind: str) -> ASYNC_GENERATOR_RESYNC_TYPE: - fetched_issues = False - async for issues_list in sonar_client.get_all_issues(): + selector = cast(SonarQubeIssueResourceConfig, event.resource_config).selector + query_params = selector.generate_request_params() + project_query_params = ( + selector.project_api_filters + and selector.project_api_filters.generate_request_params() + ) or {} + + async for issues_list in sonar_client.get_all_issues( + query_params=query_params, + project_query_params=project_query_params, + ): + logger.info(f"Received issues batch of size: {len(issues_list)}") yield issues_list - fetched_issues = True - - if not fetched_issues: - logger.error("No issues found in Sonarqube") - raise RuntimeError( - "No issues found in Sonarqube, failing the resync to avoid data loss" - ) @ocean.on_resync(ObjectKind.ANALYSIS) @ocean.on_resync(ObjectKind.SASS_ANALYSIS) async def on_saas_analysis_resync(kind: str) -> ASYNC_GENERATOR_RESYNC_TYPE: if not ocean.integration_config["sonar_is_on_premise"]: - fetched_analyses = False + logger.info("Sonar is not on-premise, processing SonarCloud on saas analysis") async for analyses_list in sonar_client.get_all_sonarcloud_analyses(): + logger.info(f"Received analysis batch of size: {len(analyses_list)}") yield analyses_list - fetched_analyses = True - - if not fetched_analyses: - logger.error("No analysis found in Sonarqube") - raise RuntimeError( - "No analysis found in Sonarqube, failing the resync to avoid data loss" - ) @ocean.on_resync(ObjectKind.ONPREM_ANALYSIS) async def on_onprem_analysis_resync(kind: str) -> ASYNC_GENERATOR_RESYNC_TYPE: if ocean.integration_config["sonar_is_on_premise"]: + logger.info("Sonar is on-premise, processing on-premise SonarQube analysis") async for analyses_list in sonar_client.get_all_sonarqube_analyses(): + logger.info(f"Received analysis batch of size: {len(analyses_list)}") yield analyses_list @ocean.on_resync(ObjectKind.PORTFOLIOS) async def on_portfolio_resync(kind: str) -> ASYNC_GENERATOR_RESYNC_TYPE: async for portfolio_list in sonar_client.get_all_portfolios(): + logger.info(f"Received portfolio batch of size: {len(portfolio_list)}") yield portfolio_list @@ -89,9 +117,10 @@ async def handle_sonarqube_webhook(webhook_data: dict[str, Any]) -> None: webhook_data.get("project", {}) ) ## making sure we're getting the right project details project_data = await sonar_client.get_single_project(project) - issues_data = await sonar_client.get_issues_by_component(project) await ocean.register_raw(ObjectKind.PROJECTS, [project_data]) - await ocean.register_raw(ObjectKind.ISSUES, issues_data) + await ocean.register_raw(ObjectKind.PROJECTS_GA, [project_data]) + async for issues_data in sonar_client.get_issues_by_component(project): + await ocean.register_raw(ObjectKind.ISSUES, issues_data) if ocean.integration_config["sonar_is_on_premise"]: onprem_analysis_data = await sonar_client.get_measures_for_all_pull_requests( diff --git a/integrations/sonarqube/pyproject.toml b/integrations/sonarqube/pyproject.toml index af849112f7..2c8400a55c 100644 --- a/integrations/sonarqube/pyproject.toml +++ b/integrations/sonarqube/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "sonarqube" -version = "0.1.121" +version = "0.1.122" description = "SonarQube projects and code quality analysis integration" authors = ["Port Team "] diff --git a/integrations/sonarqube/tests/conftest.py b/integrations/sonarqube/tests/conftest.py new file mode 100644 index 0000000000..665250f54f --- /dev/null +++ b/integrations/sonarqube/tests/conftest.py @@ -0,0 +1,103 @@ +import os +from typing import Any, Generator +from unittest.mock import MagicMock, patch + +import pytest +from port_ocean import Ocean +from port_ocean.context.event import EventContext +from port_ocean.context.ocean import initialize_port_ocean_context +from port_ocean.exceptions.context import PortOceanContextAlreadyInitializedError +from port_ocean.tests.helpers.ocean_app import get_integration_ocean_app + +from integration import SonarQubePortAppConfig + +from .fixtures import ANALYSIS, COMPONENT_PROJECTS, ISSUES, PORTFOLIOS, PURE_PROJECTS + +INTEGRATION_PATH = os.path.abspath(os.path.join(os.path.dirname(__file__), "../")) + + +@pytest.fixture +def mock_ocean_context() -> None: + """Fixture to mock the Ocean context initialization.""" + try: + mock_ocean_app = MagicMock() + mock_ocean_app.config.integration.config = { + "sonar_api_token": "token", + "sonar_url": "https://sonarqube.com", + "sonar_organization_id": "organization_id", + "sonar_is_on_premise": False, + } + mock_ocean_app.integration_router = MagicMock() + mock_ocean_app.port_client = MagicMock() + initialize_port_ocean_context(mock_ocean_app) + except PortOceanContextAlreadyInitializedError: + pass + + +@pytest.fixture +def mock_event_context() -> Generator[MagicMock, None, None]: + """Fixture to mock the event context.""" + mock_event = MagicMock(spec=EventContext) + mock_event.event_type = "test_event" + mock_event.trigger_type = "manual" + mock_event.attributes = {} + mock_event._aborted = False + mock_event._port_app_config = SonarQubePortAppConfig + + with patch("port_ocean.context.event.event", mock_event): + yield mock_event + + +def app() -> Ocean: + config = { + "event_listener": {"type": "POLLING"}, + "integration": { + "config": { + "sonar_api_token": "token", + "sonar_url": "https://sonarqube.com", + "sonar_organization_id": "organization_id", + "sonar_is_on_premise": False, + } + }, + "port": { + "client_id": "bla", + "client_secret": "bla", + }, + } + application = get_integration_ocean_app(INTEGRATION_PATH, config) + return application + + +@pytest.fixture(scope="session") +def ocean_app() -> Ocean: + return app() + + +@pytest.fixture(scope="session") +def integration_path() -> str: + return INTEGRATION_PATH + + +@pytest.fixture(scope="session") +def projects() -> list[dict[str, Any]]: + return PURE_PROJECTS + + +@pytest.fixture(scope="session") +def component_projects() -> list[dict[str, Any]]: + return COMPONENT_PROJECTS + + +@pytest.fixture(scope="session") +def issues() -> list[dict[str, Any]]: + return ISSUES + + +@pytest.fixture(scope="session") +def portfolios() -> list[dict[str, Any]]: + return PORTFOLIOS + + +@pytest.fixture(scope="session") +def analysis() -> list[dict[str, Any]]: + return ANALYSIS diff --git a/integrations/sonarqube/tests/fixtures.py b/integrations/sonarqube/tests/fixtures.py new file mode 100644 index 0000000000..f8b5730e66 --- /dev/null +++ b/integrations/sonarqube/tests/fixtures.py @@ -0,0 +1,244 @@ +from typing import Any + +PURE_PROJECTS: list[dict[str, Any]] = [ + { + "key": "project-key-1", + "name": "Project Name 1", + "qualifier": "TRK", + "visibility": "public", + "lastAnalysisDate": "2017-03-01T11:39:03+0300", + "revision": "cfb82f55c6ef32e61828c4cb3db2da12795fd767", + "managed": False, + }, + { + "key": "project-key-2", + "name": "Project Name 2", + "qualifier": "TRK", + "visibility": "private", + "lastAnalysisDate": "2017-03-02T15:21:47+0300", + "revision": "7be96a94ac0c95a61ee6ee0ef9c6f808d386a355", + "managed": False, + }, +] + + +COMPONENT_PROJECTS: list[dict[str, Any]] = [ + { + "key": "project-key-1", + "name": "My Project 1", + "qualifier": "TRK", + "isFavorite": True, + "tags": ["finance", "java"], + "visibility": "public", + "isAiCodeAssured": False, + "isAiCodeFixEnabled": False, + }, + { + "key": "project-key-2", + "name": "My Project 2", + "qualifier": "TRK", + "isFavorite": False, + "tags": [], + "visibility": "public", + "isAiCodeAssured": False, + "isAiCodeFixEnabled": False, + }, +] + +ISSUES: list[dict[str, Any]] = [ + { + "key": "01fc972e-2a3c-433e-bcae-0bd7f88f5123", + "component": "com.github.kevinsawicki:http-request:com.github.kevinsawicki.http.HttpRequest", + "project": "com.github.kevinsawicki:http-request", + "rule": "java:S1144", + "cleanCodeAttribute": "CLEAR", + "cleanCodeAttributeCategory": "INTENTIONAL", + "issueStatus": "ACCEPTED", + "prioritizedRule": False, + "impacts": [{"softwareQuality": "SECURITY", "severity": "HIGH"}], + "message": 'Remove this unused private "getKee" method.', + "messageFormattings": [{"start": 0, "end": 4, "type": "CODE"}], + "line": 81, + "hash": "a227e508d6646b55a086ee11d63b21e9", + "author": "Developer 1", + "effort": "2h1min", + "creationDate": "2013-05-13T17:55:39+0200", + "updateDate": "2013-05-13T17:55:39+0200", + "tags": ["bug"], + "comments": [ + { + "key": "7d7c56f5-7b5a-41b9-87f8-36fa70caa5ba", + "login": "john.smith", + "htmlText": "Must be "public"!", + "markdown": 'Must be "public"!', + "updatable": False, + "createdAt": "2013-05-13T18:08:34+0200", + } + ], + "attr": {"jira-issue-key": "SONAR-1234"}, + "transitions": ["reopen"], + "actions": ["comment"], + "textRange": {"startLine": 2, "endLine": 2, "startOffset": 0, "endOffset": 204}, + "flows": [ + { + "locations": [ + { + "textRange": { + "startLine": 16, + "endLine": 16, + "startOffset": 0, + "endOffset": 30, + }, + "msg": "Expected position: 5", + "msgFormattings": [{"start": 0, "end": 4, "type": "CODE"}], + } + ] + }, + { + "locations": [ + { + "textRange": { + "startLine": 15, + "endLine": 15, + "startOffset": 0, + "endOffset": 37, + }, + "msg": "Expected position: 6", + "msgFormattings": [], + } + ] + }, + ], + "quickFixAvailable": False, + "ruleDescriptionContextKey": "spring", + "codeVariants": ["windows", "linux"], + }, + { + "key": "01fc972e-2a3c-433e-bcae-0bd7f88f5123", + "component": "com.github.kevinsawicki:http-request:com.github.kevinsawicki.http.HttpRequest", + "project": "com.github.kevinsawicki:http-request", + "rule": "java:S1144", + "cleanCodeAttribute": "CLEAR", + "cleanCodeAttributeCategory": "INTENTIONAL", + "issueStatus": "ACCEPTED", + "prioritizedRule": False, + "impacts": [{"softwareQuality": "SECURITY", "severity": "HIGH"}], + "message": 'Remove this unused private "getKee" method.', + "messageFormattings": [{"start": 0, "end": 4, "type": "CODE"}], + "line": 81, + "hash": "a227e508d6646b55a086ee11d63b21e9", + "author": "Developer 1", + "effort": "2h1min", + "creationDate": "2013-05-13T17:55:39+0200", + "updateDate": "2013-05-13T17:55:39+0200", + "tags": ["bug"], + "comments": [ + { + "key": "7d7c56f5-7b5a-41b9-87f8-36fa70caa5ba", + "login": "john.smith", + "htmlText": "Must be "public"!", + "markdown": 'Must be "public"!', + "updatable": False, + "createdAt": "2013-05-13T18:08:34+0200", + } + ], + "attr": {"jira-issue-key": "SONAR-1234"}, + "transitions": ["reopen"], + "actions": ["comment"], + "textRange": {"startLine": 2, "endLine": 2, "startOffset": 0, "endOffset": 204}, + "flows": [ + { + "locations": [ + { + "textRange": { + "startLine": 16, + "endLine": 16, + "startOffset": 0, + "endOffset": 30, + }, + "msg": "Expected position: 5", + "msgFormattings": [{"start": 0, "end": 4, "type": "CODE"}], + } + ] + }, + { + "locations": [ + { + "textRange": { + "startLine": 15, + "endLine": 15, + "startOffset": 0, + "endOffset": 37, + }, + "msg": "Expected position: 6", + "msgFormattings": [], + } + ] + }, + ], + "quickFixAvailable": False, + "ruleDescriptionContextKey": "spring", + "codeVariants": ["windows", "linux"], + }, +] + +PORTFOLIOS: list[dict[str, Any]] = [ + { + "key": "apache-jakarta-commons", + "name": "Apache Jakarta Commons", + "qualifier": "VW", + "visibility": "public", + }, + { + "key": "Languages", + "name": "Languages", + "qualifier": "VW", + "visibility": "private", + }, +] + + +ANALYSIS: list[dict[str, Any]] = [ + { + "id": "AYhSC2-LY0CHkWJxvNA9", + "type": "REPORT", + "componentId": "AYhNmk00XxCL_lBVBziT", + "componentKey": "sonarsource_test_AYhCAUXoEy1XQQcbVndf", + "componentName": "test-scanner-maven", + "componentQualifier": "TRK", + "analysisId": "AYhSC3WDE6ILQDIMAPIp", + "status": "SUCCESS", + "submittedAt": "2023-05-25T10:34:21+0200", + "submitterLogin": "admin", + "startedAt": "2023-05-25T10:34:22+0200", + "executedAt": "2023-05-25T10:34:25+0200", + "executionTimeMs": 2840, + "hasScannerContext": True, + "warningCount": 2, + "warnings": [ + "The properties 'sonar.login' and 'sonar.password' are deprecated and will be removed in the future. Please pass a token with the 'sonar.token' property instead.", + 'Missing blame information for 2 files. This may lead to some features not working correctly. Please check the analysis logs and refer to the documentation.', + ], + }, + { + "id": "AYhSC2-LY0CHkWJxvNA9", + "type": "REPORT", + "componentId": "AYhNmk00XxCL_lBVBziT", + "componentKey": "sonarsource_test_AYhCAUXoEy1XQQcbVndf", + "componentName": "test-scanner-maven", + "componentQualifier": "TRK", + "analysisId": "AYhSC3WDE6ILQDIMAPIp", + "status": "SUCCESS", + "submittedAt": "2023-05-25T10:34:21+0200", + "submitterLogin": "admin", + "startedAt": "2023-05-25T10:34:22+0200", + "executedAt": "2023-05-25T10:34:25+0200", + "executionTimeMs": 2840, + "hasScannerContext": True, + "warningCount": 2, + "warnings": [ + "The properties 'sonar.login' and 'sonar.password' are deprecated and will be removed in the future. Please pass a token with the 'sonar.token' property instead.", + 'Missing blame information for 2 files. This may lead to some features not working correctly. Please check the analysis logs and refer to the documentation.', + ], + }, +] diff --git a/integrations/sonarqube/tests/test_client.py b/integrations/sonarqube/tests/test_client.py index fa62a5a31b..95aa55339b 100644 --- a/integrations/sonarqube/tests/test_client.py +++ b/integrations/sonarqube/tests/test_client.py @@ -1,8 +1,44 @@ -from typing import Any +from typing import Any, TypedDict +from unittest.mock import AsyncMock, MagicMock, patch +import httpx import pytest +from loguru import logger +from port_ocean.context.event import event_context -from client import turn_sequence_to_chunks +from client import SonarQubeClient, turn_sequence_to_chunks + +from .fixtures import PURE_PROJECTS + + +class HttpxResponses(TypedDict): + status_code: int + json: dict[str, Any] + + +class MockHttpxClient: + def __init__(self, responses: list[HttpxResponses] = []) -> None: + self.responses = [ + httpx.Response( + status_code=response["status_code"], + json=response["json"], + request=httpx.Request("GET", "https://myorg.atlassian.net"), + ) + for response in responses + ] + self._current_response_index = 0 + + async def request( + self, *args: tuple[Any], **kwargs: dict[str, Any] + ) -> httpx.Response: + if self._current_response_index == len(self.responses): + logger.error(f"Response index {self._current_response_index}") + logger.error(f"Responses length: {len(self.responses)}") + raise httpx.HTTPError("No more responses") + + response = self.responses[self._current_response_index] + self._current_response_index += 1 + return response @pytest.mark.parametrize( @@ -18,3 +54,828 @@ def test_turn_sequence_to_chunks( input: list[Any], output: list[list[Any]], chunk_size: int ) -> None: assert list(turn_sequence_to_chunks(input, chunk_size)) == output + + +@patch("client.base64.b64encode", return_value=b"token") +def test_sonarqube_client_will_produce_right_auth_header( + _mock_b64encode: Any, + mock_ocean_context: Any, + monkeypatch: Any, +) -> None: + values = [ + ( + SonarQubeClient( + "https://sonarqube.com", + "token", + "organization_id", + "app_host", + False, + ), + { + "headers": { + "Authorization": "Bearer token", + "Content-Type": "application/json", + } + }, + ), + ( + SonarQubeClient( + "https://sonarqube.com", + "token", + None, + "app_host", + False, + ), + { + "headers": { + "Authorization": "Basic token", + "Content-Type": "application/json", + } + }, + ), + ] + for sonarqube_client, expected_output in values: + sonarqube_client.http_client = MockHttpxClient([]) # type: ignore + assert sonarqube_client.api_auth_params == expected_output + + +async def test_sonarqube_client_will_send_api_request( + mock_ocean_context: Any, + monkeypatch: Any, +) -> None: + sonarqube_client = SonarQubeClient( + "https://sonarqube.com", + "token", + "organization_id", + "app_host", + False, + ) + + sonarqube_client.http_client = MockHttpxClient( # type: ignore + [ + {"status_code": 200, "json": PURE_PROJECTS[0]}, + {"status_code": 200, "json": PURE_PROJECTS[1]}, + ] + ) + + response = await sonarqube_client._send_api_request( + "/api/projects/search", + "GET", + ) + assert response == PURE_PROJECTS[0] + + +async def test_sonarqube_client_will_repeatedly_make_pagination_request( + projects: list[dict[str, Any]], monkeypatch: Any, mock_ocean_context: Any +) -> None: + sonarqube_client = SonarQubeClient( + "https://sonarqube.com", + "token", + "organization_id", + "app_host", + False, + ) + async with event_context("test_event"): + sonarqube_client.http_client = MockHttpxClient( # type: ignore + [ + { + "status_code": 200, + "json": { + "paging": {"pageIndex": 1, "pageSize": 1, "total": 2}, + "components": PURE_PROJECTS, + }, + }, + { + "status_code": 200, + "json": { + "paging": {"pageIndex": 2, "pageSize": 1, "total": 2}, + "components": projects, + }, + }, + ] + ) + + count = 0 + async for _ in sonarqube_client._send_paginated_request( + "/api/projects/search", + "GET", + "components", + ): + count += 1 + + +async def test_pagination_with_large_dataset( + mock_ocean_context: Any, monkeypatch: Any +) -> None: + sonarqube_client = SonarQubeClient( + "https://sonarqube.com", + "token", + "organization_id", + "app_host", + False, + ) + mock_get_single_project = AsyncMock() + mock_get_single_project.side_effect = lambda key: key + + # Mock three pages of results + mock_responses = [ + { + "status_code": 200, + "json": { + "paging": {"pageIndex": 1, "pageSize": 2, "total": 6}, + "components": [{"key": "project1"}, {"key": "project2"}], + }, + }, + { + "status_code": 200, + "json": { + "paging": {"pageIndex": 2, "pageSize": 2, "total": 6}, + "components": [{"key": "project3"}, {"key": "project4"}], + }, + }, + { + "status_code": 200, + "json": { + "paging": {"pageIndex": 3, "pageSize": 2, "total": 6}, + "components": [{"key": "project5"}, {"key": "project6"}], + }, + }, + ] + + async with event_context("test_event"): + + monkeypatch.setattr( + sonarqube_client, "get_single_project", mock_get_single_project + ) + + sonarqube_client.http_client = MockHttpxClient(mock_responses) # type: ignore + + project_keys: list[Any] = [] + async for components in sonarqube_client.get_components(): + project_keys.extend(comp["key"] for comp in components) + + assert len(project_keys) == 6 + assert project_keys == [ + "project1", + "project2", + "project3", + "project4", + "project5", + "project6", + ] + + +async def test_get_components_is_called_with_correct_params( + mock_ocean_context: Any, + component_projects: list[dict[str, Any]], + monkeypatch: Any, +) -> None: + sonarqube_client = SonarQubeClient( + "https://sonarqube.com", + "token", + "organization_id", + "app_host", + False, + ) + mock_paginated_request = MagicMock() + mock_paginated_request.__aiter__.return_value = () + + async with event_context("test_event"): + sonarqube_client.http_client = MockHttpxClient( # type: ignore + [ + { + "status_code": 200, + "json": { + "paging": {"pageIndex": 1, "pageSize": 1, "total": 2}, + "components": component_projects, + }, + }, + ] + ) + + monkeypatch.setattr( + sonarqube_client, "_send_paginated_request", mock_paginated_request + ) + + async for _ in sonarqube_client.get_components(): + pass + + mock_paginated_request.assert_any_call( + endpoint="components/search_projects", + data_key="components", + method="GET", + query_params=None, + ) + + +async def test_get_single_component_is_called_with_correct_params( + mock_ocean_context: Any, + monkeypatch: Any, +) -> None: + sonarqube_client = SonarQubeClient( + "https://sonarqube.com", + "token", + "organization_id", + "app_host", + False, + ) + mock_paginated_request = AsyncMock() + + sonarqube_client.http_client = MockHttpxClient( # type: ignore + [ + {"status_code": 200, "json": PURE_PROJECTS[0]}, + ] + ) + + monkeypatch.setattr(sonarqube_client, "_send_api_request", mock_paginated_request) + + await sonarqube_client.get_single_component(PURE_PROJECTS[0]) + + mock_paginated_request.assert_any_call( + endpoint="components/show", query_params={"component": PURE_PROJECTS[0]["key"]} + ) + + +async def test_get_measures_is_called_with_correct_params( + mock_ocean_context: Any, + monkeypatch: Any, +) -> None: + sonarqube_client = SonarQubeClient( + "https://sonarqube.com", + "token", + "organization_id", + "app_host", + False, + ) + mock_paginated_request = AsyncMock() + mock_paginated_request.return_value = {} + # mock_paginated_request.get.return_value = {} + + sonarqube_client.http_client = MockHttpxClient( # type: ignore + [ + {"status_code": 200, "json": PURE_PROJECTS[0]}, + ] + ) + + monkeypatch.setattr(sonarqube_client, "_send_api_request", mock_paginated_request) + + await sonarqube_client.get_measures(PURE_PROJECTS[0]["key"]) + + mock_paginated_request.assert_any_call( + endpoint="measures/component", + query_params={"component": PURE_PROJECTS[0]["key"], "metricKeys": ""}, + ) + + +async def test_get_branches_is_called_with_correct_params( + mock_ocean_context: Any, + monkeypatch: Any, +) -> None: + sonarqube_client = SonarQubeClient( + "https://sonarqube.com", + "token", + "organization_id", + "app_host", + False, + ) + mock_paginated_request = AsyncMock() + mock_paginated_request.return_value = {} + # mock_paginated_request.get.return_value = {} + + sonarqube_client.http_client = MockHttpxClient( # type: ignore + [ + {"status_code": 200, "json": PURE_PROJECTS[0]}, + ] + ) + + monkeypatch.setattr(sonarqube_client, "_send_api_request", mock_paginated_request) + + await sonarqube_client.get_branches(PURE_PROJECTS[0]["key"]) + + mock_paginated_request.assert_any_call( + endpoint="project_branches/list", + query_params={"project": PURE_PROJECTS[0]["key"]}, + ) + + +async def test_get_single_project_is_called_with_correct_params( + mock_ocean_context: Any, + monkeypatch: Any, +) -> None: + sonarqube_client = SonarQubeClient( + "https://sonarqube.com", + "token", + "organization_id", + "app_host", + False, + ) + + mock_get_measures = AsyncMock() + mock_get_measures.return_value = {} + mock_get_branches = AsyncMock() + mock_get_branches.return_value = [{"isMain": True}] + + sonarqube_client.http_client = MockHttpxClient([]) # type: ignore + monkeypatch.setattr(sonarqube_client, "get_measures", mock_get_measures) + monkeypatch.setattr(sonarqube_client, "get_branches", mock_get_branches) + + await sonarqube_client.get_single_project(PURE_PROJECTS[0]) + + mock_get_measures.assert_any_call(PURE_PROJECTS[0]["key"]) + + mock_get_branches.assert_any_call(PURE_PROJECTS[0]["key"]) + + +async def test_projects_will_return_correct_data( + mock_event_context: Any, mock_ocean_context: Any, monkeypatch: Any +) -> None: + async with event_context("test_event"): + sonarqube_client = SonarQubeClient( + "https://sonarqube.com", + "token", + "organization_id", + "app_host", + False, + ) + mock_paginated_request = MagicMock() + mock_paginated_request.__aiter__.return_value = PURE_PROJECTS[0] + + monkeypatch.setattr( + sonarqube_client, "_send_paginated_request", mock_paginated_request + ) + + async for _ in sonarqube_client.get_projects({}): + pass + + mock_paginated_request.assert_any_call( + endpoint="projects/search", + data_key="components", + method="GET", + query_params={"organization": sonarqube_client.organization_id}, + ) + + +async def test_get_analysis_by_project_processes_data_correctly( + mock_ocean_context: Any, + monkeypatch: Any, +) -> None: + sonarqube_client = SonarQubeClient( + "https://sonarqube.com", + "token", + "organization_id", + "app_host", + False, + ) + + mock_response = { + "paging": {"pageIndex": 1, "pageSize": 1, "total": 1}, + "activityFeed": [ + { + "type": "analysis", + "data": { + "branch": { + "name": "main", + "analysisDate": "2024-01-01", + "commit": "abc123", + } + }, + }, + {"type": "not_analysis", "data": {}}, + ], + } + + sonarqube_client.http_client = MockHttpxClient( + [{"status_code": 200, "json": mock_response}] # type: ignore + ) + + component = {"key": "test-project"} + results = [] + async for analysis_data in sonarqube_client.get_analysis_by_project(component): + results.extend(analysis_data) + + assert len(results) == 1 + assert results[0]["__branchName"] == "main" + assert results[0]["__analysisDate"] == "2024-01-01" + assert results[0]["__commit"] == "abc123" + assert results[0]["__component"] == component + assert results[0]["__project"] == "test-project" + + +async def test_get_all_portfolios_processes_subportfolios( + mock_ocean_context: Any, + monkeypatch: Any, +) -> None: + mock_get_portfolio_details = AsyncMock() + mock_get_portfolio_details.side_effect = lambda key: {"key": key, "subViews": []} + sonarqube_client = SonarQubeClient( + "https://sonarqube.com", + "token", + None, + "app_host", + False, + ) + + monkeypatch.setattr( + sonarqube_client, "_get_portfolio_details", mock_get_portfolio_details + ) + + portfolio_response = {"views": [{"key": "portfolio1"}, {"key": "portfolio2"}]} + + sonarqube_client.http_client = MockHttpxClient( + [ # type: ignore + {"status_code": 200, "json": portfolio_response}, + ] + ) + + portfolio_keys = set() + async for portfolios in sonarqube_client.get_all_portfolios(): + for portfolio in portfolios: + portfolio_keys.add(portfolio.get("key")) + + assert portfolio_keys == {"portfolio1", "portfolio2"} + + +def test_sanity_check_handles_errors( + mock_ocean_context: Any, + monkeypatch: Any, +) -> None: + sonarqube_client = SonarQubeClient( + "https://sonarqube.com", + "token", + "organization_id", + "app_host", + False, + ) + + # Test successful response + with patch("httpx.get") as mock_get: + mock_get.return_value = httpx.Response( + status_code=200, + json={"status": "UP", "version": "1.0"}, + headers={"content-type": "application/json"}, + request=httpx.Request("GET", "https://sonarqube.com"), + ) + sonarqube_client.sanity_check() + + # Test HTTP error + with patch("httpx.get") as mock_get: + mock_get.side_effect = httpx.HTTPStatusError( + "Error", + request=httpx.Request("GET", "https://sonarqube.com"), + response=httpx.Response( + 500, request=httpx.Request("GET", "https://sonarqube.com") + ), + ) + with pytest.raises(httpx.HTTPStatusError): + sonarqube_client.sanity_check() + + +async def test_get_pull_requests_for_project( + mock_ocean_context: Any, + monkeypatch: Any, +) -> None: + sonarqube_client = SonarQubeClient( + "https://sonarqube.com", + "token", + "organization_id", + "app_host", + False, + ) + + mock_prs = [ + {"key": "pr1", "title": "First PR"}, + {"key": "pr2", "title": "Second PR"}, + ] + + sonarqube_client.http_client = MockHttpxClient( + [{"status_code": 200, "json": {"pullRequests": mock_prs}}] # type: ignore + ) + + result = await sonarqube_client.get_pull_requests_for_project("project1") + assert result == mock_prs + assert len(result) == 2 + + +async def test_get_pull_request_measures( + mock_ocean_context: Any, + monkeypatch: Any, +) -> None: + sonarqube_client = SonarQubeClient( + "https://sonarqube.com", + "token", + "organization_id", + "app_host", + False, + ) + + sonarqube_client.metrics = ["coverage", "bugs"] + mock_measures = [ + {"metric": "coverage", "value": "85.5"}, + {"metric": "bugs", "value": "12"}, + ] + + sonarqube_client.http_client = MockHttpxClient( + [ # type: ignore + { + "status_code": 200, + "json": {"component": {"key": "project1", "measures": mock_measures}}, + } + ] + ) + + result = await sonarqube_client.get_pull_request_measures("project1", "pr1") + assert result == mock_measures + + +async def test_get_analysis_for_task_handles_missing_data( + mock_ocean_context: Any, + monkeypatch: Any, +) -> None: + sonarqube_client = SonarQubeClient( + "https://sonarqube.com", + "token", + "organization_id", + "app_host", + False, + ) + + # Mock responses for both task and analysis requests + sonarqube_client.http_client = MockHttpxClient( + [ # type: ignore + {"status_code": 200, "json": {"task": {"analysisId": "analysis1"}}}, + {"status_code": 200, "json": {"activityFeed": []}}, # Empty analysis data + ] + ) + + webhook_data = {"taskId": "task1", "project": {"key": "project1"}} + + result = await sonarqube_client.get_analysis_for_task(webhook_data) + assert result == {} # Should return empty dict when no analysis found + + +async def test_get_issues_by_component_handles_404( + mock_ocean_context: Any, + monkeypatch: Any, +) -> None: + sonarqube_client = SonarQubeClient( + "https://sonarqube.com", + "token", + "organization_id", + "app_host", + False, + ) + + sonarqube_client.http_client = MockHttpxClient( + [ # type: ignore + {"status_code": 404, "json": {"errors": [{"msg": "Component not found"}]}} + ] + ) + + with pytest.raises(httpx.HTTPStatusError) as exc_info: + async for _ in sonarqube_client.get_issues_by_component({"key": "nonexistent"}): + pass + + assert exc_info.value.response.status_code == 404 + + +async def test_get_measures_empty_metrics( + mock_ocean_context: Any, + monkeypatch: Any, +) -> None: + sonarqube_client = SonarQubeClient( + "https://sonarqube.com", + "token", + "organization_id", + "app_host", + False, + ) + + sonarqube_client.metrics = [] # Empty metrics list + + sonarqube_client.http_client = MockHttpxClient( + [ # type: ignore + { + "status_code": 200, + "json": {"component": {"key": "project1", "measures": []}}, + } + ] + ) + + result = await sonarqube_client.get_measures("project1") + assert result == [] + + +async def test_get_branches_main_branch_missing( + mock_ocean_context: Any, + monkeypatch: Any, +) -> None: + sonarqube_client = SonarQubeClient( + "https://sonarqube.com", + "token", + "organization_id", + "app_host", + False, + ) + + # Mock branches without a main branch + sonarqube_client.http_client = MockHttpxClient( + [ # type: ignore + { + "status_code": 200, + "json": { + "branches": [ + {"name": "feature1", "isMain": False}, + {"name": "feature2", "isMain": False}, + ] + }, + } + ] + ) + + project = {"key": "project1"} + result = await sonarqube_client.get_branches(project["key"]) + assert len(result) == 2 + assert all(not branch["isMain"] for branch in result) + + +async def test_create_webhook_payload_for_project_no_organization( + mock_ocean_context: Any, + monkeypatch: Any, +) -> None: + """Test webhook payload creation without organization""" + sonarqube_client = SonarQubeClient( + "https://sonarqube.com", + "token", + None, # No organization + "http://app.host", + False, + ) + + sonarqube_client.webhook_invoke_url = "http://app.host/webhook" + sonarqube_client.http_client = MockHttpxClient( + [ # type: ignore + {"status_code": 200, "json": {"webhooks": []}} # No existing webhooks + ] + ) + + result = await sonarqube_client._create_webhook_payload_for_project("project1") + assert result == {"name": "Port Ocean Webhook", "project": "project1"} + + +async def test_create_webhook_payload_for_project_with_organization( + mock_ocean_context: Any, + monkeypatch: Any, +) -> None: + """Test webhook payload creation with organization""" + sonarqube_client = SonarQubeClient( + "https://sonarqube.com", + "token", + "test-org", + "http://app.host", + False, + ) + + sonarqube_client.webhook_invoke_url = "http://app.host/webhook" + sonarqube_client.http_client = MockHttpxClient( + [{"status_code": 200, "json": {"webhooks": []}}] # type: ignore + ) + + result = await sonarqube_client._create_webhook_payload_for_project("project1") + assert result == { + "name": "Port Ocean Webhook", + "project": "project1", + "organization": "test-org", + } + + +async def test_create_webhook_payload_existing_webhook( + mock_ocean_context: Any, + monkeypatch: Any, +) -> None: + """Test webhook payload creation when webhook already exists""" + sonarqube_client = SonarQubeClient( + "https://sonarqube.com", + "token", + None, + "http://app.host", + False, + ) + + sonarqube_client.webhook_invoke_url = "http://app.host/webhook" + sonarqube_client.http_client = MockHttpxClient( + [ # type: ignore + { + "status_code": 200, + "json": { + "webhooks": [ + { + "url": "http://app.host/webhook" + } # Existing webhook with same URL + ] + }, + } + ] + ) + + result = await sonarqube_client._create_webhook_payload_for_project("project1") + assert result == {} # Should return empty dict when webhook exists + + +async def test_create_webhooks_for_projects( + mock_ocean_context: Any, + monkeypatch: Any, +) -> None: + """Test webhook creation for multiple projects""" + sonarqube_client = SonarQubeClient( + "https://sonarqube.com", + "token", + None, + "http://app.host", + False, + ) + + sonarqube_client.webhook_invoke_url = "http://app.host/webhook" + + # Mock responses for multiple webhook creations + mock_responses = [ + {"status_code": 200, "json": {"webhook": "created1"}}, + {"status_code": 200, "json": {"webhook": "created2"}}, + ] + + sonarqube_client.http_client = MockHttpxClient(mock_responses) # type: ignore + + webhook_payloads = [ + {"name": "Port Ocean Webhook", "project": "project1"}, + {"name": "Port Ocean Webhook", "project": "project2"}, + ] + + await sonarqube_client._create_webhooks_for_projects(webhook_payloads) + + +async def test_get_or_create_webhook_url_error_handling( + mock_ocean_context: Any, + monkeypatch: Any, +) -> None: + """Test webhook creation error handling""" + sonarqube_client = SonarQubeClient( + "https://sonarqube.com", + "token", + "test-org", + "http://app.host", + False, + ) + + sonarqube_client.webhook_invoke_url = "http://app.host/webhook" + + # Mock responses including an error + mock_responses = [ + # Get projects response + { + "status_code": 200, + "json": { + "paging": {"pageIndex": 1, "pageSize": 1, "total": 1}, + "components": [{"key": "project1"}], + }, + }, + # Check webhooks - returns error + {"status_code": 404, "json": {"errors": [{"msg": "Project not found"}]}}, + ] + + async with event_context("test_event"): + sonarqube_client.http_client = MockHttpxClient(mock_responses) # type: ignore + + with pytest.raises(httpx.HTTPStatusError) as exc_info: + await sonarqube_client.get_or_create_webhook_url() + + assert exc_info.value.response.status_code == 404 + + +async def test_create_webhook_payload_for_project_different_url( + mock_ocean_context: Any, + monkeypatch: Any, +) -> None: + """Test webhook payload creation when different webhook URL exists""" + sonarqube_client = SonarQubeClient( + "https://sonarqube.com", + "token", + None, + "http://app.host", + False, + ) + + sonarqube_client.webhook_invoke_url = "http://app.host/webhook" + sonarqube_client.http_client = MockHttpxClient( + [ # type: ignore + { + "status_code": 200, + "json": { + "webhooks": [ + {"url": "http://different.url/webhook"} # Different webhook URL + ] + }, + } + ] + ) + + result = await sonarqube_client._create_webhook_payload_for_project("project1") + assert result == {"name": "Port Ocean Webhook", "project": "project1"} diff --git a/integrations/sonarqube/tests/test_sample.py b/integrations/sonarqube/tests/test_sample.py deleted file mode 100644 index dc80e299c8..0000000000 --- a/integrations/sonarqube/tests/test_sample.py +++ /dev/null @@ -1,2 +0,0 @@ -def test_example() -> None: - assert 1 == 1 diff --git a/integrations/sonarqube/tests/test_sync.py b/integrations/sonarqube/tests/test_sync.py new file mode 100644 index 0000000000..c90e1a784c --- /dev/null +++ b/integrations/sonarqube/tests/test_sync.py @@ -0,0 +1,57 @@ +# from typing import Any +# from unittest.mock import AsyncMock + +# from port_ocean import Ocean +# from port_ocean.tests.helpers.ocean_app import ( +# get_integation_resource_configs, +# get_raw_result_on_integration_sync_resource_config, +# ) + +# from client import SonarQubeClient + + +# async def test_full_sync_produces_correct_response_from_api( +# monkeypatch: Any, +# ocean_app: Ocean, +# integration_path: str, +# issues: list[dict[str, Any]], +# projects: list[dict[str, Any]], +# component_projects: list[dict[str, Any]], +# analysis: list[dict[str, Any]], +# portfolios: list[dict[str, Any]], +# ) -> None: +# projects_mock = AsyncMock() +# projects_mock.return_value = projects +# component_projects_mock = AsyncMock() +# component_projects_mock.return_value = component_projects +# issues_mock = AsyncMock() +# issues_mock.return_value = issues +# saas_analysis_mock = AsyncMock() +# saas_analysis_mock.return_value = analysis +# on_onprem_analysis_resync_mock = AsyncMock() +# on_onprem_analysis_resync_mock.return_value = analysis +# on_portfolio_resync_mock = AsyncMock() +# on_portfolio_resync_mock.return_value = portfolios + +# monkeypatch.setattr(SonarQubeClient, "get_projects", projects_mock) +# monkeypatch.setattr(SonarQubeClient, "get_components", component_projects_mock) +# monkeypatch.setattr(SonarQubeClient, "get_all_issues", issues_mock) +# monkeypatch.setattr( +# SonarQubeClient, "get_all_sonarcloud_analyses", saas_analysis_mock +# ) +# monkeypatch.setattr( +# SonarQubeClient, "get_all_sonarqube_analyses", on_onprem_analysis_resync_mock +# ) +# monkeypatch.setattr(SonarQubeClient, "get_all_portfolios", on_portfolio_resync_mock) +# resource_configs = get_integation_resource_configs(integration_path) +# for resource_config in resource_configs: +# print(resource_config) +# results = await get_raw_result_on_integration_sync_resource_config( +# ocean_app, resource_config +# ) +# assert len(results) > 0 +# entities, errors = results +# assert len(errors) == 0 +# # the factories have several entities each +# # all in one batch +# assert len(list(entities)) == 1 diff --git a/integrations/sonarqube/utils.py b/integrations/sonarqube/utils.py new file mode 100644 index 0000000000..bc43ec39d3 --- /dev/null +++ b/integrations/sonarqube/utils.py @@ -0,0 +1,19 @@ +from typing import Any + +from client import SonarQubeClient +from integration import SonarQubeComponentProjectSelector + + +def produce_component_params( + client: SonarQubeClient, + selector: SonarQubeComponentProjectSelector, +) -> dict[str, Any]: + component_query_params: dict[str, Any] = {} + if client.organization_id: + component_query_params["organization"] = client.organization_id + + ## Handle query_params based on environment + if client.is_onpremise and selector: + + component_query_params.update(selector.generate_request_params()) + return component_query_params