From b3f8f25517df166ff3b48339ae61c1a47b811289 Mon Sep 17 00:00:00 2001 From: Tom Tankilevitch <59158507+Tankilevitch@users.noreply.github.com> Date: Mon, 19 Aug 2024 12:40:22 +0300 Subject: [PATCH] [Gitlab] Fix still processing a file larger than the allowed size (#913) --- integrations/gitlab/CHANGELOG.md | 9 +++++++++ .../gitlab/gitlab_integration/gitlab_service.py | 17 +++++++++++++---- integrations/gitlab/pyproject.toml | 2 +- 3 files changed, 23 insertions(+), 5 deletions(-) diff --git a/integrations/gitlab/CHANGELOG.md b/integrations/gitlab/CHANGELOG.md index 1e23969075..3883ef0fce 100644 --- a/integrations/gitlab/CHANGELOG.md +++ b/integrations/gitlab/CHANGELOG.md @@ -7,6 +7,15 @@ this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.htm +0.1.106 (2024-08-19) +==================== + +### Bug Fixes + +- Fixed an issue when we were still processing a file larger than the allowed file size +- Added more verbosity to the logs of the file kind + + 0.1.105 (2024-08-15) =================== diff --git a/integrations/gitlab/gitlab_integration/gitlab_service.py b/integrations/gitlab/gitlab_integration/gitlab_service.py index df32e0c4f8..326cdb0842 100644 --- a/integrations/gitlab/gitlab_integration/gitlab_service.py +++ b/integrations/gitlab/gitlab_integration/gitlab_service.py @@ -604,16 +604,25 @@ async def get_and_parse_single_file( self, project: Project, file_path: str, branch: str ) -> dict[str, Any] | None: try: + logger.info( + f"Processing file {file_path} in project {project.path_with_namespace}" + ) project_file = await AsyncFetcher.fetch_single( project.files.get, file_path, branch ) + logger.info( + f"Fetched file {file_path} in project {project.path_with_namespace}" + ) project_file = typing.cast(ProjectFile, project_file) parsed_file = self._parse_file_content(project_file) project_file_dict = project_file.asdict() - if parsed_file: - # Update the content with the parsed content. Useful for JSON and YAML files that can be further processed using itemsToParse - project_file_dict["content"] = parsed_file + if not parsed_file: + # if the file is too large to be processed, we return None + return None + + # Update the content with the parsed content. Useful for JSON and YAML files that can be further processed using itemsToParse + project_file_dict["content"] = parsed_file return {"file": project_file_dict, "repo": project.asdict()} except Exception as e: @@ -628,7 +637,7 @@ async def get_all_files_in_project( branch = project.default_branch try: file_paths = await self._get_file_paths(project, path, branch, True) - logger.debug( + logger.info( f"Found {len(file_paths)} files in project {project.path_with_namespace} files: {file_paths}" ) files = [] diff --git a/integrations/gitlab/pyproject.toml b/integrations/gitlab/pyproject.toml index 6f33c9219a..f8b1ec5f6f 100644 --- a/integrations/gitlab/pyproject.toml +++ b/integrations/gitlab/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "gitlab" -version = "0.1.105" +version = "0.1.106" description = "Gitlab integration for Port using Port-Ocean Framework" authors = ["Yair Siman-Tov "]