diff --git a/.github/workflows/ci_pipeline.yml b/.github/workflows/ci_pipeline.yml index 94d231e73..4581d37f0 100644 --- a/.github/workflows/ci_pipeline.yml +++ b/.github/workflows/ci_pipeline.yml @@ -135,7 +135,7 @@ jobs: - name: Install Dependencies run: | - pip install poetry wandb tree_sitter + pip install poetry wandb tree_sitter diff-cover poetry install - name: Build Executable run: make installer diff --git a/Makefile b/Makefile index e4834b2eb..5f29c6cb3 100644 --- a/Makefile +++ b/Makefile @@ -1,5 +1,6 @@ # Makefile SITE_PACKAGES=$(shell python -c "import wandb, os; print(os.path.dirname(wandb.__file__))") +DIFF_COVER_TEMPLATES=$(shell python -c "import site; print(site.getsitepackages()[0])") TOML_FILES=$(shell find cover_agent/settings -name "*.toml" | sed 's/.*/-\-add-data "&:."/' | tr '\n' ' ') .PHONY: test build installer @@ -22,6 +23,7 @@ installer: --add-data "cover_agent/version.txt:." \ $(TOML_FILES) \ --add-data "$(SITE_PACKAGES)/vendor:wandb/vendor" \ + --add-data "$(DIFF_COVER_TEMPLATES)/diff_cover/templates:diff_cover/templates" \ --hidden-import=tiktoken_ext.openai_public \ --hidden-import=tiktoken_ext \ --hidden-import=wandb \ diff --git a/cover_agent/UnitTestValidator.py b/cover_agent/UnitTestValidator.py index 0a92220da..7cb7aff76 100644 --- a/cover_agent/UnitTestValidator.py +++ b/cover_agent/UnitTestValidator.py @@ -4,6 +4,8 @@ import logging import os +from diff_cover.diff_cover_tool import main as diff_cover_main + from cover_agent.AICaller import AICaller from cover_agent.CustomLogger import CustomLogger from cover_agent.FilePreprocessor import FilePreprocessor @@ -690,20 +692,32 @@ def post_process_coverage_report(self, time_of_test_command: int): return report def generate_diff_coverage_report(self): - # Run the diff-cover command to generate a JSON diff coverage report - coverage_filename = os.path.basename(self.code_coverage_report_path) - coverage_command = f"diff-cover --json-report {self.diff_coverage_report_name} --compare-branch={self.comparison_branch} {coverage_filename}" - # Log and execute the diff coverage command - self.logger.info(f'Running diff coverage command: "{coverage_command}"') - stdout, stderr, exit_code, _ = Runner.run_command( - command=coverage_command, cwd=self.test_command_dir - ) - - # Ensure the diff command executed successfully - assert exit_code == 0, ( - f'Fatal: Error running diff coverage command. Are you sure the command is correct? "{coverage_command}"' - f"\nExit code {exit_code}. \nStdout: \n{stdout} \nStderr: \n{stderr}" - ) + """ + Generates a JSON diff coverage report using the diff-cover tool. + This method runs the diff-cover command with the specified arguments to generate + a JSON report that shows the coverage differences between the current branch and + the specified comparison branch. + Args: + None + Returns: + None + Raises: + Exception: If an error occurs while running the diff-cover command. + """ + + diff_cover_args = [ + "diff-cover", + "--json-report", + self.diff_cover_report_path, + "--compare-branch={}".format(self.comparison_branch), + self.code_coverage_report_path + ] + + self.logger.info(f'Running diff coverage module with args: "{diff_cover_args}"') + try: + diff_cover_main(diff_cover_args) + except Exception as e: + self.logger.error(f"Error running diff-cover: {e}") def get_current_coverage(self): - return self.current_coverage_report.total_coverage + return self.current_coverage_report.total_coverage \ No newline at end of file diff --git a/tests/test_UnitTestValidator.py b/tests/test_UnitTestValidator.py index 3e34023f4..5abf74106 100644 --- a/tests/test_UnitTestValidator.py +++ b/tests/test_UnitTestValidator.py @@ -102,7 +102,7 @@ def test_validate_test_pass_no_coverage_increase_with_prompt(self): result = generator.validate_test(test_to_validate) assert result["status"] == "FAIL" - assert result["reason"] == "Coverage did not increase. Maybe the test did run but did not increase coverage, or maybe the test execution was skipped due to some problem" + assert "Coverage did not increase" in result["reason"] assert result["exit_code"] == 0 def test_initial_test_suite_analysis_with_prompt_builder(self): @@ -182,7 +182,8 @@ def test_post_process_coverage_report_without_flags(self): coverage_report = generator.post_process_coverage_report(datetime.datetime.now()) assert coverage_report.total_coverage == 0.7 - def test_generate_diff_coverage_report(self): + + def test_generate_diff_coverage_report_success(self): with tempfile.NamedTemporaryFile(suffix=".py", delete=False) as temp_source_file: generator = UnitTestValidator( source_file_path=temp_source_file.name, @@ -190,8 +191,31 @@ def test_generate_diff_coverage_report(self): code_coverage_report_path="coverage.xml", test_command="pytest", llm_model="gpt-3", - diff_coverage=True + diff_coverage=True, + comparison_branch="main" ) - with patch.object(Runner, 'run_command', return_value=("", "", 0, datetime.datetime.now())): + with patch("cover_agent.UnitTestValidator.diff_cover_main") as mock_diff_cover_main: + generator.generate_diff_coverage_report() + mock_diff_cover_main.assert_called_once_with([ + "diff-cover", + "--json-report", + generator.diff_cover_report_path, + "--compare-branch=main", + "coverage.xml" + ]) + + def test_generate_diff_coverage_report_failure(self): + with tempfile.NamedTemporaryFile(suffix=".py", delete=False) as temp_source_file: + generator = UnitTestValidator( + source_file_path=temp_source_file.name, + test_file_path="test_test.py", + code_coverage_report_path="coverage.xml", + test_command="pytest", + llm_model="gpt-3", + diff_coverage=True, + comparison_branch="main" + ) + with patch("cover_agent.UnitTestValidator.diff_cover_main", side_effect=Exception("Mock exception")), \ + patch.object(generator.logger, 'error') as mock_logger_error: generator.generate_diff_coverage_report() - assert generator.diff_cover_report_path.endswith("diff-cover-report.json") \ No newline at end of file + mock_logger_error.assert_called_once_with("Error running diff-cover: Mock exception") \ No newline at end of file