Report failed parsing (#75)

* report even if no results

* pygoat findings only if changset

* update unit test

.github/workflows/codemod_pygoat.yml

@@ -38,4 +38,4 @@ jobs:
       - name: Run Codemodder
         run: codemodder --output output.codetf pygoat
       - name: Check PyGoat Findings
-        run: make webgoat-test
+        run: make pygoat-test

Makefile

@@ -8,8 +8,8 @@ test:
 integration-test:
 	${PYTEST} integration_tests
 
-webgoat-test:
-	${PYTEST} -v ci_tests/test_webgoat_findings.py
+pygoat-test:
+	${PYTEST} -v ci_tests/test_pygoat_findings.py
 
 lint:
 	pylint -v codemodder core_codemods tests integration_tests

ci_tests/test_webgoat_findings.py → ci_tests/test_pygoat_findings.py

@@ -17,17 +17,17 @@
 
 
 @pytest.fixture(scope="session")
-def webgoat_findings():
+def pygoat_findings():
     with open("output.codetf") as ff:
         results = json.load(ff)
 
-    yield set([x["codemod"] for x in results["results"]])
+    yield set([x["codemod"] for x in results["results"] if x["changeset"]])
 
 
-def test_num_webgoat_findings(webgoat_findings):
-    assert len(webgoat_findings) == len(EXPECTED_FINDINGS)
+def test_num_pygoat_findings(pygoat_findings):
+    assert len(pygoat_findings) == len(EXPECTED_FINDINGS)
 
 
 @pytest.mark.parametrize("finding", EXPECTED_FINDINGS)
-def test_webgoat_findings(webgoat_findings, finding):
-    assert finding in webgoat_findings
+def test_pygoat_findings(pygoat_findings, finding):
+    assert finding in pygoat_findings

integration_tests/base_test.py

@@ -175,7 +175,7 @@ def _run_idempotency_chec(self, command):
         if pathlib.Path(self.output_path).exists():
             with open(self.output_path, "r", encoding="utf-8") as f:
                 codetf = json.load(f)
-            assert codetf["results"] == []
+            assert codetf["results"][0]["changeset"] == []
 
         with open(self.code_path, "r", encoding="utf-8") as f:
             still_new_code = f.read()

src/codemodder/context.py

@@ -75,17 +75,16 @@ def add_dependency(self, dependency: str):
     def compile_results(self, codemods: list[CodemodExecutorWrapper]):
         results = []
         for codemod in codemods:
-            if not (changeset := self._results_by_codemod.get(codemod.id)):
-                continue
-
             data = {
                 "codemod": codemod.id,
                 "summary": codemod.summary,
                 "description": codemod.description,
                 "references": codemod.references,
                 "properties": {},
-                "failedFiles": [],
-                "changeset": [change.to_json() for change in changeset],
+                "failedFiles": [str(file) for file in self.get_failures(codemod.id)],
+                "changeset": [
+                    change.to_json() for change in self.get_results(codemod.id)
+                ],
             }
 
             results.append(data)

tests/test_main.py → tests/test_codemodder.py

@@ -32,19 +32,30 @@ def test_cst_parsing_fails(self, mock_reporting, mock_parse):
             "tests/samples/",
             "--output",
             "here.txt",
+            "--codemod-include",
+            "requests-verify",
+            "--path-include",
+            "*request.py",
         ]
 
         res = run(args)
 
         assert res == 0
         mock_parse.assert_called()
 
-        # We still report for now even if parsing failed
         mock_reporting.assert_called_once()
         args_to_reporting = mock_reporting.call_args_list[0][0]
         assert len(args_to_reporting) == 4
         _, _, _, results_by_codemod = args_to_reporting
-        assert results_by_codemod == []
+        assert results_by_codemod != []
+
+        requests_report = results_by_codemod[0]
+        assert requests_report["changeset"] == []
+        assert len(requests_report["failedFiles"]) == 2
+        assert sorted(requests_report["failedFiles"]) == [
+            "tests/samples/make_request.py",
+            "tests/samples/unverified_request.py",
+        ]
 
     @mock.patch("codemodder.codemodder.update_code")
     @mock.patch("codemodder.codemods.base_codemod.semgrep_run", side_effect=semgrep_run)
@@ -79,10 +90,9 @@ def test_reporting(self, mock_reporting, dry_run):
         args_to_reporting = mock_reporting.call_args_list[0][0]
         assert len(args_to_reporting) == 4
         _, _, _, results_by_codemod = args_to_reporting
-        # assert len(results_by_codemod) == 2
 
-        for codemod_results in results_by_codemod:
-            assert len(codemod_results["changeset"]) > 0
+        registry = load_registered_codemods()
+        assert len(results_by_codemod) == len(registry.codemods)
 
     @mock.patch("codemodder.codemods.base_codemod.semgrep_run")
     def test_no_codemods_to_run(self, mock_semgrep_run):