Implement JSON schema validation for CodeTF results

.pre-commit-config.yaml

@@ -34,6 +34,7 @@ repos:
      args: [--disable-error-code=has-type,--disable-error-code=import-not-found]
      additional_dependencies:
        [
+         "types-jsonschema~=4.21.0",
          "types-mock==5.0.*",
          "types-PyYAML==6.0",
          "types-toml~=0.10",

integration_tests/conftest.py

@@ -0,0 +1,11 @@
+import json
+
+import pytest
+import requests
+
+
+@pytest.fixture(scope="module")
+def codetf_schema():
+    schema_path = "https://raw.githubusercontent.com/pixee/codemodder-specs/main/codetf.schema.json"
+    response = requests.get(schema_path)
+    yield json.loads(response.text)

pyproject.toml

@@ -51,6 +51,7 @@ test = [
     "coverage>=7.3,<7.5",
     "Flask<4",
     "Jinja2~=3.1.2",
+    "jsonschema~=4.21.0",
     "lxml>=4.9.3,<5.2.0",
     "mock==5.1.*",
     "pre-commit<4",
@@ -60,6 +61,7 @@ test = [
     "pytest-mock~=3.12.0",
     "pytest-randomly==3.*",
     "pytest-xdist==3.*",
+    "requests~=2.31.0",
     "security~=1.2.0",
     "types-mock==5.1.*",
     "django>=4,<6",

src/codemodder/codemods/test/integration_utils.py

@@ -6,6 +6,7 @@
 from types import ModuleType
 
 import git
+import jsonschema
 
 from codemodder import __version__, registry
 
@@ -99,7 +100,8 @@ def _assert_results_fields(self, results, output_path):
         result = results[0]
         assert result["codemod"] == self.codemod_instance.id
         assert result["references"] == [
-            ref.to_json() for ref in self.codemod_instance.references
+            ref.model_dump(exclude_none=True)
+            for ref in self.codemod_instance.references
         ]
 
         # TODO: once we add description for each url.
@@ -118,15 +120,15 @@ def _assert_results_fields(self, results, output_path):
 
         assert len(change["changes"]) == self.num_changes
         line_change = change["changes"][0]
-        assert line_change["lineNumber"] == str(self.expected_line_change)
+        assert line_change["lineNumber"] == int(self.expected_line_change)
         assert line_change["description"] == self.change_description
-        assert line_change["packageActions"] == []
-        assert line_change["properties"] == {}
 
-    def _assert_codetf_output(self):
+    def _assert_codetf_output(self, codetf_schema):
         with open(self.output_path, "r", encoding="utf-8") as f:
             codetf = json.load(f)
 
+        jsonschema.validate(codetf, codetf_schema)
+
         assert sorted(codetf.keys()) == ["results", "run"]
         run = codetf["run"]
         self._assert_run_fields(run, self.output_path)
@@ -149,7 +151,7 @@ def check_code_after(self) -> ModuleType:
             path=self.code_path, allowed_exceptions=self.allowed_exceptions
         )
 
-    def test_file_rewritten(self):
+    def test_file_rewritten(self, codetf_schema):
         """
         Tests that file is re-written correctly with new code and correct codetf output.
 
@@ -183,7 +185,7 @@ def test_file_rewritten(self):
 
         self.check_code_after()
         self.check_dependencies_after()
-        self._assert_codetf_output()
+        self._assert_codetf_output(codetf_schema)
         pathlib.Path(self.output_path).unlink(missing_ok=True)
         self._run_idempotency_chec(command)
 

tests/test_codetf.py

@@ -1,6 +1,25 @@
+import json
+from pathlib import Path
+
+import jsonschema
 import pytest
+import requests
+
+from codemodder.codetf import Change, ChangeSet, CodeTF, DiffSide, Result
+
+
+@pytest.fixture(autouse=True)
+def disable_write_report():
+    """
+    Override conftest to enable results to be written to disk for these tests.
+    """
+
 
-from codemodder.codetf import Change, ChangeSet, DiffSide
+@pytest.fixture(autouse=True, scope="module")
+def codetf_schema():
+    schema_path = "https://raw.githubusercontent.com/pixee/codemodder-specs/main/codetf.schema.json"
+    response = requests.get(schema_path)
+    yield json.loads(response.text)
 
 
 def test_change():
@@ -37,3 +56,58 @@ def test_change_diffside(side):
 
     assert change.diffSide == side
     assert change.model_dump()["diffSide"] == side
+
+
+def test_write_codetf(tmpdir, mocker, codetf_schema):
+    path = tmpdir / "test.codetf.json"
+
+    assert not path.exists()
+
+    context = mocker.MagicMock(directory=Path("/foo/bar/whatever"))
+    codetf = CodeTF.build(context, 42, [], [])
+    retval = codetf.write_report(path)
+
+    assert retval == 0
+    assert path.exists()
+
+    data = path.read_text(encoding="utf-8")
+    CodeTF.model_validate_json(data)
+
+    jsonschema.validate(json.loads(data), codetf_schema)
+
+
+def test_write_codetf_with_results(tmpdir, mocker, codetf_schema):
+    path = tmpdir / "test.codetf.json"
+
+    assert not path.exists()
+
+    context = mocker.MagicMock(directory=Path("/foo/bar/whatever"))
+    results = [
+        Result(
+            codemod="test",
+            summary="test",
+            description="test",
+            changeset=[
+                ChangeSet(
+                    path="test",
+                    diff="--- a/test\n+++ b/test\n@@ -1,1 +1,1 @@\n-1\n+2\n",
+                    changes=[
+                        Change(
+                            lineNumber=1,
+                            description="Change 1 to 2",
+                        ),
+                    ],
+                ),
+            ],
+        )
+    ]
+    codetf = CodeTF.build(context, 42, [], results)
+    retval = codetf.write_report(path)
+
+    assert retval == 0
+    assert path.exists()
+
+    data = path.read_text(encoding="utf-8")
+    CodeTF.model_validate_json(data)
+
+    jsonschema.validate(json.loads(data), codetf_schema)