Requirements.txt writer adds hashes and a way to remember to update c…

…odemodder deps (#273)

* bump security version

* update tests to reference dependencies

* requirements writer can write dependency hashes

* add get_hashes script

* add codemod_dependencies.txt

* add timeout to request

.pre-commit-config.yaml

@@ -37,6 +37,7 @@ repos:
          "types-mock==5.0.*",
          "types-PyYAML==6.0",
          "types-toml~=0.10",
+         "types-requests~=2.13",
     ]
 - repo: https://github.com/astral-sh/ruff-pre-commit
   rev: v0.2.2

integration_tests/test_flask_enable_csrf_protection.py

@@ -3,6 +3,7 @@
     BaseIntegrationTest,
     original_and_expected_from_code_path,
 )
+from codemodder.dependency import FlaskWTF
 
 
 class TestFlaskEnableCSRFProtection(BaseIntegrationTest):
@@ -34,3 +35,15 @@ class TestFlaskEnableCSRFProtection(BaseIntegrationTest):
     expected_line_change = "3"
     change_description = FlaskEnableCSRFProtection.change_description
     num_changed_files = 2
+
+    requirements_path = "tests/samples/requirements.txt"
+    original_requirements = "# file used to test dependency management\nrequests==2.31.0\nblack==23.7.*\nmypy~=1.4\npylint>1\n"
+    expected_new_reqs = (
+        f"# file used to test dependency management\n"
+        "requests==2.31.0\n"
+        "black==23.7.*\n"
+        "mypy~=1.4\n"
+        "pylint>1\n"
+        f"{FlaskWTF.requirement} \\\n"
+        f"{FlaskWTF.build_hashes()}"
+    )

integration_tests/test_process_sandbox.py

@@ -3,6 +3,7 @@
     BaseIntegrationTest,
     original_and_expected_from_code_path,
 )
+from codemodder.dependency import Security
 
 
 class TestProcessSandbox(BaseIntegrationTest):
@@ -26,4 +27,12 @@ class TestProcessSandbox(BaseIntegrationTest):
 
     requirements_path = "tests/samples/requirements.txt"
     original_requirements = "# file used to test dependency management\nrequests==2.31.0\nblack==23.7.*\nmypy~=1.4\npylint>1\n"
-    expected_new_reqs = "# file used to test dependency management\nrequests==2.31.0\nblack==23.7.*\nmypy~=1.4\npylint>1\nsecurity~=1.2.0\n"
+    expected_new_reqs = (
+        f"# file used to test dependency management\n"
+        "requests==2.31.0\n"
+        "black==23.7.*\n"
+        "mypy~=1.4\n"
+        "pylint>1\n"
+        f"{Security.requirement} \\\n"
+        f"{Security.build_hashes()}"
+    )

integration_tests/test_url_sandbox.py

@@ -3,6 +3,7 @@
     BaseIntegrationTest,
     original_and_expected_from_code_path,
 )
+from codemodder.dependency import Security
 
 
 class TestUrlSandbox(BaseIntegrationTest):
@@ -36,4 +37,12 @@ class TestUrlSandbox(BaseIntegrationTest):
 
     requirements_path = "tests/samples/requirements.txt"
     original_requirements = "# file used to test dependency management\nrequests==2.31.0\nblack==23.7.*\nmypy~=1.4\npylint>1\n"
-    expected_new_reqs = "# file used to test dependency management\nrequests==2.31.0\nblack==23.7.*\nmypy~=1.4\npylint>1\nsecurity~=1.2.0\n"
+    expected_new_reqs = (
+        f"# file used to test dependency management\n"
+        "requests==2.31.0\n"
+        "black==23.7.*\n"
+        "mypy~=1.4\n"
+        "pylint>1\n"
+        f"{Security.requirement} \\\n"
+        f"{Security.build_hashes()}"
+    )

integration_tests/test_use_defusedxml.py

@@ -3,6 +3,7 @@
     BaseIntegrationTest,
     original_and_expected_from_code_path,
 )
+from codemodder.dependency import DefusedXML
 
 
 class TestUseDefusedXml(BaseIntegrationTest):
@@ -39,4 +40,12 @@ class TestUseDefusedXml(BaseIntegrationTest):
 
     requirements_path = "tests/samples/requirements.txt"
     original_requirements = "# file used to test dependency management\nrequests==2.31.0\nblack==23.7.*\nmypy~=1.4\npylint>1\n"
-    expected_new_reqs = "# file used to test dependency management\nrequests==2.31.0\nblack==23.7.*\nmypy~=1.4\npylint>1\ndefusedxml~=0.7.1\n"
+    expected_new_reqs = (
+        f"# file used to test dependency management\n"
+        "requests==2.31.0\n"
+        "black==23.7.*\n"
+        "mypy~=1.4\n"
+        "pylint>1\n"
+        f"{DefusedXML.requirement} \\\n"
+        f"{DefusedXML.build_hashes()}"
+    )

pyproject.toml

@@ -42,6 +42,7 @@ Repository = "https://github.com/pixee/codemodder-python"
 [project.scripts]
 codemodder = "codemodder.codemodder:main"
 generate-docs = 'codemodder.scripts.generate_docs:main'
+get-hashes = 'codemodder.scripts.get_hashes:main'
 
 [project.optional-dependencies]
 test = [

src/codemodder/dependency.py

@@ -1,4 +1,4 @@
-from dataclasses import dataclass
+from dataclasses import dataclass, field
 
 from packaging.requirements import Requirement
 
@@ -16,6 +16,7 @@ class Dependency:
     _license: License
     oss_link: str
     package_link: str
+    hashes: list[str] = field(default_factory=list)
 
     @property
     def name(self) -> str:
@@ -33,12 +34,19 @@ def build_description(self) -> str:
 [More facts]({self.package_link})
 """
 
+    def build_hashes(self) -> str:
+        return " \\\n".join(f"{' '*4}--hash=sha256:{sha256}" for sha256 in self.hashes)
+
     def __hash__(self):
         return hash(self.requirement)
 
 
 FlaskWTF = Dependency(
-    Requirement("flask-wtf~=1.2.0"),
+    Requirement("flask-wtf==1.2.0"),
+    hashes=[
+        "96e6f091c641c9944ba7dba2957c84797b630006aa926c99507fbd790069772b",
+        "e5dcf9f3cb80ee6ca8bb68de9ea467e7d613a708ebc5e130b9b02996e06c0d54",
+    ],
     description="""\
             This package integrates WTForms into Flask. WTForms provides data validation and and CSRF protection which helps harden applications.
 """,
@@ -51,7 +59,11 @@ def __hash__(self):
 )
 
 DefusedXML = Dependency(
-    Requirement("defusedxml~=0.7.1"),
+    Requirement("defusedxml==0.7.1"),
+    hashes=[
+        "a352e7e428770286cc899e2542b6cdaedb2b4953ff269a210103ec58f6198a61",
+        "1bb3032db185915b62d7c6209c5a8792be6a32ab2fedacc84e01b52c51aa3e69",
+    ],
     description="""\
 This package is [recommended by the Python community](https://docs.python.org/3/library/xml.html#the-defusedxml-package) \
 to protect against XML vulnerabilities.\
@@ -65,7 +77,11 @@ def __hash__(self):
 )
 
 Security = Dependency(
-    Requirement("security~=1.2.0"),
+    Requirement("security==1.2.1"),
+    hashes=[
+        "4ca5f8cfc6b836e2192a84bb5a28b72c17f3cd1abbfe3281f917394c6e6c9238",
+        "0a9dc7b457330e6d0f92bdae3603fecb85394beefad0fd3b5058758a58781ded",
+    ],
     description="""This library holds security tools for protecting Python API calls.""",
     _license=License(
         "MIT",

src/codemodder/dependency_management/codemod_dependencies.txt

@@ -0,0 +1,9 @@
+# This is a temporary solution to use dependabot to alert us
+# when the dependencies our codemods inject (for example `security`, `defusedxml`, etc) have a version update.
+
+# If this file gets a dependabot update PR, we must also update the corresponding
+# dependency in dependency.py. Be sure to update the version AND the hashes.
+# Run `get-hashes pkg==version` to get the hashes.
+defusedxml==0.7.1
+flask-wtf==1.2.0
+security==1.2.1

src/codemodder/dependency_management/requirements_txt_writer.py

@@ -20,7 +20,10 @@ def add_to_file(
         if not original_lines[-1].endswith("\n"):
             original_lines[-1] += "\n"
 
-        requirement_lines = [f"{dep.requirement}\n" for dep in dependencies]
+        requirement_lines = [
+            f"{dep.requirement} \\\n{dep.build_hashes()}" for dep in dependencies
+        ]
+
         updated_lines = original_lines + requirement_lines
 
         diff = create_diff(original_lines, updated_lines)

src/codemodder/scripts/get_hashes.py

@@ -0,0 +1,48 @@
+import requests
+import sys
+
+
+def get_package_hashes(package_name: str, version: str) -> list[str]:
+    """
+    Fetch the SHA256 hashes for a given package version from PyPI.
+    """
+    url = f"https://pypi.org/pypi/{package_name}/{version}/json"
+    response = requests.get(url, timeout=60)
+    hashes = []
+
+    if response.status_code == 200:
+        data = response.json()
+        for release in data.get("urls", []):
+            sha256 = release.get("digests", {}).get("sha256")
+            if sha256:
+                hashes.append(sha256)
+    else:
+        print(f"Failed to fetch data for {package_name}=={version}", file=sys.stderr)
+
+    return hashes
+
+
+def main():
+    if len(sys.argv) < 2:
+        print("Usage: python script.py package1==version package2==version")
+        sys.exit(1)
+    for arg in sys.argv[1:]:
+        if "==" not in arg:
+            print(
+                f"Invalid format '{arg}'. Expected format: PackageName==Version",
+                file=sys.stderr,
+            )
+            continue
+
+        package_name, version = arg.split("==", 1)
+        hashes = get_package_hashes(package_name, version)
+        if hashes:
+            print(f"SHA256 hashes for {package_name}=={version}:")
+            for hash_value in hashes:
+                print(hash_value)
+        else:
+            print(f"No hashes found for {package_name}=={version}")
+
+
+if __name__ == "__main__":
+    main()

tests/dependency_management/test_pyproject_writer.py

@@ -45,7 +45,7 @@ def test_update_pyproject_dependencies(tmpdir, dry_run):
     dependencies = [DefusedXML, Security]
     changeset = writer.write(dependencies, dry_run=dry_run)
 
-    updated_pyproject = """\
+    updated_pyproject = f"""\
         [build-system]
         requires = ["setuptools", "setuptools_scm>=8"]
         build-backend = "setuptools.build_meta"
@@ -61,8 +61,8 @@ def test_update_pyproject_dependencies(tmpdir, dry_run):
             "libcst~=1.1.0",
             "pylint~=3.0.0",
             "PyYAML~=6.0.0",
-            "defusedxml~=0.7.1",
-            "security~=1.2.0",
+            "{DefusedXML.requirement}",
+            "{Security.requirement}",
         ]
     """
 
@@ -79,8 +79,8 @@ def test_update_pyproject_dependencies(tmpdir, dry_run):
         """     "libcst~=1.1.0",\n"""
         """     "pylint~=3.0.0",\n"""
         """     "PyYAML~=6.0.0",\n"""
-        """+    "defusedxml~=0.7.1",\n"""
-        """+    "security~=1.2.0",\n"""
+        f"""+    "{DefusedXML.requirement}",\n"""
+        f"""+    "{Security.requirement}",\n"""
         " ]\n "
     )
     assert changeset.diff == res
@@ -136,7 +136,7 @@ def test_add_same_dependency_only_once(tmpdir):
     dependencies = [Security, Security]
     writer.write(dependencies)
 
-    updated_pyproject = """\
+    updated_pyproject = f"""\
         [build-system]
         requires = ["setuptools", "setuptools_scm>=8"]
         build-backend = "setuptools.build_meta"
@@ -150,7 +150,7 @@ def test_add_same_dependency_only_once(tmpdir):
             "libcst~=1.1.0",
             "pylint~=3.0.0",
             "PyYAML~=6.0.0",
-            "security~=1.2.0",
+            "{Security.requirement}",
         ]
     """
 

tests/dependency_management/test_requirements_txt_writer.py

@@ -32,7 +32,7 @@ def test_add_dependencies_preserve_comments(self, tmpdir, dry_run):
         assert dependency_file.read_text(encoding="utf-8") == (
             contents
             if dry_run
-            else "# comment\n\nrequests\ndefusedxml~=0.7.1\nsecurity~=1.2.0\n"
+            else f"# comment\n\nrequests\n{DefusedXML.requirement} \\\n{DefusedXML.build_hashes()}{Security.requirement} \\\n{Security.build_hashes()}"
         )
 
         assert changeset is not None
@@ -44,8 +44,10 @@ def test_add_dependencies_preserve_comments(self, tmpdir, dry_run):
             " # comment\n"
             " \n"
             " requests\n"
-            "+defusedxml~=0.7.1\n"
-            "+security~=1.2.0\n"
+            f"+{DefusedXML.requirement} \\\n"
+            f"{DefusedXML.build_hashes()}\n"
+            f"+{Security.requirement} \\\n"
+            f"{Security.build_hashes()}"
         )
         assert len(changeset.changes) == 2
         change_one = changeset.changes[0]
@@ -81,12 +83,12 @@ def test_add_same_dependency_only_once(self, tmpdir):
         assert len(changeset.changes) == 1
 
         assert dependency_file.read_text(encoding="utf-8") == (
-            "requests\nsecurity~=1.2.0\n"
+            f"requests\n{Security.requirement} \\\n{Security.build_hashes()}"
         )
 
     def test_dont_add_existing_dependency(self, tmpdir):
         dependency_file = Path(tmpdir) / "requirements.txt"
-        contents = "requests\nsecurity~=1.2.0\n"
+        contents = f"requests\n{Security.requirement}\n"
         dependency_file.write_text(contents, encoding="utf-8")
 
         store = PackageStore(
@@ -138,7 +140,7 @@ def test_dependency_file_no_terminating_newline(self, tmpdir):
 
         assert (
             dependency_file.read_text(encoding="utf-8")
-            == "# comment\n\nrequests\ndefusedxml~=0.7.1\nsecurity~=1.2.0\n"
+            == f"# comment\n\nrequests\n{DefusedXML.requirement} \\\n{DefusedXML.build_hashes()}{Security.requirement} \\\n{Security.build_hashes()}"
         )
 
         assert changeset is not None
@@ -150,6 +152,8 @@ def test_dependency_file_no_terminating_newline(self, tmpdir):
             " # comment\n"
             " \n"
             " requests\n"
-            "+defusedxml~=0.7.1\n"
-            "+security~=1.2.0\n"
+            f"+{DefusedXML.requirement} \\\n"
+            f"{DefusedXML.build_hashes()}\n"
+            f"+{Security.requirement} \\\n"
+            f"{Security.build_hashes()}"
         )