diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index fedaa7fc..efe1ac11 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -37,4 +37,5 @@ repos: "types-mock==5.0.*", "types-PyYAML==6.0", "types-toml~=0.10", + "types-requests~=2.13", ] diff --git a/integration_tests/test_flask_enable_csrf_protection.py b/integration_tests/test_flask_enable_csrf_protection.py index 5e26f5da..2962af12 100644 --- a/integration_tests/test_flask_enable_csrf_protection.py +++ b/integration_tests/test_flask_enable_csrf_protection.py @@ -3,6 +3,7 @@ BaseIntegrationTest, original_and_expected_from_code_path, ) +from codemodder.dependency import FlaskWTF class TestFlaskEnableCSRFProtection(BaseIntegrationTest): @@ -34,3 +35,15 @@ class TestFlaskEnableCSRFProtection(BaseIntegrationTest): expected_line_change = "3" change_description = FlaskEnableCSRFProtection.change_description num_changed_files = 2 + + requirements_path = "tests/samples/requirements.txt" + original_requirements = "# file used to test dependency management\nrequests==2.31.0\nblack==23.7.*\nmypy~=1.4\npylint>1\n" + expected_new_reqs = ( + f"# file used to test dependency management\n" + "requests==2.31.0\n" + "black==23.7.*\n" + "mypy~=1.4\n" + "pylint>1\n" + f"{FlaskWTF.requirement} \\\n" + f"{FlaskWTF.build_hashes()}" + ) diff --git a/integration_tests/test_process_sandbox.py b/integration_tests/test_process_sandbox.py index ba6aa5e3..f5ffe020 100644 --- a/integration_tests/test_process_sandbox.py +++ b/integration_tests/test_process_sandbox.py @@ -3,6 +3,7 @@ BaseIntegrationTest, original_and_expected_from_code_path, ) +from codemodder.dependency import Security class TestProcessSandbox(BaseIntegrationTest): @@ -26,4 +27,12 @@ class TestProcessSandbox(BaseIntegrationTest): requirements_path = "tests/samples/requirements.txt" original_requirements = "# file used to test dependency management\nrequests==2.31.0\nblack==23.7.*\nmypy~=1.4\npylint>1\n" - expected_new_reqs = "# file used to test dependency management\nrequests==2.31.0\nblack==23.7.*\nmypy~=1.4\npylint>1\nsecurity~=1.2.0\n" + expected_new_reqs = ( + f"# file used to test dependency management\n" + "requests==2.31.0\n" + "black==23.7.*\n" + "mypy~=1.4\n" + "pylint>1\n" + f"{Security.requirement} \\\n" + f"{Security.build_hashes()}" + ) diff --git a/integration_tests/test_url_sandbox.py b/integration_tests/test_url_sandbox.py index 4fba2507..0863923b 100644 --- a/integration_tests/test_url_sandbox.py +++ b/integration_tests/test_url_sandbox.py @@ -3,6 +3,7 @@ BaseIntegrationTest, original_and_expected_from_code_path, ) +from codemodder.dependency import Security class TestUrlSandbox(BaseIntegrationTest): @@ -36,4 +37,12 @@ class TestUrlSandbox(BaseIntegrationTest): requirements_path = "tests/samples/requirements.txt" original_requirements = "# file used to test dependency management\nrequests==2.31.0\nblack==23.7.*\nmypy~=1.4\npylint>1\n" - expected_new_reqs = "# file used to test dependency management\nrequests==2.31.0\nblack==23.7.*\nmypy~=1.4\npylint>1\nsecurity~=1.2.0\n" + expected_new_reqs = ( + f"# file used to test dependency management\n" + "requests==2.31.0\n" + "black==23.7.*\n" + "mypy~=1.4\n" + "pylint>1\n" + f"{Security.requirement} \\\n" + f"{Security.build_hashes()}" + ) diff --git a/integration_tests/test_use_defusedxml.py b/integration_tests/test_use_defusedxml.py index 681371ee..48dd2040 100644 --- a/integration_tests/test_use_defusedxml.py +++ b/integration_tests/test_use_defusedxml.py @@ -3,6 +3,7 @@ BaseIntegrationTest, original_and_expected_from_code_path, ) +from codemodder.dependency import DefusedXML class TestUseDefusedXml(BaseIntegrationTest): @@ -39,4 +40,12 @@ class TestUseDefusedXml(BaseIntegrationTest): requirements_path = "tests/samples/requirements.txt" original_requirements = "# file used to test dependency management\nrequests==2.31.0\nblack==23.7.*\nmypy~=1.4\npylint>1\n" - expected_new_reqs = "# file used to test dependency management\nrequests==2.31.0\nblack==23.7.*\nmypy~=1.4\npylint>1\ndefusedxml~=0.7.1\n" + expected_new_reqs = ( + f"# file used to test dependency management\n" + "requests==2.31.0\n" + "black==23.7.*\n" + "mypy~=1.4\n" + "pylint>1\n" + f"{DefusedXML.requirement} \\\n" + f"{DefusedXML.build_hashes()}" + ) diff --git a/pyproject.toml b/pyproject.toml index be8f1176..909c2778 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -42,6 +42,7 @@ Repository = "https://github.com/pixee/codemodder-python" [project.scripts] codemodder = "codemodder.codemodder:main" generate-docs = 'codemodder.scripts.generate_docs:main' +get-hashes = 'codemodder.scripts.get_hashes:main' [project.optional-dependencies] test = [ diff --git a/src/codemodder/dependency.py b/src/codemodder/dependency.py index 4a8f86c4..03e90e86 100644 --- a/src/codemodder/dependency.py +++ b/src/codemodder/dependency.py @@ -1,4 +1,4 @@ -from dataclasses import dataclass +from dataclasses import dataclass, field from packaging.requirements import Requirement @@ -16,6 +16,7 @@ class Dependency: _license: License oss_link: str package_link: str + hashes: list[str] = field(default_factory=list) @property def name(self) -> str: @@ -33,12 +34,19 @@ def build_description(self) -> str: [More facts]({self.package_link}) """ + def build_hashes(self) -> str: + return " \\\n".join(f"{' '*4}--hash=sha256:{sha256}" for sha256 in self.hashes) + def __hash__(self): return hash(self.requirement) FlaskWTF = Dependency( - Requirement("flask-wtf~=1.2.0"), + Requirement("flask-wtf==1.2.0"), + hashes=[ + "96e6f091c641c9944ba7dba2957c84797b630006aa926c99507fbd790069772b", + "e5dcf9f3cb80ee6ca8bb68de9ea467e7d613a708ebc5e130b9b02996e06c0d54", + ], description="""\ This package integrates WTForms into Flask. WTForms provides data validation and and CSRF protection which helps harden applications. """, @@ -51,7 +59,11 @@ def __hash__(self): ) DefusedXML = Dependency( - Requirement("defusedxml~=0.7.1"), + Requirement("defusedxml==0.7.1"), + hashes=[ + "a352e7e428770286cc899e2542b6cdaedb2b4953ff269a210103ec58f6198a61", + "1bb3032db185915b62d7c6209c5a8792be6a32ab2fedacc84e01b52c51aa3e69", + ], description="""\ This package is [recommended by the Python community](https://docs.python.org/3/library/xml.html#the-defusedxml-package) \ to protect against XML vulnerabilities.\ @@ -65,7 +77,11 @@ def __hash__(self): ) Security = Dependency( - Requirement("security~=1.2.0"), + Requirement("security==1.2.1"), + hashes=[ + "4ca5f8cfc6b836e2192a84bb5a28b72c17f3cd1abbfe3281f917394c6e6c9238", + "0a9dc7b457330e6d0f92bdae3603fecb85394beefad0fd3b5058758a58781ded", + ], description="""This library holds security tools for protecting Python API calls.""", _license=License( "MIT", diff --git a/src/codemodder/dependency_management/codemod_dependencies.txt b/src/codemodder/dependency_management/codemod_dependencies.txt new file mode 100644 index 00000000..08305e92 --- /dev/null +++ b/src/codemodder/dependency_management/codemod_dependencies.txt @@ -0,0 +1,9 @@ +# This is a temporary solution to use dependabot to alert us +# when the dependencies our codemods inject (for example `security`, `defusedxml`, etc) have a version update. + +# If this file gets a dependabot update PR, we must also update the corresponding +# dependency in dependency.py. Be sure to update the version AND the hashes. +# Run `get-hashes pkg==version` to get the hashes. +defusedxml==0.7.1 +flask-wtf==1.2.0 +security==1.2.1 diff --git a/src/codemodder/dependency_management/requirements_txt_writer.py b/src/codemodder/dependency_management/requirements_txt_writer.py index 8f7a2dd4..dda68a9b 100644 --- a/src/codemodder/dependency_management/requirements_txt_writer.py +++ b/src/codemodder/dependency_management/requirements_txt_writer.py @@ -20,7 +20,10 @@ def add_to_file( if not original_lines[-1].endswith("\n"): original_lines[-1] += "\n" - requirement_lines = [f"{dep.requirement}\n" for dep in dependencies] + requirement_lines = [ + f"{dep.requirement} \\\n{dep.build_hashes()}" for dep in dependencies + ] + updated_lines = original_lines + requirement_lines diff = create_diff(original_lines, updated_lines) diff --git a/src/codemodder/scripts/get_hashes.py b/src/codemodder/scripts/get_hashes.py new file mode 100644 index 00000000..0e08a2fb --- /dev/null +++ b/src/codemodder/scripts/get_hashes.py @@ -0,0 +1,48 @@ +import requests +import sys + + +def get_package_hashes(package_name: str, version: str) -> list[str]: + """ + Fetch the SHA256 hashes for a given package version from PyPI. + """ + url = f"https://pypi.org/pypi/{package_name}/{version}/json" + response = requests.get(url, timeout=60) + hashes = [] + + if response.status_code == 200: + data = response.json() + for release in data.get("urls", []): + sha256 = release.get("digests", {}).get("sha256") + if sha256: + hashes.append(sha256) + else: + print(f"Failed to fetch data for {package_name}=={version}", file=sys.stderr) + + return hashes + + +def main(): + if len(sys.argv) < 2: + print("Usage: python script.py package1==version package2==version") + sys.exit(1) + for arg in sys.argv[1:]: + if "==" not in arg: + print( + f"Invalid format '{arg}'. Expected format: PackageName==Version", + file=sys.stderr, + ) + continue + + package_name, version = arg.split("==", 1) + hashes = get_package_hashes(package_name, version) + if hashes: + print(f"SHA256 hashes for {package_name}=={version}:") + for hash_value in hashes: + print(hash_value) + else: + print(f"No hashes found for {package_name}=={version}") + + +if __name__ == "__main__": + main() diff --git a/tests/dependency_management/test_pyproject_writer.py b/tests/dependency_management/test_pyproject_writer.py index ec819021..7acabb87 100644 --- a/tests/dependency_management/test_pyproject_writer.py +++ b/tests/dependency_management/test_pyproject_writer.py @@ -45,7 +45,7 @@ def test_update_pyproject_dependencies(tmpdir, dry_run): dependencies = [DefusedXML, Security] changeset = writer.write(dependencies, dry_run=dry_run) - updated_pyproject = """\ + updated_pyproject = f"""\ [build-system] requires = ["setuptools", "setuptools_scm>=8"] build-backend = "setuptools.build_meta" @@ -61,8 +61,8 @@ def test_update_pyproject_dependencies(tmpdir, dry_run): "libcst~=1.1.0", "pylint~=3.0.0", "PyYAML~=6.0.0", - "defusedxml~=0.7.1", - "security~=1.2.0", + "{DefusedXML.requirement}", + "{Security.requirement}", ] """ @@ -79,8 +79,8 @@ def test_update_pyproject_dependencies(tmpdir, dry_run): """ "libcst~=1.1.0",\n""" """ "pylint~=3.0.0",\n""" """ "PyYAML~=6.0.0",\n""" - """+ "defusedxml~=0.7.1",\n""" - """+ "security~=1.2.0",\n""" + f"""+ "{DefusedXML.requirement}",\n""" + f"""+ "{Security.requirement}",\n""" " ]\n " ) assert changeset.diff == res @@ -136,7 +136,7 @@ def test_add_same_dependency_only_once(tmpdir): dependencies = [Security, Security] writer.write(dependencies) - updated_pyproject = """\ + updated_pyproject = f"""\ [build-system] requires = ["setuptools", "setuptools_scm>=8"] build-backend = "setuptools.build_meta" @@ -150,7 +150,7 @@ def test_add_same_dependency_only_once(tmpdir): "libcst~=1.1.0", "pylint~=3.0.0", "PyYAML~=6.0.0", - "security~=1.2.0", + "{Security.requirement}", ] """ diff --git a/tests/dependency_management/test_requirements_txt_writer.py b/tests/dependency_management/test_requirements_txt_writer.py index ee04529f..38cce529 100644 --- a/tests/dependency_management/test_requirements_txt_writer.py +++ b/tests/dependency_management/test_requirements_txt_writer.py @@ -32,7 +32,7 @@ def test_add_dependencies_preserve_comments(self, tmpdir, dry_run): assert dependency_file.read_text(encoding="utf-8") == ( contents if dry_run - else "# comment\n\nrequests\ndefusedxml~=0.7.1\nsecurity~=1.2.0\n" + else f"# comment\n\nrequests\n{DefusedXML.requirement} \\\n{DefusedXML.build_hashes()}{Security.requirement} \\\n{Security.build_hashes()}" ) assert changeset is not None @@ -44,8 +44,10 @@ def test_add_dependencies_preserve_comments(self, tmpdir, dry_run): " # comment\n" " \n" " requests\n" - "+defusedxml~=0.7.1\n" - "+security~=1.2.0\n" + f"+{DefusedXML.requirement} \\\n" + f"{DefusedXML.build_hashes()}\n" + f"+{Security.requirement} \\\n" + f"{Security.build_hashes()}" ) assert len(changeset.changes) == 2 change_one = changeset.changes[0] @@ -81,12 +83,12 @@ def test_add_same_dependency_only_once(self, tmpdir): assert len(changeset.changes) == 1 assert dependency_file.read_text(encoding="utf-8") == ( - "requests\nsecurity~=1.2.0\n" + f"requests\n{Security.requirement} \\\n{Security.build_hashes()}" ) def test_dont_add_existing_dependency(self, tmpdir): dependency_file = Path(tmpdir) / "requirements.txt" - contents = "requests\nsecurity~=1.2.0\n" + contents = f"requests\n{Security.requirement}\n" dependency_file.write_text(contents, encoding="utf-8") store = PackageStore( @@ -138,7 +140,7 @@ def test_dependency_file_no_terminating_newline(self, tmpdir): assert ( dependency_file.read_text(encoding="utf-8") - == "# comment\n\nrequests\ndefusedxml~=0.7.1\nsecurity~=1.2.0\n" + == f"# comment\n\nrequests\n{DefusedXML.requirement} \\\n{DefusedXML.build_hashes()}{Security.requirement} \\\n{Security.build_hashes()}" ) assert changeset is not None @@ -150,6 +152,8 @@ def test_dependency_file_no_terminating_newline(self, tmpdir): " # comment\n" " \n" " requests\n" - "+defusedxml~=0.7.1\n" - "+security~=1.2.0\n" + f"+{DefusedXML.requirement} \\\n" + f"{DefusedXML.build_hashes()}\n" + f"+{Security.requirement} \\\n" + f"{Security.build_hashes()}" ) diff --git a/tests/dependency_management/test_setup_py_writer.py b/tests/dependency_management/test_setup_py_writer.py index 59c45a18..39912e25 100644 --- a/tests/dependency_management/test_setup_py_writer.py +++ b/tests/dependency_management/test_setup_py_writer.py @@ -8,11 +8,8 @@ PackageStore, FileType, ) -from packaging.requirements import Requirement from codemodder.dependency import DefusedXML, Security -TEST_DEPENDENCIES = [Requirement("defusedxml==0.7.1"), Requirement("security~=1.2.0")] - def test_update_setuppy_comma_single_element_newline(tmpdir): original = """ @@ -46,7 +43,7 @@ def test_update_setuppy_comma_single_element_newline(tmpdir): dependencies = [DefusedXML, Security] writer.write(dependencies, dry_run=False) - after = """ + after = f""" from setuptools import setup setup( name="test pkg", @@ -54,14 +51,14 @@ def test_update_setuppy_comma_single_element_newline(tmpdir): long_description="...", author="Pixee", packages=find_packages("src"), - package_dir={"": "src"}, + package_dir={{"": "src"}}, python_requires=">3.6", install_requires=[ "protobuf>=3.12,<3.18; python_version < '3'", - "defusedxml~=0.7.1", - "security~=1.2.0", + "{DefusedXML.requirement}", + "{Security.requirement}", ], - entry_points={}, + entry_points={{}}, ) """ assert dependency_file.read() == dedent(after) @@ -97,7 +94,7 @@ def test_update_setuppy_comma_single_element_inline(tmpdir): dependencies = [DefusedXML, Security] writer.write(dependencies, dry_run=False) - after = """ + after = f""" from setuptools import setup setup( name="test pkg", @@ -105,10 +102,10 @@ def test_update_setuppy_comma_single_element_inline(tmpdir): long_description="...", author="Pixee", packages=find_packages("src"), - package_dir={"": "src"}, + package_dir={{"": "src"}}, python_requires=">3.6", - install_requires=["protobuf>=3.12,<3.18; python_version < '3'", "defusedxml~=0.7.1", "security~=1.2.0"], - entry_points={}, + install_requires=["protobuf>=3.12,<3.18; python_version < '3'", "{DefusedXML.requirement}", "{Security.requirement}"], + entry_points={{}}, ) """ assert dependency_file.read() == dedent(after) @@ -150,7 +147,7 @@ def test_update_setuppy_dependencies(tmpdir, dry_run): dependencies = [DefusedXML, Security] changeset = writer.write(dependencies, dry_run=dry_run) - after = """ + after = f""" from setuptools import setup setup( name="test pkg", @@ -158,17 +155,17 @@ def test_update_setuppy_dependencies(tmpdir, dry_run): long_description="...", author="Pixee", packages=find_packages("src"), - package_dir={"": "src"}, + package_dir={{"": "src"}}, python_requires=">3.6", install_requires=[ "protobuf>=3.12,<3.18; python_version < '3'", "protobuf>=3.12,<4; python_version >= '3'", "psutil>=5.7,<6", "requests>=2.4.2,<3", - "defusedxml~=0.7.1", - "security~=1.2.0", + "{DefusedXML.requirement}", + "{Security.requirement}", ], - entry_points={}, + entry_points={{}}, ) """ assert dependency_file.read() == (dedent(original) if dry_run else dedent(after)) @@ -182,8 +179,8 @@ def test_update_setuppy_dependencies(tmpdir, dry_run): """ "protobuf>=3.12,<4; python_version >= '3'",\n""" """ "psutil>=5.7,<6",\n""" """ "requests>=2.4.2,<3",\n""" - """+ "defusedxml~=0.7.1",\n""" - """+ "security~=1.2.0",\n""" + f"""+ "{DefusedXML.requirement}",\n""" + f"""+ "{Security.requirement}",\n""" " ],\n " " entry_points={},\n" " )\n" @@ -411,15 +408,16 @@ def test_setup_call_requirements_separate(tmpdir): dependencies = [DefusedXML, Security] changeset = writer.write(dependencies) - after = """ + after = f""" from setuptools import setup requirements = [ "protobuf>=3.12,<3.18; python_version < '3'", "protobuf>=3.12,<4; python_version >= '3'", "psutil>=5.7,<6", "requests>=2.4.2,<3", - "defusedxml==0.7.1", - "security~=1.2.0", + "{DefusedXML.requirement}", + "{Security.requirement}", + ] setup( name="test pkg", @@ -430,7 +428,7 @@ def test_setup_call_requirements_separate(tmpdir): package_dir={"": "src"}, python_requires=">3.6", install_requires=requirements, - entry_points={}, + entry_points={{}}, ) """ assert dependency_file.read() == dedent(after) @@ -446,8 +444,8 @@ def test_setup_call_requirements_separate(tmpdir): """ "psutil>=5.7,<6",\n""" """- "requests>=2.4.2,<3"\n""" """+ "requests>=2.4.2,<3",\n""" - """+ "defusedxml~=0.7.1",\n""" - """+ "security~=1.2.0",\n""" + f"""+ "{DefusedXML.requirement}",\n""" + f"""+ "{Security.requirement}",\n""" " ],\n " " entry_points={},\n" " )\n" diff --git a/tests/dependency_management/test_setupcfgt_writer.py b/tests/dependency_management/test_setupcfgt_writer.py index 48b665ac..ec1a8ce3 100644 --- a/tests/dependency_management/test_setupcfgt_writer.py +++ b/tests/dependency_management/test_setupcfgt_writer.py @@ -43,7 +43,7 @@ def test_update_dependencies(tmpdir, dry_run): dependencies = [DefusedXML, Security] changeset = writer.write(dependencies, dry_run=dry_run) - updated_setupcfg = """\ + updated_setupcfg = f"""\ [metadata] name = my_package version = attr: my_package.VERSION @@ -56,8 +56,8 @@ def test_update_dependencies(tmpdir, dry_run): install_requires = requests importlib-metadata; python_version<"3.8" - defusedxml~=0.7.1 - security~=1.2.0 + {DefusedXML.requirement} + {Security.requirement} """ assert setup_cfg.read() == ( @@ -73,8 +73,8 @@ def test_update_dependencies(tmpdir, dry_run): """ install_requires =\n""" """ requests\n""" """ importlib-metadata; python_version<"3.8"\n""" - """+ defusedxml~=0.7.1\n""" - """+ security~=1.2.0\n""" + f"""+ {DefusedXML.requirement}\n""" + f"""+ {Security.requirement}\n""" ) assert changeset.diff == res assert len(changeset.changes) == 2 @@ -127,7 +127,7 @@ def test_add_same_dependency_only_once(tmpdir): dependencies = [Security, Security] writer.write(dependencies) - updated_setupcfg = """\ + updated_setupcfg = f"""\ [metadata] name = my_package version = attr: my_package.VERSION @@ -140,7 +140,7 @@ def test_add_same_dependency_only_once(tmpdir): install_requires = requests importlib-metadata; python_version<"3.8" - security~=1.2.0 + {Security.requirement} """ assert setup_cfg.read() == dedent(updated_setupcfg) @@ -306,7 +306,7 @@ def test_cfg_inline_dependencies(tmpdir): dependencies = [Security, Security] changeset = writer.write(dependencies) - updated_setupcfg = """\ + updated_setupcfg = f"""\ [metadata] name = my_package version = attr: my_package.VERSION @@ -316,7 +316,7 @@ def test_cfg_inline_dependencies(tmpdir): [options] include_package_data = True python_requires = >=3.7 - install_requires = requests, importlib-metadata; python_version<"3.8", security~=1.2.0, + install_requires = requests, importlib-metadata; python_version<"3.8", {Security.requirement}, """ assert setup_cfg.read() == dedent(updated_setupcfg) @@ -329,7 +329,7 @@ def test_cfg_inline_dependencies(tmpdir): """ include_package_data = True\n""" """ python_requires = >=3.7\n""" """-install_requires = requests, importlib-metadata; python_version<"3.8"\n""" - """+install_requires = requests, importlib-metadata; python_version<"3.8", security~=1.2.0,\n""" + f"""+install_requires = requests, importlib-metadata; python_version<"3.8", {Security.requirement},\n""" ) assert changeset.diff == res assert len(changeset.changes) == 1