Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Requirements.txt writer adds hashes and a way to remember to update codemodder deps #273

Merged
merged 6 commits into from
Feb 21, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions .pre-commit-config.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -37,4 +37,5 @@ repos:
"types-mock==5.0.*",
"types-PyYAML==6.0",
"types-toml~=0.10",
"types-requests~=2.13",
]
13 changes: 13 additions & 0 deletions integration_tests/test_flask_enable_csrf_protection.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
BaseIntegrationTest,
original_and_expected_from_code_path,
)
from codemodder.dependency import FlaskWTF


class TestFlaskEnableCSRFProtection(BaseIntegrationTest):
Expand Down Expand Up @@ -34,3 +35,15 @@ class TestFlaskEnableCSRFProtection(BaseIntegrationTest):
expected_line_change = "3"
change_description = FlaskEnableCSRFProtection.change_description
num_changed_files = 2

requirements_path = "tests/samples/requirements.txt"
original_requirements = "# file used to test dependency management\nrequests==2.31.0\nblack==23.7.*\nmypy~=1.4\npylint>1\n"
expected_new_reqs = (
f"# file used to test dependency management\n"
"requests==2.31.0\n"
"black==23.7.*\n"
"mypy~=1.4\n"
"pylint>1\n"
f"{FlaskWTF.requirement} \\\n"
f"{FlaskWTF.build_hashes()}"
)
11 changes: 10 additions & 1 deletion integration_tests/test_process_sandbox.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
BaseIntegrationTest,
original_and_expected_from_code_path,
)
from codemodder.dependency import Security


class TestProcessSandbox(BaseIntegrationTest):
Expand All @@ -26,4 +27,12 @@ class TestProcessSandbox(BaseIntegrationTest):

requirements_path = "tests/samples/requirements.txt"
original_requirements = "# file used to test dependency management\nrequests==2.31.0\nblack==23.7.*\nmypy~=1.4\npylint>1\n"
expected_new_reqs = "# file used to test dependency management\nrequests==2.31.0\nblack==23.7.*\nmypy~=1.4\npylint>1\nsecurity~=1.2.0\n"
expected_new_reqs = (
f"# file used to test dependency management\n"
"requests==2.31.0\n"
"black==23.7.*\n"
"mypy~=1.4\n"
"pylint>1\n"
f"{Security.requirement} \\\n"
f"{Security.build_hashes()}"
)
11 changes: 10 additions & 1 deletion integration_tests/test_url_sandbox.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
BaseIntegrationTest,
original_and_expected_from_code_path,
)
from codemodder.dependency import Security


class TestUrlSandbox(BaseIntegrationTest):
Expand Down Expand Up @@ -36,4 +37,12 @@ class TestUrlSandbox(BaseIntegrationTest):

requirements_path = "tests/samples/requirements.txt"
original_requirements = "# file used to test dependency management\nrequests==2.31.0\nblack==23.7.*\nmypy~=1.4\npylint>1\n"
expected_new_reqs = "# file used to test dependency management\nrequests==2.31.0\nblack==23.7.*\nmypy~=1.4\npylint>1\nsecurity~=1.2.0\n"
expected_new_reqs = (
f"# file used to test dependency management\n"
"requests==2.31.0\n"
"black==23.7.*\n"
"mypy~=1.4\n"
"pylint>1\n"
f"{Security.requirement} \\\n"
f"{Security.build_hashes()}"
)
11 changes: 10 additions & 1 deletion integration_tests/test_use_defusedxml.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
BaseIntegrationTest,
original_and_expected_from_code_path,
)
from codemodder.dependency import DefusedXML


class TestUseDefusedXml(BaseIntegrationTest):
Expand Down Expand Up @@ -39,4 +40,12 @@ class TestUseDefusedXml(BaseIntegrationTest):

requirements_path = "tests/samples/requirements.txt"
original_requirements = "# file used to test dependency management\nrequests==2.31.0\nblack==23.7.*\nmypy~=1.4\npylint>1\n"
expected_new_reqs = "# file used to test dependency management\nrequests==2.31.0\nblack==23.7.*\nmypy~=1.4\npylint>1\ndefusedxml~=0.7.1\n"
expected_new_reqs = (
f"# file used to test dependency management\n"
"requests==2.31.0\n"
"black==23.7.*\n"
"mypy~=1.4\n"
"pylint>1\n"
f"{DefusedXML.requirement} \\\n"
f"{DefusedXML.build_hashes()}"
)
1 change: 1 addition & 0 deletions pyproject.toml
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,7 @@ Repository = "https://github.com/pixee/codemodder-python"
[project.scripts]
codemodder = "codemodder.codemodder:main"
generate-docs = 'codemodder.scripts.generate_docs:main'
get-hashes = 'codemodder.scripts.get_hashes:main'

[project.optional-dependencies]
test = [
Expand Down
24 changes: 20 additions & 4 deletions src/codemodder/dependency.py
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
from dataclasses import dataclass
from dataclasses import dataclass, field

from packaging.requirements import Requirement

Expand All @@ -16,6 +16,7 @@ class Dependency:
_license: License
oss_link: str
package_link: str
hashes: list[str] = field(default_factory=list)

@property
def name(self) -> str:
Expand All @@ -33,12 +34,19 @@ def build_description(self) -> str:
[More facts]({self.package_link})
"""

def build_hashes(self) -> str:
return " \\\n".join(f"{' '*4}--hash=sha256:{sha256}" for sha256 in self.hashes)

def __hash__(self):
return hash(self.requirement)


FlaskWTF = Dependency(
Requirement("flask-wtf~=1.2.0"),
Requirement("flask-wtf==1.2.0"),
hashes=[
"96e6f091c641c9944ba7dba2957c84797b630006aa926c99507fbd790069772b",
"e5dcf9f3cb80ee6ca8bb68de9ea467e7d613a708ebc5e130b9b02996e06c0d54",
],
description="""\
This package integrates WTForms into Flask. WTForms provides data validation and and CSRF protection which helps harden applications.
""",
Expand All @@ -51,7 +59,11 @@ def __hash__(self):
)

DefusedXML = Dependency(
Requirement("defusedxml~=0.7.1"),
Requirement("defusedxml==0.7.1"),
hashes=[
"a352e7e428770286cc899e2542b6cdaedb2b4953ff269a210103ec58f6198a61",
"1bb3032db185915b62d7c6209c5a8792be6a32ab2fedacc84e01b52c51aa3e69",
],
description="""\
This package is [recommended by the Python community](https://docs.python.org/3/library/xml.html#the-defusedxml-package) \
to protect against XML vulnerabilities.\
Expand All @@ -65,7 +77,11 @@ def __hash__(self):
)

Security = Dependency(
Requirement("security~=1.2.0"),
Requirement("security==1.2.1"),
hashes=[
"4ca5f8cfc6b836e2192a84bb5a28b72c17f3cd1abbfe3281f917394c6e6c9238",
"0a9dc7b457330e6d0f92bdae3603fecb85394beefad0fd3b5058758a58781ded",
],
description="""This library holds security tools for protecting Python API calls.""",
_license=License(
"MIT",
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
# This is a temporary solution to use dependabot to alert us
# when the dependencies our codemods inject (for example `security`, `defusedxml`, etc) have a version update.

# If this file gets a dependabot update PR, we must also update the corresponding
# dependency in dependency.py. Be sure to update the version AND the hashes.
# Run `get-hashes pkg==version` to get the hashes.
defusedxml==0.7.1
flask-wtf==1.2.0
security==1.2.1
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,10 @@ def add_to_file(
if not original_lines[-1].endswith("\n"):
original_lines[-1] += "\n"

requirement_lines = [f"{dep.requirement}\n" for dep in dependencies]
requirement_lines = [
f"{dep.requirement} \\\n{dep.build_hashes()}" for dep in dependencies
]

updated_lines = original_lines + requirement_lines

diff = create_diff(original_lines, updated_lines)
Expand Down
48 changes: 48 additions & 0 deletions src/codemodder/scripts/get_hashes.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,48 @@
import requests
import sys


def get_package_hashes(package_name: str, version: str) -> list[str]:
"""
Fetch the SHA256 hashes for a given package version from PyPI.
"""
url = f"https://pypi.org/pypi/{package_name}/{version}/json"
response = requests.get(url, timeout=60)
hashes = []

if response.status_code == 200:
data = response.json()
for release in data.get("urls", []):
sha256 = release.get("digests", {}).get("sha256")
if sha256:
hashes.append(sha256)
else:
print(f"Failed to fetch data for {package_name}=={version}", file=sys.stderr)

return hashes


def main():
if len(sys.argv) < 2:
print("Usage: python script.py package1==version package2==version")
sys.exit(1)
for arg in sys.argv[1:]:
if "==" not in arg:
print(
f"Invalid format '{arg}'. Expected format: PackageName==Version",
file=sys.stderr,
)
continue

package_name, version = arg.split("==", 1)
hashes = get_package_hashes(package_name, version)
if hashes:
print(f"SHA256 hashes for {package_name}=={version}:")
for hash_value in hashes:
print(hash_value)
else:
print(f"No hashes found for {package_name}=={version}")


if __name__ == "__main__":
main()
14 changes: 7 additions & 7 deletions tests/dependency_management/test_pyproject_writer.py
Original file line number Diff line number Diff line change
Expand Up @@ -45,7 +45,7 @@ def test_update_pyproject_dependencies(tmpdir, dry_run):
dependencies = [DefusedXML, Security]
changeset = writer.write(dependencies, dry_run=dry_run)

updated_pyproject = """\
updated_pyproject = f"""\
[build-system]
requires = ["setuptools", "setuptools_scm>=8"]
build-backend = "setuptools.build_meta"
Expand All @@ -61,8 +61,8 @@ def test_update_pyproject_dependencies(tmpdir, dry_run):
"libcst~=1.1.0",
"pylint~=3.0.0",
"PyYAML~=6.0.0",
"defusedxml~=0.7.1",
"security~=1.2.0",
"{DefusedXML.requirement}",
"{Security.requirement}",
]
"""

Expand All @@ -79,8 +79,8 @@ def test_update_pyproject_dependencies(tmpdir, dry_run):
""" "libcst~=1.1.0",\n"""
""" "pylint~=3.0.0",\n"""
""" "PyYAML~=6.0.0",\n"""
"""+ "defusedxml~=0.7.1",\n"""
"""+ "security~=1.2.0",\n"""
f"""+ "{DefusedXML.requirement}",\n"""
f"""+ "{Security.requirement}",\n"""
" ]\n "
)
assert changeset.diff == res
Expand Down Expand Up @@ -136,7 +136,7 @@ def test_add_same_dependency_only_once(tmpdir):
dependencies = [Security, Security]
writer.write(dependencies)

updated_pyproject = """\
updated_pyproject = f"""\
[build-system]
requires = ["setuptools", "setuptools_scm>=8"]
build-backend = "setuptools.build_meta"
Expand All @@ -150,7 +150,7 @@ def test_add_same_dependency_only_once(tmpdir):
"libcst~=1.1.0",
"pylint~=3.0.0",
"PyYAML~=6.0.0",
"security~=1.2.0",
"{Security.requirement}",
]
"""

Expand Down
20 changes: 12 additions & 8 deletions tests/dependency_management/test_requirements_txt_writer.py
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,7 @@ def test_add_dependencies_preserve_comments(self, tmpdir, dry_run):
assert dependency_file.read_text(encoding="utf-8") == (
contents
if dry_run
else "# comment\n\nrequests\ndefusedxml~=0.7.1\nsecurity~=1.2.0\n"
else f"# comment\n\nrequests\n{DefusedXML.requirement} \\\n{DefusedXML.build_hashes()}{Security.requirement} \\\n{Security.build_hashes()}"
)

assert changeset is not None
Expand All @@ -44,8 +44,10 @@ def test_add_dependencies_preserve_comments(self, tmpdir, dry_run):
" # comment\n"
" \n"
" requests\n"
"+defusedxml~=0.7.1\n"
"+security~=1.2.0\n"
f"+{DefusedXML.requirement} \\\n"
f"{DefusedXML.build_hashes()}\n"
f"+{Security.requirement} \\\n"
f"{Security.build_hashes()}"
)
assert len(changeset.changes) == 2
change_one = changeset.changes[0]
Expand Down Expand Up @@ -81,12 +83,12 @@ def test_add_same_dependency_only_once(self, tmpdir):
assert len(changeset.changes) == 1

assert dependency_file.read_text(encoding="utf-8") == (
"requests\nsecurity~=1.2.0\n"
f"requests\n{Security.requirement} \\\n{Security.build_hashes()}"
)

def test_dont_add_existing_dependency(self, tmpdir):
dependency_file = Path(tmpdir) / "requirements.txt"
contents = "requests\nsecurity~=1.2.0\n"
contents = f"requests\n{Security.requirement}\n"
dependency_file.write_text(contents, encoding="utf-8")

store = PackageStore(
Expand Down Expand Up @@ -138,7 +140,7 @@ def test_dependency_file_no_terminating_newline(self, tmpdir):

assert (
dependency_file.read_text(encoding="utf-8")
== "# comment\n\nrequests\ndefusedxml~=0.7.1\nsecurity~=1.2.0\n"
== f"# comment\n\nrequests\n{DefusedXML.requirement} \\\n{DefusedXML.build_hashes()}{Security.requirement} \\\n{Security.build_hashes()}"
)

assert changeset is not None
Expand All @@ -150,6 +152,8 @@ def test_dependency_file_no_terminating_newline(self, tmpdir):
" # comment\n"
" \n"
" requests\n"
"+defusedxml~=0.7.1\n"
"+security~=1.2.0\n"
f"+{DefusedXML.requirement} \\\n"
f"{DefusedXML.build_hashes()}\n"
f"+{Security.requirement} \\\n"
f"{Security.build_hashes()}"
)
Loading
Loading