diff --git a/src/codemodder/codemodder.py b/src/codemodder/codemodder.py index ff1cd746..56a05551 100644 --- a/src/codemodder/codemodder.py +++ b/src/codemodder/codemodder.py @@ -16,7 +16,7 @@ from codemodder.change import ChangeSet from codemodder.code_directory import file_line_patterns, match_files from codemodder.context import CodemodExecutionContext -from codemodder.diff import create_diff as create_diff_from_lines +from codemodder.diff import create_diff_from_tree from codemodder.executor import CodemodExecutorWrapper from codemodder.project_analysis.python_repo_manager import PythonRepoManager from codemodder.report.codetf_reporter import report_default @@ -48,16 +48,6 @@ def find_semgrep_results( return run_semgrep(context, yaml_files) -def create_diff(original_tree: cst.Module, new_tree: cst.Module) -> str: - """ - Create a diff between the original and output trees. - """ - return create_diff_from_lines( - original_tree.code.splitlines(keepends=True), - new_tree.code.splitlines(keepends=True), - ) - - def apply_codemod_to_file( base_directory: Path, file_context, @@ -78,7 +68,7 @@ def apply_codemod_to_file( if output_tree.deep_equals(source_tree): return False - diff = create_diff(source_tree, output_tree) + diff = create_diff_from_tree(source_tree, output_tree) change_set = ChangeSet( str(file_context.file_path.relative_to(base_directory)), diff, diff --git a/src/codemodder/context.py b/src/codemodder/context.py index 8b83bce8..7d452ac2 100644 --- a/src/codemodder/context.py +++ b/src/codemodder/context.py @@ -6,7 +6,6 @@ from codemodder.change import ChangeSet from codemodder.dependency import Dependency -from codemodder.dependency_management import DependencyManager from codemodder.executor import CodemodExecutorWrapper from codemodder.file_context import FileContext from codemodder.logging import logger, log_list @@ -102,6 +101,9 @@ def process_dependencies(self, codemod_id: str): ) return + # pylint: disable-next=cyclic-import + from codemodder.dependency_management import DependencyManager + dm = DependencyManager(dependencies_store, self.directory) if (changeset := dm.write(list(dependencies), self.dry_run)) is not None: self.add_results(codemod_id, [changeset]) diff --git a/src/codemodder/dependency_management/base_dependency_writer.py b/src/codemodder/dependency_management/base_dependency_writer.py index 07129b5d..d7cb807b 100644 --- a/src/codemodder/dependency_management/base_dependency_writer.py +++ b/src/codemodder/dependency_management/base_dependency_writer.py @@ -2,9 +2,10 @@ from pathlib import Path from typing import Optional from codemodder.project_analysis.file_parsers.package_store import PackageStore -from codemodder.change import ChangeSet +from codemodder.change import Action, Change, ChangeSet, PackageAction, Result from codemodder.dependency import Dependency from packaging.requirements import Requirement +from typing import Callable, Union, List class DependencyWriter(metaclass=ABCMeta): @@ -38,3 +39,25 @@ def add(self, dependencies: list[Dependency]) -> list[Dependency]: self.dependency_store.dependencies.add(requirement) new.append(new_dep) return new + + def build_changes( + self, + dependencies: list[Dependency], + line_number_strategy: Callable, + strategy_arg: Union[int, List[str], List[int]], + ) -> list[Change]: + return [ + Change( + lineNumber=line_number_strategy(strategy_arg, i), + description=dep.build_description(), + # Contextual comments should be added to the right side of split diffs + properties={ + "contextual_description": True, + "contextual_description_position": "right", + }, + packageActions=[ + PackageAction(Action.ADD, Result.COMPLETED, str(dep.requirement)) + ], + ) + for i, dep in enumerate(dependencies) + ] diff --git a/src/codemodder/dependency_management/dependency_manager.py b/src/codemodder/dependency_management/dependency_manager.py index 73d40238..c8ed4230 100644 --- a/src/codemodder/dependency_management/dependency_manager.py +++ b/src/codemodder/dependency_management/dependency_manager.py @@ -5,7 +5,14 @@ RequirementsTxtWriter, ) from codemodder.dependency_management.pyproject_writer import PyprojectWriter -from codemodder.project_analysis.file_parsers.package_store import PackageStore +from codemodder.dependency_management.setup_py_writer import ( + SetupPyWriter, +) + +from codemodder.project_analysis.file_parsers.package_store import ( + PackageStore, + FileType, +) from pathlib import Path @@ -24,14 +31,16 @@ def write( Write `dependencies` to the appropriate location in the project. """ match self.dependencies_store.type: - case "requirements.txt": + case FileType.REQ_TXT: return RequirementsTxtWriter( self.dependencies_store, self.parent_directory ).write(dependencies, dry_run) - case "pyproject.toml": + case FileType.TOML: return PyprojectWriter( self.dependencies_store, self.parent_directory ).write(dependencies, dry_run) - case "setup.py": - pass + case FileType.SETUP_PY: + return SetupPyWriter( + self.dependencies_store, self.parent_directory + ).write(dependencies, dry_run) return None diff --git a/src/codemodder/dependency_management/pyproject_writer.py b/src/codemodder/dependency_management/pyproject_writer.py index e869fe28..cbdb64b2 100644 --- a/src/codemodder/dependency_management/pyproject_writer.py +++ b/src/codemodder/dependency_management/pyproject_writer.py @@ -2,9 +2,14 @@ from typing import Optional from copy import deepcopy from codemodder.dependency import Dependency -from codemodder.change import Action, Change, ChangeSet, PackageAction, Result +from codemodder.change import ChangeSet from codemodder.dependency_management.base_dependency_writer import DependencyWriter from codemodder.diff import create_diff_and_linenums +from codemodder.logging import logger + + +def added_line_nums_strategy(lines, i): + return lines[i] class PyprojectWriter(DependencyWriter): @@ -19,6 +24,7 @@ def add_to_file( [f"{dep.requirement}" for dep in dependencies] ) except tomlkit.exceptions.NonExistentKey: + logger.debug("Unable to add dependencies to pyproject.toml file.") return None diff, added_line_nums = create_diff_and_linenums( @@ -29,21 +35,9 @@ def add_to_file( with open(self.path, "w", encoding="utf-8") as f: tomlkit.dump(pyproject, f) - changes = [ - Change( - lineNumber=added_line_nums[i], - description=dep.build_description(), - # Contextual comments should be added to the right side of split diffs - properties={ - "contextual_description": True, - "contextual_description_position": "right", - }, - packageActions=[ - PackageAction(Action.ADD, Result.COMPLETED, str(dep.requirement)) - ], - ) - for i, dep in enumerate(dependencies) - ] + changes = self.build_changes( + dependencies, added_line_nums_strategy, added_line_nums + ) return ChangeSet( str(self.path.relative_to(self.parent_directory)), diff, diff --git a/src/codemodder/dependency_management/requirements_txt_writer.py b/src/codemodder/dependency_management/requirements_txt_writer.py index aaa2c884..49786dad 100644 --- a/src/codemodder/dependency_management/requirements_txt_writer.py +++ b/src/codemodder/dependency_management/requirements_txt_writer.py @@ -1,10 +1,14 @@ from typing import Optional from codemodder.dependency_management.base_dependency_writer import DependencyWriter -from codemodder.change import Action, Change, ChangeSet, PackageAction, Result +from codemodder.change import ChangeSet from codemodder.diff import create_diff from codemodder.dependency import Dependency +def original_lines_strategy(original_lines, i): + return len(original_lines) + i + 1 + + class RequirementsTxtWriter(DependencyWriter): def add_to_file( self, dependencies: list[Dependency], dry_run: bool = False @@ -23,21 +27,9 @@ def add_to_file( with open(self.path, "w", encoding="utf-8") as f: f.writelines(updated_lines) - changes = [ - Change( - lineNumber=len(original_lines) + i + 1, - description=dep.build_description(), - # Contextual comments should be added to the right side of split diffs - properties={ - "contextual_description": True, - "contextual_description_position": "right", - }, - packageActions=[ - PackageAction(Action.ADD, Result.COMPLETED, str(dep.requirement)) - ], - ) - for i, dep in enumerate(dependencies) - ] + changes = self.build_changes( + dependencies, original_lines_strategy, original_lines + ) return ChangeSet( str(self.path.relative_to(self.parent_directory)), diff, diff --git a/src/codemodder/dependency_management/setup_py_codemod.py b/src/codemodder/dependency_management/setup_py_writer.py similarity index 58% rename from src/codemodder/dependency_management/setup_py_codemod.py rename to src/codemodder/dependency_management/setup_py_writer.py index 0f3a239a..cbc7956a 100644 --- a/src/codemodder/dependency_management/setup_py_codemod.py +++ b/src/codemodder/dependency_management/setup_py_writer.py @@ -1,12 +1,59 @@ import libcst as cst from libcst.codemod import CodemodContext from libcst import matchers +from typing import Optional from codemodder.codemods.api import BaseCodemod from codemodder.codemods.base_codemod import ReviewGuidance from codemodder.codemods.utils import is_setup_py_file from codemodder.codemods.utils_mixin import NameResolutionMixin from codemodder.file_context import FileContext from packaging.requirements import Requirement +from codemodder.dependency import Dependency +from codemodder.dependency_management.base_dependency_writer import DependencyWriter +from codemodder.change import ChangeSet +from codemodder.diff import create_diff_from_tree + + +def fixed_line_number_strategy(line_num_changed, _): + return line_num_changed + + +class SetupPyWriter(DependencyWriter): + def add_to_file( + self, dependencies: list[Dependency], dry_run: bool = False + ) -> Optional[ChangeSet]: + input_tree = self._parse_file() + wrapper = cst.MetadataWrapper(input_tree) + file_context = FileContext(self.parent_directory, self.path, [], [], []) + + codemod = SetupPyAddDependencies( + CodemodContext(wrapper=wrapper), + file_context, + dependencies=[dep.requirement for dep in dependencies], + ) + + output_tree = codemod.transform_module(input_tree) + if codemod.line_num_changed is None: + return None + + diff = create_diff_from_tree(input_tree, output_tree) + + if not dry_run: + with open(self.path, "w", encoding="utf-8") as f: + f.write(output_tree.code) + + changes = self.build_changes( + dependencies, fixed_line_number_strategy, codemod.line_num_changed + ) + return ChangeSet( + str(self.path.relative_to(self.parent_directory)), + diff, + changes=changes, + ) + + def _parse_file(self): + with open(self.path, encoding="utf-8") as f: + return cst.parse_module(f.read()) class SetupPyAddDependencies(BaseCodemod, NameResolutionMixin): @@ -26,6 +73,7 @@ def __init__( NameResolutionMixin.__init__(self) self.filename = self.file_context.file_path self.dependencies = dependencies + self.line_num_changed = None def visit_Module(self, _: cst.Module) -> bool: """ @@ -54,6 +102,14 @@ def replace_arg(self, original_node: cst.Call): return new_args def add_dependencies_to_arg(self, arg: cst.Arg) -> cst.Arg: + if not arg.value.elements: + # If there are no current dependencies, don't do anything + return arg + + # we add the new dependencies in the same line as the last + # dependency listed in install_requires + self.line_num_changed = self.lineno_for_node(arg.value.elements[-1]) - 1 + new_dependencies = [ cst.Element(value=cst.SimpleString(value=f'"{str(dep)}"')) for dep in self.dependencies diff --git a/src/codemodder/diff.py b/src/codemodder/diff.py index d0aa25d8..6e740d61 100644 --- a/src/codemodder/diff.py +++ b/src/codemodder/diff.py @@ -1,4 +1,5 @@ import difflib +import libcst as cst def create_diff(original_lines: list[str], new_lines: list[str]) -> str: @@ -6,6 +7,16 @@ def create_diff(original_lines: list[str], new_lines: list[str]) -> str: return difflines_to_str(diff_lines) +def create_diff_from_tree(original_tree: cst.Module, new_tree: cst.Module) -> str: + """ + Create a diff between the original and output trees. + """ + return create_diff( + original_tree.code.splitlines(keepends=True), + new_tree.code.splitlines(keepends=True), + ) + + def create_diff_and_linenums( original_lines: list[str], new_lines: list[str] ) -> tuple[str, list[int]]: diff --git a/src/codemodder/project_analysis/file_parsers/base_parser.py b/src/codemodder/project_analysis/file_parsers/base_parser.py index 7019aa21..beb7023d 100644 --- a/src/codemodder/project_analysis/file_parsers/base_parser.py +++ b/src/codemodder/project_analysis/file_parsers/base_parser.py @@ -12,7 +12,7 @@ def __init__(self, parent_directory: Path): @property @abstractmethod - def file_name(self): + def file_type(self): ... # pragma: no cover def _parse_dependencies(self, dependencies: List[str]): @@ -28,7 +28,7 @@ def _parse_file(self, file: Path): ... # pragma: no cover def find_file_locations(self) -> List[Path]: - return list(Path(self.parent_directory).rglob(self.file_name)) + return list(Path(self.parent_directory).rglob(self.file_type.value)) def parse(self) -> list[PackageStore]: """ diff --git a/src/codemodder/project_analysis/file_parsers/package_store.py b/src/codemodder/project_analysis/file_parsers/package_store.py index 7c9b9cb9..5fc29bee 100644 --- a/src/codemodder/project_analysis/file_parsers/package_store.py +++ b/src/codemodder/project_analysis/file_parsers/package_store.py @@ -1,10 +1,18 @@ from dataclasses import dataclass +from enum import Enum from packaging.requirements import Requirement +class FileType(Enum): + REQ_TXT = "requirements.txt" + TOML = "pyproject.toml" + SETUP_PY = "setup.py" + SETUP_CFG = "setup.cfg" + + @dataclass class PackageStore: - type: str + type: FileType file: str dependencies: set[Requirement] py_versions: list[str] diff --git a/src/codemodder/project_analysis/file_parsers/pyproject_toml_file_parser.py b/src/codemodder/project_analysis/file_parsers/pyproject_toml_file_parser.py index 9fc4a9a5..fbe13d26 100644 --- a/src/codemodder/project_analysis/file_parsers/pyproject_toml_file_parser.py +++ b/src/codemodder/project_analysis/file_parsers/pyproject_toml_file_parser.py @@ -1,4 +1,7 @@ -from codemodder.project_analysis.file_parsers.package_store import PackageStore +from codemodder.project_analysis.file_parsers.package_store import ( + PackageStore, + FileType, +) from pathlib import Path import toml @@ -7,8 +10,8 @@ class PyprojectTomlParser(BaseParser): @property - def file_name(self): - return "pyproject.toml" + def file_type(self): + return FileType.TOML def _parse_dependencies_from_toml(self, toml_data: dict): # todo: handle cases for @@ -27,7 +30,7 @@ def _parse_file(self, file: Path): # todo: handle no "project" in data return PackageStore( - type=self.file_name, + type=self.file_type, file=str(file), dependencies=set(self._parse_dependencies_from_toml(data)), py_versions=self._parse_py_versions(data), diff --git a/src/codemodder/project_analysis/file_parsers/requirements_txt_file_parser.py b/src/codemodder/project_analysis/file_parsers/requirements_txt_file_parser.py index e6d24ec8..7d1ba2f9 100644 --- a/src/codemodder/project_analysis/file_parsers/requirements_txt_file_parser.py +++ b/src/codemodder/project_analysis/file_parsers/requirements_txt_file_parser.py @@ -1,7 +1,10 @@ from typing import Optional from packaging.requirements import InvalidRequirement -from codemodder.project_analysis.file_parsers.package_store import PackageStore +from codemodder.project_analysis.file_parsers.package_store import ( + PackageStore, + FileType, +) from pathlib import Path from .base_parser import BaseParser import chardet @@ -10,15 +13,14 @@ class RequirementsTxtParser(BaseParser): @property - def file_name(self): - return "requirements.txt" + def file_type(self): + return FileType.REQ_TXT def _parse_file(self, file: Path) -> Optional[PackageStore]: try: with open(file, "rb") as f: whole_file = f.read() enc = chardet.detect(whole_file) - lines = [] if enc["confidence"] > 0.9: encoding = enc.get("encoding") decoded = whole_file.decode(encoding.lower()) if encoding else "" @@ -26,7 +28,7 @@ def _parse_file(self, file: Path) -> Optional[PackageStore]: else: raise UnicodeError() return PackageStore( - type=self.file_name, + type=self.file_type, file=str(file), dependencies=set(self._parse_dependencies(lines)), # requirements.txt files do not declare py versions explicitly diff --git a/src/codemodder/project_analysis/file_parsers/setup_cfg_file_parser.py b/src/codemodder/project_analysis/file_parsers/setup_cfg_file_parser.py index 99124dbd..dc8bfad5 100644 --- a/src/codemodder/project_analysis/file_parsers/setup_cfg_file_parser.py +++ b/src/codemodder/project_analysis/file_parsers/setup_cfg_file_parser.py @@ -1,4 +1,7 @@ -from codemodder.project_analysis.file_parsers.package_store import PackageStore +from codemodder.project_analysis.file_parsers.package_store import ( + PackageStore, + FileType, +) from pathlib import Path import configparser @@ -7,8 +10,8 @@ class SetupCfgParser(BaseParser): @property - def file_name(self): - return "setup.cfg" + def file_type(self): + return FileType.SETUP_CFG def _parse_dependencies_from_cfg(self, config: configparser.ConfigParser): # todo: handle cases for @@ -30,7 +33,7 @@ def _parse_file(self, file: Path): # todo: handle no config, no "options" in config return PackageStore( - type=self.file_name, + type=self.file_type, file=str(file), dependencies=set(self._parse_dependencies_from_cfg(config)), py_versions=self._parse_py_versions(config), diff --git a/src/codemodder/project_analysis/file_parsers/setup_py_file_parser.py b/src/codemodder/project_analysis/file_parsers/setup_py_file_parser.py index 7569216d..6ad3fde3 100644 --- a/src/codemodder/project_analysis/file_parsers/setup_py_file_parser.py +++ b/src/codemodder/project_analysis/file_parsers/setup_py_file_parser.py @@ -1,4 +1,7 @@ -from codemodder.project_analysis.file_parsers.package_store import PackageStore +from codemodder.project_analysis.file_parsers.package_store import ( + PackageStore, + FileType, +) from codemodder.utils.utils import clean_simplestring from pathlib import Path import libcst as cst @@ -11,8 +14,8 @@ class SetupPyParser(BaseParser): @property - def file_name(self): - return "setup.py" + def file_type(self): + return FileType.SETUP_PY def _parse_dependencies(self, dependencies): return [ @@ -39,7 +42,7 @@ def _parse_file(self, file: Path): # todo: handle no python_requires, install_requires return PackageStore( - type=self.file_name, + type=self.file_type, file=str(file), dependencies=set( self._parse_dependencies_from_cst(visitor.install_requires) diff --git a/tests/dependency_management/test_pyproject_writer.py b/tests/dependency_management/test_pyproject_writer.py index fa8fe5b3..f3ccffb3 100644 --- a/tests/dependency_management/test_pyproject_writer.py +++ b/tests/dependency_management/test_pyproject_writer.py @@ -3,7 +3,10 @@ from codemodder.dependency_management.pyproject_writer import PyprojectWriter from codemodder.dependency import DefusedXML, Security -from codemodder.project_analysis.file_parsers.package_store import PackageStore +from codemodder.project_analysis.file_parsers.package_store import ( + PackageStore, + FileType, +) @pytest.mark.parametrize("dry_run", [True, False]) @@ -13,6 +16,8 @@ def test_update_pyproject_dependencies(tmpdir, dry_run): requires = ["setuptools", "setuptools_scm>=8"] build-backend = "setuptools.build_meta" + # some comment + [project] dynamic = ["version"] name = "codemodder" @@ -29,7 +34,7 @@ def test_update_pyproject_dependencies(tmpdir, dry_run): pyproject_toml.write(dedent(orig_pyproject)) store = PackageStore( - type="requirements.txt", + type=FileType.REQ_TXT, file=str(pyproject_toml), dependencies=set(), py_versions=[">=3.10.0"], @@ -44,6 +49,8 @@ def test_update_pyproject_dependencies(tmpdir, dry_run): requires = ["setuptools", "setuptools_scm>=8"] build-backend = "setuptools.build_meta" + # some comment + [project] dynamic = ["version"] name = "codemodder" @@ -67,7 +74,7 @@ def test_update_pyproject_dependencies(tmpdir, dry_run): res = ( "--- \n" "+++ \n" - "@@ -11,5 +11,7 @@\n" + "@@ -13,5 +13,7 @@\n" """ "libcst~=1.1.0",\n""" """ "pylint~=3.0.0",\n""" """ "PyYAML~=6.0.0",\n""" @@ -79,14 +86,14 @@ def test_update_pyproject_dependencies(tmpdir, dry_run): assert len(changeset.changes) == 2 change_one = changeset.changes[0] - assert change_one.lineNumber == 14 + assert change_one.lineNumber == 16 assert change_one.description == DefusedXML.build_description() assert change_one.properties == { "contextual_description": True, "contextual_description_position": "right", } change_two = changeset.changes[1] - assert change_two.lineNumber == 15 + assert change_two.lineNumber == 17 assert change_two.description == Security.build_description() assert change_two.properties == { "contextual_description": True, @@ -116,7 +123,7 @@ def test_add_same_dependency_only_once(tmpdir): pyproject_toml.write(dedent(orig_pyproject)) store = PackageStore( - type="requirements.txt", + type=FileType.REQ_TXT, file=str(pyproject_toml), dependencies=set(), py_versions=[">=3.10.0"], @@ -169,7 +176,7 @@ def test_dont_add_existing_dependency(tmpdir): pyproject_toml.write(dedent(orig_pyproject)) store = PackageStore( - type="requirements.txt", + type=FileType.REQ_TXT, file=str(pyproject_toml), dependencies=set([Security.requirement]), py_versions=[">=3.10.0"], @@ -195,7 +202,7 @@ def test_pyproject_no_dependencies(tmpdir): pyproject_toml.write(dedent(orig_pyproject)) store = PackageStore( - type="requirements.txt", + type=FileType.REQ_TXT, file=str(pyproject_toml), dependencies=set(), py_versions=[">=3.10.0"], diff --git a/tests/dependency_management/test_requirements_txt_writer.py b/tests/dependency_management/test_requirements_txt_writer.py index fd5eb541..4f6a31da 100644 --- a/tests/dependency_management/test_requirements_txt_writer.py +++ b/tests/dependency_management/test_requirements_txt_writer.py @@ -3,7 +3,10 @@ from codemodder.dependency_management.requirements_txt_writer import ( RequirementsTxtWriter, ) -from codemodder.project_analysis.file_parsers.package_store import PackageStore +from codemodder.project_analysis.file_parsers.package_store import ( + PackageStore, + FileType, +) from codemodder.dependency import DefusedXML, Security @@ -14,7 +17,7 @@ def test_add_dependencies_preserve_comments(self, tmpdir, dry_run): dependency_file = Path(tmpdir) / "requirements.txt" dependency_file.write_text(contents, encoding="utf-8") store = PackageStore( - type="requirements.txt", + type=FileType.REQ_TXT, file=str(dependency_file), dependencies=set(), py_versions=[], @@ -62,7 +65,7 @@ def test_add_same_dependency_only_once(self, tmpdir): dependency_file.write_text("requests\n", encoding="utf-8") store = PackageStore( - type="requirements.txt", + type=FileType.REQ_TXT, file=str(dependency_file), dependencies=set(), py_versions=[], @@ -82,7 +85,7 @@ def test_dont_add_existing_dependency(self, tmpdir): dependency_file.write_text(contents, encoding="utf-8") store = PackageStore( - type="requirements.txt", + type=FileType.REQ_TXT, file=str(dependency_file), dependencies=set([Security.requirement]), py_versions=[], @@ -99,7 +102,7 @@ def test_dependency_file_no_terminating_newline(self, tmpdir): dependency_file.write_text(contents, encoding="utf-8") store = PackageStore( - type="requirements.txt", + type=FileType.REQ_TXT, file=str(dependency_file), dependencies=set(), py_versions=[], diff --git a/tests/dependency_management/test_setup_py_codemod.py b/tests/dependency_management/test_setup_py_codemod.py deleted file mode 100644 index b9133303..00000000 --- a/tests/dependency_management/test_setup_py_codemod.py +++ /dev/null @@ -1,218 +0,0 @@ -import pytest -import libcst as cst -from codemodder.dependency_management.setup_py_codemod import SetupPyAddDependencies -from libcst.codemod import CodemodContext -from tests.codemods.base_codemod_test import BaseCodemodTest -from packaging.requirements import Requirement -from pathlib import Path - -TEST_DEPENDENCIES = [Requirement("defusedxml==0.7.1"), Requirement("security~=1.2.0")] - - -class TestSetupPyCodemod(BaseCodemodTest): - codemod = SetupPyAddDependencies - - def initialize_codemod(self, input_tree): - """This codemod is initialized with different args than other codemods.""" - wrapper = cst.MetadataWrapper(input_tree) - codemod_instance = self.codemod( - CodemodContext(wrapper=wrapper), - self.file_context, - dependencies=TEST_DEPENDENCIES, - ) - return codemod_instance - - def test_setup_call(self, tmpdir): - before = """ - from setuptools import setup - setup( - name="test pkg", - description="testing", - long_description="...", - author="Pixee", - packages=find_packages("src"), - package_dir={"": "src"}, - python_requires=">3.6", - install_requires=[ - "protobuf>=3.12,<3.18; python_version < '3'", - "protobuf>=3.12,<4; python_version >= '3'", - "psutil>=5.7,<6", - "requests>=2.4.2,<3" - ], - entry_points={}, - ) - """ - - after = """ - from setuptools import setup - setup( - name="test pkg", - description="testing", - long_description="...", - author="Pixee", - packages=find_packages("src"), - package_dir={"": "src"}, - python_requires=">3.6", - install_requires=[ - "protobuf>=3.12,<3.18; python_version < '3'", - "protobuf>=3.12,<4; python_version >= '3'", - "psutil>=5.7,<6", - "requests>=2.4.2,<3", "defusedxml==0.7.1", "security~=1.2.0" - ], - entry_points={}, - ) - """ - tmp_file_path = Path(tmpdir / "setup.py") - self.run_and_assert_filepath(tmpdir, tmp_file_path, before, after) - - def test_other_setup_func(self, tmpdir): - before = """ - from something import setup - setup( - name="test pkg", - install_requires=[ - "protobuf>=3.12,<3.18; python_version < '3'", - "protobuf>=3.12,<4; python_version >= '3'", - "psutil>=5.7,<6", - "requests>=2.4.2,<3" - ], - entry_points={}, - ) - """ - tmp_file_path = Path(tmpdir / "setup.py") - self.run_and_assert_filepath(tmpdir, tmp_file_path, before, before) - - def test_not_setup_file(self, tmpdir): - before = """ - from setuptools import setup - setup( - name="test pkg", - description="testing", - long_description="...", - author="Pixee", - packages=find_packages("src"), - package_dir={"": "src"}, - python_requires=">3.6", - install_requires=[ - "protobuf>=3.12,<3.18; python_version < '3'", - "protobuf>=3.12,<4; python_version >= '3'", - "psutil>=5.7,<6", - "requests>=2.4.2,<3" - ], - entry_points={}, - ) - """ - tmp_file_path = Path(tmpdir / "not-setup.py") - self.run_and_assert_filepath(tmpdir, tmp_file_path, before, before) - - def test_setup_call_no_install_requires(self, tmpdir): - before = """ - from setuptools import setup - setup( - name="test pkg", - description="testing", - long_description="...", - author="Pixee", - packages=find_packages("src"), - package_dir={"": "src"}, - python_requires=">3.6", - ) - """ - tmp_file_path = Path(tmpdir / "setup.py") - self.run_and_assert_filepath(tmpdir, tmp_file_path, before, before) - - def test_setup_no_existing_requirements(self, tmpdir): - before = """ - from setuptools import setup - setup( - name="test pkg", - description="testing", - long_description="...", - author="Pixee", - packages=find_packages("src"), - package_dir={"": "src"}, - python_requires=">3.6", - install_requires=[], - entry_points={}, - ) - """ - - after = """ - from setuptools import setup - setup( - name="test pkg", - description="testing", - long_description="...", - author="Pixee", - packages=find_packages("src"), - package_dir={"": "src"}, - python_requires=">3.6", - install_requires=["defusedxml==0.7.1", "security~=1.2.0"], - entry_points={}, - ) - """ - tmp_file_path = Path(tmpdir / "setup.py") - self.run_and_assert_filepath(tmpdir, tmp_file_path, before, after) - - def test_setup_call_bad_install_requires(self, tmpdir): - before = """ - from setuptools import setup - setup( - name="test pkg", - description="testing", - long_description="...", - author="Pixee", - packages=find_packages("src"), - package_dir={"": "src"}, - python_requires=">3.6", - install_requires="some-package", - ) - """ - tmp_file_path = Path(tmpdir / "setup.py") - self.run_and_assert_filepath(tmpdir, tmp_file_path, before, before) - - @pytest.mark.skip("Need to add support.") - def test_setup_call_requirements_separate(self, tmpdir): - before = """ - from setuptools import setup - requirements = [ - "protobuf>=3.12,<3.18; python_version < '3'", - "protobuf>=3.12,<4; python_version >= '3'", - "psutil>=5.7,<6", - "requests>=2.4.2,<3" - ] - setup( - name="test pkg", - description="testing", - long_description="...", - author="Pixee", - packages=find_packages("src"), - package_dir={"": "src"}, - python_requires=">3.6", - install_requires=requirements, - entry_points={}, - ) - """ - - after = """ - from setuptools import setup - requirements = [ - "protobuf>=3.12,<3.18; python_version < '3'", - "protobuf>=3.12,<4; python_version >= '3'", - "psutil>=5.7,<6", - "requests>=2.4.2,<3", "defusedxml==0.7.1", "security~=1.2.0" - ] - setup( - name="test pkg", - description="testing", - long_description="...", - author="Pixee", - packages=find_packages("src"), - package_dir={"": "src"}, - python_requires=">3.6", - install_requires=requirements, - entry_points={}, - ) - """ - tmp_file_path = Path(tmpdir / "setup.py") - self.run_and_assert_filepath(tmpdir, tmp_file_path, before, after) diff --git a/tests/dependency_management/test_setup_py_writer.py b/tests/dependency_management/test_setup_py_writer.py new file mode 100644 index 00000000..bda76d93 --- /dev/null +++ b/tests/dependency_management/test_setup_py_writer.py @@ -0,0 +1,360 @@ +import pytest +from textwrap import dedent +from codemodder.dependency_management.setup_py_writer import SetupPyWriter +from codemodder.project_analysis.file_parsers.package_store import ( + PackageStore, + FileType, +) +from packaging.requirements import Requirement +from codemodder.dependency import DefusedXML, Security + +TEST_DEPENDENCIES = [Requirement("defusedxml==0.7.1"), Requirement("security~=1.2.0")] + + +@pytest.mark.parametrize("dry_run", [True, False]) +def test_update_setuppy_dependencies(tmpdir, dry_run): + original = """ + from setuptools import setup + setup( + name="test pkg", + description="testing", + long_description="...", + author="Pixee", + packages=find_packages("src"), + package_dir={"": "src"}, + python_requires=">3.6", + install_requires=[ + "protobuf>=3.12,<3.18; python_version < '3'", + "protobuf>=3.12,<4; python_version >= '3'", + "psutil>=5.7,<6", + "requests>=2.4.2,<3" + ], + entry_points={}, + ) + """ + + dependency_file = tmpdir.join("setup.py") + dependency_file.write(dedent(original)) + + store = PackageStore( + type=FileType.SETUP_PY, + file=str(dependency_file), + dependencies=set(), + py_versions=[">=3.6"], + ) + + writer = SetupPyWriter(store, tmpdir) + dependencies = [DefusedXML, Security] + changeset = writer.write(dependencies, dry_run=dry_run) + + after = """ + from setuptools import setup + setup( + name="test pkg", + description="testing", + long_description="...", + author="Pixee", + packages=find_packages("src"), + package_dir={"": "src"}, + python_requires=">3.6", + install_requires=[ + "protobuf>=3.12,<3.18; python_version < '3'", + "protobuf>=3.12,<4; python_version >= '3'", + "psutil>=5.7,<6", + "requests>=2.4.2,<3", "defusedxml~=0.7.1", "security~=1.2.0" + ], + entry_points={}, + ) + """ + assert dependency_file.read() == (dedent(original) if dry_run else dedent(after)) + + assert changeset is not None + assert changeset.path == dependency_file.basename + res = ( + "--- \n" + "+++ \n" + "@@ -12,7 +12,7 @@\n" + """ "protobuf>=3.12,<3.18; python_version < '3'",\n""" + """ "protobuf>=3.12,<4; python_version >= '3'",\n""" + """ "psutil>=5.7,<6",\n""" + """- "requests>=2.4.2,<3"\n""" + """+ "requests>=2.4.2,<3", "defusedxml~=0.7.1", "security~=1.2.0"\n""" + " ],\n " + " entry_points={},\n" + " )\n" + ) + assert changeset.diff == res + assert len(changeset.changes) == 2 + change_one = changeset.changes[0] + + assert change_one.lineNumber == 14 + assert change_one.description == DefusedXML.build_description() + assert change_one.properties == { + "contextual_description": True, + "contextual_description_position": "right", + } + change_two = changeset.changes[1] + assert change_two.lineNumber == 14 + assert change_two.description == Security.build_description() + assert change_two.properties == { + "contextual_description": True, + "contextual_description_position": "right", + } + + +def test_other_setup_func(tmpdir): + original = """ + from something import setup + setup( + name="test pkg", + install_requires=[ + "protobuf>=3.12,<3.18; python_version < '3'", + "protobuf>=3.12,<4; python_version >= '3'", + "psutil>=5.7,<6", + "requests>=2.4.2,<3" + ], + entry_points={}, + ) + """ + + dependency_file = tmpdir.join("setup.py") + dependency_file.write(dedent(original)) + + store = PackageStore( + type=FileType.SETUP_PY, + file=str(dependency_file), + dependencies=set(), + py_versions=[">=3.6"], + ) + + writer = SetupPyWriter(store, tmpdir) + dependencies = [DefusedXML, Security] + changeset = writer.write(dependencies) + assert dependency_file.read() == dedent(original) + assert changeset is None + + +def test_not_setup_file(tmpdir): + original = """ + from setuptools import setup + setup( + name="test pkg", + description="testing", + long_description="...", + author="Pixee", + packages=find_packages("src"), + package_dir={"": "src"}, + python_requires=">3.6", + install_requires=[ + "protobuf>=3.12,<3.18; python_version < '3'", + "protobuf>=3.12,<4; python_version >= '3'", + "psutil>=5.7,<6", + "requests>=2.4.2,<3" + ], + entry_points={}, + ) + """ + + dependency_file = tmpdir.join("not-setup.py") + dependency_file.write(dedent(original)) + + store = PackageStore( + type=FileType.SETUP_PY, + file=str(dependency_file), + dependencies=set(), + py_versions=[">=3.6"], + ) + + writer = SetupPyWriter(store, tmpdir) + dependencies = [DefusedXML, Security] + changeset = writer.write(dependencies) + assert dependency_file.read() == dedent(original) + assert changeset is None + + +def test_setup_call_no_install_requires(tmpdir): + original = """ + from setuptools import setup + setup( + name="test pkg", + description="testing", + long_description="...", + author="Pixee", + packages=find_packages("src"), + package_dir={"": "src"}, + python_requires=">3.6", + ) + """ + + dependency_file = tmpdir.join("setup.py") + dependency_file.write(dedent(original)) + + store = PackageStore( + type=FileType.SETUP_PY, + file=str(dependency_file), + dependencies=set(), + py_versions=[">=3.6"], + ) + + writer = SetupPyWriter(store, tmpdir) + dependencies = [DefusedXML, Security] + changeset = writer.write(dependencies) + assert dependency_file.read() == dedent(original) + assert changeset is None + + +def test_setup_no_existing_requirements(tmpdir): + original = """ + from setuptools import setup + setup( + name="test pkg", + description="testing", + long_description="...", + author="Pixee", + packages=find_packages("src"), + package_dir={"": "src"}, + python_requires=">3.6", + install_requires=[], + entry_points={}, + ) + """ + dependency_file = tmpdir.join("setup.py") + dependency_file.write(dedent(original)) + + store = PackageStore( + type=FileType.SETUP_PY, + file=str(dependency_file), + dependencies=set(), + py_versions=[">=3.6"], + ) + + writer = SetupPyWriter(store, tmpdir) + dependencies = [DefusedXML, Security] + changeset = writer.write(dependencies) + + assert dependency_file.read() == dedent(original) + assert changeset is None + + +def test_setup_call_bad_install_requires(tmpdir): + original = """ + from setuptools import setup + setup( + name="test pkg", + description="testing", + long_description="...", + author="Pixee", + packages=find_packages("src"), + package_dir={"": "src"}, + python_requires=">3.6", + install_requires="some-package", + ) + """ + dependency_file = tmpdir.join("setup.py") + dependency_file.write(dedent(original)) + + store = PackageStore( + type=FileType.SETUP_PY, + file=str(dependency_file), + dependencies=set(), + py_versions=[">=3.6"], + ) + + writer = SetupPyWriter(store, tmpdir) + dependencies = [DefusedXML, Security] + changeset = writer.write(dependencies) + + assert dependency_file.read() == dedent(original) + assert changeset is None + + +@pytest.mark.skip("Need to add support.") +def test_setup_call_requirements_separate(tmpdir): + original = """ + from setuptools import setup + requirements = [ + "protobuf>=3.12,<3.18; python_version < '3'", + "protobuf>=3.12,<4; python_version >= '3'", + "psutil>=5.7,<6", + "requests>=2.4.2,<3" + ] + setup( + name="test pkg", + description="testing", + long_description="...", + author="Pixee", + packages=find_packages("src"), + package_dir={"": "src"}, + python_requires=">3.6", + install_requires=requirements, + entry_points={}, + ) + """ + dependency_file = tmpdir.join("setup.py") + dependency_file.write(dedent(original)) + + store = PackageStore( + type=FileType.SETUP_PY, + file=str(dependency_file), + dependencies=set(), + py_versions=[">=3.6"], + ) + + writer = SetupPyWriter(store, tmpdir) + dependencies = [DefusedXML, Security] + changeset = writer.write(dependencies) + + after = """ + from setuptools import setup + requirements = [ + "protobuf>=3.12,<3.18; python_version < '3'", + "protobuf>=3.12,<4; python_version >= '3'", + "psutil>=5.7,<6", + "requests>=2.4.2,<3", "defusedxml==0.7.1", "security~=1.2.0" + ] + setup( + name="test pkg", + description="testing", + long_description="...", + author="Pixee", + packages=find_packages("src"), + package_dir={"": "src"}, + python_requires=">3.6", + install_requires=requirements, + entry_points={}, + ) + """ + assert dependency_file.read() == dedent(after) + + assert changeset is not None + assert changeset.path == dependency_file.basename + res = ( + "--- \n" + "+++ \n" + "@@ -12,7 +12,7 @@\n" + """ "protobuf>=3.12,<3.18; python_version < '3'",\n""" + """ "protobuf>=3.12,<4; python_version >= '3'",\n""" + """ "psutil>=5.7,<6",\n""" + """- "requests>=2.4.2,<3"\n""" + """+ "requests>=2.4.2,<3", "defusedxml~=0.7.1", "security~=1.2.0"\n""" + " ],\n " + " entry_points={},\n" + " )\n" + ) + assert changeset.diff == res + assert len(changeset.changes) == 2 + change_one = changeset.changes[0] + + assert change_one.lineNumber == 14 + assert change_one.description == DefusedXML.build_description() + assert change_one.properties == { + "contextual_description": True, + "contextual_description_position": "right", + } + change_two = changeset.changes[1] + assert change_two.lineNumber == 14 + assert change_two.description == Security.build_description() + assert change_two.properties == { + "contextual_description": True, + "contextual_description_position": "right", + } diff --git a/tests/project_analysis/file_parsers/test_pyproject_toml_file_parser.py b/tests/project_analysis/file_parsers/test_pyproject_toml_file_parser.py index 06fdad9d..600c46d5 100644 --- a/tests/project_analysis/file_parsers/test_pyproject_toml_file_parser.py +++ b/tests/project_analysis/file_parsers/test_pyproject_toml_file_parser.py @@ -55,8 +55,8 @@ def test_parse(self, pkg_with_pyproject_toml): found = parser.parse() assert len(found) == 1 store = found[0] - assert store.type == "pyproject.toml" - assert store.file == str(pkg_with_pyproject_toml / parser.file_name) + assert store.type.value == "pyproject.toml" + assert store.file == str(pkg_with_pyproject_toml / parser.file_type.value) assert store.py_versions == [">=3.10.0"] assert len(store.dependencies) == 6 @@ -65,8 +65,10 @@ def test_parse_no_python(self, pkg_with_pyproject_toml_no_python): found = parser.parse() assert len(found) == 1 store = found[0] - assert store.type == "pyproject.toml" - assert store.file == str(pkg_with_pyproject_toml_no_python / parser.file_name) + assert store.type.value == "pyproject.toml" + assert store.file == str( + pkg_with_pyproject_toml_no_python / parser.file_type.value + ) assert store.py_versions == [] assert len(store.dependencies) == 1 diff --git a/tests/project_analysis/file_parsers/test_requirements_txt_file_parser.py b/tests/project_analysis/file_parsers/test_requirements_txt_file_parser.py index dde1647b..eb6625fe 100644 --- a/tests/project_analysis/file_parsers/test_requirements_txt_file_parser.py +++ b/tests/project_analysis/file_parsers/test_requirements_txt_file_parser.py @@ -7,8 +7,8 @@ def test_parse(self, pkg_with_reqs_txt): found = parser.parse() assert len(found) == 1 store = found[0] - assert store.type == "requirements.txt" - assert store.file == str(pkg_with_reqs_txt / parser.file_name) + assert store.type.value == "requirements.txt" + assert store.file == str(pkg_with_reqs_txt / parser.file_type.value) assert store.py_versions == [] assert len(store.dependencies) == 4 @@ -17,8 +17,8 @@ def test_parse_utf_16(self, pkg_with_reqs_txt_utf_16): found = parser.parse() assert len(found) == 1 store = found[0] - assert store.type == "requirements.txt" - assert store.file == str(pkg_with_reqs_txt_utf_16 / parser.file_name) + assert store.type.value == "requirements.txt" + assert store.file == str(pkg_with_reqs_txt_utf_16 / parser.file_type.value) assert store.py_versions == [] assert len(store.dependencies) == 4 diff --git a/tests/project_analysis/file_parsers/test_setup_cfg_file_parser.py b/tests/project_analysis/file_parsers/test_setup_cfg_file_parser.py index 11a54490..1d285d0f 100644 --- a/tests/project_analysis/file_parsers/test_setup_cfg_file_parser.py +++ b/tests/project_analysis/file_parsers/test_setup_cfg_file_parser.py @@ -38,8 +38,8 @@ def test_parse(self, pkg_with_setup_cfg): found = parser.parse() assert len(found) == 1 store = found[0] - assert store.type == "setup.cfg" - assert store.file == str(pkg_with_setup_cfg / parser.file_name) + assert store.type.value == "setup.cfg" + assert store.file == str(pkg_with_setup_cfg / parser.file_type.value) assert store.py_versions == [">=3.7"] assert len(store.dependencies) == 2 diff --git a/tests/project_analysis/file_parsers/test_setup_py_file_parser.py b/tests/project_analysis/file_parsers/test_setup_py_file_parser.py index bff062c7..93244a2c 100644 --- a/tests/project_analysis/file_parsers/test_setup_py_file_parser.py +++ b/tests/project_analysis/file_parsers/test_setup_py_file_parser.py @@ -45,8 +45,8 @@ def test_parse(self, pkg_with_setup_py): found = parser.parse() assert len(found) == 1 store = found[0] - assert store.type == "setup.py" - assert store.file == str(pkg_with_setup_py / parser.file_name) + assert store.type.value == "setup.py" + assert store.file == str(pkg_with_setup_py / parser.file_type.value) assert store.py_versions == [">3.6"] assert len(store.dependencies) == 4 diff --git a/tests/test_codemodder.py b/tests/test_codemodder.py index 5dac795d..46aed80c 100644 --- a/tests/test_codemodder.py +++ b/tests/test_codemodder.py @@ -3,7 +3,8 @@ import libcst as cst -from codemodder.codemodder import create_diff, run, find_semgrep_results +from codemodder.codemodder import run, find_semgrep_results +from codemodder.diff import create_diff_from_tree from codemodder.semgrep import run as semgrep_run from codemodder.registry import load_registered_codemods from codemodder.result import ResultSet @@ -232,7 +233,7 @@ def test_diff_newline_edge_case(self): source_tree = cst.parse_module(source) result_tree = cst.parse_module(result) - diff = create_diff(source_tree, result_tree) + diff = create_diff_from_tree(source_tree, result_tree) assert ( diff == """\