From 31d0ac22a344de605deed4d9ab68b5e3f8cf59fe Mon Sep 17 00:00:00 2001 From: clavedeluna Date: Tue, 24 Oct 2023 13:42:38 -0300 Subject: [PATCH 1/8] add repo manager w reqs txt file parser --- src/codemodder/codemodder.py | 5 ++ src/codemodder/context.py | 3 ++ src/codemodder/dependency_manager.py | 1 - src/codemodder/file_parsers/__init__.py | 1 + src/codemodder/file_parsers/package_store.py | 10 ++++ .../requirements_txt_file_parser.py | 46 +++++++++++++++++++ src/codemodder/python_repo_manager.py | 20 ++++++++ tests/codemods/file_parsers/__init__.py | 0 .../test_requirements_txt_file_parser.py | 18 ++++++++ tests/conftest.py | 9 ++++ tests/test_python_repo_manager.py | 8 ++++ 11 files changed, 120 insertions(+), 1 deletion(-) create mode 100644 src/codemodder/file_parsers/__init__.py create mode 100644 src/codemodder/file_parsers/package_store.py create mode 100644 src/codemodder/file_parsers/requirements_txt_file_parser.py create mode 100644 src/codemodder/python_repo_manager.py create mode 100644 tests/codemods/file_parsers/__init__.py create mode 100644 tests/codemods/file_parsers/test_requirements_txt_file_parser.py create mode 100644 tests/test_python_repo_manager.py diff --git a/src/codemodder/codemodder.py b/src/codemodder/codemodder.py index 69603d3a..7f11e4cc 100644 --- a/src/codemodder/codemodder.py +++ b/src/codemodder/codemodder.py @@ -16,6 +16,7 @@ from codemodder.code_directory import file_line_patterns, match_files from codemodder.context import CodemodExecutionContext from codemodder.executor import CodemodExecutorWrapper +from codemodder.python_repo_manager import PythonRepoManager from codemodder.report.codetf_reporter import report_default @@ -130,12 +131,16 @@ def run(original_args) -> int: log_section("startup") logger.info("codemodder: python/%s", __VERSION__) + repo_manager = PythonRepoManager(Path(argv.directory)) context = CodemodExecutionContext( Path(argv.directory), argv.dry_run, argv.verbose, codemod_registry, + repo_manager, ) + # todo: move elsewhere? + repo_manager.package_stores # TODO: this should be a method of CodemodExecutionContext codemods_to_run = codemod_registry.match_codemods( diff --git a/src/codemodder/context.py b/src/codemodder/context.py index 07ddec6e..ddd14f48 100644 --- a/src/codemodder/context.py +++ b/src/codemodder/context.py @@ -9,6 +9,7 @@ from codemodder.executor import CodemodExecutorWrapper from codemodder.logging import logger, log_list from codemodder.registry import CodemodRegistry +from codemodder.python_repo_manager import PythonRepoManager DEPENDENCY_NOTIFICATION = """``` @@ -39,6 +40,7 @@ def __init__( dry_run: bool, verbose: bool, registry: CodemodRegistry, + repo_manager: PythonRepoManager, ): self.directory = directory self.dry_run = dry_run @@ -47,6 +49,7 @@ def __init__( self._failures_by_codemod = {} self.dependencies = {} self.registry = registry + self.repo_manager = repo_manager def add_result(self, codemod_name, change_set): self._results_by_codemod.setdefault(codemod_name, []).append(change_set) diff --git a/src/codemodder/dependency_manager.py b/src/codemodder/dependency_manager.py index 94810f5a..b83d224c 100644 --- a/src/codemodder/dependency_manager.py +++ b/src/codemodder/dependency_manager.py @@ -15,7 +15,6 @@ class DependencyManager: _new_requirements: list[Dependency] def __init__(self, parent_directory: Path): - """One-time class initialization.""" self.parent_directory = parent_directory self.dependency_file_changed = False self._lines = [] diff --git a/src/codemodder/file_parsers/__init__.py b/src/codemodder/file_parsers/__init__.py new file mode 100644 index 00000000..65d879ad --- /dev/null +++ b/src/codemodder/file_parsers/__init__.py @@ -0,0 +1 @@ +from .requirements_txt_file_parser import RequirementsTxtParser diff --git a/src/codemodder/file_parsers/package_store.py b/src/codemodder/file_parsers/package_store.py new file mode 100644 index 00000000..ce39c760 --- /dev/null +++ b/src/codemodder/file_parsers/package_store.py @@ -0,0 +1,10 @@ +from dataclasses import dataclass +from packaging.requirements import Requirement + + +@dataclass +class PackageStore: + type: str + file: str + dependencies: list[Requirement] + py_versions: list[str] diff --git a/src/codemodder/file_parsers/requirements_txt_file_parser.py b/src/codemodder/file_parsers/requirements_txt_file_parser.py new file mode 100644 index 00000000..e33dfcf0 --- /dev/null +++ b/src/codemodder/file_parsers/requirements_txt_file_parser.py @@ -0,0 +1,46 @@ +from codemodder.file_parsers.package_store import PackageStore +from packaging.requirements import Requirement +from pathlib import Path +from typing import List + + +class RequirementsTxtParser: + def __init__(self, parent_directory: Path): + self.parent_directory = parent_directory + self.file_name = "requirements.txt" + + def find_file_locations(self) -> List[Path]: + return list(Path(self.parent_directory).rglob(self.file_name)) + + def _parse_dependencies(self, lines: List[str]): + return [ + Requirement(line) + for x in lines + # Skip empty lines and comments + if (line := x.strip()) and not line.startswith("#") + ] + + def _parse_file(self, file: Path): + with open(file, "r", encoding="utf-8") as f: + lines = f.readlines() + + return PackageStore( + type="requirements_txt", + file=str(file), + dependencies=self._parse_dependencies(lines), + # requirements.txt files do not declare py versions explicitly + # though we could create a heuristic by analyzing each dependency + # and extracting py versions from them. + py_versions=[], + ) + + def parse(self) -> list[PackageStore]: + """ + Find 0 or more requirements.txt files within a project repo. + """ + stores = [] + req_files = self.find_file_locations() + for file in req_files: + store = self._parse_file(file) + stores.append(store) + return stores diff --git a/src/codemodder/python_repo_manager.py b/src/codemodder/python_repo_manager.py new file mode 100644 index 00000000..4868390c --- /dev/null +++ b/src/codemodder/python_repo_manager.py @@ -0,0 +1,20 @@ +from functools import cached_property +from pathlib import Path +from codemodder.file_parsers import RequirementsTxtParser +from codemodder.file_parsers.package_store import PackageStore + + +class PythonRepoManager: + def __init__(self, parent_directory: Path): + self.parent_directory = parent_directory + self._potential_stores = [RequirementsTxtParser] + + @cached_property + def package_stores(self) -> list[PackageStore]: + return self._parse_all_stores() + + def _parse_all_stores(self) -> list[PackageStore]: + discovered_pkg_stores: list[PackageStore] = [] + for store in self._potential_stores: + discovered_pkg_stores.extend(store(self.parent_directory).parse()) + return discovered_pkg_stores diff --git a/tests/codemods/file_parsers/__init__.py b/tests/codemods/file_parsers/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/codemods/file_parsers/test_requirements_txt_file_parser.py b/tests/codemods/file_parsers/test_requirements_txt_file_parser.py new file mode 100644 index 00000000..98edcc72 --- /dev/null +++ b/tests/codemods/file_parsers/test_requirements_txt_file_parser.py @@ -0,0 +1,18 @@ +from codemodder.file_parsers import RequirementsTxtParser + + +class TestRequirementsTxtParser: + def test_parse(self, dir_with_pkg_managers): + parser = RequirementsTxtParser(dir_with_pkg_managers) + found = parser.parse() + assert len(found) == 1 + store = found[0] + assert store.type == "requirements_txt" + assert store.file == dir_with_pkg_managers / parser.file_name + assert store.py_versions == [] + assert len(store.dependencies) == 4 + + def test_parse_no_file(self, dir_with_pkg_managers): + parser = RequirementsTxtParser(dir_with_pkg_managers / "foo") + found = parser.parse() + assert len(found) == 0 diff --git a/tests/conftest.py b/tests/conftest.py index 22a7facd..56998584 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -49,3 +49,12 @@ def disable_write_dependencies(): dm_write.start() yield dm_write.stop() + + +@pytest.fixture(scope="module") +def dir_with_pkg_managers(tmp_path_factory): + base_dir = tmp_path_factory.mktemp("foo") + req_file = base_dir / "requirements.txt" + reqs = "# comment\nrequests==2.31.0\nblack==23.7.*\nmypy~=1.4\npylint>1\n" + req_file.write_text(reqs) + return base_dir diff --git a/tests/test_python_repo_manager.py b/tests/test_python_repo_manager.py new file mode 100644 index 00000000..d34628f6 --- /dev/null +++ b/tests/test_python_repo_manager.py @@ -0,0 +1,8 @@ +from codemodder.python_repo_manager import PythonRepoManager + + +class TestPythonRepoManager: + def test_package_stores(self, dir_with_pkg_managers): + rm = PythonRepoManager(dir_with_pkg_managers) + stores = rm.package_stores + assert len(stores) == 1 From 8bc6b5c9a9ee4723d2f0ae686166a518c653b084 Mon Sep 17 00:00:00 2001 From: clavedeluna Date: Tue, 24 Oct 2023 14:35:24 -0300 Subject: [PATCH 2/8] add pyproject.toml parser --- .pre-commit-config.yaml | 1 + pyproject.toml | 1 + src/codemodder/file_parsers/__init__.py | 1 + src/codemodder/file_parsers/base_parser.py | 35 +++++++++++++++++++ .../pyproject_toml_file_parser.py | 35 +++++++++++++++++++ .../requirements_txt_file_parser.py | 27 ++------------ src/codemodder/python_repo_manager.py | 4 +-- 7 files changed, 78 insertions(+), 26 deletions(-) create mode 100644 src/codemodder/file_parsers/base_parser.py create mode 100644 src/codemodder/file_parsers/pyproject_toml_file_parser.py diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 47e59b6b..f7e991ff 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -33,4 +33,5 @@ repos: [ "types-mock==5.0.*", "types-PyYAML==6.0", + "types-toml~=0.10", ] diff --git a/pyproject.toml b/pyproject.toml index 0e98d0b9..5536340f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -16,6 +16,7 @@ dependencies = [ "python-json-logger~=2.0.0", "PyYAML~=6.0.0", "semgrep~=1.46.0", + "toml~=0.10.2", "wrapt~=1.15.0", ] diff --git a/src/codemodder/file_parsers/__init__.py b/src/codemodder/file_parsers/__init__.py index 65d879ad..ed55451b 100644 --- a/src/codemodder/file_parsers/__init__.py +++ b/src/codemodder/file_parsers/__init__.py @@ -1 +1,2 @@ from .requirements_txt_file_parser import RequirementsTxtParser +from .pyproject_toml_file_parser import PyprojectTomlParser diff --git a/src/codemodder/file_parsers/base_parser.py b/src/codemodder/file_parsers/base_parser.py new file mode 100644 index 00000000..4dff3da9 --- /dev/null +++ b/src/codemodder/file_parsers/base_parser.py @@ -0,0 +1,35 @@ +from pathlib import Path +from typing import List +from .package_store import PackageStore +from packaging.requirements import Requirement + + +class BaseParser: + def __init__(self, parent_directory: Path): + self.parent_directory = parent_directory + self.file_name: str = "" + + def _parse_dependencies(self, lines: List[str]): + return [ + Requirement(line) + for x in lines + # Skip empty lines and comments + if (line := x.strip()) and not line.startswith("#") + ] + + def _parse_file(self, file: Path): + raise NotImplementedError + + def find_file_locations(self) -> List[Path]: + return list(Path(self.parent_directory).rglob(self.file_name)) + + def parse(self) -> list[PackageStore]: + """ + Find 0 or more project config or dependency files within a project repo. + """ + stores = [] + req_files = self.find_file_locations() + for file in req_files: + store = self._parse_file(file) + stores.append(store) + return stores diff --git a/src/codemodder/file_parsers/pyproject_toml_file_parser.py b/src/codemodder/file_parsers/pyproject_toml_file_parser.py new file mode 100644 index 00000000..de557dde --- /dev/null +++ b/src/codemodder/file_parsers/pyproject_toml_file_parser.py @@ -0,0 +1,35 @@ +from codemodder.file_parsers.package_store import PackageStore +from packaging.requirements import Requirement +from pathlib import Path +from typing import List +import toml + +from .base_parser import BaseParser + + +class PyprojectTomlParser(BaseParser): + def __init__(self, parent_directory: Path): + super().__init__(parent_directory) + self.file_name = "pyproject.toml" + + def _parse_dependencies_from_toml(self, toml_data: dict): + # todo: handle cases for + # 1. no dependencies + return self._parse_dependencies(toml_data["project"]["dependencies"]) + + def _parse_py_versions(self, toml_data: dict): + # todo: handle cases for + # 1. no requires-python + # 2. various requires-python such as "">3.5.2"", ">=3.11.1,<3.11.2" + return toml_data["project"]["requires-python"] + + def _parse_file(self, file: Path): + data = toml.load(file) + # todo: handle no "project" in data + + return PackageStore( + type="pyproject_toml", + file=str(file), + dependencies=self._parse_dependencies_from_toml(data), + py_versions=self._parse_py_versions(data), + ) diff --git a/src/codemodder/file_parsers/requirements_txt_file_parser.py b/src/codemodder/file_parsers/requirements_txt_file_parser.py index e33dfcf0..46393060 100644 --- a/src/codemodder/file_parsers/requirements_txt_file_parser.py +++ b/src/codemodder/file_parsers/requirements_txt_file_parser.py @@ -2,24 +2,14 @@ from packaging.requirements import Requirement from pathlib import Path from typing import List +from .base_parser import BaseParser -class RequirementsTxtParser: +class RequirementsTxtParser(BaseParser): def __init__(self, parent_directory: Path): - self.parent_directory = parent_directory + super().__init__(parent_directory) self.file_name = "requirements.txt" - def find_file_locations(self) -> List[Path]: - return list(Path(self.parent_directory).rglob(self.file_name)) - - def _parse_dependencies(self, lines: List[str]): - return [ - Requirement(line) - for x in lines - # Skip empty lines and comments - if (line := x.strip()) and not line.startswith("#") - ] - def _parse_file(self, file: Path): with open(file, "r", encoding="utf-8") as f: lines = f.readlines() @@ -33,14 +23,3 @@ def _parse_file(self, file: Path): # and extracting py versions from them. py_versions=[], ) - - def parse(self) -> list[PackageStore]: - """ - Find 0 or more requirements.txt files within a project repo. - """ - stores = [] - req_files = self.find_file_locations() - for file in req_files: - store = self._parse_file(file) - stores.append(store) - return stores diff --git a/src/codemodder/python_repo_manager.py b/src/codemodder/python_repo_manager.py index 4868390c..9cd09958 100644 --- a/src/codemodder/python_repo_manager.py +++ b/src/codemodder/python_repo_manager.py @@ -1,13 +1,13 @@ from functools import cached_property from pathlib import Path -from codemodder.file_parsers import RequirementsTxtParser +from codemodder.file_parsers import RequirementsTxtParser, PyprojectTomlParser from codemodder.file_parsers.package_store import PackageStore class PythonRepoManager: def __init__(self, parent_directory: Path): self.parent_directory = parent_directory - self._potential_stores = [RequirementsTxtParser] + self._potential_stores = [RequirementsTxtParser, PyprojectTomlParser] @cached_property def package_stores(self) -> list[PackageStore]: From 8de212b8f9298aa205cc7af29e90aa82a782d5e2 Mon Sep 17 00:00:00 2001 From: clavedeluna Date: Tue, 24 Oct 2023 14:51:12 -0300 Subject: [PATCH 3/8] add parser for setup.cfg file --- src/codemodder/file_parsers/__init__.py | 1 + .../pyproject_toml_file_parser.py | 4 +-- .../requirements_txt_file_parser.py | 2 +- .../file_parsers/setup_cfg_file_parser.py | 36 +++++++++++++++++++ src/codemodder/python_repo_manager.py | 12 +++++-- 5 files changed, 49 insertions(+), 6 deletions(-) create mode 100644 src/codemodder/file_parsers/setup_cfg_file_parser.py diff --git a/src/codemodder/file_parsers/__init__.py b/src/codemodder/file_parsers/__init__.py index ed55451b..a0f1567c 100644 --- a/src/codemodder/file_parsers/__init__.py +++ b/src/codemodder/file_parsers/__init__.py @@ -1,2 +1,3 @@ from .requirements_txt_file_parser import RequirementsTxtParser from .pyproject_toml_file_parser import PyprojectTomlParser +from .setup_cfg_file_parser import SetupCfgParser diff --git a/src/codemodder/file_parsers/pyproject_toml_file_parser.py b/src/codemodder/file_parsers/pyproject_toml_file_parser.py index de557dde..cfc08621 100644 --- a/src/codemodder/file_parsers/pyproject_toml_file_parser.py +++ b/src/codemodder/file_parsers/pyproject_toml_file_parser.py @@ -1,7 +1,5 @@ from codemodder.file_parsers.package_store import PackageStore -from packaging.requirements import Requirement from pathlib import Path -from typing import List import toml from .base_parser import BaseParser @@ -28,7 +26,7 @@ def _parse_file(self, file: Path): # todo: handle no "project" in data return PackageStore( - type="pyproject_toml", + type=self.file_name, file=str(file), dependencies=self._parse_dependencies_from_toml(data), py_versions=self._parse_py_versions(data), diff --git a/src/codemodder/file_parsers/requirements_txt_file_parser.py b/src/codemodder/file_parsers/requirements_txt_file_parser.py index 46393060..dcb9b998 100644 --- a/src/codemodder/file_parsers/requirements_txt_file_parser.py +++ b/src/codemodder/file_parsers/requirements_txt_file_parser.py @@ -15,7 +15,7 @@ def _parse_file(self, file: Path): lines = f.readlines() return PackageStore( - type="requirements_txt", + type=self.file_name, file=str(file), dependencies=self._parse_dependencies(lines), # requirements.txt files do not declare py versions explicitly diff --git a/src/codemodder/file_parsers/setup_cfg_file_parser.py b/src/codemodder/file_parsers/setup_cfg_file_parser.py new file mode 100644 index 00000000..ae46cfff --- /dev/null +++ b/src/codemodder/file_parsers/setup_cfg_file_parser.py @@ -0,0 +1,36 @@ +from codemodder.file_parsers.package_store import PackageStore +from pathlib import Path +import configparser + +from .base_parser import BaseParser + + +class SetupCfgParser(BaseParser): + def __init__(self, parent_directory: Path): + super().__init__(parent_directory) + self.file_name = "setup.cfg" + + def _parse_dependencies_from_toml(self, config: configparser.ConfigParser): + # todo: handle cases for + # 1. no dependencies, no options dict + dependency_lines = config["options"]["install_requires"].split("\n") + return self._parse_dependencies(dependency_lines) + + def _parse_py_versions(self, config: configparser.ConfigParser): + # todo: handle cases for + # 1. no options/ no requires-python + # 2. various requires-python such as "">3.5.2"", ">=3.11.1,<3.11.2" + return config["options"]["python_requires"] + + def _parse_file(self, file: Path): + config = configparser.ConfigParser() + config.read(file) + + # todo: handle no config, no "options" in config + + return PackageStore( + type=self.file_name, + file=str(file), + dependencies=self._parse_dependencies_from_toml(config), + py_versions=self._parse_py_versions(config), + ) diff --git a/src/codemodder/python_repo_manager.py b/src/codemodder/python_repo_manager.py index 9cd09958..5d31d770 100644 --- a/src/codemodder/python_repo_manager.py +++ b/src/codemodder/python_repo_manager.py @@ -1,13 +1,21 @@ from functools import cached_property from pathlib import Path -from codemodder.file_parsers import RequirementsTxtParser, PyprojectTomlParser +from codemodder.file_parsers import ( + RequirementsTxtParser, + PyprojectTomlParser, + SetupCfgParser, +) from codemodder.file_parsers.package_store import PackageStore class PythonRepoManager: def __init__(self, parent_directory: Path): self.parent_directory = parent_directory - self._potential_stores = [RequirementsTxtParser, PyprojectTomlParser] + self._potential_stores = [ + RequirementsTxtParser, + PyprojectTomlParser, + SetupCfgParser, + ] @cached_property def package_stores(self) -> list[PackageStore]: From 9d68e0382a18bbc9909b4da9afe89d8be9424f75 Mon Sep 17 00:00:00 2001 From: clavedeluna Date: Wed, 25 Oct 2023 07:36:13 -0300 Subject: [PATCH 4/8] add setup.py parser --- src/codemodder/codemodder.py | 2 +- src/codemodder/context.py | 2 +- src/codemodder/project_analysis/__init__.py | 0 .../file_parsers/__init__.py | 1 + .../file_parsers/base_parser.py | 0 .../file_parsers/package_store.py | 0 .../pyproject_toml_file_parser.py | 2 +- .../requirements_txt_file_parser.py | 2 +- .../file_parsers/setup_cfg_file_parser.py | 3 +- .../file_parsers/setup_py_file_parser.py | 75 +++++++++++++++++++ .../project_analysis/file_parsers/utils.py | 5 ++ .../python_repo_manager.py | 6 +- .../test_requirements_txt_file_parser.py | 2 +- tests/test_python_repo_manager.py | 2 +- 14 files changed, 93 insertions(+), 9 deletions(-) create mode 100644 src/codemodder/project_analysis/__init__.py rename src/codemodder/{ => project_analysis}/file_parsers/__init__.py (78%) rename src/codemodder/{ => project_analysis}/file_parsers/base_parser.py (100%) rename src/codemodder/{ => project_analysis}/file_parsers/package_store.py (100%) rename src/codemodder/{ => project_analysis}/file_parsers/pyproject_toml_file_parser.py (92%) rename src/codemodder/{ => project_analysis}/file_parsers/requirements_txt_file_parser.py (91%) rename src/codemodder/{ => project_analysis}/file_parsers/setup_cfg_file_parser.py (89%) create mode 100644 src/codemodder/project_analysis/file_parsers/setup_py_file_parser.py create mode 100644 src/codemodder/project_analysis/file_parsers/utils.py rename src/codemodder/{ => project_analysis}/python_repo_manager.py (81%) diff --git a/src/codemodder/codemodder.py b/src/codemodder/codemodder.py index 7f11e4cc..911d4fd1 100644 --- a/src/codemodder/codemodder.py +++ b/src/codemodder/codemodder.py @@ -16,7 +16,7 @@ from codemodder.code_directory import file_line_patterns, match_files from codemodder.context import CodemodExecutionContext from codemodder.executor import CodemodExecutorWrapper -from codemodder.python_repo_manager import PythonRepoManager +from codemodder.project_analysis.python_repo_manager import PythonRepoManager from codemodder.report.codetf_reporter import report_default diff --git a/src/codemodder/context.py b/src/codemodder/context.py index ddd14f48..9f883e3e 100644 --- a/src/codemodder/context.py +++ b/src/codemodder/context.py @@ -9,7 +9,7 @@ from codemodder.executor import CodemodExecutorWrapper from codemodder.logging import logger, log_list from codemodder.registry import CodemodRegistry -from codemodder.python_repo_manager import PythonRepoManager +from codemodder.project_analysis.python_repo_manager import PythonRepoManager DEPENDENCY_NOTIFICATION = """``` diff --git a/src/codemodder/project_analysis/__init__.py b/src/codemodder/project_analysis/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/src/codemodder/file_parsers/__init__.py b/src/codemodder/project_analysis/file_parsers/__init__.py similarity index 78% rename from src/codemodder/file_parsers/__init__.py rename to src/codemodder/project_analysis/file_parsers/__init__.py index a0f1567c..719ca7eb 100644 --- a/src/codemodder/file_parsers/__init__.py +++ b/src/codemodder/project_analysis/file_parsers/__init__.py @@ -1,3 +1,4 @@ from .requirements_txt_file_parser import RequirementsTxtParser from .pyproject_toml_file_parser import PyprojectTomlParser from .setup_cfg_file_parser import SetupCfgParser +from .setup_py_file_parser import SetupPyParser diff --git a/src/codemodder/file_parsers/base_parser.py b/src/codemodder/project_analysis/file_parsers/base_parser.py similarity index 100% rename from src/codemodder/file_parsers/base_parser.py rename to src/codemodder/project_analysis/file_parsers/base_parser.py diff --git a/src/codemodder/file_parsers/package_store.py b/src/codemodder/project_analysis/file_parsers/package_store.py similarity index 100% rename from src/codemodder/file_parsers/package_store.py rename to src/codemodder/project_analysis/file_parsers/package_store.py diff --git a/src/codemodder/file_parsers/pyproject_toml_file_parser.py b/src/codemodder/project_analysis/file_parsers/pyproject_toml_file_parser.py similarity index 92% rename from src/codemodder/file_parsers/pyproject_toml_file_parser.py rename to src/codemodder/project_analysis/file_parsers/pyproject_toml_file_parser.py index cfc08621..acf39fa5 100644 --- a/src/codemodder/file_parsers/pyproject_toml_file_parser.py +++ b/src/codemodder/project_analysis/file_parsers/pyproject_toml_file_parser.py @@ -1,4 +1,4 @@ -from codemodder.file_parsers.package_store import PackageStore +from codemodder.project_analysis.file_parsers.package_store import PackageStore from pathlib import Path import toml diff --git a/src/codemodder/file_parsers/requirements_txt_file_parser.py b/src/codemodder/project_analysis/file_parsers/requirements_txt_file_parser.py similarity index 91% rename from src/codemodder/file_parsers/requirements_txt_file_parser.py rename to src/codemodder/project_analysis/file_parsers/requirements_txt_file_parser.py index dcb9b998..25d85fac 100644 --- a/src/codemodder/file_parsers/requirements_txt_file_parser.py +++ b/src/codemodder/project_analysis/file_parsers/requirements_txt_file_parser.py @@ -1,4 +1,4 @@ -from codemodder.file_parsers.package_store import PackageStore +from codemodder.project_analysis.file_parsers.package_store import PackageStore from packaging.requirements import Requirement from pathlib import Path from typing import List diff --git a/src/codemodder/file_parsers/setup_cfg_file_parser.py b/src/codemodder/project_analysis/file_parsers/setup_cfg_file_parser.py similarity index 89% rename from src/codemodder/file_parsers/setup_cfg_file_parser.py rename to src/codemodder/project_analysis/file_parsers/setup_cfg_file_parser.py index ae46cfff..014454b1 100644 --- a/src/codemodder/file_parsers/setup_cfg_file_parser.py +++ b/src/codemodder/project_analysis/file_parsers/setup_cfg_file_parser.py @@ -1,4 +1,4 @@ -from codemodder.file_parsers.package_store import PackageStore +from codemodder.project_analysis.file_parsers.package_store import PackageStore from pathlib import Path import configparser @@ -13,6 +13,7 @@ def __init__(self, parent_directory: Path): def _parse_dependencies_from_toml(self, config: configparser.ConfigParser): # todo: handle cases for # 1. no dependencies, no options dict + # setup_requires, tests_require, extras_require dependency_lines = config["options"]["install_requires"].split("\n") return self._parse_dependencies(dependency_lines) diff --git a/src/codemodder/project_analysis/file_parsers/setup_py_file_parser.py b/src/codemodder/project_analysis/file_parsers/setup_py_file_parser.py new file mode 100644 index 00000000..bd12eb56 --- /dev/null +++ b/src/codemodder/project_analysis/file_parsers/setup_py_file_parser.py @@ -0,0 +1,75 @@ +from codemodder.project_analysis.file_parsers.package_store import PackageStore +from pathlib import Path +import libcst as cst +from libcst import matchers +from packaging.requirements import Requirement + +from .base_parser import BaseParser +from .utils import clean_simplestring + + +class SetupPyParser(BaseParser): + def __init__(self, parent_directory: Path): + super().__init__(parent_directory) + self.file_name = "setup.py" + + def _parse_dependencies(self, cst_dependencies): + return [ + Requirement(line) + for x in cst_dependencies + # Skip empty lines and comments + if (line := clean_simplestring(x.value)) and not line.startswith("#") + ] + + def _parse_dependencies_from_cst(self, cst_dependencies): + # todo: handle cases for + # 1. no dependencies, + return self._parse_dependencies(cst_dependencies) + + def _parse_py_versions(self, version_str): + return version_str.strip() + + def _parse_file(self, file: Path): + visitor = SetupCallVisotor() + with open(str(file), "r", encoding="utf-8") as f: + module = cst.parse_module(f.read()) + module.visit(visitor) + + # todo: handle no python_requires, install_requires + + return PackageStore( + type=self.file_name, + file=str(file), + dependencies=self._parse_dependencies_from_cst(visitor.install_requires), + py_versions=self._parse_py_versions(visitor.python_requires), + ) + + +class SetupCallVisotor(cst.CSTVisitor): + def __init__(self): + self.python_requires = None + self.install_requires = None + # todo setup_requires, tests_require, extras_require + + def visit_Call(self, node: cst.Call) -> None: + # todo: only handle setup from setuptools, not others tho unlikely + if matchers.matches(node.func, cst.Name(value="setup")): + visitor = SetupArgVisitor() + node.visit(visitor) + self.python_requires = visitor.python_requires + self.install_requires = visitor.install_requires + + +class SetupArgVisitor(cst.CSTVisitor): + def __init__(self): + self.python_requires = None + self.install_requires = None + + def visit_Arg(self, node: cst.Arg) -> None: + if matchers.matches(node.keyword, cst.Name(value="python_requires")): + # todo: this works for `python_requires=">=3.7",` but what about + # a list of versions? + self.python_requires = node.value.value + if matchers.matches(node.keyword, cst.Name(value="install_requires")): + # todo: could it be something other than a list? + self.install_requires = node.value.elements diff --git a/src/codemodder/project_analysis/file_parsers/utils.py b/src/codemodder/project_analysis/file_parsers/utils.py new file mode 100644 index 00000000..42073e4c --- /dev/null +++ b/src/codemodder/project_analysis/file_parsers/utils.py @@ -0,0 +1,5 @@ +import libcst as cst + + +def clean_simplestring(node: cst.SimpleString) -> str: + return node.value.strip('"') diff --git a/src/codemodder/python_repo_manager.py b/src/codemodder/project_analysis/python_repo_manager.py similarity index 81% rename from src/codemodder/python_repo_manager.py rename to src/codemodder/project_analysis/python_repo_manager.py index 5d31d770..c73e4656 100644 --- a/src/codemodder/python_repo_manager.py +++ b/src/codemodder/project_analysis/python_repo_manager.py @@ -1,11 +1,12 @@ from functools import cached_property from pathlib import Path -from codemodder.file_parsers import ( +from codemodder.project_analysis.file_parsers import ( RequirementsTxtParser, PyprojectTomlParser, SetupCfgParser, + SetupPyParser, ) -from codemodder.file_parsers.package_store import PackageStore +from codemodder.project_analysis.file_parsers.package_store import PackageStore class PythonRepoManager: @@ -15,6 +16,7 @@ def __init__(self, parent_directory: Path): RequirementsTxtParser, PyprojectTomlParser, SetupCfgParser, + SetupPyParser, ] @cached_property diff --git a/tests/codemods/file_parsers/test_requirements_txt_file_parser.py b/tests/codemods/file_parsers/test_requirements_txt_file_parser.py index 98edcc72..ef8b269c 100644 --- a/tests/codemods/file_parsers/test_requirements_txt_file_parser.py +++ b/tests/codemods/file_parsers/test_requirements_txt_file_parser.py @@ -1,4 +1,4 @@ -from codemodder.file_parsers import RequirementsTxtParser +from codemodder.project_analysis.file_parsers import RequirementsTxtParser class TestRequirementsTxtParser: diff --git a/tests/test_python_repo_manager.py b/tests/test_python_repo_manager.py index d34628f6..c6359f76 100644 --- a/tests/test_python_repo_manager.py +++ b/tests/test_python_repo_manager.py @@ -1,4 +1,4 @@ -from codemodder.python_repo_manager import PythonRepoManager +from codemodder.project_analysis.python_repo_manager import PythonRepoManager class TestPythonRepoManager: From b1b8dbf41a522c5ecfe8681d57f4bbff97ff8515 Mon Sep 17 00:00:00 2001 From: clavedeluna Date: Wed, 25 Oct 2023 08:26:55 -0300 Subject: [PATCH 5/8] testing --- src/codemodder/codemodder.py | 4 +- .../pyproject_toml_file_parser.py | 4 +- .../file_parsers/setup_cfg_file_parser.py | 2 +- .../file_parsers/setup_py_file_parser.py | 4 +- .../project_analysis/file_parsers/utils.py | 4 +- tests/codemods/base_codemod_test.py | 2 + .../test_requirements_txt_file_parser.py | 18 ------ tests/conftest.py | 2 +- .../__init__.py | 0 .../project_analysis/file_parsers/__init__.py | 0 .../test_pyproject_toml_file_parser.py | 47 ++++++++++++++++ .../test_requirements_txt_file_parser.py | 18 ++++++ .../test_setup_cfg_file_parser.py | 48 ++++++++++++++++ .../file_parsers/test_setup_py_file_parser.py | 55 +++++++++++++++++++ .../test_python_repo_manager.py | 4 +- 15 files changed, 184 insertions(+), 28 deletions(-) delete mode 100644 tests/codemods/file_parsers/test_requirements_txt_file_parser.py rename tests/{codemods/file_parsers => project_analysis}/__init__.py (100%) create mode 100644 tests/project_analysis/file_parsers/__init__.py create mode 100644 tests/project_analysis/file_parsers/test_pyproject_toml_file_parser.py create mode 100644 tests/project_analysis/file_parsers/test_requirements_txt_file_parser.py create mode 100644 tests/project_analysis/file_parsers/test_setup_cfg_file_parser.py create mode 100644 tests/project_analysis/file_parsers/test_setup_py_file_parser.py rename tests/{ => project_analysis}/test_python_repo_manager.py (61%) diff --git a/src/codemodder/codemodder.py b/src/codemodder/codemodder.py index 911d4fd1..00852a3f 100644 --- a/src/codemodder/codemodder.py +++ b/src/codemodder/codemodder.py @@ -139,8 +139,8 @@ def run(original_args) -> int: codemod_registry, repo_manager, ) - # todo: move elsewhere? - repo_manager.package_stores + # todo: enable when ready + # repo_manager.package_stores # TODO: this should be a method of CodemodExecutionContext codemods_to_run = codemod_registry.match_codemods( diff --git a/src/codemodder/project_analysis/file_parsers/pyproject_toml_file_parser.py b/src/codemodder/project_analysis/file_parsers/pyproject_toml_file_parser.py index acf39fa5..8b67c1ef 100644 --- a/src/codemodder/project_analysis/file_parsers/pyproject_toml_file_parser.py +++ b/src/codemodder/project_analysis/file_parsers/pyproject_toml_file_parser.py @@ -18,8 +18,8 @@ def _parse_dependencies_from_toml(self, toml_data: dict): def _parse_py_versions(self, toml_data: dict): # todo: handle cases for # 1. no requires-python - # 2. various requires-python such as "">3.5.2"", ">=3.11.1,<3.11.2" - return toml_data["project"]["requires-python"] + # 2. multiple requires-python such as "">3.5.2"", ">=3.11.1,<3.11.2" + return [toml_data["project"]["requires-python"]] def _parse_file(self, file: Path): data = toml.load(file) diff --git a/src/codemodder/project_analysis/file_parsers/setup_cfg_file_parser.py b/src/codemodder/project_analysis/file_parsers/setup_cfg_file_parser.py index 014454b1..1c94dbb8 100644 --- a/src/codemodder/project_analysis/file_parsers/setup_cfg_file_parser.py +++ b/src/codemodder/project_analysis/file_parsers/setup_cfg_file_parser.py @@ -21,7 +21,7 @@ def _parse_py_versions(self, config: configparser.ConfigParser): # todo: handle cases for # 1. no options/ no requires-python # 2. various requires-python such as "">3.5.2"", ">=3.11.1,<3.11.2" - return config["options"]["python_requires"] + return [config["options"]["python_requires"]] def _parse_file(self, file: Path): config = configparser.ConfigParser() diff --git a/src/codemodder/project_analysis/file_parsers/setup_py_file_parser.py b/src/codemodder/project_analysis/file_parsers/setup_py_file_parser.py index bd12eb56..ae444ffe 100644 --- a/src/codemodder/project_analysis/file_parsers/setup_py_file_parser.py +++ b/src/codemodder/project_analysis/file_parsers/setup_py_file_parser.py @@ -27,11 +27,13 @@ def _parse_dependencies_from_cst(self, cst_dependencies): return self._parse_dependencies(cst_dependencies) def _parse_py_versions(self, version_str): - return version_str.strip() + # todo: handle for multiple versions + return [clean_simplestring(version_str)] def _parse_file(self, file: Path): visitor = SetupCallVisotor() with open(str(file), "r", encoding="utf-8") as f: + # todo: handle failure in parsing module = cst.parse_module(f.read()) module.visit(visitor) diff --git a/src/codemodder/project_analysis/file_parsers/utils.py b/src/codemodder/project_analysis/file_parsers/utils.py index 42073e4c..60e1da33 100644 --- a/src/codemodder/project_analysis/file_parsers/utils.py +++ b/src/codemodder/project_analysis/file_parsers/utils.py @@ -1,5 +1,7 @@ import libcst as cst -def clean_simplestring(node: cst.SimpleString) -> str: +def clean_simplestring(node: cst.SimpleString | str) -> str: + if isinstance(node, str): + return node.strip('"') return node.value.strip('"') diff --git a/tests/codemods/base_codemod_test.py b/tests/codemods/base_codemod_test.py index f98e10e1..5cdae259 100644 --- a/tests/codemods/base_codemod_test.py +++ b/tests/codemods/base_codemod_test.py @@ -33,6 +33,7 @@ def run_and_assert_filepath(self, root, file_path, input_code, expected): dry_run=True, verbose=False, registry=mock.MagicMock(), + repo_manager=mock.MagicMock(), ) self.file_context = FileContext( file_path, @@ -80,6 +81,7 @@ def run_and_assert_filepath(self, root, file_path, input_code, expected): dry_run=True, verbose=False, registry=mock.MagicMock(), + repo_manager=mock.MagicMock(), ) input_tree = cst.parse_module(input_code) all_results = self.results_by_id_filepath(input_code, file_path) diff --git a/tests/codemods/file_parsers/test_requirements_txt_file_parser.py b/tests/codemods/file_parsers/test_requirements_txt_file_parser.py deleted file mode 100644 index ef8b269c..00000000 --- a/tests/codemods/file_parsers/test_requirements_txt_file_parser.py +++ /dev/null @@ -1,18 +0,0 @@ -from codemodder.project_analysis.file_parsers import RequirementsTxtParser - - -class TestRequirementsTxtParser: - def test_parse(self, dir_with_pkg_managers): - parser = RequirementsTxtParser(dir_with_pkg_managers) - found = parser.parse() - assert len(found) == 1 - store = found[0] - assert store.type == "requirements_txt" - assert store.file == dir_with_pkg_managers / parser.file_name - assert store.py_versions == [] - assert len(store.dependencies) == 4 - - def test_parse_no_file(self, dir_with_pkg_managers): - parser = RequirementsTxtParser(dir_with_pkg_managers / "foo") - found = parser.parse() - assert len(found) == 0 diff --git a/tests/conftest.py b/tests/conftest.py index 56998584..c366bea1 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -52,7 +52,7 @@ def disable_write_dependencies(): @pytest.fixture(scope="module") -def dir_with_pkg_managers(tmp_path_factory): +def pkg_with_reqs_txt(tmp_path_factory): base_dir = tmp_path_factory.mktemp("foo") req_file = base_dir / "requirements.txt" reqs = "# comment\nrequests==2.31.0\nblack==23.7.*\nmypy~=1.4\npylint>1\n" diff --git a/tests/codemods/file_parsers/__init__.py b/tests/project_analysis/__init__.py similarity index 100% rename from tests/codemods/file_parsers/__init__.py rename to tests/project_analysis/__init__.py diff --git a/tests/project_analysis/file_parsers/__init__.py b/tests/project_analysis/file_parsers/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/project_analysis/file_parsers/test_pyproject_toml_file_parser.py b/tests/project_analysis/file_parsers/test_pyproject_toml_file_parser.py new file mode 100644 index 00000000..2c1bdaa0 --- /dev/null +++ b/tests/project_analysis/file_parsers/test_pyproject_toml_file_parser.py @@ -0,0 +1,47 @@ +import pytest +from codemodder.project_analysis.file_parsers import PyprojectTomlParser + + +@pytest.fixture(scope="module") +def pkg_with_pyproject_toml(tmp_path_factory): + base_dir = tmp_path_factory.mktemp("foo") + toml_file = base_dir / "pyproject.toml" + toml = """\ + [build-system] + requires = ["setuptools"] + build-backend = "setuptools.build_meta" + + [project] + name = "pkg for testing" + version = "0.60.0" + requires-python = ">=3.10.0" + readme = "README.md" + license = {file = "LICENSE"} + dependencies = [ + "isort~=5.12.0", + "libcst~=1.1.0", + "PyYAML~=6.0.0", + "semgrep<2", + "toml~=0.10.2", + "wrapt~=1.15.0", + ] + """ + toml_file.write_text(toml) + return base_dir + + +class TestPyprojectTomlParser: + def test_parse(self, pkg_with_pyproject_toml): + parser = PyprojectTomlParser(pkg_with_pyproject_toml) + found = parser.parse() + assert len(found) == 1 + store = found[0] + assert store.type == "pyproject.toml" + assert store.file == str(pkg_with_pyproject_toml / parser.file_name) + assert store.py_versions == [">=3.10.0"] + assert len(store.dependencies) == 6 + + def test_parse_no_file(self, pkg_with_pyproject_toml): + parser = PyprojectTomlParser(pkg_with_pyproject_toml / "foo") + found = parser.parse() + assert len(found) == 0 diff --git a/tests/project_analysis/file_parsers/test_requirements_txt_file_parser.py b/tests/project_analysis/file_parsers/test_requirements_txt_file_parser.py new file mode 100644 index 00000000..468e0b00 --- /dev/null +++ b/tests/project_analysis/file_parsers/test_requirements_txt_file_parser.py @@ -0,0 +1,18 @@ +from codemodder.project_analysis.file_parsers import RequirementsTxtParser + + +class TestRequirementsTxtParser: + def test_parse(self, pkg_with_reqs_txt): + parser = RequirementsTxtParser(pkg_with_reqs_txt) + found = parser.parse() + assert len(found) == 1 + store = found[0] + assert store.type == "requirements.txt" + assert store.file == str(pkg_with_reqs_txt / parser.file_name) + assert store.py_versions == [] + assert len(store.dependencies) == 4 + + def test_parse_no_file(self, pkg_with_reqs_txt): + parser = RequirementsTxtParser(pkg_with_reqs_txt / "foo") + found = parser.parse() + assert len(found) == 0 diff --git a/tests/project_analysis/file_parsers/test_setup_cfg_file_parser.py b/tests/project_analysis/file_parsers/test_setup_cfg_file_parser.py new file mode 100644 index 00000000..729bd2ad --- /dev/null +++ b/tests/project_analysis/file_parsers/test_setup_cfg_file_parser.py @@ -0,0 +1,48 @@ +import pytest +from codemodder.project_analysis.file_parsers import SetupCfgParser + + +@pytest.fixture(scope="module") +def pkg_with_setup_cfg(tmp_path_factory): + base_dir = tmp_path_factory.mktemp("foo") + setup_cfg = base_dir / "setup.cfg" + cfg = """\ + [metadata] + name = test pkg + version = 123 + author = Some Name + author_email = idk@gmail.com + description = My package description + license = BSD-3-Clause + classifiers = + Framework :: Django + Programming Language :: Python :: 3 + + [options] + zip_safe = False + include_package_data = True + packages = find: + python_requires = >=3.7 + install_requires = + requests + importlib-metadata; python_version<"3.8" + """ + setup_cfg.write_text(cfg) + return base_dir + + +class TestSetupCfgParser: + def test_parse(self, pkg_with_setup_cfg): + parser = SetupCfgParser(pkg_with_setup_cfg) + found = parser.parse() + assert len(found) == 1 + store = found[0] + assert store.type == "setup.cfg" + assert store.file == str(pkg_with_setup_cfg / parser.file_name) + assert store.py_versions == [">=3.7"] + assert len(store.dependencies) == 2 + + def test_parse_no_file(self, pkg_with_setup_cfg): + parser = SetupCfgParser(pkg_with_setup_cfg / "foo") + found = parser.parse() + assert len(found) == 0 diff --git a/tests/project_analysis/file_parsers/test_setup_py_file_parser.py b/tests/project_analysis/file_parsers/test_setup_py_file_parser.py new file mode 100644 index 00000000..2cee31be --- /dev/null +++ b/tests/project_analysis/file_parsers/test_setup_py_file_parser.py @@ -0,0 +1,55 @@ +import pytest +from codemodder.project_analysis.file_parsers import SetupPyParser + + +@pytest.fixture(scope="module") +def pkg_with_setup_py(tmp_path_factory): + base_dir = tmp_path_factory.mktemp("foo") + setup_py = base_dir / "setup.py" + data = """\ +# -*- coding: utf-8 -*- +# a comment +from sys import platform, version_info + +root_dir = path.abspath(path.dirname(__file__)) + +print(root_dir) + +setup( + name="test pkg", + description="testing", + long_description=read("README.md"), + # The project's main homepage. + # Author details + author="Pixee", + packages=find_packages("src"), + package_dir={"": "src"}, + python_requires=">3.6", + install_requires=[ + "protobuf>=3.12,<3.18; python_version < '3'", + "protobuf>=3.12,<4; python_version >= '3'", + "psutil>=5.7,<6", + "requests>=2.4.2,<3", + ], + entry_points={}, +) + """ + setup_py.write_text(data) + return base_dir + + +class TestSetupPyParser: + def test_parse(self, pkg_with_setup_py): + parser = SetupPyParser(pkg_with_setup_py) + found = parser.parse() + assert len(found) == 1 + store = found[0] + assert store.type == "setup.py" + assert store.file == str(pkg_with_setup_py / parser.file_name) + assert store.py_versions == [">3.6"] + assert len(store.dependencies) == 4 + + def test_parse_no_file(self, pkg_with_setup_py): + parser = SetupPyParser(pkg_with_setup_py / "foo") + found = parser.parse() + assert len(found) == 0 diff --git a/tests/test_python_repo_manager.py b/tests/project_analysis/test_python_repo_manager.py similarity index 61% rename from tests/test_python_repo_manager.py rename to tests/project_analysis/test_python_repo_manager.py index c6359f76..41730385 100644 --- a/tests/test_python_repo_manager.py +++ b/tests/project_analysis/test_python_repo_manager.py @@ -2,7 +2,7 @@ class TestPythonRepoManager: - def test_package_stores(self, dir_with_pkg_managers): - rm = PythonRepoManager(dir_with_pkg_managers) + def test_package_stores(self, pkg_with_reqs_txt): + rm = PythonRepoManager(pkg_with_reqs_txt) stores = rm.package_stores assert len(stores) == 1 From 58f0e1f9cad5fb62a1cdb35910fe1cc7a5ea9822 Mon Sep 17 00:00:00 2001 From: clavedeluna Date: Wed, 25 Oct 2023 08:39:02 -0300 Subject: [PATCH 6/8] use abstract --- src/codemodder/context.py | 2 +- .../file_parsers/base_parser.py | 17 ++++++++++++----- .../file_parsers/pyproject_toml_file_parser.py | 6 +++--- .../requirements_txt_file_parser.py | 8 +++----- .../file_parsers/setup_cfg_file_parser.py | 6 +++--- .../file_parsers/setup_py_file_parser.py | 10 +++++----- .../test_pyproject_toml_file_parser.py | 1 + .../file_parsers/test_setup_cfg_file_parser.py | 1 + .../file_parsers/test_setup_py_file_parser.py | 1 + 9 files changed, 30 insertions(+), 22 deletions(-) diff --git a/src/codemodder/context.py b/src/codemodder/context.py index 9f883e3e..15f71d05 100644 --- a/src/codemodder/context.py +++ b/src/codemodder/context.py @@ -41,7 +41,7 @@ def __init__( verbose: bool, registry: CodemodRegistry, repo_manager: PythonRepoManager, - ): + ): # pylint: disable=too-many-arguments self.directory = directory self.dry_run = dry_run self.verbose = verbose diff --git a/src/codemodder/project_analysis/file_parsers/base_parser.py b/src/codemodder/project_analysis/file_parsers/base_parser.py index 4dff3da9..d2a79a98 100644 --- a/src/codemodder/project_analysis/file_parsers/base_parser.py +++ b/src/codemodder/project_analysis/file_parsers/base_parser.py @@ -1,24 +1,31 @@ +from abc import ABC, abstractmethod + from pathlib import Path from typing import List from .package_store import PackageStore from packaging.requirements import Requirement -class BaseParser: +class BaseParser(ABC): def __init__(self, parent_directory: Path): self.parent_directory = parent_directory - self.file_name: str = "" - def _parse_dependencies(self, lines: List[str]): + @property + @abstractmethod + def file_name(self): + ... + + def _parse_dependencies(self, dependencies: List[str]): return [ Requirement(line) - for x in lines + for x in dependencies # Skip empty lines and comments if (line := x.strip()) and not line.startswith("#") ] + @abstractmethod def _parse_file(self, file: Path): - raise NotImplementedError + ... def find_file_locations(self) -> List[Path]: return list(Path(self.parent_directory).rglob(self.file_name)) diff --git a/src/codemodder/project_analysis/file_parsers/pyproject_toml_file_parser.py b/src/codemodder/project_analysis/file_parsers/pyproject_toml_file_parser.py index 8b67c1ef..8cb5c60e 100644 --- a/src/codemodder/project_analysis/file_parsers/pyproject_toml_file_parser.py +++ b/src/codemodder/project_analysis/file_parsers/pyproject_toml_file_parser.py @@ -6,9 +6,9 @@ class PyprojectTomlParser(BaseParser): - def __init__(self, parent_directory: Path): - super().__init__(parent_directory) - self.file_name = "pyproject.toml" + @property + def file_name(self): + return "pyproject.toml" def _parse_dependencies_from_toml(self, toml_data: dict): # todo: handle cases for diff --git a/src/codemodder/project_analysis/file_parsers/requirements_txt_file_parser.py b/src/codemodder/project_analysis/file_parsers/requirements_txt_file_parser.py index 25d85fac..4fa69041 100644 --- a/src/codemodder/project_analysis/file_parsers/requirements_txt_file_parser.py +++ b/src/codemodder/project_analysis/file_parsers/requirements_txt_file_parser.py @@ -1,14 +1,12 @@ from codemodder.project_analysis.file_parsers.package_store import PackageStore -from packaging.requirements import Requirement from pathlib import Path -from typing import List from .base_parser import BaseParser class RequirementsTxtParser(BaseParser): - def __init__(self, parent_directory: Path): - super().__init__(parent_directory) - self.file_name = "requirements.txt" + @property + def file_name(self): + return "requirements.txt" def _parse_file(self, file: Path): with open(file, "r", encoding="utf-8") as f: diff --git a/src/codemodder/project_analysis/file_parsers/setup_cfg_file_parser.py b/src/codemodder/project_analysis/file_parsers/setup_cfg_file_parser.py index 1c94dbb8..5a6cb3f8 100644 --- a/src/codemodder/project_analysis/file_parsers/setup_cfg_file_parser.py +++ b/src/codemodder/project_analysis/file_parsers/setup_cfg_file_parser.py @@ -6,9 +6,9 @@ class SetupCfgParser(BaseParser): - def __init__(self, parent_directory: Path): - super().__init__(parent_directory) - self.file_name = "setup.cfg" + @property + def file_name(self): + return "setup.cfg" def _parse_dependencies_from_toml(self, config: configparser.ConfigParser): # todo: handle cases for diff --git a/src/codemodder/project_analysis/file_parsers/setup_py_file_parser.py b/src/codemodder/project_analysis/file_parsers/setup_py_file_parser.py index ae444ffe..a80efdcd 100644 --- a/src/codemodder/project_analysis/file_parsers/setup_py_file_parser.py +++ b/src/codemodder/project_analysis/file_parsers/setup_py_file_parser.py @@ -9,14 +9,14 @@ class SetupPyParser(BaseParser): - def __init__(self, parent_directory: Path): - super().__init__(parent_directory) - self.file_name = "setup.py" + @property + def file_name(self): + return "setup.py" - def _parse_dependencies(self, cst_dependencies): + def _parse_dependencies(self, dependencies): return [ Requirement(line) - for x in cst_dependencies + for x in dependencies # Skip empty lines and comments if (line := clean_simplestring(x.value)) and not line.startswith("#") ] diff --git a/tests/project_analysis/file_parsers/test_pyproject_toml_file_parser.py b/tests/project_analysis/file_parsers/test_pyproject_toml_file_parser.py index 2c1bdaa0..39e92173 100644 --- a/tests/project_analysis/file_parsers/test_pyproject_toml_file_parser.py +++ b/tests/project_analysis/file_parsers/test_pyproject_toml_file_parser.py @@ -1,3 +1,4 @@ +# pylint: disable=redefined-outer-name import pytest from codemodder.project_analysis.file_parsers import PyprojectTomlParser diff --git a/tests/project_analysis/file_parsers/test_setup_cfg_file_parser.py b/tests/project_analysis/file_parsers/test_setup_cfg_file_parser.py index 729bd2ad..11a54490 100644 --- a/tests/project_analysis/file_parsers/test_setup_cfg_file_parser.py +++ b/tests/project_analysis/file_parsers/test_setup_cfg_file_parser.py @@ -1,3 +1,4 @@ +# pylint: disable=redefined-outer-name import pytest from codemodder.project_analysis.file_parsers import SetupCfgParser diff --git a/tests/project_analysis/file_parsers/test_setup_py_file_parser.py b/tests/project_analysis/file_parsers/test_setup_py_file_parser.py index 2cee31be..bff062c7 100644 --- a/tests/project_analysis/file_parsers/test_setup_py_file_parser.py +++ b/tests/project_analysis/file_parsers/test_setup_py_file_parser.py @@ -1,3 +1,4 @@ +# pylint: disable=redefined-outer-name import pytest from codemodder.project_analysis.file_parsers import SetupPyParser From e141197f2ffd30161007f16eecda6a4a1397b7f5 Mon Sep 17 00:00:00 2001 From: clavedeluna Date: Wed, 25 Oct 2023 09:16:01 -0300 Subject: [PATCH 7/8] ignore cov in base class --- src/codemodder/project_analysis/file_parsers/base_parser.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/codemodder/project_analysis/file_parsers/base_parser.py b/src/codemodder/project_analysis/file_parsers/base_parser.py index d2a79a98..0a8e1bc4 100644 --- a/src/codemodder/project_analysis/file_parsers/base_parser.py +++ b/src/codemodder/project_analysis/file_parsers/base_parser.py @@ -13,7 +13,7 @@ def __init__(self, parent_directory: Path): @property @abstractmethod def file_name(self): - ... + ... # pragma: no cover def _parse_dependencies(self, dependencies: List[str]): return [ @@ -25,7 +25,7 @@ def _parse_dependencies(self, dependencies: List[str]): @abstractmethod def _parse_file(self, file: Path): - ... + ... # pragma: no cover def find_file_locations(self) -> List[Path]: return list(Path(self.parent_directory).rglob(self.file_name)) From 1ee9b1bd7264787c4abeadbe5ace8d6e89d04af4 Mon Sep 17 00:00:00 2001 From: clavedeluna Date: Wed, 25 Oct 2023 15:11:02 -0300 Subject: [PATCH 8/8] review feedback --- src/codemodder/context.py | 1 + .../project_analysis/file_parsers/setup_cfg_file_parser.py | 4 ++-- .../project_analysis/file_parsers/setup_py_file_parser.py | 4 ++-- src/codemodder/project_analysis/file_parsers/utils.py | 2 +- 4 files changed, 6 insertions(+), 5 deletions(-) diff --git a/src/codemodder/context.py b/src/codemodder/context.py index 15f71d05..3f97f89a 100644 --- a/src/codemodder/context.py +++ b/src/codemodder/context.py @@ -33,6 +33,7 @@ class CodemodExecutionContext: # pylint: disable=too-many-instance-attributes dry_run: bool = False verbose: bool = False registry: CodemodRegistry + repo_manager: PythonRepoManager def __init__( self, diff --git a/src/codemodder/project_analysis/file_parsers/setup_cfg_file_parser.py b/src/codemodder/project_analysis/file_parsers/setup_cfg_file_parser.py index 5a6cb3f8..5f715681 100644 --- a/src/codemodder/project_analysis/file_parsers/setup_cfg_file_parser.py +++ b/src/codemodder/project_analysis/file_parsers/setup_cfg_file_parser.py @@ -10,7 +10,7 @@ class SetupCfgParser(BaseParser): def file_name(self): return "setup.cfg" - def _parse_dependencies_from_toml(self, config: configparser.ConfigParser): + def _parse_dependencies_from_cfg(self, config: configparser.ConfigParser): # todo: handle cases for # 1. no dependencies, no options dict # setup_requires, tests_require, extras_require @@ -32,6 +32,6 @@ def _parse_file(self, file: Path): return PackageStore( type=self.file_name, file=str(file), - dependencies=self._parse_dependencies_from_toml(config), + dependencies=self._parse_dependencies_from_cfg(config), py_versions=self._parse_py_versions(config), ) diff --git a/src/codemodder/project_analysis/file_parsers/setup_py_file_parser.py b/src/codemodder/project_analysis/file_parsers/setup_py_file_parser.py index a80efdcd..2fbd78d7 100644 --- a/src/codemodder/project_analysis/file_parsers/setup_py_file_parser.py +++ b/src/codemodder/project_analysis/file_parsers/setup_py_file_parser.py @@ -31,7 +31,7 @@ def _parse_py_versions(self, version_str): return [clean_simplestring(version_str)] def _parse_file(self, file: Path): - visitor = SetupCallVisotor() + visitor = SetupCallVisitor() with open(str(file), "r", encoding="utf-8") as f: # todo: handle failure in parsing module = cst.parse_module(f.read()) @@ -47,7 +47,7 @@ def _parse_file(self, file: Path): ) -class SetupCallVisotor(cst.CSTVisitor): +class SetupCallVisitor(cst.CSTVisitor): def __init__(self): self.python_requires = None self.install_requires = None diff --git a/src/codemodder/project_analysis/file_parsers/utils.py b/src/codemodder/project_analysis/file_parsers/utils.py index 60e1da33..4513523e 100644 --- a/src/codemodder/project_analysis/file_parsers/utils.py +++ b/src/codemodder/project_analysis/file_parsers/utils.py @@ -4,4 +4,4 @@ def clean_simplestring(node: cst.SimpleString | str) -> str: if isinstance(node, str): return node.strip('"') - return node.value.strip('"') + return node.raw_value