Added new tests for requirements parser

src/codemodder/project_analysis/file_parsers/requirements_txt_file_parser.py

@@ -1,4 +1,6 @@
 from typing import Optional
+
+from packaging.requirements import InvalidRequirement
 from codemodder.project_analysis.file_parsers.package_store import PackageStore
 from pathlib import Path
 from .base_parser import BaseParser
@@ -15,7 +17,7 @@ def _parse_file(self, file: Path) -> Optional[PackageStore]:
         try:
             with open(file, "rb") as f:
                 whole_file = f.read()
-                enc = chardet.detect(f.read())
+                enc = chardet.detect(whole_file)
                 lines = []
                 if enc["confidence"] > 0.9:
                     encoding = enc.get("encoding")
@@ -32,6 +34,6 @@ def _parse_file(self, file: Path) -> Optional[PackageStore]:
                     # and extracting py versions from them.
                     py_versions=[],
                 )
-        except (UnicodeError, OSError):
+        except (UnicodeError, OSError, InvalidRequirement):
             logger.debug("Error parsing file: %s", file)
         return None

tests/conftest.py

@@ -60,3 +60,22 @@ def pkg_with_reqs_txt(tmp_path_factory):
     reqs = "# comment\nrequests==2.31.0\nblack==23.7.*\nmypy~=1.4\npylint>1\n"
     req_file.write_text(reqs)
     return base_dir
+
+
+@pytest.fixture(scope="module")
+def pkg_with_reqs_txt_utf_16(tmp_path_factory):
+    base_dir = tmp_path_factory.mktemp("foo")
+    req_file = base_dir / "requirements.txt"
+    reqs = "# comment\nrequests==2.31.0\nblack==23.7.*\nmypy~=1.4\npylint>1\n"
+    req_file.write_text(reqs)
+    return base_dir
+
+
+@pytest.fixture(scope="module")
+def pkg_with_reqs_txt_unknown_encoding(tmp_path_factory):
+    base_dir = tmp_path_factory.mktemp("foo")
+    req_file = base_dir / "requirements.txt"
+    # invalid utf8 string
+    reqs = "\xf0\x28\x8c\xbc"
+    req_file.write_text(reqs)
+    return base_dir

tests/project_analysis/file_parsers/test_requirements_txt_file_parser.py

@@ -12,6 +12,21 @@ def test_parse(self, pkg_with_reqs_txt):
         assert store.py_versions == []
         assert len(store.dependencies) == 4
 
+    def test_parse_utf_16(self, pkg_with_reqs_txt_utf_16):
+        parser = RequirementsTxtParser(pkg_with_reqs_txt_utf_16)
+        found = parser.parse()
+        assert len(found) == 1
+        store = found[0]
+        assert store.type == "requirements.txt"
+        assert store.file == str(pkg_with_reqs_txt_utf_16 / parser.file_name)
+        assert store.py_versions == []
+        assert len(store.dependencies) == 4
+
+    def test_parse_unknown_encoding(self, pkg_with_reqs_txt_unknown_encoding):
+        parser = RequirementsTxtParser(pkg_with_reqs_txt_unknown_encoding)
+        found = parser.parse()
+        assert len(found) == 0
+
     def test_parse_no_file(self, pkg_with_reqs_txt):
         parser = RequirementsTxtParser(pkg_with_reqs_txt / "foo")
         found = parser.parse()