From 6c0c667c7988a6ceeb763e525d5691a64fab5501 Mon Sep 17 00:00:00 2001 From: Sebastian Csar Date: Thu, 23 Nov 2023 12:12:31 -0800 Subject: [PATCH 1/4] Swap tomllib for toml. Fixes #339 -- the toml package does not support heterogenous arrays, which are now allowed in the toml spec. This commit swaps in tomllib from the standard library for Python >= 3.11 and its backport tomli for older versions. --- .gitignore | 1 + setup.py | 2 +- tests/test_config.py | 80 +++++++++++++++++++++++++++++++++++++------- vulture/config.py | 9 +++-- 4 files changed, 75 insertions(+), 17 deletions(-) diff --git a/.gitignore b/.gitignore index df191195..0ec272f6 100644 --- a/.gitignore +++ b/.gitignore @@ -10,3 +10,4 @@ vulture.egg-info/ .pytest_cache/ .tox/ .venv/ +.vscode/ diff --git a/setup.py b/setup.py index 3914be97..48549415 100644 --- a/setup.py +++ b/setup.py @@ -47,7 +47,7 @@ def find_version(*parts): "Programming Language :: Python :: Implementation :: PyPy", "Topic :: Software Development :: Quality Assurance", ], - install_requires=["toml"], + install_requires=["tomli >= 1.1.0; python_version < '3.11'"], entry_points={"console_scripts": ["vulture = vulture.core:main"]}, python_requires=">=3.8", packages=setuptools.find_packages(exclude=["tests"]), diff --git a/tests/test_config.py b/tests/test_config.py index 494433de..30e21984 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -2,7 +2,7 @@ Unit tests for config file and CLI argument parsing. """ -from io import StringIO +from io import BytesIO from textwrap import dedent import pytest @@ -62,9 +62,10 @@ def test_toml_config(): sort_by_size=True, verbose=True, ) - data = StringIO( - dedent( - """\ + data = BytesIO( + bytes( + dedent( + """\ [tool.vulture] exclude = ["file*.py", "dir/"] ignore_decorators = ["deco1", "deco2"] @@ -75,6 +76,50 @@ def test_toml_config(): verbose = true paths = ["path1", "path2"] """ + ), + "utf-8", + ) + ) + result = _parse_toml(data) + assert isinstance(result, dict) + assert result == expected + + +def test_toml_config_with_heterogenous_array(): + """ + Ensure parsing of TOML files results in a valid config object, even if some + other part of the file contains an array of mixed types. + """ + expected = dict( + paths=["path1", "path2"], + exclude=["file*.py", "dir/"], + ignore_decorators=["deco1", "deco2"], + ignore_names=["name1", "name2"], + make_whitelist=True, + min_confidence=10, + sort_by_size=True, + verbose=True, + ) + data = BytesIO( + bytes( + dedent( + """\ + [tool.foo] + # comment for good measure + problem_array = [{ a = 1}, [2,3,4], "foo"] + + [tool.vulture] + exclude = ["file*.py", "dir/"] + ignore_decorators = ["deco1", "deco2"] + ignore_names = ["name1", "name2"] + make_whitelist = true + min_confidence = 10 + sort_by_size = true + verbose = true + paths = ["path1", "path2"] + """ + ), + "utf-8", ) ) result = _parse_toml(data) @@ -87,9 +132,10 @@ def test_config_merging(): If we have both CLI args and a ``pyproject.toml`` file, the CLI args should have precedence. """ - toml = StringIO( - dedent( - """\ + toml = BytesIO( + bytes( + dedent( + """\ [tool.vulture] exclude = ["toml_exclude"] ignore_decorators = ["toml_deco"] @@ -100,6 +146,8 @@ def test_config_merging(): verbose = false paths = ["toml_path"] """ + ), + "utf-8", ) ) cliargs = [ @@ -131,13 +179,16 @@ def test_config_merging_missing(): If we have set a boolean value in the TOML file, but not on the CLI, we want the TOML value to be taken. """ - toml = StringIO( - dedent( - """\ + toml = BytesIO( + bytes( + dedent( + """\ [tool.vulture] verbose = true ignore_names = ["name1"] """ + ), + "utf-8", ) ) cliargs = [ @@ -153,12 +204,15 @@ def test_config_merging_toml_paths_only(): If we have paths in the TOML but not on the CLI, the TOML paths should be used. """ - toml = StringIO( - dedent( - """\ + toml = BytesIO( + bytes( + dedent( + """\ [tool.vulture] paths = ["path1", "path2"] """ + ), + "utf-8", ) ) cliargs = [ diff --git a/vulture/config.py b/vulture/config.py index 4aa0d2d5..4e193fe2 100644 --- a/vulture/config.py +++ b/vulture/config.py @@ -5,7 +5,10 @@ import argparse import pathlib -import toml +try: + import tomllib +except ModuleNotFoundError: + import tomli as tomllib from .version import __version__ @@ -76,7 +79,7 @@ def _parse_toml(infile): verbose = true paths = ["path1", "path2"] """ - data = toml.load(infile) + data = tomllib.load(infile) settings = data.get("tool", {}).get("vulture", {}) _check_input_config(settings) return settings @@ -194,7 +197,7 @@ def make_config(argv=None, tomlfile=None): else: toml_path = pathlib.Path("pyproject.toml").resolve() if toml_path.is_file(): - with open(toml_path) as fconfig: + with open(toml_path, "rb") as fconfig: config = _parse_toml(fconfig) detected_toml_path = str(toml_path) else: From 4b2a42116a6a766b09744a8d8f0352f8d3abd54d Mon Sep 17 00:00:00 2001 From: Sebastian Csar Date: Thu, 23 Nov 2023 12:27:22 -0800 Subject: [PATCH 2/4] Update changelog with PR reference. --- CHANGELOG.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index c7938149..0037111f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,6 @@ +# 2.11 (2023-11-13) +* Switch to tomllib/tomli to support heterogenous arrays. (Sebastian Csar, #340) + # 2.10 (2023-10-06) * Drop support for Python 3.7 (Jendrik Seipp, #323). From bcc2e2d802e60be4f7198b3f93b43758822f8e48 Mon Sep 17 00:00:00 2001 From: Sebastian Csar Date: Fri, 24 Nov 2023 10:10:44 -0800 Subject: [PATCH 3/4] Apply suggestions from code review Co-authored-by: Jendrik Seipp --- CHANGELOG.md | 4 ++-- tests/test_config.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0037111f..660b0d07 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,5 @@ -# 2.11 (2023-11-13) -* Switch to tomllib/tomli to support heterogenous arrays. (Sebastian Csar, #340) +# next (unreleased) +* Switch to tomllib/tomli to support heterogeneous arrays (Sebastian Csar, #340). # 2.10 (2023-10-06) diff --git a/tests/test_config.py b/tests/test_config.py index 30e21984..7872b082 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -106,7 +106,7 @@ def test_toml_config_with_heterogenous_array(): """\ [tool.foo] # comment for good measure - problem_array = [{ a = 1}, [2,3,4], "foo"] + problem_array = [{a = 1}, [2,3,4], "foo"] [tool.vulture] exclude = ["file*.py", "dir/"] From 54eade576626a6eefc33e6f24453630074b6604a Mon Sep 17 00:00:00 2001 From: Sebastian Csar Date: Fri, 24 Nov 2023 10:13:08 -0800 Subject: [PATCH 4/4] Pull toml BytesIO creation out into a function. --- tests/test_config.py | 52 +++++++++++++++++++------------------------- 1 file changed, 22 insertions(+), 30 deletions(-) diff --git a/tests/test_config.py b/tests/test_config.py index 7872b082..04ce7259 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -17,6 +17,13 @@ ) +def get_toml_bytes(toml_str: str) -> BytesIO: + """ + Wrap a string in BytesIO to play the role of the incoming config stream. + """ + return BytesIO(bytes(toml_str, "utf-8")) + + def test_cli_args(): """ Ensure that CLI arguments are converted to a config object. @@ -62,10 +69,9 @@ def test_toml_config(): sort_by_size=True, verbose=True, ) - data = BytesIO( - bytes( - dedent( - """\ + data = get_toml_bytes( + dedent( + """\ [tool.vulture] exclude = ["file*.py", "dir/"] ignore_decorators = ["deco1", "deco2"] @@ -76,8 +82,6 @@ def test_toml_config(): verbose = true paths = ["path1", "path2"] """ - ), - "utf-8", ) ) result = _parse_toml(data) @@ -100,10 +104,9 @@ def test_toml_config_with_heterogenous_array(): sort_by_size=True, verbose=True, ) - data = BytesIO( - bytes( - dedent( - """\ + data = get_toml_bytes( + dedent( + """\ [tool.foo] # comment for good measure problem_array = [{a = 1}, [2,3,4], "foo"] @@ -118,8 +121,6 @@ def test_toml_config_with_heterogenous_array(): verbose = true paths = ["path1", "path2"] """ - ), - "utf-8", ) ) result = _parse_toml(data) @@ -132,10 +133,9 @@ def test_config_merging(): If we have both CLI args and a ``pyproject.toml`` file, the CLI args should have precedence. """ - toml = BytesIO( - bytes( - dedent( - """\ + toml = get_toml_bytes( + dedent( + """\ [tool.vulture] exclude = ["toml_exclude"] ignore_decorators = ["toml_deco"] @@ -146,8 +146,6 @@ def test_config_merging(): verbose = false paths = ["toml_path"] """ - ), - "utf-8", ) ) cliargs = [ @@ -179,16 +177,13 @@ def test_config_merging_missing(): If we have set a boolean value in the TOML file, but not on the CLI, we want the TOML value to be taken. """ - toml = BytesIO( - bytes( - dedent( - """\ + toml = get_toml_bytes( + dedent( + """\ [tool.vulture] verbose = true ignore_names = ["name1"] """ - ), - "utf-8", ) ) cliargs = [ @@ -204,15 +199,12 @@ def test_config_merging_toml_paths_only(): If we have paths in the TOML but not on the CLI, the TOML paths should be used. """ - toml = BytesIO( - bytes( - dedent( - """\ + toml = get_toml_bytes( + dedent( + """\ [tool.vulture] paths = ["path1", "path2"] """ - ), - "utf-8", ) ) cliargs = [