diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 7e0bc4a8..90403a09 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -28,12 +28,12 @@ jobs: fail-fast: false matrix: include: - - python-version: '3.6' - os: ubuntu-20.04 - - python-version: '3.10' - os: ubuntu-22.04 - python-version: '3.11' os: ubuntu-22.04 + - python-version: '3.12' + os: ubuntu-22.04 + - python-version: '3.13' + os: ubuntu-22.04 runs-on: ${{ matrix.os }} steps: - uses: actions/checkout@v4 @@ -58,21 +58,6 @@ jobs: pip install 'virtualenv<20.22' 'tox==4.5.1' tox-gh-actions tox --workdir .github/workflows/.tox --recreate - # tox >= 4.0.0 is needed for using optional-dependencies from pyproject.toml, which is - # is not available for python <= 3.6, so use the python3.8 of Ubuntu-20.04 to install it: - - name: Run tox for 3.6 and 3.8 on ${{ matrix.os }}'s 3.8 to get 'extras' from pyproject.toml) - if: ${{ matrix.python-version == 2.7 || matrix.python-version == 3.6 }} - run: | - set -xv;curl -sSL https://bootstrap.pypa.io/get-pip.py -o get-pip.py - python3.8 get-pip.py - # The alternative is installing python3-pip but we don't need full pip function for now: - # sudo apt-get update && sudo apt-get install -y python3-pip - # Let tox-gh-actions get the environment(s) to run tests with from tox.ini: - # Use tox==4.5.1: tox>=4 is needed for reading the extras from pyproject.toml - # Warning: tox>=4.5.2 depends on virutalenv>=20.23, which breaks Python 2.7: - python3.8 -m pip install 'virtualenv<20.22' 'tox==4.5.1' tox-gh-actions - tox --workdir .github/workflows/.tox --recreate - - name: Select the coverage file for upload if: | ( matrix.python-version == '3.6' || matrix.python-version == '3.11' ) && diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 4dea24a9..36b678db 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -101,13 +101,6 @@ repos: - toml - repo: local hooks: - - id: run-pyre - name: run-pyre (expect this to take 30 seconds) - entry: python pyre_runner.py - types: [python] - language: python - log_file: ".git/pre-commit-pyre.log" - additional_dependencies: [pyre-check,mock] - id: pytype name: pytype (may take up to two minutes) entry: sh -c "pytype >/dev/tty" diff --git a/.pylintrc b/.pylintrc index dc41d223..5b5816c0 100644 --- a/.pylintrc +++ b/.pylintrc @@ -65,6 +65,7 @@ disable=W0142,W0703,C0111,R0201,W0603,W0613,W0212,W0141, len-as-condition, no-else-return, raise-missing-from, + too-many-positional-arguments, too-many-branches, too-many-nested-blocks, too-many-statements, @@ -222,8 +223,9 @@ defining-attr-methods=__init__,__new__,setUp [DESIGN] -# Maximum number of arguments for function / method -max-args=100 +# Maximum number of arguments for function / method. +# defaults to: max-args=5 +max-args=10 # Argument names that match this expression will be ignored. Default to name # with leading underscore diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 910602de..5d53d115 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1,3 +1,14 @@ +# Development setup + +## Create a virtual environment with the test dependencies + +```bash +python -m venv .venv +. .venv/bin/activate +pip install pip-tools==7.3.0 +pip-compile --extra=test,mypy,pyright,pytype,tox -o - pyproject.toml | pip install -r /dev/stdin +``` + ## Development setup on Fedora 37 On Fedora 37, the `tox` rpm installs all Python versions. @@ -6,9 +17,26 @@ But this `tox` is older, so install `tox==4.5.1` using `pip` (see below) ```bash sudo dnf install tox;sudo rpm -e tox ``` +But preferably use `tox` from the virtual environment instead. + +## Development setup on Ubuntu 24.04 + +Prefer the virtual environment. Alternatively, an option is to use `pipx`: + +```bash +sudo apt install pipx +pipx install tox; pipx install 'pytest<7';pipx install pylint +pipx inject pytest pytest-{forked,localftpserver,pythonpath,subprocess,timeout} pyfakefs pytest_httpserver six mock +pipx inject pylint pyfakefs six mock pytest{,_forked,-localftpserver} +``` + +Use the `deadsnakes` ppa to install Python versions like 3.8 and 3.11 (see below) ## Development setup on Ubuntu 22.04 +Usage of to install +other python versions. + ```bash sudo apt update sudo apt install software-properties-common python{2,3}-dev @@ -50,13 +78,12 @@ This project uses `tox` to run the tests for different python versions. Intro: > A comprehensive beginner's introduction to `tox`":_ > https://www.seanh.cc/2018/09/01/tox-tutorial/ -`tox` runs `pytest`, `pylint` and static analysis using `mypy`, `pyre`, `pytype`, and `pyright`. +`tox` runs `pytest`, `pylint` and static analysis using `mypy`, `pytype`, and `pyright`. Links: - https://mypy.readthedocs.io/en/stable/ - https://microsoft.github.io/pyright/ - https://google.github.io/pytype/ -- https://pyre-check.org/ With `tox`, developers can run the full test suite for Python 2.7 and 3.x. The same test suite is used in GitHub CI: @@ -78,7 +105,7 @@ Using pip-tools, you can extract the requirements and extras from `pyptoject.tom ```bash PYTHON=python3.10 -EXTRAS=.,test,mypy,pyre,pytype,tox +EXTRAS=.,test,mypy,pyright,pytype,tox PFLAGS="--no-warn-conflicts" $PYTHON -m pip install pip-tools==7.3.0 $PYTHON -m piptools compile --extra=$EXTRAS -o - pyproject.toml | diff --git a/README.md b/README.md index 2f24022a..fa00abca 100644 --- a/README.md +++ b/README.md @@ -28,7 +28,7 @@ The Continuous Integration Tests feature: - Checking of the combined coverage against the diff to master: Fails if changes are not covered! - Pylint report in the GitHub Action Summary page, with Warning and Error annotatios, even in the code review. - Check that changes don't generate pylint warnings (if warning classes which are enabled in .pylintrc) -- Static analysis using `mypy`, `pyre` and `pytype` +- Static analysis using `mypy`, `pylint`, `pyright` and `pytype` This enforces that any change (besides whitespace): @@ -72,7 +72,6 @@ For the installation of the general development dependencies, visit [INSTALL.md] - Test with `python2.7 -m pytest` - Run `mypy` (without any arguments - The configuration is in `pyproject.toml`) - Run `./pytype_runner.py` -- Run `./pyre_runner.py` - Run `tox -e py36-lint` and fix any `Pylint` warnings - Run `tox -e py310-covcombine-check` and fix any missing diff-coverage. - Run `tox` for the full CI test suite @@ -86,7 +85,7 @@ The list of `virtualenvs` configured in tox can be shown using this command: `to $ tox -av default environments: py36-lint -> Run in a py36 virtualenv: Run pylint and fail on warnings remaining on lines in the diff to master -py311-pyre -> Run in a py311 virtualenv: Run pyre for static analyis, only passes using: tox -e py311-pyre +py311-pyright -> Run in a py311 virtualenv: Run pyright for static analyis py38-pytype -> Run in a py38 virtualenv: Run pytype for static analyis, intro: https://youtu.be/abvW0mOrDiY py310-covcombine-check -> Run in a py310 virtualenv: Generate combined coverage reports with py27-test coverage merged Run mypy for static analyis @@ -100,7 +99,7 @@ test -> Run in a python virtualenv: Run pytest in this environ If you have only one version of Python3, that works too. Use: `tox -e py-test` -## Static analysis using mypy, pyre, pyright and pytype +## Static analysis using mypy, pylint, pyright and pytype The preconditions for using static analysis with `mypy` (which passes now but has only a few type comments) and `pyright` are present now and `mypy` is enabled in `tox` diff --git a/pyproject.toml b/pyproject.toml index 474a6e80..fee467da 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -71,11 +71,6 @@ mypy = [ "types-six", "types-toml", ] -# pyre introduced two false-postives recently, pin it to prevent further surprises: -pyre = [ - "pyre-check == 0.9.21", - "pyre-extensions == 0.0.30", -] pytype = [ "pandas", "pytype", @@ -116,11 +111,12 @@ show_error_context = true error_summary = true files = ["xcp", "tests/test_*.py"] mypy_path = "stubs" -python_version = "3.10" +python_version = "3.11" warn_redundant_casts = true warn_return_any = true warn_unreachable = true warn_unused_configs = true +explicit_package_bases = true disallow_any_unimported = true disallow_any_explicit = false disallow_any_generics = true @@ -198,7 +194,7 @@ exclude = [ extraPaths = ["stubs"] include = ["xcp", "tests"] pythonPlatform = "Linux" -pythonVersion = "3.6" +pythonVersion = "3.11" reportFunctionMemberAccess = true reportGeneralTypeIssues = "warning" reportOptionalMemberAccess = "warning" @@ -211,7 +207,7 @@ stubPath = "stubs" inputs = ["xcp", "tests", "./*.py"] keepgoing = true platform = "linux" -python_version = "3.10" +python_version = "3.11" pythonpath = ".:stubs" disable = ["ignored-type-comment"] overriding_parameter_count_checks = true diff --git a/pyre_runner.py b/pyre_runner.py deleted file mode 100755 index a67d2427..00000000 --- a/pyre_runner.py +++ /dev/null @@ -1,62 +0,0 @@ -#!/usr/bin/env python -""" -Run a one-time pyre static analysis check without needing a .pyre_configuration -Gets the paths dynamically so it can be used in tox and GitHub CI -""" -import os -import sys -import time - -import mock - -me = os.path.basename(__file__) + ":" - -pyre_typesched = os.environ.get("PYRE_TYPESHED", None) -if pyre_typesched and os.path.exists(pyre_typesched + "/stdlib/os/path.pyi"): - print("Using {env:PYRE_TYPESHED}:", pyre_typesched) -else: - pyre_typesched = sys.path[-1] + "/mypy/typeshed" - if os.path.exists(pyre_typesched + "/stdlib/os/path.pyi"): - print("Using python_lib:", pyre_typesched) - else: - pyre_typesched = "/tmp/typeshed" - if os.path.exists(pyre_typesched + "/stdlib/os/path.pyi"): - print("Using:", pyre_typesched) - else: - clone = "git clone --depth 1 https://github.com/python/typeshed " - print(me, "Falling back to:", clone + pyre_typesched) - ret = os.system(clone + pyre_typesched) - if ret or not os.path.exists(pyre_typesched + "/stdlib/os/path.pyi"): - print(me, "Could not find or clone typeshed, giving up.") - sys.exit(0) - -command = ( - "pyre", - "--source-directory", - "xcp", - "--source-directory", - "tests", - "--search-path", - "stubs", - "--search-path", - ".", - "--search-path", - os.path.dirname(mock.__file__), - "--typeshed", - pyre_typesched, - "check", -) -cmd = " ".join(command) -print(me, "Running:", cmd) -start_time = time.time() -ret = os.system(cmd) -duration = time.time() - start_time -r = os.waitstatus_to_exitcode(ret) # type: ignore[module-addr] # newer versions have it -if r == 0: - print(me, f"OK pyre took: {duration:.1f}s") -else: - print(me, "Ran:", cmd) - print(me, "exit code:", r) - if os.environ.get("ACT", None): - time.sleep(10) -sys.exit(r) diff --git a/pytest.ini b/pytest.ini index e07456ee..0906991a 100644 --- a/pytest.ini +++ b/pytest.ini @@ -8,10 +8,11 @@ # These are the most of the needed pytest plugins, unfortunately this list does # not support ;python_version<=3.0 or ;python_version>3.0. Therefore, it can # only list plugins available for all tested python versions (2.7, 3.6 ... 3.11): +# pytest-localftpserver is also used, but its installation is not checked +# to to its installation not being detected on Ubuntu 24.04: required_plugins = pytest_httpserver pytest-forked - pytest-localftpserver pytest-pythonpath pytest-subprocess pytest-timeout diff --git a/stubs/pyfakefs/__init__.pyi b/stubs/pyfakefs/__init__.pyi deleted file mode 100644 index e69de29b..00000000 diff --git a/stubs/pyfakefs/fake_filesystem.pyi b/stubs/pyfakefs/fake_filesystem.pyi deleted file mode 100644 index 466eb8b1..00000000 --- a/stubs/pyfakefs/fake_filesystem.pyi +++ /dev/null @@ -1,151 +0,0 @@ -from _typeshed import Incomplete # pylint: disable=import-error - - -def set_uid(uid) -> None: ... -def set_gid(gid) -> None: ... -def reset_ids() -> None: ... -def is_root(): ... - - -class FakeFilesystem: - path_separator: Incomplete - alternative_path_separator: Incomplete - patcher: Incomplete - is_windows_fs: Incomplete - is_macos: Incomplete - is_case_sensitive: Incomplete - root: Incomplete - cwd: Incomplete - umask: Incomplete - open_files: Incomplete - mount_points: Incomplete - dev_null: Incomplete - def __init__( - self, - path_separator=..., - total_size: Incomplete | None = ..., - patcher: Incomplete | None = ..., - ) -> None: ... - @property - def is_linux(self): ... - def reset(self, total_size: Incomplete | None = ...) -> None: ... - def pause(self) -> None: ... - def resume(self) -> None: ... - def line_separator(self): ... - def raise_os_error( - self, errno, filename: Incomplete | None = ..., winerror: Incomplete | None = ... - ) -> None: ... - def raise_io_error(self, errno, filename: Incomplete | None = ...) -> None: ... - def add_mount_point(self, path, total_size: Incomplete | None = ...): ... - def get_disk_usage(self, path: Incomplete | None = ...): ... - def set_disk_usage(self, total_size, path: Incomplete | None = ...) -> None: ... - def change_disk_usage(self, usage_change, file_path, st_dev) -> None: ... - def stat(self, entry_path, follow_symlinks: bool = ...): ... - def chmod(self, path, mode, follow_symlinks: bool = ...) -> None: ... - def utime( - self, - path, - times: Incomplete | None = ..., - ns: Incomplete | None = ..., - follow_symlinks: bool = ..., - ) -> None: ... - def get_open_file(self, file_des): ... - def has_open_file(self, file_object): ... - def normcase(self, path): ... - def normpath(self, path): ... - def absnormpath(self, path): ... - def splitpath(self, path): ... - def splitdrive(self, path): ... - def joinpaths(self, *paths): ... - def ends_with_path_separator(self, file_path): ... - def is_filepath_ending_with_separator(self, path): ... - def exists(self, file_path, check_link: bool = ...): ... - def resolve_path(self, file_path, allow_fd: bool = ..., raw_io: bool = ...): ... - def get_object_from_normpath(self, file_path, check_read_perm: bool = ...): ... - def get_object(self, file_path, check_read_perm: bool = ...): ... - def resolve( - self, - file_path, - follow_symlinks: bool = ..., - allow_fd: bool = ..., - check_read_perm: bool = ..., - ): ... - def lresolve(self, path): ... - def add_object(self, file_path, file_object, error_fct: Incomplete | None = ...) -> None: ... - def rename(self, old_file_path, new_file_path, force_replace: bool = ...) -> None: ... - def remove_object(self, file_path) -> None: ... - def make_string_path(self, path): ... - def create_dir(self, directory_path, perm_bits=...): ... - def create_file( - self, - file_path, - st_mode=..., - contents: str = ..., - st_size: Incomplete | None = ..., - create_missing_dirs: bool = ..., - apply_umask: bool = ..., - encoding: Incomplete | None = ..., - errors: Incomplete | None = ..., - side_effect: Incomplete | None = ..., - ): ... - def add_real_file( - self, source_path, read_only: bool = ..., target_path: Incomplete | None = ... - ): ... - def add_real_symlink(self, source_path, target_path: Incomplete | None = ...): ... - def add_real_directory( - self, - source_path, - read_only: bool = ..., - lazy_read: bool = ..., - target_path: Incomplete | None = ..., - ): ... - def add_real_paths( - self, path_list, read_only: bool = ..., lazy_dir_read: bool = ... - ) -> None: ... - def create_file_internally( - self, - file_path, - st_mode=..., - contents: str = ..., - st_size: Incomplete | None = ..., - create_missing_dirs: bool = ..., - apply_umask: bool = ..., - encoding: Incomplete | None = ..., - errors: Incomplete | None = ..., - read_from_real_fs: bool = ..., - raw_io: bool = ..., - side_effect: Incomplete | None = ..., - ): ... - def create_symlink(self, file_path, link_target, create_missing_dirs: bool = ...): ... - def link(self, old_path, new_path, follow_symlinks: bool = ...): ... - def readlink(self, path): ... - def makedir(self, dir_name, mode=...) -> None: ... - def makedirs(self, dir_name, mode=..., exist_ok: bool = ...) -> None: ... - def isdir(self, path, follow_symlinks: bool = ...): ... - def isfile(self, path, follow_symlinks: bool = ...): ... - def islink(self, path): ... - def confirmdir(self, target_directory): ... - def remove(self, path) -> None: ... - def rmdir(self, target_directory, allow_symlink: bool = ...) -> None: ... - def listdir(self, target_directory): ... - -class FakeFileOpen: - __name__: str - filesystem: Incomplete - raw_io: Incomplete - def __init__( - self, filesystem, delete_on_close: bool = ..., use_io: bool = ..., raw_io: bool = ... - ) -> None: ... - def __call__(self, *args, **kwargs): ... - def call( - self, - file_, - mode: str = ..., - buffering: int = ..., - encoding: Incomplete | None = ..., - errors: Incomplete | None = ..., - newline: Incomplete | None = ..., - closefd: bool = ..., - opener: Incomplete | None = ..., - open_modes: Incomplete | None = ..., - ): ... diff --git a/stubs/pyfakefs/fake_filesystem_unittest.pyi b/stubs/pyfakefs/fake_filesystem_unittest.pyi deleted file mode 100644 index aade18b0..00000000 --- a/stubs/pyfakefs/fake_filesystem_unittest.pyi +++ /dev/null @@ -1,20 +0,0 @@ -from types import ModuleType -from typing import Any, Callable, Dict, List, Optional, Union - -from _typeshed import Incomplete # pylint: disable=import-error - -Patcher = Incomplete -PatchMode = Incomplete - -def patchfs( - _func: Optional[Incomplete] = ..., - *, - additional_skip_names: Optional[List[Union[str, ModuleType]]] = ..., - modules_to_reload: Optional[List[ModuleType]] = ..., - modules_to_patch: Optional[Dict[str, ModuleType]] = ..., - allow_root_user: bool = ..., - use_known_patches: bool = ..., - patch_open_code: PatchMode = ..., - patch_default_args: bool = ..., - use_cache: bool = ..., -) -> Callable[[Any], Any]: ... diff --git a/stubs/pytest.pyi b/stubs/pytest.pyi deleted file mode 100644 index 6e31256b..00000000 --- a/stubs/pytest.pyi +++ /dev/null @@ -1,8 +0,0 @@ -# pylint: disable=reimported,no-name-in-module,unused-import,function-redefined.redefined-builtin -from _pytest.python_api import raises -from _typeshed import Incomplete as fixture -from _typeshed import Incomplete as mark - -def skip(msg: str = "", *, allow_module_level: bool = False): ... - -__all__ = ["mark", "fixture", "skip", "raises"] diff --git a/stubs/werkzeug/wrappers.pyi b/stubs/werkzeug/wrappers.pyi deleted file mode 100644 index f4a46d17..00000000 --- a/stubs/werkzeug/wrappers.pyi +++ /dev/null @@ -1,3 +0,0 @@ -from _typeshed import Incomplete # pylint: disable=import-error,no-name-in-module -Request = Incomplete -Response = Incomplete diff --git a/tests/conftest.py b/tests/conftest.py index 1a2a4974..da395853 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -5,10 +5,10 @@ This module is run automatically by pytest to define and enable fixtures. """ -# pyre-ignore-all-errors[21] import warnings -import pytest # pyre does not find the module when run by tox -e py311-pyre +import pytest + @pytest.fixture(autouse=True) def set_warnings(): diff --git a/tests/httpserver_testcase.py b/tests/httpserver_testcase.py index b9dc5ceb..cad59524 100644 --- a/tests/httpserver_testcase.py +++ b/tests/httpserver_testcase.py @@ -1,4 +1,5 @@ import os +import sys import unittest from typing import Callable, Optional @@ -11,9 +12,10 @@ from pytest_httpserver import HTTPServer from werkzeug.wrappers import Request, Response - ErrorHandler = Optional[Callable[[Request], Response]] + ErrorHandler = Optional[Callable[[Request], Response | None]] except ImportError: pytest.skip(allow_module_level=True) + sys.exit(0) # Let pyright know that this is a dead end class HTTPServerTestCase(unittest.TestCase): @@ -31,7 +33,7 @@ def tearDownClass(cls): @classmethod def serve_file(cls, root, file_path, error_handler=None, real_path=None): - # type:(str, str, Optional[Callable[[Request], Response]], Optional[str]) -> None + # type:(str, str, ErrorHandler, Optional[str]) -> None """Expect a GET request and handle it using the local pytest_httpserver.HTTPServer""" def handle_get(request): diff --git a/tests/test_accessor.py b/tests/test_accessor.py index 93f5d609..885d5762 100644 --- a/tests/test_accessor.py +++ b/tests/test_accessor.py @@ -1,14 +1,16 @@ import unittest - -from pyfakefs.fake_filesystem import FakeFilesystem +from typing import TYPE_CHECKING import xcp.accessor from .test_mountingaccessor import check_binary_read, check_binary_write +if TYPE_CHECKING: + import pyfakefs + def test_file_accessor(fs): - # type:(FakeFilesystem) -> None + # type(pyfakefs.fake_filesystem.FakeFilesystem) -> None """Test FileAccessor.writeFile(), .openAddress and .access using pyfakefs""" accessor = xcp.accessor.createAccessor("file://repo/", False) assert isinstance(accessor, xcp.accessor.FileAccessor) @@ -18,7 +20,7 @@ def test_file_accessor(fs): class TestAccessor(unittest.TestCase): def setUp(self): - """Provide the refrence content of the repo/.treeinfo file for check_repo_access()""" + """Provide the reference content of the repo/.treeinfo file for check_repo_access()""" with open("tests/data/repo/.treeinfo", "rb") as dot_treeinfo: self.reference_treeinfo = dot_treeinfo.read() @@ -35,6 +37,7 @@ def check_repo_access(self, a): self.assertFalse(a.access('no_such_file')) self.assertEqual(a.lastError, 404) a.finish() + a.finish() # Cover the code handing a 2nd call of accessor.finish() def test_filesystem_accessor_access(self): """Test FilesystemAccessor.access()""" diff --git a/tests/test_bootloader.py b/tests/test_bootloader.py index 82500336..23b21a5e 100644 --- a/tests/test_bootloader.py +++ b/tests/test_bootloader.py @@ -41,8 +41,7 @@ def test_grub2(self): universal_newlines=True) assert proc.stdout for line in proc.stdout: - # pylint: disable-next=deprecated-method - self.assertRegexpMatches(line, r"^(5a6,13$|>)") + self.assertRegex(line, r"^(5a6,13$|>)") # Replaced removed assert call proc.stdout.close() proc.wait() diff --git a/tests/test_cpiofile.py b/tests/test_cpiofile.py index cd36a2f2..0371d2df 100644 --- a/tests/test_cpiofile.py +++ b/tests/test_cpiofile.py @@ -1,5 +1,4 @@ # suppress false positive on pytest missing pytest.raises(): -# pyre-ignore-all-errors[16] """ Test various modes of creating and extracting CpioFile using different compression types, opening the archive as stream and as file, using pyfakefs as filesystem without diff --git a/tests/test_ftpaccessor.py b/tests/test_ftpaccessor.py index a5b0b303..340b759c 100644 --- a/tests/test_ftpaccessor.py +++ b/tests/test_ftpaccessor.py @@ -21,13 +21,14 @@ from io import BytesIO import pytest -import pytest_localftpserver # pylint: disable=unused-import # Ensure that it is installed +import pytest_localftpserver # Ensure that it is installed from six import ensure_binary, ensure_str import xcp.accessor binary_data = b"\x80\x91\xaa\xb0\xb1\xb2\xb3\xb4\xb5\xb6\xb7\xb8\xb9\xcc\xdd\xee\xff" text_data = "✋➔Hello Accessor from the 🗺, download and verify ✅ me!" +assert pytest_localftpserver def upload_textfile(ftpserver, accessor): diff --git a/tests/test_httpaccessor.py b/tests/test_httpaccessor.py index d0248f7f..c6bd38fa 100644 --- a/tests/test_httpaccessor.py +++ b/tests/test_httpaccessor.py @@ -43,7 +43,6 @@ def http_get_request_data(self, url, read_file, error_handler): def assert_http_get_request_data(self, url, read_file, error_handler): # type:(str, str, ErrorHandler) -> HTTPAccessor - # pyre-ignore[23]: silence false positive with self.http_get_request_data(url, read_file, error_handler) as (httpaccessor, ref): http_accessor_filehandle = httpaccessor.openAddress(read_file) if sys.version_info >= (3, 0): diff --git a/tests/test_ifrename_logic.py b/tests/test_ifrename_logic.py index 5d445f3c..f6966054 100644 --- a/tests/test_ifrename_logic.py +++ b/tests/test_ifrename_logic.py @@ -1,4 +1,4 @@ -# pyright: reportGeneralTypeIssues=false +# pyright: reportGeneralTypeIssues=false,reportAttributeAccessIssue=false # pytype: disable=attribute-error from __future__ import print_function from __future__ import unicode_literals diff --git a/tests/test_logger.py b/tests/test_logger.py index af8cd5c3..3a14521d 100644 --- a/tests/test_logger.py +++ b/tests/test_logger.py @@ -10,8 +10,16 @@ from xcp.logger import openLog -def test_openLog_mock_open(): - """Cover xcp.logger.openLog.open_with_codec_handling and check the arguments used for open()""" +def test_openLog_mock_open(fs): + # type(FakeFilesystem) -> None + """ + - Covers xcp.logger.openLog.open_with_codec_handling and + - checks the arguments it uses for open() + + Because needs to call openLog() which creates a logfile, this test uses + the pytest pyfakefs fixture 'fs' which wraps creating it in a virtual fs. + With it, this tests does not create a log file on the filesystem of the host. + """ fh = StringIO() with patch("xcp.compat.open", mock_open()) as open_mock: open_mock.return_value = fh @@ -31,3 +39,4 @@ def test_openLog_mock_stdin(): assert openLog("test.log") is True os.close(slave_fd) os.close(master_fd) + open_mock.assert_called_once_with("test.log", "a", **open_utf8) diff --git a/tests/test_mountingaccessor.py b/tests/test_mountingaccessor.py index 9dea2d3f..54bb6303 100644 --- a/tests/test_mountingaccessor.py +++ b/tests/test_mountingaccessor.py @@ -20,6 +20,7 @@ import pytest pytest.skip(allow_module_level=True) + sys.exit(0) # Let pyright know that this is a dead end binary_data = b"\x00\x1b\x5b\x95\xb1\xb2\xb3\xb4\xb5\xb6\xb7\xb8\xb9\xcc\xdd\xee\xff" diff --git a/tests/test_pci.py b/tests/test_pci.py index 1b723538..d3278507 100644 --- a/tests/test_pci.py +++ b/tests/test_pci.py @@ -11,6 +11,8 @@ if sys.version_info >= (3, 6): from pytest_subprocess.fake_process import FakeProcess +else: + pytest.skip(allow_module_level=True) class TestInvalid(unittest.TestCase): @@ -31,6 +33,7 @@ class TestValid(unittest.TestCase): def test_null_with_segment(self): + assert PCI.is_valid("0000:00:00.0") is True c = PCI("0000:00:00.0") self.assertEqual(c.segment, 0) @@ -43,6 +46,7 @@ def test_null_with_segment(self): def test_null_without_segment(self): + assert PCI.is_valid("00:00.0") is True c = PCI("00:00.0") self.assertEqual(c.segment, 0) @@ -54,6 +58,7 @@ def test_null_without_segment(self): def test_valid(self): + assert PCI.is_valid("8765:43:1f.3") is True c = PCI("8765:43:1f.3") self.assertEqual(c.segment, 0x8765) @@ -61,14 +66,31 @@ def test_valid(self): self.assertEqual(c.device, 0x1f) self.assertEqual(c.function, 0x3) + def test_valid_index(self): + + assert PCI.is_valid("8765:43:1f.3[0]") is True + assert PCI.is_valid("1234:56:01.7[1]") is True + c = PCI("1234:56:01.7[1]") + + self.assertEqual(c.segment, 0x1234) + self.assertEqual(c.bus, 0x56) + self.assertEqual(c.device, 0x01) + self.assertEqual(c.function, 0x7) + self.assertEqual(c.index, 0x1) + def test_equality(self): self.assertEqual(PCI("0000:00:00.0"), PCI("00:00.0")) + assert PCI("1234:56:01.7[1]") != PCI("1234:56:01.7[2]") + assert PCI("1234:56:01.2") >= PCI("1234:56:01.2") + assert PCI("1234:56:01.1") <= PCI("1234:56:01.2") + assert PCI("1234:56:01.3") > PCI("1234:56:01.2") + assert PCI("1234:56:01.1") < PCI("1234:56:02.2") if sys.version_info >= (2, 7): def assert_videoclass_devices(ids, devs): # type: (PCIIds, PCIDevices) -> None - """Verification function for checking the otuput of PCIDevices.findByClass()""" + """Verification function for checking the output of PCIDevices.findByClass()""" video_class = ids.lookupClass('Display controller') assert video_class == ["03"] sorted_devices = sorted(devs.findByClass(video_class), @@ -76,6 +98,7 @@ def assert_videoclass_devices(ids, devs): # type: (PCIIds, PCIDevices) -> None # Assert devs.findByClass() finding 3 GPUs from tests/data/lspci-mn in our mocked PCIIds DB: assert len(sorted_devices) == 3 + assert len(devs.findByClass("03", "80")) == 2 # For each of the found devices, assert these expected values: for (video_dev, @@ -164,6 +187,7 @@ def test_videoclass_by_mock_calls(fp): def mock_lspci_using_open_testfile(fp): """Mock xcp.pci.PCIDevices.Popen() using open(tests/data/lspci-mn)""" with open("tests/data/lspci-mn", "rb") as fake_data: - assert isinstance(fp, FakeProcess) + if sys.version_info >= (3, 6): + assert isinstance(fp, FakeProcess) fp.register_subprocess(["lspci", "-mn"], stdout=fake_data.read()) return PCIDevices() diff --git a/tox.ini b/tox.ini index d7386335..0491f56e 100644 --- a/tox.ini +++ b/tox.ini @@ -1,11 +1,17 @@ [tox] -# This is the order how tox runs the tests when used interactively during development. -# Run the tests which uncover issues most often first! For example: -# 1. python 2.7 and 3.10 coverage test for changed, but not covered lines and mypy check -# 2. python 3.6 test and pylint warnings from changed lines -# 3. pytype (needs Python 3.8 for best results) -# 4. pyre and pyright checks, pytest test report as markdown for GitHub Actions summary -envlist = py38-covcombine-check, py36-lint-test, py310-pytype, py311-pyre-mdreport +# Set the envlist: +# Defines the environments that tox runs by default (when -e is not used) +# +# Default order of execution when using `tox` without `-e `: +# +# 1. pytest using python 3.11 for code coverage with diff-cover and mypy check +# 2. pytype (needs Python 3.10 or newer) +# 3. pylint and pyright checks (any modern Python version is fine) +# +# .github/workflows/main.yml is set up to test with 3.11, 3.12 and 3.13 in parallel. +# Therefore, use three environments: One with 3.11, one with 3.12 and one with 3.13: +# +envlist = py311-covcp-check-mdreport, py312-cov-pytype, py313-cov-lint-pyright isolated_build = true skip_missing_interpreters = true requires = @@ -28,9 +34,9 @@ commands = # https://github.com/actions/toolkit/blob/main/docs/commands.md#problem-matchers echo "::add-matcher::.github/workflows/PYTHONWARNINGS-problemMatcher.json" pytest --cov -v --new-first -x --show-capture=all -rA - sh -c 'ls -l {env:COVERAGE_FILE}' sh -c 'if [ -n "{env:PYTEST_MD_REPORT_OUTPUT}" -a -n "{env:GITHUB_STEP_SUMMARY}" ];then \ - sed -i "s/tests\(.*py\)/[&](&)/" {env:PYTEST_MD_REPORT_OUTPUT}; sed "/title/,/\/style/d" \ + mkdir -p $(dirname "{env:GITHUB_STEP_SUMMARY:.git/sum.md}"); \ + sed "s/tests\(.*py\)/[&](&)/" \ {env:PYTEST_MD_REPORT_OUTPUT} >{env:GITHUB_STEP_SUMMARY:.git/sum.md};fi' [testenv] @@ -42,15 +48,15 @@ description = Run in a {basepython} virtualenv: lint: {[lint]description} mdreport: Make a test report (which is shown in the GitHub Actions Summary Page) test: {[test]description} - # https://pypi.org/project/pyre-check/ pyre intro: https://youtu.be/0FSXS5kw2m4 - pyre: Run pyre for static analyis, only passes using: tox -e py311-pyre + # See https://microsoft.github.io/pyright/#/configuration for more in pyright + pyright: Run pyright for static analyis check: Run mypy for static analyis pytype: Run pytype for static analyis, intro: https://youtu.be/abvW0mOrDiY # checkers(mypy) need the pytest dependices as well: extras = {check,pytype}: {[check]extras} - {cov,covcp,covcombine,fox,check,lint,test,pytype,pyre,mdreport}: {[test]extras} - {cov,covcp,covcombine,fox}: {[cov]extras} + {cov,covcp,covcombine,fox,check,lint,test,pytype,pyright,mdreport}: {[test]extras} + {cov,covcp,covcombine,fox}: {[cov]extras} deps = mdreport: pytest-md-report {py27-test,py27-cov}: pyftpdlib @@ -58,9 +64,7 @@ deps = {cov,covcp,covcombine,fox}: coverage[toml] {cov,covcp,covcombine,fox}: diff-cover {lint,fox}: {[lint]deps} - pyre: pyre-check - pyre: pyre-extensions - pyre: pyright + pyright: pyright pytype: {[pytype]deps} allowlist_externals = {cov,covcp,covcombine,fox,check,lint,test,mdreport}: echo @@ -81,7 +85,6 @@ passenv = covcp: HOME check: MYPY_FORCE_COLOR check: MYPY_FORCE_TERMINAL_WIDTH - pyre: PYRE_TYPESHED {fox,check,pytype}: TERM fox: DISPLAY fox: XAUTHORITY @@ -104,7 +107,7 @@ setenv = {[cov]setenv} commands = lint: {[lint]commands} - pyre: {[pyre]commands} + pyright: {[pyright]commands} check: {[check]commands} pytype: {[pytype]commands} {cov,covcp,covcombine,check,fox,test,mdreport}: {[test]commands} @@ -189,6 +192,8 @@ python = 3.9: py39 3.10: py310 3.11: py311 + 3.12: py312 + 3.13: py313 [check] extras = mypy @@ -200,17 +205,17 @@ commands = ignore = W191,W293,W504,E101,E126,E127,E201,E202,E203,E221,E222,E226,E227,E241,E251,E261,E262,E265,E301,E302,E303,E305,E722,W391,E401,E402,E741 max-line-length = 129 -[pyre] +[pyright] commands = - pyre: python3.11 --version -V # Needs py311-pyre, does not work with py310-pyre - python pyre_runner.py - -pyright + pyright --version + # To make pyright checks optional, change the next line to '-pyright': + pyright [pytype] deps = pytype pandas commands = - python3.10 -V # Needs python <= 3.10, and 3.10 is needed to parse new "|" syntax + # Needs python >= 3.10: Needed to parse the newer syntax for "Type2 | Type2" pytype --version # Runs pytype -j auto -k --config .github/workflows/pytype.cfg and parses the output: python3 pytype_runner.py # When switching versions, update .github/workflows/pytype.cfg too! diff --git a/xcp/accessor.py b/xcp/accessor.py index 9583dfe7..afdb643d 100644 --- a/xcp/accessor.py +++ b/xcp/accessor.py @@ -24,7 +24,6 @@ """accessor - provide common interface to access methods""" import errno -# pyre-ignore-all-errors[6,16] import ftplib import io import os @@ -315,7 +314,8 @@ def finish(self): self.cleanup = False self.ftp = None - def access(self, path): # pylint: disable=arguments-differ,arguments-renamed + # pylint: disable-next=arguments-differ,arguments-renamed + def access(self, path): # pyright: ignore[reportIncompatibleMethodOverride] try: logger.debug("Testing "+path) self._cleanup() diff --git a/xcp/bootloader.py b/xcp/bootloader.py index 353cc7ff..81721c82 100644 --- a/xcp/bootloader.py +++ b/xcp/bootloader.py @@ -32,7 +32,6 @@ import xcp.cmd -# pyre-ignore-all-errors[21] try: # xenserver-release.rpm puts a branding.py into our xcp installation directory: from xcp import branding # type:ignore[attr-defined] # pytype: disable=import-error except ImportError: # For CI, use stubs/branding.py (./stubs is added to pythonpath) diff --git a/xcp/cpiofile.py b/xcp/cpiofile.py index 995da94c..ff8f5c7c 100644 --- a/xcp/cpiofile.py +++ b/xcp/cpiofile.py @@ -36,8 +36,6 @@ Derived from Lars Gustäbel's tarfile.py """ from __future__ import print_function -# pyre is not as good as other static analysis tools in inferring the correct types: -# pyre-ignore-all-errors[6,16] __version__ = "0.1" __author__ = "Simon Rowe"