From fdd0de1273daa3c9c11d41d803f0050cde28ab9d Mon Sep 17 00:00:00 2001 From: Daniel Hollas Date: Mon, 18 Nov 2024 18:16:43 +0100 Subject: [PATCH] Support new Python versions (#73) * Support and test Python 3.7-3.12 * Temporarily pin aiida-core <2.3 * Refactor /cleanup CI * Use uv in CI * Run editable install only for one version * Update developer guide * Ignore metadata inputs * recursive extras, setuptools dependency * Upgrade mypy, docs with py38 * Fix doc build * Update pylint --- .aiida-testing-config.yml | 13 ++-- .ci/install_aiida_github.sh | 6 -- .github/workflows/ci.yml | 73 +++++++++++++---------- aiida_testing/_config.py | 4 +- aiida_testing/archive_cache/_utils.py | 10 ++-- aiida_testing/mock_code/_fixtures.py | 2 +- aiida_testing/mock_code/_hasher.py | 2 +- docs/source/conf.py | 34 ++--------- docs/source/developer_guide/index.rst | 17 +----- pyproject.toml | 21 +++---- tests/archive_cache/test_archive_cache.py | 4 +- 11 files changed, 74 insertions(+), 112 deletions(-) delete mode 100755 .ci/install_aiida_github.sh diff --git a/.aiida-testing-config.yml b/.aiida-testing-config.yml index 0370b81..8510deb 100644 --- a/.aiida-testing-config.yml +++ b/.aiida-testing-config.yml @@ -4,9 +4,10 @@ mock_code: archive_cache: ignore: calcjob_attributes: - #These attributes have to be ignored to be able to run the export cache tests - #while reusing the same test archive, migrating it to the needed version, - #since they are only present in newer versions - #The test archives have version 0.8 - - environment_variables_double_quotes #This option was introduced in aiida-core 2.0 - - submit_script_filename #This option was introduced in aiida-core 1.2.1 (archive version 0.9) + # These attributes have to be ignored to be able to run the export cache tests + # while reusing the same test archive, migrating it to the needed version, + # since they are only present in newer versions + # The test archives have version 0.8 + - environment_variables_double_quotes # This option was introduced in aiida-core 2.0 + - submit_script_filename # This option was introduced in aiida-core 1.2.1 (archive version 0.9) + - metadata_inputs # Added in aiida-core 2.3.0 diff --git a/.ci/install_aiida_github.sh b/.ci/install_aiida_github.sh deleted file mode 100755 index 397bbe4..0000000 --- a/.ci/install_aiida_github.sh +++ /dev/null @@ -1,6 +0,0 @@ -#!/bin/bash -git clone https://github.com/aiidateam/aiida_core ../aiida_core -cd ../aiida_core -git checkout $AIIDA_DEVELOP_GIT_HASH -pip install -e .[docs,pre_commit,testing] -cd ${TRAVIS_BUILD_DIR} diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 203fd28..1f9ffe5 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,6 +1,9 @@ -name: ci +name: CI on: [push, pull_request] +env: + UV_VER: "0.5.2" + FORCE_COLOR: 1 jobs: @@ -9,12 +12,7 @@ jobs: timeout-minutes: 30 strategy: matrix: - python-version: ['3.9', '3.10'] - editable_install_option: ['-e', ''] - include: - #Test for aiida-core 1.X (since aiida-core 2.X does not support python 3.7) - - python-version: '3.7' - editable_install_option: '-e' + python-version: ['3.7', '3.8', '3.9', '3.10', '3.11', '3.12'] fail-fast: false services: @@ -37,16 +35,19 @@ jobs: - 5672:5672 steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v1 + uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} - - name: Install python dependencies + - name: Set up uv + uses: astral-sh/setup-uv@v3 + with: + version: ${{ env.UV_VER }} + - name: Install package run: | - pip install --upgrade pip - pip install ${{ matrix.editable_install_option }} .[testing] + uv pip install --system -e .[testing] reentry scan -r aiida || true - name: Run test suite @@ -61,16 +62,19 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 15 steps: - - uses: actions/checkout@v1 - - name: Set up Python 3.7 - uses: actions/setup-python@v1 + - uses: actions/checkout@v4 + - name: Set up Python + uses: actions/setup-python@v5 with: - python-version: 3.7 - - name: Install python dependencies - run: | - pip install --upgrade pip - pip install -e .[docs] - reentry scan -r aiida || true + python-version: '3.12' + + - name: Set up uv + uses: astral-sh/setup-uv@v3 + with: + version: ${{ env.UV_VER }} + - name: Install package + run: uv pip install --system -e .[docs] + - name: Build docs run: cd docs && make @@ -78,17 +82,20 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 15 steps: - - uses: actions/checkout@v1 - - name: Set up Python 3.8 - uses: actions/setup-python@v1 + - uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 with: - python-version: 3.8 - - name: Install python dependencies - run: | - pip install --upgrade pip - pip install -e .[pre_commit] - reentry scan -r aiida || true + python-version: '3.12' + + - name: Set up uv + uses: astral-sh/setup-uv@v3 + with: + version: ${{ env.UV_VER }} + + - name: Install package + run: uv pip install --system -e .[pre_commit] + - name: Run pre-commit - run: | - pre-commit install - pre-commit run --all-files || ( git status --short ; git diff ; exit 1 ) + uses: pre-commit/action@v3.0.1 diff --git a/aiida_testing/_config.py b/aiida_testing/_config.py index f3d63a1..50f1564 100644 --- a/aiida_testing/_config.py +++ b/aiida_testing/_config.py @@ -48,7 +48,7 @@ def validate(self): return self.schema(self._dict) @classmethod - def from_file(cls): + def from_file(cls) -> 'Config': """ Parses the configuration file ``.aiida-testing-config.yml``. @@ -58,7 +58,7 @@ def from_file(cls): cwd = pathlib.Path().cwd() config: ty.Dict[str, str] for dir_path in [cwd, *cwd.parents]: - config_file_path = (dir_path / CONFIG_FILE_NAME) + config_file_path = dir_path / CONFIG_FILE_NAME if config_file_path.exists(): with open(config_file_path, encoding='utf8') as config_file: config = yaml.load(config_file, Loader=yaml.SafeLoader) diff --git a/aiida_testing/archive_cache/_utils.py b/aiida_testing/archive_cache/_utils.py index 78c6c3b..fc55367 100644 --- a/aiida_testing/archive_cache/_utils.py +++ b/aiida_testing/archive_cache/_utils.py @@ -115,9 +115,11 @@ def import_with_migrate( raise except ImportError: - from aiida.tools.importexport import export as create_archive #type: ignore[import,no-redef] - from aiida.tools.importexport import import_data as import_archive #type: ignore[no-redef] - import_archive = partial(import_archive, extras_mode_existing='ncu', extras_mode_new='import') + from aiida.tools.importexport import export as create_archive # type: ignore[import-not-found,no-redef] + from aiida.tools.importexport import import_data as import_archive # type: ignore[no-redef] + import_archive = partial( + import_archive, extras_mode_existing='ncu', extras_mode_new='import' + ) # type: ignore[call-arg] def import_with_migrate( archive_path: ty.Union[str, pathlib.Path], @@ -133,7 +135,7 @@ def import_with_migrate( from aiida.tools.importexport import EXPORT_VERSION, IncompatibleArchiveVersionError # these are only availbale after aiida >= 1.5.0, maybe rely on verdi import instead from aiida.tools.importexport import detect_archive_type - from aiida.tools.importexport.archive.migrators import get_migrator #type: ignore[import] + from aiida.tools.importexport.archive.migrators import get_migrator # type: ignore[import-not-found] try: import_archive(archive_path, *args, **kwargs) diff --git a/aiida_testing/mock_code/_fixtures.py b/aiida_testing/mock_code/_fixtures.py index c5da6d4..8e5fe4e 100644 --- a/aiida_testing/mock_code/_fixtures.py +++ b/aiida_testing/mock_code/_fixtures.py @@ -121,7 +121,7 @@ def mock_code_factory( aiida_localhost, testing_config, testing_config_action, mock_regenerate_test_data, mock_fail_on_missing, mock_disable_mpi, monkeypatch, request: pytest.FixtureRequest, tmp_path: pathlib.Path -): # pylint: disable=too-many-arguments,redefined-outer-name,unused-argument,too-many-statements +): # pylint: disable=all """ Fixture to create a mock AiiDA Code. diff --git a/aiida_testing/mock_code/_hasher.py b/aiida_testing/mock_code/_hasher.py index fa7893e..b20e528 100644 --- a/aiida_testing/mock_code/_hasher.py +++ b/aiida_testing/mock_code/_hasher.py @@ -44,7 +44,7 @@ def __call__(self, cwd: Path) -> str: return md5sum.hexdigest() - def modify_content(self, path: Path, content: bytes) -> ty.Optional[bytes]: # pylint: disable=no-self-use,unused-argument + def modify_content(self, path: Path, content: bytes) -> ty.Optional[bytes]: # pylint: disable=unused-argument """A sub-class hook to modify the contents of the file, before hashing. If None is returned, the file is ignored, when generating the hash. diff --git a/docs/source/conf.py b/docs/source/conf.py index 70e8e5b..c00572c 100755 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -1,5 +1,3 @@ -# -*- coding: utf-8 -*- -# # Sphinx configuration for aiida-testing # # This file is execfile()d with the current directory set to its @@ -16,12 +14,7 @@ import time import contextlib -from aiida.manage.configuration import load_documentation_profile - import aiida_testing - -load_documentation_profile() - # -- AiiDA-related setup -------------------------------------------------- # If extensions (or modules to document with autodoc) are in another directory, @@ -37,7 +30,6 @@ with contextlib.suppress(ImportError): import sphinx_rtd_theme html_theme = 'sphinx_rtd_theme' - html_theme_path = [sphinx_rtd_theme.get_html_theme_path()] # -- General configuration ------------------------------------------------ @@ -56,7 +48,7 @@ intersphinx_mapping = { 'python': ('https://docs.python.org/3', None), - 'aiida': ('http://aiida-core.readthedocs.io/en/v1.6.8/', None), + 'aiida': ('https://aiida.readthedocs.io/projects/aiida-core/en/stable/', None), 'pytest': ('https://docs.pytest.org/en/latest/', None) } @@ -66,24 +58,18 @@ # The suffix of source filenames. source_suffix = '.rst' -# The encoding of source files. -#source_encoding = 'utf-8-sig' - # The master toctree document. -#~ master_doc = 'index' master_doc = 'index' # General information about the project. -project = u'aiida-testing' +project = 'aiida-testing' copyright_first_year = "2019" copyright_owners = "The AiiDA Team" current_year = str(time.localtime().tm_year) -copyright_year_string = current_year if current_year == copyright_first_year else "{}-{}".format( - copyright_first_year, current_year -) +copyright_year_string = current_year if current_year == copyright_first_year else f"{copyright_first_year}-{current_year}" # pylint: disable=redefined-builtin -copyright = u'{}, {}. All rights reserved'.format(copyright_year_string, copyright_owners) +copyright = f'{copyright_year_string}, {copyright_owners}. All rights reserved' # The version info for the project you're documenting, acts as replacement for # |version| and |release|, also used in various other places throughout the @@ -146,9 +132,7 @@ # Theme options are theme-specific and customize the look and feel of a theme # further. For a list of options available for each theme, see the # documentation. -html_theme_options = { - 'display_version': True, -} +# html_theme_options = {} # Add any paths that contain custom themes here, relative to this directory. #~ html_theme_path = ["."] @@ -221,9 +205,6 @@ #html_file_suffix = None # Language to be used for generating the HTML full-text search index. -# Sphinx supports the following languages: -# 'da', 'de', 'en', 'es', 'fi', 'fr', 'hu', 'it', 'ja' -# 'nl', 'no', 'pt', 'ro', 'ru', 'sv', 'tr' html_search_language = 'en' # A dictionary with options for the search language support, empty by default. @@ -292,11 +273,6 @@ def run_apidoc(_): apidoc_dir = os.path.join(source_dir, 'apidoc') package_dir = os.path.join(source_dir, os.pardir, os.pardir, 'aiida_testing') - # In #1139, they suggest the route below, but this ended up - # calling sphinx-build, not sphinx-apidoc - #from sphinx.apidoc import main - #main([None, '-e', '-o', apidoc_dir, package_dir, '--force']) - import subprocess cmd_path = 'sphinx-apidoc' if hasattr(sys, 'real_prefix'): # Check to see if we are in a virtualenv diff --git a/docs/source/developer_guide/index.rst b/docs/source/developer_guide/index.rst index 4f70c2e..c7314a4 100644 --- a/docs/source/developer_guide/index.rst +++ b/docs/source/developer_guide/index.rst @@ -34,8 +34,7 @@ Enable enable automatic checks of code sanity and coding style:: pre-commit install After this, the `yapf `_ formatter, -the `pylint `_ linter, the -`prospector `_ code analyzer, and +the `pylint `_ linter, and the `mypy `_ static type checker will run at every commit. @@ -71,17 +70,3 @@ Of course, you can also build the documentation locally:: pip install -e .[docs] cd docs make - -PyPI release -++++++++++++ - -The process for creating a distribution and uploading it to PyPI is:: - - pip install twine - python setup.py sdist - twine upload dist/* - -This can only be done by people who are registered as ``aiida-testing`` -maintainers on PyPI. After this, you (and everyone else) should be able to:: - - pip install aiida-testing diff --git a/pyproject.toml b/pyproject.toml index 69369c2..dc1dd04 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -26,6 +26,8 @@ classifiers = [ "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", "Topic :: Scientific/Engineering :: Physics", "Framework :: AiiDA", "Framework :: Pytest", @@ -39,10 +41,12 @@ keywords = [ ] urls = {Homepage = "https://aiida-testing.readthedocs.io/"} requires-python = ">=3.7" +# Note the dependency on setuptools due to pkg_resources dependencies = [ - "aiida-core>=1.0.0,<3.0.0", - "pytest>=3.6", + "aiida-core>=1.0.0,<2.3", + "pytest>=7.0", "voluptuous~=0.12", + "setuptools", ] [project.optional-dependencies] @@ -55,20 +59,13 @@ testing = [ ] pre_commit = [ "pre-commit", - "pylint~=2.12.2", - "mypy==0.930", + "pylint~=3.3.1", + "mypy==1.13", "types-setuptools==65.7.0.3", "types-PyYAML", ] dev = [ - "sphinx", - "sphinx-rtd-theme", - "pgtest~=1.3.1", - "aiida-diff", - "pytest-datadir", - "pre-commit", - "pylint~=2.12.2", - "mypy==0.930", + 'aiida-testing[testing,pre_commit,docs]', ] [project.scripts] diff --git a/tests/archive_cache/test_archive_cache.py b/tests/archive_cache/test_archive_cache.py index a1ce7a7..dc0da31 100644 --- a/tests/archive_cache/test_archive_cache.py +++ b/tests/archive_cache/test_archive_cache.py @@ -153,7 +153,7 @@ def test_mock_hash_codes(mock_code_factory, clear_database, liberal_hash): def test_enable_archive_cache( archive_path, aiida_local_code_factory, generate_diff_inputs, enable_archive_cache, clear_database, check_diff_workchain -): +): # pylint: disable=too-many-positional-arguments """ Basic test of the enable_archive_cache fixture """ @@ -171,7 +171,7 @@ def test_enable_archive_cache( def test_enable_archive_cache_non_existent( aiida_local_code_factory, generate_diff_inputs, enable_archive_cache, clear_database, tmp_path_factory, check_diff_workchain -): +): # pylint: disable=too-many-positional-arguments """ Test of the enable_archive_cache fixture that creation of the archive and overwriting of the archive works correctly