From 10d7e6b247024a642a5c566b3a7020e4eb9a97b4 Mon Sep 17 00:00:00 2001 From: Ian Good Date: Sun, 27 Feb 2022 11:57:25 -0500 Subject: [PATCH] Use invoke to standardize checks and tasks --- .github/release-drafter.yml | 6 --- .github/workflows/python-package.yml | 22 ++++----- .github/workflows/python-publish.yml | 10 ++--- .github/workflows/release-drafter.yml | 14 ------ .lvimrc | 3 -- MANIFEST.in | 4 +- doc/requirements.txt | 1 - doc/source/conf.py | 15 ++++++- pymapadmin/commands/health.py | 3 +- pymapadmin/commands/mailbox.py | 2 +- pymapadmin/commands/system.py | 7 +-- pymapadmin/commands/user.py | 8 ++-- pymapadmin/local.py | 5 ++- pyproject.toml | 25 +++++++++++ requirements-dev.txt | 3 +- setup.cfg | 14 ------ setup.py | 2 +- tasks/__init__.py | 65 +++++++++++++++++++++++++++ tasks/check.py | 19 ++++++++ tasks/doc.py | 34 ++++++++++++++ tasks/lint.py | 27 +++++++++++ tasks/test.py | 21 +++++++++ tasks/types.py | 21 +++++++++ test/test_health.py | 18 +++++--- test/test_mailbox.py | 11 +++-- test/test_system.py | 14 ++++-- test/test_user.py | 16 ++++--- 27 files changed, 299 insertions(+), 91 deletions(-) delete mode 100644 .github/release-drafter.yml delete mode 100644 .github/workflows/release-drafter.yml delete mode 100644 .lvimrc create mode 100644 pyproject.toml create mode 100644 tasks/__init__.py create mode 100644 tasks/check.py create mode 100644 tasks/doc.py create mode 100644 tasks/lint.py create mode 100644 tasks/test.py create mode 100644 tasks/types.py diff --git a/.github/release-drafter.yml b/.github/release-drafter.yml deleted file mode 100644 index 4e5af4a..0000000 --- a/.github/release-drafter.yml +++ /dev/null @@ -1,6 +0,0 @@ -exclude-labels: - - 'skip-changelog' -template: | - ## Change Log - - $CHANGES diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index 18b6283..2c73966 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -21,19 +21,15 @@ jobs: uses: actions/setup-python@v2 with: python-version: ${{ matrix.python-version }} - - name: Install dependencies - run: | - python -m pip install --upgrade pip setuptools wheel coveralls - python -m pip install --upgrade -r requirements-dev.txt - - name: Lint with flake8 + - name: Install build tools run: | - flake8 pymapadmin test - - name: Type checking with mypy + python -m pip install --upgrade pip setuptools wheel invoke coveralls + - name: Install package and dependencies run: | - mypy pymapadmin test - - name: Test with pytest + invoke install + - name: Run test suites, type checks, and linters run: | - py.test --cov=pymapadmin + invoke validate - name: Report test coverage to Coveralls if: success() env: @@ -53,9 +49,7 @@ jobs: python-version: '3.10' - name: Install dependencies run: | - python -m pip install --upgrade pip setuptools wheel - python -m pip install --upgrade -r requirements-dev.txt - python -m pip install --upgrade -r doc/requirements.txt + python -m pip install --upgrade pip setuptools wheel invoke - name: Build the Sphinx documentation run: | - make -C doc html + invoke install doc.install doc.build diff --git a/.github/workflows/python-publish.yml b/.github/workflows/python-publish.yml index d8dfa48..1a4ceaa 100644 --- a/.github/workflows/python-publish.yml +++ b/.github/workflows/python-publish.yml @@ -15,7 +15,7 @@ jobs: uses: actions/setup-python@v2 with: python-version: '3.10' - - name: Install dependencies + - name: Install build tools run: | python -m pip install --upgrade pip setuptools wheel twine - name: Build and publish @@ -36,14 +36,12 @@ jobs: uses: actions/setup-python@v2 with: python-version: '3.10' - - name: Install dependencies + - name: Install build tools run: | - python -m pip install --upgrade pip setuptools wheel - python -m pip install --upgrade -r requirements-dev.txt - python -m pip install --upgrade -r doc/requirements.txt + python -m pip install --upgrade pip setuptools wheel invoke - name: Build the Sphinx documentation run: | - make -C doc html + invoke install doc.install doc.build - name: Deploy to GitHub Pages if: success() uses: peaceiris/actions-gh-pages@v3 diff --git a/.github/workflows/release-drafter.yml b/.github/workflows/release-drafter.yml deleted file mode 100644 index d6f59e6..0000000 --- a/.github/workflows/release-drafter.yml +++ /dev/null @@ -1,14 +0,0 @@ -name: draft - -on: - push: - branches: [ main ] - -jobs: - update_release_draft: - - runs-on: ubuntu-latest - steps: - - uses: release-drafter/release-drafter@v5 - env: - GITHUB_TOKEN: ${{ github.token }} diff --git a/.lvimrc b/.lvimrc deleted file mode 100644 index 5f6e825..0000000 --- a/.lvimrc +++ /dev/null @@ -1,3 +0,0 @@ -let g:ale_fixers = { -\ 'python': ['autopep8'], -\} diff --git a/MANIFEST.in b/MANIFEST.in index 10b961f..072e094 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,2 +1,4 @@ -include README.md LICENSE.md pymapadmin/py.typed +include README.md LICENSE.md pyproject.toml pymapadmin/py.typed recursive-include pymapadmin *.pyi +recursive-include test *.py +prune tasks diff --git a/doc/requirements.txt b/doc/requirements.txt index eb6efc6..c577a66 100644 --- a/doc/requirements.txt +++ b/doc/requirements.txt @@ -1,3 +1,2 @@ sphinx -sphinx-autodoc-typehints cloud_sptheme diff --git a/doc/source/conf.py b/doc/source/conf.py index 611870b..49c726a 100644 --- a/doc/source/conf.py +++ b/doc/source/conf.py @@ -43,7 +43,7 @@ 'sphinx.ext.intersphinx', 'sphinx.ext.napoleon', 'sphinx.ext.githubpages', - 'sphinx_autodoc_typehints', + 'sphinx.ext.viewcode', ] # Add any paths that contain templates here, relative to this directory. @@ -70,10 +70,23 @@ # so a file named "default.css" will overwrite the builtin "default.css". html_static_path = ['_static'] +# Theme options are theme-specific and customize the look and feel of a theme +# further. For a list of options available for each theme, see the +# documentation. +# +if csp.is_cloud_theme(html_theme): + html_theme_options = { + 'borderless_decor': True, + 'sidebarwidth': '3in', + 'hyphenation_language': 'en', + } + # -- Extension configuration ------------------------------------------------- autodoc_member_order = 'bysource' autodoc_default_flags = ['show-inheritance'] +autodoc_typehints = 'description' +autodoc_typehints_format = 'short' napoleon_numpy_docstring = False # -- Options for intersphinx extension --------------------------------------- diff --git a/pymapadmin/commands/health.py b/pymapadmin/commands/health.py index 8a215a2..3252efb 100644 --- a/pymapadmin/commands/health.py +++ b/pymapadmin/commands/health.py @@ -28,9 +28,10 @@ class CheckCommand(HealthCommand[HealthCheckRequest, @classmethod def add_subparser(cls, name: str, subparsers: Any) \ -> ArgumentParser: # pragma: no cover - return subparsers.add_parser( + argparser: ArgumentParser = subparsers.add_parser( name, description=cls.__doc__, help='check the server health') + return argparser @property def method(self) -> MethodProtocol[HealthCheckRequest, diff --git a/pymapadmin/commands/mailbox.py b/pymapadmin/commands/mailbox.py index 0ced6ae..6465832 100644 --- a/pymapadmin/commands/mailbox.py +++ b/pymapadmin/commands/mailbox.py @@ -35,7 +35,7 @@ class AppendCommand(MailboxCommand[AppendRequest, AppendResponse]): @classmethod def add_subparser(cls, name: str, subparsers: Any) \ -> ArgumentParser: # pragma: no cover - subparser = subparsers.add_parser( + subparser: ArgumentParser = subparsers.add_parser( name, description=cls.__doc__, help='append a message to a mailbox') subparser.add_argument('--from', metavar='ADDRESS', dest='sender', diff --git a/pymapadmin/commands/system.py b/pymapadmin/commands/system.py index 695fa21..3499053 100644 --- a/pymapadmin/commands/system.py +++ b/pymapadmin/commands/system.py @@ -33,7 +33,7 @@ class SaveArgsCommand(Command): @classmethod def add_subparser(cls, name: str, subparsers: Any) \ -> ArgumentParser: # pragma: no cover - subparser = subparsers.add_parser( + subparser: ArgumentParser = subparsers.add_parser( name, description=cls.__doc__, help='save connection arguments to config file') return subparser @@ -53,7 +53,7 @@ class LoginCommand(SystemCommand[LoginRequest, LoginResponse]): @classmethod def add_subparser(cls, name: str, subparsers: Any) \ -> ArgumentParser: # pragma: no cover - subparser = subparsers.add_parser( + subparser: ArgumentParser = subparsers.add_parser( name, description=cls.__doc__, help='login as a user') subparser.add_argument('-s', '--save', action='store_true', @@ -110,9 +110,10 @@ class PingCommand(SystemCommand[PingRequest, PingResponse]): @classmethod def add_subparser(cls, name: str, subparsers: Any) \ -> ArgumentParser: # pragma: no cover - return subparsers.add_parser( + argparser: ArgumentParser = subparsers.add_parser( name, description=cls.__doc__, help='ping the server') + return argparser @property def method(self) -> MethodProtocol[PingRequest, PingResponse]: diff --git a/pymapadmin/commands/user.py b/pymapadmin/commands/user.py index 05d572c..17d3bfd 100644 --- a/pymapadmin/commands/user.py +++ b/pymapadmin/commands/user.py @@ -28,7 +28,7 @@ class GetUserCommand(UserCommand[GetUserRequest, UserResponse]): @classmethod def add_subparser(cls, name: str, subparsers: Any) \ -> ArgumentParser: # pragma: no cover - subparser = subparsers.add_parser( + subparser: ArgumentParser = subparsers.add_parser( name, description=cls.__doc__, help='get a user') subparser.add_argument('username', help='the user name') @@ -48,7 +48,7 @@ class SetUserCommand(UserCommand[SetUserRequest, UserResponse]): @classmethod def add_subparser(cls, name: str, subparsers: Any) \ -> ArgumentParser: # pragma: no cover - subparser = subparsers.add_parser( + subparser: ArgumentParser = subparsers.add_parser( name, description=cls.__doc__, help='assign a password to a user') subparser.add_argument('--password-file', type=FileType('r'), @@ -70,7 +70,7 @@ def getpass(self) -> str | None: if self.args.no_password: return None elif self.args.password_file: - line = self.args.password_file.readline() + line: str = self.args.password_file.readline() return line.rstrip('\r\n') else: return getpass.getpass() @@ -100,7 +100,7 @@ class DeleteUserCommand(UserCommand[DeleteUserRequest, UserResponse]): @classmethod def add_subparser(cls, name: str, subparsers: Any) \ -> ArgumentParser: # pragma: no cover - subparser = subparsers.add_parser( + subparser: ArgumentParser = subparsers.add_parser( name, description=cls.__doc__, help='delete a user') subparser.add_argument('username', help='the user name') diff --git a/pymapadmin/local.py b/pymapadmin/local.py index 0f36876..5a3a474 100644 --- a/pymapadmin/local.py +++ b/pymapadmin/local.py @@ -15,13 +15,14 @@ class _AddAction(Action): - def __init__(self, local_file: LocalFile, *args, **kwargs) -> None: + def __init__(self, local_file: LocalFile, + *args: Any, **kwargs: Any) -> None: kwargs['metavar'] = 'PATH' super().__init__(*args, **kwargs) self._local_file = local_file def __call__(self, parser: ArgumentParser, namespace: Namespace, - values: Any, option_string: str = None) -> None: + values: Any, option_string: str | None = None) -> None: setattr(namespace, self.dest, values) self._local_file.add(values) diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..0b8dbf8 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,25 @@ +[build-system] +requires = ['setuptools', 'wheel'] + +[tool.mypy] +strict = true +files = ['pymapadmin', 'test'] + +[[tool.mypy.overrides]] +module = 'google.rpc.*' +ignore_missing_imports = true + +[tool.pytest.ini_options] +testpaths = 'test' +asyncio_mode = 'auto' +norecursedirs = 'doc' + +[tool.coverage.report] +fail_under = 90 +omit = ['*/main.py', '*/config.py', '*/local.py', '*/grpc/*'] +exclude_lines = [ + 'pragma: no cover', + 'NotImplemented', + '^\s*...\s*$', + 'def __repr__', +] diff --git a/requirements-dev.txt b/requirements-dev.txt index 4e1c78a..52e1943 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -4,10 +4,11 @@ autopep8 pytest pytest-asyncio pytest-cov +bandit[toml] rope # unreleased typing changes -grpclib == 0.4.3rc1 +grpclib == 0.4.3rc2 # stubs types-setuptools diff --git a/setup.cfg b/setup.cfg index f0327cc..adde45b 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,16 +1,2 @@ -[mypy] -files = pymapadmin, test -ignore_missing_imports = True - [flake8] exclude = pymapadmin/grpc - -[tool:pytest] -asyncio_mode = auto - -[coverage:report] -omit = */main.py, */config.py, */local.py, */grpc/* -exclude_lines = - pragma: no cover - NotImplementedError - ^\s*...\s*$ diff --git a/setup.py b/setup.py index 701aed4..7f4601f 100644 --- a/setup.py +++ b/setup.py @@ -48,7 +48,7 @@ 'Programming Language :: Python :: 3.10'], python_requires='~=3.10', include_package_data=True, - packages=find_packages(), + packages=find_packages(include=('pymapadmin', 'pymapadmin.*')), install_requires=[ 'grpclib', 'protobuf', 'typing-extensions'], extras_require={ diff --git a/tasks/__init__.py b/tasks/__init__.py new file mode 100644 index 0000000..56a31ed --- /dev/null +++ b/tasks/__init__.py @@ -0,0 +1,65 @@ +# type: ignore + +import os +import os.path +from shlex import join +from invoke import task, Collection + +from . import check, doc, lint, test, types + + +@task +def clean(ctx, full=False): + """Delete all the standard build and validate artifacts.""" + if full: + ctx.run('git clean -dfX') + else: + anywhere = ['__pycache__'] + top_level = [ + '.coverage', + '.mypy_cache', + '.pytest_cache', + 'dist', + 'doc/build/'] + for name in anywhere: + for path in [ctx.package, 'test']: + subpaths = [os.path.join(subpath, name) + for subpath, dirs, names in os.walk(path) + if name in dirs or name in names] + for subpath in subpaths: + ctx.run(join(['rm', '-rf', subpath])) + for name in top_level: + ctx.run(join(['rm', '-rf', name])) + + +@task +def install(ctx, dev=True, update=False): + """Install the library and all development tools.""" + choice = 'dev' if dev else 'all' + if update: + ctx.run('pip install -U -r requirements-{}.txt'.format(choice)) + else: + ctx.run('pip install -r requirements-{}.txt'.format(choice)) + + +@task(test.all, types.all, lint.all) +def validate(ctx): + """Run all tests, type checks, and linters.""" + pass + + +ns = Collection(clean, install) +ns.add_task(validate, default=True) +ns.add_collection(check) +ns.add_collection(test) +ns.add_collection(types) +ns.add_collection(lint) +ns.add_collection(doc) + +ns.configure({ + 'package': 'pymapadmin', + 'run': { + 'echo': True, + 'pty': True, + } +}) diff --git a/tasks/check.py b/tasks/check.py new file mode 100644 index 0000000..43d7052 --- /dev/null +++ b/tasks/check.py @@ -0,0 +1,19 @@ +# type: ignore + +import warnings + +from invoke import task, Collection + + +@task +def check_import(ctx): + """Check that the library can be imported.""" + try: + __import__(ctx.package) + except Exception: + warnings.warn('Could not import {!r}, ' + 'task may fail'.format(ctx.package)) + + +ns = Collection() +ns.add_task(check_import, default=True) diff --git a/tasks/doc.py b/tasks/doc.py new file mode 100644 index 0000000..4d5fc07 --- /dev/null +++ b/tasks/doc.py @@ -0,0 +1,34 @@ +# type: ignore + +from invoke import task, Collection + + +@task +def install(ctx, update=False): + """Install the tools needed to build the docs.""" + if update: + ctx.run('pip install -U -r doc/requirements.txt') + elif not ctx.run('which sphinx-build', hide=True, warn=True): + ctx.run('pip install -r doc/requirements.txt') + + +@task(install) +def clean(ctx): + """Clean up the doc build directory.""" + ctx.run('make -C doc clean') + + +@task(install) +def build(ctx): + """Build the HTML docs.""" + ctx.run('make -C doc html') + + +@task(install, build) +def open(ctx): + """Open the docs in a browser (on macOS).""" + ctx.run('open doc/build/html/index.html') + + +ns = Collection(install, clean, open) +ns.add_task(build, default=True) diff --git a/tasks/lint.py b/tasks/lint.py new file mode 100644 index 0000000..0f221ed --- /dev/null +++ b/tasks/lint.py @@ -0,0 +1,27 @@ +# type: ignore + +from invoke import task, Collection + +from .check import check_import + + +@task(check_import) +def flake8(ctx): + """Run the flake8 linter.""" + ctx.run('flake8 {} test {} *.py'.format(ctx.package, __package__)) + + +@task(check_import) +def bandit(ctx): + """Run the bandit linter.""" + ctx.run('bandit -qr {}'.format(ctx.package)) + + +@task(flake8, bandit) +def all(ctx): + """Run all linters.""" + pass + + +ns = Collection(flake8, bandit) +ns.add_task(all, default=True) diff --git a/tasks/test.py b/tasks/test.py new file mode 100644 index 0000000..92645d4 --- /dev/null +++ b/tasks/test.py @@ -0,0 +1,21 @@ +# type: ignore + +from invoke import task, Collection + +from .check import check_import + + +@task(check_import) +def pytest(ctx): + """Run the unit tests with py.test.""" + ctx.run('py.test --cov={} --cov-report=term-missing'.format(ctx.package)) + + +@task(pytest) +def all(ctx): + """Run all test utilities.""" + pass + + +ns = Collection(pytest) +ns.add_task(all, default=True) diff --git a/tasks/types.py b/tasks/types.py new file mode 100644 index 0000000..2d1a0ab --- /dev/null +++ b/tasks/types.py @@ -0,0 +1,21 @@ +# type: ignore + +from invoke import task, Collection + +from .check import check_import + + +@task(check_import) +def mypy(ctx): + """Run the mypy type checker.""" + ctx.run('mypy {} test'.format(ctx.package)) + + +@task(mypy) +def all(ctx): + """Run all the type checker tools.""" + pass + + +ns = Collection(mypy) +ns.add_task(all, default=True) diff --git a/test/test_health.py b/test/test_health.py index 5acecd7..5ce5bf9 100644 --- a/test/test_health.py +++ b/test/test_health.py @@ -1,22 +1,28 @@ from io import StringIO from argparse import Namespace +from typing import Any -from grpclib.testing import ChannelFor -from pymapadmin.commands.health import CheckCommand from grpclib.health.v1.health_grpc import HealthBase from grpclib.health.v1.health_pb2 import HealthCheckRequest, \ HealthCheckResponse +from grpclib.server import Stream +from grpclib.testing import ChannelFor + +from pymapadmin.commands.health import CheckCommand + +from handler import MockHandler -from handler import RequestT, ResponseT, MockHandler +_CheckStream = Stream[HealthCheckRequest, HealthCheckResponse] +_WatchStream = Stream[HealthCheckRequest, HealthCheckResponse] -class Handler(HealthBase, MockHandler[RequestT, ResponseT]): +class Handler(HealthBase, MockHandler[Any, Any]): - async def Check(self, stream) -> None: + async def Check(self, stream: _CheckStream) -> None: await self._run(stream) - async def Watch(self, stream) -> None: + async def Watch(self, stream: _WatchStream) -> None: raise NotImplementedError() diff --git a/test/test_mailbox.py b/test/test_mailbox.py index c89453a..a5cd69e 100644 --- a/test/test_mailbox.py +++ b/test/test_mailbox.py @@ -1,19 +1,24 @@ from io import BytesIO, StringIO from argparse import Namespace +from typing import Any +from grpclib.server import Stream from grpclib.testing import ChannelFor + from pymapadmin.commands.mailbox import AppendCommand from pymapadmin.grpc.admin_grpc import MailboxBase from pymapadmin.grpc.admin_pb2 import Result, FAILURE, \ AppendRequest, AppendResponse -from handler import RequestT, ResponseT, MockHandler +from handler import MockHandler + +_AppendStream = Stream[AppendRequest, AppendResponse] -class Handler(MailboxBase, MockHandler[RequestT, ResponseT]): +class Handler(MailboxBase, MockHandler[Any, Any]): - async def Append(self, stream) -> None: + async def Append(self, stream: _AppendStream) -> None: await self._run(stream) diff --git a/test/test_system.py b/test/test_system.py index 8e5e2b2..c348212 100644 --- a/test/test_system.py +++ b/test/test_system.py @@ -1,22 +1,28 @@ from io import StringIO from argparse import Namespace +from typing import Any +from grpclib.server import Stream from grpclib.testing import ChannelFor + from pymapadmin.commands.system import LoginCommand, PingCommand from pymapadmin.grpc.admin_grpc import SystemBase from pymapadmin.grpc.admin_pb2 import LoginRequest, LoginResponse, \ PingRequest, PingResponse, Result, FAILURE -from handler import RequestT, ResponseT, MockHandler +from handler import MockHandler + +_PingStream = Stream[PingRequest, PingResponse] +_LoginStream = Stream[LoginRequest, LoginResponse] -class Handler(SystemBase, MockHandler[RequestT, ResponseT]): +class Handler(SystemBase, MockHandler[Any, Any]): - async def Ping(self, stream) -> None: + async def Ping(self, stream: _PingStream) -> None: await self._run(stream) - async def Login(self, stream) -> None: + async def Login(self, stream: _LoginStream) -> None: await self._run(stream) diff --git a/test/test_user.py b/test/test_user.py index 8ca125b..45849aa 100644 --- a/test/test_user.py +++ b/test/test_user.py @@ -1,8 +1,10 @@ from io import StringIO from argparse import Namespace +from typing import Any import pytest +from grpclib.server import Stream from grpclib.testing import ChannelFor from pymapadmin.commands.user import GetUserCommand, SetUserCommand, \ @@ -11,18 +13,22 @@ from pymapadmin.grpc.admin_pb2 import \ GetUserRequest, SetUserRequest, DeleteUserRequest, UserResponse -from handler import RequestT, ResponseT, MockHandler +from handler import MockHandler +_GetUserStream = Stream[GetUserRequest, UserResponse] +_SetUserStream = Stream[SetUserRequest, UserResponse] +_DeleteUserStream = Stream[DeleteUserRequest, UserResponse] -class Handler(UserBase, MockHandler[RequestT, ResponseT]): - async def GetUser(self, stream) -> None: +class Handler(UserBase, MockHandler[Any, UserResponse]): + + async def GetUser(self, stream: _GetUserStream) -> None: await self._run(stream) - async def SetUser(self, stream) -> None: + async def SetUser(self, stream: _SetUserStream) -> None: await self._run(stream) - async def DeleteUser(self, stream) -> None: + async def DeleteUser(self, stream: _DeleteUserStream) -> None: await self._run(stream)