diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 5d0f177a..5ef8ff23 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -35,8 +35,8 @@ jobs: - id: dependencies run: | - pip install -r requirements.txt - pip install -r requirements.dev.txt -c constraints.txt + pip install . + pip install ".[dev]" - id: pylint run: make lint @@ -80,8 +80,8 @@ jobs: sudo apt-get install -y libsnappy-dev postgresql-10 postgresql-11 postgresql-12 postgresql-13 postgresql-14 postgresql-15 postgresql-16 # Setup common python dependencies python -m pip install --upgrade pip - pip install -r requirements.txt - pip install -r requirements.dev.txt -c constraints.txt + pip install . + pip install ".[constraints]" pip install -e . - id: unittest diff --git a/.gitignore b/.gitignore index bd3031a5..0ff4b571 100644 --- a/.gitignore +++ b/.gitignore @@ -21,3 +21,4 @@ __pycache__/ *.orig /pghoard-rpm-src/ coverage.xml +.hypothesis diff --git a/Makefile b/Makefile index 68d60d82..322a0d60 100644 --- a/Makefile +++ b/Makefile @@ -1,58 +1,59 @@ short_ver = $(shell git describe --abbrev=0) long_ver = $(shell git describe --long 2>/dev/null || echo $(short_ver)-0-unknown-g`git describe --always`) -generated = pghoard/version.py - -all: $(generated) PYTHON ?= python3 PYTHON_SOURCE_DIRS = pghoard/ test/ PYTEST_ARG ?= -v +.PHONY: pghoard/version.py +python-build: + $(PYTHON) -m build + +.PHONY: dev-deps +dev-deps: + pip install . + pip install ".[dev]" + .PHONY: unittest -unittest: version +unittest: dev-deps $(PYTHON) -m pytest -vv test/ .PHONY: lint -lint: version +lint: dev-deps $(PYTHON) -m pylint --rcfile .pylintrc $(PYTHON_SOURCE_DIRS) .PHONY: mypy -mypy: version +mypy: dev-deps $(PYTHON) -m mypy $(PYTHON_SOURCE_DIRS) .PHONY: fmt -fmt: version +fmt: dev-deps unify --quote '"' --recursive --in-place $(PYTHON_SOURCE_DIRS) isort $(PYTHON_SOURCE_DIRS) yapf --parallel --recursive --in-place $(PYTHON_SOURCE_DIRS) .PHONY: coverage -coverage: version +coverage: dev-deps $(PYTHON) -m pytest $(PYTEST_ARG) --cov-report term-missing --cov-report xml:coverage.xml \ --cov pghoard test/ .PHONY: clean clean: $(RM) -r *.egg-info/ build/ dist/ rpm/ - $(RM) ../pghoard_* test-*.xml coverage.xml $(generated) - -pghoard/version.py: version.py - $(PYTHON) $^ $@ + $(RM) ../pghoard_* test-*.xml coverage.xml pghoard/version.py -.PHONY: version -version: pghoard/version.py .PHONY: deb -deb: $(generated) +deb: cp debian/changelog.in debian/changelog dch -v $(long_ver) --distribution unstable "Automatically built .deb" dpkg-buildpackage -A -uc -us .PHONY: rpm -rpm: $(generated) +rpm: python-build git archive --output=pghoard-rpm-src.tar --prefix=pghoard/ HEAD # add generated files to the tar, they're not in git repository - tar -r -f pghoard-rpm-src.tar --transform=s,pghoard/,pghoard/pghoard/, $(generated) + tar -r -f pghoard-rpm-src.tar --transform=s,pghoard/,pghoard/pghoard/, pghoard/version.py rpmbuild -bb pghoard.spec \ --define '_topdir $(PWD)/rpm' \ --define '_sourcedir $(CURDIR)' \ diff --git a/constraints.txt b/constraints.txt deleted file mode 100644 index 13e4400b..00000000 --- a/constraints.txt +++ /dev/null @@ -1,85 +0,0 @@ -astroid==2.5.8 -attrs==22.2.0 -azure-core==1.26.3 -azure-storage-blob==12.15.0 -bcrypt==4.0.1 -boto3==1.26.96 -botocore==1.29.96 -botocore-stubs==1.29.96 -cachetools==5.3.0 -certifi==2022.12.7 -cffi==1.15.1 -charset-normalizer==3.1.0 -coverage==6.5.0 -coveralls==3.3.1 -cryptography==39.0.2 -docopt==0.6.2 -exceptiongroup==1.1.1 -execnet==1.9.0 -freezegun==1.2.2 -google-api-core==2.11.0 -google-api-python-client==2.82.0 -googleapis-common-protos==1.59.0 -google-auth==2.16.2 -google-auth-httplib2==0.1.0 -httplib2==0.21.0 -idna==3.4 -iniconfig==2.0.0 -isodate==0.6.1 -isort==5.7.0 -jmespath==1.0.1 -lazy-object-proxy==1.9.0 -mccabe==0.6.1 -mock==5.0.1 -mypy==1.1.1 -mypy-extensions==1.0.0 -oauth2client==4.1.3 -packaging==23.0 -paramiko==3.1.0 -pluggy==1.0.0 -protobuf==4.22.1 -psycopg2==2.9.5 -pyasn1==0.4.8 -pyasn1-modules==0.2.8 -pycparser==2.21 -pydantic==1.10.6 -pylint==2.7.2 -pylint-quotes==0.2.1 -PyNaCl==1.5.0 -pyparsing==3.0.9 -pytest==7.2.2 -pytest-cov==4.0.0 -pytest-mock==3.10.0 -pytest-timeout==2.1.0 -pytest-xdist==3.2.1 -python-dateutil==2.8.2 -python-snappy==0.6.1 -python-systemd==0.0.9 -PyYAML==6.0 -requests==2.28.2 -responses==0.23.1 -rohmu==1.0.10 -rsa==4.9 -s3transfer==0.6.0 -six==1.16.0 -toml==0.10.2 -tomli==2.0.1 -types-awscrt==0.16.13 -types-botocore==1.0.2 -types-httplib2==0.21.0.5 -types-mock==5.0.0.5 -types-paramiko==3.0.0.4 -types-psycopg2==2.9.21.8 -types-python-dateutil==2.8.19.10 -types-PyYAML==6.0.12.8 -types-requests==2.28.11.15 -types-six==1.16.21.7 -types-urllib3==1.26.25.8 -typing_extensions==4.7.1 -unify==0.5 -untokenize==0.1.1 -uritemplate==4.1.1 -urllib3==1.26.15 -wrapt==1.12.1 -yapf==0.30.0 -zstandard==0.20.0 diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 00000000..6bf84298 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,174 @@ +[build-system] +requires = ["hatchling", "hatch-vcs"] +build-backend = "hatchling.build" + +[project] +name = "pghoard" +authors = [ + { name="Aiven", email="opensource@aiven.io" }, + { name="Hannu Valtonen", email="hannu.valtonen@ohmu.fi" } +] +description = "PostgreSQL automatic backup/restore service daemon." +readme = "README.rst" +requires-python = ">=3.8" +classifiers=[ + "Development Status :: 5 - Production/Stable", + "Intended Audience :: Developers", + "Intended Audience :: Information Technology", + "Intended Audience :: System Administrators", + "License :: OSI Approved :: Apache Software License", + "Programming Language :: Python :: 3.8", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Topic :: Database :: Database Engines/Servers", + "Topic :: Software Development :: Libraries", +] +license = { text = "Apache License 2.0" } +dynamic = ["version"] +dependencies = [ + "cryptography", + "psycopg2-binary >= 2.8.0", + "pydantic", + "python-dateutil", + "python-snappy >= 0.5", + "python-systemd", + "requests >= 1.2.0", + "rohmu >= 1.0.7", + "zstandard >= 0.11.1", +] + +[project.optional-dependencies] +dev = [ + "boto3", + "mock", + "mypy", + "pylint", + "pylint-quotes", + "pytest", + "pytest-cov", + "pytest-mock", + "pytest-timeout", + "pytest-xdist", + "yapf==0.30.0", + "isort==5.7.0", + "coverage", + "coveralls", + "freezegun>=1.2", + "responses", + "unify", + "types-botocore", + "types-httplib2", + "types-mock", + "types-paramiko", + "types-psycopg2", + "types-python-dateutil", + "types-requests", + "types-six", +] +contraints = [ + "astroid==2.5.8", + "attrs==22.2.0", + "azure-core==1.26.3", + "azure-storage-blob==12.15.0", + "bcrypt==4.0.1", + "boto3==1.26.96", + "botocore==1.29.96", + "botocore-stubs==1.29.96", + "cachetools==5.3.0", + "certifi==2022.12.7", + "cffi==1.15.1", + "charset-normalizer==3.1.0", + "coverage==6.5.0", + "coveralls==3.3.1", + "cryptography==39.0.2", + "docopt==0.6.2", + "exceptiongroup==1.1.1", + "execnet==1.9.0", + "freezegun==1.2.2", + "google-api-core==2.11.0", + "google-api-python-client==2.82.0", + "googleapis-common-protos==1.59.0", + "google-auth==2.16.2", + "google-auth-httplib2==0.1.0", + "httplib2==0.21.0", + "idna==3.4", + "iniconfig==2.0.0", + "isodate==0.6.1", + "isort==5.7.0", + "jmespath==1.0.1", + "lazy-object-proxy==1.9.0", + "mccabe==0.6.1", + "mock==5.0.1", + "mypy==1.1.1", + "mypy-extensions==1.0.0", + "oauth2client==4.1.3", + "packaging==23.0", + "paramiko==3.1.0", + "pluggy==1.0.0", + "protobuf==4.22.1", + "psycopg2==2.9.5", + "pyasn1==0.4.8", + "pyasn1-modules==0.2.8", + "pycparser==2.21", + "pydantic==1.10.6", + "pylint==2.7.2", + "pylint-quotes==0.2.1", + "PyNaCl==1.5.0", + "pyparsing==3.0.9", + "pytest==7.2.2", + "pytest-cov==4.0.0", + "pytest-mock==3.10.0", + "pytest-timeout==2.1.0", + "pytest-xdist==3.2.1", + "python-dateutil==2.8.2", + "python-snappy==0.6.1", + "python-systemd==0.0.9", + "PyYAML==6.0", + "requests==2.28.2", + "responses==0.23.1", + "rohmu==1.0.10", + "rsa==4.9", + "s3transfer==0.6.0", + "six==1.16.0", + "toml==0.10.2", + "tomli==2.0.1", + "types-awscrt==0.16.13", + "types-botocore==1.0.2", + "types-httplib2==0.21.0.5", + "types-mock==5.0.0.5", + "types-paramiko==3.0.0.4", + "types-psycopg2==2.9.21.8", + "types-python-dateutil==2.8.19.10", + "types-PyYAML==6.0.12.8", + "types-requests==2.28.11.15", + "types-six==1.16.21.7", + "types-urllib3==1.26.25.8", + "typing_extensions==4.7.1", + "unify==0.5", + "untokenize==0.1.1", + "uritemplate==4.1.1", + "urllib3==1.26.15", + "wrapt==1.12.1", + "yapf==0.30.0", + "zstandard==0.20.0", +] + +[project.urls] +"Homepage" = "https://github.com/Aiven-Open/pghoard/" +"Bug Tracker" = "https://github.com/Aiven-Open/pghoard/issues" + +[project.scripts] +pghoard = "pghoard.pghoard:main" +pghoard_archive_cleanup = "pghoard.archive_cleanup:main" +pghoard_archive_sync = "pghoard.archive_sync:main" +pghoard_create_keys = "pghoard.create_keys:main" +pghoard_gnutaremu = "pghoard.gnutaremu:main" +pghoard_postgres_command = "pghoard.postgres_command:main" +pghoard_restore = "pghoard.restore:main" + +[tool.hatch.version] +source = "vcs" + +[tool.hatch.build.hooks.vcs] +version-file = "pghoard/version.py" diff --git a/requirements.dev.txt b/requirements.dev.txt deleted file mode 100644 index 659afdb9..00000000 --- a/requirements.dev.txt +++ /dev/null @@ -1,26 +0,0 @@ -# Use pip for build requirements to harmonize between OS versions -boto3 -mock -mypy -pylint>=2.4.3,<=2.7.2 -pylint-quotes -pytest -pytest-cov -pytest-mock -pytest-timeout -pytest-xdist -yapf==0.30.0 -isort==5.7.0 -coverage -coveralls -freezegun>=1.2 -responses -unify -types-botocore -types-httplib2 -types-mock -types-paramiko -types-psycopg2 -types-python-dateutil -types-requests -types-six diff --git a/requirements.txt b/requirements.txt deleted file mode 100644 index 6544ec8c..00000000 --- a/requirements.txt +++ /dev/null @@ -1,5 +0,0 @@ -psycopg2 -pydantic -python-systemd -requests -rohmu >= 1.0.7 diff --git a/setup.py b/setup.py deleted file mode 100644 index 6aa38506..00000000 --- a/setup.py +++ /dev/null @@ -1,63 +0,0 @@ -from setuptools import setup, find_packages -import os -import version - - -readme_path = os.path.join(os.path.dirname(__file__), "README.rst") -with open(readme_path, "r") as fp: - readme_text = fp.read() - - -version_for_setup_py = version.get_project_version("pghoard/version.py") -version_for_setup_py = ".dev".join(version_for_setup_py.split("-", 2)[:2]) - - -setup( - name="pghoard", - version=version_for_setup_py, - zip_safe=False, - packages=find_packages(exclude=["test"]), - install_requires=[ - "cryptography", - "psycopg2 >= 2.8.0", - "pydantic", - "python-dateutil", - "python-snappy >= 0.5", - "requests >= 1.2.0", - "zstandard >= 0.11.1", - ], - extras_require={}, - dependency_links=[], - package_data={}, - entry_points={ - "console_scripts": [ - "pghoard = pghoard.pghoard:main", - "pghoard_archive_cleanup = pghoard.archive_cleanup:main", - "pghoard_archive_sync = pghoard.archive_sync:main", - "pghoard_create_keys = pghoard.create_keys:main", - "pghoard_gnutaremu = pghoard.gnutaremu:main", - "pghoard_postgres_command = pghoard.postgres_command:main", - "pghoard_restore = pghoard.restore:main", - ], - }, - author="Hannu Valtonen", - author_email="hannu.valtonen@ohmu.fi", - license="Apache 2.0", - platforms=["POSIX", "MacOS"], - description="PostgreSQL automatic backup/restore service daemon", - long_description=readme_text, - url="https://github.com/aiven/pghoard/", - classifiers=[ - "Development Status :: 5 - Production/Stable", - "Intended Audience :: Developers", - "Intended Audience :: Information Technology", - "Intended Audience :: System Administrators", - "License :: OSI Approved :: Apache Software License", - "Programming Language :: Python :: 3.8", - "Programming Language :: Python :: 3.9", - "Programming Language :: Python :: 3.10", - "Programming Language :: Python :: 3.11", - "Topic :: Database :: Database Engines/Servers", - "Topic :: Software Development :: Libraries", - ], -) diff --git a/update-constraints b/update-constraints index 2f78a98c..c3d56330 100755 --- a/update-constraints +++ b/update-constraints @@ -1,7 +1,7 @@ #!/bin/bash -ex TEMPDIR="$(mktemp -d)" python -m venv "$TEMPDIR" -"$TEMPDIR"/bin/pip install -r requirements.txt -"$TEMPDIR"/bin/pip install -r requirements.dev.txt +"$TEMPDIR"/bin/pip install . +"$TEMPDIR"/bin/pip install ".[dev]" "$TEMPDIR"/bin/pip freeze | grep -v "pkg-resources" | sort > constraints.txt rm -rf "$TEMPDIR" diff --git a/version.py b/version.py deleted file mode 100644 index a971cedc..00000000 --- a/version.py +++ /dev/null @@ -1,60 +0,0 @@ -# Copied from https://github.com/ohmu/ohmu_common_py version.py version 0.0.1-0-unknown-fa54b44 -""" -pghoard - version detection and version.py __version__ generation - -Copyright (c) 2015 Ohmu Ltd -See LICENSE for details -""" - -import imp -import os -import subprocess - - -def save_version(new_ver, old_ver, version_file): - if not new_ver: - return False - version_file = os.path.join(os.path.dirname(__file__), version_file) - if not old_ver or new_ver != old_ver: - with open(version_file, "w") as fp: - fp.write("__version__ = '{}'\n".format(new_ver)) - return True - - -def get_project_version(version_file): - version_file = os.path.join(os.path.dirname(os.path.realpath(__file__)), version_file) - try: - module = imp.load_source("verfile", version_file) - file_ver = module.__version__ - except IOError: - file_ver = None - - os.chdir(os.path.dirname(__file__) or ".") - try: - git_out = subprocess.check_output(["git", "describe", "--always"], - stderr=getattr(subprocess, "DEVNULL", None)) - except (OSError, subprocess.CalledProcessError): - pass - else: - git_ver = git_out.splitlines()[0].strip().decode("utf-8") - if "." not in git_ver: - git_ver = "0.0.1-0-unknown-{}".format(git_ver) - if save_version(git_ver, file_ver, version_file): - return git_ver - - makefile = os.path.join(os.path.dirname(__file__), "Makefile") - if os.path.exists(makefile): - with open(makefile, "r") as fp: - lines = fp.readlines() - short_ver = [line.split("=", 1)[1].strip() for line in lines if line.startswith("short_ver")][0] - if save_version(short_ver, file_ver, version_file): - return short_ver - - if not file_ver: - raise Exception("version not available from git or from file {!r}".format(version_file)) - - return file_ver - -if __name__ == "__main__": - import sys - get_project_version(sys.argv[1])