From 9cabf260752d0f92b2c63b85a76b61bfa3a90918 Mon Sep 17 00:00:00 2001 From: Henning Kage Date: Thu, 29 Feb 2024 08:37:40 +0100 Subject: [PATCH 001/109] Added missing RTD configuration --- readthedocs.yaml | 15 +++++++++++++++ 1 file changed, 15 insertions(+) create mode 100644 readthedocs.yaml diff --git a/readthedocs.yaml b/readthedocs.yaml new file mode 100644 index 0000000..3ad832b --- /dev/null +++ b/readthedocs.yaml @@ -0,0 +1,15 @@ +version: 2 + +sphinx: + configuration: docs/conf.py + +formats: all + +build: + os: "ubuntu-latest" + tools: + python: "3.10" + +python: + install: + - requirements: requirements.txt From c696c3a1d20c6f7f3efafb21746e7deb0ee27be4 Mon Sep 17 00:00:00 2001 From: Henning Kage Date: Thu, 29 Feb 2024 08:38:37 +0100 Subject: [PATCH 002/109] Updated the RTD configuration --- readthedocs.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/readthedocs.yaml b/readthedocs.yaml index 3ad832b..8832cf3 100644 --- a/readthedocs.yaml +++ b/readthedocs.yaml @@ -6,7 +6,7 @@ sphinx: formats: all build: - os: "ubuntu-latest" + os: "ubuntu-20.04" tools: python: "3.10" From 35bf354989a1fa4ff610ca926efdea50903cb464 Mon Sep 17 00:00:00 2001 From: Henning Kage Date: Thu, 29 Feb 2024 08:41:14 +0100 Subject: [PATCH 003/109] Updated the RTD configuration --- readthedocs.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/readthedocs.yaml b/readthedocs.yaml index 8832cf3..8001c37 100644 --- a/readthedocs.yaml +++ b/readthedocs.yaml @@ -8,7 +8,7 @@ formats: all build: os: "ubuntu-20.04" tools: - python: "3.10" + python: "3.8" python: install: From 3d4ed71fbe5f9d6f8a9110c080e0e8c55463964e Mon Sep 17 00:00:00 2001 From: bobslee Date: Thu, 29 Feb 2024 09:10:40 +0100 Subject: [PATCH 004/109] Update the license (copyright) period. --- LICENSE.rst | 2 +- docs/conf.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/LICENSE.rst b/LICENSE.rst index ad18bfc..a37e625 100644 --- a/LICENSE.rst +++ b/LICENSE.rst @@ -3,7 +3,7 @@ License The MIT License -Copyright (c) 2019-2021, Rheinwerk Verlag GmbH +Copyright (c) 2019-2024, Rheinwerk Verlag GmbH Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/docs/conf.py b/docs/conf.py index cf7f31d..ef62ba3 100755 --- a/docs/conf.py +++ b/docs/conf.py @@ -58,7 +58,7 @@ # General information about the project. project = u'pganonymize' -copyright = u'2019, Rheinwerk Verlag GmbH, Henning Kage' +copyright = u'2019-2024, Rheinwerk Verlag GmbH, Henning Kage' # The version info for the project you're documenting, acts as replacement # for |version| and |release|, also used in various other places throughout From 9226deabe12a17431ada3faae38b64867706637b Mon Sep 17 00:00:00 2001 From: Henning Kage Date: Tue, 12 Mar 2024 15:31:45 +0100 Subject: [PATCH 005/109] #53: Changed CHANGELOG and CONTRIBUTING to markdown --- CHANGELOG.md | 110 +++++++++++++++++++++++++++++++++++++ CHANGELOG.rst | 133 --------------------------------------------- CONTRIBUTING.md | 42 ++++++++++++++ CONTRIBUTING.rst | 52 ------------------ docs/changelog.md | 1 + docs/changelog.rst | 1 - docs/conf.py | 4 +- 7 files changed, 155 insertions(+), 188 deletions(-) create mode 100644 CHANGELOG.md delete mode 100644 CHANGELOG.rst create mode 100644 CONTRIBUTING.md delete mode 100644 CONTRIBUTING.rst create mode 120000 docs/changelog.md delete mode 100644 docs/changelog.rst diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..ef11e66 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,110 @@ +# Changelog + +## Development + +## 0.11.0 (2024-02-29) + +* [#52 Add update_json provider](https://github.com/rheinwerk-verlag/pganonymize/pull/52) ([bobslee](https://github.com/bobslee)) + +## 0.10.0 (2022-11-29) + +* [#49 Configure psycopg2 to support UUID objects](https://github.com/rheinwerk-verlag/pganonymize/pull/49) +* [#48 Add support for localized "Faker" data](https://github.com/rheinwerk-verlag/pganonymize/pull/48) + +## 0.9.0 (2022-11-23) + +* [#46 Broken Python 2.7 compatibility](https://github.com/rheinwerk-verlag/pganonymize/pull/46) +* [#45 Add partial masked provider](https://github.com/rheinwerk-verlag/pganonymize/pull/45) ([Tilley](https://github.com/Tilley/)) +* [#44 Pass kwargs through to faker functions from schema](https://github.com/rheinwerk-verlag/pganonymize/pull/44)([Tilley](https://github.com/Tilley>)) + +## 0.8.0 (2022-03-15) + +* [#39 Renamed project to "pganonymize"](https://github.com/rheinwerk-verlag/pganonymize/issues/39) +* [#38 Allow environment variables in schema definition](https://github.com/rheinwerk-verlag/pganonymize/pull/38) ([nurikk](https://github.com/nurikk)) + +## 0.7.0 (2021-11-30) + +* [#34 Subprocess "run" being used on Python2.7](https://github.com/rheinwerk-verlag/pganonymize/issues/34) +* [#35 parmap no longer supports Python 2.7](https://github.com/rheinwerk-verlag/pganonymize/issues/35) + * Dropped Python 3.5 support + * Pinned libraries Python 2.7 +* [#32 Fixed pg_dump arguments](https://github.com/rheinwerk-verlag/pganonymize/pull/32) ([korsar182](https://github.com/korsar182)) +* Simplified provider registration (no metaclass usage anymore) + +## 0.6.1 (2021-07-13) + +* Added missing dependencies for the `setup.py` + +## 0.6.0 (2021-07-13) + +* [#28 Add json support](https://github.com/rheinwerk-verlag/pganonymize/pull/25) ([nurikk](https://github.com/nurikk)) +* [#27 Better anonymisation](https://github.com/rheinwerk-verlag/pganonymize/pull/25) ([nurikk](https://github.com/nurikk)) +* [#25 Remove column specification for `cursor.copy_from` call](https://github.com/rheinwerk-verlag/pganonymize/pull/25) ([nurikk](https://github.com/nurikk)) + +## 0.5.0 (2021-06-30) + +* [#22 Fix table and column name quotes in `cursor.copy_from` call](https://github.com/rheinwerk-verlag/pganonymize/pull/22) ([nurikk](https://github.com/nurikk)) +* [#23 Allow uniq faker](https://github.com/rheinwerk-verlag/pganonymize/pull/23) ([nurikk](https://github.com/nurikk)) + +## 0.4.1 (2021-05-27) + +* [#19 Make chunk size in the table definition dynamic](https://github.com/rheinwerk-verlag/pganonymize/pull/19) ([halilkaya](https://github.com/halilkaya)) + +## 0.4.0 (2021-05-05) + +* [#18 Specify (SQL WHERE) search_condition, to filter the table for rows to be anonymized](https://github.com/rheinwerk-verlag/pganonymize/pull/18) (`bobslee `_) +* [#17 Fix anonymizing error if there is a JSONB column in a table](https://github.com/rheinwerk-verlag/pganonymize/pull/17) ([koptelovav](https://github.com/koptelovav)) + +## 0.3.3 (2021-04-16) + +* [#16 Preserve column and table cases during the copy process](https://github.com/rheinwerk-verlag/pganonymize/issues/16) + +## 0.3.2 (2021-01-25) + +* [#15 Fix for exclude bug](https://github.com/rheinwerk-verlag/pganonymize/pull/15) ([abhinavvaidya90](https://github.com/abhinavvaidya90)) + +## 0.3.1 (2020-12-04) + +* [#13 Fixed a syntax error if no truncated tables are defined](https://github.com/rheinwerk-verlag/pganonymize/pull/13) ([ray-man](https://github.com/ray-man)) + +## 0.3.0 (2020-02-11) + +* Use [`python-poetry`](https://github.com/python-poetry/poetry) for requirements management +* Added commandline argument to list all available providers (#4) +* Added commandline argument to create a dump file (#5) +* Execute table truncation in one statement to avoid foreign key constraint errors (thanks to [W1ldPo1nter](https://github.com/W1ldPo1nter)) + +## 0.2.4 (2020-01-03) + +* Fixed several issues with the usage of ``dict.keys`` and Python 3 + +## 0.2.3 (2020-01-02) + +* Fixed the wrong cStringIO import for Python 3 +* Removed Travis-CI file in favor of the Github actions + +## 0.2.2 (2020-01-02) + +* Hide the progressbar completely if verbose is set to ``False`` +* Restructured the requirement files and added flake8 to Travis CI + +## 0.2.1 (2019-12-20) + +* Added field based, regular expression excludes (to skip data under certain conditions). + Currently only regular expressions are supported and the exclusion affects the whole row, + not just one single column. + +## 0.2.0 (2019-12-20) + +* Added provider classes +* Added new providers: + * choice - returns a random list element + * mask - replaces the original value with a static sign + +## 0.1.1 (2019-12-18) + +Changed setup.py + +## 0.1.0 (2019-12-16) + +Initial release of the prototype diff --git a/CHANGELOG.rst b/CHANGELOG.rst deleted file mode 100644 index 7f73dc5..0000000 --- a/CHANGELOG.rst +++ /dev/null @@ -1,133 +0,0 @@ -Changelog -========= - -Development ------------ - -0.11.0 (2024-02-29) -------------------- - -* `#52 `_: Add update_json provider (`bobslee `_) - -0.10.0 (2022-11-29) -------------------- - -* `#49 `_: Configure psycopg2 to support UUID objects -* `#48 `_: Add support for localized "Faker" data - -0.9.0 (2022-11-23) ------------------- - -* `#46 `_: Broken Python 2.7 compatibility -* `#45 `_: Add partial masked provider (`Tilley `_) -* `#44 `_: Pass kwargs through to faker functions from schema (`Tilley `_) - -0.8.0 (2022-03-15) ------------------- - -* `#39 `_: Renamed project to "pganonymize" -* `#38 `_: Allow environment variables in schema definition (`nurikk `_) - -0.7.0 (2021-11-30) ------------------- - -* `#34 `_: Subprocess "run" being used on Python2.7 -* `#35 `_: parmap no longer supports Python 2.7 - * Dropped Python 3.5 support - * Pinned libraries Python 2.7 -* `#32 `_: Fixed pg_dump arguments (`korsar182 `_) -* Simplified provider registration (no metaclass usage anymore) - -0.6.1 (2021-07-13) ------------------- - -* Added missing dependencies for the `setup.py` - -0.6.0 (2021-07-13) ------------------- - -* `#28 `_: Add json support (`nurikk `_) -* `#27 `_: Better anonymisation (`nurikk `_) -* `#25 `_: Remove column specification for `cursor.copy_from` call (`nurikk `_) - -0.5.0 (2021-06-30) ------------------- - -* `#22 `_: Fix table and column name quotes in `cursor.copy_from` call (`nurikk `_) -* `#23 `_: Allow uniq faker (`nurikk `_) - -0.4.1 (2021-05-27) ------------------- - -* `#19 `_: Make chunk size in the table definition dynamic (`halilkaya `_) - -0.4.0 (2021-05-05) ------------------- - -* `#18 `_: Specify (SQL WHERE) search_condition, to filter the table for rows to be anonymized (`bobslee `_) -* `#17 `_: Fix anonymizing error if there is a JSONB column in a table (`koptelovav `_) - -0.3.3 (2021-04-16) ------------------- - -* `#16 `_: Preserve column and table cases during the copy process - -0.3.2 (2021-01-25) ------------------- - -* `#15 `_: Fix for exclude bug (`abhinavvaidya90 `_) - -0.3.1 (2020-12-04) ------------------- - -* `#13 `_: Fixed a syntax error if no truncated tables are defined (`ray-man `_) - -0.3.0 (2020-02-11) ------------------- - -* Use `python-poetry `_ for requirements management -* Added commandline argument to list all available providers (#4) -* Added commandline argument to create a dump file (#5) -* Execute table truncation in one statement to avoid foreign key constraint errors (thanks to `W1ldPo1nter `_) - -0.2.4 (2020-01-03) ------------------- - -* Fixed several issues with the usage of ``dict.keys`` and Python 3 - -0.2.3 (2020-01-02) ------------------- - -* Fixed the wrong cStringIO import for Python 3 -* Removed Travis-CI file in favor of the Github actions - -0.2.2 (2020-01-02) ------------------- - -* Hide the progressbar completely if verbose is set to ``False`` -* Restructured the requirement files and added flake8 to Travis CI - -0.2.1 (2019-12-20) ------------------- - -* Added field based, regular expression excludes (to skip data under certain conditions). - Currently only regular expressions are supported and the exclusion affects the whole row, - not just one single column. - -0.2.0 (2019-12-20) ------------------- - -* Added provider classes -* Added new providers: - * choice - returns a random list element - * mask - replaces the original value with a static sign - -0.1.1 (2019-12-18) ------------------- - -Changed setup.py - -0.1.0 (2019-12-16) ------------------- - -Initial release of the prototype diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000..5dce426 --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,42 @@ +# Contributing to PostgreSQL Anonymizer + +First of all: thanks for your interest in this project and taking the time to contribute. + +The following document is a small set of guidelines for contributing to this project. They are guidelines and no rules. + +## Reporting bugs + +If you have found a bug, please check the project's +[issue](https://github.com/rheinwerk-verlag/postgresql-anonymizer/issues) page first and feel free to create a +[new issue](https://github.com/rheinwerk-verlag/postgresql-anonymizer/issues/new), if no one else has reported it yet. + +## Making changes + +Create a fork if you want to make changes or clone the repo if you want a readonly access to the current development +version: + +```bash +$ git clone git@github.com:rheinwerk-verlag/postgresql-anonymizer.git +$ cd postgresql-anonymizer +``` + +For the development use a virtualenv or install the requirements directly: + +```bash +$ sudo pip install -r requirements.txt +``` + +## Coding style + +We have created an [EditorConfig](https://editorconfig.org/) file for this project that should be usable for most IDEs. +Otherwise please make sure to adhere to the specifications from the config file. + +## Creating a pull request + +Before creating a pull request make sure to check: + +* existing docstrings have been updated +* new code has valid docstrings +* whether existing [tests](https://github.com/rheinwerk-verlag/postgresql-anonymizer/tree/development/tests) have to be fixed +* new tests have to be written first +* the documentation (in particular the [Sphinx documentation](https://github.com/rheinwerk-verlag/postgresql-anonymizer/tree/development/docs)) has to be modified diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst deleted file mode 100644 index ed67cb9..0000000 --- a/CONTRIBUTING.rst +++ /dev/null @@ -1,52 +0,0 @@ -Contributing to PostgreSQL Anonymizer -===================================== - -First of all: thanks for your interest in this project and taking the time to contribute. - -The following document is a small set of guidelines for contributing to this project. They are guidelines and no rules. - -Reporting bugs ---------------- - -If you have found a bug, please check the project's `issue`_ page first and feel free to create a `new issue`_, if no -one else has reported it yet. - -Making changes --------------- - -Create a fork if you want to make changes or clone the repo if you want a readonly access to the current development -version: - -.. code-block:: bash - - $ git clone git@github.com:rheinwerk-verlag/postgresql-anonymizer.git - $ cd postgresql-anonymizer - -For the development use a virtualenv or install the requirements directly: - -.. code-block:: bash - - $ sudo pip install -r requirements.txt - -Coding style ------------- - -We have created an `EditorConfig`_ file for this project that should be usable for most IDEs. Otherwise please make -sure to adhere to the specifications from the config file. - -Creating a pull request ------------------------ - -Before creating a pull request make sure to check: - -* existing docstrings have been updated -* new code has valid docstrings -* whether existing `tests`_ have to be fixed -* new tests have to be written first -* the documentation (in particular the `Sphinx documentation`_) has to be modified - -.. _issue: https://github.com/rheinwerk-verlag/postgresql-anonymizer/issues -.. _new issue: https://github.com/rheinwerk-verlag/postgresql-anonymizer/issues/new -.. _EditorConfig: https://editorconfig.org/ -.. _tests: https://github.com/rheinwerk-verlag/postgresql-anonymizer/tree/development/tests -.. _Sphinx documentation: https://github.com/rheinwerk-verlag/postgresql-anonymizer/tree/development/docs diff --git a/docs/changelog.md b/docs/changelog.md new file mode 120000 index 0000000..04c99a5 --- /dev/null +++ b/docs/changelog.md @@ -0,0 +1 @@ +../CHANGELOG.md \ No newline at end of file diff --git a/docs/changelog.rst b/docs/changelog.rst deleted file mode 100644 index 565b052..0000000 --- a/docs/changelog.rst +++ /dev/null @@ -1 +0,0 @@ -.. include:: ../CHANGELOG.rst diff --git a/docs/conf.py b/docs/conf.py index ef62ba3..4e1da00 100755 --- a/docs/conf.py +++ b/docs/conf.py @@ -40,7 +40,7 @@ # Add any Sphinx extension module names here, as strings. They can be # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom ones. -extensions = ['sphinx.ext.autodoc', 'sphinx.ext.intersphinx', 'sphinx.ext.viewcode'] +extensions = ['sphinx.ext.autodoc', 'sphinx.ext.intersphinx', 'sphinx.ext.viewcode', 'myst_parser'] # Add any paths that contain templates here, relative to this directory. templates_path = ['_templates'] @@ -48,7 +48,7 @@ #source_parsers = {} # The suffix of source filenames. -source_suffix = ['.rst'] +source_suffix = ['.rst', '.md'] # The encoding of source files. #source_encoding = 'utf-8-sig' From 2e5950804ac96540ae9f71d396cbb4f4851b3b2c Mon Sep 17 00:00:00 2001 From: Henning Kage Date: Tue, 12 Mar 2024 15:32:11 +0100 Subject: [PATCH 006/109] #53: Added splitted Python 2 and 3 requirements --- requirements.txt | 11 +- requirements/Makefile | 24 ++++ requirements/dev-py2.in | 9 ++ requirements/dev-py2.txt | 108 +++++++++++++++ requirements/dev-py3.in | 10 ++ requirements/dev-py3.txt | 124 ++++++++++++++++++ .../tox-py2.in | 0 requirements/tox-py2.txt | 36 +++++ requirements/tox-py3.in | 5 + requirements/tox-py3.txt | 38 ++++++ setup.py | 12 +- tox.ini | 24 +--- 12 files changed, 367 insertions(+), 34 deletions(-) create mode 100644 requirements/Makefile create mode 100644 requirements/dev-py2.in create mode 100644 requirements/dev-py2.txt create mode 100644 requirements/dev-py3.in create mode 100644 requirements/dev-py3.txt rename requirements-tox.txt => requirements/tox-py2.in (100%) create mode 100644 requirements/tox-py2.txt create mode 100644 requirements/tox-py3.in create mode 100644 requirements/tox-py3.txt diff --git a/requirements.txt b/requirements.txt index 3bd0569..f9ab379 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,8 +1,3 @@ --r requirements-tox.txt -flake8==3.9.2 -isort==5.10.1 -pgcopy>=1.5.0 -pytest==6.2.5 -Sphinx==4.3.0 -sphinx-rtd-theme==1.0.0 -tqdm>=4.61.1 +# This file is here because many Platforms as a Service look for +# requirements.txt in the root directory of a project. +-r requirements/dev-py3.txt diff --git a/requirements/Makefile b/requirements/Makefile new file mode 100644 index 0000000..de6c20e --- /dev/null +++ b/requirements/Makefile @@ -0,0 +1,24 @@ +REQUIREMENTS_SOURCES := $(wildcard *.in) +REQUIREMENTS := $(REQUIREMENTS_SOURCES:.in=.txt) +PIP_INDEX_URL ?= https://pypi.python.org/simple + +all: $(REQUIREMENTS) + +dev-py2.txt: tox-py2.txt +dev-py3.txt: tox-py3.txt + +%.txt: %.in + pip-compile \ + --verbose \ + --rebuild \ + --index-url=$(PIP_INDEX_URL) \ + --output-file=$@ $< + sed -i -e 's|^--|#--|' $@ + +check: + @which pip-compile > /dev/null + +clean: check + rm -f *.txt + +.PHONY: all check clean diff --git a/requirements/dev-py2.in b/requirements/dev-py2.in new file mode 100644 index 0000000..0713cf5 --- /dev/null +++ b/requirements/dev-py2.in @@ -0,0 +1,9 @@ +-r tox-py2.in + +flake8==3.9.2 +isort==5.10.1 +pgcopy>=1.5.0 +pytest==6.2.5 +Sphinx==4.3.0 +sphinx-rtd-theme==1.0.0 +tqdm>=4.61.1 diff --git a/requirements/dev-py2.txt b/requirements/dev-py2.txt new file mode 100644 index 0000000..2de3f07 --- /dev/null +++ b/requirements/dev-py2.txt @@ -0,0 +1,108 @@ +# +# This file is autogenerated by pip-compile with Python 3.10 +# by the following command: +# +# pip-compile --index-url=https://pypi.python.org/simple --output-file=dev-py2.txt dev-py2.in +# +#--index-url https://pypi.python.org/simple + +alabaster==0.7.16 + # via sphinx +attrs==23.2.0 + # via pytest +babel==2.14.0 + # via sphinx +certifi==2024.2.2 + # via requests +charset-normalizer==3.3.2 + # via requests +coverage==4.5.4 + # via + # -r tox-py2.in + # pytest-cov +docutils==0.17.1 + # via + # sphinx + # sphinx-rtd-theme +flake8==3.9.2 + # via -r dev-py2.in +idna==3.6 + # via requests +imagesize==1.4.1 + # via sphinx +iniconfig==2.0.0 + # via pytest +isort==5.10.1 + # via -r dev-py2.in +jinja2==3.1.3 + # via sphinx +markupsafe==2.1.5 + # via jinja2 +mccabe==0.6.1 + # via flake8 +mock==3.0.5 + # via -r tox-py2.in +packaging==24.0 + # via + # pytest + # sphinx +pgcopy==1.5.0 + # via -r dev-py2.in +pluggy==1.4.0 + # via pytest +psycopg2==2.9.9 + # via pgcopy +py==1.11.0 + # via pytest +pycodestyle==2.7.0 + # via flake8 +pyflakes==2.3.1 + # via flake8 +pygments==2.17.2 + # via sphinx +pytest==6.2.5 + # via + # -r dev-py2.in + # pytest-cov + # pytest-pythonpath +pytest-cov==2.8.1 + # via -r tox-py2.in +pytest-pythonpath==0.7.3 + # via -r tox-py2.in +pytz==2024.1 + # via pgcopy +requests==2.31.0 + # via sphinx +six==1.12.0 + # via + # -r tox-py2.in + # mock +snowballstemmer==2.2.0 + # via sphinx +sphinx==4.3.0 + # via + # -r dev-py2.in + # sphinx-rtd-theme +sphinx-rtd-theme==1.0.0 + # via -r dev-py2.in +sphinxcontrib-applehelp==1.0.8 + # via sphinx +sphinxcontrib-devhelp==1.0.6 + # via sphinx +sphinxcontrib-htmlhelp==2.0.5 + # via sphinx +sphinxcontrib-jsmath==1.0.1 + # via sphinx +sphinxcontrib-qthelp==1.0.7 + # via sphinx +sphinxcontrib-serializinghtml==1.1.10 + # via sphinx +toml==0.10.2 + # via pytest +tqdm==4.66.2 + # via -r dev-py2.in +urllib3==2.2.1 + # via requests + +# The following packages are considered to be unsafe in a requirements file: +# setuptools diff --git a/requirements/dev-py3.in b/requirements/dev-py3.in new file mode 100644 index 0000000..2ec1ed0 --- /dev/null +++ b/requirements/dev-py3.in @@ -0,0 +1,10 @@ +-r tox-py3.in + +flake8 +isort +pgcopy +pytest +myst-parser +Sphinx +sphinx-rtd-theme +tqdm diff --git a/requirements/dev-py3.txt b/requirements/dev-py3.txt new file mode 100644 index 0000000..3452065 --- /dev/null +++ b/requirements/dev-py3.txt @@ -0,0 +1,124 @@ +# +# This file is autogenerated by pip-compile with Python 3.10 +# by the following command: +# +# pip-compile --index-url=https://pypi.python.org/simple --output-file=dev-py3.txt dev-py3.in +# +#--index-url https://pypi.python.org/simple + +alabaster==0.7.16 + # via sphinx +attrs==23.2.0 + # via pytest +babel==2.14.0 + # via sphinx +certifi==2024.2.2 + # via requests +charset-normalizer==3.3.2 + # via requests +coverage[toml]==7.4.3 + # via + # -r tox-py3.in + # pytest-cov +docutils==0.20.1 + # via + # myst-parser + # sphinx + # sphinx-rtd-theme +flake8==7.0.0 + # via -r dev-py3.in +idna==3.6 + # via requests +imagesize==1.4.1 + # via sphinx +iniconfig==2.0.0 + # via pytest +isort==5.13.2 + # via -r dev-py3.in +jinja2==3.1.3 + # via + # myst-parser + # sphinx +markdown-it-py==3.0.0 + # via + # mdit-py-plugins + # myst-parser +markupsafe==2.1.5 + # via jinja2 +mccabe==0.7.0 + # via flake8 +mdit-py-plugins==0.4.0 + # via myst-parser +mdurl==0.1.2 + # via markdown-it-py +mock==5.1.0 + # via -r tox-py3.in +myst-parser==2.0.0 + # via -r dev-py3.in +packaging==24.0 + # via + # pytest + # sphinx +pgcopy==1.5.0 + # via -r dev-py3.in +pluggy==1.4.0 + # via pytest +psycopg2==2.9.9 + # via pgcopy +py==1.11.0 + # via pytest +pycodestyle==2.11.1 + # via flake8 +pyflakes==3.2.0 + # via flake8 +pygments==2.17.2 + # via sphinx +pytest==6.2.5 + # via + # -r dev-py3.in + # pytest-cov + # pytest-pythonpath +pytest-cov==4.1.0 + # via -r tox-py3.in +pytest-pythonpath==0.7.4 + # via -r tox-py3.in +pytz==2024.1 + # via pgcopy +pyyaml==6.0.1 + # via myst-parser +requests==2.31.0 + # via sphinx +six==1.16.0 + # via -r tox-py3.in +snowballstemmer==2.2.0 + # via sphinx +sphinx==7.2.6 + # via + # -r dev-py3.in + # myst-parser + # sphinx-rtd-theme + # sphinxcontrib-jquery +sphinx-rtd-theme==2.0.0 + # via -r dev-py3.in +sphinxcontrib-applehelp==1.0.8 + # via sphinx +sphinxcontrib-devhelp==1.0.6 + # via sphinx +sphinxcontrib-htmlhelp==2.0.5 + # via sphinx +sphinxcontrib-jquery==4.1 + # via sphinx-rtd-theme +sphinxcontrib-jsmath==1.0.1 + # via sphinx +sphinxcontrib-qthelp==1.0.7 + # via sphinx +sphinxcontrib-serializinghtml==1.1.10 + # via sphinx +toml==0.10.2 + # via pytest +tomli==2.0.1 + # via coverage +tqdm==4.66.2 + # via -r dev-py3.in +urllib3==2.2.1 + # via requests diff --git a/requirements-tox.txt b/requirements/tox-py2.in similarity index 100% rename from requirements-tox.txt rename to requirements/tox-py2.in diff --git a/requirements/tox-py2.txt b/requirements/tox-py2.txt new file mode 100644 index 0000000..b3d48d0 --- /dev/null +++ b/requirements/tox-py2.txt @@ -0,0 +1,36 @@ +# +# This file is autogenerated by pip-compile with Python 3.10 +# by the following command: +# +# pip-compile --index-url=https://pypi.python.org/simple --output-file=tox-py2.txt tox-py2.in +# +#--index-url https://pypi.python.org/simple + +coverage==4.5.4 + # via + # -r tox-py2.in + # pytest-cov +exceptiongroup==1.2.0 + # via pytest +iniconfig==2.0.0 + # via pytest +mock==3.0.5 + # via -r tox-py2.in +packaging==24.0 + # via pytest +pluggy==1.4.0 + # via pytest +pytest==8.1.1 + # via + # pytest-cov + # pytest-pythonpath +pytest-cov==2.8.1 + # via -r tox-py2.in +pytest-pythonpath==0.7.3 + # via -r tox-py2.in +six==1.12.0 + # via + # -r tox-py2.in + # mock +tomli==2.0.1 + # via pytest diff --git a/requirements/tox-py3.in b/requirements/tox-py3.in new file mode 100644 index 0000000..8e53f2a --- /dev/null +++ b/requirements/tox-py3.in @@ -0,0 +1,5 @@ +coverage +mock +pytest-cov +pytest-pythonpath +six==1.16.0 diff --git a/requirements/tox-py3.txt b/requirements/tox-py3.txt new file mode 100644 index 0000000..3cf2d0f --- /dev/null +++ b/requirements/tox-py3.txt @@ -0,0 +1,38 @@ +# +# This file is autogenerated by pip-compile with Python 3.10 +# by the following command: +# +# pip-compile --index-url=https://pypi.python.org/simple --output-file=tox-py3.txt tox-py3.in +# +#--index-url https://pypi.python.org/simple + +attrs==23.2.0 + # via pytest +coverage[toml]==7.4.3 + # via + # -r tox-py3.in + # pytest-cov +iniconfig==2.0.0 + # via pytest +mock==5.1.0 + # via -r tox-py3.in +packaging==24.0 + # via pytest +pluggy==1.4.0 + # via pytest +py==1.11.0 + # via pytest +pytest==6.2.5 + # via + # pytest-cov + # pytest-pythonpath +pytest-cov==4.1.0 + # via -r tox-py3.in +pytest-pythonpath==0.7.4 + # via -r tox-py3.in +six==1.16.0 + # via -r tox-py3.in +toml==0.10.2 + # via pytest +tomli==2.0.1 + # via coverage diff --git a/setup.py b/setup.py index b83774e..907987a 100755 --- a/setup.py +++ b/setup.py @@ -43,13 +43,13 @@ def run(self): install_requires = [ 'faker', 'faker>=3.0,<4.0; python_version=="2.7"', - 'parmap; python_version>="3.6"', - 'parmap==1.5.2; python_version<"3.6"', + 'parmap; python_version>="3"', + 'parmap==1.5.2; python_version<"3"', 'pgcopy', 'psycopg2', - 'psycopg2>=2.8.4,<2.9; python_version<"3.6"', + 'psycopg2>=2.8.4,<2.9; python_version<"3"', 'pyyaml', - 'pyyaml>=5.4.1,<6.0; python_version<"3.6"', + 'pyyaml>=5.4.1,<6.0; python_version<"3"', 'tqdm' ] @@ -86,11 +86,11 @@ def run(self): 'Programming Language :: Python :: 2', 'Programming Language :: Python :: 2.7', 'Programming Language :: Python :: 3', - 'Programming Language :: Python :: 3.6', - 'Programming Language :: Python :: 3.7', 'Programming Language :: Python :: 3.8', 'Programming Language :: Python :: 3.9', 'Programming Language :: Python :: 3.10', + 'Programming Language :: Python :: 3.11', + 'Programming Language :: Python :: 3.12', 'Topic :: Database' ], packages=find_packages(include=['pganonymize*']), diff --git a/tox.ini b/tox.ini index 167f743..a817498 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,6 @@ [tox] -envlist = flake8,py27,py36,py37,py38,py39,py310 +requires = virtualenv<20.22.0 +envlist = flake8,py27,py38,py39,py310,py311,py312 [testenv:flake8] deps = flake8 @@ -9,24 +10,7 @@ commands = flake8 {toxinidir}/pganonymize {toxinidir}/tests setenv = PYTHONPATH = {toxinidir} deps = - # faker - py27: faker<4 - py{36,37,38,39,310}: faker>=9.9.0 - # parmap - py27: parmap==1.5.2 - py{36,37,38,39,310}: parmap>=1.5.2 - # psycopg2-binary - py27: psycopg2-binary==2.8.4 - py{36,37,38,39,310}: psycopg2-binary>=2.9.2 - # pyyaml - py27: pyyaml<6 - py{36,37,38,39,310}: pyyaml>=6 - # pytest - py27: pytest==4.0.2 - py27: attrs<19.2 - py27: more-itertools<8.11 - py{36,37,38,39,310}: pytest==6.2.5 - # Common requirements - -r{toxinidir}/requirements-tox.txt + py27: -r{toxinidir}/requirements/tox-py2.txt + py{38,39,310,311,312}: -r{toxinidir}/requirements/tox-py3.txt commands = py.test --basetemp={envtmpdir} {posargs} From a8e0e53c107548e0840b995db23cfaa30884d200 Mon Sep 17 00:00:00 2001 From: Henning Kage Date: Tue, 12 Mar 2024 15:40:34 +0100 Subject: [PATCH 007/109] #53: Added changelog entry --- CHANGELOG.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index ef11e66..15d5c7b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,11 @@ ## Development +* [#53 Python 2.7 tests are failing](https://github.com/rheinwerk-verlag/pganonymize/pull/53) + * Dropped Python 3.6, 3.7 support + * Split requirements into Python 2 and 3 + * Use markdown for the changelog and contributing docs + ## 0.11.0 (2024-02-29) * [#52 Add update_json provider](https://github.com/rheinwerk-verlag/pganonymize/pull/52) ([bobslee](https://github.com/bobslee)) From 5eba64bea11a9ebe9951331559342e3a1e3641a9 Mon Sep 17 00:00:00 2001 From: Henning Kage Date: Tue, 12 Mar 2024 15:51:51 +0100 Subject: [PATCH 008/109] #53: Updated the workflows --- .github/workflows/publish.yml | 2 +- .github/workflows/test.yml | 69 ++++++++++++++++++----------------- 2 files changed, 36 insertions(+), 35 deletions(-) diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 411221c..b026c49 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -12,7 +12,7 @@ jobs: - name: Set up Python uses: actions/setup-python@v4 with: - python-version: '3.10' + python-version: '3.12' - name: Install dependencies run: | python -m pip install --upgrade pip diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 001aeba..b1d53f4 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -3,49 +3,50 @@ name: Test on: [push] jobs: - linting: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - uses: actions/setup-python@v4 - with: - python-version: '3.10' - - name: Install dependencies - run: | - python -m pip install --upgrade pip - pip install tox - - name: Lint - run: tox -e flake8 tests: - needs: linting runs-on: ubuntu-latest strategy: matrix: - python-version: ['2.7', '3.6', '3.7', '3.8', '3.9', '3.10'] + image: + - 'python:2.7-buster' + - 'python:3.8-bookworm' + - 'python:3.9-bookworm' + - 'python:3.10-bookworm' + - 'python:3.11-bookworm' + - 'python:3.12-bookworm' + container: + image: ${{ matrix.image }} env: PYTHON: ${{ matrix.python-version }} steps: - uses: actions/checkout@v3 - - name: Set up Python ${{ matrix.python-version }} - id: setup-python - uses: actions/setup-python@v4 - with: - python-version: ${{ matrix.python-version }} - - name: Install dependencies - run: | - python -m pip install --upgrade pip - pip install --use-pep517 tox "coverage<5" - - name: Run tests + - name: Set environment variables run: | - export TOXENV=$(echo "py${{ matrix.python-version }}" | sed 's/\.//g') - tox -- -p no:warnings + echo "PYTHON=$(echo '${{ matrix.image }}' | sed -r 's/^python:([0-9]+)\.([0-9]+).*$/\1.\2/g')" >> $GITHUB_ENV + echo "TOXFACTOR=$(echo '${{ matrix.image }}' | sed -r 's/^python:([0-9]+)\.([0-9]+).*$/py\1\2/g')" >> $GITHUB_ENV + - name: Install importlib-metadata for older Python versions + run: pip install "importlib-metadata<3" + if: >- + matrix.image == 'python:2.7-buster' || + matrix.image == 'python:3.4-stretch' || + matrix.image == 'python:3.5-buster' || + matrix.image == 'python:3.6-bullseye' || + matrix.image == 'python:3.7-bookworm' + - name: Install test utilities + run: pip install "tox<4" tox-factor "coverage<5" + - name: Lint with flake8 + run: tox -e flake8 + if: matrix.image == 'python:3.12-bookworm' + - name: Test via tox + run: tox -- -p no:warnings - name: Generate coverage report - run: coverage html + run: coverage xml if: ${{ success() }} - - name: Upload coverage data - uses: actions/upload-artifact@v3 + - name: Upload coverage dataq + uses: codecov/codecov-action@v3 with: - name: coverage - path: htmlcov - if-no-files-found: ignore - if: ${{ success() }} + env_vars: PYTHON + files: coverage.xml + flags: unittests + verbose: true + if: ${{ success() }} From 968cb5a018775138368d19b56f08d185eb32bbf0 Mon Sep 17 00:00:00 2001 From: Henning Kage Date: Tue, 12 Mar 2024 16:23:35 +0100 Subject: [PATCH 009/109] #53: Removed obsolete Python version --- .github/workflows/test.yml | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index b1d53f4..2a72cea 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -26,12 +26,7 @@ jobs: echo "TOXFACTOR=$(echo '${{ matrix.image }}' | sed -r 's/^python:([0-9]+)\.([0-9]+).*$/py\1\2/g')" >> $GITHUB_ENV - name: Install importlib-metadata for older Python versions run: pip install "importlib-metadata<3" - if: >- - matrix.image == 'python:2.7-buster' || - matrix.image == 'python:3.4-stretch' || - matrix.image == 'python:3.5-buster' || - matrix.image == 'python:3.6-bullseye' || - matrix.image == 'python:3.7-bookworm' + if: matrix.image == 'python:2.7-buster' - name: Install test utilities run: pip install "tox<4" tox-factor "coverage<5" - name: Lint with flake8 @@ -42,7 +37,7 @@ jobs: - name: Generate coverage report run: coverage xml if: ${{ success() }} - - name: Upload coverage dataq + - name: Upload coverage data uses: codecov/codecov-action@v3 with: env_vars: PYTHON From 7087fdfcaa450aedfd04fc8d54754667787113f8 Mon Sep 17 00:00:00 2001 From: Henning Kage Date: Thu, 14 Mar 2024 08:16:37 +0100 Subject: [PATCH 010/109] Added environment for the publish action --- .github/workflows/publish.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 411221c..5d5ca5b 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -7,6 +7,7 @@ on: jobs: publish: runs-on: ubuntu-latest + environment: production steps: - uses: actions/checkout@v2 - name: Set up Python From 812453ece45bf346727ab232a68516874219103f Mon Sep 17 00:00:00 2001 From: Henning Kage Date: Thu, 14 Mar 2024 08:32:55 +0100 Subject: [PATCH 011/109] #55: Fixed the test workflow --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 2a72cea..1ca5b4e 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -44,4 +44,4 @@ jobs: files: coverage.xml flags: unittests verbose: true - if: ${{ success() }} + if: ${{ success() }} From 90c1630215bca3df6104b5cdd12880cfb4920381 Mon Sep 17 00:00:00 2001 From: Henning Kage Date: Thu, 14 Mar 2024 08:39:42 +0100 Subject: [PATCH 012/109] #55: Fixed the Github links for the contributing page --- CONTRIBUTING.md | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 5dce426..97b11e4 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -7,8 +7,8 @@ The following document is a small set of guidelines for contributing to this pro ## Reporting bugs If you have found a bug, please check the project's -[issue](https://github.com/rheinwerk-verlag/postgresql-anonymizer/issues) page first and feel free to create a -[new issue](https://github.com/rheinwerk-verlag/postgresql-anonymizer/issues/new), if no one else has reported it yet. +[issue](https://github.com/rheinwerk-verlag/pganonymize/issues) page first and feel free to create a +[new issue](https://github.com/rheinwerk-verlag/pganonymize/issues/new), if no one else has reported it yet. ## Making changes @@ -16,8 +16,8 @@ Create a fork if you want to make changes or clone the repo if you want a readon version: ```bash -$ git clone git@github.com:rheinwerk-verlag/postgresql-anonymizer.git -$ cd postgresql-anonymizer +$ git clone git@github.com:rheinwerk-verlag/pganonymize.git +$ cd pganonymize ``` For the development use a virtualenv or install the requirements directly: @@ -37,6 +37,6 @@ Before creating a pull request make sure to check: * existing docstrings have been updated * new code has valid docstrings -* whether existing [tests](https://github.com/rheinwerk-verlag/postgresql-anonymizer/tree/development/tests) have to be fixed +* whether existing [tests](https://github.com/rheinwerk-verlag/pganonymize/tree/development/tests) have to be fixed * new tests have to be written first -* the documentation (in particular the [Sphinx documentation](https://github.com/rheinwerk-verlag/postgresql-anonymizer/tree/development/docs)) has to be modified +* the documentation (in particular the [Sphinx documentation](https://github.com/rheinwerk-verlag/pganonymize/tree/development/docs)) has to be modified From fdbe485e3a09f85920bdc31c8f68f3cb9e0a07c6 Mon Sep 17 00:00:00 2001 From: Henning Kage Date: Thu, 14 Mar 2024 08:43:30 +0100 Subject: [PATCH 013/109] #53: Use the python 3.12 slim image --- Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index 40ffeba..e78fe78 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,9 +1,9 @@ -FROM python:3.8.1-slim +FROM python:3.12.2-slim LABEL maintainer="webteam@rheinwerk-verlag.de" RUN apt-get update -y \ - && apt-get upgrade -y \ + && apt-get upgrade -y \ && apt-get install -y libpq-dev python3-pip \ && pip install -U pip \ && pip install pganonymize psycopg2-binary \ From 351375c512561c02c4d3e4db422090300903e1db Mon Sep 17 00:00:00 2001 From: Henning Kage Date: Thu, 14 Mar 2024 09:34:45 +0100 Subject: [PATCH 014/109] #53: Updated the changelog --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 15d5c7b..464e532 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,7 +4,7 @@ * [#53 Python 2.7 tests are failing](https://github.com/rheinwerk-verlag/pganonymize/pull/53) * Dropped Python 3.6, 3.7 support - * Split requirements into Python 2 and 3 + * Split development requirements for Python 2 and 3 * Use markdown for the changelog and contributing docs ## 0.11.0 (2024-02-29) From 28ca811e309d1a601fcc836eaeeed1e6c32cf0b4 Mon Sep 17 00:00:00 2001 From: Henning Kage Date: Thu, 14 Mar 2024 09:37:50 +0100 Subject: [PATCH 015/109] #53: Updated the dev requirements --- requirements/dev-py2.in | 9 ++-- requirements/dev-py2.txt | 103 ++++++++++++++++----------------------- requirements/dev-py3.txt | 22 +++++++-- requirements/tox-py2.in | 10 +++- requirements/tox-py2.txt | 50 +++++++++++++++---- requirements/tox-py3.in | 5 ++ requirements/tox-py3.txt | 20 +++++++- 7 files changed, 136 insertions(+), 83 deletions(-) diff --git a/requirements/dev-py2.in b/requirements/dev-py2.in index 0713cf5..774290f 100644 --- a/requirements/dev-py2.in +++ b/requirements/dev-py2.in @@ -1,9 +1,8 @@ -r tox-py2.in +click<8 # Version 8 drops Python 2.7 support flake8==3.9.2 -isort==5.10.1 +isort==4.3.21 +pip-tools<6 pgcopy>=1.5.0 -pytest==6.2.5 -Sphinx==4.3.0 -sphinx-rtd-theme==1.0.0 -tqdm>=4.61.1 +psycopg2==2.8.6 diff --git a/requirements/dev-py2.txt b/requirements/dev-py2.txt index 2de3f07..ba6940d 100644 --- a/requirements/dev-py2.txt +++ b/requirements/dev-py2.txt @@ -6,103 +6,82 @@ # #--index-url https://pypi.python.org/simple -alabaster==0.7.16 - # via sphinx -attrs==23.2.0 +atomicwrites==1.4.1 # via pytest -babel==2.14.0 - # via sphinx -certifi==2024.2.2 - # via requests -charset-normalizer==3.3.2 - # via requests -coverage==4.5.4 +attrs==21.4.0 # via # -r tox-py2.in - # pytest-cov -docutils==0.17.1 + # pytest +click==7.1.2 + # via + # -r dev-py2.in + # pip-tools +coverage==4.5.1 # via - # sphinx - # sphinx-rtd-theme + # -r tox-py2.in + # pytest-cov +faker==3.0.1 + # via -r tox-py2.in flake8==3.9.2 # via -r dev-py2.in -idna==3.6 - # via requests -imagesize==1.4.1 - # via sphinx -iniconfig==2.0.0 - # via pytest -isort==5.10.1 +isort==4.3.21 # via -r dev-py2.in -jinja2==3.1.3 - # via sphinx -markupsafe==2.1.5 - # via jinja2 mccabe==0.6.1 # via flake8 mock==3.0.5 # via -r tox-py2.in -packaging==24.0 +more-itertools==5.0.0 # via + # -r tox-py2.in # pytest - # sphinx +parmap==1.5.2 + # via -r tox-py2.in pgcopy==1.5.0 + # via + # -r dev-py2.in + # -r tox-py2.in +pip-tools==5.5.0 # via -r dev-py2.in -pluggy==1.4.0 +pluggy==0.13.1 # via pytest -psycopg2==2.9.9 - # via pgcopy +psycopg2==2.8.6 + # via + # -r dev-py2.in + # pgcopy py==1.11.0 # via pytest pycodestyle==2.7.0 # via flake8 pyflakes==2.3.1 # via flake8 -pygments==2.17.2 - # via sphinx -pytest==6.2.5 +pytest==3.10.1 # via - # -r dev-py2.in + # -r tox-py2.in # pytest-cov # pytest-pythonpath pytest-cov==2.8.1 # via -r tox-py2.in pytest-pythonpath==0.7.3 # via -r tox-py2.in +python-dateutil==2.9.0.post0 + # via faker pytz==2024.1 # via pgcopy -requests==2.31.0 - # via sphinx +pyyaml==5.3.1 + # via -r tox-py2.in six==1.12.0 # via # -r tox-py2.in + # faker # mock -snowballstemmer==2.2.0 - # via sphinx -sphinx==4.3.0 - # via - # -r dev-py2.in - # sphinx-rtd-theme -sphinx-rtd-theme==1.0.0 - # via -r dev-py2.in -sphinxcontrib-applehelp==1.0.8 - # via sphinx -sphinxcontrib-devhelp==1.0.6 - # via sphinx -sphinxcontrib-htmlhelp==2.0.5 - # via sphinx -sphinxcontrib-jsmath==1.0.1 - # via sphinx -sphinxcontrib-qthelp==1.0.7 - # via sphinx -sphinxcontrib-serializinghtml==1.1.10 - # via sphinx -toml==0.10.2 - # via pytest -tqdm==4.66.2 - # via -r dev-py2.in -urllib3==2.2.1 - # via requests + # more-itertools + # pytest + # python-dateutil +text-unidecode==1.3 + # via faker +tqdm==4.64.1 + # via -r tox-py2.in # The following packages are considered to be unsafe in a requirements file: +# pip # setuptools diff --git a/requirements/dev-py3.txt b/requirements/dev-py3.txt index 3452065..b5a5ed2 100644 --- a/requirements/dev-py3.txt +++ b/requirements/dev-py3.txt @@ -25,6 +25,8 @@ docutils==0.20.1 # myst-parser # sphinx # sphinx-rtd-theme +faker==24.2.0 + # via -r tox-py3.in flake8==7.0.0 # via -r dev-py3.in idna==3.6 @@ -59,8 +61,12 @@ packaging==24.0 # via # pytest # sphinx +parmap==1.7.0 + # via -r tox-py3.in pgcopy==1.5.0 - # via -r dev-py3.in + # via + # -r dev-py3.in + # -r tox-py3.in pluggy==1.4.0 # via pytest psycopg2==2.9.9 @@ -82,14 +88,20 @@ pytest-cov==4.1.0 # via -r tox-py3.in pytest-pythonpath==0.7.4 # via -r tox-py3.in +python-dateutil==2.9.0.post0 + # via faker pytz==2024.1 # via pgcopy pyyaml==6.0.1 - # via myst-parser + # via + # -r tox-py3.in + # myst-parser requests==2.31.0 # via sphinx six==1.16.0 - # via -r tox-py3.in + # via + # -r tox-py3.in + # python-dateutil snowballstemmer==2.2.0 # via sphinx sphinx==7.2.6 @@ -119,6 +131,8 @@ toml==0.10.2 tomli==2.0.1 # via coverage tqdm==4.66.2 - # via -r dev-py3.in + # via + # -r dev-py3.in + # -r tox-py3.in urllib3==2.2.1 # via requests diff --git a/requirements/tox-py2.in b/requirements/tox-py2.in index 4aca0a6..60447b7 100644 --- a/requirements/tox-py2.in +++ b/requirements/tox-py2.in @@ -1,5 +1,13 @@ -coverage==4.5.4 +attrs<22.1.0 # Version 22.1.0 drops Python 2.7 support +coverage==4.5.1 +Faker==3.0.1 mock==3.0.5 +more-itertools==5.0.0 # Version 5 is the last one to support Python 2.7 +parmap==1.5.2 +pgcopy +pytest<4 pytest-cov==2.8.1 pytest-pythonpath==0.7.3 +pyyaml==5.3.1 # Version 5 is the last one to support Python 2.7 (and 5.4.1 cannot be built using Python 3) six==1.12.0 +tqdm==4.64.1 diff --git a/requirements/tox-py2.txt b/requirements/tox-py2.txt index b3d48d0..6b7d0c5 100644 --- a/requirements/tox-py2.txt +++ b/requirements/tox-py2.txt @@ -6,31 +6,61 @@ # #--index-url https://pypi.python.org/simple -coverage==4.5.4 +atomicwrites==1.4.1 + # via pytest +attrs==21.4.0 + # via + # -r tox-py2.in + # pytest +coverage==4.5.1 # via # -r tox-py2.in # pytest-cov -exceptiongroup==1.2.0 - # via pytest -iniconfig==2.0.0 - # via pytest +faker==3.0.1 + # via -r tox-py2.in mock==3.0.5 # via -r tox-py2.in -packaging==24.0 +more-itertools==5.0.0 + # via + # -r tox-py2.in + # pytest +parmap==1.5.2 + # via -r tox-py2.in +pgcopy==1.5.0 + # via -r tox-py2.in +pluggy==0.13.1 # via pytest -pluggy==1.4.0 +psycopg2==2.9.9 + # via pgcopy +py==1.11.0 # via pytest -pytest==8.1.1 +pytest==3.10.1 # via + # -r tox-py2.in # pytest-cov # pytest-pythonpath pytest-cov==2.8.1 # via -r tox-py2.in pytest-pythonpath==0.7.3 # via -r tox-py2.in +python-dateutil==2.9.0.post0 + # via faker +pytz==2024.1 + # via pgcopy +pyyaml==5.3.1 + # via -r tox-py2.in six==1.12.0 # via # -r tox-py2.in + # faker # mock -tomli==2.0.1 - # via pytest + # more-itertools + # pytest + # python-dateutil +text-unidecode==1.3 + # via faker +tqdm==4.64.1 + # via -r tox-py2.in + +# The following packages are considered to be unsafe in a requirements file: +# setuptools diff --git a/requirements/tox-py3.in b/requirements/tox-py3.in index 8e53f2a..2b01fe6 100644 --- a/requirements/tox-py3.in +++ b/requirements/tox-py3.in @@ -1,5 +1,10 @@ coverage +Faker mock +parmap +pgcopy pytest-cov pytest-pythonpath +pyyaml six==1.16.0 +tqdm diff --git a/requirements/tox-py3.txt b/requirements/tox-py3.txt index 3cf2d0f..781c5e7 100644 --- a/requirements/tox-py3.txt +++ b/requirements/tox-py3.txt @@ -12,14 +12,22 @@ coverage[toml]==7.4.3 # via # -r tox-py3.in # pytest-cov +faker==24.2.0 + # via -r tox-py3.in iniconfig==2.0.0 # via pytest mock==5.1.0 # via -r tox-py3.in packaging==24.0 # via pytest +parmap==1.7.0 + # via -r tox-py3.in +pgcopy==1.5.0 + # via -r tox-py3.in pluggy==1.4.0 # via pytest +psycopg2==2.9.9 + # via pgcopy py==1.11.0 # via pytest pytest==6.2.5 @@ -30,9 +38,19 @@ pytest-cov==4.1.0 # via -r tox-py3.in pytest-pythonpath==0.7.4 # via -r tox-py3.in -six==1.16.0 +python-dateutil==2.9.0.post0 + # via faker +pytz==2024.1 + # via pgcopy +pyyaml==6.0.1 # via -r tox-py3.in +six==1.16.0 + # via + # -r tox-py3.in + # python-dateutil toml==0.10.2 # via pytest tomli==2.0.1 # via coverage +tqdm==4.66.2 + # via -r tox-py3.in From 022f92b3de56d7fe081092a2d816e662600df812 Mon Sep 17 00:00:00 2001 From: Henning Kage Date: Thu, 14 Mar 2024 09:41:43 +0100 Subject: [PATCH 016/109] #53: Updated the dev requirements --- requirements/dev-py2.txt | 1 + requirements/dev-py3.txt | 4 +++- requirements/tox-py2.in | 1 + requirements/tox-py2.txt | 6 ++++-- requirements/tox-py3.in | 1 + requirements/tox-py3.txt | 4 +++- 6 files changed, 13 insertions(+), 4 deletions(-) diff --git a/requirements/dev-py2.txt b/requirements/dev-py2.txt index ba6940d..8aef9a3 100644 --- a/requirements/dev-py2.txt +++ b/requirements/dev-py2.txt @@ -47,6 +47,7 @@ pluggy==0.13.1 psycopg2==2.8.6 # via # -r dev-py2.in + # -r tox-py2.in # pgcopy py==1.11.0 # via pytest diff --git a/requirements/dev-py3.txt b/requirements/dev-py3.txt index b5a5ed2..0f9ff2c 100644 --- a/requirements/dev-py3.txt +++ b/requirements/dev-py3.txt @@ -70,7 +70,9 @@ pgcopy==1.5.0 pluggy==1.4.0 # via pytest psycopg2==2.9.9 - # via pgcopy + # via + # -r tox-py3.in + # pgcopy py==1.11.0 # via pytest pycodestyle==2.11.1 diff --git a/requirements/tox-py2.in b/requirements/tox-py2.in index 60447b7..6ead7e8 100644 --- a/requirements/tox-py2.in +++ b/requirements/tox-py2.in @@ -5,6 +5,7 @@ mock==3.0.5 more-itertools==5.0.0 # Version 5 is the last one to support Python 2.7 parmap==1.5.2 pgcopy +psycopg2==2.8.6 pytest<4 pytest-cov==2.8.1 pytest-pythonpath==0.7.3 diff --git a/requirements/tox-py2.txt b/requirements/tox-py2.txt index 6b7d0c5..cc2da33 100644 --- a/requirements/tox-py2.txt +++ b/requirements/tox-py2.txt @@ -30,8 +30,10 @@ pgcopy==1.5.0 # via -r tox-py2.in pluggy==0.13.1 # via pytest -psycopg2==2.9.9 - # via pgcopy +psycopg2==2.8.6 + # via + # -r tox-py2.in + # pgcopy py==1.11.0 # via pytest pytest==3.10.1 diff --git a/requirements/tox-py3.in b/requirements/tox-py3.in index 2b01fe6..8283136 100644 --- a/requirements/tox-py3.in +++ b/requirements/tox-py3.in @@ -3,6 +3,7 @@ Faker mock parmap pgcopy +psycopg2 pytest-cov pytest-pythonpath pyyaml diff --git a/requirements/tox-py3.txt b/requirements/tox-py3.txt index 781c5e7..b0c8d0e 100644 --- a/requirements/tox-py3.txt +++ b/requirements/tox-py3.txt @@ -27,7 +27,9 @@ pgcopy==1.5.0 pluggy==1.4.0 # via pytest psycopg2==2.9.9 - # via pgcopy + # via + # -r tox-py3.in + # pgcopy py==1.11.0 # via pytest pytest==6.2.5 From 9c5c27a964b309dc5524758d972a0da4c7f044cf Mon Sep 17 00:00:00 2001 From: Henning Kage Date: Thu, 14 Mar 2024 10:54:53 +0100 Subject: [PATCH 017/109] #53: Changed the coverage job --- .github/workflows/test.yml | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 1ca5b4e..878980a 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -35,13 +35,12 @@ jobs: - name: Test via tox run: tox -- -p no:warnings - name: Generate coverage report - run: coverage xml + run: coverage html if: ${{ success() }} - name: Upload coverage data - uses: codecov/codecov-action@v3 + uses: actions/upload-artifact@v3 with: - env_vars: PYTHON - files: coverage.xml - flags: unittests - verbose: true + name: coverage + path: htmlcov + if-no-files-found: ignore if: ${{ success() }} From a858ea3768c458fbec1e1706424426ea262d2cbe Mon Sep 17 00:00:00 2001 From: Henning Kage Date: Thu, 14 Mar 2024 11:01:20 +0100 Subject: [PATCH 018/109] #53: Re-added obsolete Python 3 versions --- .github/workflows/test.yml | 10 +++++++++- setup.py | 4 ++++ tox.ini | 4 ++-- 3 files changed, 15 insertions(+), 3 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 878980a..eee0ffe 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -9,6 +9,10 @@ jobs: matrix: image: - 'python:2.7-buster' + - 'python:3.4-stretch' + - 'python:3.5-buster' + - 'python:3.6-bullseye' + - 'python:3.7-bookworm' - 'python:3.8-bookworm' - 'python:3.9-bookworm' - 'python:3.10-bookworm' @@ -26,7 +30,11 @@ jobs: echo "TOXFACTOR=$(echo '${{ matrix.image }}' | sed -r 's/^python:([0-9]+)\.([0-9]+).*$/py\1\2/g')" >> $GITHUB_ENV - name: Install importlib-metadata for older Python versions run: pip install "importlib-metadata<3" - if: matrix.image == 'python:2.7-buster' + matrix.image == 'python:2.7-buster' || + matrix.image == 'python:3.4-stretch' || + matrix.image == 'python:3.5-buster' || + matrix.image == 'python:3.6-bullseye' || + matrix.image == 'python:3.7-bookworm' - name: Install test utilities run: pip install "tox<4" tox-factor "coverage<5" - name: Lint with flake8 diff --git a/setup.py b/setup.py index 907987a..d3cfc53 100755 --- a/setup.py +++ b/setup.py @@ -86,6 +86,10 @@ def run(self): 'Programming Language :: Python :: 2', 'Programming Language :: Python :: 2.7', 'Programming Language :: Python :: 3', + 'Programming Language :: Python :: 3.4', + 'Programming Language :: Python :: 3.5', + 'Programming Language :: Python :: 3.6', + 'Programming Language :: Python :: 3.7', 'Programming Language :: Python :: 3.8', 'Programming Language :: Python :: 3.9', 'Programming Language :: Python :: 3.10', diff --git a/tox.ini b/tox.ini index a817498..c0890ed 100644 --- a/tox.ini +++ b/tox.ini @@ -1,6 +1,6 @@ [tox] requires = virtualenv<20.22.0 -envlist = flake8,py27,py38,py39,py310,py311,py312 +envlist = flake8,py27,py34,py35,py36,py37,py38,py39,py310,py311,py312 [testenv:flake8] deps = flake8 @@ -11,6 +11,6 @@ setenv = PYTHONPATH = {toxinidir} deps = py27: -r{toxinidir}/requirements/tox-py2.txt - py{38,39,310,311,312}: -r{toxinidir}/requirements/tox-py3.txt + py{34,35,36,37,38,39,310,311,312}: -r{toxinidir}/requirements/tox-py3.txt commands = py.test --basetemp={envtmpdir} {posargs} From 5e522abc27adb2e74396254af13cbc85f749af4d Mon Sep 17 00:00:00 2001 From: Henning Kage Date: Thu, 14 Mar 2024 11:03:03 +0100 Subject: [PATCH 019/109] #53: Fixed the test job --- .github/workflows/test.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index eee0ffe..8d4225d 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -30,6 +30,7 @@ jobs: echo "TOXFACTOR=$(echo '${{ matrix.image }}' | sed -r 's/^python:([0-9]+)\.([0-9]+).*$/py\1\2/g')" >> $GITHUB_ENV - name: Install importlib-metadata for older Python versions run: pip install "importlib-metadata<3" + if: >- matrix.image == 'python:2.7-buster' || matrix.image == 'python:3.4-stretch' || matrix.image == 'python:3.5-buster' || From fad690ce3b2c19aa7bb032e7f64c61abb35d5cab Mon Sep 17 00:00:00 2001 From: Henning Kage Date: Thu, 14 Mar 2024 11:07:07 +0100 Subject: [PATCH 020/109] #53: Pinned attrs for Python 3.4/3.5 --- requirements/tox-py2.in | 2 +- requirements/tox-py3.in | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/requirements/tox-py2.in b/requirements/tox-py2.in index 6ead7e8..ddd3d48 100644 --- a/requirements/tox-py2.in +++ b/requirements/tox-py2.in @@ -9,6 +9,6 @@ psycopg2==2.8.6 pytest<4 pytest-cov==2.8.1 pytest-pythonpath==0.7.3 -pyyaml==5.3.1 # Version 5 is the last one to support Python 2.7 (and 5.4.1 cannot be built using Python 3) +pyyaml==5.3.1 # Version 5 is the last one to support Python 2.7 (and 5.4.1 cannot be built using Python 3) six==1.12.0 tqdm==4.64.1 diff --git a/requirements/tox-py3.in b/requirements/tox-py3.in index 8283136..263d5fc 100644 --- a/requirements/tox-py3.in +++ b/requirements/tox-py3.in @@ -1,3 +1,4 @@ +attrs<22.2.0 # Version 22.2.0 drops Python 3.5 support coverage Faker mock From 6c96429ad4911b3fcfd2063bcfcd40d46839f498 Mon Sep 17 00:00:00 2001 From: Henning Kage Date: Thu, 14 Mar 2024 11:10:04 +0100 Subject: [PATCH 021/109] #53: Pinned attrs for Python 3.4/3.5 --- requirements/dev-py2.txt | 2 +- requirements/dev-py3.txt | 10 +++++----- requirements/tox-py2.txt | 2 +- requirements/tox-py3.txt | 10 +++++----- 4 files changed, 12 insertions(+), 12 deletions(-) diff --git a/requirements/dev-py2.txt b/requirements/dev-py2.txt index 8aef9a3..18c48f7 100644 --- a/requirements/dev-py2.txt +++ b/requirements/dev-py2.txt @@ -1,5 +1,5 @@ # -# This file is autogenerated by pip-compile with Python 3.10 +# This file is autogenerated by pip-compile with Python 3.11 # by the following command: # # pip-compile --index-url=https://pypi.python.org/simple --output-file=dev-py2.txt dev-py2.in diff --git a/requirements/dev-py3.txt b/requirements/dev-py3.txt index 0f9ff2c..762f5e7 100644 --- a/requirements/dev-py3.txt +++ b/requirements/dev-py3.txt @@ -1,5 +1,5 @@ # -# This file is autogenerated by pip-compile with Python 3.10 +# This file is autogenerated by pip-compile with Python 3.11 # by the following command: # # pip-compile --index-url=https://pypi.python.org/simple --output-file=dev-py3.txt dev-py3.in @@ -8,8 +8,10 @@ alabaster==0.7.16 # via sphinx -attrs==23.2.0 - # via pytest +attrs==22.1.0 + # via + # -r tox-py3.in + # pytest babel==2.14.0 # via sphinx certifi==2024.2.2 @@ -130,8 +132,6 @@ sphinxcontrib-serializinghtml==1.1.10 # via sphinx toml==0.10.2 # via pytest -tomli==2.0.1 - # via coverage tqdm==4.66.2 # via # -r dev-py3.in diff --git a/requirements/tox-py2.txt b/requirements/tox-py2.txt index cc2da33..2655246 100644 --- a/requirements/tox-py2.txt +++ b/requirements/tox-py2.txt @@ -1,5 +1,5 @@ # -# This file is autogenerated by pip-compile with Python 3.10 +# This file is autogenerated by pip-compile with Python 3.11 # by the following command: # # pip-compile --index-url=https://pypi.python.org/simple --output-file=tox-py2.txt tox-py2.in diff --git a/requirements/tox-py3.txt b/requirements/tox-py3.txt index b0c8d0e..d8974c3 100644 --- a/requirements/tox-py3.txt +++ b/requirements/tox-py3.txt @@ -1,13 +1,15 @@ # -# This file is autogenerated by pip-compile with Python 3.10 +# This file is autogenerated by pip-compile with Python 3.11 # by the following command: # # pip-compile --index-url=https://pypi.python.org/simple --output-file=tox-py3.txt tox-py3.in # #--index-url https://pypi.python.org/simple -attrs==23.2.0 - # via pytest +attrs==22.1.0 + # via + # -r tox-py3.in + # pytest coverage[toml]==7.4.3 # via # -r tox-py3.in @@ -52,7 +54,5 @@ six==1.16.0 # python-dateutil toml==0.10.2 # via pytest -tomli==2.0.1 - # via coverage tqdm==4.66.2 # via -r tox-py3.in From e4a73c13aa0c8877af3dc84982ee312d7a021755 Mon Sep 17 00:00:00 2001 From: Henning Kage Date: Thu, 14 Mar 2024 11:13:52 +0100 Subject: [PATCH 022/109] #53: Pinned attrs for Python 3.4/3.5 --- requirements/dev-py3.txt | 2 +- requirements/tox-py3.in | 2 +- requirements/tox-py3.txt | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/requirements/dev-py3.txt b/requirements/dev-py3.txt index 762f5e7..74f1dce 100644 --- a/requirements/dev-py3.txt +++ b/requirements/dev-py3.txt @@ -8,7 +8,7 @@ alabaster==0.7.16 # via sphinx -attrs==22.1.0 +attrs==21.1.0 # via # -r tox-py3.in # pytest diff --git a/requirements/tox-py3.in b/requirements/tox-py3.in index 263d5fc..5a2a3a4 100644 --- a/requirements/tox-py3.in +++ b/requirements/tox-py3.in @@ -1,4 +1,4 @@ -attrs<22.2.0 # Version 22.2.0 drops Python 3.5 support +attrs==21.1.0 coverage Faker mock diff --git a/requirements/tox-py3.txt b/requirements/tox-py3.txt index d8974c3..577708a 100644 --- a/requirements/tox-py3.txt +++ b/requirements/tox-py3.txt @@ -6,7 +6,7 @@ # #--index-url https://pypi.python.org/simple -attrs==22.1.0 +attrs==21.1.0 # via # -r tox-py3.in # pytest From ac3947dff132b9ffccfb20f16eab50d58bd0dc89 Mon Sep 17 00:00:00 2001 From: Henning Kage Date: Thu, 14 Mar 2024 11:26:34 +0100 Subject: [PATCH 023/109] #53: Updated requirements --- requirements/dev-py3.txt | 4 +--- requirements/tox-py3.in | 1 - requirements/tox-py3.txt | 4 +--- tox.ini | 1 + 4 files changed, 3 insertions(+), 7 deletions(-) diff --git a/requirements/dev-py3.txt b/requirements/dev-py3.txt index 74f1dce..90a893f 100644 --- a/requirements/dev-py3.txt +++ b/requirements/dev-py3.txt @@ -9,9 +9,7 @@ alabaster==0.7.16 # via sphinx attrs==21.1.0 - # via - # -r tox-py3.in - # pytest + # via pytest babel==2.14.0 # via sphinx certifi==2024.2.2 diff --git a/requirements/tox-py3.in b/requirements/tox-py3.in index 5a2a3a4..8283136 100644 --- a/requirements/tox-py3.in +++ b/requirements/tox-py3.in @@ -1,4 +1,3 @@ -attrs==21.1.0 coverage Faker mock diff --git a/requirements/tox-py3.txt b/requirements/tox-py3.txt index 577708a..52a8372 100644 --- a/requirements/tox-py3.txt +++ b/requirements/tox-py3.txt @@ -7,9 +7,7 @@ #--index-url https://pypi.python.org/simple attrs==21.1.0 - # via - # -r tox-py3.in - # pytest + # via pytest coverage[toml]==7.4.3 # via # -r tox-py3.in diff --git a/tox.ini b/tox.ini index c0890ed..01f7b0e 100644 --- a/tox.ini +++ b/tox.ini @@ -11,6 +11,7 @@ setenv = PYTHONPATH = {toxinidir} deps = py27: -r{toxinidir}/requirements/tox-py2.txt + # Common requirements py{34,35,36,37,38,39,310,311,312}: -r{toxinidir}/requirements/tox-py3.txt commands = py.test --basetemp={envtmpdir} {posargs} From 64671fd1f7a68744f84fb3c9d6bfd08061cc85bc Mon Sep 17 00:00:00 2001 From: Henning Kage Date: Thu, 14 Mar 2024 11:47:14 +0100 Subject: [PATCH 024/109] #53: Simplified requirements --- requirements-tox.txt | 5 ++ requirements.txt | 16 ++++- requirements/Makefile | 24 ------- requirements/dev-py2.in | 8 --- requirements/dev-py2.txt | 88 ------------------------- requirements/dev-py3.in | 10 --- requirements/dev-py3.txt | 138 --------------------------------------- requirements/tox-py2.in | 14 ---- requirements/tox-py2.txt | 68 ------------------- requirements/tox-py3.in | 11 ---- requirements/tox-py3.txt | 56 ---------------- tox.ini | 13 +++- 12 files changed, 29 insertions(+), 422 deletions(-) create mode 100644 requirements-tox.txt delete mode 100644 requirements/Makefile delete mode 100644 requirements/dev-py2.in delete mode 100644 requirements/dev-py2.txt delete mode 100644 requirements/dev-py3.in delete mode 100644 requirements/dev-py3.txt delete mode 100644 requirements/tox-py2.in delete mode 100644 requirements/tox-py2.txt delete mode 100644 requirements/tox-py3.in delete mode 100644 requirements/tox-py3.txt diff --git a/requirements-tox.txt b/requirements-tox.txt new file mode 100644 index 0000000..0491ec7 --- /dev/null +++ b/requirements-tox.txt @@ -0,0 +1,5 @@ +coverage==4.5.4 +mock==3.0.5 +pytest-cov==2.8.1 +pytest-pythonpath==0.7.3 +six>=1.9 diff --git a/requirements.txt b/requirements.txt index f9ab379..4470c7c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,13 @@ -# This file is here because many Platforms as a Service look for -# requirements.txt in the root directory of a project. --r requirements/dev-py3.txt +-r requirements-tox.txt + +Faker +flake8==3.9.2 +isort==5.10.1 +parmap +pgcopy +psycopg2 +pytest==6.2.5 +pyyaml +Sphinx==7.2.6 +sphinx-rtd-theme==2.0.0 +tqdm diff --git a/requirements/Makefile b/requirements/Makefile deleted file mode 100644 index de6c20e..0000000 --- a/requirements/Makefile +++ /dev/null @@ -1,24 +0,0 @@ -REQUIREMENTS_SOURCES := $(wildcard *.in) -REQUIREMENTS := $(REQUIREMENTS_SOURCES:.in=.txt) -PIP_INDEX_URL ?= https://pypi.python.org/simple - -all: $(REQUIREMENTS) - -dev-py2.txt: tox-py2.txt -dev-py3.txt: tox-py3.txt - -%.txt: %.in - pip-compile \ - --verbose \ - --rebuild \ - --index-url=$(PIP_INDEX_URL) \ - --output-file=$@ $< - sed -i -e 's|^--|#--|' $@ - -check: - @which pip-compile > /dev/null - -clean: check - rm -f *.txt - -.PHONY: all check clean diff --git a/requirements/dev-py2.in b/requirements/dev-py2.in deleted file mode 100644 index 774290f..0000000 --- a/requirements/dev-py2.in +++ /dev/null @@ -1,8 +0,0 @@ --r tox-py2.in - -click<8 # Version 8 drops Python 2.7 support -flake8==3.9.2 -isort==4.3.21 -pip-tools<6 -pgcopy>=1.5.0 -psycopg2==2.8.6 diff --git a/requirements/dev-py2.txt b/requirements/dev-py2.txt deleted file mode 100644 index 18c48f7..0000000 --- a/requirements/dev-py2.txt +++ /dev/null @@ -1,88 +0,0 @@ -# -# This file is autogenerated by pip-compile with Python 3.11 -# by the following command: -# -# pip-compile --index-url=https://pypi.python.org/simple --output-file=dev-py2.txt dev-py2.in -# -#--index-url https://pypi.python.org/simple - -atomicwrites==1.4.1 - # via pytest -attrs==21.4.0 - # via - # -r tox-py2.in - # pytest -click==7.1.2 - # via - # -r dev-py2.in - # pip-tools -coverage==4.5.1 - # via - # -r tox-py2.in - # pytest-cov -faker==3.0.1 - # via -r tox-py2.in -flake8==3.9.2 - # via -r dev-py2.in -isort==4.3.21 - # via -r dev-py2.in -mccabe==0.6.1 - # via flake8 -mock==3.0.5 - # via -r tox-py2.in -more-itertools==5.0.0 - # via - # -r tox-py2.in - # pytest -parmap==1.5.2 - # via -r tox-py2.in -pgcopy==1.5.0 - # via - # -r dev-py2.in - # -r tox-py2.in -pip-tools==5.5.0 - # via -r dev-py2.in -pluggy==0.13.1 - # via pytest -psycopg2==2.8.6 - # via - # -r dev-py2.in - # -r tox-py2.in - # pgcopy -py==1.11.0 - # via pytest -pycodestyle==2.7.0 - # via flake8 -pyflakes==2.3.1 - # via flake8 -pytest==3.10.1 - # via - # -r tox-py2.in - # pytest-cov - # pytest-pythonpath -pytest-cov==2.8.1 - # via -r tox-py2.in -pytest-pythonpath==0.7.3 - # via -r tox-py2.in -python-dateutil==2.9.0.post0 - # via faker -pytz==2024.1 - # via pgcopy -pyyaml==5.3.1 - # via -r tox-py2.in -six==1.12.0 - # via - # -r tox-py2.in - # faker - # mock - # more-itertools - # pytest - # python-dateutil -text-unidecode==1.3 - # via faker -tqdm==4.64.1 - # via -r tox-py2.in - -# The following packages are considered to be unsafe in a requirements file: -# pip -# setuptools diff --git a/requirements/dev-py3.in b/requirements/dev-py3.in deleted file mode 100644 index 2ec1ed0..0000000 --- a/requirements/dev-py3.in +++ /dev/null @@ -1,10 +0,0 @@ --r tox-py3.in - -flake8 -isort -pgcopy -pytest -myst-parser -Sphinx -sphinx-rtd-theme -tqdm diff --git a/requirements/dev-py3.txt b/requirements/dev-py3.txt deleted file mode 100644 index 90a893f..0000000 --- a/requirements/dev-py3.txt +++ /dev/null @@ -1,138 +0,0 @@ -# -# This file is autogenerated by pip-compile with Python 3.11 -# by the following command: -# -# pip-compile --index-url=https://pypi.python.org/simple --output-file=dev-py3.txt dev-py3.in -# -#--index-url https://pypi.python.org/simple - -alabaster==0.7.16 - # via sphinx -attrs==21.1.0 - # via pytest -babel==2.14.0 - # via sphinx -certifi==2024.2.2 - # via requests -charset-normalizer==3.3.2 - # via requests -coverage[toml]==7.4.3 - # via - # -r tox-py3.in - # pytest-cov -docutils==0.20.1 - # via - # myst-parser - # sphinx - # sphinx-rtd-theme -faker==24.2.0 - # via -r tox-py3.in -flake8==7.0.0 - # via -r dev-py3.in -idna==3.6 - # via requests -imagesize==1.4.1 - # via sphinx -iniconfig==2.0.0 - # via pytest -isort==5.13.2 - # via -r dev-py3.in -jinja2==3.1.3 - # via - # myst-parser - # sphinx -markdown-it-py==3.0.0 - # via - # mdit-py-plugins - # myst-parser -markupsafe==2.1.5 - # via jinja2 -mccabe==0.7.0 - # via flake8 -mdit-py-plugins==0.4.0 - # via myst-parser -mdurl==0.1.2 - # via markdown-it-py -mock==5.1.0 - # via -r tox-py3.in -myst-parser==2.0.0 - # via -r dev-py3.in -packaging==24.0 - # via - # pytest - # sphinx -parmap==1.7.0 - # via -r tox-py3.in -pgcopy==1.5.0 - # via - # -r dev-py3.in - # -r tox-py3.in -pluggy==1.4.0 - # via pytest -psycopg2==2.9.9 - # via - # -r tox-py3.in - # pgcopy -py==1.11.0 - # via pytest -pycodestyle==2.11.1 - # via flake8 -pyflakes==3.2.0 - # via flake8 -pygments==2.17.2 - # via sphinx -pytest==6.2.5 - # via - # -r dev-py3.in - # pytest-cov - # pytest-pythonpath -pytest-cov==4.1.0 - # via -r tox-py3.in -pytest-pythonpath==0.7.4 - # via -r tox-py3.in -python-dateutil==2.9.0.post0 - # via faker -pytz==2024.1 - # via pgcopy -pyyaml==6.0.1 - # via - # -r tox-py3.in - # myst-parser -requests==2.31.0 - # via sphinx -six==1.16.0 - # via - # -r tox-py3.in - # python-dateutil -snowballstemmer==2.2.0 - # via sphinx -sphinx==7.2.6 - # via - # -r dev-py3.in - # myst-parser - # sphinx-rtd-theme - # sphinxcontrib-jquery -sphinx-rtd-theme==2.0.0 - # via -r dev-py3.in -sphinxcontrib-applehelp==1.0.8 - # via sphinx -sphinxcontrib-devhelp==1.0.6 - # via sphinx -sphinxcontrib-htmlhelp==2.0.5 - # via sphinx -sphinxcontrib-jquery==4.1 - # via sphinx-rtd-theme -sphinxcontrib-jsmath==1.0.1 - # via sphinx -sphinxcontrib-qthelp==1.0.7 - # via sphinx -sphinxcontrib-serializinghtml==1.1.10 - # via sphinx -toml==0.10.2 - # via pytest -tqdm==4.66.2 - # via - # -r dev-py3.in - # -r tox-py3.in -urllib3==2.2.1 - # via requests diff --git a/requirements/tox-py2.in b/requirements/tox-py2.in deleted file mode 100644 index ddd3d48..0000000 --- a/requirements/tox-py2.in +++ /dev/null @@ -1,14 +0,0 @@ -attrs<22.1.0 # Version 22.1.0 drops Python 2.7 support -coverage==4.5.1 -Faker==3.0.1 -mock==3.0.5 -more-itertools==5.0.0 # Version 5 is the last one to support Python 2.7 -parmap==1.5.2 -pgcopy -psycopg2==2.8.6 -pytest<4 -pytest-cov==2.8.1 -pytest-pythonpath==0.7.3 -pyyaml==5.3.1 # Version 5 is the last one to support Python 2.7 (and 5.4.1 cannot be built using Python 3) -six==1.12.0 -tqdm==4.64.1 diff --git a/requirements/tox-py2.txt b/requirements/tox-py2.txt deleted file mode 100644 index 2655246..0000000 --- a/requirements/tox-py2.txt +++ /dev/null @@ -1,68 +0,0 @@ -# -# This file is autogenerated by pip-compile with Python 3.11 -# by the following command: -# -# pip-compile --index-url=https://pypi.python.org/simple --output-file=tox-py2.txt tox-py2.in -# -#--index-url https://pypi.python.org/simple - -atomicwrites==1.4.1 - # via pytest -attrs==21.4.0 - # via - # -r tox-py2.in - # pytest -coverage==4.5.1 - # via - # -r tox-py2.in - # pytest-cov -faker==3.0.1 - # via -r tox-py2.in -mock==3.0.5 - # via -r tox-py2.in -more-itertools==5.0.0 - # via - # -r tox-py2.in - # pytest -parmap==1.5.2 - # via -r tox-py2.in -pgcopy==1.5.0 - # via -r tox-py2.in -pluggy==0.13.1 - # via pytest -psycopg2==2.8.6 - # via - # -r tox-py2.in - # pgcopy -py==1.11.0 - # via pytest -pytest==3.10.1 - # via - # -r tox-py2.in - # pytest-cov - # pytest-pythonpath -pytest-cov==2.8.1 - # via -r tox-py2.in -pytest-pythonpath==0.7.3 - # via -r tox-py2.in -python-dateutil==2.9.0.post0 - # via faker -pytz==2024.1 - # via pgcopy -pyyaml==5.3.1 - # via -r tox-py2.in -six==1.12.0 - # via - # -r tox-py2.in - # faker - # mock - # more-itertools - # pytest - # python-dateutil -text-unidecode==1.3 - # via faker -tqdm==4.64.1 - # via -r tox-py2.in - -# The following packages are considered to be unsafe in a requirements file: -# setuptools diff --git a/requirements/tox-py3.in b/requirements/tox-py3.in deleted file mode 100644 index 8283136..0000000 --- a/requirements/tox-py3.in +++ /dev/null @@ -1,11 +0,0 @@ -coverage -Faker -mock -parmap -pgcopy -psycopg2 -pytest-cov -pytest-pythonpath -pyyaml -six==1.16.0 -tqdm diff --git a/requirements/tox-py3.txt b/requirements/tox-py3.txt deleted file mode 100644 index 52a8372..0000000 --- a/requirements/tox-py3.txt +++ /dev/null @@ -1,56 +0,0 @@ -# -# This file is autogenerated by pip-compile with Python 3.11 -# by the following command: -# -# pip-compile --index-url=https://pypi.python.org/simple --output-file=tox-py3.txt tox-py3.in -# -#--index-url https://pypi.python.org/simple - -attrs==21.1.0 - # via pytest -coverage[toml]==7.4.3 - # via - # -r tox-py3.in - # pytest-cov -faker==24.2.0 - # via -r tox-py3.in -iniconfig==2.0.0 - # via pytest -mock==5.1.0 - # via -r tox-py3.in -packaging==24.0 - # via pytest -parmap==1.7.0 - # via -r tox-py3.in -pgcopy==1.5.0 - # via -r tox-py3.in -pluggy==1.4.0 - # via pytest -psycopg2==2.9.9 - # via - # -r tox-py3.in - # pgcopy -py==1.11.0 - # via pytest -pytest==6.2.5 - # via - # pytest-cov - # pytest-pythonpath -pytest-cov==4.1.0 - # via -r tox-py3.in -pytest-pythonpath==0.7.4 - # via -r tox-py3.in -python-dateutil==2.9.0.post0 - # via faker -pytz==2024.1 - # via pgcopy -pyyaml==6.0.1 - # via -r tox-py3.in -six==1.16.0 - # via - # -r tox-py3.in - # python-dateutil -toml==0.10.2 - # via pytest -tqdm==4.66.2 - # via -r tox-py3.in diff --git a/tox.ini b/tox.ini index 01f7b0e..b5f5cca 100644 --- a/tox.ini +++ b/tox.ini @@ -10,8 +10,17 @@ commands = flake8 {toxinidir}/pganonymize {toxinidir}/tests setenv = PYTHONPATH = {toxinidir} deps = - py27: -r{toxinidir}/requirements/tox-py2.txt + # Python 2.7 + py27: attrs<19.2 + py27: faker<4 + py27: more-itertools<8.11 + py27: parmap==1.5.2 + py27: psycopg2-binary==2.8.4 + py27: pyyaml<6 # Common requirements - py{34,35,36,37,38,39,310,311,312}: -r{toxinidir}/requirements/tox-py3.txt + -r{toxinidir}/requirements-tox.txt commands = py.test --basetemp={envtmpdir} {posargs} +passenv = + TOXENV + GITHUB_* From b83ea1d9b7b776d1cb7e186f8243d548e238078b Mon Sep 17 00:00:00 2001 From: Henning Kage Date: Thu, 14 Mar 2024 11:52:26 +0100 Subject: [PATCH 025/109] #53: Updated requirements --- tox.ini | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/tox.ini b/tox.ini index b5f5cca..c47aa23 100644 --- a/tox.ini +++ b/tox.ini @@ -10,13 +10,17 @@ commands = flake8 {toxinidir}/pganonymize {toxinidir}/tests setenv = PYTHONPATH = {toxinidir} deps = - # Python 2.7 - py27: attrs<19.2 + # faker py27: faker<4 - py27: more-itertools<8.11 + # parmap py27: parmap==1.5.2 + # psycopg2 py27: psycopg2-binary==2.8.4 - py27: pyyaml<6 + # PyYAML + py{27,34}: pyyaml<6 + # pytest + py27: attrs<19.2 + py27: more-itertools<8.11 # Common requirements -r{toxinidir}/requirements-tox.txt commands = From ec65f3c2f48189913a4a6952cc36fd1eb7f4cf36 Mon Sep 17 00:00:00 2001 From: Henning Kage Date: Thu, 14 Mar 2024 12:14:47 +0100 Subject: [PATCH 026/109] #53: Updated requirements --- tox.ini | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index c47aa23..ac5e11c 100644 --- a/tox.ini +++ b/tox.ini @@ -17,7 +17,8 @@ deps = # psycopg2 py27: psycopg2-binary==2.8.4 # PyYAML - py{27,34}: pyyaml<6 + py27: pyyaml<6 + py34: pyyaml<5.3 # pytest py27: attrs<19.2 py27: more-itertools<8.11 From 221606521821bc0d6d33be181ead00f635db82e0 Mon Sep 17 00:00:00 2001 From: Henning Kage Date: Thu, 14 Mar 2024 12:20:25 +0100 Subject: [PATCH 027/109] #53: Updated requirements --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index ac5e11c..a225ac0 100644 --- a/tox.ini +++ b/tox.ini @@ -13,7 +13,7 @@ deps = # faker py27: faker<4 # parmap - py27: parmap==1.5.2 + py{27,34}: parmap==1.5.2 # psycopg2 py27: psycopg2-binary==2.8.4 # PyYAML From be4b4a534fd34d4bea2d56863a2a95049f514762 Mon Sep 17 00:00:00 2001 From: Henning Kage Date: Thu, 14 Mar 2024 12:29:35 +0100 Subject: [PATCH 028/109] #53: Updated tox requirements --- tox.ini | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tox.ini b/tox.ini index a225ac0..5ebc153 100644 --- a/tox.ini +++ b/tox.ini @@ -20,6 +20,8 @@ deps = py27: pyyaml<6 py34: pyyaml<5.3 # pytest + py{27,34,35}: pytest==4.0.2 + py{36,37,38,39,310,311,312}: pytest==6.2.5 py27: attrs<19.2 py27: more-itertools<8.11 # Common requirements From 413dc2f250af470af0b3282d1035eafcb9476b24 Mon Sep 17 00:00:00 2001 From: Henning Kage Date: Thu, 14 Mar 2024 12:33:39 +0100 Subject: [PATCH 029/109] #53: Updated tox requirements --- tox.ini | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tox.ini b/tox.ini index 5ebc153..3b609b0 100644 --- a/tox.ini +++ b/tox.ini @@ -20,8 +20,8 @@ deps = py27: pyyaml<6 py34: pyyaml<5.3 # pytest - py{27,34,35}: pytest==4.0.2 - py{36,37,38,39,310,311,312}: pytest==6.2.5 + py{27,34}: pytest==4.6.11 + py{35,36,37,38,39,310,311,312}: pytest==6.2.5 py27: attrs<19.2 py27: more-itertools<8.11 # Common requirements From 0db301d4055ec5ec17be8d9feb083bb970127944 Mon Sep 17 00:00:00 2001 From: Henning Kage Date: Thu, 14 Mar 2024 12:49:02 +0100 Subject: [PATCH 030/109] #53: Updated tox requirements --- tox.ini | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tox.ini b/tox.ini index 3b609b0..63f3512 100644 --- a/tox.ini +++ b/tox.ini @@ -20,8 +20,8 @@ deps = py27: pyyaml<6 py34: pyyaml<5.3 # pytest - py{27,34}: pytest==4.6.11 - py{35,36,37,38,39,310,311,312}: pytest==6.2.5 + py{27,34,35}: pytest==4.6.11 + py{36,37,38,39,310,311,312}: pytest==6.2.5 py27: attrs<19.2 py27: more-itertools<8.11 # Common requirements From 7e17dd3478c25dc36eddfc47a3f797799848c104 Mon Sep 17 00:00:00 2001 From: Henning Kage Date: Thu, 14 Mar 2024 13:44:27 +0100 Subject: [PATCH 031/109] #53: Updated tox requirements --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index 63f3512..acd4baa 100644 --- a/tox.ini +++ b/tox.ini @@ -13,7 +13,7 @@ deps = # faker py27: faker<4 # parmap - py{27,34}: parmap==1.5.2 + py{27,34,35}: parmap==1.5.2 # psycopg2 py27: psycopg2-binary==2.8.4 # PyYAML From ce5620e1cee73ff3c86ebd52cc699ede52bbe38d Mon Sep 17 00:00:00 2001 From: Henning Kage Date: Thu, 14 Mar 2024 14:35:50 +0100 Subject: [PATCH 032/109] #53: Updated tox requirements --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index acd4baa..ae7718a 100644 --- a/tox.ini +++ b/tox.ini @@ -11,7 +11,7 @@ setenv = PYTHONPATH = {toxinidir} deps = # faker - py27: faker<4 + py{27,34}: faker<4 # parmap py{27,34,35}: parmap==1.5.2 # psycopg2 From 657f1b069dd641ffb49a0fb8471bf03219ae5eaf Mon Sep 17 00:00:00 2001 From: Henning Kage Date: Thu, 14 Mar 2024 14:38:24 +0100 Subject: [PATCH 033/109] #53: Updated tox requirements --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index ae7718a..d875e41 100644 --- a/tox.ini +++ b/tox.ini @@ -11,7 +11,7 @@ setenv = PYTHONPATH = {toxinidir} deps = # faker - py{27,34}: faker<4 + py{27,34}: faker==3.0.1 # parmap py{27,34,35}: parmap==1.5.2 # psycopg2 From 6fb567d78e5dec51a23a3a8c955ebf6d66f246fa Mon Sep 17 00:00:00 2001 From: Henning Kage Date: Thu, 14 Mar 2024 15:05:25 +0100 Subject: [PATCH 034/109] #53: Updated tox requirements --- tox.ini | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index d875e41..b804a7f 100644 --- a/tox.ini +++ b/tox.ini @@ -11,7 +11,8 @@ setenv = PYTHONPATH = {toxinidir} deps = # faker - py{27,34}: faker==3.0.1 + py27: faker==3.0.1 + py34: faker==4.1.1 # parmap py{27,34,35}: parmap==1.5.2 # psycopg2 From a20f1bcc629bfbaa312175f0d0568d4b185102df Mon Sep 17 00:00:00 2001 From: Henning Kage Date: Thu, 14 Mar 2024 15:08:05 +0100 Subject: [PATCH 035/109] #53: Updated tox requirements --- tox.ini | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tox.ini b/tox.ini index b804a7f..ae7718a 100644 --- a/tox.ini +++ b/tox.ini @@ -11,8 +11,7 @@ setenv = PYTHONPATH = {toxinidir} deps = # faker - py27: faker==3.0.1 - py34: faker==4.1.1 + py{27,34}: faker<4 # parmap py{27,34,35}: parmap==1.5.2 # psycopg2 From 1053d42ff79c4cc5ddeda6a7d585029165b4e2c2 Mon Sep 17 00:00:00 2001 From: Henning Kage Date: Thu, 14 Mar 2024 15:18:14 +0100 Subject: [PATCH 036/109] #53: Removed Python 3.4 support --- .github/workflows/test.yml | 2 -- setup.py | 1 - tox.ini | 9 ++++----- 3 files changed, 4 insertions(+), 8 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 8d4225d..8b1ef89 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -9,7 +9,6 @@ jobs: matrix: image: - 'python:2.7-buster' - - 'python:3.4-stretch' - 'python:3.5-buster' - 'python:3.6-bullseye' - 'python:3.7-bookworm' @@ -32,7 +31,6 @@ jobs: run: pip install "importlib-metadata<3" if: >- matrix.image == 'python:2.7-buster' || - matrix.image == 'python:3.4-stretch' || matrix.image == 'python:3.5-buster' || matrix.image == 'python:3.6-bullseye' || matrix.image == 'python:3.7-bookworm' diff --git a/setup.py b/setup.py index d3cfc53..4ce1eed 100755 --- a/setup.py +++ b/setup.py @@ -86,7 +86,6 @@ def run(self): 'Programming Language :: Python :: 2', 'Programming Language :: Python :: 2.7', 'Programming Language :: Python :: 3', - 'Programming Language :: Python :: 3.4', 'Programming Language :: Python :: 3.5', 'Programming Language :: Python :: 3.6', 'Programming Language :: Python :: 3.7', diff --git a/tox.ini b/tox.ini index ae7718a..6562800 100644 --- a/tox.ini +++ b/tox.ini @@ -1,6 +1,6 @@ [tox] requires = virtualenv<20.22.0 -envlist = flake8,py27,py34,py35,py36,py37,py38,py39,py310,py311,py312 +envlist = flake8,py27,py35,py36,py37,py38,py39,py310,py311,py312 [testenv:flake8] deps = flake8 @@ -11,16 +11,15 @@ setenv = PYTHONPATH = {toxinidir} deps = # faker - py{27,34}: faker<4 + py27: faker<4 # parmap - py{27,34,35}: parmap==1.5.2 + py{27,35}: parmap==1.5.2 # psycopg2 py27: psycopg2-binary==2.8.4 # PyYAML py27: pyyaml<6 - py34: pyyaml<5.3 # pytest - py{27,34,35}: pytest==4.6.11 + py{27,35}: pytest==4.6.11 py{36,37,38,39,310,311,312}: pytest==6.2.5 py27: attrs<19.2 py27: more-itertools<8.11 From a40ef63511c2253d81cd140c64c09323d1b6ffbb Mon Sep 17 00:00:00 2001 From: Henning Kage Date: Thu, 14 Mar 2024 15:26:26 +0100 Subject: [PATCH 037/109] #53: Fixed tox requirements --- tox.ini | 1 + 1 file changed, 1 insertion(+) diff --git a/tox.ini b/tox.ini index 6562800..e354363 100644 --- a/tox.ini +++ b/tox.ini @@ -16,6 +16,7 @@ deps = py{27,35}: parmap==1.5.2 # psycopg2 py27: psycopg2-binary==2.8.4 + py{35,36,37,38,39,310,311,312}: psycopg2-binary>=2.9.2 # PyYAML py27: pyyaml<6 # pytest From 9eedf3296b86d299254d487517febbe3bf360168 Mon Sep 17 00:00:00 2001 From: Henning Kage Date: Thu, 14 Mar 2024 15:28:19 +0100 Subject: [PATCH 038/109] #53: Fixed tox requirements --- tox.ini | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tox.ini b/tox.ini index e354363..161b051 100644 --- a/tox.ini +++ b/tox.ini @@ -15,8 +15,8 @@ deps = # parmap py{27,35}: parmap==1.5.2 # psycopg2 - py27: psycopg2-binary==2.8.4 - py{35,36,37,38,39,310,311,312}: psycopg2-binary>=2.9.2 + py{27,35}: psycopg2-binary==2.8.4 + py{36,37,38,39,310,311,312}: psycopg2-binary>=2.9.2 # PyYAML py27: pyyaml<6 # pytest From de1a69eb70054f9a378f816c78672b6b3617563a Mon Sep 17 00:00:00 2001 From: Henning Kage Date: Thu, 14 Mar 2024 15:41:47 +0100 Subject: [PATCH 039/109] #53: Fixed tox requirements --- .github/workflows/test.yml | 2 -- requirements-tox.txt | 2 +- requirements.txt | 13 ++++--------- tox.ini | 19 +++++++++++-------- 4 files changed, 16 insertions(+), 20 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 8b1ef89..e2ca119 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -9,7 +9,6 @@ jobs: matrix: image: - 'python:2.7-buster' - - 'python:3.5-buster' - 'python:3.6-bullseye' - 'python:3.7-bookworm' - 'python:3.8-bookworm' @@ -31,7 +30,6 @@ jobs: run: pip install "importlib-metadata<3" if: >- matrix.image == 'python:2.7-buster' || - matrix.image == 'python:3.5-buster' || matrix.image == 'python:3.6-bullseye' || matrix.image == 'python:3.7-bookworm' - name: Install test utilities diff --git a/requirements-tox.txt b/requirements-tox.txt index 0491ec7..4aca0a6 100644 --- a/requirements-tox.txt +++ b/requirements-tox.txt @@ -2,4 +2,4 @@ coverage==4.5.4 mock==3.0.5 pytest-cov==2.8.1 pytest-pythonpath==0.7.3 -six>=1.9 +six==1.12.0 diff --git a/requirements.txt b/requirements.txt index 4470c7c..3bd0569 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,13 +1,8 @@ -r requirements-tox.txt - -Faker flake8==3.9.2 isort==5.10.1 -parmap -pgcopy -psycopg2 +pgcopy>=1.5.0 pytest==6.2.5 -pyyaml -Sphinx==7.2.6 -sphinx-rtd-theme==2.0.0 -tqdm +Sphinx==4.3.0 +sphinx-rtd-theme==1.0.0 +tqdm>=4.61.1 diff --git a/tox.ini b/tox.ini index 161b051..27ef2ff 100644 --- a/tox.ini +++ b/tox.ini @@ -1,6 +1,6 @@ [tox] requires = virtualenv<20.22.0 -envlist = flake8,py27,py35,py36,py37,py38,py39,py310,py311,py312 +envlist = flake8,py27,py36,py37,py38,py39,py310,py311,py312 [testenv:flake8] deps = flake8 @@ -12,18 +12,21 @@ setenv = deps = # faker py27: faker<4 + py{36,37,38,39,310}: faker>=9.9.0 # parmap - py{27,35}: parmap==1.5.2 - # psycopg2 - py{27,35}: psycopg2-binary==2.8.4 - py{36,37,38,39,310,311,312}: psycopg2-binary>=2.9.2 - # PyYAML + py27: parmap==1.5.2 + py{36,37,38,39,310}: parmap>=1.5.2 + # psycopg2-binary + py27: psycopg2-binary==2.8.4 + py{36,37,38,39,310}: psycopg2-binary>=2.9.2 + # pyyaml py27: pyyaml<6 + py{36,37,38,39,310}: pyyaml>=6 # pytest - py{27,35}: pytest==4.6.11 - py{36,37,38,39,310,311,312}: pytest==6.2.5 + py27: pytest==4.0.2 py27: attrs<19.2 py27: more-itertools<8.11 + py{36,37,38,39,310}: pytest==6.2.5 # Common requirements -r{toxinidir}/requirements-tox.txt commands = From 8054c8b23abf37d94f5355f14dd139346aa45d40 Mon Sep 17 00:00:00 2001 From: Henning Kage Date: Thu, 14 Mar 2024 15:53:32 +0100 Subject: [PATCH 040/109] #53: Updated Sphinx doc generation --- .gitignore | 3 ++- Makefile | 59 ++++++++++++++++++++++++++++++++++++++++++---------- docs/api.rst | 2 +- 3 files changed, 51 insertions(+), 13 deletions(-) diff --git a/.gitignore b/.gitignore index 5783e32..7332cba 100644 --- a/.gitignore +++ b/.gitignore @@ -55,10 +55,11 @@ coverage.xml # Sphinx documentation docs/_build/ +docs/_api # PyBuilder target/ # IDEs .idea/ -.vscode/ \ No newline at end of file +.vscode/ diff --git a/Makefile b/Makefile index eebb1a3..f7fbf8d 100644 --- a/Makefile +++ b/Makefile @@ -41,24 +41,61 @@ clean-pyc: ## remove Python file artifacts clean-test: ## remove test and coverage artifacts rm -fr .tox/ + rm -f .coverage + rm -fr reports/ + +test: ## run tests quickly with the default Python + python setup.py test + +test-all: ## run tests on every Python version with tox + tox + +pylint: ## run style checks and static analysis with pylint + @-mkdir -p reports/ + @-pylint $(PYTHON_PACKAGE) -r n --msg-template="{path}:{line}: [{msg_id}({symbol}), {obj}] {msg}" > reports/pylint.txt + @echo "See reports/pylint.txt" + @-pylint $(PYTHON_PACKAGE) flake8: ## run style checks and static analysis with flake8 - @flake8 + @-mkdir -p reports/ + flake8 $(PYTHON_PACKAGE) $(TESTS_PACKAGE) --format='%(path)s:%(row)d: [%(code)s(%(code)s), ] %(text)s' --output-file=reports/flake8.txt --tee + +docstrings: ## check docstring presence and style conventions with pydocstyle + pydocstyle $(PYTHON_PACKAGE) + +lint: flake8 docstrings pylint + +coverage: ## check code coverage quickly with the default Python + py.test --cov-report html:reports/htmlcov --cov-report xml:reports/coverage.xml + @echo "See reports/htmlcov/index.html" + +metrics: ## print code metrics with radon + radon raw -s $(PYTHON_PACKAGE) $(TEST_PACKAGE) + radon cc -s $(PYTHON_PACKAGE) $(TEST_PACKAGE) + radon mi -s $(PYTHON_PACKAGE) $(TEST_PACKAGE) + +docs: ## generate Sphinx HTML documentation, including API docs + @if python -c 'import sys; sys.exit(sys.version_info[0]<3)'; then \ + rm -rf docs/_api; \ + sphinx-apidoc --no-toc -o docs/_api $(PYTHON_PACKAGE) "**/tests" "**/migrations" "**/south_migrations"; \ + $(MAKE) -C docs clean; \ + $(MAKE) -C docs html; \ + echo "See docs/_build/html/index.html"; \ + else \ + echo "Please build the docs using Python 3."; \ + fi + +docs-open: + $(BROWSER) docs/_build/html/index.html + +docs-all: docs docs-open release: clean ## package and upload a release - python setup.py sdist upload - python setup.py bdist_wheel upload + python setup.py release upload dist: clean ## builds source and wheel package - python setup.py sdist - python setup.py bdist_wheel + python setup.py release ls -l dist install: clean ## install the package to the active Python's site-packages python setup.py install - -test: - @pytest - -test-all: ## run tests on every Python version with tox - @tox diff --git a/docs/api.rst b/docs/api.rst index 4ac86b4..22f0faa 100644 --- a/docs/api.rst +++ b/docs/api.rst @@ -4,4 +4,4 @@ API .. toctree:: :maxdepth: 4 - pganonymize + _api/pganonymize From ce9ba832b2bc3af44a22b32a83f44e86f31b8962 Mon Sep 17 00:00:00 2001 From: Henning Kage Date: Thu, 14 Mar 2024 15:56:27 +0100 Subject: [PATCH 041/109] #53: Updated tox requirements --- tox.ini | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tox.ini b/tox.ini index 27ef2ff..09189c0 100644 --- a/tox.ini +++ b/tox.ini @@ -12,13 +12,13 @@ setenv = deps = # faker py27: faker<4 - py{36,37,38,39,310}: faker>=9.9.0 + py{36,37,38,39,310,311,312}: faker>=9.9.0 # parmap py27: parmap==1.5.2 - py{36,37,38,39,310}: parmap>=1.5.2 + py{36,37,38,39,310,311,312}: parmap>=1.5.2 # psycopg2-binary py27: psycopg2-binary==2.8.4 - py{36,37,38,39,310}: psycopg2-binary>=2.9.2 + py{36,37,38,39,310,311,312}: psycopg2-binary>=2.9.2 # pyyaml py27: pyyaml<6 py{36,37,38,39,310}: pyyaml>=6 @@ -26,7 +26,7 @@ deps = py27: pytest==4.0.2 py27: attrs<19.2 py27: more-itertools<8.11 - py{36,37,38,39,310}: pytest==6.2.5 + py{36,37,38,39,310,311,312}: pytest==6.2.5 # Common requirements -r{toxinidir}/requirements-tox.txt commands = From 9564f14634ed6bc3fd748ce1ac2de7d1b58cc3f8 Mon Sep 17 00:00:00 2001 From: Henning Kage Date: Thu, 14 Mar 2024 16:04:16 +0100 Subject: [PATCH 042/109] #53: Updated tox requirements --- setup.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/setup.py b/setup.py index 4ce1eed..50c63c5 100755 --- a/setup.py +++ b/setup.py @@ -43,13 +43,13 @@ def run(self): install_requires = [ 'faker', 'faker>=3.0,<4.0; python_version=="2.7"', - 'parmap; python_version>="3"', - 'parmap==1.5.2; python_version<"3"', + 'parmap; python_version>="3.6"', + 'parmap==1.5.2; python_version<"3.6"', 'pgcopy', 'psycopg2', - 'psycopg2>=2.8.4,<2.9; python_version<"3"', + 'psycopg2>=2.8.4,<2.9; python_version<"3.6"', 'pyyaml', - 'pyyaml>=5.4.1,<6.0; python_version<"3"', + 'pyyaml>=5.4.1,<6.0; python_version<"3.6"', 'tqdm' ] @@ -86,7 +86,6 @@ def run(self): 'Programming Language :: Python :: 2', 'Programming Language :: Python :: 2.7', 'Programming Language :: Python :: 3', - 'Programming Language :: Python :: 3.5', 'Programming Language :: Python :: 3.6', 'Programming Language :: Python :: 3.7', 'Programming Language :: Python :: 3.8', From 1dc6800e1125defeeb2fcab0779310dec8b2e30b Mon Sep 17 00:00:00 2001 From: Henning Kage Date: Thu, 14 Mar 2024 16:22:38 +0100 Subject: [PATCH 043/109] #53: Fixed the test workflow --- .github/workflows/test.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index e2ca119..a4a99f8 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -18,8 +18,6 @@ jobs: - 'python:3.12-bookworm' container: image: ${{ matrix.image }} - env: - PYTHON: ${{ matrix.python-version }} steps: - uses: actions/checkout@v3 - name: Set environment variables From 0e4899f8375476487b68f4c013f6553e2eeab544 Mon Sep 17 00:00:00 2001 From: Henning Kage Date: Thu, 14 Mar 2024 16:48:00 +0100 Subject: [PATCH 044/109] #53: Added test for PyPI publish via trusted publisher --- .github/workflows/publish.yml | 32 +++++++++++++------------------- 1 file changed, 13 insertions(+), 19 deletions(-) diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index b026c49..27b76a3 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -1,26 +1,20 @@ name: Release -on: - release: - types: [created] +#on: +# release: +# types: [created] jobs: publish: + name: Upload release to PyPI runs-on: ubuntu-latest + environment: + name: pypi + url: https://pypi.org/project/pganonymize/ + permissions: + id-token: write steps: - - uses: actions/checkout@v2 - - name: Set up Python - uses: actions/setup-python@v4 - with: - python-version: '3.12' - - name: Install dependencies - run: | - python -m pip install --upgrade pip - pip install setuptools wheel twine - - name: Build and publish - env: - TWINE_USERNAME: __token__ - TWINE_PASSWORD: ${{ secrets.PYPI_TOKEN }} - run: | - python setup.py sdist bdist_wheel - twine upload dist/* + - name: Publish package distributions to PyPI + uses: pypa/gh-action-pypi-publish@release/v1 + with: + repository-url: https://test.pypi.org/legacy/ From c2f17a3daba0e0e4ac779bdfbb231cd9cc988e6d Mon Sep 17 00:00:00 2001 From: Henning Kage Date: Thu, 14 Mar 2024 16:48:48 +0100 Subject: [PATCH 045/109] #53: Added test for PyPI publish via trusted publisher --- .github/workflows/publish.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 27b76a3..45fbead 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -1,5 +1,7 @@ name: Release +on: [push] + #on: # release: # types: [created] From 05315517adce1a7568545a763fd188bcb1a608a1 Mon Sep 17 00:00:00 2001 From: Henning Kage Date: Thu, 14 Mar 2024 17:06:23 +0100 Subject: [PATCH 046/109] #53: Added test for PyPI publish via trusted publisher --- .github/workflows/publish.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 45fbead..ed985ef 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -11,7 +11,8 @@ jobs: name: Upload release to PyPI runs-on: ubuntu-latest environment: - name: pypi + #name: pypi + name: testpypi url: https://pypi.org/project/pganonymize/ permissions: id-token: write From 7cd107b87d1c3e720b29fc803529dd4c073df7c3 Mon Sep 17 00:00:00 2001 From: Henning Kage Date: Thu, 14 Mar 2024 17:09:09 +0100 Subject: [PATCH 047/109] #53: Added test for PyPI publish via trusted publisher --- .github/workflows/publish.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index ed985ef..401c799 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -12,7 +12,8 @@ jobs: runs-on: ubuntu-latest environment: #name: pypi - name: testpypi + #name: testpypi + name: production url: https://pypi.org/project/pganonymize/ permissions: id-token: write From 635922a4fa16cc0b1b29adfa3f90a6f92e4dc204 Mon Sep 17 00:00:00 2001 From: Henning Kage Date: Thu, 14 Mar 2024 17:13:29 +0100 Subject: [PATCH 048/109] #53: Added test for PyPI publish via trusted publisher --- .github/workflows/publish.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 401c799..3b3f069 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -11,9 +11,8 @@ jobs: name: Upload release to PyPI runs-on: ubuntu-latest environment: - #name: pypi - #name: testpypi - name: production + #name: production + name: testing url: https://pypi.org/project/pganonymize/ permissions: id-token: write From 74abdd0c0aa5eda937be7c7e6840ed869a4e99ec Mon Sep 17 00:00:00 2001 From: Henning Kage Date: Thu, 14 Mar 2024 17:21:05 +0100 Subject: [PATCH 049/109] #53: Added test workflow --- .github/workflows/publish.yml | 13 ++++--------- .github/workflows/test-publish.yml | 23 +++++++++++++++++++++++ 2 files changed, 27 insertions(+), 9 deletions(-) create mode 100644 .github/workflows/test-publish.yml diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 3b3f069..990b915 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -1,23 +1,18 @@ name: Release -on: [push] - -#on: -# release: -# types: [created] +on: + release: + types: [created] jobs: publish: name: Upload release to PyPI runs-on: ubuntu-latest environment: - #name: production - name: testing + name: production url: https://pypi.org/project/pganonymize/ permissions: id-token: write steps: - name: Publish package distributions to PyPI uses: pypa/gh-action-pypi-publish@release/v1 - with: - repository-url: https://test.pypi.org/legacy/ diff --git a/.github/workflows/test-publish.yml b/.github/workflows/test-publish.yml new file mode 100644 index 0000000..d5eebfa --- /dev/null +++ b/.github/workflows/test-publish.yml @@ -0,0 +1,23 @@ +name: Test Release + +on: [push] + +jobs: + publish: + name: Upload release to TestPyPI + runs-on: ubuntu-latest + environment: + name: testing + url: https://test.pypi.org/p/pganonymize/ + permissions: + id-token: write # IMPORTANT: mandatory for trusted publishing + steps: + - name: Download all the dists + uses: actions/download-artifact@v3 + with: + name: python-package-distributions + path: dist/ + - name: Publish package distributions to PyPI + uses: pypa/gh-action-pypi-publish@release/v1 + with: + repository-url: https://test.pypi.org/legacy/ From 68b2d494d471e82b2b413b3cb0816ef7227dc79f Mon Sep 17 00:00:00 2001 From: Henning Kage Date: Thu, 14 Mar 2024 17:27:40 +0100 Subject: [PATCH 050/109] #53: Added test workflow --- .github/workflows/publish.yml | 2 +- .github/workflows/test-publish.yml | 17 ++++++++++++----- 2 files changed, 13 insertions(+), 6 deletions(-) diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 990b915..dc4cde6 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -12,7 +12,7 @@ jobs: name: production url: https://pypi.org/project/pganonymize/ permissions: - id-token: write + id-token: write # IMPORTANT: mandatory for trusted publishing steps: - name: Publish package distributions to PyPI uses: pypa/gh-action-pypi-publish@release/v1 diff --git a/.github/workflows/test-publish.yml b/.github/workflows/test-publish.yml index d5eebfa..cf7daf5 100644 --- a/.github/workflows/test-publish.yml +++ b/.github/workflows/test-publish.yml @@ -4,19 +4,26 @@ on: [push] jobs: publish: - name: Upload release to TestPyPI + name: Publish Python 🐍 distribution 📦 to TestPyPI runs-on: ubuntu-latest + needs: build environment: name: testing url: https://test.pypi.org/p/pganonymize/ permissions: id-token: write # IMPORTANT: mandatory for trusted publishing steps: - - name: Download all the dists - uses: actions/download-artifact@v3 + - uses: actions/checkout@v2 + - name: Set up Python + uses: actions/setup-python@v4 with: - name: python-package-distributions - path: dist/ + python-version: '3.11' + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install setuptools wheel + - name: Build and publish + run: python setup.py sdist bdist_wheel - name: Publish package distributions to PyPI uses: pypa/gh-action-pypi-publish@release/v1 with: From a9e40a698adf0618b230e937b42dbf37e5e06cea Mon Sep 17 00:00:00 2001 From: Henning Kage Date: Thu, 14 Mar 2024 17:30:17 +0100 Subject: [PATCH 051/109] #53: Added test workflow --- .github/workflows/test-publish.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/test-publish.yml b/.github/workflows/test-publish.yml index cf7daf5..33ab032 100644 --- a/.github/workflows/test-publish.yml +++ b/.github/workflows/test-publish.yml @@ -6,7 +6,6 @@ jobs: publish: name: Publish Python 🐍 distribution 📦 to TestPyPI runs-on: ubuntu-latest - needs: build environment: name: testing url: https://test.pypi.org/p/pganonymize/ From 94fbbe83813f50109a78fd20af8e06349acff5b5 Mon Sep 17 00:00:00 2001 From: Henning Kage Date: Thu, 14 Mar 2024 17:34:26 +0100 Subject: [PATCH 052/109] #53: Added test workflow --- .github/workflows/test-publish.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test-publish.yml b/.github/workflows/test-publish.yml index 33ab032..7bd438c 100644 --- a/.github/workflows/test-publish.yml +++ b/.github/workflows/test-publish.yml @@ -4,7 +4,7 @@ on: [push] jobs: publish: - name: Publish Python 🐍 distribution 📦 to TestPyPI + name: Upload release to TestPyPI runs-on: ubuntu-latest environment: name: testing From d5cc13edf18c69f57f86aa9cc37d95cd86f89a2f Mon Sep 17 00:00:00 2001 From: Henning Kage Date: Thu, 14 Mar 2024 17:38:22 +0100 Subject: [PATCH 053/109] #53: Updated publish workflow --- .github/workflows/publish.yml | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index dc4cde6..260977b 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -14,5 +14,16 @@ jobs: permissions: id-token: write # IMPORTANT: mandatory for trusted publishing steps: + - uses: actions/checkout@v2 + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: '3.11' + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install setuptools wheel + - name: Build and publish + run: python setup.py sdist bdist_wheel - name: Publish package distributions to PyPI uses: pypa/gh-action-pypi-publish@release/v1 From f1da0148740fd2ddf8cf2165f49c5e4eecd199fb Mon Sep 17 00:00:00 2001 From: Henning Kage Date: Thu, 14 Mar 2024 18:03:25 +0100 Subject: [PATCH 054/109] #53: Updated publish workflow --- .github/workflows/test-publish.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test-publish.yml b/.github/workflows/test-publish.yml index 7bd438c..0858445 100644 --- a/.github/workflows/test-publish.yml +++ b/.github/workflows/test-publish.yml @@ -1,6 +1,6 @@ name: Test Release -on: [push] +on: workflow_dispatch jobs: publish: From 6d1474ec8595e85aecf548fab9768676d74be125 Mon Sep 17 00:00:00 2001 From: Henning Kage Date: Thu, 14 Mar 2024 18:06:05 +0100 Subject: [PATCH 055/109] #53: Removed test workflow --- .github/workflows/test-publish.yml | 29 ----------------------------- 1 file changed, 29 deletions(-) delete mode 100644 .github/workflows/test-publish.yml diff --git a/.github/workflows/test-publish.yml b/.github/workflows/test-publish.yml deleted file mode 100644 index 0858445..0000000 --- a/.github/workflows/test-publish.yml +++ /dev/null @@ -1,29 +0,0 @@ -name: Test Release - -on: workflow_dispatch - -jobs: - publish: - name: Upload release to TestPyPI - runs-on: ubuntu-latest - environment: - name: testing - url: https://test.pypi.org/p/pganonymize/ - permissions: - id-token: write # IMPORTANT: mandatory for trusted publishing - steps: - - uses: actions/checkout@v2 - - name: Set up Python - uses: actions/setup-python@v4 - with: - python-version: '3.11' - - name: Install dependencies - run: | - python -m pip install --upgrade pip - pip install setuptools wheel - - name: Build and publish - run: python setup.py sdist bdist_wheel - - name: Publish package distributions to PyPI - uses: pypa/gh-action-pypi-publish@release/v1 - with: - repository-url: https://test.pypi.org/legacy/ From 5eee00ca0b4d602a1b076b1dc99f0d9c6187031f Mon Sep 17 00:00:00 2001 From: Henning Kage Date: Thu, 14 Mar 2024 18:13:49 +0100 Subject: [PATCH 056/109] #56: Updated workflows and added changelog --- .github/workflows/publish.yml | 36 +++++++++++----------- .github/workflows/test.yml | 56 +++++++++++++++++------------------ CHANGELOG.rst | 2 ++ 3 files changed, 48 insertions(+), 46 deletions(-) diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 5d5ca5b..260977b 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -6,22 +6,24 @@ on: jobs: publish: + name: Upload release to PyPI runs-on: ubuntu-latest - environment: production + environment: + name: production + url: https://pypi.org/project/pganonymize/ + permissions: + id-token: write # IMPORTANT: mandatory for trusted publishing steps: - - uses: actions/checkout@v2 - - name: Set up Python - uses: actions/setup-python@v4 - with: - python-version: '3.10' - - name: Install dependencies - run: | - python -m pip install --upgrade pip - pip install setuptools wheel twine - - name: Build and publish - env: - TWINE_USERNAME: __token__ - TWINE_PASSWORD: ${{ secrets.PYPI_TOKEN }} - run: | - python setup.py sdist bdist_wheel - twine upload dist/* + - uses: actions/checkout@v2 + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: '3.11' + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install setuptools wheel + - name: Build and publish + run: python setup.py sdist bdist_wheel + - name: Publish package distributions to PyPI + uses: pypa/gh-action-pypi-publish@release/v1 diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 001aeba..a4a99f8 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -3,42 +3,40 @@ name: Test on: [push] jobs: - linting: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - uses: actions/setup-python@v4 - with: - python-version: '3.10' - - name: Install dependencies - run: | - python -m pip install --upgrade pip - pip install tox - - name: Lint - run: tox -e flake8 tests: - needs: linting runs-on: ubuntu-latest strategy: matrix: - python-version: ['2.7', '3.6', '3.7', '3.8', '3.9', '3.10'] - env: - PYTHON: ${{ matrix.python-version }} + image: + - 'python:2.7-buster' + - 'python:3.6-bullseye' + - 'python:3.7-bookworm' + - 'python:3.8-bookworm' + - 'python:3.9-bookworm' + - 'python:3.10-bookworm' + - 'python:3.11-bookworm' + - 'python:3.12-bookworm' + container: + image: ${{ matrix.image }} steps: - uses: actions/checkout@v3 - - name: Set up Python ${{ matrix.python-version }} - id: setup-python - uses: actions/setup-python@v4 - with: - python-version: ${{ matrix.python-version }} - - name: Install dependencies + - name: Set environment variables run: | - python -m pip install --upgrade pip - pip install --use-pep517 tox "coverage<5" - - name: Run tests - run: | - export TOXENV=$(echo "py${{ matrix.python-version }}" | sed 's/\.//g') - tox -- -p no:warnings + echo "PYTHON=$(echo '${{ matrix.image }}' | sed -r 's/^python:([0-9]+)\.([0-9]+).*$/\1.\2/g')" >> $GITHUB_ENV + echo "TOXFACTOR=$(echo '${{ matrix.image }}' | sed -r 's/^python:([0-9]+)\.([0-9]+).*$/py\1\2/g')" >> $GITHUB_ENV + - name: Install importlib-metadata for older Python versions + run: pip install "importlib-metadata<3" + if: >- + matrix.image == 'python:2.7-buster' || + matrix.image == 'python:3.6-bullseye' || + matrix.image == 'python:3.7-bookworm' + - name: Install test utilities + run: pip install "tox<4" tox-factor "coverage<5" + - name: Lint with flake8 + run: tox -e flake8 + if: matrix.image == 'python:3.12-bookworm' + - name: Test via tox + run: tox -- -p no:warnings - name: Generate coverage report run: coverage html if: ${{ success() }} diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 7f73dc5..65d9479 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -4,6 +4,8 @@ Changelog Development ----------- +* `#56 `_: Add Trusted Publisher Management workflow + 0.11.0 (2024-02-29) ------------------- From ec579f472834dc41b29d26591c64be5a3434dd80 Mon Sep 17 00:00:00 2001 From: Henning Kage Date: Thu, 14 Mar 2024 18:15:20 +0100 Subject: [PATCH 057/109] #56: Reverted test workflow changes --- .github/workflows/test.yml | 56 ++++++++++++++++++++------------------ 1 file changed, 29 insertions(+), 27 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index a4a99f8..001aeba 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -3,40 +3,42 @@ name: Test on: [push] jobs: + linting: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-python@v4 + with: + python-version: '3.10' + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install tox + - name: Lint + run: tox -e flake8 tests: + needs: linting runs-on: ubuntu-latest strategy: matrix: - image: - - 'python:2.7-buster' - - 'python:3.6-bullseye' - - 'python:3.7-bookworm' - - 'python:3.8-bookworm' - - 'python:3.9-bookworm' - - 'python:3.10-bookworm' - - 'python:3.11-bookworm' - - 'python:3.12-bookworm' - container: - image: ${{ matrix.image }} + python-version: ['2.7', '3.6', '3.7', '3.8', '3.9', '3.10'] + env: + PYTHON: ${{ matrix.python-version }} steps: - uses: actions/checkout@v3 - - name: Set environment variables + - name: Set up Python ${{ matrix.python-version }} + id: setup-python + uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python-version }} + - name: Install dependencies run: | - echo "PYTHON=$(echo '${{ matrix.image }}' | sed -r 's/^python:([0-9]+)\.([0-9]+).*$/\1.\2/g')" >> $GITHUB_ENV - echo "TOXFACTOR=$(echo '${{ matrix.image }}' | sed -r 's/^python:([0-9]+)\.([0-9]+).*$/py\1\2/g')" >> $GITHUB_ENV - - name: Install importlib-metadata for older Python versions - run: pip install "importlib-metadata<3" - if: >- - matrix.image == 'python:2.7-buster' || - matrix.image == 'python:3.6-bullseye' || - matrix.image == 'python:3.7-bookworm' - - name: Install test utilities - run: pip install "tox<4" tox-factor "coverage<5" - - name: Lint with flake8 - run: tox -e flake8 - if: matrix.image == 'python:3.12-bookworm' - - name: Test via tox - run: tox -- -p no:warnings + python -m pip install --upgrade pip + pip install --use-pep517 tox "coverage<5" + - name: Run tests + run: | + export TOXENV=$(echo "py${{ matrix.python-version }}" | sed 's/\.//g') + tox -- -p no:warnings - name: Generate coverage report run: coverage html if: ${{ success() }} From a875626023ae30d4087fc8d397ca69937d5e3b4a Mon Sep 17 00:00:00 2001 From: Henning Kage Date: Thu, 14 Mar 2024 18:28:14 +0100 Subject: [PATCH 058/109] #53: Removed the new publish workflow changes --- .github/workflows/publish.yml | 36 +++++++++++++++++------------------ 1 file changed, 17 insertions(+), 19 deletions(-) diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 260977b..5d5ca5b 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -6,24 +6,22 @@ on: jobs: publish: - name: Upload release to PyPI runs-on: ubuntu-latest - environment: - name: production - url: https://pypi.org/project/pganonymize/ - permissions: - id-token: write # IMPORTANT: mandatory for trusted publishing + environment: production steps: - - uses: actions/checkout@v2 - - name: Set up Python - uses: actions/setup-python@v4 - with: - python-version: '3.11' - - name: Install dependencies - run: | - python -m pip install --upgrade pip - pip install setuptools wheel - - name: Build and publish - run: python setup.py sdist bdist_wheel - - name: Publish package distributions to PyPI - uses: pypa/gh-action-pypi-publish@release/v1 + - uses: actions/checkout@v2 + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: '3.10' + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install setuptools wheel twine + - name: Build and publish + env: + TWINE_USERNAME: __token__ + TWINE_PASSWORD: ${{ secrets.PYPI_TOKEN }} + run: | + python setup.py sdist bdist_wheel + twine upload dist/* From 37bb118df12a2caf5110493352a3e2615aeb2f1b Mon Sep 17 00:00:00 2001 From: Henning Kage Date: Thu, 14 Mar 2024 18:28:49 +0100 Subject: [PATCH 059/109] #53: Removed the new publish workflow changes --- .github/workflows/publish.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 5d5ca5b..411221c 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -7,7 +7,6 @@ on: jobs: publish: runs-on: ubuntu-latest - environment: production steps: - uses: actions/checkout@v2 - name: Set up Python From f77553ff12f7a2700e9b60e20b08d3aa5b1e07ab Mon Sep 17 00:00:00 2001 From: Henning Kage Date: Thu, 14 Mar 2024 18:30:30 +0100 Subject: [PATCH 060/109] #53: Updated the changelog --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 464e532..61c324c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,7 +2,7 @@ ## Development -* [#53 Python 2.7 tests are failing](https://github.com/rheinwerk-verlag/pganonymize/pull/53) +* [#53 Python 2.7 tests are failing](https://github.com/rheinwerk-verlag/pganonymize/issues/53) * Dropped Python 3.6, 3.7 support * Split development requirements for Python 2 and 3 * Use markdown for the changelog and contributing docs From f9564c68109530e575af5fb864ea0627202a747c Mon Sep 17 00:00:00 2001 From: Henning Kage Date: Fri, 15 Mar 2024 10:50:41 +0100 Subject: [PATCH 061/109] #56: Updated the build step name --- .github/workflows/publish.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 260977b..0ced969 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -23,7 +23,7 @@ jobs: run: | python -m pip install --upgrade pip pip install setuptools wheel - - name: Build and publish + - name: Build run: python setup.py sdist bdist_wheel - name: Publish package distributions to PyPI uses: pypa/gh-action-pypi-publish@release/v1 From c0c537117812ac44a5e06e3cc93185f1d6ed4265 Mon Sep 17 00:00:00 2001 From: Henning Kage Date: Fri, 15 Mar 2024 12:54:51 +0100 Subject: [PATCH 062/109] #53: Updated the checkout and setup-python action versions --- .github/workflows/publish.yml | 4 ++-- .github/workflows/test.yml | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 0ced969..9fb10fa 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -14,9 +14,9 @@ jobs: permissions: id-token: write # IMPORTANT: mandatory for trusted publishing steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - name: Set up Python - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: '3.11' - name: Install dependencies diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 001aeba..e9de181 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -6,8 +6,8 @@ jobs: linting: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 - - uses: actions/setup-python@v4 + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 with: python-version: '3.10' - name: Install dependencies @@ -25,10 +25,10 @@ jobs: env: PYTHON: ${{ matrix.python-version }} steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Set up Python ${{ matrix.python-version }} id: setup-python - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} - name: Install dependencies From a6441b35ba06e6baf34bf898589872eef23057fd Mon Sep 17 00:00:00 2001 From: Henning Kage Date: Fri, 15 Mar 2024 13:45:23 +0100 Subject: [PATCH 063/109] #53: Updated checkout action to v4 --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index a4a99f8..3ee3890 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -19,7 +19,7 @@ jobs: container: image: ${{ matrix.image }} steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Set environment variables run: | echo "PYTHON=$(echo '${{ matrix.image }}' | sed -r 's/^python:([0-9]+)\.([0-9]+).*$/\1.\2/g')" >> $GITHUB_ENV From 5314471b0372c82bd310a5d38eb93d1b6f042648 Mon Sep 17 00:00:00 2001 From: Henning Kage Date: Fri, 15 Mar 2024 13:49:18 +0100 Subject: [PATCH 064/109] #53: Skip installing the default dependencies from the setup.py --- tox.ini | 1 + 1 file changed, 1 insertion(+) diff --git a/tox.ini b/tox.ini index 09189c0..05e5aa8 100644 --- a/tox.ini +++ b/tox.ini @@ -7,6 +7,7 @@ deps = flake8 commands = flake8 {toxinidir}/pganonymize {toxinidir}/tests [testenv] +skip_install = True setenv = PYTHONPATH = {toxinidir} deps = From 751a44a7b95029e317b7c09e8bc51f9e1c6c6af0 Mon Sep 17 00:00:00 2001 From: Henning Kage Date: Fri, 15 Mar 2024 13:54:11 +0100 Subject: [PATCH 065/109] #53: Added missing pgcopy test dependency --- requirements-tox.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements-tox.txt b/requirements-tox.txt index 4aca0a6..fbe30bc 100644 --- a/requirements-tox.txt +++ b/requirements-tox.txt @@ -1,5 +1,6 @@ coverage==4.5.4 mock==3.0.5 +pgcopy pytest-cov==2.8.1 pytest-pythonpath==0.7.3 six==1.12.0 From 8f4eddfca6a7adf270f9c4af09ee1a9890ad9bda Mon Sep 17 00:00:00 2001 From: Henning Kage Date: Fri, 15 Mar 2024 13:56:11 +0100 Subject: [PATCH 066/109] #53: Added missing tqdm requirement --- requirements-tox.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements-tox.txt b/requirements-tox.txt index fbe30bc..c64bfaf 100644 --- a/requirements-tox.txt +++ b/requirements-tox.txt @@ -4,3 +4,4 @@ pgcopy pytest-cov==2.8.1 pytest-pythonpath==0.7.3 six==1.12.0 +tqdm From b91880150e5ebb6c4437f86e2ac6ad73116dddbb Mon Sep 17 00:00:00 2001 From: Henning Kage Date: Fri, 15 Mar 2024 13:58:41 +0100 Subject: [PATCH 067/109] #53: Fixed the pyyaml dependency for the tox.ini --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index 05e5aa8..c9b25e3 100644 --- a/tox.ini +++ b/tox.ini @@ -22,7 +22,7 @@ deps = py{36,37,38,39,310,311,312}: psycopg2-binary>=2.9.2 # pyyaml py27: pyyaml<6 - py{36,37,38,39,310}: pyyaml>=6 + py{36,37,38,39,310,311,312}: pyyaml>=6 # pytest py27: pytest==4.0.2 py27: attrs<19.2 From a4cd4183cff535aa08d425c81911a423c2ea17fe Mon Sep 17 00:00:00 2001 From: Henning Kage Date: Fri, 15 Mar 2024 14:32:01 +0100 Subject: [PATCH 068/109] #53: Fixed the pgcopy requirement for tox --- requirements-tox.txt | 1 - tox.ini | 2 ++ 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/requirements-tox.txt b/requirements-tox.txt index c64bfaf..fa15e9d 100644 --- a/requirements-tox.txt +++ b/requirements-tox.txt @@ -1,6 +1,5 @@ coverage==4.5.4 mock==3.0.5 -pgcopy pytest-cov==2.8.1 pytest-pythonpath==0.7.3 six==1.12.0 diff --git a/tox.ini b/tox.ini index c9b25e3..0b43f13 100644 --- a/tox.ini +++ b/tox.ini @@ -28,6 +28,8 @@ deps = py27: attrs<19.2 py27: more-itertools<8.11 py{36,37,38,39,310,311,312}: pytest==6.2.5 + # pgcopy + pgcopy # Common requirements -r{toxinidir}/requirements-tox.txt commands = From 636244adb0ffcd26e17ae123abd03754c00540c0 Mon Sep 17 00:00:00 2001 From: Henning Kage Date: Fri, 15 Mar 2024 14:35:53 +0100 Subject: [PATCH 069/109] #53: Fixed the pgcopy requirement for tox --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index 0b43f13..cbdd1e4 100644 --- a/tox.ini +++ b/tox.ini @@ -29,7 +29,7 @@ deps = py27: more-itertools<8.11 py{36,37,38,39,310,311,312}: pytest==6.2.5 # pgcopy - pgcopy + py{27,36,37,38,39,310,311,312}: pgcopy # Common requirements -r{toxinidir}/requirements-tox.txt commands = From f7dba50cc085ecdb618600fc39a99d6a8ce86f43 Mon Sep 17 00:00:00 2001 From: Henning Kage Date: Fri, 15 Mar 2024 14:41:43 +0100 Subject: [PATCH 070/109] #53: Fixed the psycopg2 requirement --- tox.ini | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tox.ini b/tox.ini index cbdd1e4..e406cce 100644 --- a/tox.ini +++ b/tox.ini @@ -11,15 +11,15 @@ skip_install = True setenv = PYTHONPATH = {toxinidir} deps = + # psycopg2-binary + py27: psycopg2-binary==2.8.4 + py{36,37,38,39,310,311,312}: psycopg2-binary>=2.9.2 # faker py27: faker<4 py{36,37,38,39,310,311,312}: faker>=9.9.0 # parmap py27: parmap==1.5.2 py{36,37,38,39,310,311,312}: parmap>=1.5.2 - # psycopg2-binary - py27: psycopg2-binary==2.8.4 - py{36,37,38,39,310,311,312}: psycopg2-binary>=2.9.2 # pyyaml py27: pyyaml<6 py{36,37,38,39,310,311,312}: pyyaml>=6 From 0a69a2d82ca7bb92daa359a7d83bb18986ec4522 Mon Sep 17 00:00:00 2001 From: Henning Kage Date: Fri, 15 Mar 2024 15:14:11 +0100 Subject: [PATCH 071/109] #53: Try to install psycopg2 instead of the binary --- .github/workflows/test.yml | 2 ++ tox.ini | 6 +++--- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 3ee3890..f40f334 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -24,6 +24,8 @@ jobs: run: | echo "PYTHON=$(echo '${{ matrix.image }}' | sed -r 's/^python:([0-9]+)\.([0-9]+).*$/\1.\2/g')" >> $GITHUB_ENV echo "TOXFACTOR=$(echo '${{ matrix.image }}' | sed -r 's/^python:([0-9]+)\.([0-9]+).*$/py\1\2/g')" >> $GITHUB_ENV + - name: Install psycopg2 requirements + run: sudo apt-get install python-dev - name: Install importlib-metadata for older Python versions run: pip install "importlib-metadata<3" if: >- diff --git a/tox.ini b/tox.ini index e406cce..5a624c7 100644 --- a/tox.ini +++ b/tox.ini @@ -11,9 +11,6 @@ skip_install = True setenv = PYTHONPATH = {toxinidir} deps = - # psycopg2-binary - py27: psycopg2-binary==2.8.4 - py{36,37,38,39,310,311,312}: psycopg2-binary>=2.9.2 # faker py27: faker<4 py{36,37,38,39,310,311,312}: faker>=9.9.0 @@ -30,6 +27,9 @@ deps = py{36,37,38,39,310,311,312}: pytest==6.2.5 # pgcopy py{27,36,37,38,39,310,311,312}: pgcopy + # psycopg2 + py27: psycopg2 + py{36,37,38,39,310,311,312}: psycopg2>=2.9.2 # Common requirements -r{toxinidir}/requirements-tox.txt commands = From 5868814a757c1d1817228dfea2a544fc37c44201 Mon Sep 17 00:00:00 2001 From: Henning Kage Date: Fri, 15 Mar 2024 15:15:37 +0100 Subject: [PATCH 072/109] #53: Fixed the psycopg2 requirement --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index f40f334..1edd3a1 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -25,7 +25,7 @@ jobs: echo "PYTHON=$(echo '${{ matrix.image }}' | sed -r 's/^python:([0-9]+)\.([0-9]+).*$/\1.\2/g')" >> $GITHUB_ENV echo "TOXFACTOR=$(echo '${{ matrix.image }}' | sed -r 's/^python:([0-9]+)\.([0-9]+).*$/py\1\2/g')" >> $GITHUB_ENV - name: Install psycopg2 requirements - run: sudo apt-get install python-dev + run: apt-get install python-dev - name: Install importlib-metadata for older Python versions run: pip install "importlib-metadata<3" if: >- From 154a3f4b4d0213446edc5f088662f813c5959879 Mon Sep 17 00:00:00 2001 From: Henning Kage Date: Fri, 15 Mar 2024 15:20:18 +0100 Subject: [PATCH 073/109] #53: Fixed the psycopg2 requirement --- .github/workflows/test.yml | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 1edd3a1..39659da 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -24,8 +24,12 @@ jobs: run: | echo "PYTHON=$(echo '${{ matrix.image }}' | sed -r 's/^python:([0-9]+)\.([0-9]+).*$/\1.\2/g')" >> $GITHUB_ENV echo "TOXFACTOR=$(echo '${{ matrix.image }}' | sed -r 's/^python:([0-9]+)\.([0-9]+).*$/py\1\2/g')" >> $GITHUB_ENV - - name: Install psycopg2 requirements - run: apt-get install python-dev + - name: Install psycopg2 requirements (Python 2) + run: apt-get install python2-dev + if: matrix.image == 'python:2.7-buster' + - name: Install psycopg2 requirements (Python 3) + run: apt-get install python3-dev + if: matrix.image != 'python:2.7-buster' - name: Install importlib-metadata for older Python versions run: pip install "importlib-metadata<3" if: >- From 8385e98ab81d6489acb16a99c8c18e476235d2ab Mon Sep 17 00:00:00 2001 From: Henning Kage Date: Fri, 15 Mar 2024 15:29:11 +0100 Subject: [PATCH 074/109] #56: Disabled Python 2.7 and 3.6 temporarily --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index e9de181..ffc85b5 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -21,7 +21,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: ['2.7', '3.6', '3.7', '3.8', '3.9', '3.10'] + python-version: ['3.7', '3.8', '3.9', '3.10'] env: PYTHON: ${{ matrix.python-version }} steps: From 5631e2597d9d10d2dd7d7115b726a563476c7f65 Mon Sep 17 00:00:00 2001 From: Henning Kage Date: Mon, 18 Mar 2024 11:05:29 +0100 Subject: [PATCH 075/109] #53: Update Python header files --- .github/workflows/test.yml | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 39659da..edbaad5 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -24,12 +24,10 @@ jobs: run: | echo "PYTHON=$(echo '${{ matrix.image }}' | sed -r 's/^python:([0-9]+)\.([0-9]+).*$/\1.\2/g')" >> $GITHUB_ENV echo "TOXFACTOR=$(echo '${{ matrix.image }}' | sed -r 's/^python:([0-9]+)\.([0-9]+).*$/py\1\2/g')" >> $GITHUB_ENV - - name: Install psycopg2 requirements (Python 2) - run: apt-get install python2-dev - if: matrix.image == 'python:2.7-buster' - - name: Install psycopg2 requirements (Python 3) - run: apt-get install python3-dev - if: matrix.image != 'python:2.7-buster' + - name: Install psycopg2 requirements + run: | + apt update + apt-get install python2-dev - name: Install importlib-metadata for older Python versions run: pip install "importlib-metadata<3" if: >- From bcaeef0278849c1cce6af0eeeb78f4cf91f90cb2 Mon Sep 17 00:00:00 2001 From: Henning Kage Date: Mon, 18 Mar 2024 11:07:20 +0100 Subject: [PATCH 076/109] #53: Force install --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index edbaad5..592611d 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -27,7 +27,7 @@ jobs: - name: Install psycopg2 requirements run: | apt update - apt-get install python2-dev + apt-get -y install python2-dev - name: Install importlib-metadata for older Python versions run: pip install "importlib-metadata<3" if: >- From 88b4d3c37439e4ec4491fa3fe1e885ee034325b5 Mon Sep 17 00:00:00 2001 From: Henning Kage Date: Mon, 18 Mar 2024 11:08:29 +0100 Subject: [PATCH 077/109] #53: Fixed typo --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 592611d..2c3b30a 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -27,7 +27,7 @@ jobs: - name: Install psycopg2 requirements run: | apt update - apt-get -y install python2-dev + apt-get -y install python-dev - name: Install importlib-metadata for older Python versions run: pip install "importlib-metadata<3" if: >- From 7d74252f23a9d109653aafc8fcd917f2d8c35b71 Mon Sep 17 00:00:00 2001 From: Henning Kage Date: Mon, 18 Mar 2024 11:11:57 +0100 Subject: [PATCH 078/109] #53: Install psycopg2 requirements for Python 2 and 3 --- .github/workflows/test.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 2c3b30a..58f1bd3 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -28,6 +28,12 @@ jobs: run: | apt update apt-get -y install python-dev + if: matrix.image == 'python:2.7-buster' + - name: Install psycopg2 requirements + run: | + apt update + apt-get -y install python3-dev + if: matrix.image != 'python:2.7-buster' - name: Install importlib-metadata for older Python versions run: pip install "importlib-metadata<3" if: >- From 77fb5ac7b0bd42b3b12c8fed95454dbacb550869 Mon Sep 17 00:00:00 2001 From: Henning Kage Date: Mon, 18 Mar 2024 11:15:28 +0100 Subject: [PATCH 079/109] #53: Use Codecov for the coverage report --- .github/workflows/test.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 58f1bd3..72d367a 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -48,12 +48,12 @@ jobs: - name: Test via tox run: tox -- -p no:warnings - name: Generate coverage report - run: coverage html + run: coverage xml if: ${{ success() }} - - name: Upload coverage data - uses: actions/upload-artifact@v3 + - name: Upload coverage report + uses: codecov/codecov-action@v4 with: - name: coverage - path: htmlcov - if-no-files-found: ignore + files: coverage.xml + flags: unittests + token: ${{ secrets.CODECOV_TOKEN }} if: ${{ success() }} From e4ace25e2fe1b566834967dc5016f3fa4798a8c6 Mon Sep 17 00:00:00 2001 From: Henning Kage Date: Mon, 18 Mar 2024 11:29:10 +0100 Subject: [PATCH 080/109] #53: Updated pytest versions --- tox.ini | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index 5a624c7..1427a2c 100644 --- a/tox.ini +++ b/tox.ini @@ -24,7 +24,8 @@ deps = py27: pytest==4.0.2 py27: attrs<19.2 py27: more-itertools<8.11 - py{36,37,38,39,310,311,312}: pytest==6.2.5 + py{36,37}: pytest==7.1.0 + py{38,39,310,311,312}: pytest>=7.4 # pgcopy py{27,36,37,38,39,310,311,312}: pgcopy # psycopg2 From be46c072aa0248f273169a171b0cabdecf088055 Mon Sep 17 00:00:00 2001 From: Henning Kage Date: Mon, 18 Mar 2024 11:31:18 +0100 Subject: [PATCH 081/109] #53: Updated pytest versions --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index 1427a2c..5a6c8e5 100644 --- a/tox.ini +++ b/tox.ini @@ -24,7 +24,7 @@ deps = py27: pytest==4.0.2 py27: attrs<19.2 py27: more-itertools<8.11 - py{36,37}: pytest==7.1.0 + py{36,37}: pytest==6.2.5 py{38,39,310,311,312}: pytest>=7.4 # pgcopy py{27,36,37,38,39,310,311,312}: pgcopy From 90ef262da91a66950f38928968fe7ed10d194928 Mon Sep 17 00:00:00 2001 From: Henning Kage Date: Mon, 18 Mar 2024 11:34:00 +0100 Subject: [PATCH 082/109] #53: Updated pytest versions --- README.rst | 5 ++++- tox.ini | 2 +- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/README.rst b/README.rst index d06acd2..0996dba 100644 --- a/README.rst +++ b/README.rst @@ -8,7 +8,7 @@ anonymization. The tool requires a direct PostgreSQL connection to perform the a .. class:: no-web no-pdf - |python| |license| |pypi| |downloads| |build| |health| + |python| |license| |pypi| |downloads| |build| |codecov| |health| .. image:: docs/_static/demo.gif @@ -180,6 +180,9 @@ After that you can pass a schema file to the container, using Docker volumes, an .. |build| image:: https://github.com/rheinwerk-verlag/postgresql-anonymizer/workflows/Test/badge.svg :target: https://github.com/rheinwerk-verlag/pganonymize/actions +.. |codecov| image:: https://github.com/rheinwerk-verlag/postgresql-anonymizer/workflows/Test/badge.svg + :target: https://codecov.io/gh/rheinwerk-verlag/pganonymize + .. |health| image:: https://snyk.io/advisor/python/pganonymize/badge.svg :target: https://snyk.io/advisor/python/pganonymize :alt: pganonymize diff --git a/tox.ini b/tox.ini index 5a6c8e5..9546c24 100644 --- a/tox.ini +++ b/tox.ini @@ -25,7 +25,7 @@ deps = py27: attrs<19.2 py27: more-itertools<8.11 py{36,37}: pytest==6.2.5 - py{38,39,310,311,312}: pytest>=7.4 + py{38,39,310,311,312}: pytest # pgcopy py{27,36,37,38,39,310,311,312}: pgcopy # psycopg2 From ae347cbf8b0543b8d2ee092d43839fa833af81ca Mon Sep 17 00:00:00 2001 From: Henning Kage Date: Mon, 18 Mar 2024 11:50:05 +0100 Subject: [PATCH 083/109] #53: Updated pytest versions --- requirements-tox.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-tox.txt b/requirements-tox.txt index fa15e9d..ee35e40 100644 --- a/requirements-tox.txt +++ b/requirements-tox.txt @@ -1,6 +1,6 @@ coverage==4.5.4 mock==3.0.5 pytest-cov==2.8.1 -pytest-pythonpath==0.7.3 +pytest-pythonpath==0.7.4 six==1.12.0 tqdm From 0718ee32f4bfaf849d63907961d13d3e09119e47 Mon Sep 17 00:00:00 2001 From: Henning Kage Date: Mon, 18 Mar 2024 12:24:38 +0100 Subject: [PATCH 084/109] #53: Updated six version --- requirements-tox.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-tox.txt b/requirements-tox.txt index ee35e40..d5c4673 100644 --- a/requirements-tox.txt +++ b/requirements-tox.txt @@ -2,5 +2,5 @@ coverage==4.5.4 mock==3.0.5 pytest-cov==2.8.1 pytest-pythonpath==0.7.4 -six==1.12.0 +six==1.16.0 tqdm From 96efcd540f45ed1d1120a12fb1e0c1df9643453a Mon Sep 17 00:00:00 2001 From: Henning Kage Date: Mon, 18 Mar 2024 12:28:32 +0100 Subject: [PATCH 085/109] #53: Updated the badge URLs --- README.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.rst b/README.rst index 0996dba..fdefc4a 100644 --- a/README.rst +++ b/README.rst @@ -177,10 +177,10 @@ After that you can pass a schema file to the container, using Docker volumes, an :target: https://pepy.tech/project/pganonymize :alt: Download count -.. |build| image:: https://github.com/rheinwerk-verlag/postgresql-anonymizer/workflows/Test/badge.svg +.. |build| image:: https://github.com/rheinwerk-verlag/pganonymize/workflows/test.yml/badge.svg :target: https://github.com/rheinwerk-verlag/pganonymize/actions -.. |codecov| image:: https://github.com/rheinwerk-verlag/postgresql-anonymizer/workflows/Test/badge.svg +.. |codecov| image:: https://github.com/rheinwerk-verlag/pganonymize/workflows/test.yml/badge.svg :target: https://codecov.io/gh/rheinwerk-verlag/pganonymize .. |health| image:: https://snyk.io/advisor/python/pganonymize/badge.svg From 7aeaded0d763238839febadda4866fbed52ab12c Mon Sep 17 00:00:00 2001 From: Henning Kage Date: Mon, 18 Mar 2024 12:30:08 +0100 Subject: [PATCH 086/109] #53: Updated the README --- README.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.rst b/README.rst index fdefc4a..a27e8c4 100644 --- a/README.rst +++ b/README.rst @@ -184,6 +184,6 @@ After that you can pass a schema file to the container, using Docker volumes, an :target: https://codecov.io/gh/rheinwerk-verlag/pganonymize .. |health| image:: https://snyk.io/advisor/python/pganonymize/badge.svg - :target: https://snyk.io/advisor/python/pganonymize - :alt: pganonymize + :target: https://snyk.io/advisor/python/pganonymize + :alt: pganonymize From af369a9ab08306b557f7acf79b6b593d680855a4 Mon Sep 17 00:00:00 2001 From: Henning Kage Date: Mon, 18 Mar 2024 12:32:08 +0100 Subject: [PATCH 087/109] #53: Updated the README --- README.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.rst b/README.rst index a27e8c4..bc7c224 100644 --- a/README.rst +++ b/README.rst @@ -177,10 +177,10 @@ After that you can pass a schema file to the container, using Docker volumes, an :target: https://pepy.tech/project/pganonymize :alt: Download count -.. |build| image:: https://github.com/rheinwerk-verlag/pganonymize/workflows/test.yml/badge.svg +.. |build| image:: https://github.com/rheinwerk-verlag/pganonymize/actions/workflows/test.yml/badge.svg :target: https://github.com/rheinwerk-verlag/pganonymize/actions -.. |codecov| image:: https://github.com/rheinwerk-verlag/pganonymize/workflows/test.yml/badge.svg +.. |codecov| image:: https://github.com/rheinwerk-verlag/pganonymize/actions/workflows/test.yml/badge.svg :target: https://codecov.io/gh/rheinwerk-verlag/pganonymize .. |health| image:: https://snyk.io/advisor/python/pganonymize/badge.svg From 12d363a449cb2b6cdf120ee08616bf429f9b5c71 Mon Sep 17 00:00:00 2001 From: Henning Kage Date: Mon, 18 Mar 2024 12:46:10 +0100 Subject: [PATCH 088/109] Updated requirements for the Sphinx documentation and fixed some warnings --- docs/pganonymize.rst | 62 -------------------------------------------- docs/schema.rst | 2 +- requirements.txt | 6 +++-- 3 files changed, 5 insertions(+), 65 deletions(-) delete mode 100644 docs/pganonymize.rst diff --git a/docs/pganonymize.rst b/docs/pganonymize.rst deleted file mode 100644 index f460a56..0000000 --- a/docs/pganonymize.rst +++ /dev/null @@ -1,62 +0,0 @@ -pganonymize package -==================== - -Submodules ----------- - -pganonymize.cli module ------------------------ - -.. automodule:: pganonymize.cli - :members: - :undoc-members: - :show-inheritance: - -pganonymize.constants module ------------------------------ - -.. automodule:: pganonymize.constants - :members: - :undoc-members: - :show-inheritance: - -pganonymize.exceptions module ------------------------------- - -.. automodule:: pganonymize.exceptions - :members: - :undoc-members: - :show-inheritance: - -pganonymize.providers module ------------------------------ - -.. automodule:: pganonymize.providers - :members: - :undoc-members: - :show-inheritance: - -pganonymize.utils module -------------------------- - -.. automodule:: pganonymize.utils - :members: - :undoc-members: - :show-inheritance: - -pganonymize.version module ---------------------------- - -.. automodule:: pganonymize.version - :members: - :undoc-members: - :show-inheritance: - - -Module contents ---------------- - -.. automodule:: pganonymize - :members: - :undoc-members: - :show-inheritance: diff --git a/docs/schema.rst b/docs/schema.rst index 866a44e..b8a4280 100644 --- a/docs/schema.rst +++ b/docs/schema.rst @@ -82,7 +82,7 @@ anonymization process. Each field entry has its own ``provider`` that defines ho - email: provider: name: md5 - append: @localhost + append: "@localhost" ``excludes`` ~~~~~~~~~~~~ diff --git a/requirements.txt b/requirements.txt index 3bd0569..652745b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,8 +1,10 @@ -r requirements-tox.txt flake8==3.9.2 isort==5.10.1 +myst-parser +parmap pgcopy>=1.5.0 pytest==6.2.5 -Sphinx==4.3.0 -sphinx-rtd-theme==1.0.0 +Sphinx==7.2.6 +sphinx-rtd-theme==2.0.0 tqdm>=4.61.1 From eed97aa79299f05a1d1a2551a06576ac69e8ef17 Mon Sep 17 00:00:00 2001 From: Henning Kage Date: Mon, 18 Mar 2024 12:52:49 +0100 Subject: [PATCH 089/109] Updated the changelog --- CHANGELOG.md | 3 --- 1 file changed, 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 547b043..022fcf7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,9 +3,6 @@ ## Development * [#53 Python 2.7 tests are failing](https://github.com/rheinwerk-verlag/pganonymize/issues/53) - * Dropped Python 3.6, 3.7 support - * Split development requirements for Python 2 and 3 - * Use markdown for the changelog and contributing docs * [#56 Add Trusted Publisher Management workflow](https://github.com/rheinwerk-verlag/pganonymize/issues/56) ## 0.11.0 (2024-02-29) From b97b08ebca82716f1ade32c961a3bb9f22ab889d Mon Sep 17 00:00:00 2001 From: Henning Kage Date: Mon, 18 Mar 2024 12:58:10 +0100 Subject: [PATCH 090/109] Updated the README --- README.rst | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/README.rst b/README.rst index bc7c224..72020f2 100644 --- a/README.rst +++ b/README.rst @@ -10,7 +10,8 @@ anonymization. The tool requires a direct PostgreSQL connection to perform the a |python| |license| |pypi| |downloads| |build| |codecov| |health| -.. image:: docs/_static/demo.gif +.. figure:: https://raw.githubusercontent.com/rheinwerk-verlag/planning-poker/main/docs/_static/demo.gif + :width: 100% .. contents:: @@ -180,7 +181,7 @@ After that you can pass a schema file to the container, using Docker volumes, an .. |build| image:: https://github.com/rheinwerk-verlag/pganonymize/actions/workflows/test.yml/badge.svg :target: https://github.com/rheinwerk-verlag/pganonymize/actions -.. |codecov| image:: https://github.com/rheinwerk-verlag/pganonymize/actions/workflows/test.yml/badge.svg +.. |codecov| image:: https://codecov.io/gh/rheinwerk-verlag/pganonymize/branch/main/graph/badge.svg :target: https://codecov.io/gh/rheinwerk-verlag/pganonymize .. |health| image:: https://snyk.io/advisor/python/pganonymize/badge.svg From 521a1c8fce724c03fe82ef5ff0f4263549b082ee Mon Sep 17 00:00:00 2001 From: Henning Kage Date: Mon, 18 Mar 2024 12:59:01 +0100 Subject: [PATCH 091/109] Updated the README --- README.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.rst b/README.rst index 72020f2..a9fb45d 100644 --- a/README.rst +++ b/README.rst @@ -10,7 +10,7 @@ anonymization. The tool requires a direct PostgreSQL connection to perform the a |python| |license| |pypi| |downloads| |build| |codecov| |health| -.. figure:: https://raw.githubusercontent.com/rheinwerk-verlag/planning-poker/main/docs/_static/demo.gif +.. figure:: https://raw.githubusercontent.com/rheinwerk-verlag/pganonymize/main/docs/_static/demo.gif :width: 100% .. contents:: From bc0a51469be1f6e466f0bd9cff5da31ea037ffbb Mon Sep 17 00:00:00 2001 From: Francois Blackburn Date: Wed, 1 May 2024 11:50:28 -0400 Subject: [PATCH 092/109] README: fix old broken link for schema doc --- README.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.rst b/README.rst index a9fb45d..586931c 100644 --- a/README.rst +++ b/README.rst @@ -162,7 +162,7 @@ After that you can pass a schema file to the container, using Docker volumes, an .. _uuid4: https://www.postgresql.org/docs/current/datatype-uuid.html .. _documentation: https://pganonymize.readthedocs.io/en/latest/ -.. _schema documentation: https://python-postgresql-anonymizer.readthedocs.io/en/latest/schema.html +.. _schema documentation: https://pganonymize.readthedocs.io/en/latest/schema.html .. _YAML sample schema: https://github.com/rheinwerk-verlag/pganonymize/blob/master/sample_schema.yml .. |python| image:: https://img.shields.io/pypi/pyversions/pganonymize From 27cf033b5d6d845bd9f96389ac85f4a65d151a7c Mon Sep 17 00:00:00 2001 From: Francois Blackburn Date: Wed, 1 May 2024 11:19:03 -0400 Subject: [PATCH 093/109] add password support to pg_dump command why: to avoid pg_dump to ask about password when you already provide it in the command --- pganonymize/utils.py | 6 +++++- tests/test_cli.py | 2 +- tests/test_utils.py | 6 ++++++ 3 files changed, 12 insertions(+), 2 deletions(-) diff --git a/pganonymize/utils.py b/pganonymize/utils.py index 11be694..fe046ec 100644 --- a/pganonymize/utils.py +++ b/pganonymize/utils.py @@ -271,8 +271,12 @@ def create_database_dump(filename, db_args): :param str filename: Path to the dumpfile that should be created :param dict db_args: A dictionary with database related information """ + env_vars = '' + if db_args.get('password'): + env_vars += 'PGPASSWORD={password}'.format(password=db_args['password']) arguments = '-d {dbname} -U {user} -h {host} -p {port}'.format(**db_args) - cmd = 'pg_dump -Fc -Z 9 {args} -f {filename}'.format( + cmd = '{env_vars} pg_dump -Fc -Z 9 {args} -f {filename}'.format( + env_vars=env_vars, args=arguments, filename=filename ) diff --git a/tests/test_cli.py b/tests/test_cli.py index 616b6b5..d2d592c 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -56,7 +56,7 @@ class TestCli(object): call('UPDATE "auth_user" t SET "first_name" = s."first_name", "last_name" = s."last_name", "email" = s."email" FROM "tmp_auth_user" s WHERE t."id" = s."id"') # noqa ], 1, - [call('pg_dump -Fc -Z 9 -d db -U root -h localhost -p 5432 -f ./dump.sql', shell=True)] + [call('PGPASSWORD=my-cool-password pg_dump -Fc -Z 9 -d db -U root -h localhost -p 5432 -f ./dump.sql', shell=True)] ], ['--list-providers', diff --git a/tests/test_utils.py b/tests/test_utils.py index ac5cdc5..4c8ec7c 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -239,3 +239,9 @@ def test(self, mock_call): create_database_dump('/tmp/dump.gz', {'dbname': 'database', 'user': 'foo', 'host': 'localhost', 'port': 5432}) mock_call.assert_called_once_with('pg_dump -Fc -Z 9 -d database -U foo -h localhost -p 5432 -f /tmp/dump.gz', shell=True) + + @patch('pganonymize.utils.subprocess.call') + def test_with_password(self, mock_call): + create_database_dump('/tmp/dump.gz', {'dbname': 'database', 'user': 'foo', 'host': 'localhost', 'port': 5432, 'password': 'pass'}) + mock_call.assert_called_once_with('PGPASSWORD=pass pg_dump -Fc -Z 9 -d database -U foo -h localhost -p 5432 -f /tmp/dump.gz', + shell=True) From 3bc18b0cd92a9b738a3b45b5d15495ee6ac6ec00 Mon Sep 17 00:00:00 2001 From: Francois Blackburn Date: Wed, 1 May 2024 11:32:11 -0400 Subject: [PATCH 094/109] remove empty space for pg_dump cmd when no password why: there is no functional issue, but it avoid to break an existing test --- pganonymize/utils.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pganonymize/utils.py b/pganonymize/utils.py index fe046ec..bd9d647 100644 --- a/pganonymize/utils.py +++ b/pganonymize/utils.py @@ -275,8 +275,8 @@ def create_database_dump(filename, db_args): if db_args.get('password'): env_vars += 'PGPASSWORD={password}'.format(password=db_args['password']) arguments = '-d {dbname} -U {user} -h {host} -p {port}'.format(**db_args) - cmd = '{env_vars} pg_dump -Fc -Z 9 {args} -f {filename}'.format( - env_vars=env_vars, + cmd = '{env_vars}pg_dump -Fc -Z 9 {args} -f {filename}'.format( + env_vars='{} '.format(env_vars) if env_vars else '', args=arguments, filename=filename ) From ecddb220ab04582966de3cdc5f071f157769b74a Mon Sep 17 00:00:00 2001 From: Henning Kage Date: Mon, 10 Jun 2024 11:09:41 +0200 Subject: [PATCH 095/109] Added changelog entry --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 022fcf7..699899e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,7 @@ ## Development +* [#58 Support pg dump password](https://github.com/rheinwerk-verlag/pganonymize/pull/58) ([fblackburn1](https://github.com/fblackburn1)) * [#53 Python 2.7 tests are failing](https://github.com/rheinwerk-verlag/pganonymize/issues/53) * [#56 Add Trusted Publisher Management workflow](https://github.com/rheinwerk-verlag/pganonymize/issues/56) From 87ba7773eeaecebc3148c91a002832121fe55617 Mon Sep 17 00:00:00 2001 From: Henning Kage Date: Mon, 10 Jun 2024 11:18:08 +0200 Subject: [PATCH 096/109] Added README notes --- README.rst | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/README.rst b/README.rst index 586931c..690ae30 100644 --- a/README.rst +++ b/README.rst @@ -135,6 +135,23 @@ Example call: --dump-file=/tmp/dump.gz \ -v +So that the password for dumping does not have to be entered manually, it can also be entered as an environment var +``PGPASSWORD``: + +.. code-block:: + + $ PGPASSWORD=password pganonymize --schema=myschema.yml \ + --dbname=test_database \ + --user=username \ + --password=mysecret \ + --host=db.host.example.com \ + --dump-file=/tmp/dump.gz \ + -v + +.. warning:: + + Currently only the ``dump-file`` operation supports environment variables. + Docker ~~~~~~ From 05fc7a7108bfad08b7f2017bc82171d48547a981 Mon Sep 17 00:00:00 2001 From: Francois Blackburn Date: Wed, 5 Jun 2024 09:26:36 -0400 Subject: [PATCH 097/109] allow to enable/disable multiprocessing why: multiprocessing is incompatible with fake.unique.* provider --- README.rst | 1 + pganonymize/cli.py | 13 ++++++++++++- pganonymize/utils.py | 36 ++++++++++++++++++++++++++++++------ tests/test_cli.py | 10 +++++----- 4 files changed, 48 insertions(+), 12 deletions(-) diff --git a/README.rst b/README.rst index 690ae30..f2f7d05 100644 --- a/README.rst +++ b/README.rst @@ -87,6 +87,7 @@ Usage --dump-file DUMP_FILE Create a database dump file with the given name --init-sql INIT_SQL SQL to run before starting anonymization + --parallel Data anonymization is done in parallel Despite the database connection values, you will have to define a YAML schema file, that includes all anonymization rules for that database. Take a look at the `schema documentation`_ or the diff --git a/pganonymize/cli.py b/pganonymize/cli.py index 4a85c6b..abfd79d 100644 --- a/pganonymize/cli.py +++ b/pganonymize/cli.py @@ -47,6 +47,12 @@ def get_arg_parser(): default=False) parser.add_argument('--dump-file', help='Create a database dump file with the given name') parser.add_argument('--init-sql', help='SQL to run before starting anonymization', default=False) + parser.add_argument( + '--parallel', + action='store_true', + help='Parallelize anonymization of value. WARNING: `fake.unique.*` providers are not compatible with this option', + default=False, + ) return parser @@ -75,7 +81,12 @@ def main(args): start_time = time.time() truncate_tables(connection) - anonymize_tables(connection, verbose=args.verbose, dry_run=args.dry_run) + anonymize_tables( + connection, + verbose=args.verbose, + dry_run=args.dry_run, + parallel=args.parallel, + ) if not args.dry_run: connection.commit() diff --git a/pganonymize/utils.py b/pganonymize/utils.py index bd9d647..576586d 100644 --- a/pganonymize/utils.py +++ b/pganonymize/utils.py @@ -24,13 +24,14 @@ psycopg2.extras.register_uuid() -def anonymize_tables(connection, verbose=False, dry_run=False): +def anonymize_tables(connection, verbose=False, dry_run=False, parallel=False): """ Anonymize a list of tables according to the schema definition. :param connection: A database connection instance. :param bool verbose: Display logging information and a progress bar. :param bool dry_run: Script is running in dry-run mode, no commit expected. + :param bool parallel: Data anonymization is done in parallel. """ definitions = config.schema.get('tables', []) for definition in definitions: @@ -44,8 +45,19 @@ def anonymize_tables(connection, verbose=False, dry_run=False): primary_key = table_definition.get('primary_key', DEFAULT_PRIMARY_KEY) total_count = get_table_count(connection, table_name, dry_run) chunk_size = table_definition.get('chunk_size', DEFAULT_CHUNK_SIZE) - build_and_then_import_data(connection, table_name, primary_key, columns, excludes, - search, total_count, chunk_size, verbose=verbose, dry_run=dry_run) + build_and_then_import_data( + connection, + table_name, + primary_key, + columns, + excludes, + search, + total_count, + chunk_size, + verbose=verbose, + dry_run=dry_run, + parallel=parallel, + ) end_time = time.time() logging.info('{} anonymization took {:.2f}s'.format(table_name, end_time - start_time)) @@ -63,8 +75,19 @@ def process_row(row, columns, excludes): return row -def build_and_then_import_data(connection, table, primary_key, columns, - excludes, search, total_count, chunk_size, verbose=False, dry_run=False): +def build_and_then_import_data( + connection, + table, + primary_key, + columns, + excludes, + search, + total_count, + chunk_size, + verbose=False, + dry_run=False, + parallel=False, +): """ Select all data from a table and return it together with a list of table columns. @@ -78,6 +101,7 @@ def build_and_then_import_data(connection, table, primary_key, columns, :param int chunk_size: Number of data rows to fetch with the cursor :param bool verbose: Display logging information and a progress bar. :param bool dry_run: Script is running in dry-run mode, no commit expected. + :param bool parallel: Data anonymization is done in parallel. """ column_names = get_column_names(columns) sql_columns = SQL(', ').join([Identifier(column_name) for column_name in [primary_key] + column_names]) @@ -95,7 +119,7 @@ def build_and_then_import_data(connection, table, primary_key, columns, for i in trange(batches, desc="Processing {} batches for {}".format(batches, table), disable=not verbose): records = cursor.fetchmany(size=chunk_size) if records: - data = parmap.map(process_row, records, columns, excludes, pm_pbar=verbose) + data = parmap.map(process_row, records, columns, excludes, pm_pbar=verbose, pm_parallel=parallel) import_data(connection, temp_table, [primary_key] + column_names, filter(None, data)) apply_anonymized_data(connection, temp_table, table, primary_key, columns) diff --git a/tests/test_cli.py b/tests/test_cli.py index d2d592c..1af0a9c 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -17,7 +17,7 @@ class TestCli(object): @pytest.mark.parametrize('cli_args, expected, expected_executes, commit_calls, call_dump', [ ['--host localhost --port 5432 --user root --password my-cool-password --dbname db --schema ./tests/schemes/valid_schema.yml -v --init-sql "set work_mem=\'1GB\'"', # noqa Namespace(verbose=1, list_providers=False, schema='./tests/schemes/valid_schema.yml', dbname='db', user='root', - password='my-cool-password', host='localhost', port='5432', dry_run=False, dump_file=None, init_sql="set work_mem='1GB'"), # noqa + password='my-cool-password', host='localhost', port='5432', dry_run=False, dump_file=None, init_sql="set work_mem='1GB'", parallel=False), # noqa [call("set work_mem='1GB'"), call('TRUNCATE TABLE "django_session"'), call('SELECT COUNT(*) FROM "auth_user"'), @@ -32,7 +32,7 @@ class TestCli(object): ], ['--dry-run --host localhost --port 5432 --user root --password my-cool-password --dbname db --schema ./tests/schemes/valid_schema.yml -v --init-sql "set work_mem=\'1GB\'"', # noqa Namespace(verbose=1, list_providers=False, schema='./tests/schemes/valid_schema.yml', dbname='db', user='root', - password='my-cool-password', host='localhost', port='5432', dry_run=True, dump_file=None, init_sql="set work_mem='1GB'"), # noqa + password='my-cool-password', host='localhost', port='5432', dry_run=True, dump_file=None, init_sql="set work_mem='1GB'", parallel=False), # noqa [call("set work_mem='1GB'"), call('TRUNCATE TABLE "django_session"'), call('SELECT "id", "first_name", "last_name", "email" FROM "auth_user" LIMIT 100'), @@ -44,7 +44,7 @@ class TestCli(object): ], ['--dump-file ./dump.sql --host localhost --port 5432 --user root --password my-cool-password --dbname db --schema ./tests/schemes/valid_schema.yml -v --init-sql "set work_mem=\'1GB\'"', # noqa Namespace(verbose=1, list_providers=False, schema='./tests/schemes/valid_schema.yml', dbname='db', user='root', - password='my-cool-password', host='localhost', port='5432', dry_run=False, dump_file='./dump.sql', init_sql="set work_mem='1GB'"), # noqa + password='my-cool-password', host='localhost', port='5432', dry_run=False, dump_file='./dump.sql', init_sql="set work_mem='1GB'", parallel=False), # noqa [ call("set work_mem='1GB'"), call('TRUNCATE TABLE "django_session"'), @@ -59,9 +59,9 @@ class TestCli(object): [call('PGPASSWORD=my-cool-password pg_dump -Fc -Z 9 -d db -U root -h localhost -p 5432 -f ./dump.sql', shell=True)] ], - ['--list-providers', + ['--list-providers --parallel', Namespace(verbose=None, list_providers=True, schema='schema.yml', dbname=None, user=None, - password='', host='localhost', port='5432', dry_run=False, dump_file=None, init_sql=False), + password='', host='localhost', port='5432', dry_run=False, dump_file=None, init_sql=False, parallel=True), [], 0, [] ] ]) From d99cafa01677b599c4d7d3050977b35ee4f3cba2 Mon Sep 17 00:00:00 2001 From: Francois Blackburn Date: Wed, 5 Jun 2024 09:27:48 -0400 Subject: [PATCH 098/109] validate config before starting anonymization why: parallel and unique are incompatible --- pganonymize/cli.py | 4 +++- pganonymize/config.py | 12 ++++++++++++ pganonymize/exceptions.py | 4 ++++ tests/test_config.py | 40 +++++++++++++++++++++++++++++++++++++-- 4 files changed, 57 insertions(+), 3 deletions(-) diff --git a/pganonymize/cli.py b/pganonymize/cli.py index abfd79d..7e72fa8 100644 --- a/pganonymize/cli.py +++ b/pganonymize/cli.py @@ -6,7 +6,7 @@ import logging import time -from pganonymize.config import config +from pganonymize.config import config, validate_args_with_config from pganonymize.constants import DATABASE_ARGS, DEFAULT_SCHEMA_FILE from pganonymize.providers import provider_registry from pganonymize.utils import anonymize_tables, create_database_dump, get_connection, truncate_tables @@ -71,6 +71,8 @@ def main(args): config.schema_file = args.schema + validate_args_with_config(args, config) + pg_args = get_pg_args(args) connection = get_connection(pg_args) if args.init_sql: diff --git a/pganonymize/config.py b/pganonymize/config.py index 0879996..921bff3 100644 --- a/pganonymize/config.py +++ b/pganonymize/config.py @@ -2,6 +2,7 @@ import re import yaml +from pganonymize.exceptions import InvalidConfiguration class Config(object): @@ -55,3 +56,14 @@ def constructor_env_variables(loader, node): config = Config() + + +def validate_args_with_config(args, config): + definitions = config.schema.get('tables', []) + for definition in definitions: + table_definition = list(definition.values())[0] + columns = table_definition.get('fields', []) + for column in columns: + column_config = list(column.values())[0] + if args.parallel and column_config['provider']['name'].startswith('fake.unique'): + raise InvalidConfiguration('`--parallel` option and `fake.unique.*` providers are incompatible') diff --git a/pganonymize/exceptions.py b/pganonymize/exceptions.py index bd893c4..1f118d9 100644 --- a/pganonymize/exceptions.py +++ b/pganonymize/exceptions.py @@ -20,3 +20,7 @@ class ProviderAlreadyRegistered(PgAnonymizeException): class BadDataFormat(PgAnonymizeException): """Raised if the anonymized data cannot be copied.""" + + +class InvalidConfiguration(PgAnonymizeException): + """Raised if configuration is invalid.""" diff --git a/tests/test_config.py b/tests/test_config.py index 2b6485d..5d48816 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -1,9 +1,10 @@ import os import pytest -from mock import patch +from mock import patch, Mock -from pganonymize.config import load_schema +from pganonymize.config import load_schema, validate_args_with_config +from pganonymize.exceptions import InvalidConfiguration @pytest.mark.parametrize('file, envs, expected', [ @@ -59,3 +60,38 @@ def test_load_schema(file, envs, expected): with patch.dict(os.environ, envs): assert load_schema(file) == expected + + +def test_validate_args_with_config_when_valid(): + args = Mock(parallel=False) + schema = { + 'tables': [ + { + 'table_name': { + 'fields': [ + {'column_name': {'provider': {'name': 'fake.unique.pystr'}}} + ] + } + } + ] + } + config = Mock(schema=schema) + validate_args_with_config(args, config) + + +def test_validate_args_with_config_when_invalid(): + args = Mock(parallel=True) + schema = { + 'tables': [ + { + 'table_name': { + 'fields': [ + {'column_name': {'provider': {'name': 'fake.unique.pystr'}}} + ] + } + } + ] + } + config = Mock(schema=schema) + with pytest.raises(InvalidConfiguration): + validate_args_with_config(args, config) From f864ae3b61372cd87b217aa0e5a99d0a275fb48c Mon Sep 17 00:00:00 2001 From: Francois Blackburn Date: Wed, 5 Jun 2024 15:39:17 -0400 Subject: [PATCH 099/109] fix flake8 issues --- pganonymize/cli.py | 5 ++++- tests/test_cli.py | 4 ++-- tests/test_utils.py | 22 ++++++++++++++++------ 3 files changed, 22 insertions(+), 9 deletions(-) diff --git a/pganonymize/cli.py b/pganonymize/cli.py index 7e72fa8..9757885 100644 --- a/pganonymize/cli.py +++ b/pganonymize/cli.py @@ -50,7 +50,10 @@ def get_arg_parser(): parser.add_argument( '--parallel', action='store_true', - help='Parallelize anonymization of value. WARNING: `fake.unique.*` providers are not compatible with this option', + help=( + 'Parallelize anonymization of value.' + 'WARNING: `fake.unique.*` providers are not compatible with this option' + ), default=False, ) diff --git a/tests/test_cli.py b/tests/test_cli.py index 1af0a9c..3fa701a 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -56,12 +56,12 @@ class TestCli(object): call('UPDATE "auth_user" t SET "first_name" = s."first_name", "last_name" = s."last_name", "email" = s."email" FROM "tmp_auth_user" s WHERE t."id" = s."id"') # noqa ], 1, - [call('PGPASSWORD=my-cool-password pg_dump -Fc -Z 9 -d db -U root -h localhost -p 5432 -f ./dump.sql', shell=True)] + [call('PGPASSWORD=my-cool-password pg_dump -Fc -Z 9 -d db -U root -h localhost -p 5432 -f ./dump.sql', shell=True)] # noqa ], ['--list-providers --parallel', Namespace(verbose=None, list_providers=True, schema='schema.yml', dbname=None, user=None, - password='', host='localhost', port='5432', dry_run=False, dump_file=None, init_sql=False, parallel=True), + password='', host='localhost', port='5432', dry_run=False, dump_file=None, init_sql=False, parallel=True), # noqa [], 0, [] ] ]) diff --git a/tests/test_utils.py b/tests/test_utils.py index 4c8ec7c..27fd592 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -236,12 +236,22 @@ class TestCreateDatabaseDump(object): @patch('pganonymize.utils.subprocess.call') def test(self, mock_call): - create_database_dump('/tmp/dump.gz', {'dbname': 'database', 'user': 'foo', 'host': 'localhost', 'port': 5432}) - mock_call.assert_called_once_with('pg_dump -Fc -Z 9 -d database -U foo -h localhost -p 5432 -f /tmp/dump.gz', - shell=True) + create_database_dump( + '/tmp/dump.gz', + {'dbname': 'database', 'user': 'foo', 'host': 'localhost', 'port': 5432}, + ) + mock_call.assert_called_once_with( + 'pg_dump -Fc -Z 9 -d database -U foo -h localhost -p 5432 -f /tmp/dump.gz', + shell=True, + ) @patch('pganonymize.utils.subprocess.call') def test_with_password(self, mock_call): - create_database_dump('/tmp/dump.gz', {'dbname': 'database', 'user': 'foo', 'host': 'localhost', 'port': 5432, 'password': 'pass'}) - mock_call.assert_called_once_with('PGPASSWORD=pass pg_dump -Fc -Z 9 -d database -U foo -h localhost -p 5432 -f /tmp/dump.gz', - shell=True) + create_database_dump( + '/tmp/dump.gz', + {'dbname': 'database', 'user': 'foo', 'host': 'localhost', 'port': 5432, 'password': 'pass'}, + ) + mock_call.assert_called_once_with( + 'PGPASSWORD=pass pg_dump -Fc -Z 9 -d database -U foo -h localhost -p 5432 -f /tmp/dump.gz', + shell=True, + ) From d72a45b98bbb4a8d957799a78ab84144d32cdc4c Mon Sep 17 00:00:00 2001 From: Henning Kage Date: Fri, 14 Jun 2024 12:23:47 +0200 Subject: [PATCH 100/109] Added CHANGELOG entry --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 699899e..84b6540 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,7 @@ ## Development +* [#62 Allow to configure parallel option](https://github.com/rheinwerk-verlag/pganonymize/pull/62) ([fblackburn1](https://github.com/fblackburn1)) * [#58 Support pg dump password](https://github.com/rheinwerk-verlag/pganonymize/pull/58) ([fblackburn1](https://github.com/fblackburn1)) * [#53 Python 2.7 tests are failing](https://github.com/rheinwerk-verlag/pganonymize/issues/53) * [#56 Add Trusted Publisher Management workflow](https://github.com/rheinwerk-verlag/pganonymize/issues/56) From f30f9266bb95fcc962aac773adf06ad18caa6cc6 Mon Sep 17 00:00:00 2001 From: Francois Blackburn Date: Fri, 7 Jun 2024 14:39:23 -0400 Subject: [PATCH 101/109] use long syntax for pg_dump command why: explicit is always better, by removing confusion on uncommon options --- pganonymize/utils.py | 4 ++-- tests/test_cli.py | 2 +- tests/test_utils.py | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/pganonymize/utils.py b/pganonymize/utils.py index 576586d..1a13aa5 100644 --- a/pganonymize/utils.py +++ b/pganonymize/utils.py @@ -298,8 +298,8 @@ def create_database_dump(filename, db_args): env_vars = '' if db_args.get('password'): env_vars += 'PGPASSWORD={password}'.format(password=db_args['password']) - arguments = '-d {dbname} -U {user} -h {host} -p {port}'.format(**db_args) - cmd = '{env_vars}pg_dump -Fc -Z 9 {args} -f {filename}'.format( + arguments = '--dbname {dbname} --username {user} --host {host} --port {port}'.format(**db_args) + cmd = '{env_vars}pg_dump --format custom --compress 9 {args} --file {filename}'.format( env_vars='{} '.format(env_vars) if env_vars else '', args=arguments, filename=filename diff --git a/tests/test_cli.py b/tests/test_cli.py index 3fa701a..9aab9ba 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -56,7 +56,7 @@ class TestCli(object): call('UPDATE "auth_user" t SET "first_name" = s."first_name", "last_name" = s."last_name", "email" = s."email" FROM "tmp_auth_user" s WHERE t."id" = s."id"') # noqa ], 1, - [call('PGPASSWORD=my-cool-password pg_dump -Fc -Z 9 -d db -U root -h localhost -p 5432 -f ./dump.sql', shell=True)] # noqa + [call('PGPASSWORD=my-cool-password pg_dump --format custom --compress 9 --dbname db --username root --host localhost --port 5432 --file ./dump.sql', shell=True)] # noqa ], ['--list-providers --parallel', diff --git a/tests/test_utils.py b/tests/test_utils.py index 27fd592..4ac1a1a 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -241,7 +241,7 @@ def test(self, mock_call): {'dbname': 'database', 'user': 'foo', 'host': 'localhost', 'port': 5432}, ) mock_call.assert_called_once_with( - 'pg_dump -Fc -Z 9 -d database -U foo -h localhost -p 5432 -f /tmp/dump.gz', + 'pg_dump --format custom --compress 9 --dbname database --username foo --host localhost --port 5432 --file /tmp/dump.gz', shell=True, ) @@ -252,6 +252,6 @@ def test_with_password(self, mock_call): {'dbname': 'database', 'user': 'foo', 'host': 'localhost', 'port': 5432, 'password': 'pass'}, ) mock_call.assert_called_once_with( - 'PGPASSWORD=pass pg_dump -Fc -Z 9 -d database -U foo -h localhost -p 5432 -f /tmp/dump.gz', + 'PGPASSWORD=pass pg_dump --format custom --compress 9 --dbname database --username foo --host localhost --port 5432 --file /tmp/dump.gz', shell=True, ) From 5a10c7495c593a0aff68bb3cd617cb1774a34e92 Mon Sep 17 00:00:00 2001 From: Francois Blackburn Date: Fri, 7 Jun 2024 14:51:05 -0400 Subject: [PATCH 102/109] add dump-options to modify pg_dump command --- README.rst | 4 +++- pganonymize/cli.py | 4 +++- pganonymize/utils.py | 9 +++++---- tests/test_cli.py | 14 +++++++------- tests/test_utils.py | 29 ++++++++++++++++++++--------- 5 files changed, 38 insertions(+), 22 deletions(-) diff --git a/README.rst b/README.rst index f2f7d05..c2427da 100644 --- a/README.rst +++ b/README.rst @@ -85,7 +85,9 @@ Usage --port PORT Port of the database --dry-run Don't commit changes made on the database --dump-file DUMP_FILE - Create a database dump file with the given name + Create a database dump file with the given name + --dump-options DUMP_OPTIONS + Options to pass to the pg_dump command --init-sql INIT_SQL SQL to run before starting anonymization --parallel Data anonymization is done in parallel diff --git a/pganonymize/cli.py b/pganonymize/cli.py index 9757885..9d035b0 100644 --- a/pganonymize/cli.py +++ b/pganonymize/cli.py @@ -46,6 +46,8 @@ def get_arg_parser(): parser.add_argument('--dry-run', action='store_true', help='Don\'t commit changes made on the database', default=False) parser.add_argument('--dump-file', help='Create a database dump file with the given name') + parser.add_argument('--dump-options', help='Options to pass to the pg_dump command', + default='--format custom --compress 9') parser.add_argument('--init-sql', help='SQL to run before starting anonymization', default=False) parser.add_argument( '--parallel', @@ -101,4 +103,4 @@ def main(args): logging.info('Anonymization took {:.2f}s'.format(end_time - start_time)) if args.dump_file: - create_database_dump(args.dump_file, pg_args) + create_database_dump(args.dump_file, pg_args, args.dump_options) diff --git a/pganonymize/utils.py b/pganonymize/utils.py index 1a13aa5..211720e 100644 --- a/pganonymize/utils.py +++ b/pganonymize/utils.py @@ -288,7 +288,7 @@ def truncate_tables(connection): cursor.close() -def create_database_dump(filename, db_args): +def create_database_dump(filename, db_args, dump_args): """ Create a dump file from the current database. @@ -299,10 +299,11 @@ def create_database_dump(filename, db_args): if db_args.get('password'): env_vars += 'PGPASSWORD={password}'.format(password=db_args['password']) arguments = '--dbname {dbname} --username {user} --host {host} --port {port}'.format(**db_args) - cmd = '{env_vars}pg_dump --format custom --compress 9 {args} --file {filename}'.format( + cmd = '{env_vars}pg_dump {dump_args} {db_args} --file {filename}'.format( env_vars='{} '.format(env_vars) if env_vars else '', - args=arguments, - filename=filename + dump_args=dump_args, + db_args=arguments, + filename=filename, ) logging.info('Creating database dump file "%s"', filename) subprocess.call(cmd, shell=True) diff --git a/tests/test_cli.py b/tests/test_cli.py index 9aab9ba..b1c462d 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -17,7 +17,7 @@ class TestCli(object): @pytest.mark.parametrize('cli_args, expected, expected_executes, commit_calls, call_dump', [ ['--host localhost --port 5432 --user root --password my-cool-password --dbname db --schema ./tests/schemes/valid_schema.yml -v --init-sql "set work_mem=\'1GB\'"', # noqa Namespace(verbose=1, list_providers=False, schema='./tests/schemes/valid_schema.yml', dbname='db', user='root', - password='my-cool-password', host='localhost', port='5432', dry_run=False, dump_file=None, init_sql="set work_mem='1GB'", parallel=False), # noqa + password='my-cool-password', host='localhost', port='5432', dry_run=False, dump_file=None, dump_options='--format custom --compress 9', init_sql="set work_mem='1GB'", parallel=False), # noqa [call("set work_mem='1GB'"), call('TRUNCATE TABLE "django_session"'), call('SELECT COUNT(*) FROM "auth_user"'), @@ -32,7 +32,7 @@ class TestCli(object): ], ['--dry-run --host localhost --port 5432 --user root --password my-cool-password --dbname db --schema ./tests/schemes/valid_schema.yml -v --init-sql "set work_mem=\'1GB\'"', # noqa Namespace(verbose=1, list_providers=False, schema='./tests/schemes/valid_schema.yml', dbname='db', user='root', - password='my-cool-password', host='localhost', port='5432', dry_run=True, dump_file=None, init_sql="set work_mem='1GB'", parallel=False), # noqa + password='my-cool-password', host='localhost', port='5432', dry_run=True, dump_file=None, dump_options='--format custom --compress 9', init_sql="set work_mem='1GB'", parallel=False), # noqa [call("set work_mem='1GB'"), call('TRUNCATE TABLE "django_session"'), call('SELECT "id", "first_name", "last_name", "email" FROM "auth_user" LIMIT 100'), @@ -42,9 +42,9 @@ class TestCli(object): ], 0, [] ], - ['--dump-file ./dump.sql --host localhost --port 5432 --user root --password my-cool-password --dbname db --schema ./tests/schemes/valid_schema.yml -v --init-sql "set work_mem=\'1GB\'"', # noqa + ['--dump-file ./dump.sql --dump-options "--format plain" --host localhost --port 5432 --user root --password my-cool-password --dbname db --schema ./tests/schemes/valid_schema.yml -v --init-sql "set work_mem=\'1GB\'"', # noqa Namespace(verbose=1, list_providers=False, schema='./tests/schemes/valid_schema.yml', dbname='db', user='root', - password='my-cool-password', host='localhost', port='5432', dry_run=False, dump_file='./dump.sql', init_sql="set work_mem='1GB'", parallel=False), # noqa + password='my-cool-password', host='localhost', port='5432', dry_run=False, dump_file='./dump.sql', dump_options='--format plain', init_sql="set work_mem='1GB'", parallel=False), # noqa [ call("set work_mem='1GB'"), call('TRUNCATE TABLE "django_session"'), @@ -56,14 +56,14 @@ class TestCli(object): call('UPDATE "auth_user" t SET "first_name" = s."first_name", "last_name" = s."last_name", "email" = s."email" FROM "tmp_auth_user" s WHERE t."id" = s."id"') # noqa ], 1, - [call('PGPASSWORD=my-cool-password pg_dump --format custom --compress 9 --dbname db --username root --host localhost --port 5432 --file ./dump.sql', shell=True)] # noqa + [call('PGPASSWORD=my-cool-password pg_dump --format plain --dbname db --username root --host localhost --port 5432 --file ./dump.sql', shell=True)] # noqa ], ['--list-providers --parallel', Namespace(verbose=None, list_providers=True, schema='schema.yml', dbname=None, user=None, - password='', host='localhost', port='5432', dry_run=False, dump_file=None, init_sql=False, parallel=True), # noqa + password='', host='localhost', port='5432', dry_run=False, dump_file=None, dump_options='--format custom --compress 9', init_sql=False, parallel=True), # noqa [], 0, [] - ] + ], ]) def test_cli_args(self, subprocess, patched_connect, quote_ident, cli_args, expected, expected_executes, commit_calls, call_dump): # noqa arg_parser = get_arg_parser() diff --git a/tests/test_utils.py b/tests/test_utils.py index 4ac1a1a..03f6d4a 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -236,22 +236,33 @@ class TestCreateDatabaseDump(object): @patch('pganonymize.utils.subprocess.call') def test(self, mock_call): - create_database_dump( - '/tmp/dump.gz', - {'dbname': 'database', 'user': 'foo', 'host': 'localhost', 'port': 5432}, - ) + filename = '/tmp/dump.gz' + db_args = {'dbname': 'database', 'user': 'foo', 'host': 'localhost', 'port': 5432} + dump_args = '--format custom --compress 9' + create_database_dump(filename, db_args, dump_args) mock_call.assert_called_once_with( - 'pg_dump --format custom --compress 9 --dbname database --username foo --host localhost --port 5432 --file /tmp/dump.gz', + 'pg_dump --format custom --compress 9 --dbname database --username foo --host localhost --port 5432 --file /tmp/dump.gz', # noqa shell=True, ) @patch('pganonymize.utils.subprocess.call') def test_with_password(self, mock_call): - create_database_dump( - '/tmp/dump.gz', - {'dbname': 'database', 'user': 'foo', 'host': 'localhost', 'port': 5432, 'password': 'pass'}, + filename = '/tmp/dump.gz' + db_args = {'dbname': 'database', 'user': 'foo', 'host': 'localhost', 'port': 5432, 'password': 'pass'} + dump_args = '--format custom --compress 9' + create_database_dump(filename, db_args, dump_args) + mock_call.assert_called_once_with( + 'PGPASSWORD=pass pg_dump --format custom --compress 9 --dbname database --username foo --host localhost --port 5432 --file /tmp/dump.gz', # noqa + shell=True, ) + + @patch('pganonymize.utils.subprocess.call') + def test_with_custom_dump_args(self, mock_call): + filename = '/tmp/dump.gz' + db_args = {'dbname': 'database', 'user': 'foo', 'host': 'localhost', 'port': 5432} + dump_args = '--format plain' + create_database_dump(filename, db_args, dump_args) mock_call.assert_called_once_with( - 'PGPASSWORD=pass pg_dump --format custom --compress 9 --dbname database --username foo --host localhost --port 5432 --file /tmp/dump.gz', + 'pg_dump --format plain --dbname database --username foo --host localhost --port 5432 --file /tmp/dump.gz', # noqa shell=True, ) From f57d0464d894d3c6e45bbe2feecdaf18d68c07c7 Mon Sep 17 00:00:00 2001 From: Henning Kage Date: Tue, 18 Jun 2024 13:15:01 +0200 Subject: [PATCH 103/109] Added CHANGELOG entry --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 84b6540..0ec037e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,7 @@ ## Development +* [#63 Add --dump-options arguments](https://github.com/rheinwerk-verlag/pganonymize/pull/63) ([fblackburn1](https://github.com/fblackburn1)) * [#62 Allow to configure parallel option](https://github.com/rheinwerk-verlag/pganonymize/pull/62) ([fblackburn1](https://github.com/fblackburn1)) * [#58 Support pg dump password](https://github.com/rheinwerk-verlag/pganonymize/pull/58) ([fblackburn1](https://github.com/fblackburn1)) * [#53 Python 2.7 tests are failing](https://github.com/rheinwerk-verlag/pganonymize/issues/53) From fdabc0455ac7dbac4987b039bd03477ab9983943 Mon Sep 17 00:00:00 2001 From: Henning Kage Date: Tue, 18 Jun 2024 14:16:43 +0200 Subject: [PATCH 104/109] Pinned pgcopy for Python 2.7 (version 1.6 drops Python 2.7 support) --- setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.py b/setup.py index 50c63c5..4b80752 100755 --- a/setup.py +++ b/setup.py @@ -46,6 +46,7 @@ def run(self): 'parmap; python_version>="3.6"', 'parmap==1.5.2; python_version<"3.6"', 'pgcopy', + 'pgcopy>=1.5,<1.6; python_version<"3.6"', 'psycopg2', 'psycopg2>=2.8.4,<2.9; python_version<"3.6"', 'pyyaml', From f8139afee84e306d833f5e596ece247510706b28 Mon Sep 17 00:00:00 2001 From: Henning Kage Date: Tue, 18 Jun 2024 14:19:37 +0200 Subject: [PATCH 105/109] Pinned pgcopy for Python 2.7 (version 1.6 drops Python 2.7 support) --- tox.ini | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index 9546c24..135dd6e 100644 --- a/tox.ini +++ b/tox.ini @@ -27,7 +27,8 @@ deps = py{36,37}: pytest==6.2.5 py{38,39,310,311,312}: pytest # pgcopy - py{27,36,37,38,39,310,311,312}: pgcopy + py27: pgcopy==1.5.0 + py{36,37,38,39,310,311,312}: pgcopy # psycopg2 py27: psycopg2 py{36,37,38,39,310,311,312}: psycopg2>=2.9.2 From 06919131f00193799b2caad62f95c4dc0eb73bbb Mon Sep 17 00:00:00 2001 From: Kian-Meng Ang Date: Tue, 18 Jun 2024 23:58:40 +0800 Subject: [PATCH 106/109] Fix typos Found via `typos --hidden --format brief` --- pganonymize/utils.py | 2 +- tests/test_providers.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pganonymize/utils.py b/pganonymize/utils.py index 211720e..b9fca23 100644 --- a/pganonymize/utils.py +++ b/pganonymize/utils.py @@ -379,7 +379,7 @@ def nested_set(dic, path, value, delimiter='.'): Set dictionary value by path. :param dict dic: The source dictionary - :param str path: The path withing dictionary + :param str path: The path within dictionary :param value: The value to be set :param str delimiter: The path delimiter """ diff --git a/tests/test_providers.py b/tests/test_providers.py index 4484316..0a2db9b 100644 --- a/tests/test_providers.py +++ b/tests/test_providers.py @@ -142,7 +142,7 @@ def test_alter_value_with_locale(self, mock_faker): providers.FakeProvider.alter_value('Foo', name='fake.date_of_birth', locale='de_DE') assert mock_faker['de_DE'].date_of_birth.call_count == 1 - def test_alter_value_with_unkown_locale(self): + def test_alter_value_with_unknown_locale(self): with pytest.raises(InvalidProviderArgument): providers.FakeProvider.alter_value('Foo', name='fake.date_of_birth', locale='de_DE') From 184cf5afeb5247c5177a2499b9d28ce51ef4a5f6 Mon Sep 17 00:00:00 2001 From: Henning Kage Date: Wed, 19 Jun 2024 10:06:10 +0200 Subject: [PATCH 107/109] Added CHANGELOG entry --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0ec037e..c6dd6e8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,7 +2,7 @@ ## Development -* [#63 Add --dump-options arguments](https://github.com/rheinwerk-verlag/pganonymize/pull/63) ([fblackburn1](https://github.com/fblackburn1)) +* [#64 Fix typos](https://github.com/rheinwerk-verlag/pganonymize/pull/64) ([kianmeng](https://github.com/kianmeng)) * [#62 Allow to configure parallel option](https://github.com/rheinwerk-verlag/pganonymize/pull/62) ([fblackburn1](https://github.com/fblackburn1)) * [#58 Support pg dump password](https://github.com/rheinwerk-verlag/pganonymize/pull/58) ([fblackburn1](https://github.com/fblackburn1)) * [#53 Python 2.7 tests are failing](https://github.com/rheinwerk-verlag/pganonymize/issues/53) From 6a46a96732f56cdafa23107f51a715f734a95547 Mon Sep 17 00:00:00 2001 From: Henning Kage Date: Thu, 20 Jun 2024 09:09:16 +0200 Subject: [PATCH 108/109] Added missing flake8 config for tox --- tox.ini | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index 135dd6e..4896c70 100644 --- a/tox.ini +++ b/tox.ini @@ -1,6 +1,8 @@ [tox] requires = virtualenv<20.22.0 -envlist = flake8,py27,py36,py37,py38,py39,py310,py311,py312 +envlist = + flake8 + py{27,36,37,38,39,310,311,312} [testenv:flake8] deps = flake8 @@ -39,3 +41,8 @@ commands = passenv = TOXENV GITHUB_* + +[flake8] +exclude = docs,.tox,.git,.eggs +max-line-length = 120 +ignore = E731,W504 From 0130ab12f4888168fb778eb1ee19e6dc848482b8 Mon Sep 17 00:00:00 2001 From: Henning Kage Date: Thu, 18 Jul 2024 08:21:27 +0200 Subject: [PATCH 109/109] Release 0.12.0 --- CHANGELOG.md | 2 ++ pganonymize/version.py | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c6dd6e8..c5eadea 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,8 @@ ## Development +## 0.12.0 (2024-07-18) + * [#64 Fix typos](https://github.com/rheinwerk-verlag/pganonymize/pull/64) ([kianmeng](https://github.com/kianmeng)) * [#62 Allow to configure parallel option](https://github.com/rheinwerk-verlag/pganonymize/pull/62) ([fblackburn1](https://github.com/fblackburn1)) * [#58 Support pg dump password](https://github.com/rheinwerk-verlag/pganonymize/pull/58) ([fblackburn1](https://github.com/fblackburn1)) diff --git a/pganonymize/version.py b/pganonymize/version.py index 4cad2e2..803770e 100644 --- a/pganonymize/version.py +++ b/pganonymize/version.py @@ -1,3 +1,3 @@ # -*- coding: utf-8 -*- -__version__ = '0.11.0' +__version__ = '0.12.0'