From 68aa1b34eae8a2123f005b37b03058cde322a410 Mon Sep 17 00:00:00 2001 From: Adrienne Stilp Date: Fri, 17 Nov 2023 16:20:17 -0800 Subject: [PATCH 1/8] Add requirements.txt file from staging server This is in preparation for using pip-tools -- pip-compile will try to use the versions in this file if possible. We want to start with the same versions that are installed on the staging server. --- requirements/requirements.txt | 168 ++++++++++++++++++++++++++++++++++ 1 file changed, 168 insertions(+) create mode 100644 requirements/requirements.txt diff --git a/requirements/requirements.txt b/requirements/requirements.txt new file mode 100644 index 00000000..0c6a6029 --- /dev/null +++ b/requirements/requirements.txt @@ -0,0 +1,168 @@ +alabaster==0.7.12 +argon2-cffi==21.3.0 +argon2-cffi-bindings==21.2.0 +asgiref==3.5.2 +astroid==2.12.13 +asttokens==2.2.1 +async-timeout==4.0.2 +attrs==22.1.0 +Babel==2.11.0 +backcall==0.2.0 +black==22.3.0 +cachetools==5.2.0 +certifi==2022.5.18.1 +cffi==1.15.0 +cfgv==3.3.1 +chardet==5.1.0 +charset-normalizer==2.0.12 +click==8.1.3 +colorama==0.4.6 +coverage==6.3.2 +crispy-bootstrap5==0.7 +cryptography==37.0.2 +decorator==5.1.1 +defusedxml==0.7.1 +Deprecated==1.2.13 +dill==0.3.6 +distlib==0.3.6 +Django==3.2.19 +django-allauth==0.54.0 +django-anvil-consortium-manager @ git+https://github.com/UW-GAC/django-anvil-consortium-manager.git@b6df2ba44ef96ed779550466e8da36d0db3a9d38 +django-anymail==8.5 +django-autocomplete-light==3.9.4 +django-coverage-plugin==2.0.3 +django-crispy-forms==1.14.0 +django-dbbackup==4.0.1 +django-debug-toolbar==3.4.0 +django-environ==0.8.1 +django-extensions==3.2.1 +django-filter==23.3 +django-js-asset==2.0.0 +django-login-required-middleware==0.8.0 +django-maintenance-mode==0.16.3 +django-model-utils==4.2.0 +django-redis==5.2.0 +django-simple-history==3.1.1 +django-stubs==1.10.1 +django-stubs-ext==0.7.0 +django-tables2==2.4.1 +django-tree-queries==0.13.0 +docutils==0.17.1 +dparse==0.6.2 +exceptiongroup==1.0.4 +executing==1.2.0 +factory-boy==3.2.1 +Faker==15.3.4 +fastobo==0.12.2 +filelock==3.8.2 +flake8==4.0.1 +flake8-isort==4.1.1 +fontawesomefree==6.2.1 +google-auth==2.14.1 +gunicorn==20.1.0 +hiredis==2.0.0 +identify==2.5.11 +idna==3.3 +imagesize==1.4.1 +importlib-metadata==5.2.0 +importlib-resources==5.10.0 +iniconfig==1.1.1 +ipdb==0.13.9 +ipython==8.7.0 +isort==5.11.3 +jedi==0.18.2 +Jinja2==3.1.2 +jsonschema==4.16.0 +lazy-object-proxy==1.8.0 +livereload==2.6.3 +MarkupSafe==2.1.1 +matplotlib-inline==0.1.6 +mccabe==0.6.1 +mypy==0.942 +mypy-extensions==0.4.3 +mysqlclient==2.1.0 +networkx==2.8.8 +nodeenv==1.7.0 +numpy==1.24.2 +oauthlib==3.2.1 +packaging==21.3 +pandas==1.5.2 +parso==0.8.3 +Paste==3.5.2 +pathspec==0.10.3 +pexpect==4.8.0 +pickleshare==0.7.5 +Pillow==9.3.0 +pkg_resources==0.0.0 +pkgutil_resolve_name==1.3.10 +platformdirs==2.6.0 +plotly==5.13.0 +pluggy==1.0.0 +pre-commit==2.18.1 +prompt-toolkit==3.0.36 +pronto==2.5.2 +ptyprocess==0.7.0 +pure-eval==0.2.2 +pyasn1==0.4.8 +pyasn1-modules==0.2.8 +pycodestyle==2.8.0 +pycparser==2.21 +pyflakes==2.4.0 +Pygments==2.13.0 +PyJWT==2.4.0 +pylint==2.15.9 +pylint-django==2.5.3 +pylint-plugin-utils==0.7 +pyparsing==3.0.9 +pyrsistent==0.19.2 +pytest==7.2.0 +pytest-django==4.5.2 +pytest-sugar==0.9.6 +python-dateutil==2.8.2 +python-fsutil==0.6.1 +python-slugify==6.1.2 +python3-openid==3.2.0 +pytz==2022.1 +PyYAML==6.0 +redis==4.5.4 +requests==2.31.0 +requests-oauthlib==1.3.1 +responses==0.21.0 +rsa==4.9 +ruamel.yaml==0.17.21 +ruamel.yaml.clib==0.2.7 +safety==2.1.1 +six==1.16.0 +snowballstemmer==2.2.0 +Sphinx==4.5.0 +sphinx-autobuild==2021.3.14 +sphinxcontrib-applehelp==1.0.2 +sphinxcontrib-devhelp==1.0.2 +sphinxcontrib-htmlhelp==2.0.0 +sphinxcontrib-jsmath==1.0.1 +sphinxcontrib-qthelp==1.0.3 +sphinxcontrib-serializinghtml==1.1.5 +sqlparse==0.4.2 +stack-data==0.6.2 +tablib==3.4.0 +tenacity==8.2.1 +termcolor==2.1.1 +testfixtures==6.18.5 +text-unidecode==1.3 +toml==0.10.2 +tomli==2.0.1 +tomlkit==0.11.6 +tornado==6.2 +traitlets==5.8.0 +types-pytz==2022.7.0.0 +types-PyYAML==6.0.12.2 +types-requests==2.27.25 +types-urllib3==1.26.25.4 +typing_extensions==4.4.0 +urllib3==1.26.9 +virtualenv==20.17.1 +wcwidth==0.2.5 +Werkzeug==2.1.1 +whitenoise==6.0.0 +wrapt==1.14.1 +zipp==3.10.0 From 047545618dc0d8592b1b1ffe47a585f73e934eab Mon Sep 17 00:00:00 2001 From: Adrienne Stilp Date: Fri, 17 Nov 2023 16:27:43 -0800 Subject: [PATCH 2/8] Add requirements.in file and compile new requirements.txt Add package requirements to requirements.in using base.txt and stripping versions. Then compile requirements.txt using pip-compile. Because requirements.txt already had all the staging server package versions pinnned, this step should not have changed any versions - just added packages required by pip-tools or removed packages that were not in the base requirements. --- requirements/requirements.in | 52 ++++++++ requirements/requirements.txt | 245 ++++++++++++++++++++-------------- 2 files changed, 194 insertions(+), 103 deletions(-) create mode 100644 requirements/requirements.in diff --git a/requirements/requirements.in b/requirements/requirements.in new file mode 100644 index 00000000..26bd34ec --- /dev/null +++ b/requirements/requirements.in @@ -0,0 +1,52 @@ +# For dependency management +pip-tools + +pytz # https://github.com/stub42/pytz +python-slugify # https://github.com/un33k/python-slugify +Pillow # https://github.com/python-pillow/Pillow +argon2-cffi # https://github.com/hynek/argon2_cffi +whitenoise # https://github.com/evansd/whitenoise +redis # https://github.com/andymccurdy/redis-py +hiredis # https://github.com/redis/hiredis-py +oauthlib # https://github.com/oauthlib/oauthlib + +# Django +# ------------------------------------------------------------------------------ +django # pyup: < 3.3 # https://www.djangoproject.com/ +django-environ # https://github.com/joke2k/django-environ +django-maintenance-mode # https://github.com/fabiocaccamo/django-maintenance-mode +django-model-utils # https://github.com/jazzband/django-model-utils +django-allauth # https://github.com/pennersr/django-allauth +django-crispy-forms # https://github.com/django-crispy-forms/django-crispy-forms +django-redis # https://github.com/jazzband/django-redis +django-login-required-middleware # https://github.com/CleitonDeLima/django-login-required-middleware +django-dbbackup # https://github.com/jazzband/django-dbbackup +# For time stamped models. +django-extensions # https://github.com/django-extensions/django-extensions + +# anvil_consortium_manager +django-anvil-consortium-manager @ git+https://github.com/UW-GAC/django-anvil-consortium-manager.git@v0.20.1 + +# Simple history - model history tracking +django-simple-history + +# Making HTTP requests to get dbgap study versions. +requests + +# For json schema validation. +jsonschema + +# For tree structures +django-tree-queries + +# For parsing DUO OWL files +pronto + +# mysql +mysqlclient # https://github.com/PyMySQL/mysqlclient + +# pandas for data summaries +pandas + +# for exporting django-tables2 tables to tsv. +tablib diff --git a/requirements/requirements.txt b/requirements/requirements.txt index 0c6a6029..df0eaa00 100644 --- a/requirements/requirements.txt +++ b/requirements/requirements.txt @@ -1,168 +1,207 @@ -alabaster==0.7.12 +# +# This file is autogenerated by pip-compile with Python 3.9 +# by the following command: +# +# pip-compile requirements/requirements.in +# argon2-cffi==21.3.0 + # via -r requirements/requirements.in argon2-cffi-bindings==21.2.0 + # via argon2-cffi asgiref==3.5.2 -astroid==2.12.13 -asttokens==2.2.1 + # via django async-timeout==4.0.2 + # via redis attrs==22.1.0 -Babel==2.11.0 -backcall==0.2.0 -black==22.3.0 + # via jsonschema +build==1.0.3 + # via pip-tools cachetools==5.2.0 + # via google-auth certifi==2022.5.18.1 + # via requests cffi==1.15.0 -cfgv==3.3.1 + # via + # argon2-cffi-bindings + # cryptography chardet==5.1.0 + # via pronto charset-normalizer==2.0.12 + # via requests click==8.1.3 -colorama==0.4.6 -coverage==6.3.2 + # via pip-tools crispy-bootstrap5==0.7 + # via django-anvil-consortium-manager cryptography==37.0.2 -decorator==5.1.1 + # via pyjwt defusedxml==0.7.1 -Deprecated==1.2.13 -dill==0.3.6 -distlib==0.3.6 -Django==3.2.19 + # via python3-openid +django==3.2.19 + # via + # -r requirements/requirements.in + # crispy-bootstrap5 + # django-allauth + # django-anvil-consortium-manager + # django-dbbackup + # django-extensions + # django-filter + # django-model-utils + # django-redis + # django-tables2 django-allauth==0.54.0 -django-anvil-consortium-manager @ git+https://github.com/UW-GAC/django-anvil-consortium-manager.git@b6df2ba44ef96ed779550466e8da36d0db3a9d38 -django-anymail==8.5 + # via -r requirements/requirements.in +django-anvil-consortium-manager @ git+https://github.com/UW-GAC/django-anvil-consortium-manager.git@v0.20.1 + # via -r requirements/requirements.in django-autocomplete-light==3.9.4 -django-coverage-plugin==2.0.3 + # via django-anvil-consortium-manager django-crispy-forms==1.14.0 + # via + # -r requirements/requirements.in + # crispy-bootstrap5 + # django-anvil-consortium-manager django-dbbackup==4.0.1 -django-debug-toolbar==3.4.0 + # via -r requirements/requirements.in django-environ==0.8.1 + # via -r requirements/requirements.in django-extensions==3.2.1 + # via + # -r requirements/requirements.in + # django-anvil-consortium-manager django-filter==23.3 -django-js-asset==2.0.0 + # via django-anvil-consortium-manager django-login-required-middleware==0.8.0 + # via -r requirements/requirements.in django-maintenance-mode==0.16.3 + # via -r requirements/requirements.in django-model-utils==4.2.0 + # via -r requirements/requirements.in django-redis==5.2.0 + # via -r requirements/requirements.in django-simple-history==3.1.1 -django-stubs==1.10.1 -django-stubs-ext==0.7.0 + # via + # -r requirements/requirements.in + # django-anvil-consortium-manager django-tables2==2.4.1 + # via django-anvil-consortium-manager django-tree-queries==0.13.0 -docutils==0.17.1 -dparse==0.6.2 -exceptiongroup==1.0.4 -executing==1.2.0 -factory-boy==3.2.1 -Faker==15.3.4 + # via -r requirements/requirements.in fastobo==0.12.2 -filelock==3.8.2 -flake8==4.0.1 -flake8-isort==4.1.1 + # via pronto fontawesomefree==6.2.1 -google-auth==2.14.1 -gunicorn==20.1.0 + # via django-anvil-consortium-manager +google-auth[requests]==2.14.1 + # via + # django-anvil-consortium-manager + # google-auth hiredis==2.0.0 -identify==2.5.11 + # via -r requirements/requirements.in idna==3.3 -imagesize==1.4.1 + # via requests importlib-metadata==5.2.0 -importlib-resources==5.10.0 -iniconfig==1.1.1 -ipdb==0.13.9 -ipython==8.7.0 -isort==5.11.3 -jedi==0.18.2 -Jinja2==3.1.2 + # via build jsonschema==4.16.0 -lazy-object-proxy==1.8.0 -livereload==2.6.3 -MarkupSafe==2.1.1 -matplotlib-inline==0.1.6 -mccabe==0.6.1 -mypy==0.942 -mypy-extensions==0.4.3 + # via -r requirements/requirements.in mysqlclient==2.1.0 + # via -r requirements/requirements.in networkx==2.8.8 -nodeenv==1.7.0 + # via + # django-anvil-consortium-manager + # pronto numpy==1.24.2 + # via + # django-anvil-consortium-manager + # pandas oauthlib==3.2.1 + # via + # -r requirements/requirements.in + # requests-oauthlib packaging==21.3 + # via build pandas==1.5.2 -parso==0.8.3 -Paste==3.5.2 -pathspec==0.10.3 -pexpect==4.8.0 -pickleshare==0.7.5 -Pillow==9.3.0 -pkg_resources==0.0.0 -pkgutil_resolve_name==1.3.10 -platformdirs==2.6.0 + # via -r requirements/requirements.in +pillow==9.3.0 + # via -r requirements/requirements.in +pip-tools==7.3.0 + # via -r requirements/requirements.in plotly==5.13.0 -pluggy==1.0.0 -pre-commit==2.18.1 -prompt-toolkit==3.0.36 + # via django-anvil-consortium-manager pronto==2.5.2 -ptyprocess==0.7.0 -pure-eval==0.2.2 + # via -r requirements/requirements.in pyasn1==0.4.8 + # via + # pyasn1-modules + # rsa pyasn1-modules==0.2.8 -pycodestyle==2.8.0 + # via google-auth pycparser==2.21 -pyflakes==2.4.0 -Pygments==2.13.0 -PyJWT==2.4.0 -pylint==2.15.9 -pylint-django==2.5.3 -pylint-plugin-utils==0.7 + # via cffi +pyjwt[crypto]==2.4.0 + # via + # django-allauth + # pyjwt pyparsing==3.0.9 + # via packaging +pyproject-hooks==1.0.0 + # via build pyrsistent==0.19.2 -pytest==7.2.0 -pytest-django==4.5.2 -pytest-sugar==0.9.6 + # via jsonschema python-dateutil==2.8.2 + # via + # pandas + # pronto python-fsutil==0.6.1 + # via django-maintenance-mode python-slugify==6.1.2 + # via -r requirements/requirements.in python3-openid==3.2.0 + # via django-allauth pytz==2022.1 -PyYAML==6.0 + # via + # -r requirements/requirements.in + # django + # django-anvil-consortium-manager + # django-dbbackup + # pandas redis==4.5.4 + # via + # -r requirements/requirements.in + # django-redis requests==2.31.0 + # via + # -r requirements/requirements.in + # django-allauth + # requests-oauthlib requests-oauthlib==1.3.1 -responses==0.21.0 + # via django-allauth rsa==4.9 -ruamel.yaml==0.17.21 -ruamel.yaml.clib==0.2.7 -safety==2.1.1 + # via google-auth six==1.16.0 -snowballstemmer==2.2.0 -Sphinx==4.5.0 -sphinx-autobuild==2021.3.14 -sphinxcontrib-applehelp==1.0.2 -sphinxcontrib-devhelp==1.0.2 -sphinxcontrib-htmlhelp==2.0.0 -sphinxcontrib-jsmath==1.0.1 -sphinxcontrib-qthelp==1.0.3 -sphinxcontrib-serializinghtml==1.1.5 + # via + # django-autocomplete-light + # google-auth + # python-dateutil sqlparse==0.4.2 -stack-data==0.6.2 + # via django tablib==3.4.0 + # via -r requirements/requirements.in tenacity==8.2.1 -termcolor==2.1.1 -testfixtures==6.18.5 + # via plotly text-unidecode==1.3 -toml==0.10.2 + # via python-slugify tomli==2.0.1 -tomlkit==0.11.6 -tornado==6.2 -traitlets==5.8.0 -types-pytz==2022.7.0.0 -types-PyYAML==6.0.12.2 -types-requests==2.27.25 -types-urllib3==1.26.25.4 -typing_extensions==4.4.0 + # via + # build + # pip-tools + # pyproject-hooks urllib3==1.26.9 -virtualenv==20.17.1 -wcwidth==0.2.5 -Werkzeug==2.1.1 + # via requests +wheel==0.41.3 + # via pip-tools whitenoise==6.0.0 -wrapt==1.14.1 + # via -r requirements/requirements.in zipp==3.10.0 + # via importlib-metadata + +# The following packages are considered to be unsafe in a requirements file: +# pip +# setuptools From b30bfd3dc6b0f5e85c0c49b02bfc6d57c61bf793 Mon Sep 17 00:00:00 2001 From: Adrienne Stilp Date: Fri, 17 Nov 2023 16:34:14 -0800 Subject: [PATCH 3/8] Add test requirements files Add a test-requirements.in and companion test-requirements.txt file. For test-requirements.txt, start with the pip freeze versions that I have in my current dev setup. --- requirements/test-requirements.in | 14 +++++ requirements/test-requirements.txt | 89 ++++++++++++++++++++++++++++++ 2 files changed, 103 insertions(+) create mode 100644 requirements/test-requirements.in create mode 100644 requirements/test-requirements.txt diff --git a/requirements/test-requirements.in b/requirements/test-requirements.in new file mode 100644 index 00000000..d80fc0df --- /dev/null +++ b/requirements/test-requirements.in @@ -0,0 +1,14 @@ +-c requirements.txt + +pytest # https://github.com/pytest-dev/pytest +pytest-sugar # https://github.com/Frozenball/pytest-sugar +responses # https://github.com/getsentry/responses - for mocking HTTP responses +factory-boy # https://github.com/FactoryBoy/factory_boy +pytest-django # https://github.com/pytest-dev/pytest-django +# For testing migrations +django-test-migrations +# For freezing time in tests +freezegun # https://github.com/spulec/freezegun +# Coverage of django templates +django-coverage-plugin # https://github.com/nedbat/django_coverage_plugin +coverage diff --git a/requirements/test-requirements.txt b/requirements/test-requirements.txt new file mode 100644 index 00000000..ef6ad5a5 --- /dev/null +++ b/requirements/test-requirements.txt @@ -0,0 +1,89 @@ +# +# This file is autogenerated by pip-compile with Python 3.9 +# by the following command: +# +# pip-compile requirements/test-requirements.in +# +attrs==22.1.0 + # via + # -c requirements/requirements.txt + # pytest +certifi==2022.5.18.1 + # via + # -c requirements/requirements.txt + # requests +charset-normalizer==2.0.12 + # via + # -c requirements/requirements.txt + # requests +coverage==6.3.2 + # via + # -r requirements/test-requirements.in + # django-coverage-plugin +django-coverage-plugin==2.0.3 + # via -r requirements/test-requirements.in +django-test-migrations==1.3.0 + # via -r requirements/test-requirements.in +exceptiongroup==1.0.4 + # via pytest +factory-boy==3.2.1 + # via -r requirements/test-requirements.in +faker==13.15.1 + # via factory-boy +freezegun==1.2.2 + # via -r requirements/test-requirements.in +idna==3.3 + # via + # -c requirements/requirements.txt + # requests +iniconfig==1.1.1 + # via pytest +packaging==21.3 + # via + # -c requirements/requirements.txt + # pytest + # pytest-sugar +pluggy==1.0.0 + # via pytest +pyparsing==3.0.9 + # via + # -c requirements/requirements.txt + # packaging +pytest==7.2.0 + # via + # -r requirements/test-requirements.in + # pytest-django + # pytest-sugar +pytest-django==4.5.2 + # via -r requirements/test-requirements.in +pytest-sugar==0.9.6 + # via -r requirements/test-requirements.in +python-dateutil==2.8.2 + # via + # -c requirements/requirements.txt + # faker + # freezegun +requests==2.31.0 + # via + # -c requirements/requirements.txt + # responses +responses==0.21.0 + # via -r requirements/test-requirements.in +six==1.16.0 + # via + # -c requirements/requirements.txt + # django-coverage-plugin + # python-dateutil +termcolor==1.1.0 + # via pytest-sugar +tomli==2.0.1 + # via + # -c requirements/requirements.txt + # pytest +typing-extensions==4.3.0 + # via django-test-migrations +urllib3==1.26.9 + # via + # -c requirements/requirements.txt + # requests + # responses From 805f93ba3d3698caba2238ec112b26ef36cad788 Mon Sep 17 00:00:00 2001 From: Adrienne Stilp Date: Fri, 17 Nov 2023 16:46:01 -0800 Subject: [PATCH 4/8] Add dev-requirements piptools files Add both the dev-requirements.in and compiled dev-requirements.txt files. For dev-requirements.txt, start with the pinned versions that were installed in my dev environment; this way, it's the closest to what I've been developing with. --- requirements/dev-requirements.in | 33 ++++ requirements/dev-requirements.txt | 287 ++++++++++++++++++++++++++++++ 2 files changed, 320 insertions(+) create mode 100644 requirements/dev-requirements.in create mode 100644 requirements/dev-requirements.txt diff --git a/requirements/dev-requirements.in b/requirements/dev-requirements.in new file mode 100644 index 00000000..4367a709 --- /dev/null +++ b/requirements/dev-requirements.in @@ -0,0 +1,33 @@ +-c requirements.txt +-c test-requirements.txt + +Werkzeug # https://github.com/pallets/werkzeug +ipdb # https://github.com/gotcha/ipdb + +# Testing +# ------------------------------------------------------------------------------ +mypy # https://github.com/python/mypy +django-stubs # https://github.com/typeddjango/django-stubs +types-requests + +# Documentation +# ------------------------------------------------------------------------------ +sphinx # https://github.com/sphinx-doc/sphinx +sphinx-autobuild # https://github.com/GaretJax/sphinx-autobuild + +# Code quality +# ------------------------------------------------------------------------------ +flake8 # https://github.com/PyCQA/flake8 +flake8-isort # https://github.com/gforcada/flake8-isort +black # https://github.com/psf/black +pylint-django # https://github.com/PyCQA/pylint-django +pre-commit # https://github.com/pre-commit/pre-commit + +# Security +# ------------------------------------------------------------------------------ +safety # https://github.com/pyupio/safety +setuptools # https://github.com/pypa/setuptools + +# Django +# ------------------------------------------------------------------------------ +django-debug-toolbar # https://github.com/jazzband/django-debug-toolbar diff --git a/requirements/dev-requirements.txt b/requirements/dev-requirements.txt new file mode 100644 index 00000000..4a4113b4 --- /dev/null +++ b/requirements/dev-requirements.txt @@ -0,0 +1,287 @@ +# +# This file is autogenerated by pip-compile with Python 3.9 +# by the following command: +# +# pip-compile requirements/dev-requirements.in +# +alabaster==0.7.12 + # via sphinx +appnope==0.1.3 + # via ipython +asgiref==3.5.2 + # via + # -c requirements/requirements.txt + # django +astroid==2.11.7 + # via pylint +asttokens==2.0.5 + # via stack-data +babel==2.10.3 + # via sphinx +backcall==0.2.0 + # via ipython +black==22.3.0 + # via -r requirements/dev-requirements.in +certifi==2022.5.18.1 + # via + # -c requirements/requirements.txt + # -c requirements/test-requirements.txt + # requests +cfgv==3.3.1 + # via pre-commit +charset-normalizer==2.0.12 + # via + # -c requirements/requirements.txt + # -c requirements/test-requirements.txt + # requests +click==8.1.3 + # via + # -c requirements/requirements.txt + # black + # safety +colorama==0.4.5 + # via sphinx-autobuild +decorator==5.1.1 + # via + # ipdb + # ipython +dill==0.3.5.1 + # via pylint +distlib==0.3.5 + # via virtualenv +django==3.2.19 + # via + # -c requirements/requirements.txt + # django-debug-toolbar + # django-stubs + # django-stubs-ext +django-debug-toolbar==3.4.0 + # via -r requirements/dev-requirements.in +django-stubs==1.10.1 + # via -r requirements/dev-requirements.in +django-stubs-ext==0.5.0 + # via django-stubs +docutils==0.20.1 + # via sphinx +dparse==0.6.2 + # via safety +executing==0.9.1 + # via stack-data +filelock==3.7.1 + # via virtualenv +flake8==4.0.1 + # via + # -r requirements/dev-requirements.in + # flake8-isort +flake8-isort==4.1.1 + # via -r requirements/dev-requirements.in +identify==2.5.2 + # via pre-commit +idna==3.3 + # via + # -c requirements/requirements.txt + # -c requirements/test-requirements.txt + # requests +imagesize==1.4.1 + # via sphinx +importlib-metadata==5.2.0 + # via + # -c requirements/requirements.txt + # sphinx +ipdb==0.13.9 + # via -r requirements/dev-requirements.in +ipython==8.4.0 + # via ipdb +isort==5.10.1 + # via + # flake8-isort + # pylint +jedi==0.18.1 + # via ipython +jinja2==3.1.2 + # via sphinx +lazy-object-proxy==1.7.1 + # via astroid +livereload==2.6.3 + # via sphinx-autobuild +markupsafe==2.1.1 + # via + # jinja2 + # werkzeug +matplotlib-inline==0.1.3 + # via ipython +mccabe==0.6.1 + # via + # flake8 + # pylint +mypy==0.942 + # via + # -r requirements/dev-requirements.in + # django-stubs +mypy-extensions==0.4.3 + # via + # black + # mypy +nodeenv==1.7.0 + # via pre-commit +packaging==21.3 + # via + # -c requirements/requirements.txt + # -c requirements/test-requirements.txt + # dparse + # safety + # sphinx +parso==0.8.3 + # via jedi +pathspec==0.9.0 + # via black +pexpect==4.8.0 + # via ipython +pickleshare==0.7.5 + # via ipython +platformdirs==2.5.2 + # via + # black + # pylint + # virtualenv +pre-commit==2.18.1 + # via -r requirements/dev-requirements.in +prompt-toolkit==3.0.30 + # via ipython +ptyprocess==0.7.0 + # via pexpect +pure-eval==0.2.2 + # via stack-data +pycodestyle==2.8.0 + # via flake8 +pyflakes==2.4.0 + # via flake8 +pygments==2.16.1 + # via + # ipython + # sphinx +pylint==2.14.5 + # via + # pylint-django + # pylint-plugin-utils +pylint-django==2.5.3 + # via -r requirements/dev-requirements.in +pylint-plugin-utils==0.7 + # via pylint-django +pyparsing==3.0.9 + # via + # -c requirements/requirements.txt + # -c requirements/test-requirements.txt + # packaging +pytz==2022.1 + # via + # -c requirements/requirements.txt + # babel + # django +pyyaml==6.0 + # via pre-commit +requests==2.31.0 + # via + # -c requirements/requirements.txt + # -c requirements/test-requirements.txt + # safety + # sphinx +ruamel-yaml==0.17.21 + # via safety +ruamel-yaml-clib==0.2.7 + # via ruamel-yaml +safety==2.3.5 + # via -r requirements/dev-requirements.in +six==1.16.0 + # via + # -c requirements/requirements.txt + # -c requirements/test-requirements.txt + # asttokens + # livereload +snowballstemmer==2.2.0 + # via sphinx +sphinx==7.1.0 + # via + # -r requirements/dev-requirements.in + # sphinx-autobuild +sphinx-autobuild==2021.3.14 + # via -r requirements/dev-requirements.in +sphinxcontrib-applehelp==1.0.2 + # via sphinx +sphinxcontrib-devhelp==1.0.2 + # via sphinx +sphinxcontrib-htmlhelp==2.0.0 + # via sphinx +sphinxcontrib-jsmath==1.0.1 + # via sphinx +sphinxcontrib-qthelp==1.0.3 + # via sphinx +sphinxcontrib-serializinghtml==1.1.5 + # via sphinx +sqlparse==0.4.2 + # via + # -c requirements/requirements.txt + # django + # django-debug-toolbar +stack-data==0.3.0 + # via ipython +testfixtures==6.18.5 + # via flake8-isort +toml==0.10.2 + # via + # dparse + # ipdb + # pre-commit +tomli==2.0.1 + # via + # -c requirements/requirements.txt + # -c requirements/test-requirements.txt + # black + # django-stubs + # mypy + # pylint +tomlkit==0.11.1 + # via pylint +tornado==6.2 + # via livereload +traitlets==5.3.0 + # via + # ipython + # matplotlib-inline +types-pytz==2022.1.2 + # via django-stubs +types-pyyaml==6.0.11 + # via django-stubs +types-requests==2.27.25 + # via -r requirements/dev-requirements.in +types-urllib3==1.26.20 + # via types-requests +typing-extensions==4.3.0 + # via + # -c requirements/test-requirements.txt + # astroid + # black + # django-stubs + # django-stubs-ext + # mypy + # pylint +urllib3==1.26.9 + # via + # -c requirements/requirements.txt + # -c requirements/test-requirements.txt + # requests +virtualenv==20.16.2 + # via pre-commit +wcwidth==0.2.5 + # via prompt-toolkit +werkzeug==2.2.3 + # via -r requirements/dev-requirements.in +wrapt==1.14.1 + # via astroid +zipp==3.10.0 + # via + # -c requirements/requirements.txt + # importlib-metadata + +# The following packages are considered to be unsafe in a requirements file: +# setuptools From 174fc3e8f2dda70c44b27ed4e2dcacd05793d519 Mon Sep 17 00:00:00 2001 From: Adrienne Stilp Date: Fri, 17 Nov 2023 16:47:03 -0800 Subject: [PATCH 5/8] Remove old requirements files --- requirements/base.txt | 49 ------------------------------------- requirements/local.txt | 45 ---------------------------------- requirements/production.txt | 9 ------- 3 files changed, 103 deletions(-) delete mode 100644 requirements/base.txt delete mode 100644 requirements/local.txt delete mode 100644 requirements/production.txt diff --git a/requirements/base.txt b/requirements/base.txt deleted file mode 100644 index 38784723..00000000 --- a/requirements/base.txt +++ /dev/null @@ -1,49 +0,0 @@ -pytz==2022.1 # https://github.com/stub42/pytz -python-slugify==6.1.2 # https://github.com/un33k/python-slugify -Pillow==9.3.0 # https://github.com/python-pillow/Pillow -argon2-cffi==21.3.0 # https://github.com/hynek/argon2_cffi -whitenoise==6.6.0 # https://github.com/evansd/whitenoise -redis==4.5.4 # https://github.com/andymccurdy/redis-py -hiredis==2.0.0 # https://github.com/redis/hiredis-py -oauthlib==3.2.1 # https://github.com/oauthlib/oauthlib - -# Django -# ------------------------------------------------------------------------------ -django==3.2.19 # pyup: < 3.3 # https://www.djangoproject.com/ -django-environ==0.8.1 # https://github.com/joke2k/django-environ -django-maintenance-mode==0.16.3 # https://github.com/fabiocaccamo/django-maintenance-mode -django-model-utils==4.3.1 # https://github.com/jazzband/django-model-utils -django-allauth==0.54.0 # https://github.com/pennersr/django-allauth -django-crispy-forms==1.14.0 # https://github.com/django-crispy-forms/django-crispy-forms -django-redis==5.2.0 # https://github.com/jazzband/django-redis -django-login-required-middleware==0.8.0 # https://github.com/CleitonDeLima/django-login-required-middleware -django-dbbackup==4.0.1 # https://github.com/jazzband/django-dbbackup -# For time stamped models. -django-extensions==3.2.1 # https://github.com/django-extensions/django-extensions - -# anvil_consortium_manager -git+https://github.com/UW-GAC/django-anvil-consortium-manager.git@v0.20.1 - -# Simple history - model history tracking -django-simple-history==3.1.1 - -# Making HTTP requests to get dbgap study versions. -requests==2.31.0 - -# For json schema validation. -jsonschema==4.16.0 - -# For tree structures -django-tree-queries==0.13.0 - -# For parsing DUO OWL files -pronto==2.5.2 - -# mysql -mysqlclient==2.1.0 # https://github.com/PyMySQL/mysqlclient - -# pandas for data summaries -pandas==1.5.2 - -# for exporting django-tables2 tables to tsv. -tablib==3.4.0 diff --git a/requirements/local.txt b/requirements/local.txt deleted file mode 100644 index eab7b053..00000000 --- a/requirements/local.txt +++ /dev/null @@ -1,45 +0,0 @@ --r base.txt - -Werkzeug==2.2.3 # https://github.com/pallets/werkzeug -ipdb==0.13.9 # https://github.com/gotcha/ipdb - -# Testing -# ------------------------------------------------------------------------------ -mypy==0.942 # https://github.com/python/mypy -django-stubs==1.10.1 # https://github.com/typeddjango/django-stubs -types-requests==2.31.0.10 -pytest==7.2.0 # https://github.com/pytest-dev/pytest -pytest-sugar==0.9.6 # https://github.com/Frozenball/pytest-sugar -responses==0.21.0 # https://github.com/getsentry/responses - for mocking HTTP responses -factory-boy==3.2.1 # https://github.com/FactoryBoy/factory_boy - -# Documentation -# ------------------------------------------------------------------------------ -sphinx==4.5.0 # https://github.com/sphinx-doc/sphinx -sphinx-autobuild==2021.3.14 # https://github.com/GaretJax/sphinx-autobuild - -# Code quality -# ------------------------------------------------------------------------------ -flake8==4.0.1 # https://github.com/PyCQA/flake8 -flake8-isort==4.1.1 # https://github.com/gforcada/flake8-isort -coverage==6.3.2 # https://github.com/nedbat/coveragepy -black==22.3.0 # https://github.com/psf/black -pylint-django==2.5.3 # https://github.com/PyCQA/pylint-django -pre-commit==2.18.1 # https://github.com/pre-commit/pre-commit - -# Security -# ------------------------------------------------------------------------------ -safety==2.3.5 # https://github.com/pyupio/safety -setuptools==65.5.1 # https://github.com/pypa/setuptools - -# Django -# ------------------------------------------------------------------------------ -django-debug-toolbar==4.2.0 # https://github.com/jazzband/django-debug-toolbar -django-coverage-plugin==2.0.3 # https://github.com/nedbat/django_coverage_plugin -pytest-django==4.5.2 # https://github.com/pytest-dev/pytest-django - -# Other additions -# ------------------------------------------------------------------------------ -django-test-migrations==1.3.0 -# For setting time when running tests. -freezegun==1.2.2 # https://github.com/spulec/freezegun diff --git a/requirements/production.txt b/requirements/production.txt deleted file mode 100644 index e0cb4320..00000000 --- a/requirements/production.txt +++ /dev/null @@ -1,9 +0,0 @@ -# PRECAUTION: avoid production dependencies that aren't in development - --r base.txt - -gunicorn==20.1.0 # https://github.com/benoitc/gunicorn - -# Django -# ------------------------------------------------------------------------------ -django-anymail==8.5 # https://github.com/anymail/django-anymail From 4b53e93fe2715e7f8a2d6ac45764bd0ab029a174 Mon Sep 17 00:00:00 2001 From: Adrienne Stilp Date: Fri, 17 Nov 2023 16:49:01 -0800 Subject: [PATCH 6/8] Update CI for new requirements files --- .github/workflows/ci.yml | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9eaf0c25..1b859332 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -32,8 +32,8 @@ jobs: python-version: 3.9 cache: pip cache-dependency-path: | - requirements/base.txt - requirements/local.txt + requirements/requirements.txt + requirements/test-requirements.txt # Run all pre-commit hooks on all the files. # Getting only staged files can be tricky in case a new PR is opened @@ -75,13 +75,14 @@ jobs: python-version: ${{ matrix.python-version }} cache: pip cache-dependency-path: | - requirements/base.txt - requirements/local.txt + requirements/requirements.txt + requirements/test-requirements.txt - name: Install Dependencies run: | python -m pip install --upgrade pip - pip install -r requirements/local.txt + pip install pip-tools + pip-sync requirements/requirements.txt requirements/test-requirements.txt - name: Run tests run: coverage run -p -m pytest @@ -108,13 +109,14 @@ jobs: python-version: 3.8 cache: pip cache-dependency-path: | - requirements/base.txt - requirements/local.txt + requirements/requirements.txt + requirements/test-requirements.txt - name: Install Dependencies run: | python -m pip install --upgrade pip - pip install -r requirements/local.txt + pip install pip-tools + pip-sync requirements/requirements.txt requirements/test-requirements.txt - name: Run tests run: coverage run -p -m pytest From 657df6b94130f67b758d59b3df7284c311d9020b Mon Sep 17 00:00:00 2001 From: Adrienne Stilp Date: Fri, 17 Nov 2023 16:54:20 -0800 Subject: [PATCH 7/8] Pin django version to 3.2 in requirements.in --- requirements/requirements.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements/requirements.in b/requirements/requirements.in index 26bd34ec..ed00dac1 100644 --- a/requirements/requirements.in +++ b/requirements/requirements.in @@ -12,7 +12,7 @@ oauthlib # https://github.com/oauthlib/oauthlib # Django # ------------------------------------------------------------------------------ -django # pyup: < 3.3 # https://www.djangoproject.com/ +django>=3.2,<4.0 # pyup: < 3.3 # https://www.djangoproject.com/ django-environ # https://github.com/joke2k/django-environ django-maintenance-mode # https://github.com/fabiocaccamo/django-maintenance-mode django-model-utils # https://github.com/jazzband/django-model-utils From 2a2abee9a308061c87af0861ffde67ebd044ba80 Mon Sep 17 00:00:00 2001 From: Adrienne Stilp Date: Fri, 17 Nov 2023 17:01:05 -0800 Subject: [PATCH 8/8] Update README for pip-tools changes --- README.md | 39 ++++++++++++++++++++++++++------------- 1 file changed, 26 insertions(+), 13 deletions(-) diff --git a/README.md b/README.md index 4f5fcc12..25a74495 100644 --- a/README.md +++ b/README.md @@ -16,7 +16,8 @@ - activate virtualenv: source venv/bin/activate - clone repository - cd into project root dir -- python -m pip install -r requirements/local.txt +- python -m pip install pip-tools +- pip-sync requirements/requirements.txt requirements/test-requirements.txt requirements/dev-requirements.txt - python manage.py migrate - python manage.py createsuperuser - python manage.py runserver_plus @@ -86,19 +87,31 @@ Running tests with pytest and unittest 1. git checkout -b \ (Create and switch to feature branch) -2. make changes, test changes, document changes, commit often -3. run tests: pytest, python manage.py test -4. test coverage: (see above) -5. type checks: mypy primed -6. git add your changes -7. manually run pre-commit if you did not install -8. git commit your changes -9. repeat steps 3-8 -10. merge in any changes to main +2. sync requirements: `pip-sync requirements/requirements.txt test-requirements.txt dev-requirements.txt` +3. make changes, test changes, document changes, commit often +4. run tests: pytest, python manage.py test +5. test coverage: (see above) +6. type checks: mypy primed +7. git add your changes +8. manually run pre-commit if you did not install +9. git commit your changes +10. repeat steps 3-8 +11. merge in any changes to main 1. git checkout main 2. git pull 3. git checkout \ 4. git rebase main -10. git push origin \ -11. review or request review of changes in github -12. submit pull request in github +12. git push origin \ +13. review or request review of changes in github +14. submit pull request in github + + +### Compiling requirements files + +Requriements files must be compiled in the correct order to handle layering. + +``` +pip-compile requirements/requirements.in +pip-compile requirements/test-requirements.in +pip-compile requirements/dev-requirements.in +```