From 89c4cd81cf4f0b25530689ea5f1f64983ba06a79 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 30 Apr 2024 17:15:56 +0000 Subject: [PATCH 01/24] Bump mypy from 1.9.0 to 1.10.0 Bumps [mypy](https://github.com/python/mypy) from 1.9.0 to 1.10.0. - [Changelog](https://github.com/python/mypy/blob/master/CHANGELOG.md) - [Commits](https://github.com/python/mypy/compare/1.9.0...v1.10.0) --- updated-dependencies: - dependency-name: mypy dependency-type: direct:development update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- requirements/dev-requirements.txt | 86 ++++++++++++++----------------- 1 file changed, 40 insertions(+), 46 deletions(-) diff --git a/requirements/dev-requirements.txt b/requirements/dev-requirements.txt index 08c17266..eb16052f 100644 --- a/requirements/dev-requirements.txt +++ b/requirements/dev-requirements.txt @@ -8,7 +8,7 @@ alabaster==0.7.13 # via sphinx asgiref==3.7.2 # via - # -c requirements.txt + # -c requirements/requirements.txt # django asttokens==2.4.1 # via stack-data @@ -18,23 +18,20 @@ backcall==0.2.0 # via ipython backports-zoneinfo==0.2.1 # via - # -c requirements.txt + # -c requirements/requirements.txt # django certifi==2023.11.17 # via - # -c requirements.txt - # -c test-requirements.txt + # -c requirements/requirements.txt + # -c requirements/test-requirements.txt # requests cfgv==3.4.0 # via pre-commit charset-normalizer==3.3.2 # via - # -c requirements.txt - # -c test-requirements.txt + # -c requirements/requirements.txt + # -c requirements/test-requirements.txt # requests -click==8.1.3 - # via - # -c requirements.txt colorama==0.4.6 # via sphinx-autobuild decorator==5.1.1 @@ -45,14 +42,14 @@ distlib==0.3.8 # via virtualenv django==4.2.11 # via - # -c requirements.txt + # -c requirements/requirements.txt # django-debug-toolbar # django-stubs # django-stubs-ext django-debug-toolbar==4.3.0 - # via -r dev-requirements.in + # via -r requirements/dev-requirements.in django-stubs==4.2.7 - # via -r dev-requirements.in + # via -r requirements/dev-requirements.in django-stubs-ext==4.2.7 # via django-stubs docutils==0.20.1 @@ -65,17 +62,17 @@ identify==2.5.34 # via pre-commit idna==3.3 # via - # -c requirements.txt - # -c test-requirements.txt + # -c requirements/requirements.txt + # -c requirements/test-requirements.txt # requests imagesize==1.4.1 # via sphinx importlib-metadata==7.0.0 # via - # -c requirements.txt + # -c requirements/requirements.txt # sphinx ipdb==0.13.13 - # via -r dev-requirements.in + # via -r requirements/dev-requirements.in ipython==8.12.3 # via ipdb jedi==0.19.1 @@ -90,17 +87,16 @@ markupsafe==2.1.5 # werkzeug matplotlib-inline==0.1.6 # via ipython -mypy==1.9.0 - # via -r dev-requirements.in +mypy==1.10.0 + # via -r requirements/dev-requirements.in mypy-extensions==1.0.0 - # via - # mypy + # via mypy nodeenv==1.8.0 # via pre-commit packaging==21.3 # via - # -c requirements.txt - # -c test-requirements.txt + # -c requirements/requirements.txt + # -c requirements/test-requirements.txt # sphinx parso==0.8.3 # via jedi @@ -109,10 +105,9 @@ pexpect==4.9.0 pickleshare==0.7.5 # via ipython platformdirs==4.1.0 - # via - # virtualenv + # via virtualenv pre-commit==3.5.0 - # via -r dev-requirements.in + # via -r requirements/dev-requirements.in prompt-toolkit==3.0.43 # via ipython ptyprocess==0.7.0 @@ -125,38 +120,38 @@ pygments==2.17.2 # sphinx pyparsing==3.1.1 # via - # -c requirements.txt - # -c test-requirements.txt + # -c requirements/requirements.txt + # -c requirements/test-requirements.txt # packaging pytz==2023.3.post1 # via - # -c requirements.txt + # -c requirements/requirements.txt # babel pyyaml==6.0.1 # via - # -c test-requirements.txt + # -c requirements/test-requirements.txt # pre-commit requests==2.31.0 # via - # -c requirements.txt - # -c test-requirements.txt + # -c requirements/requirements.txt + # -c requirements/test-requirements.txt # sphinx ruff==0.4.2 # via -r requirements/dev-requirements.in six==1.16.0 # via - # -c requirements.txt - # -c test-requirements.txt + # -c requirements/requirements.txt + # -c requirements/test-requirements.txt # asttokens # livereload snowballstemmer==2.2.0 # via sphinx sphinx==7.1.2 # via - # -r dev-requirements.in + # -r requirements/dev-requirements.in # sphinx-autobuild sphinx-autobuild==2021.3.14 - # via -r dev-requirements.in + # via -r requirements/dev-requirements.in sphinxcontrib-applehelp==1.0.4 # via sphinx sphinxcontrib-devhelp==1.0.2 @@ -171,15 +166,15 @@ sphinxcontrib-serializinghtml==1.1.5 # via sphinx sqlparse==0.5.0 # via - # -c requirements.txt + # -c requirements/requirements.txt # django # django-debug-toolbar stack-data==0.6.3 # via ipython tomli==2.0.1 # via - # -c requirements.txt - # -c test-requirements.txt + # -c requirements/requirements.txt + # -c requirements/test-requirements.txt # django-stubs # ipdb # mypy @@ -194,21 +189,20 @@ types-pytz==2024.1.0.20240203 types-pyyaml==6.0.12.12 # via django-stubs types-requests==2.31.0.20240406 - # via -r dev-requirements.in + # via -r requirements/dev-requirements.in typing-extensions==4.8.0 # via - # -c requirements.txt - # -c test-requirements.txt + # -c requirements/requirements.txt + # -c requirements/test-requirements.txt # asgiref - # astroid # django-stubs # django-stubs-ext # ipython # mypy urllib3==2.1.0 # via - # -c requirements.txt - # -c test-requirements.txt + # -c requirements/requirements.txt + # -c requirements/test-requirements.txt # requests # types-requests virtualenv==20.25.1 @@ -216,10 +210,10 @@ virtualenv==20.25.1 wcwidth==0.2.13 # via prompt-toolkit werkzeug==3.0.2 - # via -r dev-requirements.in + # via -r requirements/dev-requirements.in zipp==3.17.0 # via - # -c requirements.txt + # -c requirements/requirements.txt # importlib-metadata # The following packages are considered to be unsafe in a requirements file: From 269c49dce4d46f3cb229efed5c6d8827edcf6a4a Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 1 May 2024 18:13:41 +0000 Subject: [PATCH 02/24] Bump freezegun from 1.4.0 to 1.5.0 Bumps [freezegun](https://github.com/spulec/freezegun) from 1.4.0 to 1.5.0. - [Release notes](https://github.com/spulec/freezegun/releases) - [Changelog](https://github.com/spulec/freezegun/blob/master/CHANGELOG) - [Commits](https://github.com/spulec/freezegun/compare/1.4.0...1.5.0) --- updated-dependencies: - dependency-name: freezegun dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- requirements/test-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements/test-requirements.txt b/requirements/test-requirements.txt index f58f6830..790e51cb 100644 --- a/requirements/test-requirements.txt +++ b/requirements/test-requirements.txt @@ -26,7 +26,7 @@ factory-boy==3.3.0 # via -r requirements/test-requirements.in faker==23.2.1 # via factory-boy -freezegun==1.4.0 +freezegun==1.5.0 # via -r requirements/test-requirements.in idna==3.3 # via From ea6650b59d6ce3382282e14f76523dca7afe3b75 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 1 May 2024 18:17:20 +0000 Subject: [PATCH 03/24] Bump coverage from 7.4.4 to 7.5.0 Bumps [coverage](https://github.com/nedbat/coveragepy) from 7.4.4 to 7.5.0. - [Release notes](https://github.com/nedbat/coveragepy/releases) - [Changelog](https://github.com/nedbat/coveragepy/blob/master/CHANGES.rst) - [Commits](https://github.com/nedbat/coveragepy/compare/7.4.4...7.5.0) --- updated-dependencies: - dependency-name: coverage dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- requirements/test-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements/test-requirements.txt b/requirements/test-requirements.txt index f58f6830..a35177e5 100644 --- a/requirements/test-requirements.txt +++ b/requirements/test-requirements.txt @@ -12,7 +12,7 @@ charset-normalizer==3.3.2 # via # -c requirements/requirements.txt # requests -coverage==7.4.4 +coverage==7.5.0 # via # -r requirements/test-requirements.in # django-coverage-plugin From a9310a14713ce5807a217f4b7e91064ff5740c3f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 1 May 2024 18:17:22 +0000 Subject: [PATCH 04/24] Bump pytest from 8.1.1 to 8.2.0 Bumps [pytest](https://github.com/pytest-dev/pytest) from 8.1.1 to 8.2.0. - [Release notes](https://github.com/pytest-dev/pytest/releases) - [Changelog](https://github.com/pytest-dev/pytest/blob/main/CHANGELOG.rst) - [Commits](https://github.com/pytest-dev/pytest/compare/8.1.1...8.2.0) --- updated-dependencies: - dependency-name: pytest dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- requirements/test-requirements.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements/test-requirements.txt b/requirements/test-requirements.txt index f58f6830..b946b34c 100644 --- a/requirements/test-requirements.txt +++ b/requirements/test-requirements.txt @@ -44,13 +44,13 @@ packaging==21.3 # marshmallow # pytest # pytest-sugar -pluggy==1.4.0 +pluggy==1.5.0 # via pytest pyparsing==3.1.1 # via # -c requirements/requirements.txt # packaging -pytest==8.1.1 +pytest==8.2.0 # via # -r requirements/test-requirements.in # pytest-django From 1a23822f7b691a5bae86033126d68091522d07b5 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 1 May 2024 18:17:57 +0000 Subject: [PATCH 05/24] Bump django-tree-queries from 0.18.0 to 0.19.0 Bumps [django-tree-queries](https://github.com/matthiask/django-tree-queries) from 0.18.0 to 0.19.0. - [Changelog](https://github.com/feincms/django-tree-queries/blob/main/CHANGELOG.rst) - [Commits](https://github.com/matthiask/django-tree-queries/compare/0.18...0.19) --- updated-dependencies: - dependency-name: django-tree-queries dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- requirements/requirements.txt | 62 +++++++++++++++++------------------ 1 file changed, 31 insertions(+), 31 deletions(-) diff --git a/requirements/requirements.txt b/requirements/requirements.txt index 847056f7..12b6bc8d 100644 --- a/requirements/requirements.txt +++ b/requirements/requirements.txt @@ -5,7 +5,7 @@ # pip-compile requirements/requirements.in # argon2-cffi==23.1.0 - # via -r requirements.in + # via -r requirements/requirements.in argon2-cffi-bindings==21.2.0 # via argon2-cffi asgiref==3.7.2 @@ -25,7 +25,7 @@ cachetools==5.3.2 # via google-auth certifi==2023.11.17 # via - # -r requirements.in + # -r requirements/requirements.in # requests cffi==1.16.0 # via @@ -39,7 +39,7 @@ click==8.1.3 # via pip-tools crispy-bootstrap5==2024.2 # via - # -r requirements.in + # -r requirements/requirements.in # django-anvil-consortium-manager cryptography==42.0.5 # via pyjwt @@ -47,7 +47,7 @@ defusedxml==0.7.1 # via python3-openid django==4.2.11 # via - # -r requirements.in + # -r requirements/requirements.in # crispy-bootstrap5 # django-allauth # django-anvil-consortium-manager @@ -61,48 +61,48 @@ django==4.2.11 # django-picklefield # django-tables2 django-allauth==0.54.0 - # via -r requirements.in + # via -r requirements/requirements.in django-anvil-consortium-manager @ git+https://github.com/UW-GAC/django-anvil-consortium-manager.git@v0.22 - # via -r requirements.in + # via -r requirements/requirements.in django-autocomplete-light==3.11.0 # via django-anvil-consortium-manager django-constance==3.1.0 - # via -r requirements.in + # via -r requirements/requirements.in django-crispy-forms==2.1 # via - # -r requirements.in + # -r requirements/requirements.in # crispy-bootstrap5 # django-anvil-consortium-manager django-dbbackup==4.1.0 - # via -r requirements.in + # via -r requirements/requirements.in django-environ==0.10.0 - # via -r requirements.in + # via -r requirements/requirements.in django-extensions==3.2.3 # via - # -r requirements.in + # -r requirements/requirements.in # django-anvil-consortium-manager django-filter==23.5 # via django-anvil-consortium-manager django-htmx==1.17.3 - # via -r requirements.in + # via -r requirements/requirements.in django-login-required-middleware==0.9.0 - # via -r requirements.in + # via -r requirements/requirements.in django-maintenance-mode==0.21.1 - # via -r requirements.in + # via -r requirements/requirements.in django-model-utils==4.5.0 - # via -r requirements.in + # via -r requirements/requirements.in django-picklefield==3.2 # via - # -r requirements.in + # -r requirements/requirements.in # django-constance django-simple-history==3.5.0 # via - # -r requirements.in + # -r requirements/requirements.in # django-anvil-consortium-manager django-tables2==2.7.0 # via django-anvil-consortium-manager -django-tree-queries==0.18.0 - # via -r requirements.in +django-tree-queries==0.19.0 + # via -r requirements/requirements.in fastobo==0.12.3 # via pronto fontawesomefree==6.5.1 @@ -118,13 +118,13 @@ importlib-resources==6.1.1 # jsonschema # jsonschema-specifications jsonapi-requests==0.7.0 - # via -r requirements.in + # via -r requirements/requirements.in jsonschema==4.21.1 - # via -r requirements.in + # via -r requirements/requirements.in jsonschema-specifications==2023.12.1 # via jsonschema mysqlclient==2.2.4 - # via -r requirements.in + # via -r requirements/requirements.in networkx==3.1 # via # django-anvil-consortium-manager @@ -135,22 +135,22 @@ numpy==1.24.4 # pandas oauthlib==3.2.2 # via - # -r requirements.in + # -r requirements/requirements.in # requests-oauthlib packaging==21.3 # via # build # plotly pandas==2.0.3 - # via -r requirements.in + # via -r requirements/requirements.in pip-tools==7.4.1 - # via -r requirements.in + # via -r requirements/requirements.in pkgutil-resolve-name==1.3.10 # via jsonschema plotly==5.19.0 # via django-anvil-consortium-manager pronto==2.5.6 - # via -r requirements.in + # via -r requirements/requirements.in pyasn1==0.5.1 # via # pyasn1-modules @@ -186,7 +186,7 @@ referencing==0.33.0 # jsonschema-specifications requests==2.31.0 # via - # -r requirements.in + # -r requirements/requirements.in # django-allauth # django-anvil-consortium-manager # jsonapi-requests @@ -203,10 +203,10 @@ six==1.16.0 # via python-dateutil sqlparse==0.5.0 # via - # -r requirements.in + # -r requirements/requirements.in # django tablib==3.6.1 - # via -r requirements.in + # via -r requirements/requirements.in tenacity==8.2.3 # via # jsonapi-requests @@ -222,12 +222,12 @@ tzdata==2023.4 # via pandas urllib3==2.1.0 # via - # -r requirements.in + # -r requirements/requirements.in # requests wheel==0.42.0 # via pip-tools whitenoise==6.6.0 - # via -r requirements.in + # via -r requirements/requirements.in zipp==3.17.0 # via # importlib-metadata From 488404f3d14a8eed5fef6e020eed2e67b41c1d75 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 1 May 2024 19:20:30 +0000 Subject: [PATCH 06/24] Bump pronto from 2.5.6 to 2.5.7 Bumps [pronto](https://github.com/althonos/pronto) from 2.5.6 to 2.5.7. - [Release notes](https://github.com/althonos/pronto/releases) - [Changelog](https://github.com/althonos/pronto/blob/master/CHANGELOG.md) - [Commits](https://github.com/althonos/pronto/compare/v2.5.6...v2.5.7) --- updated-dependencies: - dependency-name: pronto dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- requirements/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements/requirements.txt b/requirements/requirements.txt index 12b6bc8d..b5423acc 100644 --- a/requirements/requirements.txt +++ b/requirements/requirements.txt @@ -149,7 +149,7 @@ pkgutil-resolve-name==1.3.10 # via jsonschema plotly==5.19.0 # via django-anvil-consortium-manager -pronto==2.5.6 +pronto==2.5.7 # via -r requirements/requirements.in pyasn1==0.5.1 # via From 20e61a32d0af736d7baeedb6509f684444a4af2e Mon Sep 17 00:00:00 2001 From: Adrienne Stilp Date: Wed, 1 May 2024 14:57:55 -0700 Subject: [PATCH 07/24] Combine mariadb and mysql tests into one step Use a strategy similar to the reusable ACM package. It probably won't work yet. --- .github/workflows/ci.yml | 58 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 58 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6ff4177a..0efbcfa6 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -20,6 +20,64 @@ concurrency: jobs: + tests: + runs-on: ubuntu-latest + strategy: + matrix: + python-version: [3.8] + backend: ["sqlite", "mariadb"] + mariadb-version: ["10.4"] + name: "py${{ matrix.python-version }}-${{ matrix.backend }}-${{ matrix.mariadb-version }}" + + services: + mysql: + # Always start mariadb, even if we are testing with the mysql backend. + # Github actions do not allow conditional services yet. + image: mariadb:${{ matrix.mariadb-version }} + ports: + - 3306:3306 + env: + MYSQL_ROOT_PASSWORD: root + MYSQL_DATABSE: test + options: --tmpfs /var/lib/mysql + + steps: + - name: Set environment variables + if: ${{ matrix.backend == 'mysql' }} + run: | + echo "DBNAME=test" >> $GITHUB_ENV + echo "DBUSER=root" >> $GITHUB_ENV + echo "DBPASSWORD=rootpw" >> $GITHUB_ENV + echo "DBHOST=127.0.0.1" >> $GITHUB_ENV + + - name: Checkout Code Repository + uses: actions/checkout@v4 + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + cache: pip + cache-dependency-path: | + requirements/requirements.txt + requirements/test-requirements.txt + + - name: Install Dependencies + run: | + python -m pip install --upgrade pip + pip install pip-tools + pip-sync requirements/requirements.txt requirements/test-requirements.txt + + - name: Collect staticfiles + run: python manage.py collectstatic --noinput --settings=config.settings.test + + - name: Run tests + run: coverage run -p -m pytest + env: + # We can set this to an empty string, since we'll never make an API call. + ANVIL_API_SERVICE_ACCOUNT_FILE: foo + + pytest-mariadb: runs-on: ubuntu-latest strategy: From a1e87f91f34947a391cd2612e800f9c2531ebf78 Mon Sep 17 00:00:00 2001 From: Adrienne Stilp Date: Wed, 1 May 2024 15:08:53 -0700 Subject: [PATCH 08/24] Set required ACM environment variable in CI --- .github/workflows/ci.yml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0efbcfa6..a6e3b71d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -41,6 +41,10 @@ jobs: MYSQL_DATABSE: test options: --tmpfs /var/lib/mysql + env: + # We can set this to an empty string, since we'll never make an API call. + ANVIL_API_SERVICE_ACCOUNT_FILE: foo + steps: - name: Set environment variables if: ${{ matrix.backend == 'mysql' }} @@ -73,9 +77,6 @@ jobs: - name: Run tests run: coverage run -p -m pytest - env: - # We can set this to an empty string, since we'll never make an API call. - ANVIL_API_SERVICE_ACCOUNT_FILE: foo pytest-mariadb: From a63820354be56fe529756b40a095dec9a933e176 Mon Sep 17 00:00:00 2001 From: Adrienne Stilp Date: Wed, 1 May 2024 15:10:43 -0700 Subject: [PATCH 09/24] Remove env setting since it's in the test config file --- .github/workflows/ci.yml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a6e3b71d..c39d8cd6 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -41,10 +41,6 @@ jobs: MYSQL_DATABSE: test options: --tmpfs /var/lib/mysql - env: - # We can set this to an empty string, since we'll never make an API call. - ANVIL_API_SERVICE_ACCOUNT_FILE: foo - steps: - name: Set environment variables if: ${{ matrix.backend == 'mysql' }} From 856de7f854ea047da53f4b1ea960a4cfcea3c063 Mon Sep 17 00:00:00 2001 From: Adrienne Stilp Date: Wed, 1 May 2024 15:15:19 -0700 Subject: [PATCH 10/24] Move ACM env variable setting to production file Having it in the base file was causing issues. The prod file should detect it from the environment variable, while the test file can just set it to a random string. Trying to detect it from the env variable and then set it to a random string was breaking github actions. --- config/settings/base.py | 2 +- config/settings/production.py | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/config/settings/base.py b/config/settings/base.py index 26b16d38..0ce84d43 100644 --- a/config/settings/base.py +++ b/config/settings/base.py @@ -377,7 +377,7 @@ # django-anvil-consortium-manager # ------------------------------------------------------------------------------ -ANVIL_API_SERVICE_ACCOUNT_FILE = env("ANVIL_API_SERVICE_ACCOUNT_FILE") + # Specify workspace adapters. ANVIL_WORKSPACE_ADAPTERS = [ "primed.dbgap.adapters.dbGaPWorkspaceAdapter", diff --git a/config/settings/production.py b/config/settings/production.py index d325df2e..53cbf7aa 100644 --- a/config/settings/production.py +++ b/config/settings/production.py @@ -154,6 +154,7 @@ # Your stuff... # ------------------------------------------------------------------------------ +ANVIL_API_SERVICE_ACCOUNT_FILE = env("ANVIL_API_SERVICE_ACCOUNT_FILE") # ANVIL_DBGAP_APPLICATION_GROUP_PREFIX = "PRIMED_DBGAP_ACCESS" # ANVIL_CDSA_GROUP_PREFIX = "PRIMED_CDSA_ACCESS" ANVIL_DATA_ACCESS_GROUP_PREFIX = "PRIMED" From d3bcea6b42013379592de30524a73940789df9ef Mon Sep 17 00:00:00 2001 From: Adrienne Stilp Date: Wed, 1 May 2024 15:28:02 -0700 Subject: [PATCH 11/24] Set correct environment variable for mariadb (I think) --- .github/workflows/ci.yml | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c39d8cd6..1ffe236b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -45,10 +45,7 @@ jobs: - name: Set environment variables if: ${{ matrix.backend == 'mysql' }} run: | - echo "DBNAME=test" >> $GITHUB_ENV - echo "DBUSER=root" >> $GITHUB_ENV - echo "DBPASSWORD=rootpw" >> $GITHUB_ENV - echo "DBHOST=127.0.0.1" >> $GITHUB_ENV + echo "DATABASE_URL==mysql://root:root@127.0.0.1:3306/mysql" >> $GITHUB_ENV - name: Checkout Code Repository uses: actions/checkout@v4 From 1f027e4193ebcb12df6d268426421f5d66f12bf7 Mon Sep 17 00:00:00 2001 From: Adrienne Stilp Date: Wed, 1 May 2024 15:28:13 -0700 Subject: [PATCH 12/24] Comment out mariadb service to ensure test fails Make sure that the mariadb backend test is actually trying to run with a mariadb backend. Comment out the configuration that sets up the mariadb backend, so the mariadb backend test should fail but the sqlite test should succeed. --- .github/workflows/ci.yml | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1ffe236b..c80b80f8 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -29,17 +29,17 @@ jobs: mariadb-version: ["10.4"] name: "py${{ matrix.python-version }}-${{ matrix.backend }}-${{ matrix.mariadb-version }}" - services: - mysql: - # Always start mariadb, even if we are testing with the mysql backend. - # Github actions do not allow conditional services yet. - image: mariadb:${{ matrix.mariadb-version }} - ports: - - 3306:3306 - env: - MYSQL_ROOT_PASSWORD: root - MYSQL_DATABSE: test - options: --tmpfs /var/lib/mysql + # services: + # mysql: + # # Always start mariadb, even if we are testing with the mysql backend. + # # Github actions do not allow conditional services yet. + # image: mariadb:${{ matrix.mariadb-version }} + # ports: + # - 3306:3306 + # env: + # MYSQL_ROOT_PASSWORD: rootpw + # MYSQL_DATABSE: test + # options: --tmpfs /var/lib/mysql steps: - name: Set environment variables From 35dd7b0a13d248149b241d37080ea946b0cbee47 Mon Sep 17 00:00:00 2001 From: Adrienne Stilp Date: Wed, 1 May 2024 15:31:11 -0700 Subject: [PATCH 13/24] Fix if statement when setting env variables This sets the environment variable for mariadb properly. --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c80b80f8..a2cd9517 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -43,7 +43,7 @@ jobs: steps: - name: Set environment variables - if: ${{ matrix.backend == 'mysql' }} + if: ${{ matrix.backend == 'mariadb' }} run: | echo "DATABASE_URL==mysql://root:root@127.0.0.1:3306/mysql" >> $GITHUB_ENV From 30d8a2c79c04f44a4c9219a2d0716aecad44b9b0 Mon Sep 17 00:00:00 2001 From: Adrienne Stilp Date: Wed, 1 May 2024 15:35:37 -0700 Subject: [PATCH 14/24] Restart mariadb service in CI --- .github/workflows/ci.yml | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a2cd9517..58da63c6 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -29,17 +29,17 @@ jobs: mariadb-version: ["10.4"] name: "py${{ matrix.python-version }}-${{ matrix.backend }}-${{ matrix.mariadb-version }}" - # services: - # mysql: - # # Always start mariadb, even if we are testing with the mysql backend. - # # Github actions do not allow conditional services yet. - # image: mariadb:${{ matrix.mariadb-version }} - # ports: - # - 3306:3306 - # env: - # MYSQL_ROOT_PASSWORD: rootpw - # MYSQL_DATABSE: test - # options: --tmpfs /var/lib/mysql + services: + mysql: + # Always start mariadb, even if we are testing with the mysql backend. + # Github actions do not allow conditional services yet. + image: mariadb:${{ matrix.mariadb-version }} + ports: + - 3306:3306 + env: + MYSQL_ROOT_PASSWORD: rootpw + MYSQL_DATABSE: test + options: --tmpfs /var/lib/mysql steps: - name: Set environment variables From 60a82d1a546132ad1e6338c6444a3a4449a8d41a Mon Sep 17 00:00:00 2001 From: Adrienne Stilp Date: Wed, 1 May 2024 15:42:16 -0700 Subject: [PATCH 15/24] Fix database root password and name env variable setting --- .github/workflows/ci.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 58da63c6..8edaa7cc 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -37,8 +37,7 @@ jobs: ports: - 3306:3306 env: - MYSQL_ROOT_PASSWORD: rootpw - MYSQL_DATABSE: test + MYSQL_ROOT_PASSWORD: root options: --tmpfs /var/lib/mysql steps: From 20f1531f050ab70a13a4baab1f411152b2b694d2 Mon Sep 17 00:00:00 2001 From: Adrienne Stilp Date: Wed, 1 May 2024 15:59:40 -0700 Subject: [PATCH 16/24] Try conditionally setting DATABSE_URL env variable in job.tests.env It appears that github was possibly masking the root:root part of the database url. --- .github/workflows/ci.yml | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8edaa7cc..cace9d38 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -40,11 +40,10 @@ jobs: MYSQL_ROOT_PASSWORD: root options: --tmpfs /var/lib/mysql + env: + DATABASE_URL: ${{ matrix.backend == 'sqlite' && 'sqlite:///db.sqlite3' || 'mysql://root:root@127.0.0.1:3306/mysql' }} + steps: - - name: Set environment variables - if: ${{ matrix.backend == 'mariadb' }} - run: | - echo "DATABASE_URL==mysql://root:root@127.0.0.1:3306/mysql" >> $GITHUB_ENV - name: Checkout Code Repository uses: actions/checkout@v4 From 7d07d61b857d759c145790ea15f63370a2ff9e29 Mon Sep 17 00:00:00 2001 From: Adrienne Stilp Date: Wed, 1 May 2024 16:02:31 -0700 Subject: [PATCH 17/24] Comment out mysql service to check that test fails --- .github/workflows/ci.yml | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index cace9d38..bd509608 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -29,16 +29,16 @@ jobs: mariadb-version: ["10.4"] name: "py${{ matrix.python-version }}-${{ matrix.backend }}-${{ matrix.mariadb-version }}" - services: - mysql: - # Always start mariadb, even if we are testing with the mysql backend. - # Github actions do not allow conditional services yet. - image: mariadb:${{ matrix.mariadb-version }} - ports: - - 3306:3306 - env: - MYSQL_ROOT_PASSWORD: root - options: --tmpfs /var/lib/mysql + # services: + # mysql: + # # Always start mariadb, even if we are testing with the mysql backend. + # # Github actions do not allow conditional services yet. + # image: mariadb:${{ matrix.mariadb-version }} + # ports: + # - 3306:3306 + # env: + # MYSQL_ROOT_PASSWORD: root + # options: --tmpfs /var/lib/mysql env: DATABASE_URL: ${{ matrix.backend == 'sqlite' && 'sqlite:///db.sqlite3' || 'mysql://root:root@127.0.0.1:3306/mysql' }} From ac1d036eb3382cb5da13c2ea07e3efbea8347b18 Mon Sep 17 00:00:00 2001 From: Adrienne Stilp Date: Wed, 1 May 2024 16:09:44 -0700 Subject: [PATCH 18/24] Stop testing after 20 failures for speed --- .github/workflows/ci.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index bd509608..114b1616 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -41,6 +41,8 @@ jobs: # options: --tmpfs /var/lib/mysql env: + PYTEST_ADDOPTS: "--maxfail=20" # Stop testing after too many failures. + # Conditionally set the database url based on the backend. DATABASE_URL: ${{ matrix.backend == 'sqlite' && 'sqlite:///db.sqlite3' || 'mysql://root:root@127.0.0.1:3306/mysql' }} steps: From 608efba90504bcaf5d6ac1d4e9ce6b0ab699a790 Mon Sep 17 00:00:00 2001 From: Adrienne Stilp Date: Wed, 1 May 2024 16:11:58 -0700 Subject: [PATCH 19/24] Readd the mysql service Verified that test was failing for expected reason. --- .github/workflows/ci.yml | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 114b1616..6959644c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -29,16 +29,16 @@ jobs: mariadb-version: ["10.4"] name: "py${{ matrix.python-version }}-${{ matrix.backend }}-${{ matrix.mariadb-version }}" - # services: - # mysql: - # # Always start mariadb, even if we are testing with the mysql backend. - # # Github actions do not allow conditional services yet. - # image: mariadb:${{ matrix.mariadb-version }} - # ports: - # - 3306:3306 - # env: - # MYSQL_ROOT_PASSWORD: root - # options: --tmpfs /var/lib/mysql + services: + mysql: + # Always start mariadb, even if we are testing with the mysql backend. + # Github actions do not allow conditional services yet. + image: mariadb:${{ matrix.mariadb-version }} + ports: + - 3306:3306 + env: + MYSQL_ROOT_PASSWORD: root + options: --tmpfs /var/lib/mysql env: PYTEST_ADDOPTS: "--maxfail=20" # Stop testing after too many failures. From fbe2c7f18ffac015fbb0a1026ebcc6f29daea116 Mon Sep 17 00:00:00 2001 From: Adrienne Stilp Date: Wed, 1 May 2024 16:43:44 -0700 Subject: [PATCH 20/24] Remove separate pytest-mariadb and pytest-sqlite steps from CI They are now both handled in the "test" step. --- .github/workflows/ci.yml | 101 ++------------------------------------- 1 file changed, 3 insertions(+), 98 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6959644c..713930b5 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -20,7 +20,7 @@ concurrency: jobs: - tests: + test: runs-on: ubuntu-latest strategy: matrix: @@ -71,110 +71,15 @@ jobs: - name: Run tests run: coverage run -p -m pytest - - pytest-mariadb: - runs-on: ubuntu-latest - strategy: - max-parallel: 2 - matrix: - # While we are only running for a single version - # ahead of planned upgrades we can add versions as - # needed - python-version: [3.8] - mariadb-version: ["10.4", "10.5"] - - services: - mysql: - image: mariadb:${{ matrix.mariadb-version }} - ports: - - 3306:3306 - env: - MYSQL_ROOT_PASSWORD: root - - env: - # mysql://user:password@host:port/database - DATABASE_URL: "mysql://root:root@127.0.0.1:3306/mysql" - # We can set this to an empty string, since we'll never make an API call. - ANVIL_API_SERVICE_ACCOUNT_FILE: foo - - - steps: - - - name: Checkout Code Repository - uses: actions/checkout@v4 - - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v5 - with: - python-version: ${{ matrix.python-version }} - cache: pip - cache-dependency-path: | - requirements/requirements.txt - requirements/test-requirements.txt - - - name: Install Dependencies - run: | - python -m pip install --upgrade pip - pip install pip-tools - pip-sync requirements/requirements.txt requirements/test-requirements.txt - - - name: Collect staticfiles - run: python manage.py collectstatic --noinput --settings=config.settings.test - - - name: Run tests - run: coverage run -p -m pytest - env: - # We can set this to an empty string, since we'll never make an API call. - ANVIL_API_SERVICE_ACCOUNT_FILE: foo - - - name: Upload coverage data - uses: actions/upload-artifact@v4 - with: - name: coverage-data-mysql-${{ strategy.job-index }} - path: .coverage.* - - pytest-sqlite: - runs-on: ubuntu-latest - - env: - # We can set this to an empty string, since we'll never make an API call. - ANVIL_API_SERVICE_ACCOUNT_FILE: foo - - steps: - - name: Checkout Code Repository - uses: actions/checkout@v4 - - - name: Set up Python 3.8 - uses: actions/setup-python@v5 - with: - python-version: 3.8 - cache: pip - cache-dependency-path: | - requirements/requirements.txt - requirements/test-requirements.txt - - - name: Install Dependencies - run: | - python -m pip install --upgrade pip - pip install pip-tools - pip-sync requirements/requirements.txt requirements/test-requirements.txt - - - name: Collect staticfiles - run: python manage.py collectstatic --noinput --settings=config.settings.test - - - name: Run tests - run: coverage run -p -m pytest - - name: Upload coverage data uses: actions/upload-artifact@v4 with: - name: coverage-data-sqlite-${{ strategy.job-index }} + name: coverage-data-${{ strategy.job-index }} path: .coverage.* coverage: needs: - - pytest-mariadb - - pytest-sqlite + - test runs-on: ubuntu-latest steps: - name: Check out the repo From 9ff6f1dec4c35e52bc9f6dadec09e5b2a4f53d39 Mon Sep 17 00:00:00 2001 From: Adrienne Stilp Date: Wed, 1 May 2024 16:44:39 -0700 Subject: [PATCH 21/24] Add mariadb-10.5 as a specific "include" matrix --- .github/workflows/ci.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 713930b5..a35cd69d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -27,6 +27,10 @@ jobs: python-version: [3.8] backend: ["sqlite", "mariadb"] mariadb-version: ["10.4"] + include: + - python-version: 3.8 # Possible future version. + backend: "mariadb" + mariadb-version: "10.5" name: "py${{ matrix.python-version }}-${{ matrix.backend }}-${{ matrix.mariadb-version }}" services: From 15769e99ed13e05616e9de490af179a348aaa8cf Mon Sep 17 00:00:00 2001 From: Jonas Carson Date: Fri, 3 May 2024 09:33:01 -0700 Subject: [PATCH 22/24] Add nighly cron to sync user and site data from drupal site --- primed_apps.cron | 3 +++ 1 file changed, 3 insertions(+) diff --git a/primed_apps.cron b/primed_apps.cron index ada71ccf..b08e827b 100644 --- a/primed_apps.cron +++ b/primed_apps.cron @@ -15,3 +15,6 @@ MAILTO="primedweb@uw.edu" 10 4 * * SUN . /var/www/django/primed_apps/primed-apps-activate.sh; python manage.py run_dbgap_audit --email primedconsortium@uw.edu >> cron.log 15 4 * * SUN . /var/www/django/primed_apps/primed-apps-activate.sh; python manage.py run_cdsa_audit --email primedconsortium@uw.edu >> cron.log 20 4 * * SUN . /var/www/django/primed_apps/primed-apps-activate.sh; python manage.py run_collaborative_analysis_audit --email primedconsortium@uw.edu >> cron.log + +# Nightly user data audit +0 2 * * * . /var/www/django/primed_apps/primed-apps-activate.sh; python manage.py sync-drupal-data --update --email primedweb@uw.edu >> cron.log From 0f877510802e1424ede85e0e613ef0f6cd385414 Mon Sep 17 00:00:00 2001 From: Jonas Carson Date: Fri, 3 May 2024 09:33:37 -0700 Subject: [PATCH 23/24] Use distinct so we only report inactive anvil users once --- primed/users/audit.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/primed/users/audit.py b/primed/users/audit.py index c81fd479..6e1580df 100644 --- a/primed/users/audit.py +++ b/primed/users/audit.py @@ -288,10 +288,13 @@ def _run_audit(self): if handled is False: self.errors.append(RemoveUser(local_user=uda, note=f"Over Threshold {over_threshold}")) + # Use distinct so this returns one row per Account + # instead of row per groupaccountmembership inactive_anvil_users = Account.objects.filter( Q(user__is_active=False) | Q(user__id__in=user_ids_to_check), groupaccountmembership__isnull=False, - ) + ).distinct() + for inactive_anvil_user in inactive_anvil_users: self.errors.append( InactiveAnvilUser( From 6de6e0029a66c151d565d899f7217fed1017df15 Mon Sep 17 00:00:00 2001 From: Jonas Carson Date: Fri, 3 May 2024 09:34:45 -0700 Subject: [PATCH 24/24] Add additional notes about what can and cannot be resolved by audit --- primed/templates/users/drupal_data_audit_email.html | 3 +++ primed/users/management/commands/sync-drupal-data.py | 4 ++-- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/primed/templates/users/drupal_data_audit_email.html b/primed/templates/users/drupal_data_audit_email.html index c957a32f..fa806c3a 100644 --- a/primed/templates/users/drupal_data_audit_email.html +++ b/primed/templates/users/drupal_data_audit_email.html @@ -17,6 +17,7 @@

User Audit

Verified Users - {{ user_audit.verified|length }} record(s)

Needs action - {{user_audit.needs_action|length }} record(s)

+ {% if user_audit.needs_action %} {% render_table user_audit.get_needs_action_table %} {% endif %} @@ -39,6 +40,8 @@

Sites with errors - {{site_audit.errors|length }} record(s)

{% if site_audit.errors %} {% render_table site_audit.get_errors_table %} {% endif %} +

* Users and sites that Need Action will be resolved by this script if in update mode

+

* Users and sites listed as Error need manual intervention to resolve

{% endblock content %} diff --git a/primed/users/management/commands/sync-drupal-data.py b/primed/users/management/commands/sync-drupal-data.py index d58113cc..d69236fe 100644 --- a/primed/users/management/commands/sync-drupal-data.py +++ b/primed/users/management/commands/sync-drupal-data.py @@ -94,10 +94,10 @@ def handle(self, *args, **options): f"needs_changes: {len(user_audit.needs_action)} errors: {len(user_audit.errors)}\n" ) if user_audit.needs_action: - notification_content += "Users that need syncing:\n" + notification_content += "Users that need syncing (will be resolved by this script if in update mode):\n" notification_content += user_audit.get_needs_action_table().render_to_text() if user_audit.errors: - notification_content += "Users that need intervention:\n" + notification_content += "Users that need intervention (cannot be resolved by script):\n" notification_content += user_audit.get_errors_table().render_to_text() self.stdout.write(notification_content)