diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6ff4177a..a35cd69d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -20,31 +20,34 @@ concurrency: jobs: - pytest-mariadb: + test: runs-on: ubuntu-latest strategy: - max-parallel: 2 matrix: - # While we are only running for a single version - # ahead of planned upgrades we can add versions as - # needed python-version: [3.8] - mariadb-version: ["10.4", "10.5"] + backend: ["sqlite", "mariadb"] + mariadb-version: ["10.4"] + include: + - python-version: 3.8 # Possible future version. + backend: "mariadb" + mariadb-version: "10.5" + name: "py${{ matrix.python-version }}-${{ matrix.backend }}-${{ matrix.mariadb-version }}" services: mysql: + # Always start mariadb, even if we are testing with the mysql backend. + # Github actions do not allow conditional services yet. image: mariadb:${{ matrix.mariadb-version }} ports: - 3306:3306 env: MYSQL_ROOT_PASSWORD: root + options: --tmpfs /var/lib/mysql env: - # mysql://user:password@host:port/database - DATABASE_URL: "mysql://root:root@127.0.0.1:3306/mysql" - # We can set this to an empty string, since we'll never make an API call. - ANVIL_API_SERVICE_ACCOUNT_FILE: foo - + PYTEST_ADDOPTS: "--maxfail=20" # Stop testing after too many failures. + # Conditionally set the database url based on the backend. + DATABASE_URL: ${{ matrix.backend == 'sqlite' && 'sqlite:///db.sqlite3' || 'mysql://root:root@127.0.0.1:3306/mysql' }} steps: @@ -69,60 +72,18 @@ jobs: - name: Collect staticfiles run: python manage.py collectstatic --noinput --settings=config.settings.test - - name: Run tests - run: coverage run -p -m pytest - env: - # We can set this to an empty string, since we'll never make an API call. - ANVIL_API_SERVICE_ACCOUNT_FILE: foo - - - name: Upload coverage data - uses: actions/upload-artifact@v4 - with: - name: coverage-data-mysql-${{ strategy.job-index }} - path: .coverage.* - - pytest-sqlite: - runs-on: ubuntu-latest - - env: - # We can set this to an empty string, since we'll never make an API call. - ANVIL_API_SERVICE_ACCOUNT_FILE: foo - - steps: - - name: Checkout Code Repository - uses: actions/checkout@v4 - - - name: Set up Python 3.8 - uses: actions/setup-python@v5 - with: - python-version: 3.8 - cache: pip - cache-dependency-path: | - requirements/requirements.txt - requirements/test-requirements.txt - - - name: Install Dependencies - run: | - python -m pip install --upgrade pip - pip install pip-tools - pip-sync requirements/requirements.txt requirements/test-requirements.txt - - - name: Collect staticfiles - run: python manage.py collectstatic --noinput --settings=config.settings.test - - name: Run tests run: coverage run -p -m pytest - name: Upload coverage data uses: actions/upload-artifact@v4 with: - name: coverage-data-sqlite-${{ strategy.job-index }} + name: coverage-data-${{ strategy.job-index }} path: .coverage.* coverage: needs: - - pytest-mariadb - - pytest-sqlite + - test runs-on: ubuntu-latest steps: - name: Check out the repo diff --git a/config/settings/base.py b/config/settings/base.py index 26b16d38..0ce84d43 100644 --- a/config/settings/base.py +++ b/config/settings/base.py @@ -377,7 +377,7 @@ # django-anvil-consortium-manager # ------------------------------------------------------------------------------ -ANVIL_API_SERVICE_ACCOUNT_FILE = env("ANVIL_API_SERVICE_ACCOUNT_FILE") + # Specify workspace adapters. ANVIL_WORKSPACE_ADAPTERS = [ "primed.dbgap.adapters.dbGaPWorkspaceAdapter", diff --git a/config/settings/production.py b/config/settings/production.py index d325df2e..53cbf7aa 100644 --- a/config/settings/production.py +++ b/config/settings/production.py @@ -154,6 +154,7 @@ # Your stuff... # ------------------------------------------------------------------------------ +ANVIL_API_SERVICE_ACCOUNT_FILE = env("ANVIL_API_SERVICE_ACCOUNT_FILE") # ANVIL_DBGAP_APPLICATION_GROUP_PREFIX = "PRIMED_DBGAP_ACCESS" # ANVIL_CDSA_GROUP_PREFIX = "PRIMED_CDSA_ACCESS" ANVIL_DATA_ACCESS_GROUP_PREFIX = "PRIMED" diff --git a/primed/templates/users/drupal_data_audit_email.html b/primed/templates/users/drupal_data_audit_email.html index c957a32f..fa806c3a 100644 --- a/primed/templates/users/drupal_data_audit_email.html +++ b/primed/templates/users/drupal_data_audit_email.html @@ -17,6 +17,7 @@

User Audit

Verified Users - {{ user_audit.verified|length }} record(s)

Needs action - {{user_audit.needs_action|length }} record(s)

+ {% if user_audit.needs_action %} {% render_table user_audit.get_needs_action_table %} {% endif %} @@ -39,6 +40,8 @@

Sites with errors - {{site_audit.errors|length }} record(s)

{% if site_audit.errors %} {% render_table site_audit.get_errors_table %} {% endif %} +

* Users and sites that Need Action will be resolved by this script if in update mode

+

* Users and sites listed as Error need manual intervention to resolve

{% endblock content %} diff --git a/primed/users/audit.py b/primed/users/audit.py index c81fd479..6e1580df 100644 --- a/primed/users/audit.py +++ b/primed/users/audit.py @@ -288,10 +288,13 @@ def _run_audit(self): if handled is False: self.errors.append(RemoveUser(local_user=uda, note=f"Over Threshold {over_threshold}")) + # Use distinct so this returns one row per Account + # instead of row per groupaccountmembership inactive_anvil_users = Account.objects.filter( Q(user__is_active=False) | Q(user__id__in=user_ids_to_check), groupaccountmembership__isnull=False, - ) + ).distinct() + for inactive_anvil_user in inactive_anvil_users: self.errors.append( InactiveAnvilUser( diff --git a/primed/users/management/commands/sync-drupal-data.py b/primed/users/management/commands/sync-drupal-data.py index d58113cc..d69236fe 100644 --- a/primed/users/management/commands/sync-drupal-data.py +++ b/primed/users/management/commands/sync-drupal-data.py @@ -94,10 +94,10 @@ def handle(self, *args, **options): f"needs_changes: {len(user_audit.needs_action)} errors: {len(user_audit.errors)}\n" ) if user_audit.needs_action: - notification_content += "Users that need syncing:\n" + notification_content += "Users that need syncing (will be resolved by this script if in update mode):\n" notification_content += user_audit.get_needs_action_table().render_to_text() if user_audit.errors: - notification_content += "Users that need intervention:\n" + notification_content += "Users that need intervention (cannot be resolved by script):\n" notification_content += user_audit.get_errors_table().render_to_text() self.stdout.write(notification_content) diff --git a/primed_apps.cron b/primed_apps.cron index ada71ccf..b08e827b 100644 --- a/primed_apps.cron +++ b/primed_apps.cron @@ -15,3 +15,6 @@ MAILTO="primedweb@uw.edu" 10 4 * * SUN . /var/www/django/primed_apps/primed-apps-activate.sh; python manage.py run_dbgap_audit --email primedconsortium@uw.edu >> cron.log 15 4 * * SUN . /var/www/django/primed_apps/primed-apps-activate.sh; python manage.py run_cdsa_audit --email primedconsortium@uw.edu >> cron.log 20 4 * * SUN . /var/www/django/primed_apps/primed-apps-activate.sh; python manage.py run_collaborative_analysis_audit --email primedconsortium@uw.edu >> cron.log + +# Nightly user data audit +0 2 * * * . /var/www/django/primed_apps/primed-apps-activate.sh; python manage.py sync-drupal-data --update --email primedweb@uw.edu >> cron.log diff --git a/requirements/dev-requirements.txt b/requirements/dev-requirements.txt index 08c17266..eb16052f 100644 --- a/requirements/dev-requirements.txt +++ b/requirements/dev-requirements.txt @@ -8,7 +8,7 @@ alabaster==0.7.13 # via sphinx asgiref==3.7.2 # via - # -c requirements.txt + # -c requirements/requirements.txt # django asttokens==2.4.1 # via stack-data @@ -18,23 +18,20 @@ backcall==0.2.0 # via ipython backports-zoneinfo==0.2.1 # via - # -c requirements.txt + # -c requirements/requirements.txt # django certifi==2023.11.17 # via - # -c requirements.txt - # -c test-requirements.txt + # -c requirements/requirements.txt + # -c requirements/test-requirements.txt # requests cfgv==3.4.0 # via pre-commit charset-normalizer==3.3.2 # via - # -c requirements.txt - # -c test-requirements.txt + # -c requirements/requirements.txt + # -c requirements/test-requirements.txt # requests -click==8.1.3 - # via - # -c requirements.txt colorama==0.4.6 # via sphinx-autobuild decorator==5.1.1 @@ -45,14 +42,14 @@ distlib==0.3.8 # via virtualenv django==4.2.11 # via - # -c requirements.txt + # -c requirements/requirements.txt # django-debug-toolbar # django-stubs # django-stubs-ext django-debug-toolbar==4.3.0 - # via -r dev-requirements.in + # via -r requirements/dev-requirements.in django-stubs==4.2.7 - # via -r dev-requirements.in + # via -r requirements/dev-requirements.in django-stubs-ext==4.2.7 # via django-stubs docutils==0.20.1 @@ -65,17 +62,17 @@ identify==2.5.34 # via pre-commit idna==3.3 # via - # -c requirements.txt - # -c test-requirements.txt + # -c requirements/requirements.txt + # -c requirements/test-requirements.txt # requests imagesize==1.4.1 # via sphinx importlib-metadata==7.0.0 # via - # -c requirements.txt + # -c requirements/requirements.txt # sphinx ipdb==0.13.13 - # via -r dev-requirements.in + # via -r requirements/dev-requirements.in ipython==8.12.3 # via ipdb jedi==0.19.1 @@ -90,17 +87,16 @@ markupsafe==2.1.5 # werkzeug matplotlib-inline==0.1.6 # via ipython -mypy==1.9.0 - # via -r dev-requirements.in +mypy==1.10.0 + # via -r requirements/dev-requirements.in mypy-extensions==1.0.0 - # via - # mypy + # via mypy nodeenv==1.8.0 # via pre-commit packaging==21.3 # via - # -c requirements.txt - # -c test-requirements.txt + # -c requirements/requirements.txt + # -c requirements/test-requirements.txt # sphinx parso==0.8.3 # via jedi @@ -109,10 +105,9 @@ pexpect==4.9.0 pickleshare==0.7.5 # via ipython platformdirs==4.1.0 - # via - # virtualenv + # via virtualenv pre-commit==3.5.0 - # via -r dev-requirements.in + # via -r requirements/dev-requirements.in prompt-toolkit==3.0.43 # via ipython ptyprocess==0.7.0 @@ -125,38 +120,38 @@ pygments==2.17.2 # sphinx pyparsing==3.1.1 # via - # -c requirements.txt - # -c test-requirements.txt + # -c requirements/requirements.txt + # -c requirements/test-requirements.txt # packaging pytz==2023.3.post1 # via - # -c requirements.txt + # -c requirements/requirements.txt # babel pyyaml==6.0.1 # via - # -c test-requirements.txt + # -c requirements/test-requirements.txt # pre-commit requests==2.31.0 # via - # -c requirements.txt - # -c test-requirements.txt + # -c requirements/requirements.txt + # -c requirements/test-requirements.txt # sphinx ruff==0.4.2 # via -r requirements/dev-requirements.in six==1.16.0 # via - # -c requirements.txt - # -c test-requirements.txt + # -c requirements/requirements.txt + # -c requirements/test-requirements.txt # asttokens # livereload snowballstemmer==2.2.0 # via sphinx sphinx==7.1.2 # via - # -r dev-requirements.in + # -r requirements/dev-requirements.in # sphinx-autobuild sphinx-autobuild==2021.3.14 - # via -r dev-requirements.in + # via -r requirements/dev-requirements.in sphinxcontrib-applehelp==1.0.4 # via sphinx sphinxcontrib-devhelp==1.0.2 @@ -171,15 +166,15 @@ sphinxcontrib-serializinghtml==1.1.5 # via sphinx sqlparse==0.5.0 # via - # -c requirements.txt + # -c requirements/requirements.txt # django # django-debug-toolbar stack-data==0.6.3 # via ipython tomli==2.0.1 # via - # -c requirements.txt - # -c test-requirements.txt + # -c requirements/requirements.txt + # -c requirements/test-requirements.txt # django-stubs # ipdb # mypy @@ -194,21 +189,20 @@ types-pytz==2024.1.0.20240203 types-pyyaml==6.0.12.12 # via django-stubs types-requests==2.31.0.20240406 - # via -r dev-requirements.in + # via -r requirements/dev-requirements.in typing-extensions==4.8.0 # via - # -c requirements.txt - # -c test-requirements.txt + # -c requirements/requirements.txt + # -c requirements/test-requirements.txt # asgiref - # astroid # django-stubs # django-stubs-ext # ipython # mypy urllib3==2.1.0 # via - # -c requirements.txt - # -c test-requirements.txt + # -c requirements/requirements.txt + # -c requirements/test-requirements.txt # requests # types-requests virtualenv==20.25.1 @@ -216,10 +210,10 @@ virtualenv==20.25.1 wcwidth==0.2.13 # via prompt-toolkit werkzeug==3.0.2 - # via -r dev-requirements.in + # via -r requirements/dev-requirements.in zipp==3.17.0 # via - # -c requirements.txt + # -c requirements/requirements.txt # importlib-metadata # The following packages are considered to be unsafe in a requirements file: diff --git a/requirements/requirements.txt b/requirements/requirements.txt index 847056f7..b5423acc 100644 --- a/requirements/requirements.txt +++ b/requirements/requirements.txt @@ -5,7 +5,7 @@ # pip-compile requirements/requirements.in # argon2-cffi==23.1.0 - # via -r requirements.in + # via -r requirements/requirements.in argon2-cffi-bindings==21.2.0 # via argon2-cffi asgiref==3.7.2 @@ -25,7 +25,7 @@ cachetools==5.3.2 # via google-auth certifi==2023.11.17 # via - # -r requirements.in + # -r requirements/requirements.in # requests cffi==1.16.0 # via @@ -39,7 +39,7 @@ click==8.1.3 # via pip-tools crispy-bootstrap5==2024.2 # via - # -r requirements.in + # -r requirements/requirements.in # django-anvil-consortium-manager cryptography==42.0.5 # via pyjwt @@ -47,7 +47,7 @@ defusedxml==0.7.1 # via python3-openid django==4.2.11 # via - # -r requirements.in + # -r requirements/requirements.in # crispy-bootstrap5 # django-allauth # django-anvil-consortium-manager @@ -61,48 +61,48 @@ django==4.2.11 # django-picklefield # django-tables2 django-allauth==0.54.0 - # via -r requirements.in + # via -r requirements/requirements.in django-anvil-consortium-manager @ git+https://github.com/UW-GAC/django-anvil-consortium-manager.git@v0.22 - # via -r requirements.in + # via -r requirements/requirements.in django-autocomplete-light==3.11.0 # via django-anvil-consortium-manager django-constance==3.1.0 - # via -r requirements.in + # via -r requirements/requirements.in django-crispy-forms==2.1 # via - # -r requirements.in + # -r requirements/requirements.in # crispy-bootstrap5 # django-anvil-consortium-manager django-dbbackup==4.1.0 - # via -r requirements.in + # via -r requirements/requirements.in django-environ==0.10.0 - # via -r requirements.in + # via -r requirements/requirements.in django-extensions==3.2.3 # via - # -r requirements.in + # -r requirements/requirements.in # django-anvil-consortium-manager django-filter==23.5 # via django-anvil-consortium-manager django-htmx==1.17.3 - # via -r requirements.in + # via -r requirements/requirements.in django-login-required-middleware==0.9.0 - # via -r requirements.in + # via -r requirements/requirements.in django-maintenance-mode==0.21.1 - # via -r requirements.in + # via -r requirements/requirements.in django-model-utils==4.5.0 - # via -r requirements.in + # via -r requirements/requirements.in django-picklefield==3.2 # via - # -r requirements.in + # -r requirements/requirements.in # django-constance django-simple-history==3.5.0 # via - # -r requirements.in + # -r requirements/requirements.in # django-anvil-consortium-manager django-tables2==2.7.0 # via django-anvil-consortium-manager -django-tree-queries==0.18.0 - # via -r requirements.in +django-tree-queries==0.19.0 + # via -r requirements/requirements.in fastobo==0.12.3 # via pronto fontawesomefree==6.5.1 @@ -118,13 +118,13 @@ importlib-resources==6.1.1 # jsonschema # jsonschema-specifications jsonapi-requests==0.7.0 - # via -r requirements.in + # via -r requirements/requirements.in jsonschema==4.21.1 - # via -r requirements.in + # via -r requirements/requirements.in jsonschema-specifications==2023.12.1 # via jsonschema mysqlclient==2.2.4 - # via -r requirements.in + # via -r requirements/requirements.in networkx==3.1 # via # django-anvil-consortium-manager @@ -135,22 +135,22 @@ numpy==1.24.4 # pandas oauthlib==3.2.2 # via - # -r requirements.in + # -r requirements/requirements.in # requests-oauthlib packaging==21.3 # via # build # plotly pandas==2.0.3 - # via -r requirements.in + # via -r requirements/requirements.in pip-tools==7.4.1 - # via -r requirements.in + # via -r requirements/requirements.in pkgutil-resolve-name==1.3.10 # via jsonschema plotly==5.19.0 # via django-anvil-consortium-manager -pronto==2.5.6 - # via -r requirements.in +pronto==2.5.7 + # via -r requirements/requirements.in pyasn1==0.5.1 # via # pyasn1-modules @@ -186,7 +186,7 @@ referencing==0.33.0 # jsonschema-specifications requests==2.31.0 # via - # -r requirements.in + # -r requirements/requirements.in # django-allauth # django-anvil-consortium-manager # jsonapi-requests @@ -203,10 +203,10 @@ six==1.16.0 # via python-dateutil sqlparse==0.5.0 # via - # -r requirements.in + # -r requirements/requirements.in # django tablib==3.6.1 - # via -r requirements.in + # via -r requirements/requirements.in tenacity==8.2.3 # via # jsonapi-requests @@ -222,12 +222,12 @@ tzdata==2023.4 # via pandas urllib3==2.1.0 # via - # -r requirements.in + # -r requirements/requirements.in # requests wheel==0.42.0 # via pip-tools whitenoise==6.6.0 - # via -r requirements.in + # via -r requirements/requirements.in zipp==3.17.0 # via # importlib-metadata diff --git a/requirements/test-requirements.txt b/requirements/test-requirements.txt index f58f6830..d4589f67 100644 --- a/requirements/test-requirements.txt +++ b/requirements/test-requirements.txt @@ -12,7 +12,7 @@ charset-normalizer==3.3.2 # via # -c requirements/requirements.txt # requests -coverage==7.4.4 +coverage==7.5.0 # via # -r requirements/test-requirements.in # django-coverage-plugin @@ -26,7 +26,7 @@ factory-boy==3.3.0 # via -r requirements/test-requirements.in faker==23.2.1 # via factory-boy -freezegun==1.4.0 +freezegun==1.5.0 # via -r requirements/test-requirements.in idna==3.3 # via @@ -44,13 +44,13 @@ packaging==21.3 # marshmallow # pytest # pytest-sugar -pluggy==1.4.0 +pluggy==1.5.0 # via pytest pyparsing==3.1.1 # via # -c requirements/requirements.txt # packaging -pytest==8.1.1 +pytest==8.2.0 # via # -r requirements/test-requirements.in # pytest-django