Merge pull request #559 from UW-GAC/deploy/stage

Deploy to prod

.github/workflows/ci.yml

@@ -20,31 +20,34 @@ concurrency:
 
 jobs:
 
-  pytest-mariadb:
+  test:
     runs-on: ubuntu-latest
     strategy:
-      max-parallel: 2
       matrix:
-        # While we are only running for a single version
-        # ahead of planned upgrades we can add versions as
-        # needed
         python-version: [3.8]
-        mariadb-version: ["10.4", "10.5"]
+        backend: ["sqlite", "mariadb"]
+        mariadb-version: ["10.4"]
+        include:
+          - python-version: 3.8  # Possible future version.
+            backend: "mariadb"
+            mariadb-version: "10.5"
+    name: "py${{ matrix.python-version }}-${{ matrix.backend }}-${{ matrix.mariadb-version }}"
 
     services:
       mysql:
+        # Always start mariadb, even if we are testing with the mysql backend.
+        # Github actions do not allow conditional services yet.
         image: mariadb:${{ matrix.mariadb-version }}
         ports:
           - 3306:3306
         env:
           MYSQL_ROOT_PASSWORD: root
+        options: --tmpfs /var/lib/mysql
 
     env:
-      # mysql://user:password@host:port/database
-      DATABASE_URL: "mysql://root:[email protected]:3306/mysql"
-      # We can set this to an empty string, since we'll never make an API call.
-      ANVIL_API_SERVICE_ACCOUNT_FILE: foo
-
+      PYTEST_ADDOPTS: "--maxfail=20"  # Stop testing after too many failures.
+      # Conditionally set the database url based on the backend.
+      DATABASE_URL: ${{ matrix.backend == 'sqlite' && 'sqlite:///db.sqlite3' || 'mysql://root:[email protected]:3306/mysql' }}
 
     steps:
 
@@ -69,60 +72,18 @@ jobs:
       - name: Collect staticfiles
         run: python manage.py collectstatic --noinput --settings=config.settings.test
 
-      - name: Run tests
-        run:  coverage run -p -m pytest
-        env:
-          # We can set this to an empty string, since we'll never make an API call.
-          ANVIL_API_SERVICE_ACCOUNT_FILE: foo
-
-      - name: Upload coverage data
-        uses: actions/upload-artifact@v4
-        with:
-          name: coverage-data-mysql-${{ strategy.job-index }}
-          path: .coverage.*
-
-  pytest-sqlite:
-    runs-on: ubuntu-latest
-
-    env:
-      # We can set this to an empty string, since we'll never make an API call.
-      ANVIL_API_SERVICE_ACCOUNT_FILE: foo
-
-    steps:
-      - name: Checkout Code Repository
-        uses: actions/checkout@v4
-
-      - name: Set up Python 3.8
-        uses: actions/setup-python@v5
-        with:
-          python-version: 3.8
-          cache: pip
-          cache-dependency-path: |
-            requirements/requirements.txt
-            requirements/test-requirements.txt
-
-      - name: Install Dependencies
-        run: |
-          python -m pip install --upgrade pip
-          pip install pip-tools
-          pip-sync requirements/requirements.txt requirements/test-requirements.txt
-
-      - name: Collect staticfiles
-        run: python manage.py collectstatic --noinput --settings=config.settings.test
-
       - name: Run tests
         run:  coverage run -p -m pytest
 
       - name: Upload coverage data
         uses: actions/upload-artifact@v4
         with:
-          name: coverage-data-sqlite-${{ strategy.job-index }}
+          name: coverage-data-${{ strategy.job-index }}
           path: .coverage.*
 
   coverage:
     needs:
-      - pytest-mariadb
-      - pytest-sqlite
+      - test
     runs-on: ubuntu-latest
     steps:
       - name: Check out the repo

config/settings/base.py

@@ -377,7 +377,7 @@
 
 # django-anvil-consortium-manager
 # ------------------------------------------------------------------------------
-ANVIL_API_SERVICE_ACCOUNT_FILE = env("ANVIL_API_SERVICE_ACCOUNT_FILE")
+
 # Specify workspace adapters.
 ANVIL_WORKSPACE_ADAPTERS = [
     "primed.dbgap.adapters.dbGaPWorkspaceAdapter",

config/settings/production.py

@@ -154,6 +154,7 @@
 
 # Your stuff...
 # ------------------------------------------------------------------------------
+ANVIL_API_SERVICE_ACCOUNT_FILE = env("ANVIL_API_SERVICE_ACCOUNT_FILE")
 # ANVIL_DBGAP_APPLICATION_GROUP_PREFIX = "PRIMED_DBGAP_ACCESS"
 # ANVIL_CDSA_GROUP_PREFIX = "PRIMED_CDSA_ACCESS"
 ANVIL_DATA_ACCESS_GROUP_PREFIX = "PRIMED"

primed/templates/users/drupal_data_audit_email.html

@@ -17,6 +17,7 @@ <h2>User Audit</h2>
       <h3>Verified Users - {{ user_audit.verified|length }} record(s)</h3>
 
       <h3>Needs action - {{user_audit.needs_action|length }} record(s)</h3>
+
       {% if user_audit.needs_action %}
       {% render_table user_audit.get_needs_action_table %}
       {% endif %}
@@ -39,6 +40,8 @@ <h3>Sites with errors - {{site_audit.errors|length }} record(s)</h3>
       {% if site_audit.errors %}
       {% render_table site_audit.get_errors_table %}
       {% endif %}
+      <p>* Users and sites that <b>Need Action</b> will be resolved by this script if in update mode</p>
+      <p>* Users and sites listed as <b>Error</b> need manual intervention to resolve</p>
 
 
 {% endblock content %}

primed/users/audit.py

@@ -288,10 +288,13 @@ def _run_audit(self):
             if handled is False:
                 self.errors.append(RemoveUser(local_user=uda, note=f"Over Threshold {over_threshold}"))
 
+        # Use distinct so this returns one row per Account
+        # instead of row per groupaccountmembership
         inactive_anvil_users = Account.objects.filter(
             Q(user__is_active=False) | Q(user__id__in=user_ids_to_check),
             groupaccountmembership__isnull=False,
-        )
+        ).distinct()
+
         for inactive_anvil_user in inactive_anvil_users:
             self.errors.append(
                 InactiveAnvilUser(

primed/users/management/commands/sync-drupal-data.py

@@ -94,10 +94,10 @@ def handle(self, *args, **options):
             f"needs_changes: {len(user_audit.needs_action)} errors: {len(user_audit.errors)}\n"
         )
         if user_audit.needs_action:
-            notification_content += "Users that need syncing:\n"
+            notification_content += "Users that need syncing (will be resolved by this script if in update mode):\n"
             notification_content += user_audit.get_needs_action_table().render_to_text()
         if user_audit.errors:
-            notification_content += "Users that need intervention:\n"
+            notification_content += "Users that need intervention (cannot be resolved by script):\n"
             notification_content += user_audit.get_errors_table().render_to_text()
 
         self.stdout.write(notification_content)

primed_apps.cron

@@ -15,3 +15,6 @@ MAILTO="[email protected]"
 10 4 * * SUN . /var/www/django/primed_apps/primed-apps-activate.sh; python manage.py run_dbgap_audit --email [email protected] >> cron.log
 15 4 * * SUN . /var/www/django/primed_apps/primed-apps-activate.sh; python manage.py run_cdsa_audit --email [email protected] >> cron.log
 20 4 * * SUN . /var/www/django/primed_apps/primed-apps-activate.sh; python manage.py run_collaborative_analysis_audit --email [email protected] >> cron.log
+
+# Nightly user data audit
+0 2 * * * . /var/www/django/primed_apps/primed-apps-activate.sh; python manage.py sync-drupal-data --update --email [email protected] >> cron.log

requirements/dev-requirements.txt

@@ -8,7 +8,7 @@ alabaster==0.7.13
     # via sphinx
 asgiref==3.7.2
     # via
-    #   -c requirements.txt
+    #   -c requirements/requirements.txt
     #   django
 asttokens==2.4.1
     # via stack-data
@@ -18,23 +18,20 @@ backcall==0.2.0
     # via ipython
 backports-zoneinfo==0.2.1
     # via
-    #   -c requirements.txt
+    #   -c requirements/requirements.txt
     #   django
 certifi==2023.11.17
     # via
-    #   -c requirements.txt
-    #   -c test-requirements.txt
+    #   -c requirements/requirements.txt
+    #   -c requirements/test-requirements.txt
     #   requests
 cfgv==3.4.0
     # via pre-commit
 charset-normalizer==3.3.2
     # via
-    #   -c requirements.txt
-    #   -c test-requirements.txt
+    #   -c requirements/requirements.txt
+    #   -c requirements/test-requirements.txt
     #   requests
-click==8.1.3
-    # via
-    #   -c requirements.txt
 colorama==0.4.6
     # via sphinx-autobuild
 decorator==5.1.1
@@ -45,14 +42,14 @@ distlib==0.3.8
     # via virtualenv
 django==4.2.11
     # via
-    #   -c requirements.txt
+    #   -c requirements/requirements.txt
     #   django-debug-toolbar
     #   django-stubs
     #   django-stubs-ext
 django-debug-toolbar==4.3.0
-    # via -r dev-requirements.in
+    # via -r requirements/dev-requirements.in
 django-stubs==4.2.7
-    # via -r dev-requirements.in
+    # via -r requirements/dev-requirements.in
 django-stubs-ext==4.2.7
     # via django-stubs
 docutils==0.20.1
@@ -65,17 +62,17 @@ identify==2.5.34
     # via pre-commit
 idna==3.3
     # via
-    #   -c requirements.txt
-    #   -c test-requirements.txt
+    #   -c requirements/requirements.txt
+    #   -c requirements/test-requirements.txt
     #   requests
 imagesize==1.4.1
     # via sphinx
 importlib-metadata==7.0.0
     # via
-    #   -c requirements.txt
+    #   -c requirements/requirements.txt
     #   sphinx
 ipdb==0.13.13
-    # via -r dev-requirements.in
+    # via -r requirements/dev-requirements.in
 ipython==8.12.3
     # via ipdb
 jedi==0.19.1
@@ -90,17 +87,16 @@ markupsafe==2.1.5
     #   werkzeug
 matplotlib-inline==0.1.6
     # via ipython
-mypy==1.9.0
-    # via -r dev-requirements.in
+mypy==1.10.0
+    # via -r requirements/dev-requirements.in
 mypy-extensions==1.0.0
-    # via
-    #   mypy
+    # via mypy
 nodeenv==1.8.0
     # via pre-commit
 packaging==21.3
     # via
-    #   -c requirements.txt
-    #   -c test-requirements.txt
+    #   -c requirements/requirements.txt
+    #   -c requirements/test-requirements.txt
     #   sphinx
 parso==0.8.3
     # via jedi
@@ -109,10 +105,9 @@ pexpect==4.9.0
 pickleshare==0.7.5
     # via ipython
 platformdirs==4.1.0
-    # via
-    #   virtualenv
+    # via virtualenv
 pre-commit==3.5.0
-    # via -r dev-requirements.in
+    # via -r requirements/dev-requirements.in
 prompt-toolkit==3.0.43
     # via ipython
 ptyprocess==0.7.0
@@ -125,38 +120,38 @@ pygments==2.17.2
     #   sphinx
 pyparsing==3.1.1
     # via
-    #   -c requirements.txt
-    #   -c test-requirements.txt
+    #   -c requirements/requirements.txt
+    #   -c requirements/test-requirements.txt
     #   packaging
 pytz==2023.3.post1
     # via
-    #   -c requirements.txt
+    #   -c requirements/requirements.txt
     #   babel
 pyyaml==6.0.1
     # via
-    #   -c test-requirements.txt
+    #   -c requirements/test-requirements.txt
     #   pre-commit
 requests==2.31.0
     # via
-    #   -c requirements.txt
-    #   -c test-requirements.txt
+    #   -c requirements/requirements.txt
+    #   -c requirements/test-requirements.txt
     #   sphinx
 ruff==0.4.2
     # via -r requirements/dev-requirements.in
 six==1.16.0
     # via
-    #   -c requirements.txt
-    #   -c test-requirements.txt
+    #   -c requirements/requirements.txt
+    #   -c requirements/test-requirements.txt
     #   asttokens
     #   livereload
 snowballstemmer==2.2.0
     # via sphinx
 sphinx==7.1.2
     # via
-    #   -r dev-requirements.in
+    #   -r requirements/dev-requirements.in
     #   sphinx-autobuild
 sphinx-autobuild==2021.3.14
-    # via -r dev-requirements.in
+    # via -r requirements/dev-requirements.in
 sphinxcontrib-applehelp==1.0.4
     # via sphinx
 sphinxcontrib-devhelp==1.0.2
@@ -171,15 +166,15 @@ sphinxcontrib-serializinghtml==1.1.5
     # via sphinx
 sqlparse==0.5.0
     # via
-    #   -c requirements.txt
+    #   -c requirements/requirements.txt
     #   django
     #   django-debug-toolbar
 stack-data==0.6.3
     # via ipython
 tomli==2.0.1
     # via
-    #   -c requirements.txt
-    #   -c test-requirements.txt
+    #   -c requirements/requirements.txt
+    #   -c requirements/test-requirements.txt
     #   django-stubs
     #   ipdb
     #   mypy
@@ -194,32 +189,31 @@ types-pytz==2024.1.0.20240203
 types-pyyaml==6.0.12.12
     # via django-stubs
 types-requests==2.31.0.20240406
-    # via -r dev-requirements.in
+    # via -r requirements/dev-requirements.in
 typing-extensions==4.8.0
     # via
-    #   -c requirements.txt
-    #   -c test-requirements.txt
+    #   -c requirements/requirements.txt
+    #   -c requirements/test-requirements.txt
     #   asgiref
-    #   astroid
     #   django-stubs
     #   django-stubs-ext
     #   ipython
     #   mypy
 urllib3==2.1.0
     # via
-    #   -c requirements.txt
-    #   -c test-requirements.txt
+    #   -c requirements/requirements.txt
+    #   -c requirements/test-requirements.txt
     #   requests
     #   types-requests
 virtualenv==20.25.1
     # via pre-commit
 wcwidth==0.2.13
     # via prompt-toolkit
 werkzeug==3.0.2
-    # via -r dev-requirements.in
+    # via -r requirements/dev-requirements.in
 zipp==3.17.0
     # via
-    #   -c requirements.txt
+    #   -c requirements/requirements.txt
     #   importlib-metadata
 
 # The following packages are considered to be unsafe in a requirements file: