+
+{% include "snippets/data_prep_workspace_table.html" with table=associated_data_prep_workspaces is_active=data_prep_active %}
+
{{block.super}}
{% endblock after_panel %}
diff --git a/primed/templates/cdsa/dataaffiliateagreement_detail.html b/primed/templates/cdsa/dataaffiliateagreement_detail.html
index 211b3e92..ac650255 100644
--- a/primed/templates/cdsa/dataaffiliateagreement_detail.html
+++ b/primed/templates/cdsa/dataaffiliateagreement_detail.html
@@ -20,7 +20,20 @@
- {% if messages %}
- {% for message in messages %}
-
- {% endfor %}
+ {% if config.ANNOUNCEMENT_TEXT %}
+
+
+ {{ config.ANNOUNCEMENT_TEXT }}
+
{% endif %}
- {% if not request.user.account %}
-
+ {% if request.user.is_authenticated and not request.user.account %}
+
@@ -80,5 +130,7 @@
You must link your AnVIL account before you can access any data on AnVIL.
- {% endif %}
+ {% endif %}
+
+ {% if messages %}
+ {% for message in messages %}
+
+ {% endfor %}
+ {% endif %}
+
+ {% block extra_navbar %}
+ {% endblock %}
+
{% block content %}
diff --git a/primed/templates/cdsa/cdsaworkspace_detail.html b/primed/templates/cdsa/cdsaworkspace_detail.html
index 76a7952c..3a192665 100644
--- a/primed/templates/cdsa/cdsaworkspace_detail.html
+++ b/primed/templates/cdsa/cdsaworkspace_detail.html
@@ -1,16 +1,46 @@
{% extends "anvil_consortium_manager/workspace_detail.html" %}
+{% load render_table from django_tables2 %}
{% block pills %}
{% if workspace_data_object.gsr_restricted %}
{% include "snippets/gsr_restricted_badge.html" %}
{% endif %}
+ {% if not primary_cdsa %}
+
+
+ No primary CDSA
+
+
+ {% elif primary_cdsa.requires_study_review %}
+
+
+ Study review required
+
+
+ {% endif %}
+
{{ block.super }}
{% endblock pills %}
{% block workspace_data %}
- Associated CDSA
- + {% if primary_cdsa %} + {{ primary_cdsa }} + {% else %} + — + {% endif %} +
- Study
- {{ object.cdsaworkspace.study }} @@ -55,7 +85,27 @@
- DUO consent description +
-
+
- {{ workspace_data_object.data_use_permission.abbreviation }}: {{ workspace_data_object.data_use_permission.get_short_definition }}
+ {% for x in workspace_data_object.data_use_modifiers.all %} +- {{ x.abbreviation }}: {{ x.get_short_definition }}
+ {% endfor %} +
+ {% if workspace_data_object.additional_limitations %}
+ - Additional limitations for this consent group +
-
+
- {{ workspace_data_object.additional_limitations }}
+
+ {% endif %}
+ {% if primary_cdsa.additional_limitations %}
+ - Additional limitations from CDSA +
-
+
- {{ primary_cdsa.additional_limitations }}
+
+ {% endif %}
+
+
- {{ object.cdsaworkspace.data_use_limitations }}
+
-
+
+
+{% endif %}
+
+{{ block.super }}
+
+{% endblock after_panel %}
+
{% block action_buttons %}
{% if show_update_button %}
diff --git a/primed/templates/cdsa/memberagreement_detail.html b/primed/templates/cdsa/memberagreement_detail.html
index 381b1f23..c4a4b1d9 100644
--- a/primed/templates/cdsa/memberagreement_detail.html
+++ b/primed/templates/cdsa/memberagreement_detail.html
@@ -20,7 +20,13 @@
+
+
+
+ Additional limitations
+
+
+
+ + {{ object.additional_limitations }} +
+
+{% include "snippets/data_prep_workspace_table.html" with table=associated_data_prep_workspaces is_active=data_prep_active %}
+
{{block.super}}
{% endblock after_panel %}
diff --git a/primed/templates/snippets/data_prep_workspace_table.html b/primed/templates/snippets/data_prep_workspace_table.html
new file mode 100644
index 00000000..32d04cff
--- /dev/null
+++ b/primed/templates/snippets/data_prep_workspace_table.html
@@ -0,0 +1,24 @@
+{% load render_table from django_tables2 %}
+
+{% if perms.anvil_consortium_manager.anvil_consortium_manager_staff_view %}
+
+
+
+
+
+
+
+
+
+ {% render_table table %}
+
+
+
+
+
+
+{% endif %}
diff --git a/primed/templates/users/drupal_data_audit_email.html b/primed/templates/users/drupal_data_audit_email.html
new file mode 100644
index 00000000..c957a32f
--- /dev/null
+++ b/primed/templates/users/drupal_data_audit_email.html
@@ -0,0 +1,47 @@
+
+{% load static i18n %}
+{% load render_table from django_tables2 %}
+
+
+
+ Drupal Data Audit Report
+
+
+
+
+{% block content %}
+
+ Drupal Data Audit - [applying_changes={{ apply_changes }}]
+ User Audit
+
+ Verified Users - {{ user_audit.verified|length }} record(s)
+
+ Needs action - {{user_audit.needs_action|length }} record(s)
+ {% if user_audit.needs_action %}
+ {% render_table user_audit.get_needs_action_table %}
+ {% endif %}
+
+ Errors - {{user_audit.errors|length }} record(s)
+ {% if user_audit.errors %}
+ {% render_table user_audit.get_errors_table %}
+ {% endif %}
+
+ Site Audit
+
+ Verified sites - {{ site_audit.verified|length }} record(s)
+
+ Sites that need action - {{site_audit.needs_action|length }} record(s)
+ {% if site_audit.needs_action %}
+ {% render_table site_audit.get_needs_action_table %}
+ {% endif %}
+
+ Sites with errors - {{site_audit.errors|length }} record(s)
+ {% if site_audit.errors %}
+ {% render_table site_audit.get_errors_table %}
+ {% endif %}
+
+
+{% endblock content %}
+
+
+
diff --git a/primed/users/adapters.py b/primed/users/adapters.py
index b0f6d6a3..24e6d91e 100644
--- a/primed/users/adapters.py
+++ b/primed/users/adapters.py
@@ -23,7 +23,7 @@ class SocialAccountAdapter(DefaultSocialAccountAdapter):
def is_open_for_signup(self, request: HttpRequest, sociallogin: Any):
return getattr(settings, "ACCOUNT_ALLOW_REGISTRATION", True)
- def update_user_info(self, user, extra_data: Dict):
+ def update_user_info(self, user, extra_data: Dict, apply_update=True):
drupal_username = extra_data.get("preferred_username")
drupal_email = extra_data.get("email")
first_name = extra_data.get("first_name")
@@ -52,13 +52,15 @@ def update_user_info(self, user, extra_data: Dict):
user.email = drupal_email
user_changed = True
- if user_changed is True:
+ if user_changed is True and apply_update is True:
user.save()
+ return user_changed
- def update_user_study_sites(self, user, extra_data: Dict):
+ def update_user_study_sites(self, user, extra_data: Dict, apply_update=True):
# Get list of research centers in domain table
research_center_or_site = extra_data.get("study_site_or_center")
+ user_sites_updated = False
if research_center_or_site:
if not isinstance(research_center_or_site, list):
raise ImproperlyConfigured(
@@ -79,19 +81,24 @@ def update_user_study_sites(self, user, extra_data: Dict):
continue
else:
if not user.study_sites.filter(pk=rc.pk):
- user.study_sites.add(rc)
- logger.info(
- f"[SocialAccountAdatpter:update_user_study_sites] adding user "
- f"study_sites user: {user} rc: {rc}"
- )
+ user_sites_updated = True
+ if apply_update is True:
+ user.study_sites.add(rc)
+ logger.info(
+ f"[SocialAccountAdatpter:update_user_study_sites] adding user "
+ f"study_sites user: {user} rc: {rc}"
+ )
for existing_rc in user.study_sites.all():
if existing_rc.short_name not in research_center_or_site:
- user.study_sites.remove(existing_rc)
- logger.info(
- "[SocialAccountAdatpter:update_user_study_sites] "
- f"removing study_site {existing_rc} for user {user}"
- )
+ user_sites_updated = True
+ if apply_update:
+ user.study_sites.remove(existing_rc)
+ logger.info(
+ "[SocialAccountAdatpter:update_user_study_sites] "
+ f"removing study_site {existing_rc} for user {user}"
+ )
+ return user_sites_updated
def update_user_groups(self, user, extra_data: Dict):
managed_scope_status = extra_data.get("managed_scope_status")
diff --git a/primed/users/audit.py b/primed/users/audit.py
new file mode 100644
index 00000000..371d7760
--- /dev/null
+++ b/primed/users/audit.py
@@ -0,0 +1,527 @@
+import logging
+from dataclasses import dataclass
+
+import django_tables2 as tables
+import jsonapi_requests
+from allauth.socialaccount.models import SocialAccount
+from anvil_consortium_manager.models import Account
+from django.conf import settings
+from django.contrib.auth import get_user_model
+from django.core.exceptions import ObjectDoesNotExist
+from django.db.models import Q
+from django_tables2.export import TableExport
+from oauthlib.oauth2 import BackendApplicationClient
+from requests_oauthlib import OAuth2, OAuth2Session
+
+from primed.drupal_oauth_provider.provider import CustomProvider
+from primed.primed_anvil.audit import PRIMEDAudit, PRIMEDAuditResult
+from primed.primed_anvil.models import StudySite
+
+logger = logging.getLogger(__name__)
+
+
+class TextTable(object):
+ def render_to_text(self):
+ return TableExport(export_format=TableExport.CSV, table=self).export()
+
+
+class UserAuditResultsTable(tables.Table, TextTable):
+ """A table to show results from a UserAudit instance."""
+
+ result_type = tables.Column()
+ local_user_id = tables.Column()
+ local_username = tables.Column()
+ remote_user_id = tables.Column()
+ remote_username = tables.Column()
+ changes = tables.Column()
+ note = tables.Column()
+ anvil_groups = tables.Column()
+
+ class Meta:
+ orderable = False
+
+
+@dataclass
+class UserAuditResult(PRIMEDAuditResult):
+ local_user: SocialAccount = None
+ anvil_account: Account = None
+ remote_user_data: jsonapi_requests.JsonApiObject = None
+ note: str = None
+ changes: dict = None
+ anvil_groups: list = None
+
+ def get_table_dictionary(self):
+ """Return a dictionary that can be used to populate an instance of `SiteAuditResultsTable`."""
+
+ row = {
+ "changes": self.changes,
+ "anvil_groups": self.anvil_groups,
+ "note": self.note,
+ "result_type": type(self).__name__,
+ }
+ if self.local_user:
+ row.update(
+ {
+ "local_user_id": self.local_user.user.id,
+ "local_username": self.local_user.user.username,
+ }
+ )
+ if self.remote_user_data:
+ row.update(
+ {
+ "remote_user_id": self.remote_user_data.attributes.get(
+ "drupal_internal__uid"
+ ),
+ "remote_username": self.remote_user_data.attributes.get("name"),
+ }
+ )
+ if self.anvil_account:
+ row.update(
+ {
+ "anvil_account": self.anvil_account,
+ "local_user_id": self.anvil_account.user.id,
+ }
+ )
+ return row
+
+
+@dataclass
+class VerifiedUser(UserAuditResult):
+ pass
+
+
+@dataclass
+class NewUser(UserAuditResult):
+ pass
+
+
+@dataclass
+class RemoveUser(UserAuditResult):
+ pass
+
+
+@dataclass
+class InactiveAnvilUser(UserAuditResult):
+ pass
+
+
+@dataclass
+class UpdateUser(UserAuditResult):
+ pass
+
+
+@dataclass
+class OverDeactivateThresholdUser(UserAuditResult):
+ pass
+
+
+class UserAudit(PRIMEDAudit):
+ ISSUE_TYPE_USER_INACTIVE = "User is inactive in drupal"
+ ISSUE_TYPE_USER_REMOVED_FROM_SITE = "User removed from site"
+ USER_DEACTIVATE_THRESHOLD = 3
+ results_table_class = UserAuditResultsTable
+
+ def __init__(self, apply_changes=False, ignore_deactivate_threshold=False):
+ """Initialize the audit.
+
+ Args:
+ apply_changes: Whether to make changes to align the audit
+ """
+ super().__init__()
+ self.apply_changes = apply_changes
+ self.ignore_deactivate_threshold = ignore_deactivate_threshold
+
+ def _run_audit(self):
+ """Run the audit on local and remote users."""
+ user_endpoint_url = "user/user"
+ drupal_uids = set()
+ json_api = get_drupal_json_api()
+ study_sites = get_study_sites(json_api)
+
+ user_count = 0
+ while user_endpoint_url is not None:
+
+ users_endpoint = json_api.endpoint(user_endpoint_url)
+ users_endpoint_response = users_endpoint.get()
+
+ # If there are more, there will be a 'next' link
+
+ user_endpoint_url = users_endpoint_response.content.links.get(
+ "next", {}
+ ).get("href")
+
+ for user in users_endpoint_response.data:
+ drupal_uid = user.attributes.get("drupal_internal__uid")
+ drupal_username = user.attributes.get("name")
+ drupal_email = user.attributes.get("mail")
+ drupal_firstname = user.attributes.get("field_given_first_name_s_")
+ drupal_lastname = user.attributes.get(
+ "field_examples_family_last_name_"
+ )
+ drupal_full_name = " ".join(
+ part for part in (drupal_firstname, drupal_lastname) if part
+ )
+ drupal_study_sites_rel = user.relationships.get(
+ "field_study_site_or_center"
+ )
+ drupal_user_study_site_shortnames = []
+ if drupal_study_sites_rel:
+ for dss in drupal_study_sites_rel.data:
+ study_site_uuid = dss.id
+ study_site_info = study_sites[study_site_uuid]
+
+ drupal_user_study_site_shortnames.append(
+ study_site_info["short_name"]
+ )
+ new_user_sites = StudySite.objects.filter(
+ short_name__in=drupal_user_study_site_shortnames
+ )
+ # no uid is blocked or anonymous
+ if not drupal_uid:
+ # potential blocked user, but will no longer have a drupal uid
+ # so we cover these below
+ continue
+ sa = None
+ try:
+ sa = SocialAccount.objects.get(
+ uid=user.attributes["drupal_internal__uid"],
+ provider=CustomProvider.id,
+ )
+ except ObjectDoesNotExist:
+ drupal_user = get_user_model()()
+ drupal_user.username = drupal_username
+ drupal_user.name = drupal_full_name
+ drupal_user.email = drupal_email
+ if self.apply_changes is True:
+ drupal_user.save()
+ drupal_user.study_sites.set(new_user_sites)
+ if self.apply_changes is True:
+ sa = SocialAccount.objects.create(
+ user=drupal_user,
+ uid=user.attributes["drupal_internal__uid"],
+ provider=CustomProvider.id,
+ )
+ self.needs_action.append(
+ NewUser(local_user=sa, remote_user_data=user)
+ )
+
+ if sa:
+ user_updates = {}
+ if sa.user.name != drupal_full_name:
+ user_updates.update(
+ {"name": {"old": sa.user.name, "new": drupal_full_name}}
+ )
+ sa.user.name = drupal_full_name
+ if sa.user.username != drupal_username:
+ user_updates.update(
+ {
+ "username": {
+ "old": sa.user.username,
+ "new": drupal_username,
+ }
+ }
+ )
+ sa.user.username = drupal_username
+ if sa.user.email != drupal_email:
+ user_updates.update(
+ {"email": {"old": sa.user.email, "new": drupal_email}}
+ )
+ sa.user.email = drupal_email
+
+ if sa.user.is_active is False:
+ user_updates.update({"is_active": {"old": False, "new": True}})
+ sa.user.is_active = True
+
+ prev_user_site_names = set(
+ sa.user.study_sites.all().values_list("short_name", flat=True)
+ )
+ new_user_site_names = set(drupal_user_study_site_shortnames)
+ if prev_user_site_names != new_user_site_names:
+
+ user_updates.update(
+ {
+ "sites": {
+ "old": prev_user_site_names,
+ "new": new_user_site_names,
+ }
+ }
+ )
+ # do not remove from sites by default
+ removed_sites = prev_user_site_names.difference(
+ new_user_site_names
+ )
+ new_sites = new_user_site_names.difference(prev_user_site_names)
+
+ if settings.DRUPAL_DATA_AUDIT_REMOVE_USER_SITES is True:
+ if self.apply_changes is True:
+ sa.user.study_sites.set(new_user_sites)
+ else:
+ if removed_sites:
+ self.errors.append(
+ UpdateUser(
+ local_user=sa,
+ remote_user_data=user,
+ changes=user_updates,
+ )
+ )
+ if new_sites:
+ for new_site in new_user_sites:
+ if new_site.short_name in new_user_site_names:
+ if self.apply_changes is True:
+ sa.user.study_sites.add(new_site)
+
+ if user_updates:
+ if self.apply_changes is True:
+ sa.user.save()
+
+ self.needs_action.append(
+ UpdateUser(
+ local_user=sa,
+ remote_user_data=user,
+ changes=user_updates,
+ )
+ )
+ else:
+ self.verified.append(
+ VerifiedUser(local_user=sa, remote_user_data=user)
+ )
+
+ drupal_uids.add(drupal_uid)
+ user_count += 1
+
+ # find active django accounts that are drupal based
+ # users that we did not get from drupal
+ # these may include blocked users
+
+ unaudited_drupal_accounts = SocialAccount.objects.filter(
+ provider=CustomProvider.id, user__is_active=True
+ ).exclude(uid__in=drupal_uids)
+ user_ids_to_check = []
+ count_inactive = unaudited_drupal_accounts.count()
+ over_threshold = False
+ if self.ignore_deactivate_threshold is False:
+ if count_inactive > self.USER_DEACTIVATE_THRESHOLD:
+ over_threshold = True
+
+ for uda in unaudited_drupal_accounts:
+ user_ids_to_check.append(uda.user.id)
+ handled = False
+ if settings.DRUPAL_DATA_AUDIT_DEACTIVATE_USERS is True:
+ uda.user.is_active = False
+ if over_threshold is False:
+ if self.apply_changes is True:
+ uda.user.save()
+ handled = True
+ self.needs_action.append(RemoveUser(local_user=uda))
+ if handled is False:
+ self.errors.append(
+ RemoveUser(local_user=uda, note=f"Over Threshold {over_threshold}")
+ )
+
+ inactive_anvil_users = Account.objects.filter(
+ Q(user__is_active=False) | Q(user__id__in=user_ids_to_check),
+ groupaccountmembership__isnull=False,
+ )
+ for inactive_anvil_user in inactive_anvil_users:
+ self.errors.append(
+ InactiveAnvilUser(
+ anvil_account=inactive_anvil_user,
+ anvil_groups=list(
+ inactive_anvil_user.groupaccountmembership_set.all().values_list(
+ "group__name", flat=True
+ )
+ ),
+ )
+ )
+
+
+class SiteAuditResultsTable(tables.Table, TextTable):
+ """A table to show results from a SiteAudit instance."""
+
+ result_type = tables.Column()
+ local_site_name = tables.Column()
+ remote_site_name = tables.Column()
+ changes = tables.Column()
+ note = tables.Column()
+
+ def value_local_site_name(self, value):
+ return value
+
+ class Meta:
+ orderable = False
+
+
+@dataclass
+class SiteAuditResult(PRIMEDAuditResult):
+ local_site: StudySite
+ remote_site_data: jsonapi_requests.JsonApiObject = None
+ changes: dict = None
+ note: str = None
+
+ def get_table_dictionary(self):
+ """Return a dictionary that can be used to populate an instance of `SiteAuditResultsTable`."""
+ row = {
+ "changes": self.changes,
+ "note": self.note,
+ "result_type": type(self).__name__,
+ }
+ if self.local_site:
+ row.update(
+ {
+ "local_site_name": self.local_site.short_name,
+ }
+ )
+ if self.remote_site_data:
+ row.update(
+ {
+ "remote_site_name": self.remote_site_data.get("short_name"),
+ }
+ )
+ return row
+
+
+@dataclass
+class VerifiedSite(SiteAuditResult):
+ pass
+
+
+@dataclass
+class NewSite(SiteAuditResult):
+ pass
+
+
+@dataclass
+class RemoveSite(SiteAuditResult):
+ pass
+
+
+@dataclass
+class UpdateSite(SiteAuditResult):
+ changes: dict
+
+
+class SiteAudit(PRIMEDAudit):
+ ISSUE_TYPE_LOCAL_SITE_INVALID = "Local site is invalid"
+ results_table_class = SiteAuditResultsTable
+
+ def __init__(self, apply_changes=False):
+ """Initialize the audit.
+
+ Args:
+ apply_changes: Whether to make changes to align the audit
+ """
+ super().__init__()
+ self.apply_changes = apply_changes
+
+ def _run_audit(self):
+ """Run the audit on local and remote users."""
+ valid_nodes = set()
+ json_api = get_drupal_json_api()
+ study_sites = get_study_sites(json_api=json_api)
+ for study_site_info in study_sites.values():
+
+ short_name = study_site_info["short_name"]
+ full_name = study_site_info["full_name"]
+ node_id = study_site_info["node_id"]
+ valid_nodes.add(node_id)
+
+ try:
+ study_site = StudySite.objects.get(drupal_node_id=node_id)
+ except ObjectDoesNotExist:
+ study_site = None
+ if self.apply_changes is True:
+ study_site = StudySite.objects.create(
+ drupal_node_id=node_id,
+ short_name=short_name,
+ full_name=full_name,
+ )
+ self.needs_action.append(
+ NewSite(remote_site_data=study_site_info, local_site=study_site)
+ )
+ else:
+ study_site_updates = {}
+
+ if study_site.full_name != full_name:
+ study_site_updates.update(
+ {"full_name": {"old": study_site.full_name, "new": full_name}}
+ )
+ study_site.full_name = full_name
+
+ if study_site.short_name != short_name:
+ study_site_updates.update(
+ {
+ "short_name": {
+ "old": study_site.short_name,
+ "new": short_name,
+ }
+ }
+ )
+ study_site.short_name = short_name
+
+ if study_site_updates:
+ if self.apply_changes is True:
+ study_site.save()
+ self.needs_action.append(
+ UpdateSite(
+ local_site=study_site,
+ remote_site_data=study_site_info,
+ changes=study_site_updates,
+ )
+ )
+ else:
+ self.verified.append(
+ VerifiedSite(
+ local_site=study_site, remote_site_data=study_site_info
+ )
+ )
+
+ invalid_study_sites = StudySite.objects.exclude(drupal_node_id__in=valid_nodes)
+
+ for iss in invalid_study_sites:
+ self.errors.append(
+ RemoveSite(local_site=iss, note=self.ISSUE_TYPE_LOCAL_SITE_INVALID)
+ )
+
+
+def get_drupal_json_api():
+
+ json_api_client_id = settings.DRUPAL_API_CLIENT_ID
+ json_api_client_secret = settings.DRUPAL_API_CLIENT_SECRET
+
+ token_url = f"{settings.DRUPAL_SITE_URL}/oauth/token"
+ client = BackendApplicationClient(client_id=json_api_client_id)
+ oauth = OAuth2Session(client=client)
+ api_root = f"{settings.DRUPAL_SITE_URL}/{settings.DRUPAL_API_REL_PATH}"
+
+ token = oauth.fetch_token(
+ token_url=token_url,
+ client_id=json_api_client_id,
+ client_secret=json_api_client_secret,
+ )
+
+ drupal_api = jsonapi_requests.Api.config(
+ {
+ "API_ROOT": api_root,
+ "AUTH": OAuth2(client=client, client_id=json_api_client_id, token=token),
+ "VALIDATE_SSL": True,
+ }
+ )
+ return drupal_api
+
+
+def get_study_sites(json_api):
+ study_sites_endpoint = json_api.endpoint("node/study_site_or_center")
+ study_sites_response = study_sites_endpoint.get()
+ study_sites_info = dict()
+
+ for ss in study_sites_response.data:
+ short_name = ss.attributes["title"]
+ full_name = ss.attributes["field_long_name"]
+ node_id = ss.attributes["drupal_internal__nid"]
+
+ study_sites_info[ss.id] = {
+ "node_id": node_id,
+ "short_name": short_name,
+ "full_name": full_name,
+ }
+ return study_sites_info
diff --git a/primed/users/management/__init__.py b/primed/users/management/__init__.py
new file mode 100644
index 00000000..e69de29b
diff --git a/primed/users/management/commands/__init__.py b/primed/users/management/commands/__init__.py
new file mode 100644
index 00000000..e69de29b
diff --git a/primed/users/management/commands/sync-drupal-data.py b/primed/users/management/commands/sync-drupal-data.py
new file mode 100644
index 00000000..d58113cc
--- /dev/null
+++ b/primed/users/management/commands/sync-drupal-data.py
@@ -0,0 +1,105 @@
+import logging
+
+from django.core.mail import send_mail
+from django.core.management.base import BaseCommand
+from django.http import HttpRequest
+from django.template.loader import render_to_string
+from django.utils.timezone import localtime
+
+from primed.users import audit
+
+logger = logging.getLogger(__name__)
+
+
+class Command(BaseCommand):
+ help = "Sync drupal user and domain data"
+
+ def add_arguments(self, parser):
+ parser.add_argument(
+ "--update",
+ action="store_true",
+ dest="update",
+ default=False,
+ help="Make updates to sync local data with remote. If not set, will just report.",
+ )
+ parser.add_argument(
+ "--ignore-threshold",
+ action="store_true",
+ dest="ignore_threshold",
+ default=False,
+ help="Ignore user deactivation threshold",
+ )
+
+ parser.add_argument(
+ "--email",
+ help="""Email to which to send audit result details that need action or have errors.""",
+ )
+
+ def _send_email(self, user_audit, site_audit):
+ # Send email if requested and there are problems.
+ if user_audit.ok() is False or site_audit.ok() is False:
+ # django-tables2 requires request context, so we create an empty one
+ # if we wanted to linkify any of our data we would need to do more here
+ request = HttpRequest()
+ subject = "[command:sync-drupal-data] report"
+ html_body = render_to_string(
+ "users/drupal_data_audit_email.html",
+ context={
+ "user_audit": user_audit,
+ "site_audit": site_audit,
+ "request": request,
+ "apply_changes": self.apply_changes,
+ },
+ )
+ send_mail(
+ subject,
+ "Drupal data audit problems or changes found. Please see attached report.",
+ None,
+ [self.email],
+ fail_silently=False,
+ html_message=html_body,
+ )
+
+ def handle(self, *args, **options):
+ self.apply_changes = options.get("update")
+ self.email = options["email"]
+ self.ignore_threshold = options["ignore_threshold"]
+
+ notification_content = (
+ f"[sync-drupal-data] start: Applying Changes: {self.apply_changes} "
+ f"Ignoring Threshold: {self.ignore_threshold} Start time: {localtime()}\n"
+ )
+ site_audit = audit.SiteAudit(apply_changes=self.apply_changes)
+ site_audit.run_audit()
+
+ notification_content += (
+ f"SiteAudit summary: status ok: {site_audit.ok()} verified: {len(site_audit.verified)} "
+ f"needs_changes: {len(site_audit.needs_action)} errors: {len(site_audit.errors)}\n"
+ )
+ if site_audit.needs_action:
+ notification_content += "Sites that need syncing:\n"
+ notification_content += site_audit.get_needs_action_table().render_to_text()
+ if site_audit.errors:
+ notification_content += "Sites requiring intervention:\n"
+ notification_content += site_audit.get_errors_table().render_to_text()
+
+ user_audit = audit.UserAudit(
+ apply_changes=self.apply_changes,
+ ignore_deactivate_threshold=self.ignore_threshold,
+ )
+ user_audit.run_audit()
+ notification_content += (
+ "--------------------------------------\n"
+ f"UserAudit summary: status ok: {user_audit.ok()} verified: {len(user_audit.verified)} "
+ f"needs_changes: {len(user_audit.needs_action)} errors: {len(user_audit.errors)}\n"
+ )
+ if user_audit.needs_action:
+ notification_content += "Users that need syncing:\n"
+ notification_content += user_audit.get_needs_action_table().render_to_text()
+ if user_audit.errors:
+ notification_content += "Users that need intervention:\n"
+ notification_content += user_audit.get_errors_table().render_to_text()
+
+ self.stdout.write(notification_content)
+ if self.email:
+ self._send_email(user_audit, site_audit)
diff --git a/primed/users/tests/test_audit.py b/primed/users/tests/test_audit.py
new file mode 100644
index 00000000..155d114e
--- /dev/null
+++ b/primed/users/tests/test_audit.py
@@ -0,0 +1,612 @@
+import json
+import time
+from io import StringIO
+
+import responses
+from allauth.socialaccount.models import SocialAccount
+from anvil_consortium_manager.models import (
+ Account,
+ GroupAccountMembership,
+ ManagedGroup,
+)
+from django.conf import settings
+from django.contrib.auth import get_user_model
+from django.core import mail
+from django.core.management import call_command
+from django.test import TestCase
+from marshmallow_jsonapi import Schema, fields
+
+from primed.drupal_oauth_provider.provider import CustomProvider
+from primed.users import audit
+from primed.users.models import StudySite
+
+
+class StudySiteMockObject:
+ def __init__(self, id, title, field_long_name, drupal_internal__nid) -> None:
+ self.id = id
+ self.title = title
+ self.field_long_name = field_long_name
+ self.drupal_internal__nid = drupal_internal__nid
+
+
+class UserMockObject:
+ def __init__(
+ self,
+ id,
+ display_name,
+ drupal_internal__uid,
+ name,
+ mail,
+ field_given_first_name_s_,
+ field_examples_family_last_name_,
+ field_study_site_or_center,
+ ) -> None:
+ self.id = id
+ self.display_name = display_name
+ self.drupal_internal__uid = drupal_internal__uid
+ self.name = name
+ self.mail = mail
+ self.field_given_first_name_s_ = field_given_first_name_s_
+ self.field_examples_family_last_name_ = field_examples_family_last_name_
+ self.field_study_site_or_center = field_study_site_or_center
+
+
+class StudySiteSchema(Schema):
+ id = fields.Str(dump_only=True)
+ title = fields.Str()
+ field_long_name = fields.Str()
+ drupal_internal__nid = fields.Str()
+ # document_meta = fields.DocumentMeta()
+
+ class Meta:
+ type_ = "node--study_site_or_center"
+
+
+class UserSchema(Schema):
+ id = fields.Str(dump_only=True)
+ display_name = fields.Str()
+ drupal_internal__uid = fields.Str()
+ name = fields.Str()
+ mail = fields.Str()
+ field_given_first_name_s_ = fields.Str()
+ field_examples_family_last_name_ = fields.Str()
+ field_study_site_or_center = fields.Relationship(
+ many=True, schema="StudySiteSchema", type_="node--study_site_or_center"
+ )
+
+ class Meta:
+ type_ = "users"
+
+
+# def debug_requests_on():
+# """Switches on logging of the requests module."""
+# HTTPConnection.debuglevel = 1
+
+# logging.basicConfig()
+# logging.getLogger().setLevel(logging.DEBUG)
+# requests_log = logging.getLogger("requests.packages.urllib3")
+# requests_log.setLevel(logging.DEBUG)
+# requests_log.propagate = True
+
+
+TEST_STUDY_SITE_DATA = [
+ StudySiteMockObject(
+ **{
+ "id": "1",
+ "drupal_internal__nid": "1",
+ "title": "SS1",
+ "field_long_name": "S S 1",
+ # "document_meta": {"page": {"offset": 10}},
+ }
+ ),
+ StudySiteMockObject(
+ **{
+ "id": "2",
+ "drupal_internal__nid": "2",
+ "title": "SS2",
+ "field_long_name": "S S 2",
+ # "document_meta": {"page": {"offset": 10}},
+ }
+ ),
+]
+
+TEST_USER_DATA = [
+ UserMockObject(
+ **{
+ "id": "usr1",
+ "display_name": "dnusr1",
+ "drupal_internal__uid": "usr1",
+ "name": "testuser1",
+ "mail": "testuser1@test.com",
+ "field_given_first_name_s_": "test1",
+ "field_examples_family_last_name_": "user1",
+ "field_study_site_or_center": [],
+ }
+ ),
+ # second mock object is deactivated user (no drupal uid)
+ UserMockObject(
+ **{
+ "id": "usr2",
+ "display_name": "dnusr2",
+ "drupal_internal__uid": "",
+ "name": "testuser2",
+ "mail": "testuser2@test.com",
+ "field_given_first_name_s_": "test2",
+ "field_examples_family_last_name_": "user2",
+ "field_study_site_or_center": [],
+ }
+ ),
+]
+
+
+class TestUserDataAudit(TestCase):
+ """General tests of the user audit"""
+
+ def setUp(self):
+ # debug_requests_on()
+ super().setUp()
+ fake_time = time.time()
+ self.token = {
+ "token_type": "Bearer",
+ "access_token": "asdfoiw37850234lkjsdfsdfTEST", # gitleaks:allow
+ "refresh_token": "sldvafkjw34509s8dfsdfTEST", # gitleaks:allow
+ "expires_in": 3600,
+ "expires_at": fake_time + 3600,
+ }
+
+ def add_fake_study_sites_response(self):
+ url_path = f"{settings.DRUPAL_SITE_URL}/{settings.DRUPAL_API_REL_PATH}/node/study_site_or_center/"
+ responses.get(
+ url=url_path,
+ body=json.dumps(StudySiteSchema(many=True).dump(TEST_STUDY_SITE_DATA)),
+ )
+
+ def add_fake_users_response(self):
+ url_path = (
+ f"{settings.DRUPAL_SITE_URL}/{settings.DRUPAL_API_REL_PATH}/user/user/"
+ )
+ TEST_USER_DATA[0].field_study_site_or_center = [TEST_STUDY_SITE_DATA[0]]
+ user_data = UserSchema(
+ include_data=("field_study_site_or_center",), many=True
+ ).dump(TEST_USER_DATA)
+
+ responses.get(
+ url=url_path,
+ body=json.dumps(user_data),
+ )
+
+ def add_fake_token_response(self):
+ token_url = f"{settings.DRUPAL_SITE_URL}/oauth/token"
+ responses.post(url=token_url, body=json.dumps(self.token))
+
+ def get_fake_json_api(self):
+ self.add_fake_token_response()
+ return audit.get_drupal_json_api()
+
+ @responses.activate
+ def test_get_json_api(self):
+ json_api = self.get_fake_json_api()
+ assert (
+ json_api.requests.config.AUTH._client.token["access_token"]
+ == self.token["access_token"]
+ )
+
+ @responses.activate
+ def test_get_study_sites(self):
+ json_api = self.get_fake_json_api()
+ self.add_fake_study_sites_response()
+ study_sites = audit.get_study_sites(json_api=json_api)
+
+ for test_study_site in TEST_STUDY_SITE_DATA:
+
+ assert (
+ test_study_site.field_long_name
+ == study_sites[test_study_site.drupal_internal__nid]["full_name"]
+ )
+ assert (
+ test_study_site.title
+ == study_sites[test_study_site.drupal_internal__nid]["short_name"]
+ )
+ assert (
+ test_study_site.drupal_internal__nid
+ == study_sites[test_study_site.drupal_internal__nid]["node_id"]
+ )
+
+ @responses.activate
+ def test_audit_study_sites_no_update(self):
+ self.get_fake_json_api()
+ self.add_fake_study_sites_response()
+ site_audit = audit.SiteAudit(apply_changes=False)
+ site_audit.run_audit()
+ self.assertFalse(site_audit.ok())
+ self.assertEqual(len(site_audit.errors), 0)
+ self.assertEqual(len(site_audit.needs_action), 2)
+ self.assertEqual(StudySite.objects.all().count(), 0)
+
+ @responses.activate
+ def test_audit_study_sites_with_new_sites(self):
+ self.get_fake_json_api()
+ self.add_fake_study_sites_response()
+ site_audit = audit.SiteAudit(apply_changes=True)
+ site_audit.run_audit()
+ self.assertFalse(site_audit.ok())
+ self.assertEqual(len(site_audit.needs_action), 2)
+ self.assertEqual(StudySite.objects.all().count(), 2)
+
+ assert (
+ StudySite.objects.filter(
+ short_name__in=[
+ TEST_STUDY_SITE_DATA[0].title,
+ TEST_STUDY_SITE_DATA[1].title,
+ ]
+ ).count()
+ == 2
+ )
+ assert len(site_audit.get_needs_action_table().rows) == 2
+
+ @responses.activate
+ def test_audit_study_sites_with_site_update(self):
+ StudySite.objects.create(
+ drupal_node_id=TEST_STUDY_SITE_DATA[0].drupal_internal__nid,
+ short_name="WrongShortName",
+ full_name="WrongTitle",
+ )
+ StudySite.objects.create(
+ drupal_node_id=TEST_STUDY_SITE_DATA[1].drupal_internal__nid,
+ short_name=TEST_STUDY_SITE_DATA[1].title,
+ full_name=TEST_STUDY_SITE_DATA[1].field_long_name,
+ )
+ self.get_fake_json_api()
+ self.add_fake_study_sites_response()
+ site_audit = audit.SiteAudit(apply_changes=True)
+ site_audit.run_audit()
+ self.assertFalse(site_audit.ok())
+ self.assertEqual(len(site_audit.needs_action), 1)
+ self.assertEqual(len(site_audit.verified), 1)
+ self.assertEqual(len(site_audit.errors), 0)
+ self.assertEqual(StudySite.objects.all().count(), 2)
+
+ first_test_ss = StudySite.objects.get(short_name=TEST_STUDY_SITE_DATA[0].title)
+ # did we update the long name
+ assert first_test_ss.full_name == TEST_STUDY_SITE_DATA[0].field_long_name
+ assert first_test_ss.short_name == TEST_STUDY_SITE_DATA[0].title
+
+ @responses.activate
+ def test_audit_study_sites_with_extra_site(self):
+ StudySite.objects.create(
+ drupal_node_id=99, short_name="ExtraSite", full_name="ExtraSiteLong"
+ )
+ self.get_fake_json_api()
+ self.add_fake_study_sites_response()
+ site_audit = audit.SiteAudit(apply_changes=True)
+ site_audit.run_audit()
+ self.assertFalse(site_audit.ok())
+ self.assertEqual(len(site_audit.errors), 1)
+ self.assertEqual(StudySite.objects.all().count(), 3)
+ assert len(site_audit.get_errors_table().rows) == 1
+
+ @responses.activate
+ def test_full_user_audit(self):
+ self.add_fake_token_response()
+ self.add_fake_study_sites_response()
+ self.add_fake_users_response()
+ StudySite.objects.create(
+ drupal_node_id=TEST_STUDY_SITE_DATA[0].drupal_internal__nid,
+ short_name=TEST_STUDY_SITE_DATA[0].title,
+ full_name=TEST_STUDY_SITE_DATA[0].field_long_name,
+ )
+ user_audit = audit.UserAudit(apply_changes=True)
+ user_audit.run_audit()
+
+ self.assertFalse(user_audit.ok())
+ self.assertEqual(len(user_audit.needs_action), 1)
+
+ users = get_user_model().objects.all()
+ assert users.count() == 1
+
+ assert users.first().email == TEST_USER_DATA[0].mail
+ assert users.first().username == TEST_USER_DATA[0].name
+ assert users.first().study_sites.count() == 1
+ assert (
+ users.first().study_sites.first().short_name
+ == TEST_STUDY_SITE_DATA[0].title
+ )
+
+ @responses.activate
+ def test_full_user_audit_check_only(self):
+ self.add_fake_token_response()
+ self.add_fake_study_sites_response()
+ self.add_fake_users_response()
+ StudySite.objects.create(
+ drupal_node_id=TEST_STUDY_SITE_DATA[0].drupal_internal__nid,
+ short_name=TEST_STUDY_SITE_DATA[0].title,
+ full_name=TEST_STUDY_SITE_DATA[0].field_long_name,
+ )
+ user_audit = audit.UserAudit(apply_changes=False)
+ user_audit.run_audit()
+ self.assertFalse(user_audit.ok())
+ self.assertEqual(len(user_audit.needs_action), 1)
+
+ # verify we did not actually create a user
+ users = get_user_model().objects.all()
+ assert users.count() == 0
+
+ @responses.activate
+ def test_user_audit_remove_site_inform(self):
+ self.add_fake_token_response()
+ self.add_fake_study_sites_response()
+ self.add_fake_users_response()
+ ss1 = StudySite.objects.create(
+ drupal_node_id=TEST_STUDY_SITE_DATA[1].drupal_internal__nid,
+ short_name=TEST_STUDY_SITE_DATA[1].title,
+ full_name=TEST_STUDY_SITE_DATA[1].field_long_name,
+ )
+ drupal_fullname = "{} {}".format(
+ TEST_USER_DATA[0].field_given_first_name_s_,
+ TEST_USER_DATA[0].field_examples_family_last_name_,
+ )
+ drupal_username = TEST_USER_DATA[0].name
+ drupal_email = TEST_USER_DATA[0].mail
+ new_user = get_user_model().objects.create(
+ username=drupal_username + "UPDATE",
+ email=drupal_email + "UPDATE",
+ name=drupal_fullname + "UPDATE",
+ )
+ new_user.study_sites.add(ss1)
+ SocialAccount.objects.create(
+ user=new_user,
+ uid=TEST_USER_DATA[0].drupal_internal__uid,
+ provider=CustomProvider.id,
+ )
+ user_audit = audit.UserAudit(apply_changes=False)
+ user_audit.run_audit()
+ self.assertFalse(user_audit.ok())
+ self.assertEqual(len(user_audit.errors), 1)
+
+ new_user.refresh_from_db()
+ # assert we did not remove the site
+ assert ss1 in new_user.study_sites.all()
+
+ @responses.activate
+ def test_user_audit_remove_site_act(self):
+ self.add_fake_token_response()
+ self.add_fake_study_sites_response()
+ self.add_fake_users_response()
+ ss1 = StudySite.objects.create(
+ drupal_node_id=TEST_STUDY_SITE_DATA[1].drupal_internal__nid,
+ short_name=TEST_STUDY_SITE_DATA[1].title,
+ full_name=TEST_STUDY_SITE_DATA[1].field_long_name,
+ )
+ drupal_fullname = "{} {}".format(
+ TEST_USER_DATA[0].field_given_first_name_s_,
+ TEST_USER_DATA[0].field_examples_family_last_name_,
+ )
+ drupal_username = TEST_USER_DATA[0].name
+ drupal_email = TEST_USER_DATA[0].mail
+ new_user = get_user_model().objects.create(
+ username=drupal_username + "UPDATE",
+ email=drupal_email + "UPDATE",
+ name=drupal_fullname + "UPDATE",
+ )
+ new_user.study_sites.add(ss1)
+ SocialAccount.objects.create(
+ user=new_user,
+ uid=TEST_USER_DATA[0].drupal_internal__uid,
+ provider=CustomProvider.id,
+ )
+ with self.settings(DRUPAL_DATA_AUDIT_REMOVE_USER_SITES=True):
+ user_audit = audit.UserAudit(apply_changes=True)
+ user_audit.run_audit()
+ self.assertFalse(user_audit.ok())
+ new_user.refresh_from_db()
+ # assert we did remove the site
+ assert ss1 not in new_user.study_sites.all()
+
+ @responses.activate
+ def test_user_audit_change_user(self):
+ self.add_fake_token_response()
+ self.add_fake_study_sites_response()
+ self.add_fake_users_response()
+ StudySite.objects.create(
+ drupal_node_id=TEST_STUDY_SITE_DATA[0].drupal_internal__nid,
+ short_name=TEST_STUDY_SITE_DATA[0].title,
+ full_name=TEST_STUDY_SITE_DATA[0].field_long_name,
+ )
+ drupal_fullname = "{} {}".format(
+ TEST_USER_DATA[0].field_given_first_name_s_,
+ TEST_USER_DATA[0].field_examples_family_last_name_,
+ )
+ drupal_username = TEST_USER_DATA[0].name
+ drupal_email = TEST_USER_DATA[0].mail
+ new_user = get_user_model().objects.create(
+ username=drupal_username + "UPDATE",
+ email=drupal_email + "UPDATE",
+ name=drupal_fullname + "UPDATE",
+ is_active=False,
+ )
+ SocialAccount.objects.create(
+ user=new_user,
+ uid=TEST_USER_DATA[0].drupal_internal__uid,
+ provider=CustomProvider.id,
+ )
+ user_audit = audit.UserAudit(apply_changes=True)
+ user_audit.run_audit()
+ self.assertFalse(user_audit.ok())
+ new_user.refresh_from_db()
+
+ self.assertEqual(new_user.name, drupal_fullname)
+ self.assertEqual(len(user_audit.needs_action), 1)
+
+ # test user removal
+ @responses.activate
+ def test_user_audit_remove_user_only_inform(self):
+ self.add_fake_token_response()
+ self.add_fake_study_sites_response()
+ self.add_fake_users_response()
+ StudySite.objects.create(
+ drupal_node_id=TEST_STUDY_SITE_DATA[0].drupal_internal__nid,
+ short_name=TEST_STUDY_SITE_DATA[0].title,
+ full_name=TEST_STUDY_SITE_DATA[0].field_long_name,
+ )
+
+ new_user = get_user_model().objects.create(
+ username="username2", email="useremail2", name="user fullname2"
+ )
+ SocialAccount.objects.create(
+ user=new_user,
+ uid=999,
+ provider=CustomProvider.id,
+ )
+ user_audit = audit.UserAudit(apply_changes=True)
+ user_audit.run_audit()
+ self.assertFalse(user_audit.ok())
+
+ new_user.refresh_from_db()
+ self.assertTrue(new_user.is_active)
+
+ # test user removal
+ @responses.activate
+ def test_user_audit_remove_user(self):
+ self.add_fake_token_response()
+ self.add_fake_study_sites_response()
+ self.add_fake_users_response()
+ StudySite.objects.create(
+ drupal_node_id=TEST_STUDY_SITE_DATA[0].drupal_internal__nid,
+ short_name=TEST_STUDY_SITE_DATA[0].title,
+ full_name=TEST_STUDY_SITE_DATA[0].field_long_name,
+ )
+
+ new_user = get_user_model().objects.create(
+ username="username2", email="useremail2", name="user fullname2"
+ )
+ SocialAccount.objects.create(
+ user=new_user,
+ uid=999,
+ provider=CustomProvider.id,
+ )
+ new_anvil_account = Account.objects.create(
+ user=new_user,
+ is_service_account=False,
+ )
+ new_anvil_managed_group = ManagedGroup.objects.create(
+ name="testgroup",
+ email="testgroup@testgroup.org",
+ )
+ GroupAccountMembership.objects.create(
+ group=new_anvil_managed_group,
+ account=new_anvil_account,
+ role=GroupAccountMembership.MEMBER,
+ )
+
+ with self.settings(DRUPAL_DATA_AUDIT_DEACTIVATE_USERS=True):
+ user_audit = audit.UserAudit(apply_changes=True)
+ user_audit.run_audit()
+ self.assertFalse(user_audit.ok())
+ self.assertEqual(len(user_audit.errors), 1)
+ self.assertEqual(user_audit.errors[0].anvil_account, new_anvil_account)
+ self.assertIn(
+ "InactiveAnvilUser", user_audit.get_errors_table().render_to_text()
+ )
+ self.assertEqual(len(user_audit.needs_action), 2)
+ new_user.refresh_from_db()
+ self.assertFalse(new_user.is_active)
+
+ # test user removal
+ @responses.activate
+ def test_user_audit_remove_user_threshold(self):
+ self.add_fake_token_response()
+ self.add_fake_study_sites_response()
+ self.add_fake_users_response()
+ StudySite.objects.create(
+ drupal_node_id=TEST_STUDY_SITE_DATA[0].drupal_internal__nid,
+ short_name=TEST_STUDY_SITE_DATA[0].title,
+ full_name=TEST_STUDY_SITE_DATA[0].field_long_name,
+ )
+
+ SocialAccount.objects.create(
+ user=get_user_model().objects.create(
+ username="username2", email="useremail2", name="user fullname2"
+ ),
+ uid=996,
+ provider=CustomProvider.id,
+ )
+
+ SocialAccount.objects.create(
+ user=get_user_model().objects.create(
+ username="username3", email="useremail3", name="user fullname3"
+ ),
+ uid=997,
+ provider=CustomProvider.id,
+ )
+
+ SocialAccount.objects.create(
+ user=get_user_model().objects.create(
+ username="username4", email="useremail4", name="user fullname4"
+ ),
+ uid=998,
+ provider=CustomProvider.id,
+ )
+ SocialAccount.objects.create(
+ user=get_user_model().objects.create(
+ username="username5", email="useremail5", name="user fullname5"
+ ),
+ uid=999,
+ provider=CustomProvider.id,
+ )
+ with self.settings(DRUPAL_DATA_AUDIT_DEACTIVATE_USERS=True):
+ user_audit = audit.UserAudit(apply_changes=False)
+ user_audit.run_audit()
+ self.assertFalse(user_audit.ok())
+ self.assertEqual(len(user_audit.errors), 4)
+ self.assertEqual(len(user_audit.needs_action), 1)
+ self.assertEqual(user_audit.errors[0].note, "Over Threshold True")
+ # Run again with ignore threshold, should move from error to needs action
+ user_audit = audit.UserAudit(
+ apply_changes=False, ignore_deactivate_threshold=True
+ )
+ user_audit.run_audit()
+ self.assertFalse(user_audit.ok())
+ self.assertEqual(len(user_audit.errors), 0)
+ self.assertEqual(len(user_audit.needs_action), 5)
+
+ @responses.activate
+ def test_sync_drupal_data_command(self):
+ self.add_fake_token_response()
+ self.add_fake_study_sites_response()
+ self.add_fake_users_response()
+ out = StringIO()
+ call_command("sync-drupal-data", stdout=out)
+ self.assertIn(
+ "SiteAudit summary: status ok: False verified: 0 needs_changes: 2",
+ out.getvalue(),
+ )
+
+ @responses.activate
+ def test_sync_drupal_data_command_with_issues(self):
+
+ StudySite.objects.create(
+ drupal_node_id="999999",
+ short_name=TEST_STUDY_SITE_DATA[0].title,
+ full_name=TEST_STUDY_SITE_DATA[0].field_long_name,
+ )
+
+ new_user = get_user_model().objects.create(
+ username="username2", email="useremail2", name="user fullname2"
+ )
+ SocialAccount.objects.create(
+ user=new_user,
+ uid=999,
+ provider=CustomProvider.id,
+ )
+ self.add_fake_token_response()
+ self.add_fake_study_sites_response()
+ self.add_fake_users_response()
+ out = StringIO()
+ call_command("sync-drupal-data", "--email=test@example.com", stdout=out)
+ self.assertIn("SiteAudit summary: status ok: False", out.getvalue())
+ self.assertIn("UserAudit summary: status ok: False", out.getvalue())
+
+ self.assertEqual(len(mail.outbox), 1)
+ email = mail.outbox[0]
+ self.assertEqual(email.to, ["test@example.com"])
+ self.assertEqual(email.subject, "[command:sync-drupal-data] report")
diff --git a/requirements/dev-requirements.txt b/requirements/dev-requirements.txt
index 814e7b09..f2991d16 100644
--- a/requirements/dev-requirements.txt
+++ b/requirements/dev-requirements.txt
@@ -233,7 +233,7 @@ types-pytz==2024.1.0.20240203
# via django-stubs
types-pyyaml==6.0.12.12
# via django-stubs
-types-requests==2.31.0.20240310
+types-requests==2.31.0.20240406
# via -r requirements/dev-requirements.in
typing-extensions==4.8.0
# via
@@ -257,7 +257,7 @@ virtualenv==20.25.1
# via pre-commit
wcwidth==0.2.13
# via prompt-toolkit
-werkzeug==3.0.1
+werkzeug==3.0.2
# via -r requirements/dev-requirements.in
zipp==3.17.0
# via
diff --git a/requirements/requirements.in b/requirements/requirements.in
index dfc31fc7..b977ac59 100644
--- a/requirements/requirements.in
+++ b/requirements/requirements.in
@@ -44,6 +44,9 @@ requests
# For json schema validation.
jsonschema
+# For interacting with drupal json api
+jsonapi-requests
+
# For tree structures
django-tree-queries
@@ -66,3 +69,7 @@ django-htmx
certifi>=2023.7.22
urllib3>=1.26.18
sqlparse>=0.4.4
+
+# Dynamic settings
+django-constance
+django-picklefield # Required by django-constance for database backend
diff --git a/requirements/requirements.txt b/requirements/requirements.txt
index bf4c6ad2..54ee4636 100644
--- a/requirements/requirements.txt
+++ b/requirements/requirements.txt
@@ -58,6 +58,7 @@ django==4.2.11
# django-filter
# django-htmx
# django-model-utils
+ # django-picklefield
# django-tables2
django-allauth==0.61.1
# via -r requirements/requirements.in
@@ -65,6 +66,8 @@ django-anvil-consortium-manager @ git+https://github.com/UW-GAC/django-anvil-con
# via -r requirements/requirements.in
django-autocomplete-light==3.11.0
# via django-anvil-consortium-manager
+django-constance==3.1.0
+ # via -r requirements/requirements.in
django-crispy-forms==2.1
# via
# -r requirements/requirements.in
@@ -86,15 +89,19 @@ django-login-required-middleware==0.9.0
# via -r requirements/requirements.in
django-maintenance-mode==0.21.1
# via -r requirements/requirements.in
-django-model-utils==4.4.0
+django-model-utils==4.5.0
# via -r requirements/requirements.in
+django-picklefield==3.2
+ # via
+ # -r requirements/requirements.in
+ # django-constance
django-simple-history==3.5.0
# via
# -r requirements/requirements.in
# django-anvil-consortium-manager
django-tables2==2.7.0
# via django-anvil-consortium-manager
-django-tree-queries==0.16.1
+django-tree-queries==0.18.0
# via -r requirements/requirements.in
fastobo==0.12.3
# via pronto
@@ -110,6 +117,8 @@ importlib-resources==6.1.1
# via
# jsonschema
# jsonschema-specifications
+jsonapi-requests==0.7.0
+ # via -r requirements/requirements.in
jsonschema==4.21.1
# via -r requirements/requirements.in
jsonschema-specifications==2023.12.1
@@ -151,9 +160,7 @@ pyasn1-modules==0.3.0
pycparser==2.21
# via cffi
pyjwt[crypto]==2.4.0
- # via
- # django-allauth
- # pyjwt
+ # via django-allauth
pyparsing==3.1.1
# via packaging
pyproject-hooks==1.0.0
@@ -182,6 +189,7 @@ requests==2.31.0
# -r requirements/requirements.in
# django-allauth
# django-anvil-consortium-manager
+ # jsonapi-requests
# requests-oauthlib
requests-oauthlib==1.3.1
# via django-allauth
@@ -197,10 +205,12 @@ sqlparse==0.4.4
# via
# -r requirements/requirements.in
# django
-tablib==3.5.0
+tablib==3.6.1
# via -r requirements/requirements.in
tenacity==8.2.3
- # via plotly
+ # via
+ # jsonapi-requests
+ # plotly
tomli==2.0.1
# via
# build
diff --git a/requirements/test-requirements.in b/requirements/test-requirements.in
index df760727..c5e95951 100644
--- a/requirements/test-requirements.in
+++ b/requirements/test-requirements.in
@@ -18,3 +18,5 @@ freezegun # https://github.com/spulec/freezegun
django-coverage-plugin # https://github.com/nedbat/django_coverage_plugin
# Test coverage.
coverage
+# Mock json api data
+marshmallow-jsonapi
diff --git a/requirements/test-requirements.txt b/requirements/test-requirements.txt
index e69f69b4..f58f6830 100644
--- a/requirements/test-requirements.txt
+++ b/requirements/test-requirements.txt
@@ -12,7 +12,7 @@ charset-normalizer==3.3.2
# via
# -c requirements/requirements.txt
# requests
-coverage==7.4.3
+coverage==7.4.4
# via
# -r requirements/test-requirements.in
# django-coverage-plugin
@@ -34,9 +34,14 @@ idna==3.3
# requests
iniconfig==2.0.0
# via pytest
+marshmallow==3.20.1
+ # via marshmallow-jsonapi
+marshmallow-jsonapi==0.24.0
+ # via -r requirements/test-requirements.in
packaging==21.3
# via
# -c requirements/requirements.txt
+ # marshmallow
# pytest
# pytest-sugar
pluggy==1.4.0
+
+
+
+ + +
+
+
+
+ {% render_table table %}
+
+