Merge remote-tracking branch 'origin/es/sql-user-data' into autostaging

corehq/apps/api/tests/user_resources.py

@@ -299,10 +299,7 @@ def test_update(self):
                 'language': 'pol',
                 'last_name': 'last',
                 'first_name': 'test',
-                'user_data': {
-                    'chw_id': '13/43/DFA',
-                    'commcare_profile': self.profile.id,
-                }
+                'user_data': {'chw_id': '13/43/DFA'},
             }
         )
         self.assertTrue("50253311398" in

corehq/apps/callcenter/tests/test_utils.py

@@ -224,7 +224,8 @@ def setUpClass(cls):
         cls.domain.save()
 
     def setUp(self):
-        self.user = CommCareUser.create(TEST_DOMAIN, 'user1', '***', None, None, commit=False)  # Don't commit yet
+        self.user = CommCareUser.create(TEST_DOMAIN, format_username('user1', TEST_DOMAIN),
+                                        '***', None, None, commit=False)  # Don't commit yet
 
     def tearDown(self):
         self.user.delete(self.domain.name, deleted_by=None)
@@ -346,7 +347,6 @@ def test_update_no_change(self):
         self.assertEqual(1, len(user_case.xform_ids))
 
     def test_bulk_upload_usercases(self):
-        self.user.username = format_username('bushy_top', TEST_DOMAIN)
         self.user.save()
 
         upload_record = UserUploadRecord.objects.create(
@@ -381,6 +381,7 @@ def test_bulk_upload_usercases(self):
             upload_record_id=upload_record.pk,
         )
         self.assertEqual(results['errors'], [])
+        self.assertEqual([r['flag'] for r in results['rows']], ['updated', 'created'])
 
         old_user_case = CommCareCase.objects.get_case_by_external_id(TEST_DOMAIN, self.user._id, USERCASE_TYPE)
         self.assertEqual(old_user_case.owner_id, self.user.get_id)

corehq/apps/cloudcare/tests/test_esaccessors.py

@@ -1,7 +1,7 @@
 import uuid
 from unittest.mock import MagicMock, patch
 
-from django.test import SimpleTestCase
+from django.test import TestCase
 
 from corehq.apps.cloudcare.esaccessors import login_as_user_query
 from corehq.apps.es.tests.utils import es_test
@@ -10,7 +10,7 @@
 
 
 @es_test(requires=[user_adapter])
-class TestLoginAsUserQuery(SimpleTestCase):
+class TestLoginAsUserQuery(TestCase):
 
     @classmethod
     def setUpClass(cls):
@@ -22,15 +22,23 @@ def setUpClass(cls):
         cls.domain = 'user-esaccessors-test'
 
     def _send_user_to_es(self, _id=None, username=None, user_data=None):
-        user = CommCareUser(
+        user = CommCareUser.create(
             domain=self.domain,
             username=username or self.username,
+            password='password',
+            created_by=None,
+            created_via=None,
             _id=_id or uuid.uuid4().hex,
             first_name=self.first_name,
             last_name=self.last_name,
-            user_data=user_data or {},
             is_active=True,
+
         )
+        user.save()
+        self.addCleanup(user.delete, self.domain, deleted_by=None)
+        if user_data:
+            user.get_user_data(self.domain).update(user_data)
+            user.get_user_data(self.domain).save()
 
         with patch('corehq.apps.groups.dbaccessors.get_group_id_name_map_by_user', return_value=[]):
             user_adapter.index(user, refresh=True)

corehq/apps/cloudcare/tests/test_session.py

@@ -6,39 +6,38 @@
     get_user_contributions_to_touchforms_session,
 )
 from corehq.apps.custom_data_fields.models import (
+    PROFILE_SLUG,
     CustomDataFieldsDefinition,
     CustomDataFieldsProfile,
     Field,
-    PROFILE_SLUG,
 )
-from corehq.apps.users.views.mobile.custom_data_fields import UserFieldsView
 from corehq.apps.users.models import CommCareUser, WebUser
+from corehq.apps.users.views.mobile.custom_data_fields import UserFieldsView
 from corehq.form_processor.models import CommCareCase
 
 
 class SessionUtilsTest(TestCase):
 
     def test_load_session_data_for_mobile_worker(self):
-        user = CommCareUser(
-            domain='cloudcare-tests',
-            username='[email protected]',
-            _id=uuid.uuid4().hex
-        )
+        user = CommCareUser.create('cloudcare-tests', '[email protected]',
+                                   'password', None, None)
+        self.addCleanup(user.delete, None, None)
         data = get_user_contributions_to_touchforms_session('cloudcare-tests', user)
         self.assertEqual('worker', data['username'])
         self.assertEqual(user._id, data['user_id'])
         self.assertTrue(isinstance(data['user_data'], dict))
         self.assertTrue(data['user_data']['commcare_project'], 'cloudcare-tests')
 
     def test_default_user_data(self):
-        user = CommCareUser(
-            domain='cloudcare-tests',
-            username='[email protected]',
-            _id=uuid.uuid4().hex
-        )
+        user = CommCareUser.create('cloudcare-tests', '[email protected]',
+                                   'password', None, None)
+        self.addCleanup(user.delete, None, None)
+
         user_data = get_user_contributions_to_touchforms_session('cloudcare-tests', user)['user_data']
-        for key in ['commcare_first_name', 'commcare_last_name', 'commcare_phone_number']:
-            self.assertEqual(None, user_data[key])
+        self.assertEqual('', user_data['commcare_first_name'])
+        self.assertEqual('', user_data['commcare_last_name'])
+        self.assertEqual(None, user_data['commcare_phone_number'])
+
         user.first_name = 'first'
         user.last_name = 'last'
         user_data = get_user_contributions_to_touchforms_session('cloudcare-tests', user)['user_data']
@@ -70,10 +69,7 @@ def test_user_data_profile(self):
         self.assertEqual('supernova', user_data['word'])
 
     def test_load_session_data_for_web_user(self):
-        user = WebUser(
-            username='[email protected]',
-            _id=uuid.uuid4().hex
-        )
+        user = WebUser.create(None, '[email protected]', '123', None, None)
         data = get_user_contributions_to_touchforms_session('cloudcare-tests', user)
         self.assertEqual('[email protected]', data['username'])
         self.assertEqual(user._id, data['user_id'])

corehq/apps/domain/deletion.py

@@ -349,6 +349,7 @@ def _delete_demo_user_restores(domain_name):
         'StockLevelsConfig', 'StockRestoreConfig',
     ]),
     ModelDeletion('consumption', 'DefaultConsumption', 'domain'),
+    ModelDeletion('users', 'SQLUserData', 'domain'),
     ModelDeletion('custom_data_fields', 'CustomDataFieldsDefinition', 'domain', [
         'CustomDataFieldsProfile', 'Field',
     ]),

corehq/apps/domain/tests/test_delete_domain.py

@@ -4,14 +4,14 @@
 from datetime import date, datetime, timedelta
 from decimal import Decimal
 from io import BytesIO
+from unittest.mock import patch
 
 from django.contrib.auth.models import User
 from django.core.management import call_command
 from django.db.transaction import TransactionManagementError
 from django.test import TestCase
 
 from dateutil.relativedelta import relativedelta
-from unittest.mock import patch
 
 from casexml.apps.phone.models import SyncLogSQL
 from couchforms.models import UnfinishedSubmissionStub
@@ -51,9 +51,16 @@
 from corehq.apps.cloudcare.models import ApplicationAccess
 from corehq.apps.commtrack.models import CommtrackConfig
 from corehq.apps.consumption.models import DefaultConsumption
-from corehq.apps.custom_data_fields.models import CustomDataFieldsDefinition
+from corehq.apps.custom_data_fields.models import (
+    CustomDataFieldsDefinition,
+    CustomDataFieldsProfile,
+)
 from corehq.apps.data_analytics.models import GIRRow, MALTRow
-from corehq.apps.data_dictionary.models import CaseProperty, CasePropertyAllowedValue, CaseType
+from corehq.apps.data_dictionary.models import (
+    CaseProperty,
+    CasePropertyAllowedValue,
+    CaseType,
+)
 from corehq.apps.data_interfaces.models import (
     AutomaticUpdateRule,
     CaseRuleAction,
@@ -109,25 +116,23 @@
 from corehq.apps.users.audit.change_messages import UserChangeMessage
 from corehq.apps.users.models import (
     DomainRequest,
+    HqPermissions,
     Invitation,
     PermissionInfo,
-    HqPermissions,
     RoleAssignableBy,
     RolePermission,
-    UserRole,
     UserHistory,
+    UserRole,
     WebUser,
 )
+from corehq.apps.users.user_data import SQLUserData
 from corehq.apps.users.util import SYSTEM_USER_ID
 from corehq.apps.zapier.consts import EventTypes
 from corehq.apps.zapier.models import ZapierSubscription
 from corehq.blobs import CODES, NotFound, get_blob_db
 from corehq.form_processor.backends.sql.dbaccessors import doc_type_to_state
 from corehq.form_processor.models import CommCareCase, XFormInstance
-from corehq.form_processor.tests.utils import (
-    create_case,
-    create_form_for_test,
-)
+from corehq.form_processor.tests.utils import create_case, create_form_for_test
 from corehq.motech.models import ConnectionSettings, RequestLog
 from corehq.motech.repeaters.const import RECORD_SUCCESS_STATE
 from corehq.motech.repeaters.models import (
@@ -498,19 +503,22 @@ def test_consumption(self):
         self._assert_consumption_counts(self.domain.name, 0)
         self._assert_consumption_counts(self.domain2.name, 1)
 
-    def _assert_custom_data_fields_counts(self, domain_name, count):
-        self._assert_queryset_count([
-            CustomDataFieldsDefinition.objects.filter(domain=domain_name),
-        ], count)
-
-    def test_custom_data_fields(self):
+    def test_user_data_cascading(self):
         for domain_name in [self.domain.name, self.domain2.name]:
-            CustomDataFieldsDefinition.get_or_create(domain_name, 'UserFields')
+            user = User.objects.create(username=f'mobileuser@{domain_name}.{HQ_ACCOUNT_ROOT}')
+            definition = CustomDataFieldsDefinition.get_or_create(domain_name, 'UserFields')
+            profile = CustomDataFieldsProfile.objects.create(name='myprofile', definition=definition)
+            SQLUserData.objects.create(domain=domain_name, user_id='123', django_user=user,
+                                       profile=profile, data={})
+
+        models = [User, CustomDataFieldsDefinition, CustomDataFieldsProfile, SQLUserData]
+        for model in models:
+            self.assertEqual(model.objects.count(), 2)
 
         self.domain.delete()
 
-        self._assert_custom_data_fields_counts(self.domain.name, 0)
-        self._assert_custom_data_fields_counts(self.domain2.name, 1)
+        for model in models:
+            self.assertEqual(model.objects.count(), 1)
 
     def _assert_data_analytics_counts(self, domain_name, count):
         self._assert_queryset_count([

corehq/apps/dump_reload/sql/dump.py

@@ -160,6 +160,7 @@
     FilteredModelIteratorBuilder('users.RoleAssignableBy', SimpleFilter('role__domain')),
     FilteredModelIteratorBuilder('users.RolePermission', SimpleFilter('role__domain')),
     FilteredModelIteratorBuilder('users.UserRole', SimpleFilter('domain')),
+    FilteredModelIteratorBuilder('users.SQLUserData', SimpleFilter('domain')),
     FilteredModelIteratorBuilder('locations.LocationFixtureConfiguration', SimpleFilter('domain')),
     FilteredModelIteratorBuilder('commtrack.CommtrackConfig', SimpleFilter('domain')),
     FilteredModelIteratorBuilder('commtrack.ActionConfig', SimpleFilter('commtrack_config__domain')),

corehq/apps/es/users.py

@@ -104,7 +104,7 @@ def _from_dict(self, user_dict):
         user_dict['__group_ids'] = [res.id for res in results]
         user_dict['__group_names'] = [res.name for res in results]
         user_dict['user_data_es'] = []
-        if 'user_data' in user_dict and user_dict['doc_type'] == 'CommCareUser':
+        if user_dict.get('base_doc') == 'CouchUser' and user_dict['doc_type'] == 'CommCareUser':
             user_obj = self.model_cls.wrap_correctly(user_dict)
             user_data = user_obj.get_user_data(user_obj.domain)
             for key, value in user_data.items():

corehq/apps/smsforms/tests/test_app.py

@@ -22,10 +22,12 @@
 )
 from corehq.apps.smsforms.models import SQLXFormsSession
 from corehq.apps.users.models import WebUser
+from corehq.apps.users.tests.util import patch_user_data_db_layer
 from corehq.form_processor.models import CommCareCase
 from corehq.messaging.scheduling.util import utcnow
 
 
+@patch_user_data_db_layer
 @patch('corehq.apps.smsforms.app.tfsms.start_session')
 class TestStartSession(TestCase):
     domain = "test-domain"

corehq/apps/user_importer/tests/test_importer.py

@@ -49,6 +49,7 @@
     HqPermissions,
 )
 from corehq.apps.users.model_log import UserModelAction
+from corehq.apps.users.tests.util import patch_user_data_db_layer
 from corehq.apps.users.views.mobile.custom_data_fields import UserFieldsView
 from corehq.const import USER_CHANGE_VIA_BULK_IMPORTER
 from corehq.extensions.interface import disable_extensions
@@ -677,9 +678,7 @@ def test_user_data_profile_redundant(self):
             PROFILE_SLUG: self.profile.id,
         })
         # Profile fields shouldn't actually be added to user_data
-        self.assertEqual(self.user.get_user_data(self.domain.name).raw, {
-            PROFILE_SLUG: self.profile.id,
-        })
+        self.assertEqual(self.user.get_user_data(self.domain.name).raw, {})
 
     def test_user_data_profile_blank(self):
         import_users_and_groups(
@@ -2062,6 +2061,7 @@ def test_tableau_users(self, mock_request):
             local_tableau_users.get(username='[email protected]')
 
 
+@patch_user_data_db_layer
 class TestUserChangeLogger(SimpleTestCase):
     @classmethod
     def setUpClass(cls):

corehq/apps/userreports/tests/test_choice_provider.py

@@ -6,11 +6,11 @@
 from django.utils.translation import gettext
 
 from corehq.apps.domain.shortcuts import create_domain, create_user
+from corehq.apps.es.client import manager
 from corehq.apps.es.fake.groups_fake import GroupESFake
 from corehq.apps.es.fake.users_fake import UserESFake
-from corehq.apps.es.client import manager
-from corehq.apps.es.users import user_adapter
 from corehq.apps.es.tests.utils import es_test
+from corehq.apps.es.users import user_adapter
 from corehq.apps.groups.models import Group
 from corehq.apps.locations.tests.util import LocationHierarchyTestCase
 from corehq.apps.registry.tests.utils import (
@@ -43,6 +43,7 @@
     WebUser,
 )
 from corehq.apps.users.models_role import UserRole
+from corehq.apps.users.tests.util import patch_user_data_db_layer
 from corehq.apps.users.util import normalize_username
 from corehq.util.es.testing import sync_users_to_es
 from corehq.util.test_utils import flag_disabled, flag_enabled
@@ -262,6 +263,7 @@ class UserChoiceProviderTest(SimpleTestCase, ChoiceProviderTestMixin):
     domain = 'user-choice-provider'
 
     @classmethod
+    @patch_user_data_db_layer
     def make_mobile_worker(cls, username, domain=None):
         domain = domain or cls.domain
         user = CommCareUser(username=normalize_username(username, domain),
@@ -605,7 +607,9 @@ def test_query_full_registry_access(self):
                           Choice(value='B', display='B'),
                           Choice(value='C', display='C')],
                          self.choice_provider.query(ChoiceQueryContext(query='', offset=0, user=self.web_user)))
-        self.assertEqual([], self.choice_provider.query(ChoiceQueryContext(query='D', offset=0, user=self.web_user)))
+        self.assertEqual([], self.choice_provider.query(
+            ChoiceQueryContext(query='D', offset=0, user=self.web_user)
+        ))
 
     def test_query_no_registry_access(self):
         self.assertEqual([Choice(value='A', display='A')],

corehq/apps/users/README.rst

@@ -50,11 +50,11 @@ User Data
 Users may have arbitrary data associated with them, assigned by the project and then referenced in applications.
 This user data is implemented via the ``custom_data_fields`` app, the same way as location and product data.
 
-User data is only relevant to mobile users. However, ``user_data`` is a property of ``CouchUser``
-and not ``CommCareUser`` because a legacy feature applied user data to web users. As a result of this,
-some web users do have user data saved to their documents.
+User data is being migrated to SQL to support web users, which will have a ``SQLUserData`` object for each domain
+they are a member of.  Data is accessed through the accessor ``user.get_user_data(domain)``, which returns an
+instance of ``UserData`` - a class that acts like a dictionary, but factors in data controlled by user data
+profiles and uneditable system fields.
 
-User data should be accessed via the ``metadata`` property, which takes into account user data profiles.
 
 UserRole and Permissions
 ~~~~~~~~~~~~~~~~~~~~~~~~

corehq/apps/users/management/commands/populate_sql_user_data.py

@@ -0,0 +1,27 @@
+# One-off migration, November 2023
+
+from django.contrib.auth.models import User
+from django.core.management.base import BaseCommand
+
+from corehq.apps.users.models import CouchUser
+from corehq.util.log import with_progress_bar
+from corehq.util.queries import queryset_to_iterator
+
+
+class Command(BaseCommand):
+    help = "Populate SQL user data from couch"
+
+    def handle(self, **options):
+        queryset = get_users_without_user_data()
+        for user in with_progress_bar(queryset_to_iterator(queryset, User), queryset.count()):
+            populate_user_data(user)
+
+
+def get_users_without_user_data():
+    return User.objects.filter(sqluserdata__isnull=True)
+
+
+def populate_user_data(django_user):
+    user = CouchUser.from_django_user(django_user, strict=True)
+    for domain in user.domains:  # get_domains?
+        user.get_user_data(domain).save()