Merge branch 'master' into riese/log_filter

.prettierrc

@@ -0,0 +1,14 @@
+{
+    "plugins": ["prettier-plugin-jinja-template"],
+    "overrides": [
+        {
+            "files": [
+                "*.html"
+            ],
+            "options": {
+                "parser": "jinja-template"
+            }
+        }
+    ],
+    "tabWidth": 2
+}

corehq/apps/api/odata/tests/utils.py

@@ -150,6 +150,5 @@ def get_instance(cls, domain_name):
 
 def generate_api_key_from_web_user(web_user):
     api_key = HQApiKey.objects.get_or_create(user=web_user.get_django_user())[0]
-    api_key.key = api_key.generate_key()
     api_key.save()
     return api_key

corehq/apps/api/tests/core_api.py

@@ -473,7 +473,7 @@ def test_get_user(self):
         endpoint = "%s?%s" % (self.single_endpoint(self.user._id),
                               urlencode({
                                   "username": self.user.username,
-                                  "api_key": self.api_key.key
+                                  "api_key": self.api_key.plaintext_key
                               }))
         response = self.client.get(endpoint)
         self.assertEqual(response.status_code, 200)
@@ -501,7 +501,7 @@ def test_wrong_user_api_key(self):
         endpoint = "%s?%s" % (self.single_endpoint(self.user._id),
                               urlencode({
                                   "username": self.user.username,
-                                  "api_key": other_api_key.key
+                                  "api_key": other_api_key.plaintext_key
                               }))
         response = self.client.get(endpoint)
         self.assertEqual(response.status_code, 401)
@@ -710,7 +710,7 @@ def setUp(self):
         super().setUp()
         self.endpoint = "%s?%s" % (self.single_endpoint(self.user._id), urlencode({
             "username": self.user.username,
-            "api_key": self.api_key.key
+            "api_key": self.api_key.plaintext_key
         }))
 
     def test_throttle_allowlist(self):

corehq/apps/api/tests/test_auth.py

@@ -47,7 +47,7 @@ def _get_request_with_basic_auth(self, domain=None):
         )
 
     def _construct_api_auth_header(self, username, api_key):
-        return f'ApiKey {username}:{api_key.key}'
+        return f'ApiKey {username}:{api_key.plaintext_key}'
 
     def _construct_basic_auth_header(self, username, password):
         # https://stackoverflow.com/q/5495452/8207

corehq/apps/api/tests/test_validation.py

@@ -0,0 +1,105 @@
+from django.test import TestCase
+from unittest.mock import patch
+
+from corehq.apps.api.validation import WebUserResourceValidator
+from corehq.apps.domain.models import Domain
+from corehq.apps.users.models import WebUser
+from corehq.util.test_utils import flag_enabled, flag_disabled
+
+
+class TestWebUserResourceValidator(TestCase):
+
+    @classmethod
+    def setUpClass(cls):
+        super().setUpClass()
+        cls.domain = Domain(name="test-domain", is_active=True)
+        cls.domain.save()
+        cls.addClassCleanup(cls.domain.delete)
+        cls.requesting_user = WebUser.create(cls.domain.name, "[email protected]", "123", None, None)
+        cls.validator = WebUserResourceValidator(cls.domain.name, cls.requesting_user)
+
+    @classmethod
+    def tearDownClass(cls):
+        cls.requesting_user.delete(None, None)
+        super().tearDownClass()
+
+    def test_validate_parameters(self):
+        params = {"email": "[email protected]", "role": "Admin"}
+        self.assertIsNone(self.validator.validate_parameters(params))
+
+        invalid_params = {"invalid_param": "value"}
+        self.assertEqual(self.validator.validate_parameters(invalid_params), "Invalid parameter(s): invalid_param")
+
+    @flag_enabled('TABLEAU_USER_SYNCING')
+    @patch('corehq.apps.users.models.WebUser.has_permission', return_value=True)
+    def test_validate_parameters_with_tableau_edit_permission(self, mock_has_permission):
+        params = {"email": "[email protected]", "role": "Admin", "tableau_role": "Viewer"}
+        self.assertIsNone(self.validator.validate_parameters(params))
+
+    @flag_disabled('TABLEAU_USER_SYNCING')
+    @patch('corehq.apps.users.models.WebUser.has_permission', return_value=False)
+    def test_validate_parameters_without_tableau_edit_permission(self, mock_has_permission):
+        params = {"email": "[email protected]", "role": "Admin", "tableau_role": "Viewer"}
+        self.assertEqual(self.validator.validate_parameters(params),
+                         "You do not have permission to edit Tableau Configuration.")
+
+    @patch('corehq.apps.registration.validation.domain_has_privilege', return_value=True)
+    def test_validate_parameters_with_profile_permission(self, mock_domain_has_privilege):
+        params = {"email": "[email protected]", "role": "Admin", "profile": "some_profile"}
+        self.assertIsNone(self.validator.validate_parameters(params))
+
+    @patch('corehq.apps.registration.validation.domain_has_privilege', return_value=False)
+    def test_validate_parameters_without_profile_permission(self, mock_domain_has_privilege):
+        params = {"email": "[email protected]", "role": "Admin", "profile": "some_profile"}
+        self.assertEqual(self.validator.validate_parameters(params),
+                         "This domain does not have user profile privileges.")
+
+    @patch('corehq.apps.registration.validation.domain_has_privilege', return_value=True)
+    def test_validate_parameters_with_location_privilege(self, mock_domain_has_privilege):
+        params = {"email": "[email protected]", "role": "Admin", "primary_location": "some_location"}
+        self.assertIsNone(self.validator.validate_parameters(params))
+        params = {"email": "[email protected]", "role": "Admin", "assigned_locations": "some_location"}
+        self.assertIsNone(self.validator.validate_parameters(params))
+
+    @patch('corehq.apps.registration.validation.domain_has_privilege', return_value=False)
+    def test_validate_parameters_without_location_privilege(self, mock_domain_has_privilege):
+        params = {"email": "[email protected]", "role": "Admin", "primary_location": "some_location"}
+        self.assertEqual(self.validator.validate_parameters(params),
+                         "This domain does not have locations privileges.")
+
+        params = {"email": "[email protected]", "role": "Admin", "assigned_locations": "some_location"}
+        self.assertEqual(self.validator.validate_parameters(params),
+                         "This domain does not have locations privileges.")
+
+    def test_validate_email(self):
+        self.assertIsNone(self.validator.validate_email("[email protected]", True))
+
+        self.assertEqual(self.validator.validate_email("[email protected]", True),
+                        "A user with this email address is already in "
+                        "this project or has a pending invitation.")
+
+        deactivated_user = WebUser.create(self.domain.name, "[email protected]", "123", None, None)
+        deactivated_user.is_active = False
+        deactivated_user.save()
+        self.assertEqual(self.validator.validate_email("[email protected]", True),
+                         "A user with this email address is deactivated. ")
+
+    def test_validate_locations(self):
+        with patch('corehq.apps.user_importer.validation.LocationValidator.validate_spec') as mock_validate_spec:
+            mock_validate_spec.return_value = None
+            self.assertIsNone(self.validator.validate_locations(self.requesting_user.username,
+                                                                ["loc1", "loc2"], "loc1"))
+
+            actual_spec = mock_validate_spec.call_args[0][0]
+            self.assertEqual(actual_spec['username'], self.requesting_user.username)
+            self.assertCountEqual(actual_spec['location_code'], ["loc1", "loc2"])
+
+        self.assertEqual(
+            self.validator.validate_locations(self.requesting_user.username, ["loc1", "loc2"], "loc3"),
+            "Primary location must be one of the user's locations"
+        )
+
+        self.assertEqual(
+            self.validator.validate_locations(self.requesting_user.username, ["loc1", "loc2"], ""),
+            "Primary location can't be empty if the user has any locations set"
+        )

corehq/apps/api/tests/utils.py

@@ -124,11 +124,11 @@ def _get_api_key_auth_headers(self, headers=None, username=None):
             return {}
 
         username = username or self.username
-        api_key = self.api_key.key
+        api_key = self.api_key.plaintext_key
         if username != self.username:
             web_user = WebUser.get_by_username(username)
             api_key, _ = HQApiKey.objects.get_or_create(user=WebUser.get_django_user(web_user))
-            api_key = api_key.key
+            api_key = api_key.plaintext_key
 
         return {
             'HTTP_AUTHORIZATION': f'apikey {username}:{api_key}'

corehq/apps/api/validation.py

@@ -0,0 +1,87 @@
+from memoized import memoized
+
+from corehq.apps.custom_data_fields.models import CustomDataFieldsDefinition
+from corehq.apps.reports.util import get_allowed_tableau_groups_for_domain
+from corehq.apps.user_importer.importer import SiteCodeToLocationCache
+from corehq.apps.user_importer.validation import (
+    RoleValidator,
+    ProfileValidator,
+    LocationValidator,
+    TableauGroupsValidator,
+    TableauRoleValidator,
+    CustomDataValidator,
+    EmailValidator,
+)
+from corehq.apps.users.validation import validate_primary_location_assignment
+from corehq.apps.registration.validation import AdminInvitesUserFormValidator
+
+
+class WebUserResourceValidator():
+    def __init__(self, domain, requesting_user):
+        self.domain = domain
+        self.requesting_user = requesting_user
+
+    @property
+    def roles_by_name(self):
+        from corehq.apps.users.views.utils import get_editable_role_choices
+        return {role[1]: role[0] for role in get_editable_role_choices(self.domain, self.requesting_user,
+                                                  allow_admin_role=True)}
+
+    @property
+    @memoized
+    def profiles_by_name(self):
+        from corehq.apps.users.views.mobile.custom_data_fields import UserFieldsView
+        return CustomDataFieldsDefinition.get_profiles_by_name(self.domain, UserFieldsView.field_type)
+
+    @property
+    def location_cache(self):
+        return SiteCodeToLocationCache(self.domain)
+
+    def validate_parameters(self, parameters):
+        allowed_params = ['email', 'role', 'primary_location', 'assigned_locations',
+                          'profile', 'custom_user_data', 'tableau_role', 'tableau_groups']
+        invalid_params = [param for param in parameters if param not in allowed_params]
+        if invalid_params:
+            return f"Invalid parameter(s): {', '.join(invalid_params)}"
+        return AdminInvitesUserFormValidator.validate_parameters(self.domain, self.requesting_user, parameters)
+
+    def validate_role(self, role):
+        spec = {'role': role}
+        return RoleValidator(self.domain, self.roles_by_name()).validate_spec(spec)
+
+    def validate_profile(self, new_profile_name):
+        profile_validator = ProfileValidator(self.domain, self.requesting_user, True, self.profiles_by_name())
+        spec = {'user_profile': new_profile_name}
+        return profile_validator.validate_spec(spec)
+
+    def validate_custom_data(self, custom_data, profile_name):
+        custom_data_validator = CustomDataValidator(self.domain, self.profiles_by_name())
+        spec = {'data': custom_data, 'user_profile': profile_name}
+        return custom_data_validator.validate_spec(spec)
+
+    def validate_email(self, email, is_post):
+        if is_post:
+            error = AdminInvitesUserFormValidator.validate_email(self.domain, email)
+            if error:
+                return error
+        email_validator = EmailValidator(self.domain, 'email')
+        spec = {'email': email}
+        return email_validator.validate_spec(spec)
+
+    def validate_locations(self, editable_user, assigned_location_codes, primary_location_code):
+        error = validate_primary_location_assignment(primary_location_code, assigned_location_codes)
+        if error:
+            return error
+
+        location_validator = LocationValidator(self.domain, self.requesting_user, self.location_cache, True)
+        location_codes = list(set(assigned_location_codes + [primary_location_code]))
+        spec = {'location_code': location_codes,
+                'username': editable_user}
+        return location_validator.validate_spec(spec)
+
+    def validate_tableau_group(self, tableau_groups):
+        allowed_groups_for_domain = get_allowed_tableau_groups_for_domain(self.domain) or []
+        return TableauGroupsValidator.validate_tableau_groups(allowed_groups_for_domain, tableau_groups)
+
+    def validate_tableau_role(self, tableau_role):
+        return TableauRoleValidator.validate_tableau_role(tableau_role)

corehq/apps/app_manager/fixtures/mobile_ucr.py

@@ -91,7 +91,8 @@ def __call__(self, restore_state):
             return []
 
         restore_user = restore_state.restore_user
-        apps = self._get_apps(restore_state, restore_user)
+        with restore_state.timing_context('_get_apps'):
+            apps = self._get_apps(restore_state, restore_user)
         report_configs = self._get_report_configs(apps)
         if not report_configs:
             return []
@@ -168,6 +169,7 @@ def __init__(self, domain, report_configs):
         self.reports = {}
         self.domain = domain
         self.report_configs = report_configs
+        self.add_row_index = toggles.ADD_ROW_INDEX_TO_MOBILE_UCRS.enabled(domain)
 
     def get_data(self, key, data_source):
         if key not in self.data_cache:
@@ -234,7 +236,8 @@ def __call__(self, restore_state, restore_user, needed_versions, report_configs)
         if needed_versions.intersection({MOBILE_UCR_VERSION_1, MOBILE_UCR_MIGRATING_TO_2}):
             yield _get_report_index_fixture(restore_user)
             try:
-                self.report_data_cache.load_reports()
+                with restore_state.timing_context('V1 load_reports'):
+                    self.report_data_cache.load_reports()
             except Exception:
                 logging.exception("Error fetching reports for domain", extra={
                     "domain": restore_user.domain,
@@ -275,13 +278,11 @@ def _v1_fixture(self, restore_user, report_configs, fail_hard=False):
         return root
 
     def report_config_to_fixture(self, report_config, restore_user):
-        row_index_enabled = toggles.ADD_ROW_INDEX_TO_MOBILE_UCRS.enabled(restore_user.domain)
-
         def _row_to_row_elem(
             deferred_fields, filter_options_by_field, row, index, is_total_row=False,
         ):
             row_elem = E.row(index=str(index), is_total_row=str(is_total_row))
-            if row_index_enabled:
+            if self.report_data_cache.add_row_index:
                 row_elem.append(E.column(str(index), id='row_index'))
             for k in sorted(row.keys()):
                 value = serialize(row[k])
@@ -321,15 +322,17 @@ def __call__(self, restore_state, restore_user, needed_versions, report_configs)
             yield _get_report_index_fixture(restore_user, oldest_sync_time)
 
             try:
-                self.report_data_cache.load_reports(synced_fixtures)
+                with restore_state.timing_context('V2 load_reports'):
+                    self.report_data_cache.load_reports(synced_fixtures)
             except Exception:
                 logging.exception("Error fetching reports for domain", extra={
                     "domain": restore_user.domain,
                     "report_config_ids": [config.report_id for config in synced_fixtures]
                 })
                 return []
 
-            yield from self._v2_fixtures(restore_user, synced_fixtures, restore_state.params.fail_hard)
+            with restore_state.timing_context('_v2_fixtures'):
+                yield from self._v2_fixtures(restore_state, synced_fixtures)
             for report_uuid in purged_fixture_ids:
                 yield from self._empty_v2_fixtures(report_uuid)
 
@@ -404,10 +407,12 @@ def _empty_v2_fixtures(self, report_uuid):
         yield E.fixture(id=self._report_fixture_id(report_uuid))
         yield E.fixture(id=self._report_filter_id(report_uuid))
 
-    def _v2_fixtures(self, restore_user, report_configs, fail_hard=False):
+    def _v2_fixtures(self, restore_state, report_configs):
+        fail_hard = restore_state.params.fail_hard
         for report_config in report_configs:
             try:
-                yield from self.report_config_to_fixture(report_config, restore_user)
+                with restore_state.timing_context(report_config.instance_id):
+                    yield from self.report_config_to_fixture(report_config, restore_state.restore_user)
             except ReportConfigurationNotFoundError as err:
                 logging.exception('Error generating report fixture: {}'.format(err))
                 if fail_hard:
@@ -427,7 +432,7 @@ def _v2_fixtures(self, restore_user, report_configs, fail_hard=False):
     def report_config_to_fixture(self, report_config, restore_user):
         def _row_to_row_elem(deferred_fields, filter_options_by_field, row, index, is_total_row=False):
             row_elem = E.row(index=str(index), is_total_row=str(is_total_row))
-            if toggles.ADD_ROW_INDEX_TO_MOBILE_UCRS.enabled(restore_user.domain):
+            if self.report_data_cache.add_row_index:
                 row_elem.append(E('row_index', str(index)))
             for k in sorted(row.keys()):
                 value = serialize(row[k])

corehq/apps/app_manager/models.py

@@ -181,6 +181,7 @@
 from corehq.blobs.mixin import CODES, BlobMixin
 from corehq.const import USER_DATE_FORMAT, USER_TIME_FORMAT
 from corehq.util import bitly, view_utils
+from corehq.util.metrics import metrics_counter
 from corehq.util.quickcache import quickcache
 from corehq.util.timer import TimingContext, time_method
 from corehq.util.timezones.conversions import ServerTime
@@ -4467,8 +4468,31 @@ def make_build(self, comment=None, user_id=None):
         assert copy._id
         prune_auto_generated_builds.delay(self.domain, self._id)
 
+        self.check_build_dependencies(new_build=copy)
+
         return copy
 
+    def check_build_dependencies(self, new_build):
+        """
+        Reports whether the app dependencies have been added or removed.
+        """
+
+        def has_dependencies(build):
+            return bool(
+                build.profile.get('features', {}).get('dependencies')
+            )
+
+        new_build_has_dependencies = has_dependencies(new_build)
+
+        last_build = get_latest_build_doc(self.domain, self.id)
+        last_build = self.__class__.wrap(last_build) if last_build else None
+        last_build_has_dependencies = has_dependencies(last_build) if last_build else False
+
+        if not last_build_has_dependencies and new_build_has_dependencies:
+            metrics_counter('commcare.app_build.dependencies_added')
+        elif last_build_has_dependencies and not new_build_has_dependencies:
+            metrics_counter('commcare.app_build.dependencies_removed')
+
     def convert_app_to_build(self, copy_of, user_id, comment=None):
         self.copy_of = copy_of
         built_on = datetime.datetime.utcnow()