From d10279cee67f55d2893d5fcc862f5174d79e8dbc Mon Sep 17 00:00:00 2001 From: AddisonDunn Date: Tue, 29 Oct 2024 14:44:44 -0400 Subject: [PATCH 1/4] create a FF for module badges dev --- corehq/toggles/__init__.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/corehq/toggles/__init__.py b/corehq/toggles/__init__.py index 3f3344d69185..8330e36e9118 100644 --- a/corehq/toggles/__init__.py +++ b/corehq/toggles/__init__.py @@ -2963,3 +2963,10 @@ def domain_has_privilege_from_toggle(privilege_slug, domain): tag=TAG_CUSTOM, namespaces=[NAMESPACE_DOMAIN], ) + +MODULE_BADGES = StaticToggle( + slug='module_badges', + label='USH: Show case counts from CSQL queries as badges on modules', + tag=TAG_CUSTOM, + namespaces=[NAMESPACE_DOMAIN], +) From b3b4eb81d29d154b983ea00575103cc324e70bf7 Mon Sep 17 00:00:00 2001 From: AddisonDunn Date: Tue, 29 Oct 2024 14:54:00 -0400 Subject: [PATCH 2/4] create models to store fixture config and history --- corehq/apps/domain/deletion.py | 2 + corehq/apps/dump_reload/sql/dump.py | 2 + ...tureexpression_csqlfixtureexpressionlog.py | 37 ++++++++++++ corehq/apps/fixtures/models.py | 56 ++++++++++++++++++- migrations.lock | 1 + 5 files changed, 97 insertions(+), 1 deletion(-) create mode 100644 corehq/apps/fixtures/migrations/0011_csqlfixtureexpression_csqlfixtureexpressionlog.py diff --git a/corehq/apps/domain/deletion.py b/corehq/apps/domain/deletion.py index 7db01a5be76d..c94c1ff61e05 100644 --- a/corehq/apps/domain/deletion.py +++ b/corehq/apps/domain/deletion.py @@ -468,6 +468,8 @@ def _delete_demo_user_restores(domain_name): ModelDeletion('couchforms', 'UnfinishedSubmissionStub', 'domain'), ModelDeletion('couchforms', 'UnfinishedArchiveStub', 'domain'), ModelDeletion('fixtures', 'LookupTable', 'domain'), + ModelDeletion('fixtures', 'CSQLFixtureExpression', 'domain'), + ModelDeletion('fixtures', 'CSQLFixtureExpressionLog', 'expression__domain'), CustomDeletion('ucr', delete_all_ucr_tables_for_domain, []), ModelDeletion('domain', 'OperatorCallLimitSettings', 'domain'), ModelDeletion('domain', 'SMSAccountConfirmationSettings', 'domain'), diff --git a/corehq/apps/dump_reload/sql/dump.py b/corehq/apps/dump_reload/sql/dump.py index c037922e2a92..5cc148eeffe6 100644 --- a/corehq/apps/dump_reload/sql/dump.py +++ b/corehq/apps/dump_reload/sql/dump.py @@ -222,6 +222,8 @@ FilteredModelIteratorBuilder('fixtures.LookupTable', SimpleFilter('domain')), FilteredModelIteratorBuilder('fixtures.LookupTableRow', SimpleFilter('domain')), FilteredModelIteratorBuilder('fixtures.LookupTableRowOwner', SimpleFilter('domain')), + FilteredModelIteratorBuilder('fixtures.CSQLFixtureExpression', SimpleFilter('domain')), + FilteredModelIteratorBuilder('fixtures.CSQLFixtureExpressionLog', SimpleFilter('expression__domain')), FilteredModelIteratorBuilder('hqmedia.LogoForSystemEmailsReference', SimpleFilter('domain')), FilteredModelIteratorBuilder('userreports.UCRExpression', SimpleFilter('domain')), FilteredModelIteratorBuilder('generic_inbound.ConfigurableAPI', SimpleFilter('domain')), diff --git a/corehq/apps/fixtures/migrations/0011_csqlfixtureexpression_csqlfixtureexpressionlog.py b/corehq/apps/fixtures/migrations/0011_csqlfixtureexpression_csqlfixtureexpressionlog.py new file mode 100644 index 000000000000..c2094cb8aef8 --- /dev/null +++ b/corehq/apps/fixtures/migrations/0011_csqlfixtureexpression_csqlfixtureexpressionlog.py @@ -0,0 +1,37 @@ +# Generated by Django 4.2.16 on 2024-10-29 18:29 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('fixtures', '0010_lookuptable_is_synced'), + ] + + operations = [ + migrations.CreateModel( + name='CSQLFixtureExpression', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('domain', models.CharField(default='', max_length=64)), + ('name', models.CharField(default='', max_length=64)), + ('csql', models.CharField(default='', max_length=2000)), + ('date_created', models.DateTimeField(auto_now_add=True)), + ('last_modified', models.DateTimeField(auto_now=True)), + ('deleted', models.BooleanField(default=False)), + ], + ), + migrations.CreateModel( + name='CSQLFixtureExpressionLog', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('date', models.DateTimeField(auto_now_add=True)), + ('action', models.CharField(choices=[('update', 'Updated'), ('create', 'Created'), ('delete', 'Deleted')], max_length=16)), + ('name', models.CharField(default='', max_length=64)), + ('csql', models.CharField(default='', max_length=2000)), + ('expression', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='fixtures.csqlfixtureexpression')), + ], + ), + ] diff --git a/corehq/apps/fixtures/models.py b/corehq/apps/fixtures/models.py index d8370f263253..278817bc3c9b 100644 --- a/corehq/apps/fixtures/models.py +++ b/corehq/apps/fixtures/models.py @@ -1,4 +1,5 @@ from datetime import datetime +from django.db.models import ForeignKey from functools import reduce from itertools import chain from uuid import uuid4 @@ -6,11 +7,14 @@ from attrs import define, field from django.db import models from django.db.models.expressions import RawSQL +from django.db.models.signals import post_save +from django.db.transaction import atomic +from django.dispatch import receiver +from django.utils.translation import gettext as _ from corehq.apps.groups.models import Group from corehq.sql_db.fields import CharIdField from corehq.util.jsonattrs import AttrsDict, AttrsList, list_of - from .exceptions import FixtureVersionError FIXTURE_BUCKET = 'domain-fixtures' @@ -306,3 +310,53 @@ class Meta: app_label = 'fixtures' db_table = 'fixtures_userfixturestatus' unique_together = ("user_id", "fixture_type") + + +class CSQLFixtureExpression(models.Model): + domain = models.CharField(max_length=64, default='') + name = models.CharField(max_length=64, default='') + csql = models.CharField(max_length=2000, default='') + date_created = models.DateTimeField(auto_now_add=True) + last_modified = models.DateTimeField(auto_now=True) + deleted = models.BooleanField(default=False) + + @classmethod + def by_domain(cls, domain): + return cls.objects.filter(domain=domain, deleted=False) + + @atomic + def soft_delete(self): + self.deleted = True + self.save() + CSQLFixtureExpressionLog.objects.create( + expression=self, + action=CSQLFixtureExpressionLog.DELETE, + ) + + +class CSQLFixtureExpressionLog(models.Model): + + CREATE = 'create' + DELETE = 'delete' + UPDATE = 'update' + + expression = ForeignKey(CSQLFixtureExpression, on_delete=models.CASCADE) + date = models.DateTimeField(auto_now_add=True) + action = models.CharField(max_length=16, choices=( + (UPDATE, _('Updated')), + (CREATE, _('Created')), + (DELETE, _('Deleted')), + ), null=False) + name = models.CharField(max_length=64, default='') + csql = models.CharField(max_length=2000, default='') + + +@receiver(post_save, sender=CSQLFixtureExpression) +def after_save(sender, instance, created, **kwargs): + updated_or_created = CSQLFixtureExpressionLog.CREATE if created else CSQLFixtureExpressionLog.UPDATE + CSQLFixtureExpressionLog.objects.create( + expression=instance, + action=updated_or_created, + name=instance.name, + csql=instance.csql + ) diff --git a/migrations.lock b/migrations.lock index 7dc0874ac264..30ba23133868 100644 --- a/migrations.lock +++ b/migrations.lock @@ -479,6 +479,7 @@ fixtures 0008_sqllookuptables 0009_remove_lookuptablerowowner_couch_id 0010_lookuptable_is_synced + 0011_csqlfixtureexpression_csqlfixtureexpressionlog form_processor 0001_initial 0002_xformattachmentsql From 95283d962bba0fb94d6fc1999dfb8b659bf9e672 Mon Sep 17 00:00:00 2001 From: AddisonDunn Date: Tue, 29 Oct 2024 15:01:50 -0400 Subject: [PATCH 3/4] allow editing CSQL fixture configuration via a page in fixtures new 'module badges' tab under Data --- .../fixtures/csql_fixture_configuration.html | 71 +++++++++++++++ corehq/apps/fixtures/urls.py | 3 + corehq/apps/fixtures/views.py | 90 +++++++++++++++++-- corehq/tabs/tabclasses.py | 8 +- 4 files changed, 165 insertions(+), 7 deletions(-) create mode 100644 corehq/apps/fixtures/templates/fixtures/csql_fixture_configuration.html diff --git a/corehq/apps/fixtures/templates/fixtures/csql_fixture_configuration.html b/corehq/apps/fixtures/templates/fixtures/csql_fixture_configuration.html new file mode 100644 index 000000000000..2e09152b262c --- /dev/null +++ b/corehq/apps/fixtures/templates/fixtures/csql_fixture_configuration.html @@ -0,0 +1,71 @@ +{% extends 'hqwebapp/bootstrap5/base_section.html' %} +{% load hq_shared_tags %} +{% load i18n %} + +{% js_entry_b3 "hqwebapp/js/htmx_and_alpine" %} + +{% block page_title %} + {% trans 'CSQL Fixture Configuration' %} +{% endblock %} + +{% block page_content %} +
+
+
+ +

+ + + + + + + + + + +
+ {% trans "Name" %} + + {% trans "Case Search QL Expression" %} + + {% trans "Delete" %} +
+ +
+

+
+{% endblock %} diff --git a/corehq/apps/fixtures/urls.py b/corehq/apps/fixtures/urls.py index 144b76d167cf..9453b951652c 100644 --- a/corehq/apps/fixtures/urls.py +++ b/corehq/apps/fixtures/urls.py @@ -11,6 +11,7 @@ fixture_upload_job_poll, update_tables, upload_fixture_api, + CSQLFixtureExpressionView ) from corehq.apps.hqwebapp.decorators import waf_allow @@ -25,6 +26,8 @@ url(r'^edit_lookup_tables/upload/$', waf_allow('XSS_BODY')(UploadItemLists.as_view()), name='upload_fixtures'), url(r'^edit_lookup_tables/update-tables/(?P[\w-]+)?$', update_tables, name='update_lookup_tables'), + url(r'^csql_fixture/csql_fixture_configuration/$', CSQLFixtureExpressionView.as_view(), + name=CSQLFixtureExpressionView.urlname), # upload status url(r'^upload/status/(?P(?:dl-)?[0-9a-fA-Z]{25,32})/$', diff --git a/corehq/apps/fixtures/views.py b/corehq/apps/fixtures/views.py index 4902a6dff3a5..40cf0f61d26c 100644 --- a/corehq/apps/fixtures/views.py +++ b/corehq/apps/fixtures/views.py @@ -12,6 +12,7 @@ Http404, HttpResponseBadRequest, HttpResponseRedirect, + HttpResponse, JsonResponse, ) from django.http.response import HttpResponseServerError @@ -24,7 +25,7 @@ from django.views.decorators.http import require_POST from django.views.generic.base import TemplateView -from corehq.apps.hqwebapp.decorators import waf_allow +from corehq.apps.hqwebapp.decorators import use_bootstrap5, waf_allow from dimagi.utils.decorators.view import get_file from dimagi.utils.logging import notify_exception from dimagi.utils.web import get_url_base, json_response @@ -43,11 +44,8 @@ FixtureUploadError, ) from corehq.apps.fixtures.fixturegenerators import item_lists_by_domain -from corehq.apps.fixtures.models import ( - LookupTableRow, - LookupTable, - TypeField, -) +from corehq.apps.fixtures.models import (LookupTable, LookupTableRow, CSQLFixtureExpression, + TypeField) from corehq.apps.fixtures.tasks import ( async_fixture_download, fixture_upload_async, @@ -575,3 +573,83 @@ def fixture_metadata(request, domain): Returns list of fixtures and metadata needed for itemsets in vellum """ return json_response(item_lists_by_domain(domain)) + + +@method_decorator(use_bootstrap5, name='dispatch') +class CSQLFixtureExpressionView(BaseDomainView): + urlname = 'csql_fixture_configuration' + page_title = _('CSQL Fixture Confguration') + template_name = 'fixtures/csql_fixture_configuration.html' + + @method_decorator(toggles.MODULE_BADGES.required_decorator()) + @method_decorator(require_can_edit_fixtures) + def dispatch(self, request, *args, **kwargs): + return super(CSQLFixtureExpressionView, self).dispatch(request, *args, **kwargs) + + def all_module_badge_configurations(self): + return CSQLFixtureExpression.by_domain(self.domain) + + @property + def page_context(self): + return { + 'save_url': reverse('csql_fixture_configuration', args=[self.domain]), + 'csql_fixture_configurations': + list(self.all_module_badge_configurations().values('id', 'name', 'csql')), + } + + def _post_response(self, message, div_class): + return HttpResponse((f'
' + _(message) + '
'), + content_type='text/html') + + @atomic + def post(self, request, *args, **kwargs): + try: + data = request.POST + ''' Post data format is: + { + 'name': ['name1', 'name2', 'name3'], + 'id': ['1', '2', ''], # empty string ID means new expression, missing means delete + 'csql': ['asdf', 'asdfg', 'asdfgh'],} + } + ''' + + touched_badge_ids = [] + ids_list = data.getlist('id') + name_list = data.getlist('name') + csql_list = data.getlist('csql') + + name_list_without_blanks = [name for name in name_list if name] + if len(name_list_without_blanks) != len(set(name_list_without_blanks)): + return self._post_response( + "Configuration not updated, two expressions cannot have the same name.", 'alert-warning') + + for index in range(0, len(ids_list)): + _id = ids_list[index] + name = name_list[index] + csql = csql_list[index] + + if not name_list[index] or not csql_list[index]: + if not name_list[index] and not csql_list[index]: + continue # ignore empty rows + return self._post_response("Configuration not updated, some fields are blank.", + 'alert-warning') + + if _id: + module_badge_configuration = CSQLFixtureExpression.by_domain(self.domain).get(id=_id) + module_badge_configuration.name = name + module_badge_configuration.csql = csql + module_badge_configuration.save() + else: + module_badge_configuration = CSQLFixtureExpression.objects.create( + domain=self.domain, name=name, csql=csql) + touched_badge_ids.append(module_badge_configuration.id) + for expression in CSQLFixtureExpression.by_domain(self.domain).exclude(id__in=touched_badge_ids): + expression.soft_delete() + return self._post_response("Fixture confugration updated!", 'alert-success') + except Exception as e: + notify_exception(request, message=str(e)) + return self._post_response("Configuration not updated, unknown error occurred.", 'alert-danger') + + @property + def section_url(self): + return reverse(self.urlname, args=[self.domain]) diff --git a/corehq/tabs/tabclasses.py b/corehq/tabs/tabclasses.py index 78c0d917af11..6c9e58043592 100644 --- a/corehq/tabs/tabclasses.py +++ b/corehq/tabs/tabclasses.py @@ -45,8 +45,8 @@ AttendeesListView, EventsView, ) +from corehq.apps.fixtures.views import CSQLFixtureExpressionView from corehq.apps.geospatial.dispatchers import CaseManagementMapDispatcher - from corehq.apps.hqadmin.reports import ( DeployHistoryReport, DeviceLogSoftAssertReport, @@ -625,6 +625,12 @@ def sidebar_items(self): items.extend(FixtureInterfaceDispatcher.navigation_sections( request=self._request, domain=self.domain)) + if (toggles.MODULE_BADGES.enabled(self.domain) and self.couch_user.can_edit_data()): + items.append([_('Module Badges'), [{ + 'title': _(CSQLFixtureExpressionView.page_title), + 'url': reverse(CSQLFixtureExpressionView.urlname, args=[self.domain]), + }]]) + if self._can_view_data_dictionary: items.append([DataDictionaryView.page_title, [{ 'title': DataDictionaryView.page_title, From 948fb211d41d4ca7bbb0f2f6ec49233cedf170cf Mon Sep 17 00:00:00 2001 From: AddisonDunn Date: Tue, 29 Oct 2024 15:07:51 -0400 Subject: [PATCH 4/4] test everything --- corehq/apps/fixtures/tests/test_views.py | 59 ++++++++++++++++++++++-- 1 file changed, 56 insertions(+), 3 deletions(-) diff --git a/corehq/apps/fixtures/tests/test_views.py b/corehq/apps/fixtures/tests/test_views.py index 9982b4a14a82..217280b68e70 100644 --- a/corehq/apps/fixtures/tests/test_views.py +++ b/corehq/apps/fixtures/tests/test_views.py @@ -7,11 +7,14 @@ from corehq.apps.domain.models import Domain from corehq.apps.domain.shortcuts import create_domain -from corehq.apps.users.models import WebUser - +from corehq.util.test_utils import flag_enabled +from corehq.apps.fixtures.models import CSQLFixtureExpression, CSQLFixtureExpressionLog +from corehq.apps.fixtures.views import CSQLFixtureExpressionView, update_tables +from corehq.apps.users.dbaccessors import delete_all_users +from corehq.apps.users.models import HqPermissions, WebUser +from corehq.apps.users.models_role import UserRole from ..interface import FixtureEditInterface from ..models import LookupTable, LookupTableRow, Field, TypeField -from corehq.apps.fixtures.views import update_tables DOMAIN = "lookup" USER = "test@test.com" @@ -316,3 +319,53 @@ def _create_request(self, method): request.user = request.couch_user = WebUser(is_superuser=True, is_authenticated=True, is_active=True) return request + + +@flag_enabled('MODULE_BADGES') +class TestCSQLFixtureExpressionView(TestCase): + + DOMAIN = 'test-domain' + DEFAULT_USER_PASSWORD = 'password' + USERNAME = 'username@test.com' + + @classmethod + def setUpClass(cls): + super(TestCSQLFixtureExpressionView, cls).setUpClass() + cls.domain_obj = create_domain(cls.DOMAIN) + cls.user = WebUser.create( + cls.DOMAIN, cls.USERNAME, cls.DEFAULT_USER_PASSWORD, None, None, is_admin=True + ) + cls.role = UserRole.create(cls.DOMAIN, 'Fixtures Access', HqPermissions(edit_data=True)) + cls.user.set_role(cls.DOMAIN, cls.role.get_qualified_id()) + + @classmethod + def tearDownClass(cls): + cls.user.delete(cls.DOMAIN, deleted_by=None) + cls.domain_obj.delete() + delete_all_users() + super(TestCSQLFixtureExpressionView, cls).tearDownClass() + + def _get_view_response(self, data): + self.client.login(username=self.USERNAME, password=self.DEFAULT_USER_PASSWORD) + response = self.client.post(reverse(CSQLFixtureExpressionView.urlname, args=[self.DOMAIN]), data) + return response + + @patch('django_prbac.decorators.has_privilege', return_value=True) + def test_create_update_delete(self, has_privilege): + deleted_exp = CSQLFixtureExpression.objects.create(domain=self.DOMAIN, name='deleted_exp', csql='asdf') + exp2 = CSQLFixtureExpression.objects.create(domain=self.DOMAIN, name='exp2', csql='asdf') + data = { + 'id': [exp2.id, ''], + 'name': ['exp2', 'exp3'], + 'csql': ['asdfg', 'asdf'], + } + response = self._get_view_response(data) + self.assertEqual(response.status_code, 200) + self.assertFalse(CSQLFixtureExpression.objects.filter( + domain=self.DOMAIN, name='deleted_exp', deleted=False).exists()) + CSQLFixtureExpression.objects.get(domain=self.DOMAIN, name='exp2', id=exp2.id, csql='asdfg') + CSQLFixtureExpression.objects.get(domain=self.DOMAIN, name='exp3', csql='asdf') + CSQLFixtureExpressionLog.objects.get(expression=deleted_exp, + action=CSQLFixtureExpressionLog.DELETE) + CSQLFixtureExpressionLog.objects.get(expression=exp2, action=CSQLFixtureExpressionLog.UPDATE) + CSQLFixtureExpressionLog.objects.get(name='exp3', action=CSQLFixtureExpressionLog.CREATE)