Merge branch 'master' of github.com:dimagi/commcare-hq into es/2stick…

…y2furious

.github/CODEOWNERS

@@ -41,7 +41,7 @@
 /corehq/apps/case_search/ @orangejenny @MartinRiese @esoergel @AddisonDunn @stephherbers @Robert-Costello @Jtang-1
 /corehq/apps/change_feed/ @dannyroberts
 /corehq/apps/cleanup/management/commands/populate_sql_model_from_couch_model.py @millerdev
-/corehq/apps/cloudcare/ @orangejenny @MartinRiese @AddisonDunn @stephherbers @Robert-Costello @Jtang-1 @nospame
+/corehq/apps/cloudcare/ @MartinRiese @AddisonDunn @stephherbers @Robert-Costello @Jtang-1 @nospame
 /corehq/apps/commtrack/ @esoergel
 /corehq/apps/data_dictionary/ @esoergel @orangejenny @zandre-eng
 /corehq/apps/domain/decorators.py @esoergel

.github/workflows/codeql-analysis.yml

@@ -12,8 +12,8 @@
 name: "CodeQL"
 
 on:
-  push:
-    branches: [ master ]
+#  push:
+#    branches: [ master ]
 #  pull_request:
 #    # The branches below must be a subset of the branches above
 #    branches: [ master ]
@@ -39,7 +39,7 @@ jobs:
 
     # Initializes the CodeQL tools for scanning.
     - name: Initialize CodeQL
-      uses: github/codeql-action/init@v2
+      uses: github/codeql-action/init@v3
       with:
         languages: ${{ matrix.language }}
         # If you wish to specify custom queries, you can do so here or in a config file.
@@ -50,7 +50,7 @@ jobs:
     # Autobuild attempts to build any compiled languages  (C/C++, C#, or Java).
     # If this step fails, then you should remove it and run the build manually (see below)
     - name: Autobuild
-      uses: github/codeql-action/autobuild@v2
+      uses: github/codeql-action/autobuild@v3
 
     # ℹ️ Command-line programs to run using the OS shell.
     # 📚 https://git.io/JvXDl
@@ -64,4 +64,4 @@ jobs:
     #   make release
 
     - name: Perform CodeQL Analysis
-      uses: github/codeql-action/analyze@v2
+      uses: github/codeql-action/analyze@v3

DEV_SETUP.md

@@ -766,9 +766,9 @@ ability to assign other users as superuser in the in-app Superuser Management pa
 
 In order to build JavaScript bundles with Webpack, you will need to have `yarn dev`
 running in the background. It will watch any existing Webpack Entry Point, aka modules
-included on a page using the `webpack_main` template tag.
+included on a page using the `js_entry` template tag.
 
-When you add a new entry point (`webpack_main` tag), please remember to restart `yarn dev` so
+When you add a new entry point (`js_entry` tag), please remember to restart `yarn dev` so
 that it can identify the new entry point it needs to watch.
 
 To build Webpack bundles like it's done in production environments, pleas use `yarn build`.

corehq/apps/accounting/decorators.py

@@ -1,7 +1,6 @@
-import json
 from functools import wraps
 
-from django.http import HttpResponse
+from django.http import HttpResponse, JsonResponse
 
 from django_prbac.decorators import requires_privilege
 from django_prbac.exceptions import PermissionDenied
@@ -65,7 +64,7 @@ def requires_privilege_plaintext_response(slug,
     """
     A version of the requires_privilege decorator which returns an
     HttpResponse object with HTTP Status Code of 412 by default and
-    content_type of tex/plain if the privilege is not found.
+    content_type of text/plain if the privilege is not found.
     """
     def decorate(fn):
         @wraps(fn)
@@ -114,8 +113,7 @@ def wrapped(request, *args, **kwargs):
             except PermissionDenied:
                 error_message = "You have lost access to this feature."
                 response = get_response(error_message, http_status_code)
-                return HttpResponse(json.dumps(response),
-                                    content_type="application/json", status=401)
+                return JsonResponse(response, status=401)
         return wrapped
     return decorate
 

corehq/apps/api/object_fetch_api.py

@@ -122,7 +122,7 @@ def get(self, request, domain, case_id=None, attachment_id=None):
         if attachment_meta is not None:
             mime_type = attachment_meta['content_type']
         else:
-            mime_type = "plain/text"
+            mime_type = "text/plain"
 
         return StreamingHttpResponse(streaming_content=FileWrapper(attachment_stream),
                                      content_type=mime_type)

corehq/apps/api/resources/auth.py

@@ -1,8 +1,7 @@
-import json
 from functools import wraps
 
 from django.core.exceptions import PermissionDenied
-from django.http import Http404, HttpResponse, HttpResponseForbidden
+from django.http import Http404, HttpResponseForbidden, JsonResponse
 
 from attrs import define, field
 from tastypie.authentication import Authentication
@@ -28,11 +27,9 @@ def _inner(req, *args, **kwargs):
             return view_func(req, *args, **kwargs)
         except Http404 as e:
             if str(e):
-                return HttpResponse(json.dumps({"error": str(e)}),
-                                content_type="application/json",
-                                status=404)
-            return HttpResponse(json.dumps({"error": "not authorized"}),
-                                content_type="application/json",
+                return JsonResponse({"error": str(e)},
+                                    status=404)
+            return JsonResponse({"error": "not authorized"},
                                 status=401)
     return _inner
 
@@ -192,7 +189,7 @@ def _get_auth_decorator(self, request):
 class DomainAdminAuthentication(LoginAndDomainAuthentication):
 
     def is_authenticated(self, request, **kwargs):
-        permission_check = lambda couch_user, domain: couch_user.is_domain_admin(domain)
+        permission_check = lambda couch_user, domain: couch_user.is_domain_admin(domain)  # noqa: E731
         wrappers = [
             require_permission_raw(permission_check, login_decorator=self._get_auth_decorator(request)),
             wrap_4xx_errors_for_apis,

corehq/apps/api/tests/test_user_updates.py

@@ -142,6 +142,10 @@ def test_update_groups_fails(self):
         with self.assertRaises(ValidationError):
             update(self.user, 'groups', [group._id])
 
+    def test_update_groups_with_fake_group_id_raises_exception(self):
+        with self.assertRaises(ValidationError):
+            update(self.user, 'groups', ["fake_id"])
+
     def test_update_unknown_field_raises_exception(self):
         with self.assertRaises(UpdateUserException) as cm:
             update(self.user, 'username', 'new-username')

corehq/apps/app_execution/static/app_execution/js/workflow_logs.js

@@ -5,7 +5,7 @@ hqDefine("app_execution/js/workflow_logs", [
     'hqwebapp/js/initial_page_data',
     'hqwebapp/js/tempus_dominus',
     'app_execution/js/workflow_charts',
-    'hqwebapp/js/bootstrap5/components.ko',
+    'hqwebapp/js/components/pagination',
 ], function ($, ko, initialPageData, hqTempusDominus) {
     let logsModel = function () {
         let self = {};

corehq/apps/app_manager/decorators.py

@@ -114,7 +114,8 @@ def _safe_cached_download(request, *args, **kwargs):
             if not username:
                 content_response = dict(error="app.update.not.allowed.user.logged_out",
                                         default_response=_("Please log in to the app to check for an update."))
-                return HttpResponse(status=406, content=json.dumps(content_response))
+                return HttpResponse(status=406, content=json.dumps(content_response),
+                                    content_type='application/json')
         if latest and not target:
             latest_enabled_build = _get_latest_enabled_build(domain, username, app_id, request.GET.get('profile'),
                                                              location_flag_enabled)

corehq/apps/app_manager/fields.py

@@ -15,9 +15,10 @@
 
 from corehq.apps.app_manager.analytics import get_exports_by_application
 from corehq.apps.app_manager.dbaccessors import get_app, get_apps_in_domain
+from corehq.apps.export.const import ALL_CASE_TYPE_EXPORT
+from corehq.apps.hqwebapp import crispy as hqcrispy
 from corehq.apps.registry.models import DataRegistry
 from corehq.apps.registry.utils import get_data_registry_dropdown_options
-from corehq.apps.hqwebapp import crispy as hqcrispy
 from corehq.apps.reports.analytics.esaccessors import get_case_types_for_domain
 from corehq.apps.userreports.app_manager.data_source_meta import (
     DATA_SOURCE_TYPE_CASE,
@@ -26,7 +27,6 @@
 )
 from corehq.apps.userreports.dbaccessors import get_datasources_for_domain
 from corehq.toggles import AGGREGATE_UCRS
-from corehq.apps.export.const import ALL_CASE_TYPE_EXPORT
 
 DataSource = collections.namedtuple('DataSource', ['application', 'source_type', 'source', 'registry_slug'])
 RMIDataChoice = collections.namedtuple('RMIDataChoice', ['id', 'text', 'data'])
@@ -285,8 +285,10 @@ class ApplicationDataRMIHelper(object):
     APP_TYPE_NONE = 'no_app'
     APP_TYPE_UNKNOWN = 'unknown'
 
-    def __init__(self, domain, user, as_dict=True):
+    def __init__(self, domain, project, user, as_dict=True):
         self.domain = domain
+        self.domain_object = project
+
         self.user = user
         self.as_dict = as_dict
         self.form_labels = AppFormRMIPlaceholder(
@@ -420,14 +422,16 @@ def _sort_key_form(form):
     def _all_forms(self):
         forms = []
         unknown_forms = []
-        for f in get_exports_by_form(self.domain):
+
+        for f in get_exports_by_form(self.domain, use_es=self.domain_object.exports_use_elasticsearch):
             form = f['value']
             if form.get('app_deleted') and not form.get('submissions'):
                 continue
             if 'app' in form:
                 form['has_app'] = True
                 forms.append(form)
-            else:
+            elif not self.domain_object.exports_use_elasticsearch:
+                # If the elasticsearch toggle is on, we don't care about forms without apps
                 app_id = f['key'][1] or ''
                 form['app'] = {
                     'id': app_id

corehq/apps/app_manager/static/app_manager/js/download_async_modal.js

@@ -1,4 +1,10 @@
-hqDefine('app_manager/js/download_async_modal', function () {
+hqDefine('app_manager/js/download_async_modal', [
+    'jquery',
+    'underscore',
+], function (
+    $,
+    _
+) {
     var asyncDownloader = function ($el) {
         "use strict";
         var self = {};
@@ -7,9 +13,9 @@ hqDefine('app_manager/js/download_async_modal', function () {
                              "If you see this repeatedly please report an issue.");
 
         self.$el = $el;
-        self.el_id = $el.attr("id");
-        self.$download_progress = self.$el.find("#" + self.el_id + "-download-progress");
-        self.$downloading = self.$el.find("#" + self.el_id + "-downloading");
+        self.elId = $el.attr("id");
+        self.$download_progress = self.$el.find("#" + self.elId + "-download-progress");
+        self.$downloading = self.$el.find("#" + self.elId + "-downloading");
 
         self.init = function () {
             self.download_in_progress = false;
@@ -40,29 +46,29 @@ hqDefine('app_manager/js/download_async_modal', function () {
             }
         };
 
-        self.updateProgress = function (progress_response) {
-            if (progress_response.trim().length) {
+        self.updateProgress = function (progressResponse) {
+            if (progressResponse.trim().length) {
                 self.$downloading.addClass("hide");
-                self.$download_progress.html(progress_response).removeClass("hide");
+                self.$download_progress.html(progressResponse).removeClass("hide");
             }
         };
 
-        self.isDone = function (progress_response) {
-            var ready_id = 'ready_' + self.download_poll_id,
-                error_id = 'error_' + self.download_poll_id;
-            return progress_response &&
-                progress_response.trim().length &&
-                _.any([ready_id, error_id], function (el_id) {
-                    return progress_response.indexOf(el_id) >= 0;
+        self.isDone = function (progressResponse) {
+            var readyId = 'ready_' + self.download_poll_id,
+                errorId = 'error_' + self.download_poll_id;
+            return progressResponse &&
+                progressResponse.trim().length &&
+                _.any([readyId, errorId], function (elId) {
+                    return progressResponse.indexOf(elId) >= 0;
                 });
         };
 
-        self.generateDownload = function (download_url, params) {
+        self.generateDownload = function (downloadUrl, params) {
             // prevent multiple calls
             if (!self.download_in_progress) {
                 self.download_in_progress = true;
                 $.ajax({
-                    url: download_url,
+                    url: downloadUrl,
                     type: "GET",
                     data: params,
                     dataType: "json",

corehq/apps/app_manager/static/app_manager/js/settings/commcare_settings.js

@@ -522,13 +522,3 @@ hqDefine('app_manager/js/settings/commcare_settings', function () {
         CommcareSettings: CommcareSettings,
     };
 });
-
-ko.bindingHandlers.passwordSetter = {
-    init: function (element, valueAccessor) {
-        var observableValue = valueAccessor();
-        $(element).password_setter();
-        $(element).on('textchange change', function () {
-            observableValue($(element).val());
-        });
-    },
-};

corehq/apps/app_manager/static/app_manager/js/summary/case_summary.js

@@ -7,7 +7,7 @@ hqDefine('app_manager/js/summary/case_summary',[
     'app_manager/js/summary/models',
     'app_manager/js/menu',  // enable lang switcher and "Updates to publish" banner
     'hqwebapp/js/bootstrap3/knockout_bindings.ko', // popover
-    'hqwebapp/js/bootstrap3/components.ko',    // search box
+    'hqwebapp/js/components/search_box',
 ], function ($, _, ko, initialPageData, assertProperties, models) {
 
     var caseTypeModel = function (caseType) {

corehq/apps/app_manager/static/app_manager/js/summary/form_diff.js

@@ -10,7 +10,7 @@ hqDefine('app_manager/js/summary/form_diff',[
     'hqwebapp/js/layout',
     'app_manager/js/menu',  // enable lang switcher and "Updates to publish" banner
     'hqwebapp/js/bootstrap3/knockout_bindings.ko', // popover
-    'hqwebapp/js/bootstrap3/components.ko',    // search box
+    'hqwebapp/js/components/search_box',
 ], function ($, _, ko, initialPageData, assertProperties, models, formModels, utils, layout) {
 
     $(function () {

corehq/apps/app_manager/static/app_manager/js/summary/form_summary.js

@@ -9,7 +9,7 @@ hqDefine('app_manager/js/summary/form_summary',[
     'app_manager/js/summary/utils',
     'app_manager/js/menu',  // enable lang switcher and "Updates to publish" banner
     'hqwebapp/js/bootstrap3/knockout_bindings.ko', // popover
-    'hqwebapp/js/bootstrap3/components.ko',    // search box
+    'hqwebapp/js/components/search_box',
 ], function ($, _, ko, initialPageData, assertProperties, models, formModels, utils) {
     $(function () {
         var lang = initialPageData.get('lang'),

corehq/apps/app_manager/templates/app_manager/app_view.html

@@ -20,7 +20,6 @@
     <script src="{% static 'hqwebapp/js/bootstrap3/widgets.js' %}"></script>
     <script src="{% static 'app_manager/js/supported_languages.js' %}"></script>
     <script src="{% static 'app_manager/js/settings/translations.js' %}"></script>
-    <script src="{% static 'app_manager/js/settings/password_setter.jquery.js' %}"></script>
     <script src="{% static "hqmedia/js/hqmediauploaders.js" %}"></script>
     <script src="{% static 'hqmedia/js/media_reference_models.js' %}"></script>
     <script src="{% static 'app_manager/js/app_view.js' %}"></script>
@@ -76,8 +75,7 @@
 
 {% block modals %}{{ block.super }}
   {% for uploader in uploaders %}
-    {% include 'hqmedia/partials/multimedia_uploader.html' %}
+    {% include 'hqmedia/partials/multimedia_uploader.html' with id=uploader.slug type=uploader.media_type %}
   {% endfor %}
-  {% include 'app_manager/partials/settings/password_setter_modal.html' %}
   {% include 'app_manager/partials/toggle_diff_modal.html' %}
 {% endblock %}

corehq/apps/app_manager/templates/app_manager/form_view.html

@@ -312,5 +312,6 @@
 
 {% block modals %}{{ block.super }}
   {% include "app_manager/partials/forms/form_view_modals.html" %}
-  {% include "app_manager/partials/nav_menu_media_modals.html" %}
+  {% include "hqmedia/partials/multimedia_uploader.html" with id=multimedia.upload_managers.icon.slug type="image" %}
+  {% include "hqmedia/partials/multimedia_uploader.html" with id=multimedia.upload_managers.audio.slug type="audio" %}
 {% endblock modals %}

corehq/apps/app_manager/templates/app_manager/module_view.html

@@ -197,10 +197,11 @@
 {% endblock %}
 
 {% block modals %}{{ block.super }}
-  {% include "app_manager/partials/nav_menu_media_modals.html" %}
+  {% include "hqmedia/partials/multimedia_uploader.html" with id=multimedia.upload_managers.icon.slug type="image" %}
+  {% include "hqmedia/partials/multimedia_uploader.html" with id=multimedia.upload_managers.audio.slug type="audio" %}
   {% if request|toggle_enabled:"CASE_DETAIL_PRINT" %}
     {% with print_uploader as uploader %}
-      {% include 'hqmedia/partials/multimedia_uploader.html' %}
+      {% include 'hqmedia/partials/multimedia_uploader.html' with id=uploader.slug type=uploader.media_type %}
     {% endwith %}
   {% endif %}
 {% endblock modals %}