Merge branch 'master' into bmb/data-cleaning-prototype

.github/workflows/tests.yml

@@ -50,7 +50,16 @@ jobs:
         JS_SETUP: yes
         KAFKA_HOSTNAME: kafka
         STRIPE_PRIVATE_KEY: ${{ secrets.STRIPE_PRIVATE_KEY }}
-      run: scripts/docker test --exitfirst -vv --reusedb=1 --divided-we-run=${{ matrix.DIVIDED_WE_RUN }} --showlocals --max-test-time=29 -p no:cacheprovider
+      run: >-
+        scripts/docker test
+        --exitfirst
+        --verbosity=2
+        --reusedb=1
+        --no-migrations
+        --divided-we-run=${{ matrix.DIVIDED_WE_RUN }}
+        --showlocals
+        --max-test-time=29
+        -p no:cacheprovider
     - name: "Codecov upload"
       env:
         TOKEN: ${{ secrets.CODECOV_TOKEN }}

Dockerfile

@@ -3,57 +3,57 @@
 # This Dockerfile is built as the `dimagi/commcarehq_base` image, which
 # is used for running tests.
 
-FROM python:3.9
+FROM ghcr.io/astral-sh/uv:0.5.2-python3.9-bookworm-slim
 MAINTAINER Dimagi <[email protected]>
 
 ENV PYTHONUNBUFFERED=1 \
     PYTHONUSERBASE=/vendor \
     PATH=/vendor/bin:$PATH \
-    NODE_VERSION=20.11.1
+    NODE_VERSION=20.11.1 \
+    UV_COMPILE_BYTECODE=1 \
+    UV_LINK_MODE=copy
+# UV_COMPILE_BYTECODE: Compile bytecode during installation to improve module
+#   load performance. Also suppresses a couchdbkit syntax error that happens
+#   during bytecode compilation.
+# UV_LINK_MODE: Copy from the cache instead of linking since it's a mounted volume
 
 RUN mkdir /vendor
 
-RUN curl -SLO "https://nodejs.org/dist/v$NODE_VERSION/node-v$NODE_VERSION-linux-x64.tar.gz" \
-  && tar -xzf "node-v$NODE_VERSION-linux-x64.tar.gz" -C /usr/local --strip-components=1 \
-  && rm "node-v$NODE_VERSION-linux-x64.tar.gz"
-
 RUN apt-get update \
-  && apt-get install -y --no-install-recommends \
-     default-jdk \
-     wget \
-     libxml2-dev \
-     libxmlsec1-dev \
-     libxmlsec1-openssl \
-     gettext
-
-# Install latest chrome dev package and fonts to support major
-# charsets (Chinese, Japanese, Arabic, Hebrew, Thai and a few others)
-# Note: this installs the necessary libs to make the bundled version
-# of Chromium that Puppeteer installs, work.
-RUN wget -q -O - https://dl-ssl.google.com/linux/linux_signing_key.pub | apt-key add - \
+  && apt-get install -y --no-install-recommends curl gnupg \
+  && curl https://dl-ssl.google.com/linux/linux_signing_key.pub | apt-key add - \
   && sh -c 'echo "deb [arch=amd64] http://dl.google.com/linux/chrome/deb/ stable main" >> /etc/apt/sources.list.d/google.list' \
   && apt-get update \
   && apt-get install -y --no-install-recommends \
-     google-chrome-unstable \
-     fonts-ipafont-gothic \
-     fonts-wqy-zenhei \
-     fonts-thai-tlwg \
-     fonts-kacst \
-     fonts-freefont-ttf
+     build-essential \
+     bzip2 \
+     default-jre \
+     gettext \
+     git \
+     google-chrome-stable \
+     libmagic1 \
+     libpq-dev \
+     libxml2 \
+     libxmlsec1 \
+     libxmlsec1-openssl \
+     make \
+  && rm -rf /var/lib/apt/lists/* /src/*.deb
+# build-essential allows uv to build uwsgi; increases image size by 240 MB
+# libpq-dev is for make-requirements-test.sh; increases image size by ~20 MB
+# libpq-dev can be replaced with libpq5 if pip-tools is replaced with uv in make-requirements-test.sh
 
-# Deletes all package sources, so don't apt-get install anything after this:
-RUN rm -rf /var/lib/apt/lists/* /src/*.deb
+RUN curl -SLO "https://nodejs.org/dist/v$NODE_VERSION/node-v$NODE_VERSION-linux-x64.tar.gz" \
+  && tar -xzf "node-v$NODE_VERSION-linux-x64.tar.gz" -C /usr/local --strip-components=1 \
+  && rm "node-v$NODE_VERSION-linux-x64.tar.gz"
 
 COPY requirements/test-requirements.txt package.json /vendor/
 
-# prefer https for git checkouts made by pip
-RUN git config --global url."https://".insteadOf git:// \
- && pip install --upgrade pip \
- && pip install -r /vendor/test-requirements.txt --user --upgrade \
- && rm -rf /root/.cache/pip
+RUN --mount=type=cache,target=/root/.cache/uv \
+  uv venv --allow-existing /vendor \
+  && uv pip install --prefix=/vendor -r /vendor/test-requirements.txt
 
 # this keeps the image size down, make sure to set in mocha-headless-chrome options
-#   executablePath: 'google-chrome-unstable'
+#   executablePath: 'google-chrome-stable'
 ENV PUPPETEER_SKIP_DOWNLOAD true
 
 RUN npm -g install \
@@ -67,3 +67,10 @@ RUN npm -g install \
  && cd /vendor \
  && npm shrinkwrap \
  && yarn global add phantomjs-prebuilt
+
+# For backward compatibility with commcarehq_base image containing
+# google-chrome-unstable. Can be removed after all test jobs with Gruntfile.js
+# referencing google-chrome-unstable have completed (at least a few weeks or
+# months after the PR in which this was introduced is merged). Old PRs can be
+# updated to use google-chrome-stable by merging master into them.
+RUN ln -s /usr/bin/google-chrome-stable /usr/bin/google-chrome-unstable

Gruntfile.js

@@ -70,9 +70,9 @@ module.exports = function (grunt) {
                 reporter: reporter,
             };
 
-        // For running in docker/travis
+        // For running in docker
         if (process.env.PUPPETEER_SKIP_DOWNLOAD) {
-            runnerOptions.executablePath = 'google-chrome-unstable';
+            runnerOptions.executablePath = 'google-chrome-stable';
         }
 
         grunt.log.writeln("\n");

corehq/apps/accounting/invoicing.py

@@ -83,9 +83,11 @@ def __init__(self, date_start, date_end, domain, recipients=None):
             raise InvoiceError("Domain '%s' is not a valid domain on HQ!" % domain)
 
     def create_invoices(self):
-        subscriptions = self._get_subscriptions()
-        self._ensure_full_coverage(subscriptions)
-        for subscription in subscriptions:
+        all_subscriptions = self._get_subscriptions()
+        self._ensure_full_coverage(all_subscriptions)
+        chargeable_subscriptions = [sub for sub in all_subscriptions
+                                    if sub.plan_version.plan.edition != SoftwarePlanEdition.PAUSED]
+        for subscription in chargeable_subscriptions:
             try:
                 if subscription.account.is_customer_billing_account:
                     log_accounting_info("Skipping invoice for subscription: %s, because it is part of a Customer "
@@ -106,8 +108,6 @@ def _get_subscriptions(self):
             ),
             subscriber=self.subscriber,
             date_start__lte=self.date_end,
-        ).exclude(
-            plan_version__plan__edition=SoftwarePlanEdition.PAUSED,
         ).order_by('date_start', 'date_end').all()
         return list(subscriptions)
 

corehq/apps/accounting/tests/test_invoice_factory.py

@@ -8,6 +8,7 @@
 from corehq.apps.accounting.models import (
     BillingAccount,
     DefaultProductPlan,
+    DomainUserHistory,
     SoftwarePlanEdition,
     Subscription,
 )
@@ -125,20 +126,39 @@ def test_no_coverage(self):
         community_ranges = self.invoice_factory._get_community_ranges(subscriptions)
         self.assertEqual(community_ranges, [(self.invoice_start, self.invoice_end + datetime.timedelta(days=1))])
 
+    def test_community_plan_generates_invoice(self):
+        """
+        Ensure that Community plans can generate invoices.
+        """
+        community_plan = DefaultProductPlan.get_default_plan_version()
+        subscription = Subscription.new_domain_subscription(
+            self.account, self.domain.name, community_plan,
+            date_start=self.invoice_start,
+            date_end=self.invoice_end + datetime.timedelta(days=1),
+        )
+        DomainUserHistory.objects.create(
+            domain=self.domain.name, record_date=self.invoice_end, num_users=10)
+
+        self.invoice_factory.create_invoices()
+        invoice_count = subscription.invoice_set.count()
+        self.assertEqual(invoice_count, 1)
+
     def test_paused_plan_generates_no_invoice(self):
         """
-        Ensure that paused plans do not generate invoices.
+        Ensure that Paused plans do not generate invoices.
         """
         paused_plan = generator.subscribable_plan_version(
             edition=SoftwarePlanEdition.PAUSED
         )
-        Subscription.new_domain_subscription(
+        subscription = Subscription.new_domain_subscription(
             self.account, self.domain.name, paused_plan,
             date_start=self.invoice_start,
             date_end=self.invoice_end + datetime.timedelta(days=1),
-
         )
-        self.assertListEqual(self.invoice_factory._get_subscriptions(), [])
+
+        self.invoice_factory.create_invoices()
+        invoice_count = subscription.invoice_set.count()
+        self.assertEqual(invoice_count, 0)
 
 
 class TestInvoicingMethods(BaseAccountingTest):

corehq/apps/app_manager/fixtures/mobile_ucr.py

@@ -139,12 +139,7 @@ def _get_apps(self, restore_state, restore_user):
 
         if app_aware_sync_app:
             apps = [app_aware_sync_app]
-        elif (
-            toggles.RESTORE_ACCESSIBLE_REPORTS_ONLY.enabled(restore_user.domain)
-            and restore_state.params.is_webapps
-            # only way to reliably know that this is a web apps restore, not live preview
-            and not restore_user.request_user.can_view_apps(restore_user.domain)
-        ):
+        elif toggles.RESTORE_ACCESSIBLE_REPORTS_ONLY.enabled(restore_user.domain):
             apps = []
             for app in get_web_apps_available_to_user(restore_user.domain, restore_user._couch_user):
                 if not is_remote_app(app):

corehq/apps/app_manager/templates/app_manager/app_view_release_manager.html

@@ -38,7 +38,6 @@
   {% initial_page_data 'intro_only' intro_only %}
   {% initial_page_data 'langs' app.langs %}
   {% initial_page_data 'latest_build_id' latest_build_id %}
-  {% initial_page_data 'sms_contacts' sms_contacts %}
   {% initial_page_data 'confirm' confirm %}
   {% initial_page_data 'upstream_url' upstream_url_template %}
   {% initial_page_data 'show_release_mode' show_release_mode %}

corehq/apps/app_manager/templates/app_manager/apps_base.html

@@ -62,8 +62,6 @@
   {% endcompress %}
 
   {% include 'hqwebapp/includes/atwho.html' %}
-  <script src="{% static 'bootstrap3-typeahead/bootstrap3-typeahead.min.js' %}"></script>
-  <script src="{% static 'hqwebapp/js/bootstrap-multi-typeahead.js' %}"></script>
 
   {% include 'hqwebapp/includes/ui_element_js.html' %}
   {% compress js %}

corehq/apps/app_manager/templates/app_manager/partials/releases/releases_deploy_modal_sms.html

@@ -12,11 +12,15 @@
               </span>
       </label>
     </div>
-    <form method="post" action="{% url "send_to_recipients" domain %}">{% csrf_token %}
+    <form method="post" action="{% url "send_to_recipients" domain %}">
+      {% csrf_token %}
       <div class="form-group">
         <label>{% trans "Send to" %}</label>
-        <input type="text" name="recipients" value="" class="form-control"
-               data-bind="multiTypeahead: $root.recipients" />
+        <select name="recipients" id="sms-recipients" class="form-control" multiple data-bind="staticSelect2: {}">
+          {% for contact in sms_contacts %}
+            <option>{{ contact }}</option>
+          {% endfor %}
+        </select>
       </div>
       <div class="form-group">
                 <textarea class="bitly form-control vertical-resize" name="message"

corehq/apps/app_manager/views/releases.py

@@ -208,10 +208,7 @@ def get_releases_context(request, domain, app_id):
         'can_view_cloudcare': has_privilege(request, privileges.CLOUDCARE),
         'has_mobile_workers': get_doc_count_in_domain_by_class(domain, CommCareUser) > 0,
         'latest_released_version': get_latest_released_app_version(domain, app_id),
-        'sms_contacts': (
-            get_sms_autocomplete_context(request, domain)['sms_contacts']
-            if can_send_sms else []
-        ),
+        'sms_contacts': get_sms_autocomplete_context(domain) if can_send_sms else [],
         'build_profile_access': build_profile_access,
         'application_profile_url': reverse(LanguageProfilesView.urlname, args=[domain, app_id]),
         'latest_build_id': get_latest_build_id(domain, app_id),

corehq/apps/case_importer/templates/case_importer/partials/ko_import_status.html

@@ -27,56 +27,56 @@
   <!--/ko-->
 </div>
 <!--ko if: !$root._.isEmpty(result.errors)-->
-<div class="alert alert-warning">
-  <ul class="list-unstyled">
-    <!--ko foreach: result.errors-->
-    <!--ko if: rows-->
-    <li>
-      <!--ko if: column-->
-      <p><strong>
-        <!-- ko if: rows.length == 1-->
-        {% blocktrans %}
-          <span data-bind="text: rows.length"></span> row had an invalid
-          "<span data-bind="text: column"></span>" cell and was not saved
-        {% endblocktrans %}
+  <!--ko foreach: result.errors-->
+    <div class="alert alert-warning">
+      <!--ko if: rows-->
+        <!--ko if: column-->
+          <h6>
+            <!-- ko if: rows.length == 1-->
+            {% blocktrans %}
+              <span data-bind="text: rows.length"></span> row had an invalid
+              "<span data-bind="text: column"></span>" cell and was not saved
+            {% endblocktrans %}
+            <!--/ko-->
+            <!-- ko if: rows.length > 1-->
+            {% blocktrans %}
+              <span data-bind="text: rows.length"></span> rows had invalid
+              "<span data-bind="text: column"></span>" cells and were not saved
+            {% endblocktrans %}
+            <!--/ko-->
+          </h6>
         <!--/ko-->
-        <!-- ko if: rows.length > 1-->
-        {% blocktrans %}
-          <span data-bind="text: rows.length"></span> rows had invalid
-          "<span data-bind="text: column"></span>" cells and were not saved
-        {% endblocktrans %}
+        <!--ko if: !column -->
+          <h6 data-bind="text: title"></h6>
         <!--/ko-->
-      </strong></p>
-      <!--/ko-->
-      <!--ko if: !column -->
-      <p><strong data-bind="text: title"></strong></p>
-      <!--/ko-->
-      <p data-bind="text: description">
-      </p>
-      <p data-bind="visible: sample, text: sample">
-      </p>
-      <div data-bind="visible: rows.length">
-        <button type="button"
-                class="btn btn-outline-primary btn-sm"
-                data-bs-toggle="collapse"
-                data-bind="attr: {
-                  'data-bs-target': '#rowDetailsCollapse-' + $parentContext.$index(),
-                  'aria-controls': 'rowDetailsCollapse-' + $parentContext.$index()
-                }"
-                aria-expanded="false">
-          {% trans "Toggle Affected Row(s)" %}
-        </button>
-        <div class="collapse"
-             data-bind="attr: { id: 'rowDetailsCollapse-' + $parentContext.$index() }">
-          <div class="card card-body mt-3"
-               data-bind="text: rows.join(', ')"></div>
+        <p data-bind="text: description"></p>
+        <p data-bind="visible: sample, text: sample"></p>
+        <div data-bind="visible: rows.length">
+          <button
+              type="button"
+              class="btn btn-outline-primary btn-sm"
+              data-bs-toggle="collapse"
+              data-bind="attr: {
+                'data-bs-target': '#rowDetailsCollapse-' + $parentContext.$index() + '-' + $index(),
+                'aria-controls': 'rowDetailsCollapse-' + $parentContext.$index() + '-' + $index()
+              }"
+              aria-expanded="false"
+          >
+            {% trans "Toggle Affected Row(s)" %}
+          </button>
+          <div
+              class="collapse"
+              data-bind="attr: { id: 'rowDetailsCollapse-' + $parentContext.$index() + '-' + $index() }"
+          >
+            <div
+                class="card card-body mt-3"
+                data-bind="text: rows.join(', ')"
+            ></div>
+          </div>
         </div>
-      </div>
-    </li>
-    <!--/ko-->
-    <!--/ko-->
-  </ul>
-</div>
+      <!--/ko-->
+    </div>
+  <!--/ko-->
 <!--/ko-->
 <!--/ko-->
 <!--ko if: state == $root.states.SUCCESS && !result-->