Merge branch 'master' into ml/case-deletion

.github/workflows/lint.yml

@@ -58,7 +58,7 @@ jobs:
           fetch-depth: 0
 
       - name: Set up Node.js
-        uses: actions/setup-node@v3
+        uses: actions/setup-node@v4
         with:
           node-version: 14
 

.github/workflows/tests.yml

@@ -50,6 +50,7 @@ jobs:
         NOSE_DIVIDED_WE_RUN: ${{ matrix.NOSE_DIVIDED_WE_RUN }}
         JS_SETUP: yes
         KAFKA_HOSTNAME: kafka
+        STRIPE_PRIVATE_KEY: ${{ secrets.STRIPE_PRIVATE_KEY }}
       run: scripts/docker test --noinput --stop -v --divided-we-run=${{ matrix.NOSE_DIVIDED_WE_RUN }} --divide-depth=1 --with-timing --with-flaky --threshold=10 --max-test-time=29
     - name: "Codecov upload"
       env:

DEV_SETUP.md

@@ -532,10 +532,12 @@ $ ./manage.py create_kafka_topics
 $ env CCHQ_IS_FRESH_INSTALL=1 ./manage.py migrate --noinput
 ```
 
-If you are using a partitioned database, populate the additional databases too:
+If you are using a partitioned database, populate the additional
+databases too, and configure PL/Proxy:
 
 ```sh
 $ env CCHQ_IS_FRESH_INSTALL=1 ./manage.py migrate_multi --noinput
+$ ./manage.py configure_pl_proxy_cluster --create_only
 ```
 
 You should run `./manage.py migrate` frequently, but only use the environment

Dockerfile

@@ -54,7 +54,7 @@ RUN git config --global url."https://".insteadOf git:// \
 
 # this keeps the image size down, make sure to set in mocha-headless-chrome options
 #   executablePath: 'google-chrome-unstable'
-ENV PUPPETEER_SKIP_CHROMIUM_DOWNLOAD true
+ENV PUPPETEER_SKIP_DOWNLOAD true
 
 RUN npm -g install \
     yarn \

Gruntfile.js

@@ -73,7 +73,7 @@ module.exports = function (grunt) {
             };
 
         // For running in docker/travis
-        if (process.env.PUPPETEER_SKIP_CHROMIUM_DOWNLOAD) {
+        if (process.env.PUPPETEER_SKIP_DOWNLOAD) {
             runnerOptions.executablePath = 'google-chrome-unstable';
         }
 

corehq/apps/accounting/forms.py

@@ -625,10 +625,10 @@ def __init__(self, subscription, account_id, web_user, *args, **kwargs):
             is_most_recent_version = subscription.plan_version.plan.get_version() == subscription.plan_version
             most_recent_version_text = ("is most recent version" if is_most_recent_version
                                         else "not most recent version")
-            self.fields['most_recent_version'].initial = most_recent_version_text
+            self.fields['most_recent_version'].initial = is_most_recent_version
             most_recent_version_field = hqcrispy.B3TextField(
                 'most_recent_version',
-                self.fields['most_recent_version'].initial
+                most_recent_version_text
             )
             self.fields['domain'].choices = [
                 (subscription.subscriber.domain, subscription.subscriber.domain)

corehq/apps/accounting/interface.py

@@ -945,7 +945,7 @@ class CustomerInvoiceInterface(InvoiceInterfaceBase):
         'corehq.apps.accounting.interface.IsHiddenFilter',
     ]
 
-    account = None
+    subscription = None
 
     @property
     def headers(self):

corehq/apps/accounting/invoicing.py

@@ -849,6 +849,7 @@ def _end_date_count_sms(self):
     @property
     @memoized
     def unit_cost(self):
+        """Return aggregate cost of all the excess SMS"""
         total_excess = Decimal('0.0')
         if self.is_within_monthly_limit:
             return total_excess

corehq/apps/accounting/models.py

@@ -19,6 +19,7 @@
 import stripe
 from django_prbac.models import Role
 from memoized import memoized
+from corehq.apps.accounting.utils.stripe import charge_through_stripe
 
 from corehq.apps.domain.shortcuts import publish_domain_saved
 from dimagi.ext.couchdbkit import (
@@ -562,6 +563,8 @@ def _send_autopay_card_removed_email(self, new_user, domain):
             email,
             render_to_string('accounting/email/autopay_card_removed.html', context),
             text_content=strip_tags(render_to_string('accounting/email/autopay_card_removed.html', context)),
+            domain=domain,
+            use_domain_gateway=True,
         )
 
     def _send_autopay_card_added_email(self, domain):
@@ -593,6 +596,8 @@ def _send_autopay_card_added_email(self, domain):
             email,
             render_to_string('accounting/email/invoice_autopay_setup.html', context),
             text_content=strip_tags(render_to_string('accounting/email/invoice_autopay_setup.html', context)),
+            domain=domain,
+            use_domain_gateway=True,
         )
 
     @staticmethod
@@ -3770,18 +3775,33 @@ def _remove_card_from_all_accounts(self, card):
                 account.remove_autopay_user()
 
     def create_card(self, stripe_token, billing_account, domain, autopay=False):
+        """
+        Creates and associates a new card with the Stripe customer.
+
+        This method uses a Stripe token (usually generated on the client side)
+        to securely create a new card and associate it with the customer
+        represented by this instance. Additionally, if the 'autopay' flag is
+        set to True, it sets the card to be used for automatic payments for
+        a specific billing account and domain.
+
+        Parameters:
+        - stripe_token (str): The token representing the card details, typically
+                            generated using Stripe.js on the client side.
+        - billing_account (BillingAccount): The account for which the card might
+                                            be set for automatic payments.
+        - domain (str): The domain associated with the billing account.
+        - autopay (bool, optional): Flag indicating if the card should be set for
+                                    automatic payments. Default is False.
+
+        Returns:
+        - card (stripe.Card): The newly created Stripe card object.
+        """
         customer = self.customer
         card = customer.cards.create(card=stripe_token)
-        self.set_default_card(card)
         if autopay:
             self.set_autopay(card, billing_account, domain)
         return card
 
-    def set_default_card(self, card):
-        self.customer.default_card = card
-        self.customer.save()
-        return card
-
     def set_autopay(self, card, billing_account, domain):
         """
         Sets the auto_pay status on the card for a billing account
@@ -3838,11 +3858,10 @@ def _auto_pay_card_metadata_key(billing_account):
 
     def create_charge(self, card, amount_in_dollars, description, idempotency_key=None):
         """ Charges a stripe card and returns a transaction id """
-        amount_in_cents = int((amount_in_dollars * Decimal('100')).quantize(Decimal(10)))
-        transaction_record = stripe.Charge.create(
+        transaction_record = charge_through_stripe(
             card=card,
             customer=self.customer,
-            amount=amount_in_cents,
+            amount_in_dollars=amount_in_dollars,
             currency=settings.DEFAULT_CURRENCY,
             description=description,
             idempotency_key=idempotency_key

corehq/apps/accounting/payment_handlers.py

@@ -21,6 +21,7 @@
     log_accounting_error,
     log_accounting_info,
 )
+from corehq.apps.accounting.utils.stripe import charge_through_stripe
 from corehq.const import USER_DATE_FORMAT
 
 stripe.api_key = settings.STRIPE_PRIVATE_KEY
@@ -60,11 +61,6 @@ def update_credits(self, payment_record):
         """
         raise NotImplementedError("you must implement update_credits")
 
-    @staticmethod
-    def get_amount_in_cents(amount):
-        amt_cents = amount * Decimal('100')
-        return int(amt_cents.quantize(Decimal(10)))
-
     def update_payment_information(self, account):
         account.last_payment_method = LastPayment.CC_ONE_TIME
         account.pre_or_post_pay = PreOrPostPay.POSTPAY
@@ -182,10 +178,10 @@ def cost_item_name(self):
         return _("Invoice #%s") % self.invoice.id
 
     def create_charge(self, amount, card=None, customer=None):
-        return stripe.Charge.create(
+        return charge_through_stripe(
             card=card,
             customer=customer,
-            amount=self.get_amount_in_cents(amount),
+            amount_in_dollars=amount,
             currency=settings.DEFAULT_CURRENCY,
             description="Payment for Invoice %s" % self.invoice.invoice_number,
         )
@@ -244,10 +240,10 @@ def cost_item_name(self):
         return _('Bulk Payment for project space %s' % self.domain)
 
     def create_charge(self, amount, card=None, customer=None):
-        return stripe.Charge.create(
+        return charge_through_stripe(
             card=card,
             customer=customer,
-            amount=self.get_amount_in_cents(amount),
+            amount_in_dollars=amount,
             currency=settings.DEFAULT_CURRENCY,
             description=self.cost_item_name,
         )
@@ -345,10 +341,10 @@ def get_charge_amount(self, request):
         return Decimal(request.POST['amount'])
 
     def create_charge(self, amount, card=None, customer=None):
-        return stripe.Charge.create(
+        return charge_through_stripe(
             card=card,
             customer=customer,
-            amount=self.get_amount_in_cents(amount),
+            amount_in_dollars=amount,
             currency=settings.DEFAULT_CURRENCY,
             description="Payment for %s" % self.cost_item_name,
         )

corehq/apps/accounting/tests/generator.py

@@ -114,7 +114,8 @@ def subscribable_plan_version(edition=SoftwarePlanEdition.STANDARD):
 
 @unit_testing_only
 def generate_domain_subscription(account, domain, date_start, date_end,
-                                 plan_version=None, service_type=SubscriptionType.NOT_SET, is_active=False):
+                                 plan_version=None, service_type=SubscriptionType.NOT_SET,
+                                 is_active=False, do_not_invoice=False):
     subscriber, _ = Subscriber.objects.get_or_create(domain=domain.name)
     subscription = Subscription(
         account=account,
@@ -124,6 +125,7 @@ def generate_domain_subscription(account, domain, date_start, date_end,
         date_end=date_end,
         service_type=service_type,
         is_active=is_active,
+        do_not_invoice=do_not_invoice
     )
     subscription.save()
     return subscription

corehq/apps/accounting/tests/test_autopay_with_api.py

@@ -0,0 +1,148 @@
+from django.core import mail
+
+from django_prbac.models import Role
+import stripe
+
+from dimagi.utils.dates import add_months_to_date
+
+from corehq.apps.accounting import tasks, utils
+from corehq.apps.accounting.models import (
+    Invoice,
+    PaymentRecord,
+    SoftwarePlan,
+    SoftwarePlanVersion,
+    SoftwareProductRate,
+    StripePaymentMethod,
+)
+from corehq.apps.accounting.payment_handlers import (
+    AutoPayInvoicePaymentHandler,
+)
+from corehq.apps.accounting.tests import generator
+from corehq.apps.accounting.tests.test_invoicing import BaseInvoiceTestCase
+from django.db import transaction
+from unittest import SkipTest
+from django.conf import settings
+
+
+class TestBillingAutoPay(BaseInvoiceTestCase):
+
+    @classmethod
+    def setUpClass(cls):
+        super(TestBillingAutoPay, cls).setUpClass()
+        # Dependabot-created PRs do not have access to secrets.
+        # We skip test so the tests do not fail when dependabot creates new PR for dependency upgrades.
+        # Or for developers running tests locally if they do not have stripe API key in their localsettings.
+        if not settings.STRIPE_PRIVATE_KEY:
+            raise SkipTest("Stripe API Key not set")
+        cls._generate_autopayable_entities()
+        cls._generate_non_autopayable_entities()
+        cls._generate_invoices()
+
+    @classmethod
+    def _generate_autopayable_entities(cls):
+        """
+        Create account, domain and subscription linked to the autopay user that have autopay enabled
+        """
+        cls.autopay_account = cls.account
+        cls.autopay_account.created_by_domain = cls.domain
+        cls.autopay_account.save()
+        web_user = generator.arbitrary_user(domain_name=cls.domain.name, is_active=True, is_webuser=True)
+        cls.autopay_user_email = web_user.email
+        cls.stripe_customer = stripe.Customer.create(email=cls.autopay_user_email)
+        cls.addClassCleanup(cls.stripe_customer.delete)
+        cls.payment_method = StripePaymentMethod(web_user=cls.autopay_user_email,
+                                                customer_id=cls.stripe_customer.id)
+        # cls.payment_method.save()
+        cls.card = cls.payment_method.create_card('tok_visa', cls.autopay_account, None)
+        cls.payment_method.set_autopay(cls.card, cls.autopay_account, cls.domain)
+        cls.payment_method.save()
+        cls.autopay_account.update_autopay_user(cls.autopay_user_email, cls.domain)
+
+    @classmethod
+    def _generate_non_autopayable_entities(cls):
+        """
+        Create account, domain, and subscription linked to the autopay user, but that don't have autopay enabled
+        """
+        cls.non_autopay_account = generator.billing_account(
+            web_user_creator=generator.create_arbitrary_web_user_name(is_dimagi=True),
+            web_user_contact=cls.autopay_user_email
+        )
+        cls.non_autopay_domain = generator.arbitrary_domain()
+        cls.addClassCleanup(cls.non_autopay_domain.delete)
+        # Non-autopay subscription has same parameters as the autopayable subscription
+        cheap_plan = SoftwarePlan.objects.create(name='cheap')
+        cheap_product_rate = SoftwareProductRate.objects.create(monthly_fee=100, name=cheap_plan.name)
+        cheap_plan_version = SoftwarePlanVersion.objects.create(
+            plan=cheap_plan,
+            product_rate=cheap_product_rate,
+            role=Role.objects.first(),
+        )
+        cls.non_autopay_subscription = generator.generate_domain_subscription(
+            cls.non_autopay_account,
+            cls.non_autopay_domain,
+            plan_version=cheap_plan_version,
+            date_start=cls.subscription.date_start,
+            date_end=add_months_to_date(cls.subscription.date_start, cls.subscription_length),
+        )
+
+    @classmethod
+    def _generate_invoices(cls):
+        """
+        Create invoices for both autopayable and non-autopayable subscriptions
+        """
+        # invoice date is 2 months before the end of the subscription (this is arbitrary)
+        invoice_date = utils.months_from_date(cls.subscription.date_start, cls.subscription_length - 2)
+        tasks.calculate_users_in_all_domains(invoice_date)
+        tasks.generate_invoices_based_on_date(invoice_date)
+
+    def test_get_autopayable_invoices(self):
+        """
+        Invoice.autopayable_invoices() should return invoices that can be automatically paid
+        """
+        autopayable_invoice = Invoice.objects.filter(subscription=self.subscription)
+        date_due = autopayable_invoice.first().date_due
+
+        autopayable_invoices = Invoice.autopayable_invoices(date_due)
+
+        self.assertCountEqual(autopayable_invoices, autopayable_invoice)
+
+    def test_get_autopayable_invoices_returns_nothing(self):
+        """
+        Invoice.autopayable_invoices() should not return invoices if the customer does not have an autopay method
+        """
+        not_autopayable_invoice = Invoice.objects.filter(subscription=self.non_autopay_subscription)
+        date_due = not_autopayable_invoice.first().date_due
+        autopayable_invoices = Invoice.autopayable_invoices(date_due)
+        self.assertCountEqual(autopayable_invoices, [])
+
+    def test_pay_autopayable_invoices(self):
+        original_outbox_length = len(mail.outbox)
+
+        autopayable_invoice = Invoice.objects.filter(subscription=self.subscription)
+        date_due = autopayable_invoice.first().date_due
+
+        AutoPayInvoicePaymentHandler().pay_autopayable_invoices(date_due)
+        self.assertAlmostEqual(autopayable_invoice.first().get_total(), 0)
+        self.assertEqual(len(PaymentRecord.objects.all()), 1)
+        self.assertEqual(len(mail.outbox), original_outbox_length + 1)
+
+    def test_double_charge_is_prevented_and_only_one_payment_record_created(self):
+        self.original_outbox_length = len(mail.outbox)
+        invoice = Invoice.objects.get(subscription=self.subscription)
+        original_amount = invoice.balance
+        self._run_autopay()
+        # Add balance to the same invoice so it gets paid again
+        invoice = Invoice.objects.get(subscription=self.subscription)
+        invoice.balance = original_amount
+        invoice.save()
+        # Run autopay again to test no double charge
+        with transaction.atomic(), self.assertLogs(level='ERROR') as log_cm:
+            self._run_autopay()
+            self.assertIn("[BILLING] [Autopay] Attempt to double charge invoice", "\n".join(log_cm.output))
+        self.assertEqual(len(PaymentRecord.objects.all()), 1)
+        self.assertEqual(len(mail.outbox), self.original_outbox_length + 1)
+
+    def _run_autopay(self):
+        autopayable_invoice = Invoice.objects.filter(subscription=self.subscription)
+        date_due = autopayable_invoice.first().date_due
+        AutoPayInvoicePaymentHandler().pay_autopayable_invoices(date_due)