Merge pull request #33469 from dimagi/revert-33392-ml/deleted_on_part…

…ial_index

Revert "remove usage of XFormInstance.DELETED (part 1)"

corehq/apps/data_pipeline_audit/dbacessors.py

@@ -100,7 +100,7 @@ def get_primary_db_form_counts(domain, startdate=None, enddate=None):
         for doc_type, state in doc_type_to_state.items():
             counter[doc_type] += queryset.filter(state=state).count()
 
-        where_clause = 'deleted_on IS NOT NULL'
+        where_clause = 'state & {0} = {0}'.format(XFormInstance.DELETED)
         counter['XFormInstance-Deleted'] += queryset.extra(where=[where_clause]).count()
 
     return counter

corehq/apps/data_pipeline_audit/management/commands/find_sql_forms_not_in_es.py

@@ -10,7 +10,7 @@
 import sys
 
 from django.core.management.base import BaseCommand
-from django.db.models import Q
+from django.db.models import F, Q
 from django.db.models.functions import Greatest
 
 from corehq.util.argparse_types import date_type
@@ -64,7 +64,12 @@ def iter_form_ids_by_last_modified(start_datetime, end_datetime):
     return paginate_query_across_partitioned_databases(
         XFormInstance,
         (Q(last_modified__gt=start_datetime, last_modified__lt=end_datetime)
-         & Q(state__in=[XFormInstance.NORMAL, XFormInstance.ARCHIVED])),
+         & Q(state=F('state').bitand(XFormInstance.DELETED)
+             + F('state').bitand(XFormInstance.DEPRECATED)
+             + F('state').bitand(XFormInstance.DUPLICATE)
+             + F('state').bitand(XFormInstance.ERROR)
+             + F('state').bitand(XFormInstance.SUBMISSION_ERROR_LOG)
+             + F('state'))),
         annotate=annotate,
         values=['form_id'],
         load_source='find_sql_forms_not_in_es'

corehq/apps/domain/tests/test_delete_domain.py

@@ -1063,7 +1063,7 @@ def test_archived_forms_are_deleted(self):
         self.assertEqual(len(XFormInstance.objects.get_form_ids_in_domain(self.deleted_domain.name)), 0)
 
     def test_soft_deleted_forms_are_deleted(self):
-        create_form_for_test(self.deleted_domain.name, deleted_on=datetime.now())
+        create_form_for_test(self.deleted_domain.name, state=XFormInstance.DELETED)
         call_command('hard_delete_forms_and_cases_in_domain', self.deleted_domain.name, noinput=True)
         self.assertEqual(len(XFormInstance.objects.get_form_ids_in_domain(self.deleted_domain.name)), 0)
 

corehq/form_processor/backends/sql/dbaccessors.py

@@ -8,7 +8,7 @@
 
 from django.conf import settings
 from django.db import InternalError, transaction, router
-from django.db.models import Q
+from django.db.models import F, Q
 from django.db.models.expressions import Value
 from django.db.models.functions import Concat
 
@@ -335,8 +335,9 @@ def doc_to_json(self, doc):
     def extra_filters(self, for_count=False):
         filters = []
         if not (for_count or self.include_deleted):
-            # don't include in count query since the query planner can't account for it
-            filters.append(Q(deleted_on__isnull=True))
+            # don't inlucde in count query since the query planner can't account for it
+            # hack for django: 'state & DELETED = 0' so 'state = state + state & DELETED'
+            filters.append(Q(state=F('state').bitand(XFormInstance.DELETED) + F('state')))
         if self.domain:
             filters.append(Q(domain=self.domain))
         if self.start_date is not None:

corehq/form_processor/models/forms.py

@@ -4,7 +4,7 @@
 from io import BytesIO
 
 from django.db import InternalError, models, transaction
-from django.db.models import Q
+from django.db.models import F, Q
 
 from jsonfield.fields import JSONField
 from lxml import etree
@@ -187,7 +187,8 @@ def get_deleted_form_ids_in_domain(self, domain):
         for db_name in get_db_aliases_for_partitioned_query():
             result.extend(
                 self.using(db_name)
-                .filter(domain=domain, deleted_on__isnull=False)
+                .annotate(state_deleted=F('state').bitand(XFormInstance.DELETED))
+                .filter(domain=domain, state_deleted=XFormInstance.DELETED)
                 .values_list('form_id', flat=True)
             )
         return result
@@ -209,7 +210,7 @@ def get_deleted_form_ids_for_user(self, domain, user_id):
     def _get_form_ids_for_user(self, domain, user_id, is_deleted):
         with self.model.get_plproxy_cursor(readonly=True) as cursor:
             cursor.execute(
-                'SELECT form_id FROM get_form_ids_for_user_2(%s, %s, %s)',
+                'SELECT form_id FROM get_form_ids_for_user(%s, %s, %s)',
                 [domain, user_id, is_deleted]
             )
             results = fetchall_as_namedtuple(cursor)
@@ -340,7 +341,7 @@ def soft_delete_forms(self, domain, form_ids, deletion_date=None, deletion_id=No
         deletion_date = deletion_date or datetime.utcnow()
         with self.model.get_plproxy_cursor() as cursor:
             cursor.execute(
-                'SELECT soft_delete_forms_3(%s, %s, %s, %s) as affected_count',
+                'SELECT soft_delete_forms(%s, %s, %s, %s) as affected_count',
                 [domain, form_ids, deletion_date, deletion_id]
             )
             results = fetchall_as_namedtuple(cursor)
@@ -356,7 +357,7 @@ def soft_undelete_forms(self, domain, form_ids):
         problem = 'Restored on {}'.format(datetime.utcnow())
         with self.model.get_plproxy_cursor() as cursor:
             cursor.execute(
-                'SELECT soft_undelete_forms_3(%s, %s, %s) as affected_count',
+                'SELECT soft_undelete_forms(%s, %s, %s) as affected_count',
                 [domain, form_ids, problem]
             )
             results = fetchall_as_namedtuple(cursor)
@@ -562,7 +563,9 @@ def is_submission_error_log(self):
 
     @property
     def is_deleted(self):
-        return bool(self.deleted_on)
+        # deleting a form adds the deleted state to the current state
+        # in order to support restoring the pre-deleted state.
+        return self.state & self.DELETED == self.DELETED
 
     @property
     def doc_type(self):
@@ -638,7 +641,7 @@ def get_sync_token(self):
 
     def soft_delete(self):
         type(self).objects.soft_delete_forms(self.domain, [self.form_id])
-        self.deleted_on = datetime.utcnow()
+        self.state |= self.DELETED
 
     def to_json(self, include_attachments=False):
         from ..serializers import XFormInstanceSerializer, lazy_serialize_form_attachments, \

migrations.lock

@@ -985,7 +985,6 @@ sql_accessors
  0066_drop_unused_function
  0067_livequery_sql
  0067_livequery_sql_include_deleted_indices
- 0068_remove_deleted_state
 sql_proxy_accessors
  0001_initial
  0002_add_sync_functions
@@ -1036,7 +1035,6 @@ sql_proxy_accessors
  0047_remove_get_case_models_functions
  0048_get_ledger_values_for_cases_3
  0049_drop_unused_function
- 0050_remove_deleted_state
 sql_proxy_standby_accessors
  0001_initial
 sso