Merge branch 'master' into ce/connectid-messaging-settins

dimagi · Nov 15, 2024 · c0c16a1 · c0c16a1
2 parents dce0bff + 359d2e4
commit c0c16a1
Show file tree

Hide file tree

Showing 45 changed files with 752 additions and 618 deletions.
diff --git a/changelog/0083-database_backup_retention_change.yml b/changelog/0083-database_backup_retention_change.yml
@@ -0,0 +1,23 @@
+title: Database Backup Retention Change
+key: database_backup_retention_change
+date: 2024-10-25
+optional_per_env: yes
+min_commcare_version:
+max_commcare_version:
+context: |
+  The logic used to retain database backups has been modified to better respect the settings that specify
+  how many days to keep backups.
+
+details: |
+  No action is needed. This is just to inform you that the logic determining which backups to
+  keep has been modified. This impacts the values set for services like [couchdb](https://github.com/dimagi/commcare-cloud/blob/c3abda48758fb337ca87e8039f1b97e333129351/src/commcare_cloud/ansible/roles/couchdb2/defaults/main.yml#L9-L10),
+  [postgres](https://github.com/dimagi/commcare-cloud/blob/c3abda48758fb337ca87e8039f1b97e333129351/src/commcare_cloud/ansible/roles/pg_backup/defaults/main.yml#L9-L10),
+  and [blob](https://github.com/dimagi/commcare-cloud/blob/c3abda48758fb337ca87e8039f1b97e333129351/src/commcare_cloud/ansible/roles/backups/defaults/main.yml#L5-L6).
+
+  Put simply, if the <service>_backup_days value was set to 2, 4 daily backups would exist on your
+  machine at any given time. This change reduces that value to 3 which is more in line with what
+  it means to keep backups for 2 days.
+
+
+update_steps: |
+  No action needed.
diff --git a/changelog/0084-add-geospatial-queue.yml b/changelog/0084-add-geospatial-queue.yml
@@ -0,0 +1,30 @@
+title: New Geospatial Celery Queue
+key: add-geospatial-queue
+date: 2024-11-06
+optional_per_env: no
+min_commcare_version:
+max_commcare_version: 513700958cf443a85f0480ff681979d207f22db0
+context: |
+  The Microplanning feature requires the location properties on cases to be indexed correctly so
+  that they can be used correctly in this feature. A separate queue is added to handle this indexing
+  process when the Microplanning feature is enabled for a domain. This can potentially be a long-running
+  task, and so having a separate queue ensures that other queues do not get clogged up by this indexing task.
+
+details: |
+  This creates a geospatial_queue in Celery, which will be used for indexing cases when enabling the
+  Microplanning feature.
+
+update_steps: |
+  This update should be performed before updating CommCare to a version more recent than that specified above.
+
+  First, add a queue called `geospatial_queue` to your app_processes.yml,
+  following the examples in commcare-cloud commit [a94636b1d](https://github.com/dimagi/commcare-cloud/commit/a94636b1d).
+
+  Then, to apply this change:
+
+  ```
+  commcare-cloud <env> update-supervisor-confs
+  ```
+
+  Please note that a restart is required for the change to take effect and can be done as part of the above command
+  or with a separately done deploy.
diff --git a/docs/source/changelog/0083-database_backup_retention_change.md b/docs/source/changelog/0083-database_backup_retention_change.md
@@ -0,0 +1,29 @@
+<!--THIS FILE IS AUTOGENERATED: DO NOT EDIT-->
+<!--See https://github.com/dimagi/commcare-cloud/blob/master/changelog/README.md for instructions-->
+# 83. Database Backup Retention Change
+
+**Date:** 2024-10-25
+
+**Optional per env:** _only required on some environments_
+
+
+## CommCare Version Dependency
+This change is not known to be dependent on any particular version of CommCare.
+
+
+## Change Context
+The logic used to retain database backups has been modified to better respect the settings that specify
+how many days to keep backups.
+
+## Details
+No action is needed. This is just to inform you that the logic determining which backups to
+keep has been modified. This impacts the values set for services like [couchdb](https://github.com/dimagi/commcare-cloud/blob/c3abda48758fb337ca87e8039f1b97e333129351/src/commcare_cloud/ansible/roles/couchdb2/defaults/main.yml#L9-L10),
+[postgres](https://github.com/dimagi/commcare-cloud/blob/c3abda48758fb337ca87e8039f1b97e333129351/src/commcare_cloud/ansible/roles/pg_backup/defaults/main.yml#L9-L10),
+and [blob](https://github.com/dimagi/commcare-cloud/blob/c3abda48758fb337ca87e8039f1b97e333129351/src/commcare_cloud/ansible/roles/backups/defaults/main.yml#L5-L6).
+
+Put simply, if the <service>_backup_days value was set to 2, 4 daily backups would exist on your
+machine at any given time. This change reduces that value to 3 which is more in line with what
+it means to keep backups for 2 days.
+
+## Steps to update
+No action needed.
diff --git a/docs/source/changelog/0084-add-geospatial-queue.md b/docs/source/changelog/0084-add-geospatial-queue.md
@@ -0,0 +1,38 @@
+<!--THIS FILE IS AUTOGENERATED: DO NOT EDIT-->
+<!--See https://github.com/dimagi/commcare-cloud/blob/master/changelog/README.md for instructions-->
+# 84. New Geospatial Celery Queue
+
+**Date:** 2024-11-06
+
+**Optional per env:** _required on all environments_
+
+
+## CommCare Version Dependency
+CommCare versions beyond the following commit require this change to function correctly:
+[51370095](https://github.com/dimagi/commcare-hq/commit/513700958cf443a85f0480ff681979d207f22db0)
+
+
+## Change Context
+The Microplanning feature requires the location properties on cases to be indexed correctly so
+that they can be used correctly in this feature. A separate queue is added to handle this indexing
+process when the Microplanning feature is enabled for a domain. This can potentially be a long-running
+task, and so having a separate queue ensures that other queues do not get clogged up by this indexing task.
+
+## Details
+This creates a geospatial_queue in Celery, which will be used for indexing cases when enabling the
+Microplanning feature.
+
+## Steps to update
+This update should be performed before updating CommCare to a version more recent than that specified above.
+
+First, add a queue called `geospatial_queue` to your app_processes.yml,
+following the examples in commcare-cloud commit [a94636b1d](https://github.com/dimagi/commcare-cloud/commit/a94636b1d).
+
+Then, to apply this change:
+
+```
+commcare-cloud <env> update-supervisor-confs
+```
+
+Please note that a restart is required for the change to take effect and can be done as part of the above command
+or with a separately done deploy.
diff --git a/docs/source/changelog/index.md b/docs/source/changelog/index.md
@@ -7,6 +7,20 @@ need to be applied on your environment to keep it up to date.
 
 ### Changelog
 
+#### **2024-11-06** [New Geospatial Celery Queue](0084-add-geospatial-queue.md)
+The Microplanning feature requires the location properties on cases to be indexed correctly so
+that they can be used correctly in this feature. A separate queue is added to handle this indexing
+process when the Microplanning feature is enabled for a domain. This can potentially be a long-running
+task, and so having a separate queue ensures that other queues do not get clogged up by this indexing task.
+
+
+---
+#### **2024-10-25** [Database Backup Retention Change](0083-database_backup_retention_change.md)
+The logic used to retain database backups has been modified to better respect the settings that specify
+how many days to keep backups.
+
+
+---
 #### **2024-07-25** [Unicel SMS Gateway Removal](0082-unicel-sms-gateway-removal.md)
 This is a notice specifically for projects with Unicel configured as an SMS gateway, which is no longer functional.
 Projects that are not using the Unicel gateway do not need to take any action.

diff --git a/docs/source/installation/1-quick-monolith-install.rst b/docs/source/installation/1-quick-monolith-install.rst
@@ -11,7 +11,6 @@ Prerequisites
 - A single Ubuntu 22.04 64-bit server
 - Root user to SSH into the server
 - git must be installed on the server. If not, please use https://github.com/git-guides/install-git#debianubuntu to install git
-- We recommend using Python 3.10 with commcare-cloud. Follow instructions at https://commcare-cloud.readthedocs.io/en/latest/installation/2-manual-install.html#upgrade-to-python-3-10 to upgrade.
 
 Installation Steps
 ------------------

diff --git a/docs/source/installation/2-manual-install.rst b/docs/source/installation/2-manual-install.rst
@@ -180,31 +180,6 @@ add them to the “sudo” user group. For example, if your username were
    ...
    $ sudo usermod -a -G sudo jbloggs
 
-Configure SSH
--------------
-
-If you do not have an SSH key pair already, you will need to create one.
-(Substitute “[email protected]” with your email address)
-
-::
-
-   $ ssh-keygen -t rsa -b 4096 -C "[email protected]"
-
-**Cluster only:** Copy an SSH key pair for your user to the control
-machine. For example, if the key pair you want to copy is
-``~/.ssh/id_rsa`` and ``~/.ssh/id_rsa.pub``, then the commands to copy
-the SSH key pair would be
-
-::
-
-   $ ssh-copy-id -i ~/.ssh/id_rsa.pub jbloggs@control1
-   $ scp ~/.ssh/id_rsa{,.pub} control1:.ssh/
-
-You can now log in using your SSH key:
-
-::
-
-   (jbloggs@jbloggs-pc) $ ssh control1
 
 Install CommCare Cloud
 ----------------------
@@ -283,12 +258,6 @@ Install CommCare Cloud
 9.  Copy your **public** key to ``~/environments/_authorized_keys/``.
     The filename must correspond to your username.
 
-    For example:
-
-    ::
-
-       $ cp ~/.ssh/id_rsa.pub ~/environments/_authorized_keys/$(whoami).pub
-
 10. Change “monolith.commcarehq.test” to your real domain name,
 
     ::

diff --git a/docs/source/reference/1-commcare-cloud/2-configuration.rst b/docs/source/reference/1-commcare-cloud/2-configuration.rst
@@ -190,7 +190,8 @@ Each ``<queue-name>`` must be one of the following values:
 ``linked_domain_queue``, ``reminder_case_update_queue``, ``reminder_queue``,
 ``reminder_rule_queue``, ``repeat_record_queue``, ``saved_exports_queue``,
 ``sumologic_logs_queue``, ``send_report_throttled``, ``sms_queue``,
-``submission_reprocessing_queue``, ``ucr_indicator_queue``, ``ucr_queue``.
+``submission_reprocessing_queue``, ``ucr_indicator_queue``, ``ucr_queue``,
+``geospatial_queue``.
 For all features to work, each of these queues must
 appear at least once, and up to once per host.
 

diff --git a/environments/development/app-processes.yml b/environments/development/app-processes.yml
@@ -9,7 +9,7 @@ celery_processes:
       concurrency: 1
     email_queue,reminder_case_update_queue:
       concurrency: 1
-    linked_domain_queue,reminder_rule_queue,repeat_record_queue,saved_exports_queue:
+    linked_domain_queue,reminder_rule_queue,repeat_record_queue,saved_exports_queue,geospatial_queue:
       concurrency: 1
     ucr_indicator_queue,ucr_queue:
       concurrency: 1

diff --git a/environments/echis/app-processes.yml b/environments/echis/app-processes.yml
@@ -79,6 +79,8 @@ celery_processes:
       prefetch_multiplier: 1
     logistics_reminder_queue:
       concurrency: 2
+    geospatial_queue:
+      concurrency: 1
   echis_server5:
     ucr_indicator_queue:
       concurrency: 15

diff --git a/environments/india/app-processes.yml b/environments/india/app-processes.yml
@@ -36,6 +36,8 @@ celery_processes:
       concurrency: 1
     background_queue,dashboard_comparison_queue:
       concurrency: 2
+    geospatial_queue:
+      concurrency: 1
 pillows:
   pillow5:
     AppDbChangeFeedPillow:

diff --git a/environments/pna/app-processes.yml b/environments/pna/app-processes.yml
@@ -11,7 +11,7 @@ celery_processes:
       concurrency: 2
     background_queue,export_download_queue,saved_exports_queue,analytics_queue:
       concurrency: 4
-    linked_domain_queue,ucr_queue,async_restore_queue,email_queue,case_rule_queue:
+    linked_domain_queue,ucr_queue,async_restore_queue,email_queue,case_rule_queue,geospatial_queue:
       concurrency: 1
     beat: {}
     celery_periodic:

diff --git a/environments/production/app-processes.yml b/environments/production/app-processes.yml
@@ -76,6 +76,8 @@ celery_processes:
       prefetch_multiplier: 1
     ush_background_tasks:
       concurrency: 8
+    geospatial_queue:
+      concurrency: 1
 
   celerybeat_a0:
     # not really queues

diff --git a/environments/production/terraform.yml b/environments/production/terraform.yml
@@ -67,7 +67,7 @@ servers:
     server_auto_recovery: true
 
   - server_name: "web_m_a2{i}-production"
-    server_instance_type: c6a.8xlarge
+    server_instance_type: m6a.4xlarge
     network_tier: "app-private"
     az: "a"
     volume_size: 40
@@ -77,7 +77,7 @@ servers:
     server_auto_recovery: true
 
   - server_name: "web_m_b2{i}-production"
-    server_instance_type: c6a.8xlarge
+    server_instance_type: m6a.4xlarge
     network_tier: "app-private"
     az: "b"
     volume_size: 40

diff --git a/environments/staging/app-processes.yml b/environments/staging/app-processes.yml
@@ -1,6 +1,5 @@
-formplayer_command_args: ''
 # Use the following to enable datadog tracing
-# formplayer_command_args: '-javaagent:/home/cchq/dd-java-agent.jar -Dsrc.main.java.org.javarosa.enableOpenTracing=true'
+formplayer_command_args: '-javaagent:/home/cchq/dd-java-agent.jar -Dsrc.main.java.org.javarosa.enableOpenTracing=true -Ddd.service=formplayer -Ddd.env=staging'
 
 django_command_prefix: '{{ virtualenv_home }}/bin/ddtrace-run '
 datadog_pythonagent: True
@@ -62,6 +61,8 @@ celery_processes:
       prefetch_multiplier: 1
     ush_background_tasks:
       concurrency: 2
+    geospatial_queue:
+      concurrency: 1
     beat: {}
     flower: {}
 pillows:

diff --git a/environments/staging/public.yml b/environments/staging/public.yml
@@ -3,8 +3,8 @@ monitor_celery_heartbeat: False
 
 elasticsearch_endpoint: '{{ groups.elasticsearch.0 }}'
 elasticsearch_cluster_name: 'staginges'
-elasticsearch_version: 5.6.16
-elasticsearch_download_sha256: 6b035a59337d571ab70cea72cc55225c027ad142fbb07fd8984e54261657c77f.
+elasticsearch_version: 6.8.23
+elasticsearch_download_sha256: 424af91f838f9e5f13e0292f97cbd6333535450291a621d761bd479dfc2dff78.
 elasticsearch_delete_old_logs: True
 
 kafka_version: 3.2.3
@@ -135,7 +135,7 @@ localsettings:
   COUCH_PASSWORD: "{{ COUCH_PASSWORD }}"
   COUCH_USERNAME: "{{ COUCH_USERNAME }}"
   DEPLOY_MACHINE_NAME: "{{ ansible_hostname }}"
-  ELASTICSEARCH_MAJOR_VERSION: 5
+  ELASTICSEARCH_MAJOR_VERSION: 6
   EMAIL_SMTP_HOST: email-smtp.us-east-1.amazonaws.com
   EMAIL_SMTP_PORT: 587
   EMAIL_USE_TLS: yes

diff --git a/environments/swiss/app-processes.yml b/environments/swiss/app-processes.yml
@@ -10,7 +10,7 @@ celery_processes:
       concurrency: 1  
     export_download_queue:
       concurrency: 1
-    case_import_queue,reminder_case_update_queue,linked_domain_queue:
+    case_import_queue,reminder_case_update_queue,linked_domain_queue,geospatial_queue:
       concurrency: 1
     background_queue,case_rule_queue:
       concurrency: 1