From 90be87f91c979dd22b96fcac42e5d62431f4ff4a Mon Sep 17 00:00:00 2001 From: Amit Phulera Date: Mon, 4 Sep 2023 12:37:01 +0530 Subject: [PATCH 01/15] allow python installations to get latest patch version --- src/commcare_cloud/ansible/roles/common_installs/tasks/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/commcare_cloud/ansible/roles/common_installs/tasks/main.yml b/src/commcare_cloud/ansible/roles/common_installs/tasks/main.yml index 5266cf58d7..5d27e5c39e 100644 --- a/src/commcare_cloud/ansible/roles/common_installs/tasks/main.yml +++ b/src/commcare_cloud/ansible/roles/common_installs/tasks/main.yml @@ -30,7 +30,7 @@ become: yes apt: name: - - python{{ python_version }} + - python{{ python_version }}={{ python_version }}.* - python{{ python_version }}-dev tags: - python From edff9857561ad2dfa69f3a687c77a50fe5301dcc Mon Sep 17 00:00:00 2001 From: Daniel Roberts Date: Mon, 11 Sep 2023 12:07:29 -0500 Subject: [PATCH 02/15] Make es_conf include elasticsearch service reload-daemon logic --- src/commcare_cloud/ansible/roles/elasticsearch/tasks/main.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/commcare_cloud/ansible/roles/elasticsearch/tasks/main.yml b/src/commcare_cloud/ansible/roles/elasticsearch/tasks/main.yml index 426f61c782..a2fbd9a222 100644 --- a/src/commcare_cloud/ansible/roles/elasticsearch/tasks/main.yml +++ b/src/commcare_cloud/ansible/roles/elasticsearch/tasks/main.yml @@ -108,6 +108,8 @@ daemon-reload: yes name: elasticsearch when: copy_elasticsearch_systemd_result.changed + tags: + - es_conf - debug: msg="If you're just updating the ES conf then you sill need to restart the cluster. Use the 'es_rolling_restart' playbook." when: copy_elasticsearch_conf_result.changed From ead71e1bb9028c385b34d752e45f51566dc78b47 Mon Sep 17 00:00:00 2001 From: Amit Phulera Date: Tue, 12 Sep 2023 17:09:44 +0530 Subject: [PATCH 03/15] add changelog.yml --- changelog/0074-update-to-python-3.9.18.yml | 26 ++++++++++++++++++++++ 1 file changed, 26 insertions(+) create mode 100644 changelog/0074-update-to-python-3.9.18.yml diff --git a/changelog/0074-update-to-python-3.9.18.yml b/changelog/0074-update-to-python-3.9.18.yml new file mode 100644 index 0000000000..94f241b0ce --- /dev/null +++ b/changelog/0074-update-to-python-3.9.18.yml @@ -0,0 +1,26 @@ +title: update-to-python-3.9.18 +key: update-to-python-3.9.18 +date: 2023-09-12 +optional_per_env: no +# (optional) Min version of HQ that MUST be deployed before this change can be rolled out (commit hash) +min_commcare_version: +# (optional) Max version of HQ that can be deployed before this change MUST be rolled out (commit hash) +max_commcare_version: +context: | + Installs python 3.9.18 and build a new virutalenv for CommCare HQ + +details: | + Python 3.9.18 applies some security patches to the known CVEs in earlier versions of Python 3.9. + Performing these steps will ensure that that those CVEs don't affect your environment + +update_steps: | + 1. Update to the latest commcare-cloud + 2. Install Python 3.9.18, update supervisor configs and sudoers file, and create + virtualenv for CommCare HQ: + ```bash + cchq deploy-stack --tags=python + ``` + It will do a dry run, after which you can review the changes and enter 'y' to apply. + 3. Restart services: + ```bash + cchq service commcare restart \ No newline at end of file From 2236ef057aa68d79089e2e6a2452a0607136e7bb Mon Sep 17 00:00:00 2001 From: Amit Phulera Date: Tue, 12 Sep 2023 17:09:54 +0530 Subject: [PATCH 04/15] make --- .../changelog/0074-update-to-python-3.9.18.md | 31 +++++++++++++++++++ docs/source/changelog/index.md | 5 +++ 2 files changed, 36 insertions(+) create mode 100644 docs/source/changelog/0074-update-to-python-3.9.18.md diff --git a/docs/source/changelog/0074-update-to-python-3.9.18.md b/docs/source/changelog/0074-update-to-python-3.9.18.md new file mode 100644 index 0000000000..7c9856080c --- /dev/null +++ b/docs/source/changelog/0074-update-to-python-3.9.18.md @@ -0,0 +1,31 @@ + + +# 74. update-to-python-3.9.18 + +**Date:** 2023-09-12 + +**Optional per env:** _required on all environments_ + + +## CommCare Version Dependency +This change is not known to be dependent on any particular version of CommCare. + + +## Change Context +Installs python 3.9.18 and build a new virutalenv for CommCare HQ + +## Details +Python 3.9.18 applies some security patches to the known CVEs in earlier versions of Python 3.9. +Performing these steps will ensure that that those CVEs don't affect your environment + +## Steps to update +1. Update to the latest commcare-cloud +2. Install Python 3.9.18, update supervisor configs and sudoers file, and create + virtualenv for CommCare HQ: + ```bash + cchq deploy-stack --tags=python + ``` + It will do a dry run, after which you can review the changes and enter 'y' to apply. +3. Restart services: + ```bash + cchq service commcare restart diff --git a/docs/source/changelog/index.md b/docs/source/changelog/index.md index 83f327d3be..67038ed515 100644 --- a/docs/source/changelog/index.md +++ b/docs/source/changelog/index.md @@ -7,6 +7,11 @@ need to be applied on your environment to keep it up to date. ### Changelog +#### **2023-09-12** [update-to-python-3.9.18](0074-update-to-python-3.9.18.md) +Installs python 3.9.18 and build a new virutalenv for CommCare HQ + + +--- #### **2023-06-14** [Prepare project spaces for Case List Explorer report release](0073-prepare-for-cle-ga.md) We have created a management command to assist with syncing data to the Case Search Index so that legacy projects may access data in a new report that will be made generally available. From e96ecf6869de6d6a720f2e0dea36a006be47e8e7 Mon Sep 17 00:00:00 2001 From: Amit Phulera Date: Wed, 13 Sep 2023 12:38:50 +0530 Subject: [PATCH 05/15] fix: syntax --- changelog/0074-update-to-python-3.9.18.yml | 3 ++- docs/source/changelog/0074-update-to-python-3.9.18.md | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/changelog/0074-update-to-python-3.9.18.yml b/changelog/0074-update-to-python-3.9.18.yml index 94f241b0ce..f9130bba3d 100644 --- a/changelog/0074-update-to-python-3.9.18.yml +++ b/changelog/0074-update-to-python-3.9.18.yml @@ -23,4 +23,5 @@ update_steps: | It will do a dry run, after which you can review the changes and enter 'y' to apply. 3. Restart services: ```bash - cchq service commcare restart \ No newline at end of file + cchq service commcare restart + ``` diff --git a/docs/source/changelog/0074-update-to-python-3.9.18.md b/docs/source/changelog/0074-update-to-python-3.9.18.md index 7c9856080c..93c3687a7a 100644 --- a/docs/source/changelog/0074-update-to-python-3.9.18.md +++ b/docs/source/changelog/0074-update-to-python-3.9.18.md @@ -29,3 +29,4 @@ Performing these steps will ensure that that those CVEs don't affect your enviro 3. Restart services: ```bash cchq service commcare restart + ``` From e71ef635aab251fb96d2442f59a1a388ac8b2ed2 Mon Sep 17 00:00:00 2001 From: Amit Phulera Date: Wed, 13 Sep 2023 20:31:38 +0530 Subject: [PATCH 06/15] turn on multiplexer on all indexes on swiss --- environments/swiss/public.yml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/environments/swiss/public.yml b/environments/swiss/public.yml index c548e2337b..afa435cd25 100644 --- a/environments/swiss/public.yml +++ b/environments/swiss/public.yml @@ -125,14 +125,14 @@ localsettings: ENABLE_SOFT_ASSERT_EMAILS: True ELASTICSEARCH_MAJOR_VERSION: 2 # Index Multiplexer Settings - ES_APPS_INDEX_MULTIPLEXED: False - ES_CASE_SEARCH_INDEX_MULTIPLEXED: False - ES_CASES_INDEX_MULTIPLEXED: False - ES_DOMAINS_INDEX_MULTIPLEXED: False - ES_FORMS_INDEX_MULTIPLEXED: False - ES_GROUPS_INDEX_MULTIPLEXED: False - ES_SMS_INDEX_MULTIPLEXED: False - ES_USERS_INDEX_MULTIPLEXED: False + ES_APPS_INDEX_MULTIPLEXED: True + ES_CASE_SEARCH_INDEX_MULTIPLEXED: True + ES_CASES_INDEX_MULTIPLEXED: True + ES_DOMAINS_INDEX_MULTIPLEXED: True + ES_FORMS_INDEX_MULTIPLEXED: True + ES_GROUPS_INDEX_MULTIPLEXED: True + ES_SMS_INDEX_MULTIPLEXED: True + ES_USERS_INDEX_MULTIPLEXED: True # Index Swap Settings ES_APPS_INDEX_SWAPPED: False ES_CASE_SEARCH_INDEX_SWAPPED: False From 9de97ca70547155dca204bee5e445372d6cc40eb Mon Sep 17 00:00:00 2001 From: Amit Phulera Date: Fri, 15 Sep 2023 15:04:08 +0530 Subject: [PATCH 07/15] disable inline script updates in elasticsearch --- .../roles/elasticsearch/templates/config/elasticsearch.yml.j2 | 1 - 1 file changed, 1 deletion(-) diff --git a/src/commcare_cloud/ansible/roles/elasticsearch/templates/config/elasticsearch.yml.j2 b/src/commcare_cloud/ansible/roles/elasticsearch/templates/config/elasticsearch.yml.j2 index 8b6c6ef2e5..eb6ef5212f 100644 --- a/src/commcare_cloud/ansible/roles/elasticsearch/templates/config/elasticsearch.yml.j2 +++ b/src/commcare_cloud/ansible/roles/elasticsearch/templates/config/elasticsearch.yml.j2 @@ -87,7 +87,6 @@ indices.fielddata.cache.size: {{ elasticsearch_fielddata_cache_size }} # somewhat of a security risk but required by pact custom reports script.engine.groovy.inline.aggs: true script.engine.groovy.inline.search: true -script.engine.groovy.inline.update: true {% endif %} {% endif %} From ef6f5ae363c1edff6ddbcbca02ffe5a8ace64e77 Mon Sep 17 00:00:00 2001 From: Amit Phulera Date: Fri, 15 Sep 2023 15:18:38 +0530 Subject: [PATCH 08/15] swap all indices on swiss --- environments/swiss/public.yml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/environments/swiss/public.yml b/environments/swiss/public.yml index afa435cd25..870ab96b3d 100644 --- a/environments/swiss/public.yml +++ b/environments/swiss/public.yml @@ -134,14 +134,14 @@ localsettings: ES_SMS_INDEX_MULTIPLEXED: True ES_USERS_INDEX_MULTIPLEXED: True # Index Swap Settings - ES_APPS_INDEX_SWAPPED: False - ES_CASE_SEARCH_INDEX_SWAPPED: False - ES_CASES_INDEX_SWAPPED: False - ES_DOMAINS_INDEX_SWAPPED: False - ES_FORMS_INDEX_SWAPPED: False - ES_GROUPS_INDEX_SWAPPED: False - ES_SMS_INDEX_SWAPPED: False - ES_USERS_INDEX_SWAPPED: False + ES_APPS_INDEX_SWAPPED: True + ES_CASE_SEARCH_INDEX_SWAPPED: True + ES_CASES_INDEX_SWAPPED: True + ES_DOMAINS_INDEX_SWAPPED: True + ES_FORMS_INDEX_SWAPPED: True + ES_GROUPS_INDEX_SWAPPED: True + ES_SMS_INDEX_SWAPPED: True + ES_USERS_INDEX_SWAPPED: True # Index Settings End IS_DIMAGI_ENVIRONMENT: True EULA_COMPLIANCE: True From 70ffa92c80cc819956117d4bc0299c3bfcbaea16 Mon Sep 17 00:00:00 2001 From: Daniel Miller Date: Mon, 18 Sep 2023 09:17:40 -0400 Subject: [PATCH 09/15] Allow --check and be less strict about --diff It's confusing to get a "that option is managed automatically" error when attempting to run with --check. The operator is forced to guess what will happen when lacking a thorough understanding of how the automatic option management works. It seems appropriate to perform a single --check run if that was requested by the operator. The situation is similar for --diff, although a duplicate option does not cause harm and does not change the run mode. --- .../commands/ansible/ansible_playbook.py | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/src/commcare_cloud/commands/ansible/ansible_playbook.py b/src/commcare_cloud/commands/ansible/ansible_playbook.py index a1f87f2732..f00d35f1c9 100644 --- a/src/commcare_cloud/commands/ansible/ansible_playbook.py +++ b/src/commcare_cloud/commands/ansible/ansible_playbook.py @@ -15,7 +15,7 @@ from commcare_cloud.alias import commcare_cloud from commcare_cloud.cli_utils import ask, has_arg, check_branch, print_command, has_local_connection_arg from commcare_cloud.user_utils import get_dev_username -from commcare_cloud.colors import color_error, color_notice, color_code +from commcare_cloud.colors import color_error, color_warning, color_notice, color_code from commcare_cloud.commands import shared_args from commcare_cloud.commands.ansible.helpers import ( AnsibleContext, DEPRECATED_ANSIBLE_ARGS, @@ -141,12 +141,9 @@ def ansible_playbook(playbook, *cmd_args): env_vars = ansible_context.build_env() cmd_parts += get_user_arg(public_vars, unknown_args, use_factory_auth) - if has_arg(unknown_args, '-D', '--diff') or has_arg(unknown_args, '-C', '--check'): - puts(color_error("Options --diff and --check not allowed. " - "Please remove -D, --diff, -C, --check.")) - puts(color_error("These ansible-playbook options are managed automatically " - "by commcare-cloud and cannot be set manually.")) - return 2 # exit code + if has_arg(unknown_args, '-D', '--diff'): + puts(color_warning("WARNING: Redundant --diff option.")) + puts(color_warning("This ansible-playbook option is managed automatically by commcare-cloud.")) cmd_parts += environment.secrets_backend.get_extra_ansible_args() @@ -167,6 +164,11 @@ def run_check(): def run_apply(): return ansible_playbook(playbook, *unknown_args) + if has_arg(unknown_args, '-C', '--check'): + # run once with --check if that arg was specified explicitly + with ansible_context.environment.secrets_backend.suppress_datadog_event(): + return ansible_playbook(playbook, *unknown_args) + return run_action_with_check_mode(run_check, run_apply, skip_check, quiet, always_skip_check) From 12e1293f6e83af17e7e018d5d3c20e3186eb4f40 Mon Sep 17 00:00:00 2001 From: Graham Herceg Date: Mon, 18 Sep 2023 12:59:17 -0400 Subject: [PATCH 10/15] use subprocess.run instead of check_output to wait for command to finish --- src/commcare_cloud/commands/terraform/aws.py | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/src/commcare_cloud/commands/terraform/aws.py b/src/commcare_cloud/commands/terraform/aws.py index cdd336f3a5..c75cd6bfb9 100644 --- a/src/commcare_cloud/commands/terraform/aws.py +++ b/src/commcare_cloud/commands/terraform/aws.py @@ -40,6 +40,19 @@ def check_output(cmd_parts, env, silent=False): return subprocess.check_output(cmd_parts, env=env_vars) +def run(cmd_parts, env, silent=False): + + env_vars = os.environ.copy() + env_vars.update(env) + if not silent: + cmd = ' '.join(shlex_quote(arg) for arg in cmd_parts) + print_command('{} {}'.format( + ' '.join('{}={}'.format(key, value) for key, value in env.items()), + cmd, + )) + return subprocess.run(cmd_parts, env=env_vars) + + def aws_cli(environment, cmd_parts): return json.loads( @@ -627,7 +640,7 @@ def _has_valid_session_credentials_for_sso(): def _refresh_sso_credentials(aws_session_profile): - check_output(['aws', 'sso', 'login'], env={'AWS_PROFILE': aws_session_profile}) + run(['aws', 'sso', 'login'], env={'AWS_PROFILE': aws_session_profile}) def _has_valid_v1_session_credentials(aws_profile): From 24c51bc73f43ac4edde144296c9851d09c69feb5 Mon Sep 17 00:00:00 2001 From: Graham Herceg Date: Mon, 18 Sep 2023 15:17:52 -0400 Subject: [PATCH 11/15] pass check=True to raise error if non-zero-exit code --- src/commcare_cloud/commands/terraform/aws.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/commcare_cloud/commands/terraform/aws.py b/src/commcare_cloud/commands/terraform/aws.py index c75cd6bfb9..279fbf2ccb 100644 --- a/src/commcare_cloud/commands/terraform/aws.py +++ b/src/commcare_cloud/commands/terraform/aws.py @@ -50,7 +50,8 @@ def run(cmd_parts, env, silent=False): ' '.join('{}={}'.format(key, value) for key, value in env.items()), cmd, )) - return subprocess.run(cmd_parts, env=env_vars) + # check=True to raise error if results in non-zero exit status + return subprocess.run(cmd_parts, env=env_vars, check=True) def aws_cli(environment, cmd_parts): From 4aa7cca4a04385fb0597514d14b2e2a23ec6d574 Mon Sep 17 00:00:00 2001 From: Graham Herceg Date: Mon, 18 Sep 2023 15:23:27 -0400 Subject: [PATCH 12/15] resolve lint errors --- src/commcare_cloud/commands/terraform/aws.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/commcare_cloud/commands/terraform/aws.py b/src/commcare_cloud/commands/terraform/aws.py index 279fbf2ccb..0978b2afb9 100644 --- a/src/commcare_cloud/commands/terraform/aws.py +++ b/src/commcare_cloud/commands/terraform/aws.py @@ -4,7 +4,6 @@ import json import os import subprocess -import textwrap from datetime import datetime from dateutil import parser import pytz @@ -104,7 +103,6 @@ def get_aws_resources(environment): "--region", config.region, ])] - nlb_endpoints = aws_cli(environment, [ 'aws', 'elbv2', 'describe-load-balancers', '--query', "LoadBalancers[?Type=='network'].[LoadBalancerName,DNSName]", @@ -428,7 +426,10 @@ def _aws_sign_in_with_sso(environment): aws_session_profile = '{}:session'.format(environment.terraform_config.aws_profile) # todo: add `... or if _date_modified(AWS_CONFIG_PATH) > _date_modified(AWS_CREDENTIALS_PATH)` if not _has_profile_for_sso(aws_session_profile): - puts(color_notice("Configuring SSO. To further customize, run `aws configure sso --profile {}`".format(aws_session_profile))) + puts(color_notice( + "Configuring SSO. To further customize, run `aws configure sso " + "--profile {}`".format( + aws_session_profile))) _write_profile_for_sso( aws_session_profile, sso_start_url=environment.aws_config.sso_config.sso_start_url, From d0f5c52d6392eca0bb152563c74ea070bc83becc Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 21 Sep 2023 21:17:36 +0000 Subject: [PATCH 13/15] Bump cryptography from 41.0.3 to 41.0.4 Bumps [cryptography](https://github.com/pyca/cryptography) from 41.0.3 to 41.0.4. - [Changelog](https://github.com/pyca/cryptography/blob/main/CHANGELOG.rst) - [Commits](https://github.com/pyca/cryptography/compare/41.0.3...41.0.4) --- updated-dependencies: - dependency-name: cryptography dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 0b184107c9..b6d28ea090 100644 --- a/requirements.txt +++ b/requirements.txt @@ -42,7 +42,7 @@ clint==0.5.1 # via commcare-cloud (setup.py) couchdb-cluster-admin==0.7.2 # via commcare-cloud (setup.py) -cryptography==41.0.3 +cryptography==41.0.4 # via # ansible-core # commcare-cloud (setup.py) From 7ce2d4c7ff706f464bb8234f7d8c8a873d853264 Mon Sep 17 00:00:00 2001 From: Amit Phulera Date: Fri, 22 Sep 2023 12:33:28 +0530 Subject: [PATCH 14/15] disable multiplexer swiss --- environments/swiss/public.yml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/environments/swiss/public.yml b/environments/swiss/public.yml index 870ab96b3d..8244be5d0f 100644 --- a/environments/swiss/public.yml +++ b/environments/swiss/public.yml @@ -125,14 +125,14 @@ localsettings: ENABLE_SOFT_ASSERT_EMAILS: True ELASTICSEARCH_MAJOR_VERSION: 2 # Index Multiplexer Settings - ES_APPS_INDEX_MULTIPLEXED: True - ES_CASE_SEARCH_INDEX_MULTIPLEXED: True - ES_CASES_INDEX_MULTIPLEXED: True - ES_DOMAINS_INDEX_MULTIPLEXED: True - ES_FORMS_INDEX_MULTIPLEXED: True - ES_GROUPS_INDEX_MULTIPLEXED: True - ES_SMS_INDEX_MULTIPLEXED: True - ES_USERS_INDEX_MULTIPLEXED: True + ES_APPS_INDEX_MULTIPLEXED: False # Swapped + ES_CASE_SEARCH_INDEX_MULTIPLEXED: False # Swapped + ES_CASES_INDEX_MULTIPLEXED: False # Swapped + ES_DOMAINS_INDEX_MULTIPLEXED: False # Swapped + ES_FORMS_INDEX_MULTIPLEXED: False # Swapped + ES_GROUPS_INDEX_MULTIPLEXED: False # Swapped + ES_SMS_INDEX_MULTIPLEXED: False # Swapped + ES_USERS_INDEX_MULTIPLEXED: False # Swapped # Index Swap Settings ES_APPS_INDEX_SWAPPED: True ES_CASE_SEARCH_INDEX_SWAPPED: True From ecd377860df245c9e7b0737c18f08864aec6c305 Mon Sep 17 00:00:00 2001 From: Graham Herceg Date: Mon, 25 Sep 2023 16:11:12 -0400 Subject: [PATCH 15/15] set formplayer_archive_time_spec to 3 days on prod --- environments/production/public.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/environments/production/public.yml b/environments/production/public.yml index c5d18c1682..d64d8d8ef5 100644 --- a/environments/production/public.yml +++ b/environments/production/public.yml @@ -67,6 +67,7 @@ filebeat_inputs: tags: nginx-error formplayer_java_version: "{{ java_17_bin_path }}/java" +formplayer_archive_time_spec: '3d' formplayer_purge_time_spec: '10d' formplayer_sensitive_data_logging: true formplayer_forward_ip_proxy: true