diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 108fe5b..2ad4320 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -17,7 +17,7 @@ # under the License. # -# This GitHub Actions workflow will prepare a environment with Python 3.9 +# This GitHub Actions workflow will prepare a environment with Python 3.11 # and the required Python packages, to run the CI tests # (Ansible-lint, Flake8 etc.) for Fluo-Muchos. # See also: @@ -43,7 +43,7 @@ jobs: - uses: actions/checkout@v4 - uses: actions/setup-python@v4 with: - python-version: '3.9' + python-version: '3.11' - name: Install required packages run: pip install -r ./lib/requirements.txt - name: Install common Ansible collections diff --git a/README.md b/README.md index 9834908..92b3eaa 100644 --- a/README.md +++ b/README.md @@ -28,12 +28,12 @@ Check out [Uno] for setting up Accumulo or Fluo on a single machine. Muchos requires the following common components for installation and setup: * Python 3 with a virtual environment setup. -Create a Python 3 environment and switch to it. (The CI tests using Python 3.9, +Create a Python 3 environment and switch to it. (The CI tests using Python 3.11, but this should work in later versions as well. If you encounter problems, please file an issue). ```bash cd ~ -python3.9 -m venv env +python3.11 -m venv env source env/bin/activate ``` * `ssh-agent` installed and running and ssh-agent forwarding. Note that this may diff --git a/ansible/roles/azure/tasks/create_multiple_vmss.yml b/ansible/roles/azure/tasks/create_multiple_vmss.yml index 28ccce8..21b4e3d 100644 --- a/ansible/roles/azure/tasks/create_multiple_vmss.yml +++ b/ansible/roles/azure/tasks/create_multiple_vmss.yml @@ -51,6 +51,7 @@ resource_group: "{{ resource_group }}" location: "{{ location }}" name: "{{ vmss_name }}-{{ item.name_suffix }}" + orchestration_mode: "Uniform" vm_size: "{{ item.sku }}" priority: "{{ item.vmss_priority | default('None') }}" admin_username: "{{ cluster_user }}" diff --git a/ansible/roles/azure/tasks/create_vmss.yml b/ansible/roles/azure/tasks/create_vmss.yml index 3c1d9f4..60bf1cd 100644 --- a/ansible/roles/azure/tasks/create_vmss.yml +++ b/ansible/roles/azure/tasks/create_vmss.yml @@ -56,6 +56,7 @@ resource_group: "{{ resource_group }}" location: "{{ location }}" name: "{{ vmss_name }}" + orchestration_mode: "Uniform" vm_size: "{{ vm_sku }}" priority: "{{ vmss_priority | default('None') }}" admin_username: "{{ cluster_user }}" diff --git a/conf/checksums b/conf/checksums index af03a97..11942cc 100644 --- a/conf/checksums +++ b/conf/checksums @@ -1,4 +1,4 @@ -accumulo:2.1.2:27778c1c3f1d88ab128649fd0671d3be97ba052216ab43f1169395960e8c7d16375a51f940c2262437b836ea31f83f73f08f7a3d8cadda443e5e8bb31d9b23c5 +accumulo:2.1.3:1a27a144dc31f55ccc8e081b6c1bc6cc0362a8391838c53c166cb45291ff8f35867fd8e4729aa7b2c540f8b721f8c6953281bf589fc7fe320e4dc4d20b87abc4 accumulo:2.0.1:b443839443a9f5098b55bc5c54be10c11fedbaea554ee6cd42eaa9311068c70bd611d7fc67698c91ec73da0e85b9907aa72b98d5eb4d49ea3a5d51b0c6c5785f accumulo:1.10.3:36e6795ad3720ba72fc9f4ddabf45f6d67cdc77658a181733fa2c47bfd3799f123b8840a79b538b8504a7b4bdc97fd0b52efab790395a5e022a897bc18405d0c fluo:1.2.0:037f89cd2bfdaf76a1368256c52de46d6b9a85c9c1bfc776ec4447d02c813fb2 @@ -12,11 +12,12 @@ hadoop:2.9.2:bd6060b629c2e7c5d411ae3ba2330b79cdb2f1caa87d5ea69392c0f9cb64a83464e hadoop:2.8.5:4174e7a67b614b7d5e47a1a2420cbe9a57978908f8ad0405f1d177306fb36ed87c895810f70e3c6a6cbadc76afb9303f1c49cbca67237e18c799d30f87afa57c hadoop:2.7.7:17c8917211dd4c25f78bf60130a390f9e273b0149737094e45f4ae5c917b1174b97eb90818c5df068e607835120126281bcc07514f38bd7fd3cb8e9d3db1bdde hadoop:2.6.5:001ad18d4b6d0fe542b15ddadba2d092bc97df1c4d2d797381c8d12887691898 +spark:3.4.4:405f604532c11c3793b757aae41bbae0f86f3bcab65f47dea07be9003532c382429feab67d8ed19721ee539801842b21d32685c37a97372f96d50b318f5b1bb0 spark:3.2.0:710854DDB0617EFCB9783725F027617FFEB6CE17A161D069F4008700E0027CB87E5BF6D772A21746AEAEC0DA06703A8B257DEF72951C1073BFE0A3EBE497AF71 spark:3.1.2:E7B580BC67356F1B36756B18A95E452307EB3825265616E399B1766124764D6E65AB4C666AD23C93E006D88B6C83A78A7D786BE981B4F20696697860A459879D spark:3.0.3:22F80D2D50B964FA6F87358330CCDAA4748FA1FC546272A1EFB21868DC59A815CE9BDCEE3B520B797E3740221212BAACF1A4BA5E7166D552A0990EFB9478DE32 spark:2.4.8:84B52335834F1DC0540D5156F4CCD642A23C3CA130EBD5DC1BC8FBF1CAF0F89B783DA34DABD200A0B6385176BAD3BA4261C2C6639CAFC8CA6B8FFB8D7BB3EA72 -zookeeper:3.8.3:eb1a5e79fe55fa50e36021ca8b752e40584d5e24f23985c43e9f56abfbf23c0239dce37d18f27213c9a38fcb6f68daf1a9468fc352e6ac2364faaf9b56145bdf +zookeeper:3.8.4:4d85d6f7644d5f36d9c4d65e78bd662ab35ebe1380d762c24c12b98af029027eee453437c9245dbdf2b9beb77cd6b690b69e26f91cf9d11b0a183a979c73fa43 zookeeper:3.7.2:6afbfc1afc8b9370281bd9862f37dbb1cb95ec54bb2ed4371831aa5c0f08cfee775050bd57ce5fc0836e61af27eed9f0076f54b98997dd0e15159196056e52ea zookeeper:3.6.4:dec4835a1a23fa69599091d03b35f6c9f4e545d5fa1e149658cffdd7d1044ec31e14f41189aabbf59011dfce6609e6850209008e67cfd0f26270f1558196dcd8 zookeeper:3.5.10:fbca82d93fb07534d2cd7d93a399463cfea7e772075285d103e7cda5599d14c2d9996c1e50cc66c7c0dc78f996b20e75b20d74df368eb222dbc3caa0ab2cc14b diff --git a/conf/muchos.props.example b/conf/muchos.props.example index 3851f2c..c9c6852 100644 --- a/conf/muchos.props.example +++ b/conf/muchos.props.example @@ -37,11 +37,11 @@ accumulo_instance = muchos accumulo_password = secret # Software versions (make sure you have a corresponding entry for the checksum in conf/checksums) hadoop_version = 3.3.6 -zookeeper_version = 3.8.3 -spark_version = 2.4.8 +zookeeper_version = 3.8.4 +spark_version = 3.4.4 fluo_version = 1.2.0 fluo_yarn_version = 1.0.0 -accumulo_version = 2.1.2 +accumulo_version = 2.1.3 # Specifies if software should be downloaded. If 'False', tarballs of the software above should be in conf/upload/ download_software = True # Install Hub (for GitHub) diff --git a/docs/azure-image-reference.md b/docs/azure-image-reference.md index 8d010b8..c726741 100644 --- a/docs/azure-image-reference.md +++ b/docs/azure-image-reference.md @@ -17,8 +17,8 @@ The sections below describe the various image related configurations in `muchos. The Rocky Linux images in Azure require plan information to be supplied. Currently, the Rocky Linux 8 image has been verified to work correctly. For using Rocky Linux 8 instead of Alma Linux, here is what you can use in `muchos.props`. The image plan information is mandatory for this image: - * `azure_image_reference = rockylinux|erockyenterprisesoftwarefoundationinc1653071250513|free|latest||` - * `azure_image_plan = free|rockylinux|erockyenterprisesoftwarefoundationinc1653071250513|` + * `azure_image_reference = rockylinux-x86_64|resf|8-base|latest||` + * `azure_image_plan = 8-base|rockylinux-x86_64|resf|` In addition, you might need to manually view and accept the terms of use for the Rocky Linux image, before being able to use it within Muchos. See the `azure_image_plan` section below for details. @@ -39,11 +39,11 @@ The sections below describe the various image related configurations in `muchos. ## azure_image_plan `azure_image_plan` is only needed when working with images which require payment plan information to be supplied when a VM or VMSS is being created using that image. The format of this configuration is `plan_name|product|publisher|`. Plan information for the images published by a given publisher can easily be queried by using the Azure CLI. For example, to query the plan information for a Rocky Linux image in Azure: - `az vm image show --urn "erockyenterprisesoftwarefoundationinc1653071250513:rockylinux:free:latest" --query "plan"` + `az vm image show --urn "resf:rockylinux-x86_64:8-base:latest" --query "plan"` Then using that information, `azure_image_plan` can be configured as below in muchos.props: - `azure_image_plan = free|rockylinux|erockyenterprisesoftwarefoundationinc1653071250513|` + `azure_image_plan = 8-base|rockylinux-x86_64|resf|` More information about purchase plans, and accepting terms, etc. is available [here](https://learn.microsoft.com/en-us/azure/virtual-machines/linux/cli-ps-findimage#check-the-purchase-plan-information). diff --git a/docs/azure-ubuntu-1804.md b/docs/azure-ubuntu-1804.md index 17d75cf..9726060 100644 --- a/docs/azure-ubuntu-1804.md +++ b/docs/azure-ubuntu-1804.md @@ -10,7 +10,7 @@ set up an Azure cluster, then the following steps can get you on your way. # https://docs.microsoft.com/en-us/cli/azure/install-azure-cli-apt?view=azure-cli-latest # Install Ansible Azure for Python 3. The main reason these tips were written -# was to save you time on the following steps. Muchos is tested with Python 3.9 and above. +# was to save you time on the following steps. Muchos is tested with Python 3.11 and above. # If python-pip and pip were installed and used, those would go against Python 2 # and would not work. sudo apt install python3-pip diff --git a/lib/muchos/azure.py b/lib/muchos/azure.py index ef3e2b1..b6ebb4d 100644 --- a/lib/muchos/azure.py +++ b/lib/muchos/azure.py @@ -261,14 +261,16 @@ def add_specialized_configs(self, hosts_file): "{0}dir_ordinal{1}".format( dirtype, ordinal ), - 0 - if len( - curr_default_dirs - if dirtype == "default" - else curr_worker_dirs - ) - < ordinal + 1 - else ordinal, + ( + 0 + if len( + curr_default_dirs + if dirtype == "default" + else curr_worker_dirs + ) + < ordinal + 1 + else ordinal + ), ), file=vmss_file, ) diff --git a/lib/muchos/config/azurevalidations.py b/lib/muchos/config/azurevalidations.py index e248691..efe746a 100644 --- a/lib/muchos/config/azurevalidations.py +++ b/lib/muchos/config/azurevalidations.py @@ -115,9 +115,11 @@ def validate_azure_configs(config, action): ) or all( [ - vmss.get("sku") in config.spot_capable_skus() - if vmss.get("vmss_priority") == "Low" - else True + ( + vmss.get("sku") in config.spot_capable_skus() + if vmss.get("vmss_priority") == "Low" + else True + ) for vmss in config.azure_multiple_vmss_vars.get( "vars_list", [] ) @@ -161,9 +163,11 @@ def validate_azure_configs(config, action): lambda config, client: not config.use_multiple_vmss() or all( [ - vmss.get("sku") in config.premiumio_capable_skus() - if vmss.get("data_disk_sku") == "Premium_LRS" - else True + ( + vmss.get("sku") in config.premiumio_capable_skus() + if vmss.get("data_disk_sku") == "Premium_LRS" + else True + ) for vmss in config.azure_multiple_vmss_vars.get( "vars_list", [] ) diff --git a/lib/requirements.txt b/lib/requirements.txt index 6e308ba..1fbddac 100644 --- a/lib/requirements.txt +++ b/lib/requirements.txt @@ -1,4 +1,4 @@ -flake8==6.1.0 -ansible-core==2.15.9 -ansible-lint==6.21.1 -nose2==0.14.0 +flake8==7.1.1 +ansible-core==2.18.0 +ansible-lint==24.9.2 +nose2==0.15.1 diff --git a/lib/tests/azure/test_config.py b/lib/tests/azure/test_config.py index 164a311..1e7faae 100644 --- a/lib/tests/azure/test_config.py +++ b/lib/tests/azure/test_config.py @@ -39,8 +39,8 @@ def test_azure_cluster(): ) assert c.checksum("accumulo") == ( "sha512:" - "27778c1c3f1d88ab128649fd0671d3be97ba052216ab43f1169395960e8c7d16" - "375a51f940c2262437b836ea31f83f73f08f7a3d8cadda443e5e8bb31d9b23c5" + "1a27a144dc31f55ccc8e081b6c1bc6cc0362a8391838c53c166cb45291ff8f35" + "867fd8e4729aa7b2c540f8b721f8c6953281bf589fc7fe320e4dc4d20b87abc4" ) assert c.get("azure", "vm_sku") == "Standard_D8s_v3" assert c.get("azure", "data_disk_sku") == "Standard_LRS" diff --git a/lib/tests/ec2/test_config.py b/lib/tests/ec2/test_config.py index 02f8c72..6e17793 100644 --- a/lib/tests/ec2/test_config.py +++ b/lib/tests/ec2/test_config.py @@ -33,8 +33,8 @@ def test_ec2_cluster(): ) assert c.checksum("accumulo") == ( "sha512:" - "27778c1c3f1d88ab128649fd0671d3be97ba052216ab43f1169395960e8c7d16" - "375a51f940c2262437b836ea31f83f73f08f7a3d8cadda443e5e8bb31d9b23c5" + "1a27a144dc31f55ccc8e081b6c1bc6cc0362a8391838c53c166cb45291ff8f35" + "867fd8e4729aa7b2c540f8b721f8c6953281bf589fc7fe320e4dc4d20b87abc4" ) assert c.get("ec2", "default_instance_type") == "m5d.large" assert c.get("ec2", "worker_instance_type") == "m5d.large" diff --git a/scripts/install-ansible-for-azure b/scripts/install-ansible-for-azure index 799b5ad..f05dd79 100755 --- a/scripts/install-ansible-for-azure +++ b/scripts/install-ansible-for-azure @@ -1,4 +1,4 @@ #!/usr/bin/env bash ansible-galaxy collection install community.general==7.5.0 -ansible-galaxy collection install azure.azcollection==1.18.1 -pip install -r ~/.ansible/collections/ansible_collections/azure/azcollection/requirements-azure.txt +ansible-galaxy collection install azure.azcollection==2.7.0 +pip install -r ~/.ansible/collections/ansible_collections/azure/azcollection/requirements.txt