Update Ansible core, some dependencies, checksums

.github/workflows/ci.yaml

@@ -17,7 +17,7 @@
 # under the License.
 #
 
-# This GitHub Actions workflow will prepare a environment with Python 3.9
+# This GitHub Actions workflow will prepare a environment with Python 3.11
 # and the required Python packages, to run the CI tests
 # (Ansible-lint, Flake8 etc.) for Fluo-Muchos.
 # See also:
@@ -43,7 +43,7 @@ jobs:
     - uses: actions/checkout@v4
     - uses: actions/setup-python@v4
       with:
-        python-version: '3.9'
+        python-version: '3.11'
     - name: Install required packages
       run: pip install -r ./lib/requirements.txt
     - name: Install common Ansible collections

README.md

@@ -28,12 +28,12 @@ Check out [Uno] for setting up Accumulo or Fluo on a single machine.
 Muchos requires the following common components for installation and setup:
 
 * Python 3 with a virtual environment setup.
-Create a Python 3 environment and switch to it. (The CI tests using Python 3.9,
+Create a Python 3 environment and switch to it. (The CI tests using Python 3.11,
 but this should work in later versions as well. If you encounter problems,
 please file an issue).
 ```bash
 cd ~
-python3.9 -m venv env
+python3.11 -m venv env
 source env/bin/activate
 ```
 * `ssh-agent` installed and running and ssh-agent forwarding.  Note that this may

ansible/roles/azure/tasks/create_multiple_vmss.yml

@@ -51,6 +51,7 @@
     resource_group: "{{ resource_group }}"
     location: "{{ location }}"
     name: "{{ vmss_name }}-{{ item.name_suffix }}"
+    orchestration_mode: "Uniform"
     vm_size: "{{ item.sku }}"
     priority: "{{ item.vmss_priority | default('None') }}"
     admin_username: "{{ cluster_user }}"

ansible/roles/azure/tasks/create_vmss.yml

@@ -56,6 +56,7 @@
     resource_group: "{{ resource_group }}"
     location: "{{ location }}"
     name: "{{ vmss_name }}"
+    orchestration_mode: "Uniform"
     vm_size: "{{ vm_sku }}"
     priority: "{{ vmss_priority | default('None') }}"
     admin_username: "{{ cluster_user }}"

conf/checksums

@@ -1,4 +1,4 @@
-accumulo:2.1.2:27778c1c3f1d88ab128649fd0671d3be97ba052216ab43f1169395960e8c7d16375a51f940c2262437b836ea31f83f73f08f7a3d8cadda443e5e8bb31d9b23c5
+accumulo:2.1.3:1a27a144dc31f55ccc8e081b6c1bc6cc0362a8391838c53c166cb45291ff8f35867fd8e4729aa7b2c540f8b721f8c6953281bf589fc7fe320e4dc4d20b87abc4
 accumulo:2.0.1:b443839443a9f5098b55bc5c54be10c11fedbaea554ee6cd42eaa9311068c70bd611d7fc67698c91ec73da0e85b9907aa72b98d5eb4d49ea3a5d51b0c6c5785f
 accumulo:1.10.3:36e6795ad3720ba72fc9f4ddabf45f6d67cdc77658a181733fa2c47bfd3799f123b8840a79b538b8504a7b4bdc97fd0b52efab790395a5e022a897bc18405d0c
 fluo:1.2.0:037f89cd2bfdaf76a1368256c52de46d6b9a85c9c1bfc776ec4447d02c813fb2
@@ -12,11 +12,12 @@ hadoop:2.9.2:bd6060b629c2e7c5d411ae3ba2330b79cdb2f1caa87d5ea69392c0f9cb64a83464e
 hadoop:2.8.5:4174e7a67b614b7d5e47a1a2420cbe9a57978908f8ad0405f1d177306fb36ed87c895810f70e3c6a6cbadc76afb9303f1c49cbca67237e18c799d30f87afa57c
 hadoop:2.7.7:17c8917211dd4c25f78bf60130a390f9e273b0149737094e45f4ae5c917b1174b97eb90818c5df068e607835120126281bcc07514f38bd7fd3cb8e9d3db1bdde
 hadoop:2.6.5:001ad18d4b6d0fe542b15ddadba2d092bc97df1c4d2d797381c8d12887691898
+spark:3.4.4:405f604532c11c3793b757aae41bbae0f86f3bcab65f47dea07be9003532c382429feab67d8ed19721ee539801842b21d32685c37a97372f96d50b318f5b1bb0
 spark:3.2.0:710854DDB0617EFCB9783725F027617FFEB6CE17A161D069F4008700E0027CB87E5BF6D772A21746AEAEC0DA06703A8B257DEF72951C1073BFE0A3EBE497AF71
 spark:3.1.2:E7B580BC67356F1B36756B18A95E452307EB3825265616E399B1766124764D6E65AB4C666AD23C93E006D88B6C83A78A7D786BE981B4F20696697860A459879D
 spark:3.0.3:22F80D2D50B964FA6F87358330CCDAA4748FA1FC546272A1EFB21868DC59A815CE9BDCEE3B520B797E3740221212BAACF1A4BA5E7166D552A0990EFB9478DE32
 spark:2.4.8:84B52335834F1DC0540D5156F4CCD642A23C3CA130EBD5DC1BC8FBF1CAF0F89B783DA34DABD200A0B6385176BAD3BA4261C2C6639CAFC8CA6B8FFB8D7BB3EA72
-zookeeper:3.8.3:eb1a5e79fe55fa50e36021ca8b752e40584d5e24f23985c43e9f56abfbf23c0239dce37d18f27213c9a38fcb6f68daf1a9468fc352e6ac2364faaf9b56145bdf
+zookeeper:3.8.4:4d85d6f7644d5f36d9c4d65e78bd662ab35ebe1380d762c24c12b98af029027eee453437c9245dbdf2b9beb77cd6b690b69e26f91cf9d11b0a183a979c73fa43
 zookeeper:3.7.2:6afbfc1afc8b9370281bd9862f37dbb1cb95ec54bb2ed4371831aa5c0f08cfee775050bd57ce5fc0836e61af27eed9f0076f54b98997dd0e15159196056e52ea
 zookeeper:3.6.4:dec4835a1a23fa69599091d03b35f6c9f4e545d5fa1e149658cffdd7d1044ec31e14f41189aabbf59011dfce6609e6850209008e67cfd0f26270f1558196dcd8
 zookeeper:3.5.10:fbca82d93fb07534d2cd7d93a399463cfea7e772075285d103e7cda5599d14c2d9996c1e50cc66c7c0dc78f996b20e75b20d74df368eb222dbc3caa0ab2cc14b

conf/muchos.props.example

@@ -37,11 +37,11 @@ accumulo_instance = muchos
 accumulo_password = secret
 # Software versions (make sure you have a corresponding entry for the checksum in conf/checksums)
 hadoop_version = 3.3.6
-zookeeper_version = 3.8.3
-spark_version = 2.4.8
+zookeeper_version = 3.8.4
+spark_version = 3.4.4
 fluo_version = 1.2.0
 fluo_yarn_version = 1.0.0
-accumulo_version = 2.1.2
+accumulo_version = 2.1.3
 # Specifies if software should be downloaded. If 'False', tarballs of the software above should be in conf/upload/
 download_software = True
 # Install Hub (for GitHub)

docs/azure-image-reference.md

@@ -17,8 +17,8 @@ The sections below describe the various image related configurations in `muchos.
 
   The Rocky Linux images in Azure require plan information to be supplied. Currently, the Rocky Linux 8 image has been verified to work correctly. For using Rocky Linux 8 instead of Alma Linux, here is what you can use in `muchos.props`. The image plan information is mandatory for this image:
 
-    * `azure_image_reference = rockylinux|erockyenterprisesoftwarefoundationinc1653071250513|free|latest||`
-    * `azure_image_plan = free|rockylinux|erockyenterprisesoftwarefoundationinc1653071250513|`
+    * `azure_image_reference = rockylinux-x86_64|resf|8-base|latest||`
+    * `azure_image_plan = 8-base|rockylinux-x86_64|resf|`
 
   In addition, you might need to manually view and accept the terms of use for the Rocky Linux image, before being able to use it within Muchos. See the `azure_image_plan` section below for details.
 
@@ -39,11 +39,11 @@ The sections below describe the various image related configurations in `muchos.
 ## azure_image_plan
 `azure_image_plan` is only needed when working with images which require payment plan information to be supplied when a VM or VMSS is being created using that image. The format of this configuration is `plan_name|product|publisher|`. Plan information for the images published by a given publisher can easily be queried by using the Azure CLI. For example, to query the plan information for a Rocky Linux image in Azure:
 
-   `az vm image show --urn "erockyenterprisesoftwarefoundationinc1653071250513:rockylinux:free:latest" --query "plan"`
+   `az vm image show --urn "resf:rockylinux-x86_64:8-base:latest" --query "plan"`
 
 Then using that information, `azure_image_plan` can be configured as below in muchos.props:
 
-   `azure_image_plan = free|rockylinux|erockyenterprisesoftwarefoundationinc1653071250513|`
+   `azure_image_plan = 8-base|rockylinux-x86_64|resf|`
 
 More information about purchase plans, and accepting terms, etc. is available [here](https://learn.microsoft.com/en-us/azure/virtual-machines/linux/cli-ps-findimage#check-the-purchase-plan-information).
 

docs/azure-ubuntu-1804.md

@@ -10,7 +10,7 @@ set up an Azure cluster, then the following steps can get you on your way.
 # https://docs.microsoft.com/en-us/cli/azure/install-azure-cli-apt?view=azure-cli-latest
 
 # Install Ansible Azure for Python 3.  The main reason these tips were written
-# was to save you time on the following steps.  Muchos is tested with Python 3.9 and above.
+# was to save you time on the following steps.  Muchos is tested with Python 3.11 and above.
 # If python-pip and pip were installed and used, those would go against Python 2
 # and would not work.
 sudo apt install python3-pip

lib/muchos/azure.py

@@ -261,14 +261,16 @@ def add_specialized_configs(self, hosts_file):
                                     "{0}dir_ordinal{1}".format(
                                         dirtype, ordinal
                                     ),
-                                    0
-                                    if len(
-                                        curr_default_dirs
-                                        if dirtype == "default"
-                                        else curr_worker_dirs
-                                    )
-                                    < ordinal + 1
-                                    else ordinal,
+                                    (
+                                        0
+                                        if len(
+                                            curr_default_dirs
+                                            if dirtype == "default"
+                                            else curr_worker_dirs
+                                        )
+                                        < ordinal + 1
+                                        else ordinal
+                                    ),
                                 ),
                                 file=vmss_file,
                             )

lib/muchos/config/azurevalidations.py

@@ -115,9 +115,11 @@ def validate_azure_configs(config, action):
             )
             or all(
                 [
-                    vmss.get("sku") in config.spot_capable_skus()
-                    if vmss.get("vmss_priority") == "Low"
-                    else True
+                    (
+                        vmss.get("sku") in config.spot_capable_skus()
+                        if vmss.get("vmss_priority") == "Low"
+                        else True
+                    )
                     for vmss in config.azure_multiple_vmss_vars.get(
                         "vars_list", []
                     )
@@ -161,9 +163,11 @@ def validate_azure_configs(config, action):
             lambda config, client: not config.use_multiple_vmss()
             or all(
                 [
-                    vmss.get("sku") in config.premiumio_capable_skus()
-                    if vmss.get("data_disk_sku") == "Premium_LRS"
-                    else True
+                    (
+                        vmss.get("sku") in config.premiumio_capable_skus()
+                        if vmss.get("data_disk_sku") == "Premium_LRS"
+                        else True
+                    )
                     for vmss in config.azure_multiple_vmss_vars.get(
                         "vars_list", []
                     )

lib/requirements.txt

@@ -1,4 +1,4 @@
-flake8==6.1.0
-ansible-core==2.15.9
-ansible-lint==6.21.1
-nose2==0.14.0
+flake8==7.1.1
+ansible-core==2.18.0
+ansible-lint==24.9.2
+nose2==0.15.1

lib/tests/azure/test_config.py

@@ -39,8 +39,8 @@ def test_azure_cluster():
     )
     assert c.checksum("accumulo") == (
         "sha512:"
-        "27778c1c3f1d88ab128649fd0671d3be97ba052216ab43f1169395960e8c7d16"
-        "375a51f940c2262437b836ea31f83f73f08f7a3d8cadda443e5e8bb31d9b23c5"
+        "1a27a144dc31f55ccc8e081b6c1bc6cc0362a8391838c53c166cb45291ff8f35"
+        "867fd8e4729aa7b2c540f8b721f8c6953281bf589fc7fe320e4dc4d20b87abc4"
     )
     assert c.get("azure", "vm_sku") == "Standard_D8s_v3"
     assert c.get("azure", "data_disk_sku") == "Standard_LRS"

lib/tests/ec2/test_config.py

@@ -33,8 +33,8 @@ def test_ec2_cluster():
     )
     assert c.checksum("accumulo") == (
         "sha512:"
-        "27778c1c3f1d88ab128649fd0671d3be97ba052216ab43f1169395960e8c7d16"
-        "375a51f940c2262437b836ea31f83f73f08f7a3d8cadda443e5e8bb31d9b23c5"
+        "1a27a144dc31f55ccc8e081b6c1bc6cc0362a8391838c53c166cb45291ff8f35"
+        "867fd8e4729aa7b2c540f8b721f8c6953281bf589fc7fe320e4dc4d20b87abc4"
     )
     assert c.get("ec2", "default_instance_type") == "m5d.large"
     assert c.get("ec2", "worker_instance_type") == "m5d.large"

scripts/install-ansible-for-azure

@@ -1,4 +1,4 @@
 #!/usr/bin/env bash
 ansible-galaxy collection install community.general==7.5.0
-ansible-galaxy collection install azure.azcollection==1.18.1
-pip install -r ~/.ansible/collections/ansible_collections/azure/azcollection/requirements-azure.txt
+ansible-galaxy collection install azure.azcollection==2.7.0
+pip install -r ~/.ansible/collections/ansible_collections/azure/azcollection/requirements.txt