Fix for missing schema registry URL in rest, c3 and ksql (see issue 1425)

.bumpversion-ansible.cfg

@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 7.4.0
+current_version = 7.5.3
 commit = true
 message = Bump cp-ansible Version: {current_version} → {new_version}
 tag = false

.bumpversion.cfg

@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 7.4.0
+current_version = 7.5.3
 commit = true
 message = Bump CP Version: {current_version} → {new_version}
 tag = false

.github/CODEOWNERS

@@ -1,2 +1,2 @@
 # See go/codeowners - automatically generated for confluentinc/cp-ansible:
-*	@confluentinc/cp-ansible-dev
+*	@confluentinc/ansible-team

CHANGELOG.rst

@@ -4,6 +4,60 @@ Ansible Playbooks for Confluent Platform - Release Notes
 
 .. contents:: Topics
 
+v7.5.3
+======
+
+Notable fixes
+-------------
+
+- Critical security and vulnerability issues were fixed.
+
+v7.5.2
+======
+
+Notable fixes
+-------------
+
+- Critical security and vulnerability issues were fixed.
+
+
+v7.5.1
+======
+
+Notable enhancements
+-------------
+
+- Updated default confluent cli version to 3.30.1
+- Fixed bugs in discovery to generate inventory file with appropriate security protocols
+- Ansible builtin File mode is now string instead of octal
+
+v7.5.0
+======
+
+New features
+-------------
+
+- Configure single sign-on (SSO) authentication for Control Center using OpenID Connect (OIDC)
+- Enable FIPS on Confluent Platform(CP) for RHEL8 & RHEL9 using cp-ansible.
+
+Notable enhancements
+-------------
+
+- Added capability to discover multiple workers in the connect cluster in discovery
+- Added support to specify ansible_password in discovery
+
+v7.4.1
+======
+
+Notable enhancements
+-------------
+
+- Parametrize the number of retries for MDS API requests
+- Add Broker's principals to Controller's super user list on a Kraft cluster with RBAC
+- Removed timeout configs from client properties of Kafka Broker, allowing customers to use custom timeout values
+- Archived installation of Confluent Platform on Debian 9 since the OS version reached end-of-life
+
+
 v7.4.0
 ======
 

Jenkinsfile

@@ -29,8 +29,8 @@ def confluent_release_quality = choice(name: 'CONFLUENT_RELEASE_QUALITY',
 
 // Parameter for the molecule test scenario to run
 def molecule_scenario_name = choice(name: 'SCENARIO_NAME',
-    choices: ['rbac-scram-custom-rhel', 'plaintext-rhel'],
-    defaultValue: 'rbac-scram-custom-rhel',
+    choices: ['rbac-mtls-rhel8', 'plaintext-rhel'],
+    defaultValue: 'rbac-mtls-rhel8',
     description: 'The Ansible Molecule scenario name to run',
 )
 

discovery/README.md

@@ -12,15 +12,15 @@ Though this script tries to come up with inventory which is the closed represent
 
 These are dependencies for this script and should be installed on the machine where we are executing it from. This is not a requirement for managed nodes of the cluster.
 #### Hosts
-The discovery script needs list of hosts which is part of the existing cluster on which services has to be discovered. Apart from the list of hosts, the script also need the Confluent Service names. If these service names has been updated, the same should be provided under `service_override` section.
+The discovery script needs list of hosts which is part of the existing cluster on which services has to be discovered. Apart from the list of hosts, the script also need the Confluent Service names. If these service names has been updated, the same should be provided under `service_overrides` section. The name should be passed including ".service" 
 
 ```yaml
 all:
   vars:
     ansible_connection: docker
     ansible_user: null
-    service_override:
-      zookeeper_service_name: myservice.zookeeper
+    service_overrides:
+      zookeeper_service_name: myservice-zookeeper.service
 ```
 
 ### How
@@ -95,8 +95,8 @@ For a cluster running on local docker environment
 
 vars:
   ansible_connection: docker
-  service_override:
-    zookeeper_service_name: 'custom-service-name'
+  service_overrides:
+    zookeeper_service_name: 'custom-service-name.service'
 hosts:
   all:
     - zookeeper1
@@ -112,16 +112,25 @@ hosts:
 ```
 
 #### Command Line options
-##### verbose
+##### verbosity
 To get the verbose output from script and Ansible you can set the verbosity level between 0 to 4. Where 4 means more verbose.
+```shell
+python discovery/main.py --input discovery/hosts.yml --verbosity 4
+```
 ##### limit
 Use limit flag to limit the discovery for specified list of hosts
+```shell
+python discovery/main.py --input discovery/hosts.yml --limit host1,host2
+```
 ##### output_file
 Use this flag to specify output inventory file name. Default value is inventory.yml
-
 ```shell
-python discovery/main.py --input discovery/hosts.yml --verbose 4 --limit host1,host2
+python discovery/main.py --input discovery/hosts.yml --output inventory.yml
 ```
+#### multi_threaded
+To run the discovery scripts in multi threaded environment. Default is True
+
+
 ### FQA
 * **Can I use it for older CP versions**  
 Ideally we should be using the discovery from the branch which maps to the CP cluster. However, to onboard existing cluster, one can use the latest disvoery code and use **--from_version** parameter to specify the CP cluster version

discovery/main.py

@@ -7,8 +7,9 @@
 logger = Logger.get_logger()
 
 
-def perform_pre_checks(input_context, inventory):
-    SystemValidator.validate_connection(input_context)
+def perform_pre_checks(input_context: InputContext, inventory: CPInventoryManager):
+    if not input_context.skip_validation:
+        SystemValidator.validate_connection(input_context)
 
 
 def build_system_properties(input_context: InputContext, inventory: CPInventoryManager):

discovery/manager/manager.py

@@ -32,13 +32,16 @@ def my_event_handler(self, event):
     def get_ansible_vars_from_input_context(input_context: InputContext) -> dict:
         vars = dict()
         vars['ansible_user'] = input_context.ansible_user
+        vars['ansible_password'] = input_context.ansible_password
         vars['ansible_become'] = input_context.ansible_become
         vars['ansible_connection'] = input_context.ansible_connection
         vars['ansible_become_user'] = input_context.ansible_become_user
         vars['ansible_become_method'] = input_context.ansible_become_method
         vars['ansible_ssh_extra_args'] = input_context.ansible_ssh_extra_args
         vars['ansible_python_interpreter'] = input_context.ansible_python_interpreter
         vars['ansible_ssh_private_key_file'] = input_context.ansible_ssh_private_key_file
+        vars['ansible_common_remote_group'] = input_context.ansible_common_remote_group
+        vars['ansible_become_password'] = input_context.ansible_become_password
         return vars
 
     @staticmethod
@@ -98,9 +101,7 @@ def get_service_facts(input_context: InputContext) -> dict:
     def get_service_host_mapping(input_context: InputContext, **kwargs) -> dict:
         logger.info(f"Creating service and host mapping...")
         logger.info("Configured services:")
-        logger.info(yaml.dump(ConfluentServices(input_context).get_all_service_names(),
-                              indent=2,
-                              default_flow_style=False))
+        logger.info(ConfluentServices(input_context).get_all_service_names())
 
         hosts = input_context.ansible_hosts
         if _host_group_declared_in_inventory(hosts, input_context):
@@ -238,11 +239,11 @@ def __get_service_properties_file(input_context: InputContext, service: ServiceD
 
         host = hosts[0]
         service_details = SystemPropertyManager.get_service_details(input_context, service, [host])
-        execution_command = service_details.get(host).get("status").get("ExecStart")
+        execution_command = str(service_details.get(host).get("status").get("ExecStart"))
 
         # check if we have flag based configs
         property_files = dict()
-        matches = re.findall('(--[\w\.]+\.config)*\s+([\w\/-]+\.properties)', execution_command)
+        matches = re.findall(u'(--[\w\.]+\.config)*\s+(\S+\.properties)', execution_command)
         for match in matches:
             key, path = match
             key = key.strip('--') if key else DEFAULT_KEY
@@ -252,6 +253,37 @@ def __get_service_properties_file(input_context: InputContext, service: ServiceD
             logger.error(f"Cannot find associated properties file for service {service.value.get('name')}")
         return property_files
 
+    @staticmethod
+    def slurp_remote_file(input_context: InputContext, hosts: list, file: str) -> dict:
+        """
+        Returns a map of hosts and content of given file on that host
+        :param input_context:
+        :param hosts:
+        :param file:
+        :return:
+        """
+
+        content = dict()
+        if not file:
+            return content
+
+        runner_utils = AnsibleRunnerUtils()
+        hosts, host_pattern = AnsibleRunnerUtils.get_host_and_pattern_from_host_list(hosts)
+        ansible_runner.run(
+            quiet=input_context.verbosity <= 3,
+            host_pattern=host_pattern,
+            inventory=AnsibleRunnerUtils.get_inventory_dict(input_context, hosts),
+            module="slurp",
+            module_args=f"src={file}",
+            event_handler=runner_utils.my_event_handler
+        )
+
+        response = runner_utils.result_ok
+        for host in hosts:
+            content[host] = base64.b64decode(response[host]['content']).decode('utf-8')
+
+        return content
+
     @staticmethod
     def get_property_mappings(input_context: InputContext, service: ServiceData, hosts: list) -> dict:
 
@@ -262,23 +294,10 @@ def get_property_mappings(input_context: InputContext, service: ServiceData, hos
             return mappings
 
         for key, file in seed_properties_file.items():
-
-            runner_utils = AnsibleRunnerUtils()
-            hosts, host_pattern = AnsibleRunnerUtils.get_host_and_pattern_from_host_list(hosts)
-            ansible_runner.run(
-                quiet=input_context.verbosity <= 3,
-                host_pattern=host_pattern,
-                inventory=AnsibleRunnerUtils.get_inventory_dict(input_context, hosts),
-                module="slurp",
-                module_args=f"src={file}",
-                event_handler=runner_utils.my_event_handler
-            )
-
-            response = runner_utils.result_ok
+            content = ServicePropertyManager.slurp_remote_file(input_context=input_context, hosts=hosts, file=file)
             for host in hosts:
-                properties = base64.b64decode(response[host]['content']).decode('utf-8')
                 host_properties = mappings.get(host, dict())
-                host_properties.update({key: load_properties_to_dict(properties)})
+                host_properties.update({key: load_properties_to_dict(content.get(host))})
                 mappings[host] = host_properties
 
         return mappings
@@ -314,7 +333,10 @@ def get_keystore_alias_names(input_context: InputContext, hosts: list, keystorep
     @staticmethod
     def get_jaas_file_path(input_context: InputContext, service: ServiceData, hosts: list):
         # check if overriden as env var
-        env_details = ServicePropertyManager.get_env_details(input_context, service, hosts)
+        from discovery.service import AbstractPropertyBuilder
+        env_details = AbstractPropertyBuilder.get_service_environment_variable(input_context=input_context,
+                                                                               service=service,
+                                                                               hosts=hosts)
         kafka_opts = env_details.get('KAFKA_OPTS', None)
         if kafka_opts is not None:
             if "-Djava.security.auth.login.config=" in kafka_opts:
@@ -325,7 +347,9 @@ def get_jaas_file_path(input_context: InputContext, service: ServiceData, hosts:
     @staticmethod
     def get_log_file_path(input_context: InputContext, service: ServiceData, hosts: list, log4j_opts_env_var):
         # check if overriden as env var
-        env_details = ServicePropertyManager.get_env_details(input_context, service, hosts)
+        from discovery.service import AbstractPropertyBuilder
+        env_details = AbstractPropertyBuilder.get_service_environment_variable(input_context=input_context,
+                                                                               service=service, hosts=hosts)
         log4j_opts = env_details.get(log4j_opts_env_var, None)
         if log4j_opts is not None:
             if "-Dlog4j.configuration=file:" in log4j_opts:
@@ -354,36 +378,6 @@ def get_log_file_path(input_context: InputContext, service: ServiceData, hosts:
 
         return None
 
-    @staticmethod
-    def parse_environment_details(env_command: str) -> dict:
-        env_details = dict()
-        if not env_command:
-            return env_details
-
-        tokens = ['KAFKA_HEAP_OPTS', 'KAFKA_OPTS', 'KAFKA_LOG4J_OPTS', 'LOG_DIR', 'CONFLUENT_SECURITY_MASTER_KEY']
-        for token in tokens:
-            pattern = f"{token}=(.*?)( [A-Z]{{3}}|$)"
-            match = re.search(pattern, env_command)
-            if match:
-                env_details[token] = match.group(1).rstrip()
-        return env_details
-
-    @staticmethod
-    def _get_env_from_service(input_context: InputContext, service: ServiceData, hosts: list) -> str:
-        service_facts = SystemPropertyManager.get_service_details(input_context, service, hosts)
-        service_facts = service_facts.get(hosts[0])
-        environment = service_facts.get("status", dict()).get("Environment", None)
-        if not environment:
-            logger.warning(f"Could not find any environment variable for service {service.name}")
-        return environment
-
-    @staticmethod
-    def get_env_details(input_context: InputContext, service: ServiceData, hosts: list) -> dict:
-        env_cmd = ServicePropertyManager._get_env_from_service(input_context=input_context,
-                                                               service=service,
-                                                               hosts=hosts)
-        return ServicePropertyManager.parse_environment_details(env_cmd)
-
     @staticmethod
     def get_kerberos_configurations(input_context: InputContext, hosts: list, kerberos_config_file):
         realm, kdc, admin = "", "", ""
@@ -581,22 +575,26 @@ def validate_connection(input_context: InputContext) -> None:
         hosts, host_pattern = AnsibleRunnerUtils.get_host_and_pattern_from_input_context(input_context)
         if not hosts or len(hosts) == 0:
             terminate_script("Empty host list. Please refer to documentation for correct host format")
+
+        inventory = AnsibleRunnerUtils.get_inventory_dict(input_context)
         ansible_runner.run(
             quiet=input_context.verbosity <= 3,
             host_pattern=host_pattern,
-            inventory=AnsibleRunnerUtils.get_inventory_dict(input_context),
+            inventory=inventory,
             module='ansible.builtin.ping',
             event_handler=runner_utils.my_event_handler
         )
 
         if runner_utils.result_ok:
             alive_hosts = runner_utils.result_ok.keys()
             failed_hosts.extend(hosts - alive_hosts)
-            logger.info(f"Connection was successful to:\n{yaml.dump(list(alive_hosts), indent=2, default_flow_style=False)}")
+            logger.info(
+                f"Connection was successful to:\n{yaml.dump(list(alive_hosts), indent=2, default_flow_style=False)}")
         else:
             failed_hosts.extend(hosts)
 
         if failed_hosts:
-            message = f"Could not connect to hosts:\n{yaml.dump(list(failed_hosts), indent=2, default_flow_style=False)}.\n" \
-                      f"Please verify the hostnames, ssh user and key"
+            message = f"Could not connect to hosts:\n{yaml.dump(list(failed_hosts), indent=2, default_flow_style=False)}\n" \
+                      f"Please verify the following details:\n" \
+                      f"{yaml.dump(inventory['all']['vars'], indent=2, default_flow_style=False)}"
             terminate_script(message)