From 2ad9054098c2b4c9201954cb22ddbc9a444a7bd4 Mon Sep 17 00:00:00 2001 From: Nacho Silla Date: Mon, 16 Oct 2023 11:39:38 +0200 Subject: [PATCH] Adding sushy tools to the ocp-on-libvirt solution Currently ocp-on-libvirt installs the conserver service along with the VMs to allow for IPMI remote management of the hosts, which in turn allows for PXE image boot up. In order to support ZTP deployments, virtual media image boot up is needed, and to manage it remotely, the target server BMCs must suport the redfish protocol. Sushy tools provides support for the redfish protocol for libvirt VMs. This change installs sushy tools, as implemented in the assisted-installer ansible project, when running the ocp-on-libvirt role. Depends-On: https://github.com/dci-labs/bos2-ci-config/pull/72 Depends-On: https://github.com/openshift-kni/baremetal-deploy/pull/975 Depends-On: https://github.com/redhat-partner-solutions/crucible/pull/272 Test-Args-Hints: -e enable_redfish=true -e enable_virtualmedia=true --- roles/ocp_on_libvirt/defaults/main.yml | 9 ++++ roles/ocp_on_libvirt/tasks/main.yml | 3 ++ roles/ocp_on_libvirt/tasks/redfish_setup.yml | 54 +++++++++++++++++++ roles/ocp_on_libvirt/templates/hosts.j2 | 17 ++++-- .../templates/libvirt_node.xml.j2 | 13 +++-- 5 files changed, 90 insertions(+), 6 deletions(-) create mode 100644 roles/ocp_on_libvirt/tasks/redfish_setup.yml diff --git a/roles/ocp_on_libvirt/defaults/main.yml b/roles/ocp_on_libvirt/defaults/main.yml index d49a3f315..c440c43f1 100644 --- a/roles/ocp_on_libvirt/defaults/main.yml +++ b/roles/ocp_on_libvirt/defaults/main.yml @@ -1,9 +1,18 @@ --- libvirt_image_path: /var/lib/libvirt/images +bootmode: uefi enable_conserver: false enable_legacy_vga_mode: false do_dns_config: true apps_ip_address: 192.168.123.10 api_ip_address: 192.168.123.5 dns_vip_address: 192.168.123.6 + +# REDFISH +enable_redfish: false +enable_virtualmedia: false +redfish_cache_dir: /var/lib/dci-openshift-agent +redfish_assisted_deploy_repo: https://github.com/redhat-partner-solutions/crucible.git +redfish_assisted_deploy_version: 92864fc8be3f83a80666d3c1e98d1740f17dfc30 + ... diff --git a/roles/ocp_on_libvirt/tasks/main.yml b/roles/ocp_on_libvirt/tasks/main.yml index 5e44e616a..fa5125753 100644 --- a/roles/ocp_on_libvirt/tasks/main.yml +++ b/roles/ocp_on_libvirt/tasks/main.yml @@ -30,6 +30,9 @@ vars: vbmc_host: "{{ vbmc_host_provided }}" vbmc_nodes: "{{ resources }}" + - name: Redfish Setup + ansible.builtin.include_tasks: redfish_setup.yml + when: enable_redfish | bool - name: DCI Setup ansible.builtin.include_tasks: dci_setup.yml - name: Setup conserver diff --git a/roles/ocp_on_libvirt/tasks/redfish_setup.yml b/roles/ocp_on_libvirt/tasks/redfish_setup.yml new file mode 100644 index 000000000..e083638ce --- /dev/null +++ b/roles/ocp_on_libvirt/tasks/redfish_setup.yml @@ -0,0 +1,54 @@ +- name: "Get Ansible roles path" + shell: "set -o pipefail && ansible-config dump|grep DEFAULT_ROLES_PATH|sed -e 's/.*=\\s*//'" + register: roles_path_cmd + delegate_to: localhost + +- name: "Set roles_path" + set_fact: + roles_path: "{{ roles_path_cmd.stdout }}" + +- name: "Clone/update assisted-deploy repo" + vars: + git_repo: "{{ redfish_assisted_deploy_repo }}" + git_ref: "{{ redfish_assisted_deploy_version }}" + git: + version: "{{ git_ref }}" + repo: "{{ git_repo }}" + dest: "{{ redfish_cache_dir }}/assisted_deploy_repo" + #force: true + # On RHEL8 git clone can sporadically fail with OpenSSL SSL_read: + # SSL_ERROR_SYSCALL, errno 104. + delegate_to: localhost + register: git_clone + retries: 3 + delay: 10 + until: not git_clone.failed + when: + - "redfish_cache_dir + '/assisted_deploy_repo/roles' in roles_path" + tags: + - clone_upstream_repos + +- name: Install sushy-tools + vars: + repo_root_path: "{{ playbook_dir | dirname }}" + cert_country: US + cert_state: MA + cert_locality: Westford + cert_organization: DCI + cert_organizational_unit: Lab + sushy_ignore_boot_device: false + include_role: + name: setup_sushy_tools + +- name: Get KVM hosts UUID + shell: > + virsh list --all --name --uuid | + sed -e 's/^\([^ ]*\) \([^ ]*\)$/"\2": "\1",/g' | + tr -d '\n' | + sed -e 's/^\(.*\),$/{\1}/g' + register: all_vms + become: yes + +- name: Store KVM hosts UUID + set_fact: + redfish_kvm_uuid: "{{ all_vms.stdout | from_json }}" \ No newline at end of file diff --git a/roles/ocp_on_libvirt/templates/hosts.j2 b/roles/ocp_on_libvirt/templates/hosts.j2 index f34d6f095..93fa4341b 100644 --- a/roles/ocp_on_libvirt/templates/hosts.j2 +++ b/roles/ocp_on_libvirt/templates/hosts.j2 @@ -13,6 +13,10 @@ dir="{{ '{{' }} ansible_user_dir {{ '}}' }}/clusterconfigs" #webserver_url="http://{{ ansible_fqdn }}:8080" +{% if externalMACAddress is defined %} +externalMACAddress={{ externalMACAddress }} +{% endif %} + {% if enable_lso | default(false) | bool %} ocs_install_type=internal local_storage_devices=["/dev/sdb"] @@ -28,7 +32,7 @@ labels={"cluster.ocs.openshift.io/openshift-storage": ""} [masters] {% for key, value in ironic_nodes.items() -%} {% if 'master' in key -%} -{{ key }} {{ "ansible_host=" + key + "." + ansible_fqdn if not do_dns_config|bool else "" }} name={{ key }} role=master ipmi_user={{ value.ipmi_user }} ipmi_password={{ value.ipmi_pass }} ipmi_address={{ value.ipmi_address }} ipmi_port={{ value.ipmi_port }} provision_mac={{ value.mac_address }} hardware_profile=default socket_console={{ enable_conserver }}{% if value.root_device_hint is defined %} root_device_hint={{ value.root_device_hint }}{% endif %}{% if value.root_device_hint_value is defined %} root_device_hint_value={{ value.root_device_hint_value }}{% endif %} +{{ key }} {{ "ansible_host=" + key + "." + ansible_fqdn if not do_dns_config|bool else "" }} name={{ key }} role=master ipmi_user={{ value.ipmi_user }} ipmi_password={{ value.ipmi_pass }} ipmi_address={{ value.ipmi_address }} ipmi_port={{ value.ipmi_port }} provision_mac={{ value.mac_address }} hardware_profile=default socket_console={{ enable_conserver }}{% if value.root_device_hint is defined %} root_device_hint={{ value.root_device_hint }}{% endif %}{% if value.root_device_hint_value is defined %} root_device_hint_value={{ value.root_device_hint_value }}{% endif %}{% if key in redfish_kvm_uuid | default({}) %} kvm_uuid={{ redfish_kvm_uuid[key] }} redfish_port={{ redfish_port }}{% endif %} {% endif %} {%- endfor %} @@ -40,7 +44,7 @@ ansible_ssh_extra_args='-o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/n [workers] {% for key, value in ironic_nodes.items() -%} {% if 'worker' in key -%} -{{ key }} {{ "ansible_host=" + key + "." + ansible_fqdn if not do_dns_config|bool else "" }} name={{ key }} role=worker ipmi_user={{ value.ipmi_user }} ipmi_password={{ value.ipmi_pass }} ipmi_address={{ value.ipmi_address }} ipmi_port={{ value.ipmi_port }} provision_mac={{ value.mac_address }} hardware_profile=unknown socket_console={{ enable_conserver }}{% if value.root_device_hint is defined %} root_device_hint={{ value.root_device_hint }}{% endif %}{% if value.root_device_hint_value is defined %} root_device_hint_value={{ value.root_device_hint_value }}{% endif %} +{{ key }} {{ "ansible_host=" + key + "." + ansible_fqdn if not do_dns_config|bool else "" }} name={{ key }} role=worker ipmi_user={{ value.ipmi_user }} ipmi_password={{ value.ipmi_pass }} ipmi_address={{ value.ipmi_address }} ipmi_port={{ value.ipmi_port }} provision_mac={{ value.mac_address }} hardware_profile=unknown socket_console={{ enable_conserver }}{% if value.root_device_hint is defined %} root_device_hint={{ value.root_device_hint }}{% endif %}{% if value.root_device_hint_value is defined %} root_device_hint_value={{ value.root_device_hint_value }}{% endif %}{% if key in redfish_kvm_uuid | default({}) %} kvm_uuid={{ redfish_kvm_uuid[key] }} redfish_port={{ redfish_port }}{% endif %} {% endif %} {%- endfor %} @@ -52,7 +56,7 @@ ansible_ssh_extra_args='-o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/n [provisioner] {% for key, value in ironic_nodes.items() -%} {% if 'provision' in key -%} -{{ key }}{{ "." + ansible_fqdn if not do_dns_config|bool else "" }} {{ "ansible_host=" + key + "." + ansible_fqdn if not do_dns_config|bool else "" }} name={{ key }} ansible_user={{ provisionhost_user }} prov_nic=eth0 pub_nic=eth1 ansible_ssh_common_args="-o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null" +{{ key }}{{ "." + ansible_fqdn if not do_dns_config|bool else "" }} {{ "ansible_host=" + key + "." + ansible_fqdn if not do_dns_config|bool else "" }} name={{ key }} ansible_user={{ provisionhost_user }} prov_nic=eth0 pub_nic=eth1 ansible_ssh_common_args="-o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null"{{ enable_virtualmedia | ternary(" bootstrapProvisioningIP=" + bootstrapProvisioningIP, "") }} {% endif %} {%- endfor %} @@ -73,3 +77,10 @@ ansible_ssh_extra_args='-o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/n # The following mirror entries are the default ones. If you want to add more mirror # you can uncomment this parameter and add it here. #registry_source_mirrors=["quay.io/openshift-release-dev/ocp-v4.0-art-dev", "registry.svc.ci.openshift.org/ocp/release", "quay.io/openshift-release-dev/ocp-release"] + +{% if enable_redfish | bool -%} +[kvm_hosts_redfish] +{% for key, value in ironic_nodes.items() -%} +{{ key }} +{% endfor %} +{%- endif %} \ No newline at end of file diff --git a/roles/ocp_on_libvirt/templates/libvirt_node.xml.j2 b/roles/ocp_on_libvirt/templates/libvirt_node.xml.j2 index aa9c163da..dd8faf2ab 100644 --- a/roles/ocp_on_libvirt/templates/libvirt_node.xml.j2 +++ b/roles/ocp_on_libvirt/templates/libvirt_node.xml.j2 @@ -3,8 +3,12 @@ {{ host['memory'] * 1024 | int }} {{ host['vcpus'] }} - hvm + hvm +{% if bootmode == "uefi" %} + /usr/share/edk2/ovmf/OVMF_CODE.secboot.fd + /var/lib/libvirt/qemu/nvram/{{ host['name'] }}_VARS.fd +{% endif %} {% if enable_conserver %} {% endif %} @@ -13,6 +17,9 @@ +{% if bootmode == "uefi" %} + +{% endif %} @@ -31,7 +38,7 @@ - + @@ -79,7 +86,7 @@ {% endfor %} - + {% if enable_conserver %}