From 9c916a93d371af35f5099690be4b3d7835fe911c Mon Sep 17 00:00:00 2001 From: Calin Gherghe Date: Wed, 27 Jul 2022 13:55:44 -0700 Subject: [PATCH] Reference Architecture 22.06 New Components: - FlexRAN software (v.22.03) Timer Mode on bare-metal with Ubuntu 22.04 real-time kernel - Istio mTLS key protection using SGX New Platform: - QCT (Quanta Computer) Servers with 4th Gen Intel Xeon Scalable processor MCC (40 cores) CPU (dual and single) Updates/Changes: - Intel Ethernet Operator enabled by default in all supported profiles Removed Support: - Ubuntu 21.10 as base operating system Co-authored-by: Ali Shah, Syed Faraz Co-authored-by: Gherghe, Calin Co-authored-by: Kubin, Lukas Co-authored-by: Liu, Mark Co-authored-by: Mlynek, Krystian Co-authored-by: Park, Seungweon Co-authored-by: Prokes, Jiri Co-authored-by: Puzikov, Dmitrii --- docs/flexran_guide.md | 5 + docs/generate_profiles.md | 2 +- docs/vm_config_guide.md | 2 +- generate/playbook_templates/infra_playbook.j2 | 2 + generate/playbook_templates/intel_playbook.j2 | 23 +++- .../profiles_templates/common/group_vars.j2 | 28 ++++- .../profiles_templates/common/host_vars.j2 | 17 ++- generate/profiles_templates/k8s/profiles.yml | 21 +++- .../vm/vm_host_profiles.yml | 7 ++ .../profiles_templates/vm/vms_profiles.yml | 7 ++ playbooks/k8s/k8s.yml | 2 +- playbooks/preflight.yml | 68 ++++-------- .../tasks/main.yml | 6 + .../install_realtime_kernel/tasks/main.yml | 33 ++++++ .../defaults/main.yml} | 12 +- .../tasks/main.yml | 102 +++++++++++++++++ .../update_nic_drivers/tasks/i40e.yml | 3 +- .../update_nic_drivers/tasks/ice.yml | 3 +- roles/install_dependencies/tasks/main.yml | 3 + .../intel_ethernet_operator/defaults/main.yml | 4 - roles/intel_ethernet_operator/tasks/ddp.yml | 57 ++++++++-- .../tasks/ethernet_operator.yml | 29 ++++- .../tasks/flow_config.yml | 24 ++++ roles/intel_ethernet_operator/tasks/fw.yml | 46 ++++++-- roles/intel_ethernet_operator/tasks/main.yml | 16 --- .../tasks/operator_sdk.yml | 37 ------ .../tasks/preflight_ethernet_operator.yml | 81 ++++++++++++++ roles/intel_flexran/defaults/main.yml | 57 ++++++++++ .../intel_flexran/files/kernel_cmdline_gen.sh | 38 +++++++ roles/intel_flexran/files/spr-bm-fcp-2x40 | 1 + roles/intel_flexran/files/spr-bm-q-mcc-1x32 | 1 + roles/intel_flexran/files/spr-bm-q-xcc-2x52 | 1 + .../tasks/cleanup_flexran.yml} | 7 +- roles/intel_flexran/tasks/fec_acc.yml | 24 ++++ roles/intel_flexran/tasks/flexran.yml | 64 +++++++++++ .../intel_flexran/tasks/flexran_preflight.yml | 105 ++++++++++++++++++ roles/intel_flexran/tasks/main.yml | 49 ++++++++ roles/intel_flexran/tasks/oneapi.yml | 43 +++++++ roles/intel_flexran/tasks/pf_bb.yml | 103 +++++++++++++++++ roles/intel_flexran/tasks/timer_mode.yml | 90 +++++++++++++++ roles/intel_flexran/vars/main.yml | 35 ++++++ .../defaults/main.yml | 7 +- .../tasks/check_sriov_fec_operator.yml | 13 ++- roles/intel_sriov_fec_operator/tasks/main.yml | 11 +- roles/intel_sriov_fec_operator/tasks/opm.yml | 5 +- roles/minio_install/files/main.py | 7 +- .../tasks/preflight_minio_config.yml | 2 +- roles/operator_framework/defaults/main.yml | 18 +++ roles/operator_framework/tasks/main.yml | 57 ++++++++++ .../tasks/main.yml | 8 ++ .../tasks/main.yml | 32 ++++++ .../charts/istioctl/.helmignore | 23 ++++ .../charts/istioctl/Chart.yaml | 39 +++++++ .../charts/istioctl/templates/NOTES.txt | 7 ++ .../charts/istioctl/templates/_helpers.tpl | 51 +++++++++ .../templates/istioctl-deployment.yaml | 59 ++++++++++ .../istioctl/templates/istioctl-rbac.yaml | 28 +++++ .../charts/istioctl/values.yaml | 88 +++++++++++++++ roles/service_mesh_install/defaults/main.yml | 1 + .../files/profiles/intel-cryptomb.yaml | 58 ++++++++++ .../files/profiles/intel-qat-hw.yaml | 51 +++++++++ .../files/profiles/intel-qat-sw.yaml | 17 +++ .../files/profiles/sgx-mtls.yaml | 32 ++++++ roles/service_mesh_install/tasks/cleanup.yml | 30 +++-- .../tasks/{install.yml => istio-install.yml} | 67 ++++++----- roles/service_mesh_install/tasks/main.yml | 17 ++- .../tasks/tls-splicing-and-bumping.yml | 13 +++ .../templates/istioctl-options.yml.j2 | 2 +- .../templates/istioctl-values.yaml.j2 | 15 +++ .../templates/tcpip-bypass-ebpf.yaml.j2 | 2 +- roles/service_mesh_install/vars/main.yml | 15 ++- .../tasks/main.yml | 6 + 72 files changed, 1805 insertions(+), 234 deletions(-) create mode 100644 docs/flexran_guide.md create mode 100644 roles/bootstrap/install_realtime_kernel/tasks/main.yml rename roles/{intel_sriov_fec_operator/tasks/operator_sdk.yml => bootstrap/set_intel_flexran_kernel_flags/defaults/main.yml} (69%) create mode 100644 roles/bootstrap/set_intel_flexran_kernel_flags/tasks/main.yml create mode 100644 roles/intel_ethernet_operator/tasks/flow_config.yml delete mode 100644 roles/intel_ethernet_operator/tasks/operator_sdk.yml create mode 100644 roles/intel_ethernet_operator/tasks/preflight_ethernet_operator.yml create mode 100644 roles/intel_flexran/defaults/main.yml create mode 100644 roles/intel_flexran/files/kernel_cmdline_gen.sh create mode 100644 roles/intel_flexran/files/spr-bm-fcp-2x40 create mode 100644 roles/intel_flexran/files/spr-bm-q-mcc-1x32 create mode 100644 roles/intel_flexran/files/spr-bm-q-xcc-2x52 rename roles/{intel_sriov_fec_operator/tasks/olm.yml => intel_flexran/tasks/cleanup_flexran.yml} (68%) create mode 100644 roles/intel_flexran/tasks/fec_acc.yml create mode 100644 roles/intel_flexran/tasks/flexran.yml create mode 100644 roles/intel_flexran/tasks/flexran_preflight.yml create mode 100644 roles/intel_flexran/tasks/main.yml create mode 100644 roles/intel_flexran/tasks/oneapi.yml create mode 100644 roles/intel_flexran/tasks/pf_bb.yml create mode 100644 roles/intel_flexran/tasks/timer_mode.yml create mode 100644 roles/intel_flexran/vars/main.yml create mode 100644 roles/operator_framework/defaults/main.yml create mode 100644 roles/operator_framework/tasks/main.yml create mode 100644 roles/remove_kubespray_host_dns_settings/tasks/main.yml create mode 100644 roles/service_mesh_install/charts/istioctl/.helmignore create mode 100644 roles/service_mesh_install/charts/istioctl/Chart.yaml create mode 100644 roles/service_mesh_install/charts/istioctl/templates/NOTES.txt create mode 100644 roles/service_mesh_install/charts/istioctl/templates/_helpers.tpl create mode 100644 roles/service_mesh_install/charts/istioctl/templates/istioctl-deployment.yaml create mode 100644 roles/service_mesh_install/charts/istioctl/templates/istioctl-rbac.yaml create mode 100644 roles/service_mesh_install/charts/istioctl/values.yaml create mode 100644 roles/service_mesh_install/files/profiles/intel-cryptomb.yaml create mode 100644 roles/service_mesh_install/files/profiles/intel-qat-hw.yaml create mode 100644 roles/service_mesh_install/files/profiles/intel-qat-sw.yaml create mode 100644 roles/service_mesh_install/files/profiles/sgx-mtls.yaml rename roles/service_mesh_install/tasks/{install.yml => istio-install.yml} (63%) create mode 100644 roles/service_mesh_install/templates/istioctl-values.yaml.j2 diff --git a/docs/flexran_guide.md b/docs/flexran_guide.md new file mode 100644 index 00000000..fd625456 --- /dev/null +++ b/docs/flexran_guide.md @@ -0,0 +1,5 @@ +# Intel(R) FlexRAN(TM) Readme + +A formal PDF Quick Start Guide for Intel(R) FlexRAN(TM) deployment using the RA playbooks is published at this URL: +https://networkbuilders.intel.com/solutionslibrary/network-and-cloud-edge-reference-system-architecture-flexran-software-single-server-quick-start-guide + diff --git a/docs/generate_profiles.md b/docs/generate_profiles.md index ae85a898..d817d28c 100644 --- a/docs/generate_profiles.md +++ b/docs/generate_profiles.md @@ -69,7 +69,7 @@ At the moment, Container Experience Kits supports the following profiles: * build_your_own Profile's name will be passed to the make command via the required `PROFILE` parameter. Each profile includes some specific sort of functionalities. Choose the profile that suits you the most via inspecting the examples generated [here](#creating-sample-profiles). -If you would like to know more about CEK profiles read section 2.2 in [here](https://networkbuilders.intel.com/solutionslibrary/container-bare-metal-for-2nd-3rd-generation-intel-xeon-scalable-processor). +If you would like to know more about CEK profiles read section 2.2 in [here](https://networkbuilders.intel.com/solutionslibrary/network-and-cloud-edge-container-bare-metal-reference-system-architecture-user-guide). ## Example Commands diff --git a/docs/vm_config_guide.md b/docs/vm_config_guide.md index 9e50eb96..d09af445 100644 --- a/docs/vm_config_guide.md +++ b/docs/vm_config_guide.md @@ -53,7 +53,7 @@ Next section provides VM related configuration options. The first option defines VM image distribution of cloud image, which will be used inside VMs. Currently supported distributions are: "ubuntu" and "rocky". Default is "ubuntu" Following two options define VM image version for Ubuntu and for Rocky. - Currently supported ubuntu versions are: "20.04", "21.10" and "22.04". Default is "20.04" + Currently supported ubuntu versions are: "20.04" and "22.04". Default is "20.04" Currently supported rocky version is: "8.5". Default is "8.5" Default VM image distribution is "ubuntu" and default version is "20.04" Setting for VM image can be done just on the first VM host. It is common for all VMs across all VM hosts. diff --git a/generate/playbook_templates/infra_playbook.j2 b/generate/playbook_templates/infra_playbook.j2 index 55d115a6..476fd20a 100644 --- a/generate/playbook_templates/infra_playbook.j2 +++ b/generate/playbook_templates/infra_playbook.j2 @@ -62,6 +62,8 @@ when: iommu_enabled | default(true) | bool or on_vms | default(false) | bool - role: bootstrap/set_rdt_kernel_flags when: telegraf_enabled | default(true) | bool + - role: bootstrap/set_intel_flexran_kernel_flags + when: intel_flexran_enabled | default(false) | bool {%- if playbook_name in ['full_nfv', 'remote_fp', 'on_prem', 'build_your_own'] %} - role: bootstrap/configure_sst tags: sst diff --git a/generate/playbook_templates/intel_playbook.j2 b/generate/playbook_templates/intel_playbook.j2 index 424b4fa8..a81e2140 100644 --- a/generate/playbook_templates/intel_playbook.j2 +++ b/generate/playbook_templates/intel_playbook.j2 @@ -4,9 +4,22 @@ roles: - role: cluster_defaults tags: defaults + - role: remove_kubespray_host_dns_settings + tags: remove-kubespray-host-dns-settings + when: + - remove_kubespray_host_dns_settings | default(false) | bool - role: nfd_install tags: nfd when: nfd_enabled | default(true) | bool + - role: operator_framework + tags: operator-framework + when: + - intel_ethernet_operator_enabled | default(false) | bool or + intel_sriov_fec_operator_enabled | default(false) | bool + - role: intel_ethernet_operator + tags: intel-ethernet-operator + when: + - intel_ethernet_operator_enabled | default(false) | bool - role: sriov_dp_install tags: sriov-net-dp when: @@ -18,10 +31,6 @@ - sriov_network_operator_enabled | default(true) | bool - not sriov_net_dp_enabled | default(false) | bool - not sriov_cni_enabled | default(false) | bool - - role: intel_ethernet_operator - tags: intel-ethernet-operator - when: - - intel_ethernet_operator_enabled | default(false) | bool {%- if playbook_name in ['access', 'full_nfv', 'on_prem', 'regional_dc', 'remote_fp', 'storage', 'build_your_own'] %} - role: intel_dp_operator tags: dp-operator @@ -96,6 +105,12 @@ when: - intel_sriov_fec_operator_enabled | default(false) | bool {%- endif %} +{%- if playbook_name in ['access', 'full_nfv', 'build_your_own'] %} + - role: intel_flexran + tags: intel-flexran + when: + - intel_flexran_enabled | default(false) | bool +{%- endif %} {%- if playbook_name in ['access', 'full_nfv', 'on_prem', 'regional_dc', 'remote_fp', 'build_your_own'] %} - role: service_mesh_install tags: service-mesh diff --git a/generate/profiles_templates/common/group_vars.j2 b/generate/profiles_templates/common/group_vars.j2 index 1fca57d0..e1fac6ca 100644 --- a/generate/profiles_templates/common/group_vars.j2 +++ b/generate/profiles_templates/common/group_vars.j2 @@ -265,16 +265,22 @@ kmra: {%- if service_mesh and service_mesh.enabled in ['on', 'optional'] %} # Service mesh deployment # https://istio.io/latest/docs/setup/install/istioctl/ +# Intel Istio +# https://github.com/intel/istio # for all available options, please, refer to the 'roles/service_mesh_install/vars/main.yml; +# for the options dependencies and compatibility, please, refer to the official CEK documentation; service_mesh: enabled: {% if service_mesh.enabled == 'on' %}true{% else %}false{% endif %} # enable Service Mesh - # available profiles are: 'default', 'demo', 'minimal', 'external', 'empty', 'preview' + # available profiles are: 'default', 'demo', 'minimal', 'external', 'empty', 'preview', + # 'sgx-mtls', 'intel-qat-hw', 'intel-qat-sw', 'intel-cryptomb' # if custom profile needs to be deployed, please, place the file named '.yaml' # into the directory 'roles/service_mesh_install/files/profiles/' # 'custom-ca' profile name is reserved for usage by sgx_signer if sgx_signer option is enabled - # any name provided will be overwritten in this case + # any profile name provided will be overwritten in this case profile: {% if service_mesh.sgx_signer == 'on' and arch in ['icx', 'spr'] %}custom-ca{% else %}default{% endif %} + intel_preview: + enabled: {% if service_mesh.intel_preview == 'on' %}true{% else %}false{% endif %} # enable intel istio preview {%- if service_mesh.tcpip_bypass_ebpf in ['on', 'optional'] %} tcpip_bypass_ebpf: enabled: {% if service_mesh.tcpip_bypass_ebpf == 'on' %}true{% else %}false{% endif %} # enable tcp/ip ebpf bypass demo @@ -288,6 +294,14 @@ service_mesh: enabled: {% if service_mesh.sgx_signer == 'on' %}true{% else %}false{% endif %} # enable automated key management integration name: sgx-signer {%- endif %} + {%- if service_mesh.intel_preview in ['on', 'optional'] %} + # uncomment following section and enable intel_preview if sgx-mtls profile is selected + {% if service_mesh.intel_preview == 'optional' %}#{% endif %}set: + {% if service_mesh.intel_preview == 'optional' %}# {% endif %}- values.global.proxy.sgx.enabled=true + {% if service_mesh.intel_preview == 'optional' %}# {% endif %}- values.global.proxy.sgx.certExtensionValidationEnabled=true + {% if service_mesh.intel_preview == 'optional' %}# {% endif %}- values.gateways.sgx.enabled=true + {% if service_mesh.intel_preview == 'optional' %}# {% endif %}- values.gateways.sgx.certExtensionValidationEnabled=true + {%- endif %} {% endif %} {%- if tcs in ['on', 'optional'] and @@ -358,11 +372,14 @@ firewall_enabled: {% if firewall == "on" %}true{% else %}false{% endif %} ## Proxy configuration ## #http_proxy: "http://proxy.example.com:1080" #https_proxy: "http://proxy.example.com:1080" -#additional_no_proxy: ".example.com,mirror_ip" #no need to include the following (will be added automatically): localhost, 127.0.0.1, controllerIP, nodesIPs +#additional_no_proxy: ".example.com,mirror_ip" # no need to include the following (will be added automatically): localhost, 127.0.0.1, controllerIPs, nodesIPs # (Ubuntu only) disables DNS stub listener which may cause issues on Ubuntu dns_disable_stub_listener: true +# Remove the block between ansible markers set by kubespray in dhclient & hosts files to avoid DNS & LDAP issues (connection loss) after K8s setup after reboot +remove_kubespray_host_dns_settings: true + # Kubernetes cluster name, also will be used as DNS domain cluster_name: cluster.local @@ -434,10 +451,9 @@ kube_proxy_nodeport_addresses_cidr: 127.0.0.0/8 # Docker registry running on the cluster allows us to store images not available on Docker Hub # The range of valid ports is 30000-32767 registry_enable: {% if registry == 'on' %}true{% else %}false{% endif %} -registry_nodeport: 30500 +registry_nodeport: "30500" registry_local_address: "localhost:{{ '{{' }} registry_nodeport {{ '}}' }}" {%- endif %} - {%- if cert_manager in ['on', 'optional'] %} cert_manager_enable: {% if cert_manager == 'on' %}true{% else %}false{% endif %} {%- endif %} @@ -465,7 +481,7 @@ minio_deploy_test_mode: true # true (Test Mode) - use a # Intel Cloud Native Data Plane. {%- if cndp_dp in ['on', 'optional'] %} cndp_dp_enabled: {% if cndp_dp == 'on' %}true{% else %}false{% endif %} -{% if cndp_dp == 'on' %} +{%- if cndp_dp == 'on' %} cndp_net_attach_def_enabled: true # Whether or not to create NetworkAttachmentDefinition resource. cndp_net_attach_def_conf: name: cndp-cni-afxdp0 # (Optional) Name of NetworkAttachmentDefinition resource. diff --git a/generate/profiles_templates/common/host_vars.j2 b/generate/profiles_templates/common/host_vars.j2 index 7428eb40..68c763ea 100644 --- a/generate/profiles_templates/common/host_vars.j2 +++ b/generate/profiles_templates/common/host_vars.j2 @@ -80,7 +80,7 @@ dataplane_interfaces: [] {%- if ddp in ['on', 'optional'] %} # install Intel x700 & x800 series NICs DDP packages -install_ddp_packages: {% if ddp == "on"%}true{% else %}false{% endif %} +install_ddp_packages: {% if ddp == 'on' and nic == 'fvl'%}true{% else %}false{% endif %} # If following error appears: "Flashing failed: Operation not permitted" # run deployment with update_nic_firmware: true # or @@ -148,13 +148,18 @@ intel_ethernet_operator: fw_update: {% if intel_ethernet_operator.fw_update == 'on' and nic == 'cvl' %}true{% else %}false{% endif %} # perform firmware update on PFs listed in dataplane_interfaces # NodeFlowConfig manifests local path # For more information refer to: - # https://github.com/smart-edge-open/intel-ethernet-operator/blob/main/docs/flowconfig-daemon/creating-rules.md + # https://github.com/intel/intel-ethernet-operator/blob/main/docs/flowconfig-daemon/creating-rules.md # node_flow_config_dir: /tmp/node_flow_config {% endif %} {%- if intel_sriov_fec_operator in ['on', 'optional'] %} # Wireless FEC H/W Accelerator Device (e.g. ACC100) PCI ID -fec_acc: {{ fec_acc_dev }} # must be string in [a-fA-F0-9]{4}:[a-fA-F0-9]{2}:[01][a-fA-F0-9].[0-7] format +fec_acc: "0000:27:00.0" # must be string in [a-fA-F0-9]{4}:[a-fA-F0-9]{2}:[01][a-fA-F0-9].[0-7] format +{% endif %} + +{%- if intel_flexran in ['on', 'optional'] %} +# Intel FlexRAN +intel_flexran_enabled: {% if intel_flexran == 'on' %}true{% else %}false{% endif %} # if true, deploy FlexRAN {% endif %} {%- if qat in ['on', 'optional'] %} @@ -369,13 +374,13 @@ cndp_dp_pools: # {%- if secondary_host == 'true' %} # Do not set VM image info here - do it just on the first vm_host -# secondary vm_host - do not change dhcp settings here +# Secondary vm_host - do not change dhcp settings here dhcp: [] {% else %} # Default VM image version is Ubuntu 20.04 - focal -# Supported VM image disributions ['ubuntu', 'rocky'] +# Supported VM image distributions ['ubuntu', 'rocky'] #vm_image_distribution: "ubuntu" -# Supported VM image ubuntu versions ['20.04', '21.04', '21.10', '22.04'] +# Supported VM image ubuntu versions ['20.04', '22.04'] #vm_image_version_ubuntu: "22.04" # Supported VM image rocky versions ['8.5'] #vm_image_version_rocky: "8.5" diff --git a/generate/profiles_templates/k8s/profiles.yml b/generate/profiles_templates/k8s/profiles.yml index daf8dfb4..8e12ebce 100644 --- a/generate/profiles_templates/k8s/profiles.yml +++ b/generate/profiles_templates/k8s/profiles.yml @@ -59,12 +59,14 @@ # tcpip_bypass_ebpf # tls_splicing # sgx_signer +# intel_preview # - intel_ethernet_operator # enabled # flow_config # ddp # fw_update # - intel_sriov_fec_operator +# - intel_flexran --- access: @@ -118,6 +120,7 @@ access: tcpip_bypass_ebpf: off tls_splicing: off sgx_signer: off + intel_preview: off wireguard: on multus: on firewall: optional @@ -135,6 +138,7 @@ access: ddp: optional fw_update: optional intel_sriov_fec_operator: on + intel_flexran: on basic: name: basic @@ -216,6 +220,7 @@ full_nfv: tcpip_bypass_ebpf: on tls_splicing: on sgx_signer: on + intel_preview: optional wireguard: on multus: on firewall: optional @@ -227,11 +232,12 @@ full_nfv: registry: on hugepages: on intel_ethernet_operator: - enabled: optional + enabled: on flow_config: optional - ddp: optional + ddp: on fw_update: optional intel_sriov_fec_operator: optional + intel_flexran: optional on_prem: name: on_prem @@ -274,6 +280,7 @@ on_prem: tcpip_bypass_ebpf: on tls_splicing: on sgx_signer: on + intel_preview: optional wireguard: on multus: on firewall: optional @@ -316,6 +323,7 @@ regional_dc: enabled: on tcpip_bypass_ebpf: on tls_splicing: on + intel_preview: optional wireguard: on multus: on firewall: optional @@ -373,6 +381,7 @@ remote_fp: tcpip_bypass_ebpf: optional tls_splicing: optional sgx_signer: optional + intel_preview: optional wireguard: on multus: on firewall: optional @@ -383,9 +392,9 @@ remote_fp: registry: on hugepages: on intel_ethernet_operator: - enabled: optional + enabled: on flow_config: optional - ddp: optional + ddp: on fw_update: optional storage: @@ -471,6 +480,7 @@ build_your_own: tcpip_bypass_ebpf: optional tls_splicing: optional sgx_signer: optional + intel_preview: optional wireguard: optional multus: optional firewall: optional @@ -486,4 +496,5 @@ build_your_own: flow_config: optional ddp: optional fw_update: optional - intel_sriov_fec_operator: optional \ No newline at end of file + intel_sriov_fec_operator: optional + intel_flexran: optional diff --git a/generate/profiles_templates/vm/vm_host_profiles.yml b/generate/profiles_templates/vm/vm_host_profiles.yml index 621cd6ff..112d5703 100644 --- a/generate/profiles_templates/vm/vm_host_profiles.yml +++ b/generate/profiles_templates/vm/vm_host_profiles.yml @@ -57,6 +57,7 @@ # tcpip_bypass_ebpf # tls_splicing # sgx_signer +# intel_preview # - intel_ethernet_operator # enabled # flow_config @@ -99,6 +100,7 @@ access: enabled: on tcpip_bypass_ebpf: on tls_splicing: on + intel_preview: optional wireguard: on multus: on firewall: optional @@ -190,6 +192,7 @@ full_nfv: tcpip_bypass_ebpf: on tls_splicing: on sgx_signer: optional + intel_preview: optional wireguard: on multus: on firewall: optional @@ -245,6 +248,7 @@ on_prem: tcpip_bypass_ebpf: on tls_splicing: on sgx_signer: optional + intel_preview: optional wireguard: on multus: on firewall: optional @@ -287,6 +291,7 @@ regional_dc: enabled: on tcpip_bypass_ebpf: on tls_splicing: on + intel_preview: optional wireguard: on multus: on firewall: optional @@ -343,6 +348,7 @@ remote_fp: tcpip_bypass_ebpf: optional tls_splicing: optional sgx_signer: optional + intel_preview: optional wireguard: on multus: on firewall: optional @@ -404,6 +410,7 @@ build_your_own: tcpip_bypass_ebpf: optional tls_splicing: optional sgx_signer: optional + intel_preview: optional wireguard: optional multus: optional firewall: optional diff --git a/generate/profiles_templates/vm/vms_profiles.yml b/generate/profiles_templates/vm/vms_profiles.yml index d9d73c2e..9e6e9b04 100644 --- a/generate/profiles_templates/vm/vms_profiles.yml +++ b/generate/profiles_templates/vm/vms_profiles.yml @@ -56,6 +56,7 @@ # tcpip_bypass_ebpf # tls_splicing # sgx_signer +# intel_preview # - intel_ethernet_operator # enabled # flow_config @@ -106,6 +107,7 @@ access: enabled: on tcpip_bypass_ebpf: on tls_splicing: on + intel_preview: optional wireguard: on multus: on firewall: optional @@ -197,6 +199,7 @@ full_nfv: tcpip_bypass_ebpf: on tls_splicing: on sgx_signer: optional + intel_preview: optional wireguard: on multus: on firewall: optional @@ -252,6 +255,7 @@ on_prem: tcpip_bypass_ebpf: on tls_splicing: on sgx_signer: optional + intel_preview: optional wireguard: on multus: on firewall: optional @@ -294,6 +298,7 @@ regional_dc: enabled: on tcpip_bypass_ebpf: on tls_splicing: on + intel_preview: optional wireguard: on multus: on firewall: optional @@ -350,6 +355,7 @@ remote_fp: tcpip_bypass_ebpf: optional tls_splicing: optional sgx_signer: optional + intel_preview: optional wireguard: on multus: on firewall: optional @@ -411,6 +417,7 @@ build_your_own: tcpip_bypass_ebpf: optional tls_splicing: optional sgx_signer: optional + intel_preview: optional wireguard: optional multus: optional firewall: optional diff --git a/playbooks/k8s/k8s.yml b/playbooks/k8s/k8s.yml index 95607deb..3375b26e 100644 --- a/playbooks/k8s/k8s.yml +++ b/playbooks/k8s/k8s.yml @@ -250,7 +250,7 @@ - name: install podman package package: - name: "{{ 'podman-3.3.1' if ansible_os_family == 'RedHat' else 'podman' }}" + name: podman state: present register: podman_package retries: 10 diff --git a/playbooks/preflight.yml b/playbooks/preflight.yml index 862856ac..9ab0a810 100644 --- a/playbooks/preflight.yml +++ b/playbooks/preflight.yml @@ -170,7 +170,7 @@ any_errors_fatal: true vars: cek_supported_distros: [RedHat, Rocky, Ubuntu] - cek_supported_distros_versions: ['8.5', '20.04', '21.10', '22.04'] + cek_supported_distros_versions: ['8.5', '20.04', '22.04'] cpusets_ranges: [] cpusets_discretes: [] isolcpus_ranges: [] @@ -495,7 +495,7 @@ any_errors_fatal: true vars: cstates_supported_distros: [Ubuntu] - cstates_supported_distros_versions: ['21.10', '22.04'] + cstates_supported_distros_versions: ['22.04'] phy_nics_pciids: [] gpu_pciids: - name: DG1 @@ -780,9 +780,12 @@ assert: that: - nfd_enabled - fail_msg: "SGX DP requires nfd_enabled set to 'true'" + fail_msg: "Deployment of Intel Device Plugins requires nfd_enabled set to 'true' in group_vars" success_msg: "NFD configuration verified" - when: sgx_dp_enabled is defined and sgx_dp_enabled + when: + - (qat_dp_enabled | default(false)) or + (sgx_dp_enabled | default(false)) or + (gpu_dp_enabled | default(false)) - name: check kmra.pccs.api_key presence assert: @@ -1148,45 +1151,9 @@ when: intel_power_manager is defined and intel_power_manager.enabled and container_runtime in ["crio", "containerd"] - name: check Intel Ethernet Operator configuration - block: - - name: check if ddp service is enabled - assert: - that: enable_ice_systemd_service - msg: "Please set enable_ice_systemd_service as true" - when: - - intel_ethernet_operator.ddp_update is defined and intel_ethernet_operator.ddp_update - - ansible_distribution == "Ubuntu" and ansible_distribution_version >= "22.04" - - - name: check if legacy DDP/FW update variables are false - assert: - that: - - not install_ddp_packages - - not update_nic_firmware - msg: - - "Intel Ethernet Operator is mutually exclusive with legacy DDP/FW update role." - - "Please set 'install_ddp_packages' and 'update_nic_firmware' as false" - when: - - (intel_ethernet_operator.ddp_update is defined and intel_ethernet_operator.ddp_update) or - (intel_ethernet_operator.fw_update is defined and intel_ethernet_operator.fw_update) - - - name: check if ice driver will be updated - assert: - that: update_nic_drivers - msg: "Firmware update requires update_nic_drivers set as true" - when: intel_ethernet_operator.fw_update is defined and intel_ethernet_operator.fw_update - - - name: check Hugepages settings for Flow Configuration - assert: - that: number_of_hugepages_2M >= 1024 - msg: "Flow Configuration requires at least 2Gi of 2M Hugepages." - when: dataplane_interfaces | json_query('[?flow_configuration]') - - - name: check if SRIOV Network Operator is enabled - assert: - that: sriov_network_operator_enabled - msg: "Flow Configuration requires SRIOV Network Operator to be enabled." - when: dataplane_interfaces | json_query('[?flow_configuration]') - + include_role: + name: intel_ethernet_operator + tasks_from: preflight_ethernet_operator when: intel_ethernet_operator_enabled - name: make sure isolcpus and cpusets are not enabled simultaneously @@ -1207,7 +1174,9 @@ - name: FEC Operator - check profile is access or bring_your_own assert: that: profile_name in ['access', 'full_nfv', 'bring_your_own'] - msg: "Deploying the Intel SR-IOV FEC Operator is supported only in the 'access', 'full_nfv', or 'byo' profiles. Please correct the group_vars configuration" + msg: > + "Deploying the Intel SR-IOV FEC Operator is supported only in the 'access', 'full_nfv', or 'byo' profiles. + Please correct the group_vars configuration" - name: FEC Operator - check distro is Ubuntu 22.04 (generic or realtime) assert: @@ -1226,11 +1195,6 @@ that: container_runtime == 'docker' msg: "Deploying the Intel SR-IOV FEC Operator is supported only for docker runtime. Please correct the group_vars configuration" - - name: FEC Operator - check IEO coexistence - assert: - that: "{{ not intel_ethernet_operator_enabled | default(false) }}" - msg: "Deploying the Intel SR-IOV FEC Operator is not supported along with Intel Ethernet Operator (IEO). Please disable IEO in the group_vars configuration" - # TODO # - name: FEC Operator - check Cert Manager is enabled # assert: @@ -1251,6 +1215,12 @@ when: intel_sriov_fec_operator_enabled | default(false) | bool + - name: check Intel FlexRAN requirements + include_role: + name: intel_flexran + tasks_from: flexran_preflight + when: intel_flexran_enabled | default(false) + #################################### # Prerequisites for Role specific # #################################### diff --git a/roles/bootstrap/install_qat_drivers_services/tasks/main.yml b/roles/bootstrap/install_qat_drivers_services/tasks/main.yml index 9e980aa3..14f5a538 100644 --- a/roles/bootstrap/install_qat_drivers_services/tasks/main.yml +++ b/roles/bootstrap/install_qat_drivers_services/tasks/main.yml @@ -18,6 +18,12 @@ include_role: name: install_dependencies +- name: WA for libudev-dev version issue on Ubuntu 22.04 + apt: + name: 'udev' + state: latest # noqa 403 package-latest + when: ansible_distribution == "Ubuntu" and ansible_distribution_version == "22.04" + - name: get current udev package version shell: "set -o pipefail && apt list --installed 2>/dev/null |grep '^udev' | awk 'NR==1{ print $2 }'" args: diff --git a/roles/bootstrap/install_realtime_kernel/tasks/main.yml b/roles/bootstrap/install_realtime_kernel/tasks/main.yml new file mode 100644 index 00000000..409b4cf0 --- /dev/null +++ b/roles/bootstrap/install_realtime_kernel/tasks/main.yml @@ -0,0 +1,33 @@ +## +## Copyright (c) 2020-2022 Intel Corporation. +## +## Licensed under the Apache License, Version 2.0 (the "License"); +## you may not use this file except in compliance with the License. +## You may obtain a copy of the License at +## +## http://www.apache.org/licenses/LICENSE-2.0 +## +## Unless required by applicable law or agreed to in writing, software +## distributed under the License is distributed on an "AS IS" BASIS, +## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +## See the License for the specific language governing permissions and +## limitations under the License. +## +--- +# Installing the RT kernel is intended only for FlexRAN support + +- debug: msg="NOP. Until Ubuntu 22.04 RT patch is made public by Canonical, realtime image is expected to be pre-built by user" + +# - name: Install realtime headers from the local DEB packages +# shell: "apt install -y /opt/rt-kits/*.deb" +# ignore_errors: True + +# - name: Find existing local DEB files +# find: +# paths: /opt/rt-kits/ +# patterns: "*.deb" +# register: found_debs +# - name: Install the local DEB packages +# apt: +# deb: "{{ item.path }}" +# with_items: "{{ found_debs.files }}" diff --git a/roles/intel_sriov_fec_operator/tasks/operator_sdk.yml b/roles/bootstrap/set_intel_flexran_kernel_flags/defaults/main.yml similarity index 69% rename from roles/intel_sriov_fec_operator/tasks/operator_sdk.yml rename to roles/bootstrap/set_intel_flexran_kernel_flags/defaults/main.yml index f83851e3..5e1f764b 100644 --- a/roles/intel_sriov_fec_operator/tasks/operator_sdk.yml +++ b/roles/bootstrap/set_intel_flexran_kernel_flags/defaults/main.yml @@ -14,14 +14,4 @@ ## limitations under the License. ## --- -- name: clone Operator SDK repository - git: - repo: "{{ operator_sdk_git }}" - version: "{{ operator_sdk_git_ref }}" - dest: "{{ operator_sdk_dir }}" - force: yes - -- name: build and install Operator SDK - make: - target: "install" - chdir: "{{ operator_sdk_dir }}" +intel_flexran_marker: "# intel-flexran" diff --git a/roles/bootstrap/set_intel_flexran_kernel_flags/tasks/main.yml b/roles/bootstrap/set_intel_flexran_kernel_flags/tasks/main.yml new file mode 100644 index 00000000..26647917 --- /dev/null +++ b/roles/bootstrap/set_intel_flexran_kernel_flags/tasks/main.yml @@ -0,0 +1,102 @@ +## +## Copyright (c) 2020-2022 Intel Corporation. +## +## Licensed under the Apache License, Version 2.0 (the "License"); +## you may not use this file except in compliance with the License. +## You may obtain a copy of the License at +## +## http://www.apache.org/licenses/LICENSE-2.0 +## +## Unless required by applicable law or agreed to in writing, software +## distributed under the License is distributed on an "AS IS" BASIS, +## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +## See the License for the specific language governing permissions and +## limitations under the License. +## +--- +# probe CPU +- debug: msg="CPU={{ ansible_processor[2] }} cores={{ ansible_processor_cores }} count={{ ansible_processor_count }} nproc={{ ansible_processor_nproc }} tpc={{ ansible_processor_threads_per_core }} vcpus={{ ansible_processor_vcpus }}" # noqa 204 line-length + +- name: include Intel FlexRAN role vars + include_vars: ../../intel_flexran/defaults/main.yml + +- name: create Intel FlexRAN files directory on controller + file: + path: "{{ intel_flexran_files_dir }}" + state: directory + mode: '0755' + +- name: transfer Intel FlexRAN kernel-cmdline generator to worker + copy: + src: '../../intel_flexran/files/kernel_cmdline_gen.sh' + dest: "{{ intel_flexran_files_dir }}" + mode: '0755' + +- name: generate Intel FlexRAN kernel-cmdline + shell: "./kernel_cmdline_gen.sh" # noqa 305 + args: + chdir: "{{ intel_flexran_files_dir }}" + register: generated_cmdline + changed_when: false + +- debug: msg="{{ generated_cmdline.stdout }}" + +- name: set generic Intel FlexRAN kernel flags + set_fact: +# intel_flexran_cmdline: 'GRUB_CMDLINE_LINUX="${GRUB_CMDLINE_LINUX} intel_iommu=on iommu=pt" {{ intel_flexran_marker }}' + intel_flexran_cmdline: 'GRUB_CMDLINE_LINUX="{{ generated_cmdline.stdout }}" {{ intel_flexran_marker }}' + +- name: override Intel FlexRAN kernel flags specific for 1x32 cores CPU + set_fact: + # Sam: BM: Quanta MCC SPR single socket (1x32 cores) for L2 testfile=icelake-sp: + intel_flexran_cmdline: 'GRUB_CMDLINE_LINUX="crashkernel=auto intel_iommu=on iommu=pt usbcore.autosuspend=-1 selinux=0 enforcing=0 nmi_watchdog=0 softlockup_panic=0 audit=0 cgroup_disable=memory mce=off hugepagesz=1G hugepages=40 hugepagesz=2M hugepages=0 default_hugepagesz=1G kthread_cpus=0,30-32,62-63 irqaffinity=0,30-32,62-63 nosoftlockup skew_tick=1 skew_tick=1 isolcpus=1-29,33-61 nohz_full=1-29,33-61 rcu_nocbs=1-29,33-61" {{ intel_flexran_marker }}' # noqa 204 line-length + intel_flexran_isol_cores: "1-29,33-61" + intel_flexran_cpu_supported: true + when: + - ansible_processor_count == 1 + - ansible_processor_cores == 32 + +- name: override Intel FlexRAN kernel flags specific for 2x40 cores CPUs + set_fact: + # Baoqian script: BM: FCP (2x40 cores): + intel_flexran_cmdline: 'GRUB_CMDLINE_LINUX="intel_iommu=on iommu=pt usbcore.autosuspend=-1 selinux=0 enforcing=0 nmi_watchdog=0 crashkernel=auto softlockup_panic=0 audit=0 cgroup_disable=memory tsc=nowatchdog intel_pstate=disable mce=off hugepagesz=1G hugepages=40 hugepagesz=2M hugepages=0 default_hugepagesz=1G kthread_cpus=0,80,40-79,120-159 irqaffinity=0,80,40-79,120-159 nohz=on nosoftlockup nohz_full=1-39,81-119 rcu_nocbs=1-39,81-119 rcu_nocb_poll skew_tick=1 isolcpus=1-39,81-119" {{ intel_flexran_marker }}' # noqa 204 line-length + intel_flexran_cpu_supported: true + when: + - ansible_processor_count == 2 + - ansible_processor_cores == 40 + +- name: override Intel FlexRAN kernel flags specific for 2x52 cores CPUs + set_fact: + # Sam: BM: Quanta XCC dual socket (2x52 cores) for L2 testfile=icelake-sp: + intel_flexran_cmdline: 'GRUB_CMDLINE_LINUX="intel_iommu=on iommu=pt default_hugepagesz=1G hugepagesz=1G hugepages=60 irqaffinity=0,50-52,103-104,154-156,206-207 mce=off nmi_watchdog=0 softlockup_panic=0 selinux=0 enforcing=0 audit=0 kthread_cpus=0,50-52,103-104,154-156,206-207 clock=pit no_timer_check clocksource=tsc tsc=perfect usbcore.autosuspend=-1 pci=realloc pci=assign-busses rdt=l3cat skew_tick=1 isolcpus=managed_irq,domain,1-49,53-101,105-153,157-205 intel_pstate=disable nosoftlockup tsc=nowatchdog nohz=on nohz_full=1-49,53-101,105-153,157-205 rcu_nocbs=1-49,53-101,105-153,157-205" {{ intel_flexran_marker }}' # noqa 204 line-length + intel_flexran_isol_cores: "1-49,53-101,105-153,157-205" + intel_flexran_cpu_supported: true + # Jing: XCC dual socket (2x52 cores): +# intel_flexran_cmdline: 'GRUB_CMDLINE_LINUX="intel_iommu=on iommu=pt default_hugepagesz=1G hugepagesz=1G hugepages=60 irqaffinity=0-3,100-103,104-107,204-207 mce=off nmi_watchdog=0 softlockup_panic=0 selinux=0 enforcing=0 audit=0 kthread_cpus=0-3,100-103,104-107,204-207 clock=pit no_timer_check clocksource=tsc tsc=perfect usbcore.autosuspend=-1 pci=realloc pci=assign-busses rdt=l3cat skew_tick=1 isolcpus=managed_irq,domain,4-99,108-203 intel_pstate=disable nosoftlockup tsc=nowatchdog nohz=on nohz_full=4-99,108-203 rcu_nocbs=4-99,108-203" {{ intel_flexran_marker }}' # noqa 204 line-length + when: + - ansible_processor_count == 2 + - ansible_processor_cores == 52 + +- debug: msg="{{ intel_flexran_cmdline }}" + +- name: set Intel FlexRAN kernel flags in /etc/default/grub + lineinfile: + dest: /etc/default/grub + regexp: '^GRUB_CMDLINE_LINUX="\${GRUB_CMDLINE_LINUX}(.*?)" {{ intel_flexran_marker }}$' + line: '{{ intel_flexran_cmdline }}' + state: present + mode: '0664' + notify: + - reboot server + +- name: set Intel FlexRAN cores isolation for RHEL + lineinfile: + dest: /etc/tuned/realtime-variables.conf # or /etc/tuned/realtime-virtual-host.conf + line: 'isolated_cores={{ intel_flexran_isol_cores }}' + state: present + create: yes + mode: '0664' + when: + - ansible_os_family == "RedHat" + notify: + - reboot server diff --git a/roles/bootstrap/update_nic_drivers/tasks/i40e.yml b/roles/bootstrap/update_nic_drivers/tasks/i40e.yml index d0d08478..60964ecf 100644 --- a/roles/bootstrap/update_nic_drivers/tasks/i40e.yml +++ b/roles/bootstrap/update_nic_drivers/tasks/i40e.yml @@ -30,6 +30,7 @@ when: - i40e_installed_version.stdout != i40e_driver_version - mgmt_interface_driver.stdout != i40e_driver_name + - not update_kernel - ansible_os_family == "RedHat" or (ansible_distribution == "Ubuntu" and ansible_distribution_version < "22.04") @@ -72,7 +73,7 @@ reboot_timeout: 1200 when: - (i40e_installed_version.stdout != i40e_driver_version and mgmt_interface_driver.stdout == i40e_driver_name) or - (i40e_installed_version.stdout != i40e_driver_version and ansible_distribution == "Ubuntu" and ansible_distribution_version >= "22.04") + (i40e_installed_version.stdout != i40e_driver_version and (ansible_distribution == "Ubuntu" and ansible_distribution_version >= "22.04" or update_kernel)) - name: ensure that i40e module is loaded modprobe: diff --git a/roles/bootstrap/update_nic_drivers/tasks/ice.yml b/roles/bootstrap/update_nic_drivers/tasks/ice.yml index 7c8f5a6a..ef42c5a4 100644 --- a/roles/bootstrap/update_nic_drivers/tasks/ice.yml +++ b/roles/bootstrap/update_nic_drivers/tasks/ice.yml @@ -30,6 +30,7 @@ when: - ice_installed_version.stdout != ice_driver_version - mgmt_interface_driver.stdout != ice_driver_name + - not update_kernel - ansible_os_family == "RedHat" or (ansible_distribution == "Ubuntu" and ansible_distribution_version < "22.04") @@ -72,7 +73,7 @@ reboot_timeout: 1200 when: - (ice_installed_version.stdout != ice_driver_version and mgmt_interface_driver.stdout == ice_driver_name) or - (ice_installed_version.stdout != ice_driver_version and ansible_distribution == "Ubuntu" and ansible_distribution_version >= "22.04") + (ice_installed_version.stdout != ice_driver_version and (ansible_distribution == "Ubuntu" and ansible_distribution_version >= "22.04" or update_kernel)) - name: ensure that ice module is loaded modprobe: diff --git a/roles/install_dependencies/tasks/main.yml b/roles/install_dependencies/tasks/main.yml index 564a3890..265b904b 100644 --- a/roles/install_dependencies/tasks/main.yml +++ b/roles/install_dependencies/tasks/main.yml @@ -26,3 +26,6 @@ register: pkg_mgr_results retries: 3 until: pkg_mgr_results is success + environment: + http_proxy: "{{ http_proxy }}" + https_proxy: "{{ https_proxy }}" diff --git a/roles/intel_ethernet_operator/defaults/main.yml b/roles/intel_ethernet_operator/defaults/main.yml index 803cfab7..6d070b08 100644 --- a/roles/intel_ethernet_operator/defaults/main.yml +++ b/roles/intel_ethernet_operator/defaults/main.yml @@ -18,10 +18,6 @@ intel_ethernet_operator_git: "https://github.com/intel/intel-ethernet-operator.g intel_ethernet_operator_git_ref: "v22.04" intel_ethernet_operator_dir: "{{ (project_root_dir, 'intel-ethernet-operator') | path_join }}" -operator_sdk_git: "https://github.com/operator-framework/operator-sdk.git" -operator_sdk_git_ref: "v1.18.1" -operator_sdk_dir: "{{ (project_root_dir, 'operator-sdk') | path_join }}" - uft_git: "https://github.com/intel/UFT.git" uft_git_ref: "v22.03" uft_dir: "{{ (project_root_dir, 'uft') | path_join }}" diff --git a/roles/intel_ethernet_operator/tasks/ddp.yml b/roles/intel_ethernet_operator/tasks/ddp.yml index 3693d448..b8ce9ffb 100644 --- a/roles/intel_ethernet_operator/tasks/ddp.yml +++ b/roles/intel_ethernet_operator/tasks/ddp.yml @@ -33,12 +33,13 @@ minutes: 1 - name: check status of DDP update - command: "kubectl get ethernetnodeconfig {{ node_name }} -n {{ intel_ethernet_operator_namespace }}" + command: "kubectl get ethernetnodeconfig {{ hostvars[node_name]['ansible_hostname'] }} -n {{ intel_ethernet_operator_namespace }}" register: ddp_update_info until: "'InProgress' not in ddp_update_info.stdout" - retries: 30 + retries: 60 delay: 10 - failed_when: "'Failed' in ddp_update_info.stdout" + failed_when: "'Failed' in ddp_update_info.stdout or 'InProgress' in ddp_update_info.stdout or 'NotRequested' in ddp_update_info.stdout" + changed_when: false - name: check node after reboot block: @@ -50,23 +51,52 @@ timeout: 1200 - name: check if node is ready - command: "kubectl get nodes {{ node_name }}" + command: "kubectl get nodes {{ hostvars[node_name]['ansible_hostname'] }}" register: ddp_node_readiness - retries: 30 + retries: 60 delay: 10 - until: "'NotReady' not in ddp_node_readiness.stdout" + until: + - "'NotReady' not in ddp_node_readiness.stdout" + - "'SchedulingDisabled' not in ddp_node_readiness.stdout" + changed_when: false - name: wait for pods to be up and running pause: minutes: 1 + - name: check if all pods are running + shell: set -o pipefail && kubectl get pods -A | awk 'NR != 1 { print $4 }' + args: + executable: /bin/bash + register: ddp_pods_status + retries: 30 + delay: 10 + until: + - "'Error' not in ddp_pods_status.stdout" + - "'CrashLoopBackOff' not in ddp_pods_status.stdout" + - "'Terminating' not in ddp_pods_status.stdout" + - "'ContainerCreating' not in ddp_pods_status.stdout" + - "'Pending' not in ddp_pods_status.stdout" + changed_when: false + + - name: check if all containers are running + shell: set -o pipefail && kubectl get pods -A | awk '! /Completed/ && NR != 1 { print $3 }' + args: + executable: /bin/bash + register: ddp_containers_readiness + retries: 30 + delay: 10 + until: "'0' not in ddp_containers_readiness.stdout" + changed_when: false + - name: check status of DDP update - command: "kubectl get ethernetnodeconfig {{ node_name }} -n {{ intel_ethernet_operator_namespace }}" + command: "kubectl get ethernetnodeconfig {{ hostvars[node_name]['ansible_hostname'] }} -n {{ intel_ethernet_operator_namespace }}" register: after_ddp_update_info until: "'PostUpdateReboot' not in after_ddp_update_info.stdout" retries: 30 delay: 10 - failed_when: "'Failed' in after_ddp_update_info.stdout" + failed_when: "'Failed' in after_ddp_update_info.stdout or 'PostUpdateReboot' in after_ddp_update_info.stdout or 'InProgress' in after_ddp_update_info.stdout" # noqa 204 line-length + changed_when: false when: "'PostUpdateReboot' in ddp_update_info.stdout" - name: remove DDP CR after update @@ -75,7 +105,8 @@ src: "{{ (intel_ethernet_operator_ddp_files_dir, node_name + '-ddp-update.yml') | path_join }}" - name: remove EthernetNodeConfig after update - command: kubectl delete enc {{ node_name }} -n {{ intel_ethernet_operator_namespace }} + command: kubectl delete enc {{ hostvars[node_name]['ansible_hostname'] }} -n {{ intel_ethernet_operator_namespace }} + changed_when: true - name: reload nic modules block: @@ -87,6 +118,12 @@ delegate_to: "{{ node_name }}" changed_when: false + - name: check if IRDMA module is loaded + command: "lsmod" + register: ieo_lsmod + delegate_to: "{{ node_name }}" + changed_when: false + - name: unload ICE module modprobe: name: ice @@ -95,6 +132,7 @@ when: - not hostvars[node_name]['enable_ice_systemd_service'] - mgmt_interface_driver_ieo.stdout != "ice" + - "'irdma' not in ieo_lsmod.stdout" - name: load ICE module modprobe: @@ -104,6 +142,7 @@ when: - not hostvars[node_name]['enable_ice_systemd_service'] - mgmt_interface_driver_ieo.stdout != "ice" + - "'irdma' not in ieo_lsmod.stdout" - name: create ddp service block: diff --git a/roles/intel_ethernet_operator/tasks/ethernet_operator.yml b/roles/intel_ethernet_operator/tasks/ethernet_operator.yml index 144daaa3..dad37415 100644 --- a/roles/intel_ethernet_operator/tasks/ethernet_operator.yml +++ b/roles/intel_ethernet_operator/tasks/ethernet_operator.yml @@ -98,6 +98,31 @@ state: present src: "{{ (intel_ethernet_operator_files_dir, 'subscription.yml') | path_join }}" -- name: wait for Intel Ethernet Operator to become fully available +- name: wait for Ethernet Operator deployment pause: - minutes: 1 + seconds: 100 + +- name: check if all pods are running + shell: set -o pipefail && kubectl get pods -A | awk 'NR != 1 { print $4 }' + args: + executable: /bin/bash + register: ieo_pods_status + retries: 30 + delay: 10 + until: + - "'Error' not in ieo_pods_status.stdout" + - "'CrashLoopBackOff' not in ieo_pods_status.stdout" + - "'Terminating' not in ieo_pods_status.stdout" + - "'ContainerCreating' not in ieo_pods_status.stdout" + - "'Pending' not in ieo_pods_status.stdout" + changed_when: false + +- name: check if all containers are running + shell: set -o pipefail && kubectl get pods -A | awk '! /Completed/ && NR != 1 { print $3 }' + args: + executable: /bin/bash + register: ieo_containers_readiness + retries: 30 + delay: 10 + until: "'0' not in ieo_containers_readiness.stdout" + changed_when: false diff --git a/roles/intel_ethernet_operator/tasks/flow_config.yml b/roles/intel_ethernet_operator/tasks/flow_config.yml new file mode 100644 index 00000000..65149c7b --- /dev/null +++ b/roles/intel_ethernet_operator/tasks/flow_config.yml @@ -0,0 +1,24 @@ +## +## Copyright (c) 2020-2022 Intel Corporation. +## +## Licensed under the Apache License, Version 2.0 (the "License"); +## you may not use this file except in compliance with the License. +## You may obtain a copy of the License at +## +## http://www.apache.org/licenses/LICENSE-2.0 +## +## Unless required by applicable law or agreed to in writing, software +## distributed under the License is distributed on an "AS IS" BASIS, +## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +## See the License for the specific language governing permissions and +## limitations under the License. +## +- name: build Unified Flow Tool image + include_tasks: uft.yml + when: + - inventory_hostname == groups['kube_node'][0] + +- name: flow configuration + include_tasks: flow_config_deployment.yml + when: + - inventory_hostname == groups['kube_control_plane'][0] diff --git a/roles/intel_ethernet_operator/tasks/fw.yml b/roles/intel_ethernet_operator/tasks/fw.yml index ce059ac1..7598144a 100644 --- a/roles/intel_ethernet_operator/tasks/fw.yml +++ b/roles/intel_ethernet_operator/tasks/fw.yml @@ -33,7 +33,7 @@ minutes: 2 - name: check status of FW update - command: "kubectl get ethernetnodeconfig {{ node_name }} -n {{ intel_ethernet_operator_namespace }}" + command: "kubectl get ethernetnodeconfig {{ hostvars[node_name]['ansible_hostname'] }} -n {{ intel_ethernet_operator_namespace }}" register: fw_update_info until: "'InProgress' not in fw_update_info.stdout" retries: 60 @@ -50,23 +50,54 @@ timeout: 1200 - name: check if node is ready - command: "kubectl get nodes {{ node_name }}" + command: "kubectl get nodes {{ hostvars[node_name]['ansible_hostname'] }}" register: fw_node_readiness retries: 30 delay: 10 - until: "'NotReady' and 'SchedulingDisabled' not in fw_node_readiness.stdout" + until: + - "'NotReady' not in fw_node_readiness.stdout" + - "'SchedulingDisabled' not in fw_node_readiness.stdout" + changed_when: false - name: wait for pods to be up and running pause: minutes: 1 + - name: check if all pods are running + shell: set -o pipefail && kubectl get pods -A | awk 'NR != 1 { print $4 }' + args: + executable: /bin/bash + register: fw_pods_status + retries: 30 + delay: 10 + until: + - "'Error' not in fw_pods_status.stdout" + - "'CrashLoopBackOff' not in fw_pods_status.stdout" + - "'Terminating' not in fw_pods_status.stdout" + - "'ContainerCreating' not in fw_pods_status.stdout" + - "'Pending' not in fw_pods_status.stdout" + changed_when: false + + - name: check if all containers are running + shell: set -o pipefail && kubectl get pods -A | awk '! /Completed/ && NR != 1 { print $3 }' + args: + executable: /bin/bash + register: fw_containers_readiness + retries: 30 + delay: 10 + until: "'0' not in fw_containers_readiness.stdout" + changed_when: false + - name: check status of FW update - command: "kubectl get ethernetnodeconfig {{ node_name }} -n {{ intel_ethernet_operator_namespace }}" + command: "kubectl get ethernetnodeconfig {{ hostvars[node_name]['ansible_hostname'] }} -n {{ intel_ethernet_operator_namespace }}" register: after_fw_update_info - until: "'PostUpdateReboot' and 'InProgress' not in after_fw_update_info.stdout" + until: + - "'PostUpdateReboot' not in after_fw_update_info.stdout" + - "'InProgress' not in after_fw_update_info.stdout" retries: 30 delay: 10 - failed_when: "'Failed' in after_fw_update_info.stdout" + failed_when: "'Failed' in after_fw_update_info.stdout or 'PostUpdateReboot' in after_fw_update_info.stdout or 'InProgress' in after_fw_update_info.stdout" # noqa 204 line-length + changed_when: false when: "'PostUpdateReboot' in fw_update_info.stdout" - name: remove FW CR after update @@ -75,6 +106,7 @@ src: "{{ (intel_ethernet_operator_fw_files_dir, node_name + '-fw-update.yml') | path_join }}" - name: remove EthernetNodeConfig after update - command: kubectl delete enc {{ node_name }} -n {{ intel_ethernet_operator_namespace }} + command: kubectl delete enc {{ hostvars[node_name]['ansible_hostname'] }} -n {{ intel_ethernet_operator_namespace }} + changed_when: true when: hostvars[node_name]['intel_ethernet_operator']['fw_update'] diff --git a/roles/intel_ethernet_operator/tasks/main.yml b/roles/intel_ethernet_operator/tasks/main.yml index 5eda267b..ff3387e5 100644 --- a/roles/intel_ethernet_operator/tasks/main.yml +++ b/roles/intel_ethernet_operator/tasks/main.yml @@ -18,16 +18,6 @@ include_role: name: install_dependencies -- name: install Operator-SDK - include_tasks: operator_sdk.yml - when: inventory_hostname == groups['kube_control_plane'][0] - -- name: build Unified Flow Tool image - include_tasks: uft.yml - when: - - inventory_hostname == groups['kube_node'][0] - - intel_ethernet_operator_flow_config_enabled - - name: install Intel Ethernet Operator include_tasks: ethernet_operator.yml when: inventory_hostname == groups['kube_control_plane'][0] @@ -47,9 +37,3 @@ loop_var: node_name when: - inventory_hostname == groups['kube_control_plane'][0] - -- name: flow configuration - include_tasks: flow_config_deployment.yml - when: - - inventory_hostname == groups['kube_control_plane'][0] - - intel_ethernet_operator_flow_config_enabled diff --git a/roles/intel_ethernet_operator/tasks/operator_sdk.yml b/roles/intel_ethernet_operator/tasks/operator_sdk.yml deleted file mode 100644 index 40fc4267..00000000 --- a/roles/intel_ethernet_operator/tasks/operator_sdk.yml +++ /dev/null @@ -1,37 +0,0 @@ -## -## Copyright (c) 2020-2022 Intel Corporation. -## -## Licensed under the Apache License, Version 2.0 (the "License"); -## you may not use this file except in compliance with the License. -## You may obtain a copy of the License at -## -## http://www.apache.org/licenses/LICENSE-2.0 -## -## Unless required by applicable law or agreed to in writing, software -## distributed under the License is distributed on an "AS IS" BASIS, -## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -## See the License for the specific language governing permissions and -## limitations under the License. -## ---- -- name: clone Operator-SDK repository - git: - repo: "{{ operator_sdk_git }}" - version: "{{ operator_sdk_git_ref }}" - dest: "{{ operator_sdk_dir }}" - force: yes - -- name: install Operator-SDK - make: - target: "install" - chdir: "{{ operator_sdk_dir }}" - -- name: install Operator Lifecycle Manager - command: kubectl create -f "{{ item }}" # noqa 301 - loop: - - "https://raw.githubusercontent.com/operator-framework/operator-lifecycle-manager/v0.20.0/deploy/upstream/quickstart/crds.yaml" - - "https://raw.githubusercontent.com/operator-framework/operator-lifecycle-manager/v0.20.0/deploy/upstream/quickstart/olm.yaml" - -- name: wait for Operator Lifecycle Manager - pause: - minutes: 1 diff --git a/roles/intel_ethernet_operator/tasks/preflight_ethernet_operator.yml b/roles/intel_ethernet_operator/tasks/preflight_ethernet_operator.yml new file mode 100644 index 00000000..4526ea27 --- /dev/null +++ b/roles/intel_ethernet_operator/tasks/preflight_ethernet_operator.yml @@ -0,0 +1,81 @@ +## +## Copyright (c) 2020-2022 Intel Corporation. +## +## Licensed under the Apache License, Version 2.0 (the "License"); +## you may not use this file except in compliance with the License. +## You may obtain a copy of the License at +## +## http://www.apache.org/licenses/LICENSE-2.0 +## +## Unless required by applicable law or agreed to in writing, software +## distributed under the License is distributed on an "AS IS" BASIS, +## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +## See the License for the specific language governing permissions and +## limitations under the License. +## +- name: check Intel Ethernet Operator configuration + block: + - name: check CVL NICs + shell: "set -o pipefail && lshw -businfo -numeric -C network | grep -i {{ item.bus_info }} | awk -F ':' '{ print $NF}' | sed 's/.$//' " + args: + executable: /bin/bash + register: cvl_nics + changed_when: false + loop: "{{ dataplane_interfaces }}" + + - name: fail if unsupported CVL NIC is in use + fail: + msg: "Detected E822 or E823 device with Device ID {{ present_cvl_nics | intersect(cvl_822_823_nics_ids) }}. + Intel Ethernet Operator supports only E810 Series Devices" + when: "present_cvl_nics | intersect(cvl_822_823_nics_ids)" + vars: + present_cvl_nics: "{{ cvl_nics.results | json_query('[*].stdout') }}" + cvl_822_823_nics_ids: + - "124C" # Intel(R) Ethernet Connection E823-L for backplane + - "124D" # Intel(R) Ethernet Connection E823-L for SFP + - "124E" # Intel(R) Ethernet Connection E823-L/X557-AT 10GBASE-T + - "124F" # Intel(R) Ethernet Connection E823-L 1GbE + - "151D" # Intel(R) Ethernet Connection E823-L for QSFP + - "188A" # Intel(R) Ethernet Connection E823-C for backplane + - "188B" # Intel(R) Ethernet Connection E823-C for QSFP + - "188C" # Intel(R) Ethernet Connection E823-C for SFP + - "188D" # Intel(R) Ethernet Connection E823-C/X557-AT 10GBASE-T + - "188E" # Intel(R) Ethernet Connection E823-C 1GbE + - "1890" # Intel(R) Ethernet Connection E822-C for backplane + - "1891" # Intel(R) Ethernet Connection E822-C for QSFP + - "1892" # Intel(R) Ethernet Connection E822-C for SFP + - "1893" # Intel(R) Ethernet Connection E822-C/X557-AT 10GBASE-T + - "1894" # Intel(R) Ethernet Connection E822-C 1GbE + - "1897" # Intel(R) Ethernet Connection E822-L for backplane + - "1898" # Intel(R) Ethernet Connection E822-L for SFP + - "1899" # Intel(R) Ethernet Connection E822-L/X557-AT 10GBASE-T + - "189A" # Intel(R) Ethernet Connection E822-L 1GbE + - name: check if legacy DDP/FW update variables are false + assert: + that: + - not install_ddp_packages + - not update_nic_firmware + msg: + - "Intel Ethernet Operator is mutually exclusive with legacy DDP/FW update role." + - "Please set 'install_ddp_packages' and 'update_nic_firmware' as false" + when: + - (intel_ethernet_operator.ddp_update is defined and intel_ethernet_operator.ddp_update) or + (intel_ethernet_operator.fw_update is defined and intel_ethernet_operator.fw_update) + + - name: check if ice driver will be updated + assert: + that: update_nic_drivers + msg: "Firmware update requires update_nic_drivers set as true" + when: intel_ethernet_operator.fw_update is defined and intel_ethernet_operator.fw_update + + - name: check Hugepages settings for Flow Configuration + assert: + that: number_of_hugepages_2M >= 1024 + msg: "Flow Configuration requires at least 2Gi of 2M Hugepages." + when: dataplane_interfaces | json_query('[?flow_configuration]') + + - name: check if SRIOV Network Operator is enabled + assert: + that: sriov_network_operator_enabled + msg: "Flow Configuration requires SRIOV Network Operator to be enabled." + when: dataplane_interfaces | json_query('[?flow_configuration]') diff --git a/roles/intel_flexran/defaults/main.yml b/roles/intel_flexran/defaults/main.yml new file mode 100644 index 00000000..e49e98fb --- /dev/null +++ b/roles/intel_flexran/defaults/main.yml @@ -0,0 +1,57 @@ +## +## Copyright (c) 2020-2022 Intel Corporation. +## +## Licensed under the Apache License, Version 2.0 (the "License"); +## you may not use this file except in compliance with the License. +## You may obtain a copy of the License at +## +## http://www.apache.org/licenses/LICENSE-2.0 +## +## Unless required by applicable law or agreed to in writing, software +## distributed under the License is distributed on an "AS IS" BASIS, +## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +## See the License for the specific language governing permissions and +## limitations under the License. +## +--- +# References: WIKI, RDC +# FEC = Forward Error Correction +# CR = Custom Resource +# ACC100 = Intel vRAN Dedicated H/W Accelerator Card + +# - include_role: dpdk +# when: dpdk_dir is not defined # to allow tagged execution + +# Intel FlexRAN +# intel_flexran_repo: "not public" +# intel_flexran_token: "pkg requires private access from Intel’s Developer Zone Portal" +# intel_flexran_staging_location: "/tmp/flexran/" # a directory on localhost (ansible host) +intel_flexran_ver: "22.03" +# intel_flexran_tarball: "FlexRAN-22.03.tar.gz" +# intel_flexran_tar_chk: "65e59ac1295ef392f54b80047db2efe458962fc78e5d84c5d54703439a364cda" # SHA256 +intel_flexran_dir: "{{ (project_root_dir, 'intel-flexran') | path_join }}" +intel_flexran_files_dir: "{{ (project_root_dir, 'intel-flexran-files') | path_join }}" # for ACC100 CRs, kernel cmdline, etc +intel_flexran_dpdk_ver: "21.11" +# intel_flexran_dpdk_dir: "{{ dpdk_dir }}" # as defined in host_vars +intel_flexran_dpdk_dir: "{{ (project_root_dir, 'dpdk-' + intel_flexran_dpdk_ver) | path_join }}" +# intel_flexran_dpdk_zip: "dpdk_patch-{{ intel_flexran_ver }}.patch.zip" +# intel_flexran_dpdk_zip_chk: "8870b139a3f7fbbd2f0bee1aeaeeb5e0a08fb4745b4e183bf4c9119e5d2dcdaa" # SHA256 +intel_flexran_dpdk_patch: "dpdk_patch-22.03.patch" +intel_flexran_dpdk_patch_chk: "4556ba6e5ac32d0360c0e5c71ba7fa22a6065f9e608a35e1c945691f7dfd7fe4" # SHA256 + +# Intel oneAPI Base Toolkit +# Reference: https://www.intel.com/content/www/us/en/developer/tools/oneapi/base-toolkit-download.html +intel_oneapi_ver: "2022.1.2.146" +intel_oneapi_url: "https://registrationcenter-download.intel.com/akdlm/irc_nas/18487/l_BaseKit_p_2022.1.2.146_offline.sh" +intel_oneapi_chk: "91682e4410c17a82147ce574c30e57271cc12adfab198c8547612f13d4dd21c8d77ce12153d29b3774bc27f0c6b604cd" # SHA384 +intel_oneapi_dir: "{{ (project_root_dir, 'intel-oneapi') | path_join }}" + +# Newer Release Date: April 05, 2022 +# intel_oneapi_ver: "2022.2" +# intel_oneapi_url: "https://registrationcenter-download.intel.com/akdlm/irc_nas/18673/l_BaseKit_p_2022.2.0.262_offline.sh" +# intel_oneapi_chk: "e508b0a64f048d9518cc3706e1fa3f400dbb0a07fdc0f91e02b371b18a35715fa0fad7a960dbb7fc04595f77ae65a333" # SHA384 + +# Intel ACC100 FEC CR (Mount Bryce) +# intel_acc100_cr: "acc100-cr.yaml" +# fec_acc_node: "{{ groups['kube_node'][0] }}" +# fec_acc_dev: "0000:49:00.0" # must be string in [a-fA-F0-9]{4}:[a-fA-F0-9]{2}:[01][a-fA-F0-9].[0-7] format diff --git a/roles/intel_flexran/files/kernel_cmdline_gen.sh b/roles/intel_flexran/files/kernel_cmdline_gen.sh new file mode 100644 index 00000000..520aec32 --- /dev/null +++ b/roles/intel_flexran/files/kernel_cmdline_gen.sh @@ -0,0 +1,38 @@ +#! /bin/bash + +threads_per_core=$(lscpu | grep "Thread(s) per core" | awk -F ':' '{print $2}' | xargs) +cores_per_socket=$(lscpu | grep "Core(s) per socket" | awk -F ':' '{print $2}' | xargs) +numa=$(lscpu | grep "NUMA node(s)" |awk -F ':' '{print $2}' | xargs) + +# if cores_per_socket < 2, which means no cores available to isolate for realtime app. +# and isolcpus will results in 1-0, keep it here for test purpose. + +# On socket 0, core 0 and its sibling thread core will be kept for housekeeping +# all the other cores will be isolated +# No cores isolated from socket 1. +# NUMA node0 CPU(s): 0-27,56-83 +# NUMA node1 CPU(s): 28-55,84-111 +if [ "$numa" == "1" ] ; then + if [ "$threads_per_core" == "2" ] ; then + isolcpus="1-$(( cores_per_socket - 1 )),$(( cores_per_socket + 1 ))-$(( cores_per_socket * 2 - 1 ))" + housekeeping="0,$cores_per_socket" + + else + isolcpus="1-$(( cores_per_socket - 1 ))" + housekeeping="0" + fi +elif [ "$numa" == "2" ]; then + if [ "$threads_per_core" == "2" ] ; then + isolcpus="1-$(( cores_per_socket - 1 )),$(( cores_per_socket * 2 + 1 ))-$(( cores_per_socket * 3 - 1 ))" + housekeeping="0,$(( cores_per_socket * 2 )),$(( cores_per_socket ))-$(( cores_per_socket * 2 - 1 )),$(( cores_per_socket * 3 ))-$(( cores_per_socket * 4 - 1 ))" + + else + + isolcpus="1-$(( cores_per_socket - 1 ))" + housekeeping="0,$(( cores_per_socket ))-$(( cores_per_socket * 2 - 1 ))" + fi +fi + +flexran_kernel_cmdline="intel_iommu=on iommu=pt usbcore.autosuspend=-1 selinux=0 enforcing=0 nmi_watchdog=0 crashkernel=auto softlockup_panic=0 audit=0 cgroup_disable=memory tsc=nowatchdog intel_pstate=disable mce=off hugepagesz=1G hugepages=40 hugepagesz=2M hugepages=0 default_hugepagesz=1G kthread_cpus=$housekeeping irqaffinity=$housekeeping nohz=on nosoftlockup nohz_full=$isolcpus rcu_nocbs=$isolcpus rcu_nocb_poll skew_tick=1 isolcpus=$isolcpus" + +echo "$flexran_kernel_cmdline" diff --git a/roles/intel_flexran/files/spr-bm-fcp-2x40 b/roles/intel_flexran/files/spr-bm-fcp-2x40 new file mode 100644 index 00000000..d2ab7f26 --- /dev/null +++ b/roles/intel_flexran/files/spr-bm-fcp-2x40 @@ -0,0 +1 @@ +intel_iommu=on iommu=pt usbcore.autosuspend=-1 selinux=0 enforcing=0 nmi_watchdog=0 crashkernel=auto softlockup_panic=0 audit=0 cgroup_disable=memory tsc=nowatchdog intel_pstate=disable mce=off hugepagesz=1G hugepages=40 hugepagesz=2M hugepages=0 default_hugepagesz=1G kthread_cpus=0,80,40-79,120-159 irqaffinity=0,80,40-79,120-159 nohz=on nosoftlockup nohz_full=1-39,81-119 rcu_nocbs=1-39,81-119 rcu_nocb_poll skew_tick=1 isolcpus=1-39,81-119 \ No newline at end of file diff --git a/roles/intel_flexran/files/spr-bm-q-mcc-1x32 b/roles/intel_flexran/files/spr-bm-q-mcc-1x32 new file mode 100644 index 00000000..dbd864f2 --- /dev/null +++ b/roles/intel_flexran/files/spr-bm-q-mcc-1x32 @@ -0,0 +1 @@ +intel_iommu=on iommu=pt usbcore.autosuspend=-1 selinux=0 enforcing=0 nmi_watchdog=0 softlockup_panic=0 audit=0 cgroup_disable=memory mce=off hugepagesz=1G hugepages=40 hugepagesz=2M hugepages=0 default_hugepagesz=1G kthread_cpus=0,30-32,62-63 irqaffinity=0,30-32,62-63 nosoftlockup skew_tick=1 skew_tick=1 isolcpus=1-29,33-61 nohz_full=1-29,33-61 rcu_nocbs=1-29,33-61 \ No newline at end of file diff --git a/roles/intel_flexran/files/spr-bm-q-xcc-2x52 b/roles/intel_flexran/files/spr-bm-q-xcc-2x52 new file mode 100644 index 00000000..61641a81 --- /dev/null +++ b/roles/intel_flexran/files/spr-bm-q-xcc-2x52 @@ -0,0 +1 @@ +intel_iommu=on iommu=pt default_hugepagesz=1G hugepagesz=1G hugepages=60 irqaffinity=0,50-52,103-104,154-156,206-207 mce=off nmi_watchdog=0 softlockup_panic=0 selinux=0 enforcing=0 audit=0 kthread_cpus=0,50-52,103-104,154-156,206-207 clock=pit no_timer_check clocksource=tsc tsc=perfect usbcore.autosuspend=-1 pci=realloc pci=assign-busses rdt=l3cat skew_tick=1 isolcpus=managed_irq,domain,1-49,53-101,105-153,157-205 intel_pstate=disable nosoftlockup tsc=nowatchdog nohz=on nohz_full=1-49,53-101,105-153,157-205 rcu_nocbs=1-49,53-101,105-153,157-205 \ No newline at end of file diff --git a/roles/intel_sriov_fec_operator/tasks/olm.yml b/roles/intel_flexran/tasks/cleanup_flexran.yml similarity index 68% rename from roles/intel_sriov_fec_operator/tasks/olm.yml rename to roles/intel_flexran/tasks/cleanup_flexran.yml index 9f6cb413..a15ced12 100644 --- a/roles/intel_sriov_fec_operator/tasks/olm.yml +++ b/roles/intel_flexran/tasks/cleanup_flexran.yml @@ -14,9 +14,4 @@ ## limitations under the License. ## --- -- name: install Operator Lifecycle Manager (OLM) - command: operator-sdk olm install # noqa 301 - environment: - PATH: "{{ gopath.stdout }}/bin:/usr/local/go/bin:/usr/sbin:/usr/bin:/sbin:/bin:{{ operator_sdk_dir }}" -# ignore_errors: true -# install re-run fails unless uninstalled first +- debug: msg="FlexRAN cleanup TBD" diff --git a/roles/intel_flexran/tasks/fec_acc.yml b/roles/intel_flexran/tasks/fec_acc.yml new file mode 100644 index 00000000..c0dd5b00 --- /dev/null +++ b/roles/intel_flexran/tasks/fec_acc.yml @@ -0,0 +1,24 @@ +## +## Copyright (c) 2020-2022 Intel Corporation. +## +## Licensed under the Apache License, Version 2.0 (the "License"); +## you may not use this file except in compliance with the License. +## You may obtain a copy of the License at +## +## http://www.apache.org/licenses/LICENSE-2.0 +## +## Unless required by applicable law or agreed to in writing, software +## distributed under the License is distributed on an "AS IS" BASIS, +## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +## See the License for the specific language governing permissions and +## limitations under the License. +## +--- +- name: configure vRAN FEC Acc H/W with pf_bb tool + shell: "./pf_bb_config ACC100 -c ./acc100/acc100_config_pf_4g5g.cfg" # noqa 305 + args: + chdir: "{{ (intel_flexran_dir, 'source/pf-bb-config') | path_join }}" + register: pf_bb_config + changed_when: false + +- debug: msg="{{ pf_bb_config.stdout }}" diff --git a/roles/intel_flexran/tasks/flexran.yml b/roles/intel_flexran/tasks/flexran.yml new file mode 100644 index 00000000..574856cd --- /dev/null +++ b/roles/intel_flexran/tasks/flexran.yml @@ -0,0 +1,64 @@ +## +## Copyright (c) 2020-2022 Intel Corporation. +## +## Licensed under the Apache License, Version 2.0 (the "License"); +## you may not use this file except in compliance with the License. +## You may obtain a copy of the License at +## +## http://www.apache.org/licenses/LICENSE-2.0 +## +## Unless required by applicable law or agreed to in writing, software +## distributed under the License is distributed on an "AS IS" BASIS, +## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +## See the License for the specific language governing permissions and +## limitations under the License. +## +--- +# - name: create Intel FlexRAN directory on worker node +# file: +# path: "{{ intel_flexran_dir }}" +# state: directory +# mode: '0755' + +# - name: unpack Intel FlexRAN tarball on worker node +# unarchive: +# src: "{{ (intel_flexran_staging_location, intel_flexran_tarball) | path_join }}" +# dest: "{{ intel_flexran_dir }}" +# mode: '0755' + +# - name: extract Intel FlexRAN +# shell: "echo | ./extract.sh" # noqa 301 +# args: +# chdir: "{{ intel_flexran_dir }}" + +- name: set DPDK path for Intel FlexRAN + copy: + dest: "{{ (intel_flexran_dir, '.flexran_dpdk.path') | path_join }}" + content: "{{ intel_flexran_dpdk_dir }}" + mode: '0755' + +- name: set oneAPI path for Intel FlexRAN + copy: + dest: "{{ (intel_flexran_dir, '.flexran_icx.path') | path_join }}" + content: "{{ intel_oneapi_dir }}" + mode: '0755' + +- name: build Intel FlexRAN SDK + shell: "source set_env_var.sh -d && ./flexran_build.sh -e -r 5gnr -m sdk" # noqa 305 + args: + executable: /bin/bash + chdir: "{{ intel_flexran_dir }}" + changed_when: false + +# - name: build DPDK # all this presumably already done by DPDK role. need to re-run after building sdk above?! +# shell: "source set_env_var.sh -d && cd $RTE_SDK && meson build && cd build && meson configure && ninja && ninja install" # noqa 305 +# args: +# executable: /bin/bash +# chdir: "{{ intel_flexran_dir }}" + +- name: build FlexRAN ALL for 5GNR + shell: "source set_env_var.sh -d && ./flexran_build.sh -e -r 5gnr" # noqa 305 + args: + executable: /bin/bash + chdir: "{{ intel_flexran_dir }}" + changed_when: false diff --git a/roles/intel_flexran/tasks/flexran_preflight.yml b/roles/intel_flexran/tasks/flexran_preflight.yml new file mode 100644 index 00000000..0df80dbd --- /dev/null +++ b/roles/intel_flexran/tasks/flexran_preflight.yml @@ -0,0 +1,105 @@ +## +## Copyright (c) 2020-2022 Intel Corporation. +## +## Licensed under the Apache License, Version 2.0 (the "License"); +## you may not use this file except in compliance with the License. +## You may obtain a copy of the License at +## +## http://www.apache.org/licenses/LICENSE-2.0 +## +## Unless required by applicable law or agreed to in writing, software +## distributed under the License is distributed on an "AS IS" BASIS, +## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +## See the License for the specific language governing permissions and +## limitations under the License. +## +--- +- block: +# - name: load FlexRAN vars +# include_vars: "../roles/intel_flexran/defaults/main.yml" + + # check CPU for FlexRAN + - debug: msg="CPU={{ ansible_processor[2] }} cores={{ ansible_processor_cores }} count={{ ansible_processor_count }} nproc={{ ansible_processor_nproc }} tpc={{ ansible_processor_threads_per_core }} vcpus={{ ansible_processor_vcpus }}" # noqa 204 line-length + - name: check CPU for FlexRAN + assert: + that: "ansible_processor_count == 1 and ansible_processor_cores == 32" + msg: "Intel FlexRAN proper configuration requires worker with single 32-cores SPR CPU. Deployment may proceed but is unsupported" + failed_when: false + + # check o/s for FlexRAN + - debug: msg="Linux distribution on target is {{ ansible_distribution }} {{ ansible_distribution_version }} ({{ ansible_distribution_release }}) with {{ ansible_kernel }} kernel" # noqa 204 line-length + - name: check linux distro version and kernel for FlexRAN + assert: + that: "ansible_distribution == 'Ubuntu' and ansible_distribution_version == '22.04' and 'realtime' in ansible_kernel" + msg: + - Deploying Intel FlexRAN is supported only on Ubuntu 22.04 with realtime kernel. + - Please prepare accordingly the o/s image on target or disable FlexRAN. See docs/flexran_guide.md + + # check package for FlexRAN + - debug: msg="Expecting subfolders inside '{{ intel_flexran_dir }}' on worker node" + + - name: probe for FlexRAN extraction + stat: + path: "{{ item }}" + register: flexran_dir_stats + with_items: + - "{{ (intel_flexran_dir, 'bin') | path_join }}" + - "{{ (intel_flexran_dir, 'framework') | path_join }}" + - "{{ (intel_flexran_dir, 'sdk') | path_join }}" + - "{{ (intel_flexran_dir, 'source') | path_join }}" + - "{{ (intel_flexran_dir, 'tests') | path_join }}" + - "{{ (intel_flexran_dir, 'xran') | path_join }}" + + - name: check FlexRAN folders + assert: + that: "item.stat.exists and item.stat.isdir" + msg: + - Directory '{{ item.item }}' is missing on target '{{ inventory_hostname }}' + - Deploying Intel FlexRAN requires the tarball package to be pre-extracted on the worker node. See docs/flexran_guide.md + with_items: "{{ flexran_dir_stats.results }}" + + # check DPDK patch for FlexRAN + - debug: msg="Expecting file {{ (dpdk_local_patches_dir, 'dpdk-' + dpdk_version, intel_flexran_dpdk_patch) | path_join }} on local ansible host" + + - name: probe for FlexRAN DPDK patch + delegate_to: localhost + stat: + path: "{{ (dpdk_local_patches_dir, 'dpdk-' + dpdk_version, intel_flexran_dpdk_patch) | path_join }}" + checksum_algorithm: sha256 + register: provided_flexran_dpdk_patch + + - debug: msg="{{ intel_flexran_dpdk_patch }} exists is {{ provided_flexran_dpdk_patch.stat.exists }}" + + - name: check the FlexRAN DPDK patch name + assert: + that: "provided_flexran_dpdk_patch.stat.exists" + msg: + - Mandatory file {{ (dpdk_local_patches_dir, 'dpdk-' + dpdk_version, intel_flexran_dpdk_patch) | path_join }} does NOT exist on localhost. + - Please acquire the DPDK patch.zip and unzip it in the location indicated above in order to deploy FlexRAN. See docs/flexran_guide.md + + - debug: msg="{{ intel_flexran_dpdk_patch }} checksum is {{ provided_flexran_dpdk_patch.stat.checksum }}" + + - name: check the FlexRAN DPDK patch integrity + assert: + that: "provided_flexran_dpdk_patch.stat.checksum == '{{ intel_flexran_dpdk_patch_chk }}'" + msg: + - File {{ (dpdk_local_patches_dir, 'dpdk-' + dpdk_version, intel_flexran_dpdk_patch) | path_join }} on localhost is NOT the expected one. + - Please provide the correct file. See docs/flexran_guide.md + + # check DPDK for FlexRAN + - name: check DPDK is enabled for FlexRAN + assert: + that: install_dpdk + msg: "DPDK installation is required for FlexRAN. Please make sure install_dpdk is true in the worker node host_vars file" + + - debug: msg="DPDK version is set to '{{ dpdk_version }}'" + + - name: check DPDK version for FlexRAN + assert: + that: dpdk_version == intel_flexran_dpdk_ver + msg: + - DPDK version '{{ dpdk_version }}' set in the worker node host_vars file does NOT match the DPDK version required for FlexRAN. + - Must be '{{ intel_flexran_dpdk_ver }}' + + when: + - intel_flexran_enabled | default(false) | bool diff --git a/roles/intel_flexran/tasks/main.yml b/roles/intel_flexran/tasks/main.yml new file mode 100644 index 00000000..83e62bb9 --- /dev/null +++ b/roles/intel_flexran/tasks/main.yml @@ -0,0 +1,49 @@ +## +## Copyright (c) 2020-2022 Intel Corporation. +## +## Licensed under the Apache License, Version 2.0 (the "License"); +## you may not use this file except in compliance with the License. +## You may obtain a copy of the License at +## +## http://www.apache.org/licenses/LICENSE-2.0 +## +## Unless required by applicable law or agreed to in writing, software +## distributed under the License is distributed on an "AS IS" BASIS, +## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +## See the License for the specific language governing permissions and +## limitations under the License. +## +--- +- name: preflight check for Intel FlexRAN + include_tasks: flexran_preflight.yml + when: inventory_hostname == groups['kube_node'][0] + +- name: install dependencies for Intel FlexRAN + include_role: + name: install_dependencies + +- name: deploy Intel oneAPI + include_tasks: oneapi.yml + when: inventory_hostname == groups['kube_node'][0] + +- name: deploy Intel FlexRAN + include_tasks: flexran.yml + when: inventory_hostname == groups['kube_node'][0] + +- name: deploy Intel pf_bb (Physical Function Baseband) device config app + include_tasks: pf_bb.yml + when: inventory_hostname == groups['kube_node'][0] + +- name: configure FEC Accelerator Device + include_tasks: fec_acc.yml + loop: "{{ groups['kube_node'] }}" + loop_control: + loop_var: node_name + when: + - inventory_hostname == groups['kube_node'][0] + - fec_acc is defined +# TODO: multi-host multi-acc + +- name: test Timer Mode + include_tasks: timer_mode.yml + when: inventory_hostname == groups['kube_node'][0] diff --git a/roles/intel_flexran/tasks/oneapi.yml b/roles/intel_flexran/tasks/oneapi.yml new file mode 100644 index 00000000..10369ecd --- /dev/null +++ b/roles/intel_flexran/tasks/oneapi.yml @@ -0,0 +1,43 @@ +## +## Copyright (c) 2020-2022 Intel Corporation. +## +## Licensed under the Apache License, Version 2.0 (the "License"); +## you may not use this file except in compliance with the License. +## You may obtain a copy of the License at +## +## http://www.apache.org/licenses/LICENSE-2.0 +## +## Unless required by applicable law or agreed to in writing, software +## distributed under the License is distributed on an "AS IS" BASIS, +## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +## See the License for the specific language governing permissions and +## limitations under the License. +## +--- +- name: create Intel oneAPI directory + file: + path: "{{ intel_oneapi_dir }}" + state: directory + mode: '0755' + +- name: download Intel oneAPI + get_url: + url: "{{ intel_oneapi_url }}" + dest: "{{ (intel_oneapi_dir, 'intel-oneapi-basekit-offline.sh') | path_join }}" + checksum: "sha384:{{ intel_oneapi_chk }}" + mode: '0755' + use_proxy: yes + +- name: create libnuma symlink + file: + src: "/usr/lib/x86_64-linux-gnu/libnuma.so" + dest: "/usr/lib64/libnuma.so" + state: link +# ln -s /usr/lib/x86_64-linux-gnu/libnuma.so /usr/lib64/libnuma.so + +- name: install Intel oneAPI + command: "sh {{ intel_oneapi_dir }}/intel-oneapi-basekit-offline.sh -a --silent --eula accept --install-dir {{ intel_oneapi_dir }}" + changed_when: false + failed_when: false # to allow re-run install without uninstall +# environment: +# PATH: "{{ gopath.stdout }}/bin:/usr/local/go/bin:/usr/sbin:/usr/bin:/sbin:/bin:{{ intel_oneapi_dir }}" diff --git a/roles/intel_flexran/tasks/pf_bb.yml b/roles/intel_flexran/tasks/pf_bb.yml new file mode 100644 index 00000000..6a2030ca --- /dev/null +++ b/roles/intel_flexran/tasks/pf_bb.yml @@ -0,0 +1,103 @@ +## +## Copyright (c) 2020-2022 Intel Corporation. +## +## Licensed under the Apache License, Version 2.0 (the "License"); +## you may not use this file except in compliance with the License. +## You may obtain a copy of the License at +## +## http://www.apache.org/licenses/LICENSE-2.0 +## +## Unless required by applicable law or agreed to in writing, software +## distributed under the License is distributed on an "AS IS" BASIS, +## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +## See the License for the specific language governing permissions and +## limitations under the License. +## +--- +- name: check igb_uio module is loaded + shell: "set -o pipefail && lsmod | grep -i igb_uio" + args: + executable: /bin/bash + register: igb_uio_module + changed_when: false + failed_when: false + +- name: insert igb_uio module + block: + - name: clone DPDK-KMODS repository + git: + repo: "http://dpdk.org/git/dpdk-kmods" + version: "e68a705cc5dc3d1333bbcd722fe4e9a6ba3ee648" # latest as of July 2022 + dest: "{{ (intel_flexran_dpdk_dir, 'dpdk-kmods') | path_join }}" + force: yes + +# - name: build igb_uio +# make: +# chdir: "{{ (intel_flexran_dpdk_dir, 'dpdk-kmods/linux/igb_uio') | path_join }}" + +# - name: build igb_uio with command +# command: make +# args: +# chdir: "{{ (intel_flexran_dpdk_dir, 'dpdk-kmods/linux/igb_uio') | path_join }}" + + # Using shell since the make built-in module and command (above) both get errors + - name: build igb_uio with shell + shell: "make" # noqa 305 + args: + executable: /bin/bash + chdir: "{{ (intel_flexran_dpdk_dir, 'dpdk-kmods/linux/igb_uio') | path_join }}" + changed_when: true + + - name: load uio + modprobe: + name: uio + state: present + + - name: load igb_uio + command: insmod igb_uio.ko + args: + chdir: "{{ (intel_flexran_dpdk_dir, 'dpdk-kmods/linux/igb_uio') | path_join }}" + changed_when: false + failed_when: false + + when: "'igb_uio' not in igb_uio_module.stdout" + +- debug: msg="fec_acc pciid is {{ fec_acc | regex_replace('^ip-', '') }}" + +- name: bind ACC card + shell: "./dpdk-devbind.py -b igb_uio {{ fec_acc }} && ./dpdk-devbind.py -s" # noqa 305 + args: + executable: /bin/bash + chdir: "{{ (intel_flexran_dpdk_dir, 'usertools') | path_join }}" + changed_when: false + failed_when: false + +- name: clone inih repository + git: + repo: "https://github.com/benhoyt/inih" + version: "r44" + dest: "{{ (intel_flexran_dir, 'source/inih') | path_join }}" + force: yes + +- name: build inih + make: + file: "Makefile.static" + chdir: "{{ (intel_flexran_dir, 'source/inih/extra') | path_join }}" + +- name: copy libinih.a + copy: + remote_src: yes + src: "{{ (intel_flexran_dir, 'source/inih/extra', 'libinih.a') | path_join }}" + dest: "{{ (intel_flexran_dir, 'source/inih/') | path_join }}" + mode: '0755' + +- name: clone pf-bb-config repository + git: + repo: "https://github.com/intel/pf-bb-config.git" + version: "v22.03" # may need to lock on tag or commit + dest: "{{ (intel_flexran_dir, 'source/pf-bb-config') | path_join }}" + force: yes + +- name: build pf-bb-config tool + make: + chdir: "{{ (intel_flexran_dir, 'source/pf-bb-config') | path_join }}" diff --git a/roles/intel_flexran/tasks/timer_mode.yml b/roles/intel_flexran/tasks/timer_mode.yml new file mode 100644 index 00000000..7fd03e4c --- /dev/null +++ b/roles/intel_flexran/tasks/timer_mode.yml @@ -0,0 +1,90 @@ +## +## Copyright (c) 2020-2022 Intel Corporation. +## +## Licensed under the Apache License, Version 2.0 (the "License"); +## you may not use this file except in compliance with the License. +## You may obtain a copy of the License at +## +## http://www.apache.org/licenses/LICENSE-2.0 +## +## Unless required by applicable law or agreed to in writing, software +## distributed under the License is distributed on an "AS IS" BASIS, +## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +## See the License for the specific language governing permissions and +## limitations under the License. +## +--- +- name: enable h/w FEC mode + lineinfile: + path: "{{ (intel_flexran_dir, 'bin/nr5g/gnb/l1/phycfg_timer.xml') | path_join }}" + search_string: 'dpdkBasebandFecMode' + line: " 1" + +- name: set h/w FEC device + lineinfile: + path: "{{ (intel_flexran_dir, 'bin/nr5g/gnb/l1/phycfg_timer.xml') | path_join }}" + search_string: 'dpdkBasebandDevice' + line: "{{ ' ' + fec_acc + '' }}" + +- name: check dpdkBaseband config + shell: "grep dpdkBaseband phycfg_timer.xml" # noqa 305 + args: + chdir: "{{ (intel_flexran_dir, 'bin/nr5g/gnb/l1') | path_join }}" + register: phycfg_timer_mode + changed_when: false + +- debug: msg="{{ phycfg_timer_mode.stdout }}" + +- name: check FEC Acc devices + shell: "set -o pipefail && lspci | grep -i acc" + args: + executable: /bin/bash + register: fec_acc_devs + changed_when: false + +- debug: msg={{ fec_acc_devs.stdout }} + +- debug: + msg: + - "Intel FlexRAN deployment is complete and Timer Mode configuration is done." + - "The worker node is ready for L1/L2 Tests to be executed and verified according to the Guide" + +# - name: run L1 +# shell: "source set_env_var.sh -d && cd {{ (intel_flexran_dir, 'bin/nr5g/gnb/l1') | path_join }} && ./l1.sh -e" # noqa 305 +# args: +# executable: /bin/bash +# chdir: "{{ intel_flexran_dir }}" +# async: 150 +# poll: 0 +# register: l1 + +# - name: wait l1 ready +# pause: +# seconds: 30 + +# - name: run L2 +# shell: "source set_env_var.sh -d && cd {{ (intel_flexran_dir, 'bin/nr5g/gnb/testmac') | path_join }} && ./l2.sh --testfile=icelake-sp/icxsp_mu0_10mhz_4x4_hton.cfg" # noqa 204 line-length +# args: +# executable: /bin/bash +# chdir: "{{ intel_flexran_dir }}" +# async: 120 +# poll: 0 +# register: l2 + +# - name: wait l2 finish +# pause: +# seconds: 150 + +# - name: dump l1 result +# async_status: jid="{{ l1.ansible_job_id }}" +# register: l1_result + +# - debug: msg="{{ l1_result.stdout }}" +# failed_when: false + +# - name: dump l2 result +# async_status: jid="{{ l2.ansible_job_id }}" +# register: l2_result + +# - debug: msg="{{ l2_result.stdout }}" +# failed_when: false diff --git a/roles/intel_flexran/vars/main.yml b/roles/intel_flexran/vars/main.yml new file mode 100644 index 00000000..31d7fda9 --- /dev/null +++ b/roles/intel_flexran/vars/main.yml @@ -0,0 +1,35 @@ +## +## Copyright (c) 2020-2022 Intel Corporation. +## +## Licensed under the Apache License, Version 2.0 (the "License"); +## you may not use this file except in compliance with the License. +## You may obtain a copy of the License at +## +## http://www.apache.org/licenses/LICENSE-2.0 +## +## Unless required by applicable law or agreed to in writing, software +## distributed under the License is distributed on an "AS IS" BASIS, +## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +## See the License for the specific language governing permissions and +## limitations under the License. +## +--- +install_dependencies: + Debian: + - git + - linux-headers-{{ ansible_kernel }} + - xdg-utils + - libnotify4 + - libgbm1 + - libnuma-dev + - libhugetlbfs-dev + - build-essential + - cmake + - meson + - pkgconf + - python3-pyelftools + - flex + - bison + RedHat: + - git + - make diff --git a/roles/intel_sriov_fec_operator/defaults/main.yml b/roles/intel_sriov_fec_operator/defaults/main.yml index a4f47729..b74fb524 100644 --- a/roles/intel_sriov_fec_operator/defaults/main.yml +++ b/roles/intel_sriov_fec_operator/defaults/main.yml @@ -34,12 +34,7 @@ intel_sriov_fec_operator_catalog_image: "{{ registry_local_address }}/n3000-oper # Intel ACC100 FEC CR (Mount Bryce) intel_acc100_cr: "acc100-cr.yaml" fec_acc_node: "{{ groups['kube_node'][0] }}" -fec_acc_dev: "0000:49:00.0" # must be string in [a-fA-F0-9]{4}:[a-fA-F0-9]{2}:[01][a-fA-F0-9].[0-7] format - -# Operator SDK / Lifecycle Manager (OLM) -operator_sdk_git: "https://github.com/operator-framework/operator-sdk.git" -operator_sdk_git_ref: "v1.20.0" -operator_sdk_dir: "{{ (project_root_dir, 'operator-sdk') | path_join }}" +fec_acc_dev: "{{ fec_acc }}" # defined in host_vars # Operator Package Manager (OPM) opm_url: "https://github.com/operator-framework/operator-registry/releases/download/{{ opm_ver }}/linux-amd64-opm" diff --git a/roles/intel_sriov_fec_operator/tasks/check_sriov_fec_operator.yml b/roles/intel_sriov_fec_operator/tasks/check_sriov_fec_operator.yml index 88cf959f..c130a167 100644 --- a/roles/intel_sriov_fec_operator/tasks/check_sriov_fec_operator.yml +++ b/roles/intel_sriov_fec_operator/tasks/check_sriov_fec_operator.yml @@ -25,7 +25,7 @@ register: fec_operator_pods - debug: msg={{ fec_operator_pods.stdout }} - + - name: check csv in OLM namespace (expected 1 Succeeded) command: kubectl get csv -n olm # noqa 301 register: olm_csv @@ -43,3 +43,14 @@ register: fec_operator_node - debug: msg={{ fec_operator_node.stdout }} + +- name: check FEC Acc devices on worker node (expected 1 PF + 16 VFs) + delegate_to: "{{ groups['kube_node'][0] }}" + shell: "set -o pipefail && lspci | grep -i acc" + args: + executable: /bin/bash + register: fec_operator_devs + changed_when: false + failed_when: false + +- debug: msg={{ fec_operator_devs.stdout }} diff --git a/roles/intel_sriov_fec_operator/tasks/main.yml b/roles/intel_sriov_fec_operator/tasks/main.yml index 176a2237..87ae0fed 100644 --- a/roles/intel_sriov_fec_operator/tasks/main.yml +++ b/roles/intel_sriov_fec_operator/tasks/main.yml @@ -18,13 +18,10 @@ include_role: name: install_dependencies -- name: deploy Operator SDK - include_tasks: operator_sdk.yml - when: inventory_hostname == groups['kube_control_plane'][0] - -- name: deploy Lifecycle Manager (OLM) - include_tasks: olm.yml - when: inventory_hostname == groups['kube_control_plane'][0] +- name: set FEC Acc for Intel SRIOV FEC Operator + delegate_to: localhost + include_vars: + file: "{{ (inventory_dir, 'host_vars', groups['kube_node'][0] + '.yml') | path_join }}" - name: deploy Operator Package Manager (OPM) include_tasks: opm.yml diff --git a/roles/intel_sriov_fec_operator/tasks/opm.yml b/roles/intel_sriov_fec_operator/tasks/opm.yml index 8a231630..e121f687 100644 --- a/roles/intel_sriov_fec_operator/tasks/opm.yml +++ b/roles/intel_sriov_fec_operator/tasks/opm.yml @@ -14,8 +14,9 @@ ## limitations under the License. ## --- -- get_url: - url: "{{ opm_url }}" +- name: get OPM from URL + get_url: + url: "{{ opm_url }}" dest: "{{ (opm_dir, opm_cmd) | path_join }}" checksum: "sha256:{{ opm_chk }}" mode: '0755' diff --git a/roles/minio_install/files/main.py b/roles/minio_install/files/main.py index 97ecd348..4d2c3e29 100755 --- a/roles/minio_install/files/main.py +++ b/roles/minio_install/files/main.py @@ -23,11 +23,8 @@ def tr(s): def generate_k8s_service_patch(): # define text file to open - my_file = open('parsed-ips-result.txt', 'r') - - # read text file into list - # data = [word.split(',') for word in open('parsed-ips-result.txt', 'r').readlines()] - data = my_file.read() + with open('parsed-ips-result.txt', 'r') as my_file: + data = my_file.read() # display content of text file ips = [] diff --git a/roles/minio_install/tasks/preflight_minio_config.yml b/roles/minio_install/tasks/preflight_minio_config.yml index 1d6d21bc..56d09d02 100644 --- a/roles/minio_install/tasks/preflight_minio_config.yml +++ b/roles/minio_install/tasks/preflight_minio_config.yml @@ -37,7 +37,7 @@ - name: Check minio_vf is defined and set and vf_driver uses 'iavf' (kernel driver) assert: that: - - ('vf_00' is not in item.sriov_vfs.keys() and item.default_vf_driver == 'iavf') or + - ('vf_00' is not in item.sriov_vfs.keys() and item.default_vf_driver == 'iavf') or ('vf_00' is in item.sriov_vfs.keys() and item.sriov_vfs.get('vf_00') == 'iavf') msg: >- "MinIO uses vf_driver: 'iavf' (kernel driver) with the vf_00 by default. diff --git a/roles/operator_framework/defaults/main.yml b/roles/operator_framework/defaults/main.yml new file mode 100644 index 00000000..3efee1b7 --- /dev/null +++ b/roles/operator_framework/defaults/main.yml @@ -0,0 +1,18 @@ +## +## Copyright (c) 2020-2022 Intel Corporation. +## +## Licensed under the Apache License, Version 2.0 (the "License"); +## you may not use this file except in compliance with the License. +## You may obtain a copy of the License at +## +## http://www.apache.org/licenses/LICENSE-2.0 +## +## Unless required by applicable law or agreed to in writing, software +## distributed under the License is distributed on an "AS IS" BASIS, +## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +## See the License for the specific language governing permissions and +## limitations under the License. +## +operator_sdk_git: "https://github.com/operator-framework/operator-sdk.git" +operator_sdk_git_ref: "v1.20.0" +operator_sdk_dir: "{{ (project_root_dir, 'operator-sdk') | path_join }}" diff --git a/roles/operator_framework/tasks/main.yml b/roles/operator_framework/tasks/main.yml new file mode 100644 index 00000000..f9295096 --- /dev/null +++ b/roles/operator_framework/tasks/main.yml @@ -0,0 +1,57 @@ +## +## Copyright (c) 2020-2022 Intel Corporation. +## +## Licensed under the Apache License, Version 2.0 (the "License"); +## you may not use this file except in compliance with the License. +## You may obtain a copy of the License at +## +## http://www.apache.org/licenses/LICENSE-2.0 +## +## Unless required by applicable law or agreed to in writing, software +## distributed under the License is distributed on an "AS IS" BASIS, +## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +## See the License for the specific language governing permissions and +## limitations under the License. +## +--- +- block: + - name: uninstall Operator Lifecycle Manager (OLM) + command: operator-sdk olm uninstall + environment: + PATH: "{{ gopath.stdout }}/bin:/usr/local/go/bin:/usr/sbin:/usr/bin:/sbin:/bin:{{ operator_sdk_dir }}" + changed_when: false + failed_when: false + + - name: clone Operator-SDK repository + git: + repo: "{{ operator_sdk_git }}" + version: "{{ operator_sdk_git_ref }}" + dest: "{{ operator_sdk_dir }}" + force: yes + + - name: install Operator-SDK + make: + target: "install" + chdir: "{{ operator_sdk_dir }}" + + - name: install Operator Lifecycle Manager (OLM) + command: operator-sdk olm install --version v0.20.0 + environment: + PATH: "{{ gopath.stdout }}/bin:/usr/local/go/bin:/usr/sbin:/usr/bin:/sbin:/bin:{{ operator_sdk_dir }}" + changed_when: true + + - name: check if OLM pods are running + shell: set -o pipefail && kubectl get pods -n olm | awk 'NR != 1 { print $3 }' + args: + executable: /bin/bash + register: olm_pods_status + retries: 30 + delay: 10 + until: + - "'Error' not in olm_pods_status.stdout" + - "'CrashLoopBackOff' not in olm_pods_status.stdout" + - "'Terminating' not in olm_pods_status.stdout" + - "'ContainerCreating' not in olm_pods_status.stdout" + - "'Pending' not in olm_pods_status.stdout" + changed_when: false + when: inventory_hostname == groups['kube_control_plane'][0] diff --git a/roles/platform_aware_scheduling_install/tasks/main.yml b/roles/platform_aware_scheduling_install/tasks/main.yml index 4db7cf83..78f30d2c 100644 --- a/roles/platform_aware_scheduling_install/tasks/main.yml +++ b/roles/platform_aware_scheduling_install/tasks/main.yml @@ -78,3 +78,11 @@ when: - inventory_hostname == groups['kube_control_plane'][0] +- name: restart kubelet service + service: + name: kubelet.service + state: restarted + +- name: wait for the kubelet service restarted + pause: + seconds: 30 diff --git a/roles/remove_kubespray_host_dns_settings/tasks/main.yml b/roles/remove_kubespray_host_dns_settings/tasks/main.yml new file mode 100644 index 00000000..90bc32b6 --- /dev/null +++ b/roles/remove_kubespray_host_dns_settings/tasks/main.yml @@ -0,0 +1,32 @@ +## +## Copyright (c) 2020-2022 Intel Corporation. +## +## Licensed under the Apache License, Version 2.0 (the "License"); +## you may not use this file except in compliance with the License. +## You may obtain a copy of the License at +## +## http://www.apache.org/licenses/LICENSE-2.0 +## +## Unless required by applicable law or agreed to in writing, software +## distributed under the License is distributed on an "AS IS" BASIS, +## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +## See the License for the specific language governing permissions and +## limitations under the License. +## +--- +- name: reset DNS settings in dhclient.conf + blockinfile: + path: "{{ item }}" + state: absent + marker: "# Ansible entries {mark}" + failed_when: false + with_items: + - /etc/dhclient.conf + - /etc/dhcp/dhclient.conf + +- name: reset entries in /etc/hosts + blockinfile: + path: "/etc/hosts" + state: absent + marker: "# Ansible inventory hosts {mark}" + failed_when: false diff --git a/roles/service_mesh_install/charts/istioctl/.helmignore b/roles/service_mesh_install/charts/istioctl/.helmignore new file mode 100644 index 00000000..0e8a0eb3 --- /dev/null +++ b/roles/service_mesh_install/charts/istioctl/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/roles/service_mesh_install/charts/istioctl/Chart.yaml b/roles/service_mesh_install/charts/istioctl/Chart.yaml new file mode 100644 index 00000000..8009e040 --- /dev/null +++ b/roles/service_mesh_install/charts/istioctl/Chart.yaml @@ -0,0 +1,39 @@ +## +## Copyright (c) 2020-2022 Intel Corporation. +## +## Licensed under the Apache License, Version 2.0 (the "License"); +## you may not use this file except in compliance with the License. +## You may obtain a copy of the License at +## +## http://www.apache.org/licenses/LICENSE-2.0 +## +## Unless required by applicable law or agreed to in writing, software +## distributed under the License is distributed on an "AS IS" BASIS, +## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +## See the License for the specific language governing permissions and +## limitations under the License. +## +apiVersion: v2 +name: istioctl +description: A Helm chart for Kubernetes + +# A chart can be either an 'application' or a 'library' chart. +# +# Application charts are a collection of templates that can be packaged into versioned archives +# to be deployed. +# +# Library charts provide useful utilities or functions for the chart developer. They're included as +# a dependency of application charts to inject those utilities and functions into the rendering +# pipeline. Library charts do not define any templates and therefore cannot be deployed. +type: application + +# This is the chart version. This version number should be incremented each time you make changes +# to the chart and its templates, including the app version. +# Versions are expected to follow Semantic Versioning (https://semver.org/) +version: 0.1.0 + +# This is the version number of the application being deployed. This version number should be +# incremented each time you make changes to the application. Versions are not expected to +# follow Semantic Versioning. They should reflect the version the application is using. +# It is recommended to use it with quotes. +appVersion: "1.14.0" diff --git a/roles/service_mesh_install/charts/istioctl/templates/NOTES.txt b/roles/service_mesh_install/charts/istioctl/templates/NOTES.txt new file mode 100644 index 00000000..10f82c56 --- /dev/null +++ b/roles/service_mesh_install/charts/istioctl/templates/NOTES.txt @@ -0,0 +1,7 @@ +1. Execute istioctl by running these commands: +export POD_NAME=$(kubectl get pods --namespace {{ .Release.Namespace }} -l "app.kubernetes.io/name={{ include "istioctl.name" . }},app.kubernetes.io/instance={{ .Release.Name }}" -o jsonpath="{.items[0].metadata.name}") +kubectl --namespace {{ .Release.Namespace }} exec $POD_NAME -- istioctl version +or interactively: +kubectl --namespace {{ .Release.Namespace }} exec -it $POD_NAME -- bash + +$ istioctl version diff --git a/roles/service_mesh_install/charts/istioctl/templates/_helpers.tpl b/roles/service_mesh_install/charts/istioctl/templates/_helpers.tpl new file mode 100644 index 00000000..91966207 --- /dev/null +++ b/roles/service_mesh_install/charts/istioctl/templates/_helpers.tpl @@ -0,0 +1,51 @@ +{{/* +Expand the name of the chart. +*/}} +{{- define "istioctl.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "istioctl.fullname" -}} +{{- if .Values.fullnameOverride }} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- $name := default .Chart.Name .Values.nameOverride }} +{{- if contains $name .Release.Name }} +{{- .Release.Name | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} +{{- end }} +{{- end }} +{{- end }} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "istioctl.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Common labels +*/}} +{{- define "istioctl.labels" -}} +helm.sh/chart: {{ include "istioctl.chart" . }} +{{ include "istioctl.selectorLabels" . }} +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end }} + +{{/* +Selector labels +*/}} +{{- define "istioctl.selectorLabels" -}} +app.kubernetes.io/name: {{ include "istioctl.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end }} diff --git a/roles/service_mesh_install/charts/istioctl/templates/istioctl-deployment.yaml b/roles/service_mesh_install/charts/istioctl/templates/istioctl-deployment.yaml new file mode 100644 index 00000000..8bcf0e82 --- /dev/null +++ b/roles/service_mesh_install/charts/istioctl/templates/istioctl-deployment.yaml @@ -0,0 +1,59 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "istioctl.fullname" . }} + labels: + {{- include "istioctl.labels" . | nindent 4 }} +spec: + replicas: {{ .Values.replicaCount }} + selector: + matchLabels: + {{- include "istioctl.selectorLabels" . | nindent 6 }} + template: + metadata: + {{- with .Values.podAnnotations }} + annotations: + {{- toYaml . | nindent 8 }} + {{- end }} + labels: + {{- include "istioctl.selectorLabels" . | nindent 8 }} + spec: + {{- with .Values.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + serviceAccountName: {{ include "istioctl.fullname" . }} + securityContext: + {{- toYaml .Values.podSecurityContext | nindent 8 }} + containers: + - name: {{ .Chart.Name }} + securityContext: + {{- toYaml .Values.securityContext | nindent 12 }} + image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}" + imagePullPolicy: {{ .Values.image.pullPolicy }} + resources: + {{- toYaml .Values.resources | nindent 12 }} + {{- with .Values.command }} + command: + {{- toYaml . | nindent 12 }} + {{- end }} + {{- with .Values.volumeMounts }} + volumeMounts: + {{- toYaml . | nindent 12 }} + {{- end }} + {{- with .Values.volumes }} + volumes: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} diff --git a/roles/service_mesh_install/charts/istioctl/templates/istioctl-rbac.yaml b/roles/service_mesh_install/charts/istioctl/templates/istioctl-rbac.yaml new file mode 100644 index 00000000..5ab60a52 --- /dev/null +++ b/roles/service_mesh_install/charts/istioctl/templates/istioctl-rbac.yaml @@ -0,0 +1,28 @@ +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "istioctl.fullname" . }} + namespace: {{ .Release.Namespace }} + +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ include "istioctl.fullname" . }} +rules: + {{- toYaml .Values.clusterRoleRules | nindent 2 }} + +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ include "istioctl.fullname" . }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ include "istioctl.fullname" . }} +subjects: + - kind: ServiceAccount + name: {{ include "istioctl.fullname" . }} + namespace: {{ .Release.Namespace }} diff --git a/roles/service_mesh_install/charts/istioctl/values.yaml b/roles/service_mesh_install/charts/istioctl/values.yaml new file mode 100644 index 00000000..cbfe7d0e --- /dev/null +++ b/roles/service_mesh_install/charts/istioctl/values.yaml @@ -0,0 +1,88 @@ +## +## Copyright (c) 2020-2022 Intel Corporation. +## +## Licensed under the Apache License, Version 2.0 (the "License"); +## you may not use this file except in compliance with the License. +## You may obtain a copy of the License at +## +## http://www.apache.org/licenses/LICENSE-2.0 +## +## Unless required by applicable law or agreed to in writing, software +## distributed under the License is distributed on an "AS IS" BASIS, +## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +## See the License for the specific language governing permissions and +## limitations under the License. +## + +# Default values for istioctl. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +replicaCount: 1 + +image: + repository: istio/istioctl + pullPolicy: IfNotPresent + tag: 1.14.1 + +imagePullSecrets: [] +nameOverride: "" +fullnameOverride: "" + +podAnnotations: {} + +podSecurityContext: {} + +securityContext: + runAsNonRoot: true + runAsUser: 65333 + runAsGroup: 65333 + capabilities: + drop: + - ALL + +resources: + limits: + cpu: 100m + memory: 200Mi + requests: + cpu: 50m + memory: 100Mi + +nodeSelector: {} + +tolerations: + - key: "node-role.kubernetes.io/master" + operator: "Equal" + value: "" + effect: "NoSchedule" + +affinity: + nodeAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 1 + preference: + matchExpressions: + - key: "node-role.kubernetes.io/master" + operator: In + values: [""] + +volumes: + - name: istio-profiles + hostPath: + path: /opt/cek/charts/istio/profiles/ + +volumeMounts: + - name: istio-profiles + mountPath: /opt/cek/charts/istio/profiles/ + readOnly: true + +command: + - "sh" + - "-c" + - "sleep infinity" + +clusterRoleRules: + - apiGroups: ["*"] + resources: ["*"] + verbs: ["*"] diff --git a/roles/service_mesh_install/defaults/main.yml b/roles/service_mesh_install/defaults/main.yml index b16cb9ed..6060e60c 100644 --- a/roles/service_mesh_install/defaults/main.yml +++ b/roles/service_mesh_install/defaults/main.yml @@ -17,3 +17,4 @@ service_mesh_download_url: "https://github.com/istio/istio/releases/download/{{ service_mesh.version }}/istio-{{ service_mesh.version }}-linux-amd64.tar.gz" service_mesh_release_dir: "{{ (project_root_dir, 'istio') | path_join }}" service_mesh_charts_dir: "{{ (project_root_dir, 'charts', 'istio') | path_join }}" +service_mesh_profiles_dir: "{{ (service_mesh_charts_dir, 'profiles') | path_join }}" diff --git a/roles/service_mesh_install/files/profiles/intel-cryptomb.yaml b/roles/service_mesh_install/files/profiles/intel-cryptomb.yaml new file mode 100644 index 00000000..871a3e11 --- /dev/null +++ b/roles/service_mesh_install/files/profiles/intel-cryptomb.yaml @@ -0,0 +1,58 @@ +apiVersion: install.istio.io/v1alpha1 +kind: IstioOperator +spec: + profile: default + tag: 1.14-intel.1 + hub: intel + meshConfig: + # Enable for envoy debugging + # accessLogFile: /dev/stdout + defaultConfig: + proxyStatsMatcher: + inclusionPrefixes: + - "listener" + # Allows cryptomb bucket statistics via EnvoyFilter + proxyMetadata: + BOOTSTRAP_XDS_AGENT: "true" + + components: + ingressGateways: + - enabled: true + name: istio-ingressgateway + k8s: + overlays: + - kind: Deployment + name: istio-ingressgateway + patches: + - path: spec.template.spec.containers.[name:istio-proxy].args.[-1] + value: "--concurrency=2" + # Limit CPU/MEM usage to 2 vCPUs/4 GB for QoS reasons. + # Enable CPU manager static policy in kubelet to even more deterministic results. + resources: + requests: + cpu: 2000m + memory: 4096Mi + limits: + cpu: 2000m + memory: 4096Mi + hpaSpec: + maxReplicas: 1 + minReplicas: 1 + podAnnotations: # this controls the SDS service which configures ingress gateway + proxy.istio.io/config: | + privateKeyProvider: + cryptomb: + pollDelay: 10ms + values: + # Annotate pods with + # inject.istio.io/templates: sidecar,cryptomb + # + # Note: CryptoMB doesn't have any method for guiding the workload to + # an AVX-512 enabled node, so when you annotate the pod with the + # cryptomb annotation, also set the taints correctly. + sidecarInjectorWebhook: + templates: + cryptomb: | + spec: + containers: + - name: istio-proxy diff --git a/roles/service_mesh_install/files/profiles/intel-qat-hw.yaml b/roles/service_mesh_install/files/profiles/intel-qat-hw.yaml new file mode 100644 index 00000000..697a739c --- /dev/null +++ b/roles/service_mesh_install/files/profiles/intel-qat-hw.yaml @@ -0,0 +1,51 @@ +apiVersion: install.istio.io/v1alpha1 +kind: IstioOperator +metadata: + name: qat + namespace: istio-system +spec: + profile: default + tag: 1.14-intel.1 + hub: intel + components: + ingressGateways: + - name: istio-ingressgateway + enabled: true + k8s: + # Ingress gateway needs to have IPC_LOCK capability and the + # QAT resources manually added, because the template + # injection isn't supported for gateways. + overlays: + - kind: Deployment + name: istio-ingressgateway + patches: + - path: spec.template.spec.containers.[name:istio-proxy].securityContext.capabilities.add + value: ["IPC_LOCK"] + resources: + requests: + qat.intel.com/generic: '1' + limits: + qat.intel.com/generic: '1' + podAnnotations: # this controls the SDS service which configures ingress gateway + proxy.istio.io/config: | + privateKeyProvider: + qat: + pollDelay: 20ms + values: + # Annotate pods with either + # inject.istio.io/templates: sidecar,qathw + sidecarInjectorWebhook: + templates: + qathw: | + spec: + containers: + - name: istio-proxy + securityContext: + capabilities: + add: + - IPC_LOCK + resources: + requests: + qat.intel.com/generic: '1' + limits: + qat.intel.com/generic: '1' diff --git a/roles/service_mesh_install/files/profiles/intel-qat-sw.yaml b/roles/service_mesh_install/files/profiles/intel-qat-sw.yaml new file mode 100644 index 00000000..81f9601c --- /dev/null +++ b/roles/service_mesh_install/files/profiles/intel-qat-sw.yaml @@ -0,0 +1,17 @@ +apiVersion: install.istio.io/v1alpha1 +kind: IstioOperator +spec: + profile: default + hub: intel + tag: 1.14-intel.1 + # Enable for envoy debugging + # meshConfig: + # accessLogFile: /dev/stdout + components: + ingressGateways: + - name: istio-ingressgateway + enabled: true + k8s: + env: + - name: OPENSSL_CONF + value: "/etc/ssl/openssl-avx512.cnf" diff --git a/roles/service_mesh_install/files/profiles/sgx-mtls.yaml b/roles/service_mesh_install/files/profiles/sgx-mtls.yaml new file mode 100644 index 00000000..e04aadda --- /dev/null +++ b/roles/service_mesh_install/files/profiles/sgx-mtls.yaml @@ -0,0 +1,32 @@ +apiVersion: install.istio.io/v1alpha1 +kind: IstioOperator +spec: + profile: minimal + hub: intel + tag: 1.14-intel.1 + meshConfig: + accessLogFile: /dev/stdout + enableTracing: true + defaultConfig: + proxyMetadata: + # Enable basic DNS proxying + ISTIO_META_DNS_CAPTURE: "true" + # Enable automatic address allocation, optional + ISTIO_META_DNS_AUTO_ALLOCATE: "true" + # RSA or ECDSA + # ECC_SIGNATURE_ALGORITHM: ECDSA + + components: + pilot: + k8s: + imagePullPolicy: IfNotPresent + ingressGateways: + - name: istio-ingressgateway + enabled: true + k8s: + imagePullPolicy: IfNotPresent + egressGateways: + - name: istio-egressgateway + enabled: true + k8s: + imagePullPolicy: IfNotPresent diff --git a/roles/service_mesh_install/tasks/cleanup.yml b/roles/service_mesh_install/tasks/cleanup.yml index 074ec00e..824dc92d 100644 --- a/roles/service_mesh_install/tasks/cleanup.yml +++ b/roles/service_mesh_install/tasks/cleanup.yml @@ -14,16 +14,30 @@ ## limitations under the License. ## --- +- name: find existing istioctl pod + command: | + kubectl get pods \ + --namespace {{ service_mesh.istio_namespace }} \ + -l "app.kubernetes.io/name=istioctl,app.kubernetes.io/instance=istioctl" \ + -o jsonpath="{.items[0].metadata.name}" + register: istioctl_pod + failed_when: false + changed_when: true + - name: remove existing istio resources - command: - argv: - - "{{ service_mesh_release_dir }}/bin/istioctl" - - "x" - - "uninstall" - - "--purge" - - "-y" + command: | + kubectl exec \ + --namespace {{ service_mesh.istio_namespace }} \ + {{ istioctl_pod.stdout }} -- istioctl x uninstall --purge -y failed_when: false changed_when: true + when: + - istioctl_pod.stderr | length == 0 + +- name: remove existing istioctl deployment + command: "helm delete istioctl --namespace {{ service_mesh.istio_namespace }}" + changed_when: true + failed_when: false - name: remove existing tcpip-bypass-ebpf resources command: "kubectl delete -f {{ (service_mesh_charts_dir, 'tcpip-bypass-ebpf.yaml') | path_join }}" @@ -35,7 +49,7 @@ changed_when: true failed_when: false -- name: remove existing intel-tls-splicing resources +- name: remove existing tcs cluster issuer command: "kubectl delete -f {{ (service_mesh_charts_dir, 'tcs-cluster-issuer.yaml') | path_join }}" changed_when: true failed_when: false diff --git a/roles/service_mesh_install/tasks/install.yml b/roles/service_mesh_install/tasks/istio-install.yml similarity index 63% rename from roles/service_mesh_install/tasks/install.yml rename to roles/service_mesh_install/tasks/istio-install.yml index 781560ee..5cc2a0fc 100644 --- a/roles/service_mesh_install/tasks/install.yml +++ b/roles/service_mesh_install/tasks/istio-install.yml @@ -14,26 +14,19 @@ ## limitations under the License. ## --- -- name: download and unarchive istio release - unarchive: - src: "{{ service_mesh_download_url }}" - dest: "{{ project_root_dir }}" - remote_src: yes - list_files: yes - mode: 0755 - extra_opts: - - "--transform" - - "s/^istio-{{ service_mesh.version }}/istio/" - register: istio_download - until: istio_download is not failed - retries: 5 +- name: set facts for upstream istio release + set_fact: + istio_image: "{{ service_mesh.image }}" + istio_tag: "{{ service_mesh.version }}" + when: + - not service_mesh.intel_preview.enabled -- name: add istio binaries to the $PATH - copy: - content: | - export PATH={{ service_mesh_release_dir }}/bin:$PATH - dest: /etc/profile.d/istioctl.sh - mode: 0755 +- name: set facts for intel preview istio release + set_fact: + istio_image: "{{ service_mesh.intel_preview.image }}" + istio_tag: "{{ service_mesh.intel_preview.version }}" + when: + - service_mesh.intel_preview.enabled - name: create istio custom manifests directory file: @@ -43,10 +36,15 @@ - name: copy custom istio profile manifest if exists block: + - name: create custom istio profiles directory + file: + path: "{{ service_mesh_profiles_dir }}" + state: directory + mode: 0755 - name: copy manifest copy: src: "{{ ('profiles', service_mesh.profile) | path_join }}.yaml" - dest: "{{ service_mesh_charts_dir }}" + dest: "{{ service_mesh_profiles_dir }}" owner: root mode: preserve rescue: @@ -76,17 +74,26 @@ delegate_to: localhost become: false -- name: create istio system namespace for istio resources - shell: "set -o pipefail && kubectl create ns {{ service_mesh.istio_namespace }} -o yaml --dry-run=client | kubectl apply -f -" - args: - executable: /bin/bash - changed_when: true +- name: copy istio helm chart to the controller node + copy: + src: "{{ (role_path, 'charts', 'istioctl') | path_join }}" + dest: "{{ service_mesh_charts_dir }}" + mode: 0755 + +- name: populate istio charts values templates and push to controller node + template: + src: "istioctl-values.yaml.j2" + dest: "{{ (project_root_dir, 'charts', 'istioctl-values.yaml') | path_join }}" + force: yes + mode: preserve -- name: install istio service mesh - # noqa 305 - shell is used intentionally here - shell: "{{ (service_mesh_release_dir, 'bin', 'istioctl') | path_join }} install {{ argv | join(' ') }}" - args: - executable: "/bin/bash" +- name: install intel istio helm chart + command: >- + helm upgrade -i istioctl + --namespace {{ service_mesh.istio_namespace }} + --create-namespace + -f {{ (project_root_dir, 'charts', 'istioctl-values.yaml') | path_join }} + {{ (service_mesh_charts_dir, 'istioctl') | path_join }} changed_when: true - name: remove temporary files diff --git a/roles/service_mesh_install/tasks/main.yml b/roles/service_mesh_install/tasks/main.yml index 32e83bd6..0502e164 100644 --- a/roles/service_mesh_install/tasks/main.yml +++ b/roles/service_mesh_install/tasks/main.yml @@ -46,10 +46,25 @@ hostvars[groups['kube_node'][0]]['is_spr'] - name: install istio service mesh - include_tasks: install.yml + include_tasks: istio-install.yml when: - inventory_hostname == groups['kube_control_plane'][0] +- name: fix the iptable-restore error by https://github.com/istio/istio/issues/23009 + modprobe: + name: "{{ item }}" + state: present + with_items: + - "br_netfilter" + - "nf_nat" + - "xt_REDIRECT" + - "xt_owner" + - "iptable_nat" + - "iptable_mangle" + - "iptable_filter" + when: + - ansible_distribution in ['RedHat', 'Rocky'] and inventory_hostname != groups['kube_control_plane'][0] + - name: deploy TLS splicing and bumping include_tasks: tls-splicing-and-bumping.yml when: diff --git a/roles/service_mesh_install/tasks/tls-splicing-and-bumping.yml b/roles/service_mesh_install/tasks/tls-splicing-and-bumping.yml index dcd19959..d27a9ed2 100644 --- a/roles/service_mesh_install/tasks/tls-splicing-and-bumping.yml +++ b/roles/service_mesh_install/tasks/tls-splicing-and-bumping.yml @@ -28,6 +28,19 @@ force: yes mode: preserve +- name: wait for the istio service mesh pods are in running state + shell: |- + set -o pipefail && \ + [ $(kubectl get pod -n {{ service_mesh.istio_namespace }} -l 'app in (istio-ingressgateway, istiod)' \ + | grep Running | wc -l) -eq 2 ] + args: + executable: /bin/bash + register: ready + retries: 120 + delay: 10 + until: ready.rc == 0 + changed_when: true + - name: deploy intel-tls-splicing shell: |- set -o pipefail && \ diff --git a/roles/service_mesh_install/templates/istioctl-options.yml.j2 b/roles/service_mesh_install/templates/istioctl-options.yml.j2 index 5ddacf14..22d38fef 100644 --- a/roles/service_mesh_install/templates/istioctl-options.yml.j2 +++ b/roles/service_mesh_install/templates/istioctl-options.yml.j2 @@ -56,5 +56,5 @@ argv: {% if service_mesh.profile in ['default', 'demo', 'minimal', 'external', 'empty', 'preview'] %} - --set profile={{ service_mesh.profile }} {% else %} - - --filename={{ service_mesh_charts_dir }}/{{ service_mesh.profile }}.yaml + - --filename={{ service_mesh_profiles_dir }}/{{ service_mesh.profile }}.yaml {% endif -%} diff --git a/roles/service_mesh_install/templates/istioctl-values.yaml.j2 b/roles/service_mesh_install/templates/istioctl-values.yaml.j2 new file mode 100644 index 00000000..f3a99830 --- /dev/null +++ b/roles/service_mesh_install/templates/istioctl-values.yaml.j2 @@ -0,0 +1,15 @@ +image: + repository: "{{ istio_image }}" + pullPolicy: IfNotPresent + tag: "{{ istio_tag }}" + +command: + - "sh" + - "-c" + - | + istioctl \ + install \ +{% for arg in argv %} + {{ arg }} \ +{% endfor %} + && sleep infinity diff --git a/roles/service_mesh_install/templates/tcpip-bypass-ebpf.yaml.j2 b/roles/service_mesh_install/templates/tcpip-bypass-ebpf.yaml.j2 index c514ab8a..bcc5dfc8 100644 --- a/roles/service_mesh_install/templates/tcpip-bypass-ebpf.yaml.j2 +++ b/roles/service_mesh_install/templates/tcpip-bypass-ebpf.yaml.j2 @@ -21,7 +21,7 @@ spec: effect: NoSchedule containers: - name: {{ service_mesh.tcpip_bypass_ebpf.name }} - image: {{ service_mesh.tcpip_bypass_ebpf.image }} + image: {{ service_mesh.tcpip_bypass_ebpf.image }}:{{ service_mesh.tcpip_bypass_ebpf.version }} imagePullPolicy: IfNotPresent securityContext: privileged: true diff --git a/roles/service_mesh_install/vars/main.yml b/roles/service_mesh_install/vars/main.yml index 54cea121..b892c41d 100644 --- a/roles/service_mesh_install/vars/main.yml +++ b/roles/service_mesh_install/vars/main.yml @@ -15,7 +15,12 @@ ## service_mesh_defaults: enabled: false - version: 1.13.1 + image: istio/istioctl + version: 1.14.1 + intel_preview: + enabled: false + image: intel/istioctl + version: 1.14-intel.1 context: '' filename: [] namespace: '' @@ -27,14 +32,16 @@ service_mesh_defaults: dry_run: false profile: default force: false - readiness_timeout: '10m0s' - set: [] + readiness_timeout: '20m0s' + set: + - values.global.defaultPodDisruptionBudget.enabled=false verify: true tcpip_bypass_ebpf: enabled: false name: bypass-tcpip namespace: kube-system - image: docker.io/intel/istio-tcpip-bypass@sha256:9f3944e8591d3eda51e547be039a9d5202f1403e024847273cfb98aafe31919f + image: intel/istio-tcpip-bypass + version: latest tls_splicing: enabled: false hostname: www.example.com diff --git a/roles/sriov_network_operator_install/tasks/main.yml b/roles/sriov_network_operator_install/tasks/main.yml index aa51c97e..a61bcf11 100644 --- a/roles/sriov_network_operator_install/tasks/main.yml +++ b/roles/sriov_network_operator_install/tasks/main.yml @@ -25,3 +25,9 @@ - name: configure sriov-network-operator include_tasks: sriov_network_node_policy_configure.yml when: inventory_hostname == groups['kube_control_plane'][0] + +- name: Intel Ethernet Operator - flow configuration + include_role: + name: intel_ethernet_operator + tasks_from: flow_config + when: intel_ethernet_operator_flow_config_enabled | default(false)