From 7aedebd768ed7fe541d14bd604a504f692ffe3ee Mon Sep 17 00:00:00 2001 From: Calin Gherghe Date: Tue, 22 Feb 2022 09:39:32 -0800 Subject: [PATCH] New Components: - Intel Service Mesh - MinIO Object Storage - Intel Power Manager (Power Operator) - Platform Aware Scheduling (Telemetry Aware Scheduling + GPU Aware Scheduling) - Intel DLB (Dynamic Load Balancer) New Platforms: - Taylors Falls Reference Design (Intel Xeon-D) Updates/Changes: - Playbooks and profile config files are generated automatically - Profiles list expanded with 'Storage' - RHEL bumped to version 8.5 as base operating system - Version upgrades for key components: DPDK = 21.11 Kubernetes = 1.22 Kubespray = 2.17 KMRA = 1.4 Removed support: - Intel CPU Manager for Kubernetes (CMK) - CentOS (all distro versions) as base operating system Co-authored-by: Ali Shah, Syed Faraz Co-authored-by: Gherghe, Calin Co-authored-by: Joel A Gibson Co-authored-by: Klimowicz, Patryk Co-authored-by: Konefal, Szymon Co-authored-by: Krystian M.ynek Co-authored-by: Li, Jennifer Co-authored-by: Liu, Mark Co-authored-by: Lv, Carry Co-authored-by: Park, Seungweon Co-authored-by: Pedersen, Michael Co-authored-by: Prokes, Jiri Co-authored-by: Puzikov, Dmitrii Co-authored-by: Rymsza, Monika Co-authored-by: Weber, Karol --- CONTRIBUTING.md | 73 + Makefile | 85 +- README.md | 66 +- action_plugins/validate_isolcpus.py | 2 +- docs/generate_profiles.md | 107 + docs/power_operator.md | 162 + docs/sriov.md | 2 +- docs/storage.md | 82 + docs/vm_config_guide.md | 331 + generate/playbook_templates/infra_playbook.j2 | 186 + generate/playbook_templates/intel_playbook.j2 | 128 + generate/playbook_templates/main_playbook.j2 | 15 + .../profiles_templates/common/group_vars.j2 | 377 + .../profiles_templates/common}/host_vars.j2 | 226 +- .../profiles_templates/k8s}/inventory.j2 | 3 +- .../profiles_templates/k8s}/profiles.yml | 119 +- generate/profiles_templates/vm/inventory.j2 | 22 + .../vm/vm_host_profiles.yml | 177 + .../profiles_templates/vm/vms_profiles.yml | 184 + generate/render.py | 53 + generate/render/__init__.py | 0 generate/render/common/cli.py | 43 + generate/render/common/common.py | 95 + generate/render/renderers/k8s_profiles.py | 72 + generate/render/renderers/playbook.py | 62 + generate/render/renderers/vm_profiles.py | 108 + playbooks/full_nfv.yml | 26 - playbooks/infra/access.yml | 126 - playbooks/infra/basic.yml | 96 - .../infra/container_runtime_deployment.yml | 22 + playbooks/infra/full_nfv.yml | 146 - playbooks/infra/on_prem.yml | 143 - playbooks/infra/prepare_vms.yml | 31 + playbooks/infra/redeploy_cleanup.yml | 2 +- playbooks/infra/regional_dc.yml | 98 - playbooks/infra/remote_fp.yml | 140 - playbooks/intel/access.yml | 91 - playbooks/intel/basic.yml | 78 - playbooks/intel/full_nfv.yml | 118 - playbooks/intel/on_prem.yml | 114 - playbooks/intel/regional_dc.yml | 91 - playbooks/intel/remote_fp.yml | 117 - playbooks/k8s/k8s.yml | 99 +- playbooks/k8s/kubelet-certificates.yml | 22 +- playbooks/k8s/patch_kubespray.yml | 4 +- playbooks/on_prem.yml | 26 - playbooks/preflight.yml | 499 +- playbooks/redeploy_cleanup.yml | 2 +- playbooks/regional_dc.yml | 26 - playbooks/remote_fp.yml | 26 - playbooks/test/test_roles.yml | 2 +- playbooks/{access.yml => vm.yml} | 19 +- profiles/group_vars.j2 | 251 - profiles/render.py | 138 - profiles/requirements.txt | 2 - requirements.txt | 11 + roles/bond_cni_install/defaults/main.yml | 5 +- roles/bond_cni_install/tasks/main.yml | 13 +- roles/bond_cni_install/vars/main.yml | 2 +- roles/bootstrap/ansible_host/tasks/main.yml | 53 - .../apply_kubernetes_reqs/tasks/main.yml | 18 +- .../defaults/main.yml} | 8 +- .../tasks/main.yml | 36 + .../setup_additional_grub_parameters.yml | 29 + .../vars/main.yml | 5 +- .../tasks/autogenerate_isolcpus.yml | 68 - .../configure_cpu_isolation/tasks/main.yml | 20 +- .../tasks/setup_isolcpus.yml | 8 +- .../configure_cpu_isolation/vars/main.yml | 2 +- roles/bootstrap/configure_dns/tasks/main.yml | 4 +- .../configure_hugepages/defaults/main.yml | 2 +- .../configure_hugepages/tasks/main.yml | 2 +- .../tasks/setup_hugepages.yml | 28 +- .../configure_hugepages/vars/main.yml | 2 +- .../configure_intel_pstate/tasks/main.yml | 14 +- .../tasks/setup_intel_pstate.yml | 3 +- .../tasks/setup_turbo.yml | 76 +- .../configure_intel_pstate/vars/main.yml | 2 +- .../configure_openssl/defaults/main.yml | 19 +- .../tasks/install_openssl_1.1.1_centos7.yml | 127 - .../tasks/install_req_packages_centos7.yml | 134 - .../configure_openssl/tasks/main.yml | 55 +- .../bootstrap/configure_openssl/vars/main.yml | 4 +- .../bootstrap/configure_proxy/tasks/main.yml | 15 +- ...bmra_sriov_qat_init => cek_sriov_qat_init} | 2 +- ...on_qat_confg.yml => check_qat_service.yml} | 36 +- roles/bootstrap/configure_qat/tasks/main.yml | 64 +- .../templates/bmra_sriov_qat_init.service.j2 | 12 - .../templates/cek_sriov_qat_init.service.j2 | 13 + roles/bootstrap/configure_qat/vars/main.yml | 4 +- .../configure_security/tasks/fw_debian.yaml | 63 + .../configure_security/tasks/fw_redhat.yaml | 68 + .../configure_security/tasks/main.yml | 136 +- .../configure_security/tasks/selinux.yaml | 34 + .../configure_security/vars/main.yml | 3 +- .../bootstrap/configure_sgx/defaults/main.yml | 69 + .../configure_sgx}/tasks/centos.yml | 50 +- roles/bootstrap/configure_sgx/tasks/main.yml | 68 + .../configure_sgx}/tasks/rhel.yml | 46 +- .../configure_sgx}/tasks/ubuntu.yml | 82 +- .../configure_sgx}/vars/main.yml | 4 +- .../bootstrap/configure_sst/defaults/main.yml | 15 +- .../centos8_2_install_isst_interface.yml | 158 - .../configure_sst/tasks/clx_setup_sst_bf.yml | 2 +- .../configure_sst/tasks/icx_sst_bf.yml | 2 +- .../configure_sst/tasks/icx_sst_cp.yml | 2 +- .../configure_sst/tasks/icx_sst_tf.yml | 2 +- roles/bootstrap/configure_sst/tasks/main.yml | 12 +- .../tasks/sst_bf_cp_tf_pp_setup.yml | 35 +- .../bootstrap/configure_sst/tasks/sst_pp.yml | 54 +- .../tasks/sst_pp_user_defined_setup.yml | 26 +- .../tasks/ubuntu_install_sst_tool.yml | 28 +- .../isst-load-modules-icx.service.j2 | 14 - .../defaults/main.yml | 2 +- .../tasks/dataplane-interfaces.yml | 4 +- .../tasks/main.yml | 6 +- .../golang_install/defaults/main.yml | 6 +- roles/bootstrap/golang_install/tasks/main.yml | 103 + .../install_gpu_kernel/defaults/main.yml | 8 +- .../tasks/centos82_dracut_install.yml | 44 - .../install_gpu_kernel/tasks/main.yml | 12 +- .../install_gpu_kernel/tasks/prepare_grub.yml | 2 +- .../tasks/update_kernel_config.yml | 15 +- .../install_gpu_kernel/vars/main.yml | 2 +- .../install_packages/tasks/debian.yml | 35 +- .../bootstrap/install_packages/tasks/main.yml | 7 +- .../bootstrap/install_packages/tasks/rhel.yml | 171 +- .../defaults/main.yml | 9 +- .../tasks/main.yml | 56 +- .../vars/main.yml | 2 +- .../install_rt_package/tasks/main.yml | 85 - .../templates/CentOS-RT.repo | 64 - .../set_rdt_kernel_flags/defaults/main.yml | 2 +- .../set_rdt_kernel_flags/tasks/main.yml | 2 +- .../set_sriov_kernel_flags/tasks/main.yml | 4 +- .../tasks/setup_sriov_kernel_flags.yml | 22 +- .../set_sriov_kernel_flags/vars/main.yml | 2 +- roles/bootstrap/update_grub/tasks/main.yml | 2 +- .../update_nic_drivers/defaults/main.yml | 14 +- .../update_nic_drivers/tasks/i40e.yml | 60 +- .../update_nic_drivers/tasks/iavf.yml | 60 +- .../update_nic_drivers/tasks/ice.yml | 72 +- .../update_nic_drivers/tasks/main.yml | 4 +- .../update_nic_drivers/vars/main.yml | 2 +- .../update_nic_firmware/defaults/main.yml | 33 + .../update_nic_firmware/tasks/main.yml | 18 +- .../update_nic_firmware/tasks/update.yml | 3 +- .../update_nic_firmware/vars/main.yml | 20 +- roles/cert_manager_install/defaults/main.yml | 3 +- roles/cert_manager_install/tasks/main.yml | 3 +- roles/check_machine_type/tasks/main.yml | 12 +- roles/check_machine_type/vars/main.yml | 7 +- roles/cluster_defaults/defaults/main.yml | 2 +- roles/cluster_defaults/tasks/main.yml | 8 +- .../templates/NOTES.txt | 10 - .../templates/_helpers.tpl | 32 - .../templates/daemonset.yml | 79 - .../templates/job.yml | 85 - .../templates/rbac.yml | 162 - .../templates/serviceaccount.yml | 8 - .../templates/webhook.yml | 131 - roles/cmk_install/tasks/main.yml | 364 - roles/cmk_install/tasks/reset.yml | 72 - .../cmk_install/templates/cmk-webhook.conf.j2 | 7 - .../cmk_install/templates/helm_values.yml.j2 | 21 - .../templates/kube_client_csr.yml.j2 | 10 - .../templates/kube_server_csr.yml.j2 | 10 - .../templates/webhook_client_csr.json.j2 | 10 - .../templates/webhook_server_csr.json.j2 | 12 - roles/collectd_install/defaults/main.yml | 20 +- roles/collectd_install/tasks/collectd.yml | 2 +- roles/collectd_install/tasks/copy-configs.yml | 15 +- roles/collectd_install/tasks/main.yml | 2 +- roles/collectd_install/tasks/msr-config.yml | 4 +- .../collectd_install/templates/daemonset.yml | 7 - .../plugin_configuration/turbostat.conf | 1 + roles/collectd_install/templates/psp.yml.j2 | 2 - .../containerd/defaults/main.yml | 79 + .../containerd/handlers/main.yml} | 32 +- .../containerd/meta}/main.yml | 8 +- .../containerd/tasks/containerd_repo.yml | 41 + .../containerd/tasks/main.yml | 87 + .../containerd/templates/config.toml.j2 | 55 + .../containerd/templates/http_proxy.conf.j2 | 2 + .../templates/rh_containerd.repo.j2 | 10 + .../containerd/vars/redhat.yml | 20 + .../containerd/vars/ubuntu.yml | 31 + .../containerd_common/defaults}/main.yml | 37 +- .../containerd_common/tasks/main.yml | 25 + .../containerd_common/vars/redhat.yml | 25 + .../containerd_common/vars/ubuntu.yml | 21 +- .../container_engine/crictl/defaults/main.yml | 40 + .../container_engine/crictl/handlers/main.yml | 27 + .../container_engine/crictl/tasks/crictl.yml | 71 + roles/container_engine/crictl/tasks/main.yml | 26 + .../crictl/templates/crictl.yml.j2 | 4 + roles/container_engine/crio/defaults/main.yml | 83 + roles/container_engine/crio/files/mounts.conf | 1 + roles/container_engine/crio/handlers/main.yml | 31 + .../container_engine/crio/tasks/crio_repo.yml | 66 + roles/container_engine/crio/tasks/main.yml | 154 + .../crio/templates/config.json.j2 | 17 + .../crio/templates/crictl.yml.j2 | 4 + .../crio/templates/crio.conf.j2 | 385 + .../crio/templates/http_proxy.conf.j2 | 2 + .../crio/templates/registry_mirror.conf.j2 | 11 + .../crio/vars/centos-8.yml} | 15 +- roles/container_engine/crio/vars/redhat.yml | 19 + .../crio/vars/ubuntu.yml} | 28 +- .../container_engine/docker/defaults/main.yml | 55 + .../container_engine/docker/handlers/main.yml | 38 + roles/container_engine/docker/meta/main.yml | 18 + roles/container_engine/docker/tasks/main.yml | 113 + .../container_engine/docker/tasks/systemd.yml | 56 + .../docker/templates/docker.service.j2 | 34 + .../docker/templates/docker_options.conf.j2 | 11 + .../docker/templates/http_proxy.conf.j2 | 2 + .../docker/templates/rh_docker.repo.j2 | 10 + roles/container_engine/docker/vars/redhat.yml | 39 + roles/container_engine/docker/vars/ubuntu.yml | 48 + roles/container_engine/meta/main.yml | 28 + roles/container_registry/defaults/main.yml | 10 +- roles/container_registry/tasks/main.yml | 40 +- roles/container_registry/tasks/tls.yml | 228 +- .../container-registry/configmap.yaml.j2 | 2 +- .../container-registry/deployment.yaml.j2 | 8 +- .../nginx-configmap.yaml.j2 | 2 - .../container-registry/secret.yaml.j2 | 2 +- .../container-registry/service.yaml.j2 | 2 +- .../templates/kube_registry_csr.yml.j2 | 3 +- .../templates/registry_csr.json.j2 | 13 +- roles/container_registry/vars/main.yml | 10 +- .../create_signed_k8s_certs/defaults/main.yml | 2 +- .../tasks/create_signed_k8s_certs.yml | 8 +- roles/create_signed_k8s_certs/tasks/main.yml | 2 +- .../templates/csr.json.j2 | 7 +- .../templates/kube_csr.yml.j2 | 3 +- roles/create_signed_k8s_certs/vars/main.yml | 2 +- roles/git2_install/tasks/main.yml | 56 - roles/golang_install/tasks/main.yml | 82 - roles/gpu_dp_install/defaults/main.yml | 15 +- roles/gpu_dp_install/tasks/main.yml | 13 +- .../templates/intel-gpu-plugin.yml.j2 | 11 +- roles/gpu_dp_install/vars/main.yml | 2 +- .../{vars => defaults}/main.yml | 4 +- .../install_ddp_pkgs/tasks/install_a_pkg.yml | 14 +- roles/install_ddp_pkgs/tasks/install_pkgs.yml | 2 +- roles/install_ddp_pkgs/tasks/main.yml | 3 +- roles/install_dependencies/tasks/main.yml | 2 +- roles/install_dpdk/defaults/main.yml | 22 + .../install_dpdk/tasks/install_dpdk_make.yml | 2 +- .../install_dpdk/tasks/install_dpdk_meson.yml | 16 +- roles/install_dpdk/tasks/main.yml | 37 +- roles/install_dpdk/vars/main.yml | 7 +- roles/intel_dp_operator/defaults/main.yml | 8 +- roles/intel_dp_operator/tasks/main.yml | 2 +- roles/intel_dp_operator/vars/main.yml | 2 +- roles/intel_power_manager/defaults/main.yml | 25 + roles/intel_power_manager/files/appqos.conf | 8 + roles/intel_power_manager/files/rbac.patch | 77 + roles/intel_power_manager/tasks/app_qos.yml | 103 + .../tasks/deploy_sample_pods.yml | 44 + .../tasks/deploy_shared_resources.yml | 69 + roles/intel_power_manager/tasks/main.yml | 68 + .../tasks/power_manager.yml | 150 + .../tasks/power_pod_template_helper.yml | 25 + .../templates/controller_manager.yml.j2 | 48 + .../templates/global_shared_profile.yml.j2 | 11 + .../templates/local_shared_profile.yml.j2 | 11 + .../templates/power_config.yml.j2 | 16 + .../templates/sample_power_pod.yml.j2 | 23 + .../templates/shared_workload.yml.j2 | 14 + .../vars/main.yml | 2 +- .../istio_operator_install/defaults/main.yml | 47 - .../istio_operator_install/tasks/cleanup.yml | 66 - .../istio_operator_install/tasks/install.yml | 68 - .../istioctl-operator-init-options.yml.j2 | 63 - .../istioctl-operator-remove-options.yml.j2 | 42 - .../templates/profiles/default.yaml.j2 | 185 - .../templates/profiles/empty.yaml.j2 | 17 - .../templates/profiles/external.yaml.j2 | 25 - .../templates/profiles/minimal.yaml.j2 | 12 - .../charts/kmra-apphsm/Chart.yaml | 6 +- .../templates/kmra-apphsm-configmap.yml | 9 +- .../templates/kmra-apphsm-deployment.yml | 25 +- .../templates/kmra-apphsm-qcnl-configmap.yaml | 11 + roles/kmra_install/charts/kmra-ctk/Chart.yaml | 6 +- .../templates/kmra-ctk-loadkey-configmap.yml | 6 +- .../templates/kmra-ctk-loadkey-deployment.yml | 24 + .../kmra-ctk-loadkey-qcnl-configmap.yaml | 11 + .../kmra_install/charts/kmra-pccs/Chart.yaml | 6 +- .../templates/kmra-pccs-configmap.yml | 2 +- .../templates/kmra-pccs-deployment.yml | 11 + roles/kmra_install/defaults/main.yml | 62 +- roles/kmra_install/tasks/cleanup.yml | 2 +- .../kmra_install/tasks/create_tls_secrets.yml | 9 +- roles/kmra_install/tasks/main.yml | 340 +- .../templates/kmra-apphsm-values.yaml.j2 | 14 + .../templates/kmra-ctk-values.yaml.j2 | 12 + roles/kmra_install/vars/main.yml | 2 +- roles/kube_prometheus/defaults/main.yml | 4 +- .../grafana-dashboardDefinitions.yaml | 66 +- ...grafana-telegraf-dashboardDefinitions.yaml | 70 +- .../grafana-dashboardDatasources.yaml | 2 +- .../grafana-dashboardSources.yaml | 2 +- .../node-exporter-clusterRoleBinding.yaml | 2 +- .../node-exporter-service.yaml | 2 +- .../node-exporter-serviceAccount.yaml | 2 +- .../node-exporter-serviceMonitor.yaml | 2 +- .../persistent-volume-claim-grafana.yaml | 2 +- .../kube-prometheus/prometheus-rules.yaml | 2 +- .../tasks/create-tas-demo-policy.yml | 2 +- .../kube_prometheus/tasks/kube-prometheus.yml | 5 +- roles/kube_prometheus/tasks/main.yml | 2 +- .../node-exporter-clusterRole.yaml.j2 | 2 +- .../templates/node-exporter-daemonset.yaml.j2 | 10 +- .../persistent-volume-grafana.yaml.j2 | 2 +- .../templates/prometheus-prometheus.yaml.j2 | 7 +- roles/kube_prometheus/vars/main.yml | 6 +- roles/kubespray_patch/defaults/main.yml | 2 +- roles/kubespray_patch/tasks/main.yml | 2 +- roles/kubespray_target_setup/tasks/main.yml | 8 +- .../templates/{config.yaml => config.yaml.j2} | 4 - .../templates/multus.conf.j2 | 2 +- roles/kubespray_target_setup/vars/main.yml | 2 +- roles/load_ddp/defaults/main.yml | 2 +- roles/load_ddp/tasks/load_i40e_ddp.yml | 2 +- roles/load_ddp/tasks/load_ice_ddp.yml | 2 +- roles/load_ddp/tasks/main.yml | 2 +- roles/load_ddp/tasks/update_network_card.yml | 2 +- roles/load_ddp/vars/main.yml | 2 +- .../minio_install/charts/operator/Chart.yaml | 32 + .../operator/crds/minio.min.io_tenants.yaml | 8232 +++++++++++++++++ .../charts/operator/templates/NOTES.txt | 6 + .../charts/operator/templates/_helpers.tpl | 84 + .../templates/cluster-role-binding.yaml | 12 + .../operator/templates/cluster-role.yaml | 131 + .../templates/console-deployment.yaml | 65 + .../operator/templates/console-ingress.yaml | 50 + .../operator/templates/console-service.yaml | 15 + .../charts/operator/templates/console-ui.yaml | 262 + .../templates/operator-deployment.yaml | 66 + .../operator/templates/operator-service.yaml | 15 + .../operator/templates/serviceaccount.yaml | 7 + .../minio_install/charts/operator/values.yaml | 74 + .../charts/tenant}/Chart.yaml | 24 +- .../charts/tenant/templates/NOTES.txt | 13 + .../charts/tenant/templates/_helpers.tpl | 84 + .../tenant/templates/tenant-secret.yaml | 16 + .../charts/tenant/templates/tenant.yaml | 122 + roles/minio_install/charts/tenant/values.yaml | 127 + .../defaults/main.yaml} | 27 +- .../tasks/create_blockdevicefiles.yml | 36 + .../tasks/create_nvme_partition.yml | 46 + .../tasks/create_persistentvolumes.yml | 38 + .../tasks/create_storageclass.yml} | 22 +- .../minio_install/tasks/file_blockdevice.yml | 44 + .../tasks/format_blockdevicefiles.yml | 22 + roles/minio_install/tasks/main.yml | 70 + roles/minio_install/tasks/minio_operator.yml | 78 + roles/minio_install/tasks/minio_tenant.yml | 71 + .../minio_install/tasks/mount_loopdevices.yml | 31 + .../minio_install/tasks/nvme_blockdevice.yml | 26 + .../tasks/setup_loopdevices.yml} | 12 +- .../minio_operator_custom_values.yml.j2 | 56 + .../minio_tenant_custom_values.yml.j2 | 71 + ...minio_tenant_localpersistentvolumes.yml.j2 | 25 + .../minio_tenant_storageclass.yml.j2 | 7 + roles/net_attach_defs_create/tasks/main.yml | 2 +- .../charts/node-feature-discovery/Chart.yaml | 2 +- .../templates/config.yml | 2 +- .../templates/master.yml | 8 +- .../node-feature-discovery/templates/rbac.yml | 5 +- .../templates/worker.yml | 6 + .../charts/node-feature-discovery/values.yaml | 6 +- roles/nfd_install/defaults/main.yml | 38 +- roles/nfd_install/tasks/main.yml | 26 +- .../nfd_install/templates/helm_values.yml.j2 | 97 +- roles/nfd_install/vars/main.yml | 2 +- .../openssl_engine_install/defaults/main.yml | 14 +- roles/openssl_engine_install/tasks/main.yml | 2 +- .../tasks/openssl_engine_config.yml | 242 +- .../vars/main.yml | 8 +- .../charts/gpu-aware-scheduling}/Chart.yaml | 11 +- .../templates}/.helmignore | 0 .../gpu-aware-scheduling/templates/NOTES.txt | 31 + .../templates/_helpers.tpl | 33 +- .../templates/gas-deployment.yaml | 61 + .../templates/gas-rbac-accounts.yaml | 34 + .../templates/gas-service.yaml | 14 + .../charts/gpu-aware-scheduling/values.yaml | 45 + .../telemetry-aware-scheduling}/.helmignore | 0 .../telemetry-aware-scheduling/Chart.yaml | 8 +- .../crds/tas-policy-crd.yaml | 83 + .../templates/NOTES.txt | 4 +- .../templates/_helpers.tpl | 13 +- .../templates/tas-demo-policy.yaml} | 7 +- .../templates/tas-deployment.yaml} | 21 +- .../templates/tas-rbac-accounts.yaml} | 23 - .../templates/tas-service.yaml} | 2 +- .../telemetry-aware-scheduling/values.yaml | 11 +- .../defaults/main.yml | 77 + .../tasks/backups_restore.yml | 49 + .../tasks/build_image.yml | 72 + .../tasks/create-scheduler-config.yml | 194 + .../tasks/gas.yml | 51 + .../tasks/main.yml | 80 + .../tasks/tas.yml | 71 + .../tasks/tls.yml | 89 + .../templates/gas-chart.yaml.j2 | 7 + .../templates/gas-values.yaml.j2 | 30 + .../pas-cluster-role-configmapgetter.yaml.j2 | 23 + .../templates/pas-extenders-configmap.yaml.j2 | 62 + .../templates/tas-chart.yaml.j2 | 7 + .../templates/tas-values.yaml.j2 | 32 + .../templates/tls-extender-ca.json.j2 | 11 + .../templates/tls-extender-client.json.j2 | 8 + .../templates/tls-extender.json.j2 | 17 + .../vars/main.yml | 52 + .../intel-qat-plugin/templates/NOTES.txt | 8 - .../templates/qat_plugin.yaml | 65 - .../templates/qat_plugin_configmap.yaml | 9 - roles/qat_dp_install/defaults/main.yml | 17 +- roles/qat_dp_install/files/noiommu.patch | 78 + roles/qat_dp_install/tasks/main.yml | 189 +- .../templates/intel-qat-plugin.yml.j2 | 21 + .../templates/qat_values.yml.j2 | 15 - roles/qat_dp_install/vars/main.yml | 2 +- roles/redeploy_cleanup/defaults/main.yml | 12 +- .../redeploy_cleanup/tasks/intel_cleanup.yml | 29 +- roles/redeploy_cleanup/tasks/k8s_cleanup.yml | 57 +- roles/redeploy_cleanup/tasks/main.yml | 4 +- roles/redeploy_cleanup/tasks/remove_files.yml | 4 +- roles/redeploy_cleanup/tasks/remove_minio.yml | 85 + roles/service_mesh_install/defaults/main.yml | 19 + .../files/profiles/.gitkeep | 0 roles/service_mesh_install/tasks/cleanup.yml | 43 + roles/service_mesh_install/tasks/install.yml | 101 + .../tasks/main.yml | 22 +- .../tasks/tcpip-bypass-ebpf.yml | 41 + .../templates/istioctl-options.yml.j2 | 60 + .../templates/tcpip-bypass-ebpf.yaml.j2 | 37 + roles/service_mesh_install/vars/main.yml | 37 + roles/sgx_configure_icx/defaults/main.yml | 80 - .../charts/intel-sgx-aesmd/Chart.yaml | 6 +- roles/sgx_dp_install/defaults/main.yaml | 10 +- roles/sgx_dp_install/tasks/main.yaml | 46 +- .../templates/intel-sgx-plugin.yml.j2 | 9 +- roles/sgx_dp_install/vars/main.yml | 2 +- roles/sriov_cni_install/defaults/main.yml | 6 +- roles/sriov_cni_install/tasks/main.yml | 16 +- roles/sriov_cni_install/vars/main.yml | 2 +- .../charts/sriov-net-dp/Chart.yaml | 2 +- .../charts/sriov-net-dp/values.yaml | 6 +- roles/sriov_dp_install/defaults/main.yml | 10 +- roles/sriov_dp_install/tasks/main.yml | 28 +- .../templates/helm_values.yml.j2 | 4 +- roles/sriov_dp_install/vars/main.yml | 2 +- .../charts/sriov-network-operator/Chart.yaml | 2 +- .../charts/sriov-network-operator/README.md | 2 +- ...f.io_networkattachmentdefinitions_crd.yaml | 2 +- ...vnetwork.openshift.io_sriovibnetworks.yaml | 2 +- ...openshift.io_sriovnetworknodepolicies.yaml | 2 +- ...k.openshift.io_sriovnetworknodestates.yaml | 2 +- ....openshift.io_sriovnetworkpoolconfigs.yaml | 2 +- ...iovnetwork.openshift.io_sriovnetworks.yaml | 2 +- ...ork.openshift.io_sriovoperatorconfigs.yaml | 2 +- .../defaults/main.yml | 21 +- ...ustom_sriov_network_policies_configure.yml | 51 + .../tasks/load_ddp_profile.yml | 2 +- .../tasks/main.yml | 2 +- .../sriov_network_node_policy_configure.yml | 9 +- .../tasks/sriov_network_operator_install.yml | 2 +- .../sriov_network_node_policy.yml.j2 | 1 + ...bmra_sriov_nic_init => cek_sriov_nic_init} | 4 +- roles/sriov_nic_init/tasks/bind_vf_driver.yml | 4 +- roles/sriov_nic_init/tasks/create_vfs.yml | 4 +- roles/sriov_nic_init/tasks/main.yml | 16 +- .../templates/bmra_sriov_nic_init.service.j2 | 16 - .../templates/cek_sriov_nic_init.service.j2 | 16 + roles/sriov_nic_init/vars/main.yml | 4 +- roles/tas_install/charts/tas-policy-crd.yml | 83 - .../telemetry-aware-scheduling/.helmignore | 22 - .../templates/tas-configmap.yml | 36 - roles/tas_install/defaults/main.yml | 39 - roles/tas_install/tasks/tas.yml | 205 - roles/tas_install/templates/tas-values.yml.j2 | 27 - .../charts/telegraf/Chart.yaml | 2 +- .../charts/telegraf/templates/daemonset.yml | 12 +- roles/telegraf_install/defaults/main.yml | 151 +- roles/telegraf_install/tasks/cleanup.yml | 4 +- roles/telegraf_install/tasks/main.yml | 8 +- roles/telegraf_install/tasks/msr-config.yml | 2 +- .../tasks/resctrl_wa_for_vms.yml | 24 + roles/telegraf_install/tasks/telegraf.yml | 6 +- .../telegraf_install/templates/values.yaml.j2 | 8 +- roles/telegraf_install/vars/main.yml | 145 +- roles/userspace_cni_install/defaults/main.yml | 7 +- roles/userspace_cni_install/tasks/main.yml | 4 +- .../tasks/ovs_install.yml | 133 +- .../tasks/userspace_cni_install.yml | 2 +- .../tasks/vpp_install.yml | 13 +- roles/userspace_cni_install/vars/main.yml | 3 +- roles/vm/conf_libvirt/tasks/main.yml | 73 + roles/vm/conf_libvirt/vars/main.yml | 37 + roles/vm/manage_imgs/defaults/main.yml | 26 + roles/vm/manage_imgs/tasks/main.yml | 137 + .../manage_imgs/templates/backup-user-data.j2 | 38 + roles/vm/manage_imgs/templates/meta-data.j2 | 2 + .../manage_imgs/templates/test-user-data.j2 | 7 + roles/vm/manage_imgs/templates/user-data.j2 | 68 + roles/vm/manage_vms/tasks/main.yml | 47 + roles/vm/manage_vms/tasks/optimize.yml | 42 + roles/vm/prepare_cek/tasks/main.yml | 148 + 514 files changed, 20869 insertions(+), 7770 deletions(-) create mode 100644 CONTRIBUTING.md create mode 100644 docs/generate_profiles.md create mode 100644 docs/power_operator.md create mode 100644 docs/storage.md create mode 100644 docs/vm_config_guide.md create mode 100644 generate/playbook_templates/infra_playbook.j2 create mode 100644 generate/playbook_templates/intel_playbook.j2 create mode 100644 generate/playbook_templates/main_playbook.j2 create mode 100644 generate/profiles_templates/common/group_vars.j2 rename {profiles => generate/profiles_templates/common}/host_vars.j2 (58%) rename {profiles => generate/profiles_templates/k8s}/inventory.j2 (98%) rename {profiles => generate/profiles_templates/k8s}/profiles.yml (57%) create mode 100644 generate/profiles_templates/vm/inventory.j2 create mode 100644 generate/profiles_templates/vm/vm_host_profiles.yml create mode 100644 generate/profiles_templates/vm/vms_profiles.yml create mode 100644 generate/render.py create mode 100644 generate/render/__init__.py create mode 100644 generate/render/common/cli.py create mode 100644 generate/render/common/common.py create mode 100644 generate/render/renderers/k8s_profiles.py create mode 100644 generate/render/renderers/playbook.py create mode 100644 generate/render/renderers/vm_profiles.py delete mode 100644 playbooks/full_nfv.yml delete mode 100644 playbooks/infra/access.yml delete mode 100644 playbooks/infra/basic.yml create mode 100644 playbooks/infra/container_runtime_deployment.yml delete mode 100644 playbooks/infra/full_nfv.yml delete mode 100644 playbooks/infra/on_prem.yml create mode 100644 playbooks/infra/prepare_vms.yml delete mode 100644 playbooks/infra/regional_dc.yml delete mode 100644 playbooks/infra/remote_fp.yml delete mode 100644 playbooks/intel/access.yml delete mode 100644 playbooks/intel/basic.yml delete mode 100644 playbooks/intel/full_nfv.yml delete mode 100644 playbooks/intel/on_prem.yml delete mode 100644 playbooks/intel/regional_dc.yml delete mode 100644 playbooks/intel/remote_fp.yml delete mode 100644 playbooks/on_prem.yml delete mode 100644 playbooks/regional_dc.yml delete mode 100644 playbooks/remote_fp.yml rename playbooks/{access.yml => vm.yml} (60%) delete mode 100644 profiles/group_vars.j2 delete mode 100755 profiles/render.py delete mode 100644 profiles/requirements.txt create mode 100644 requirements.txt delete mode 100644 roles/bootstrap/ansible_host/tasks/main.yml rename roles/{sgx_dp_install/charts/intel-sgx-aesmd/values.yaml => bootstrap/configure_additional_grub_parameters/defaults/main.yml} (80%) create mode 100644 roles/bootstrap/configure_additional_grub_parameters/tasks/main.yml create mode 100644 roles/bootstrap/configure_additional_grub_parameters/tasks/setup_additional_grub_parameters.yml rename roles/{git2_install => bootstrap/configure_additional_grub_parameters}/vars/main.yml (79%) delete mode 100755 roles/bootstrap/configure_cpu_isolation/tasks/autogenerate_isolcpus.yml delete mode 100644 roles/bootstrap/configure_openssl/tasks/install_openssl_1.1.1_centos7.yml delete mode 100644 roles/bootstrap/configure_openssl/tasks/install_req_packages_centos7.yml rename roles/bootstrap/configure_qat/files/{bmra_sriov_qat_init => cek_sriov_qat_init} (94%) rename roles/bootstrap/configure_qat/tasks/{asymmetric_crypto_service_on_qat_confg.yml => check_qat_service.yml} (53%) delete mode 100644 roles/bootstrap/configure_qat/templates/bmra_sriov_qat_init.service.j2 create mode 100644 roles/bootstrap/configure_qat/templates/cek_sriov_qat_init.service.j2 create mode 100644 roles/bootstrap/configure_security/tasks/fw_debian.yaml create mode 100644 roles/bootstrap/configure_security/tasks/fw_redhat.yaml create mode 100644 roles/bootstrap/configure_security/tasks/selinux.yaml create mode 100644 roles/bootstrap/configure_sgx/defaults/main.yml rename roles/{sgx_configure_icx => bootstrap/configure_sgx}/tasks/centos.yml (77%) create mode 100644 roles/bootstrap/configure_sgx/tasks/main.yml rename roles/{sgx_configure_icx => bootstrap/configure_sgx}/tasks/rhel.yml (78%) rename roles/{sgx_configure_icx => bootstrap/configure_sgx}/tasks/ubuntu.yml (64%) rename roles/{sgx_configure_icx => bootstrap/configure_sgx}/vars/main.yml (94%) delete mode 100644 roles/bootstrap/configure_sst/tasks/centos8_2_install_isst_interface.yml delete mode 100644 roles/bootstrap/configure_sst/templates/isst-load-modules-icx.service.j2 rename roles/{ => bootstrap}/golang_install/defaults/main.yml (79%) create mode 100644 roles/bootstrap/golang_install/tasks/main.yml delete mode 100644 roles/bootstrap/install_gpu_kernel/tasks/centos82_dracut_install.yml delete mode 100644 roles/bootstrap/install_rt_package/tasks/main.yml delete mode 100644 roles/bootstrap/install_rt_package/templates/CentOS-RT.repo create mode 100644 roles/bootstrap/update_nic_firmware/defaults/main.yml delete mode 100644 roles/cmk_install/charts/cpu-manager-for-kubernetes/templates/NOTES.txt delete mode 100644 roles/cmk_install/charts/cpu-manager-for-kubernetes/templates/_helpers.tpl delete mode 100644 roles/cmk_install/charts/cpu-manager-for-kubernetes/templates/daemonset.yml delete mode 100644 roles/cmk_install/charts/cpu-manager-for-kubernetes/templates/job.yml delete mode 100644 roles/cmk_install/charts/cpu-manager-for-kubernetes/templates/rbac.yml delete mode 100644 roles/cmk_install/charts/cpu-manager-for-kubernetes/templates/serviceaccount.yml delete mode 100644 roles/cmk_install/charts/cpu-manager-for-kubernetes/templates/webhook.yml delete mode 100644 roles/cmk_install/tasks/main.yml delete mode 100644 roles/cmk_install/tasks/reset.yml delete mode 100644 roles/cmk_install/templates/cmk-webhook.conf.j2 delete mode 100644 roles/cmk_install/templates/helm_values.yml.j2 delete mode 100644 roles/cmk_install/templates/kube_client_csr.yml.j2 delete mode 100644 roles/cmk_install/templates/kube_server_csr.yml.j2 delete mode 100644 roles/cmk_install/templates/webhook_client_csr.json.j2 delete mode 100644 roles/cmk_install/templates/webhook_server_csr.json.j2 create mode 100644 roles/collectd_install/templates/plugin_configuration/turbostat.conf create mode 100644 roles/container_engine/containerd/defaults/main.yml rename roles/{qat_dp_install/charts/intel-qat-plugin/values.yaml => container_engine/containerd/handlers/main.yml} (54%) rename roles/{tas_install/tasks => container_engine/containerd/meta}/main.yml (78%) create mode 100644 roles/container_engine/containerd/tasks/containerd_repo.yml create mode 100644 roles/container_engine/containerd/tasks/main.yml create mode 100644 roles/container_engine/containerd/templates/config.toml.j2 create mode 100644 roles/container_engine/containerd/templates/http_proxy.conf.j2 create mode 100644 roles/container_engine/containerd/templates/rh_containerd.repo.j2 create mode 100644 roles/container_engine/containerd/vars/redhat.yml create mode 100644 roles/container_engine/containerd/vars/ubuntu.yml rename roles/{bootstrap/install_rt_package/vars => container_engine/containerd_common/defaults}/main.yml (50%) create mode 100644 roles/container_engine/containerd_common/tasks/main.yml create mode 100644 roles/container_engine/containerd_common/vars/redhat.yml rename playbooks/basic.yml => roles/container_engine/containerd_common/vars/ubuntu.yml (57%) create mode 100644 roles/container_engine/crictl/defaults/main.yml create mode 100644 roles/container_engine/crictl/handlers/main.yml create mode 100644 roles/container_engine/crictl/tasks/crictl.yml create mode 100644 roles/container_engine/crictl/tasks/main.yml create mode 100644 roles/container_engine/crictl/templates/crictl.yml.j2 create mode 100644 roles/container_engine/crio/defaults/main.yml create mode 100644 roles/container_engine/crio/files/mounts.conf create mode 100644 roles/container_engine/crio/handlers/main.yml create mode 100644 roles/container_engine/crio/tasks/crio_repo.yml create mode 100644 roles/container_engine/crio/tasks/main.yml create mode 100644 roles/container_engine/crio/templates/config.json.j2 create mode 100644 roles/container_engine/crio/templates/crictl.yml.j2 create mode 100644 roles/container_engine/crio/templates/crio.conf.j2 create mode 100644 roles/container_engine/crio/templates/http_proxy.conf.j2 create mode 100644 roles/container_engine/crio/templates/registry_mirror.conf.j2 rename roles/{cmk_install/defaults/main.yml => container_engine/crio/vars/centos-8.yml} (73%) create mode 100644 roles/container_engine/crio/vars/redhat.yml rename roles/{kubespray_install/tasks/main.yml => container_engine/crio/vars/ubuntu.yml} (57%) create mode 100644 roles/container_engine/docker/defaults/main.yml create mode 100644 roles/container_engine/docker/handlers/main.yml create mode 100644 roles/container_engine/docker/meta/main.yml create mode 100644 roles/container_engine/docker/tasks/main.yml create mode 100644 roles/container_engine/docker/tasks/systemd.yml create mode 100644 roles/container_engine/docker/templates/docker.service.j2 create mode 100644 roles/container_engine/docker/templates/docker_options.conf.j2 create mode 100644 roles/container_engine/docker/templates/http_proxy.conf.j2 create mode 100644 roles/container_engine/docker/templates/rh_docker.repo.j2 create mode 100644 roles/container_engine/docker/vars/redhat.yml create mode 100644 roles/container_engine/docker/vars/ubuntu.yml create mode 100644 roles/container_engine/meta/main.yml delete mode 100644 roles/git2_install/tasks/main.yml delete mode 100644 roles/golang_install/tasks/main.yml rename roles/install_ddp_pkgs/{vars => defaults}/main.yml (92%) create mode 100644 roles/install_dpdk/defaults/main.yml create mode 100644 roles/intel_power_manager/defaults/main.yml create mode 100644 roles/intel_power_manager/files/appqos.conf create mode 100644 roles/intel_power_manager/files/rbac.patch create mode 100644 roles/intel_power_manager/tasks/app_qos.yml create mode 100644 roles/intel_power_manager/tasks/deploy_sample_pods.yml create mode 100644 roles/intel_power_manager/tasks/deploy_shared_resources.yml create mode 100644 roles/intel_power_manager/tasks/main.yml create mode 100644 roles/intel_power_manager/tasks/power_manager.yml create mode 100644 roles/intel_power_manager/tasks/power_pod_template_helper.yml create mode 100644 roles/intel_power_manager/templates/controller_manager.yml.j2 create mode 100644 roles/intel_power_manager/templates/global_shared_profile.yml.j2 create mode 100644 roles/intel_power_manager/templates/local_shared_profile.yml.j2 create mode 100644 roles/intel_power_manager/templates/power_config.yml.j2 create mode 100644 roles/intel_power_manager/templates/sample_power_pod.yml.j2 create mode 100644 roles/intel_power_manager/templates/shared_workload.yml.j2 rename roles/{cmk_install => intel_power_manager}/vars/main.yml (93%) delete mode 100644 roles/istio_operator_install/defaults/main.yml delete mode 100644 roles/istio_operator_install/tasks/cleanup.yml delete mode 100644 roles/istio_operator_install/tasks/install.yml delete mode 100644 roles/istio_operator_install/templates/istioctl-operator-init-options.yml.j2 delete mode 100644 roles/istio_operator_install/templates/istioctl-operator-remove-options.yml.j2 delete mode 100644 roles/istio_operator_install/templates/profiles/default.yaml.j2 delete mode 100644 roles/istio_operator_install/templates/profiles/empty.yaml.j2 delete mode 100644 roles/istio_operator_install/templates/profiles/external.yaml.j2 delete mode 100644 roles/istio_operator_install/templates/profiles/minimal.yaml.j2 create mode 100644 roles/kmra_install/charts/kmra-apphsm/templates/kmra-apphsm-qcnl-configmap.yaml create mode 100644 roles/kmra_install/charts/kmra-ctk/templates/kmra-ctk-loadkey-qcnl-configmap.yaml rename roles/kubespray_target_setup/templates/{config.yaml => config.yaml.j2} (51%) rename roles/{kubespray_install => kubespray_target_setup}/templates/multus.conf.j2 (99%) create mode 100644 roles/minio_install/charts/operator/Chart.yaml create mode 100755 roles/minio_install/charts/operator/crds/minio.min.io_tenants.yaml create mode 100755 roles/minio_install/charts/operator/templates/NOTES.txt create mode 100755 roles/minio_install/charts/operator/templates/_helpers.tpl create mode 100755 roles/minio_install/charts/operator/templates/cluster-role-binding.yaml create mode 100755 roles/minio_install/charts/operator/templates/cluster-role.yaml create mode 100755 roles/minio_install/charts/operator/templates/console-deployment.yaml create mode 100755 roles/minio_install/charts/operator/templates/console-ingress.yaml create mode 100755 roles/minio_install/charts/operator/templates/console-service.yaml create mode 100755 roles/minio_install/charts/operator/templates/console-ui.yaml create mode 100755 roles/minio_install/charts/operator/templates/operator-deployment.yaml create mode 100755 roles/minio_install/charts/operator/templates/operator-service.yaml create mode 100755 roles/minio_install/charts/operator/templates/serviceaccount.yaml create mode 100644 roles/minio_install/charts/operator/values.yaml rename roles/{qat_dp_install/charts/intel-qat-plugin => minio_install/charts/tenant}/Chart.yaml (59%) mode change 100644 => 100755 create mode 100755 roles/minio_install/charts/tenant/templates/NOTES.txt create mode 100755 roles/minio_install/charts/tenant/templates/_helpers.tpl create mode 100755 roles/minio_install/charts/tenant/templates/tenant-secret.yaml create mode 100755 roles/minio_install/charts/tenant/templates/tenant.yaml create mode 100755 roles/minio_install/charts/tenant/values.yaml rename roles/{sgx_configure_icx/tasks/main.yml => minio_install/defaults/main.yaml} (53%) mode change 100644 => 100755 create mode 100644 roles/minio_install/tasks/create_blockdevicefiles.yml create mode 100644 roles/minio_install/tasks/create_nvme_partition.yml create mode 100644 roles/minio_install/tasks/create_persistentvolumes.yml rename roles/{istio_operator_install/tasks/deploy_istio_profile.yml => minio_install/tasks/create_storageclass.yml} (55%) create mode 100644 roles/minio_install/tasks/file_blockdevice.yml create mode 100644 roles/minio_install/tasks/format_blockdevicefiles.yml create mode 100644 roles/minio_install/tasks/main.yml create mode 100644 roles/minio_install/tasks/minio_operator.yml create mode 100644 roles/minio_install/tasks/minio_tenant.yml create mode 100644 roles/minio_install/tasks/mount_loopdevices.yml create mode 100644 roles/minio_install/tasks/nvme_blockdevice.yml rename roles/{collectd_install/vars/main.yml => minio_install/tasks/setup_loopdevices.yml} (74%) create mode 100644 roles/minio_install/templates/minio_operator_custom_values.yml.j2 create mode 100644 roles/minio_install/templates/minio_tenant_custom_values.yml.j2 create mode 100644 roles/minio_install/templates/minio_tenant_localpersistentvolumes.yml.j2 create mode 100644 roles/minio_install/templates/minio_tenant_storageclass.yml.j2 rename roles/{tas_install => openssl_engine_install}/vars/main.yml (84%) rename roles/{cmk_install/charts/cpu-manager-for-kubernetes => platform_aware_scheduling_install/charts/gpu-aware-scheduling}/Chart.yaml (74%) rename roles/{cmk_install/charts/cpu-manager-for-kubernetes => platform_aware_scheduling_install/charts/gpu-aware-scheduling/templates}/.helmignore (100%) create mode 100644 roles/platform_aware_scheduling_install/charts/gpu-aware-scheduling/templates/NOTES.txt rename roles/{qat_dp_install/charts/intel-qat-plugin => platform_aware_scheduling_install/charts/gpu-aware-scheduling}/templates/_helpers.tpl (50%) create mode 100644 roles/platform_aware_scheduling_install/charts/gpu-aware-scheduling/templates/gas-deployment.yaml create mode 100644 roles/platform_aware_scheduling_install/charts/gpu-aware-scheduling/templates/gas-rbac-accounts.yaml create mode 100644 roles/platform_aware_scheduling_install/charts/gpu-aware-scheduling/templates/gas-service.yaml create mode 100644 roles/platform_aware_scheduling_install/charts/gpu-aware-scheduling/values.yaml rename roles/{qat_dp_install/charts/intel-qat-plugin => platform_aware_scheduling_install/charts/telemetry-aware-scheduling}/.helmignore (100%) rename roles/{tas_install => platform_aware_scheduling_install}/charts/telemetry-aware-scheduling/Chart.yaml (83%) create mode 100644 roles/platform_aware_scheduling_install/charts/telemetry-aware-scheduling/crds/tas-policy-crd.yaml rename roles/{tas_install => platform_aware_scheduling_install}/charts/telemetry-aware-scheduling/templates/NOTES.txt (85%) rename roles/{tas_install => platform_aware_scheduling_install}/charts/telemetry-aware-scheduling/templates/_helpers.tpl (84%) rename roles/{tas_install/templates/tas-demo-policy.yml.j2 => platform_aware_scheduling_install/charts/telemetry-aware-scheduling/templates/tas-demo-policy.yaml} (81%) rename roles/{tas_install/charts/telemetry-aware-scheduling/templates/tas-deployment.yml => platform_aware_scheduling_install/charts/telemetry-aware-scheduling/templates/tas-deployment.yaml} (73%) rename roles/{tas_install/charts/telemetry-aware-scheduling/templates/tas-rbac-accounts.yml => platform_aware_scheduling_install/charts/telemetry-aware-scheduling/templates/tas-rbac-accounts.yaml} (63%) rename roles/{tas_install/charts/telemetry-aware-scheduling/templates/tas-service.yml => platform_aware_scheduling_install/charts/telemetry-aware-scheduling/templates/tas-service.yaml} (85%) rename roles/{tas_install => platform_aware_scheduling_install}/charts/telemetry-aware-scheduling/values.yaml (86%) create mode 100644 roles/platform_aware_scheduling_install/defaults/main.yml create mode 100644 roles/platform_aware_scheduling_install/tasks/backups_restore.yml create mode 100644 roles/platform_aware_scheduling_install/tasks/build_image.yml create mode 100644 roles/platform_aware_scheduling_install/tasks/create-scheduler-config.yml create mode 100644 roles/platform_aware_scheduling_install/tasks/gas.yml create mode 100644 roles/platform_aware_scheduling_install/tasks/main.yml create mode 100644 roles/platform_aware_scheduling_install/tasks/tas.yml create mode 100644 roles/platform_aware_scheduling_install/tasks/tls.yml create mode 100644 roles/platform_aware_scheduling_install/templates/gas-chart.yaml.j2 create mode 100644 roles/platform_aware_scheduling_install/templates/gas-values.yaml.j2 create mode 100644 roles/platform_aware_scheduling_install/templates/pas-cluster-role-configmapgetter.yaml.j2 create mode 100644 roles/platform_aware_scheduling_install/templates/pas-extenders-configmap.yaml.j2 create mode 100644 roles/platform_aware_scheduling_install/templates/tas-chart.yaml.j2 create mode 100644 roles/platform_aware_scheduling_install/templates/tas-values.yaml.j2 create mode 100644 roles/platform_aware_scheduling_install/templates/tls-extender-ca.json.j2 create mode 100644 roles/platform_aware_scheduling_install/templates/tls-extender-client.json.j2 create mode 100644 roles/platform_aware_scheduling_install/templates/tls-extender.json.j2 create mode 100644 roles/platform_aware_scheduling_install/vars/main.yml delete mode 100644 roles/qat_dp_install/charts/intel-qat-plugin/templates/NOTES.txt delete mode 100644 roles/qat_dp_install/charts/intel-qat-plugin/templates/qat_plugin.yaml delete mode 100644 roles/qat_dp_install/charts/intel-qat-plugin/templates/qat_plugin_configmap.yaml create mode 100644 roles/qat_dp_install/files/noiommu.patch create mode 100644 roles/qat_dp_install/templates/intel-qat-plugin.yml.j2 delete mode 100644 roles/qat_dp_install/templates/qat_values.yml.j2 create mode 100644 roles/redeploy_cleanup/tasks/remove_minio.yml create mode 100644 roles/service_mesh_install/defaults/main.yml create mode 100644 roles/service_mesh_install/files/profiles/.gitkeep create mode 100644 roles/service_mesh_install/tasks/cleanup.yml create mode 100644 roles/service_mesh_install/tasks/install.yml rename roles/{istio_operator_install => service_mesh_install}/tasks/main.yml (58%) create mode 100644 roles/service_mesh_install/tasks/tcpip-bypass-ebpf.yml create mode 100644 roles/service_mesh_install/templates/istioctl-options.yml.j2 create mode 100644 roles/service_mesh_install/templates/tcpip-bypass-ebpf.yaml.j2 create mode 100644 roles/service_mesh_install/vars/main.yml delete mode 100644 roles/sgx_configure_icx/defaults/main.yml create mode 100644 roles/sriov_network_operator_install/tasks/custom_sriov_network_policies_configure.yml rename roles/sriov_nic_init/files/{bmra_sriov_nic_init => cek_sriov_nic_init} (95%) delete mode 100644 roles/sriov_nic_init/templates/bmra_sriov_nic_init.service.j2 create mode 100644 roles/sriov_nic_init/templates/cek_sriov_nic_init.service.j2 delete mode 100644 roles/tas_install/charts/tas-policy-crd.yml delete mode 100644 roles/tas_install/charts/telemetry-aware-scheduling/.helmignore delete mode 100644 roles/tas_install/charts/telemetry-aware-scheduling/templates/tas-configmap.yml delete mode 100644 roles/tas_install/defaults/main.yml delete mode 100644 roles/tas_install/tasks/tas.yml delete mode 100644 roles/tas_install/templates/tas-values.yml.j2 create mode 100644 roles/telegraf_install/tasks/resctrl_wa_for_vms.yml create mode 100644 roles/vm/conf_libvirt/tasks/main.yml create mode 100644 roles/vm/conf_libvirt/vars/main.yml create mode 100644 roles/vm/manage_imgs/defaults/main.yml create mode 100644 roles/vm/manage_imgs/tasks/main.yml create mode 100644 roles/vm/manage_imgs/templates/backup-user-data.j2 create mode 100644 roles/vm/manage_imgs/templates/meta-data.j2 create mode 100644 roles/vm/manage_imgs/templates/test-user-data.j2 create mode 100644 roles/vm/manage_imgs/templates/user-data.j2 create mode 100644 roles/vm/manage_vms/tasks/main.yml create mode 100644 roles/vm/manage_vms/tasks/optimize.yml create mode 100644 roles/vm/prepare_cek/tasks/main.yml diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 00000000..9950c8e1 --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,73 @@ +# Contributing to Container Experience Kits + +Anyone is welcome to contribute. + +Please file bugs under github issues. + +Please submit git commits and request PRs for review and inclusion. Titles should be clear and concise +with details provided in commit log explaining the purpose and method. Please include reference to github +issue if applicable. Providing testing details will expedite the review process. + +## Licenses + +The container-experience-kits source is covered under the Apache 2.0 License. +See http://www.apache.org/licenses/ + +Before submitting a patch, ensure there are no licensing issues by following the Developer Certificate of +Origin (DCO) process. + +The DCO is an attestation attached to every contribution. The commit log must have a Signed-off-by line +(-signoff option), which certifies that you wrote it and/or have the right to submit it. The format of the +sign-off message is expected to appear on each commit in the pull request like so: + +``` +Signed-off-by: First Last +``` + +For additional explanation, see below or at https://developercertificate.org/. + +``` +Developer Certificate of Origin +Version 1.1 + +Copyright (C) 2004, 2006 The Linux Foundation and its contributors. + +Everyone is permitted to copy and distribute verbatim copies of this +license document, but changing it is not allowed. + + +Developer's Certificate of Origin 1.1 + +By making a contribution to this project, I certify that: + +(a) The contribution was created in whole or in part by me and I + have the right to submit it under the open source license + indicated in the file; or + +(b) The contribution is based upon previous work that, to the best + of my knowledge, is covered under an appropriate open source + license and I have the right under that license to submit that + work with modifications, whether created in whole or in part + by me, under the same open source license (unless I am + permitted to submit under a different license), as indicated + in the file; or + +(c) The contribution was provided directly to me by some other + person who certified (a), (b) or (c) and I have not modified + it. + +(d) I understand and agree that this project and the contribution + are public and that a record of the contribution (including all + personal information I submit with it, including my sign-off) is + maintained indefinitely and may be redistributed consistent with + this project or the open source license(s) involved. +``` + +## Releases + +Our primary release mechanism is based on a calendar versioning standard, formatted as YY.MM. +- YY as the last two digits of the year. +- MM as the zero padded month of the year. + +Fixes for issues will most often be rolled into a follow on release. Any exceptions added outside of a release +cycle will be accompanied by a tag update for the current released version. diff --git a/Makefile b/Makefile index 3d10207c..00e28bb4 100644 --- a/Makefile +++ b/Makefile @@ -1,12 +1,87 @@ -BMRA_DIRECTORIES_WITH_SHELL_FILES ?= roles/ examples/ playbooks/infra/ playbooks/intel/ +CEK_DIRECTORIES_WITH_SHELL_FILES ?= roles/ examples/ playbooks/infra/ playbooks/intel/ +ARCH ?= 'icx' +PLAYBOOKS_DIRS = playbooks playbooks/infra playbooks/intel +PLAYBOOK_NAMES = access basic full_nfv on_prem regional_dc remote_fp storage + +# set default target available with simple 'make' command +.DEFAULT_GOAL := examples + +.PHONY: shellcheck ansible-lint all-profiles clean clean-playbooks help k8s-profiles vm-profiles shellcheck: - find $(BMRA_DIRECTORIES_WITH_SHELL_FILES) -type f \( -name '*.sh' -o -name '*.bash' -o -name '*.ksh' -o -name '*.bashrc' -o -name '*.bash_profile' -o -name '*.bash_login' -o -name '*.bash_logout' \) \ + find $(CEK_DIRECTORIES_WITH_SHELL_FILES) -type f \( -name '*.sh' -o -name '*.bash' -o -name '*.ksh' -o -name '*.bashrc' -o -name '*.bash_profile' -o -name '*.bash_login' -o -name '*.bash_logout' \) \ | xargs shellcheck ansible-lint: ansible-lint playbooks/* roles/* -c .ansible-lint -profile ?= '' -bmra-profiles: - python3 profiles/render.py --config profiles/profiles.yml --host profiles/host_vars.j2 --group profiles/group_vars.j2 --inventory profiles/inventory.j2 --output examples -p $(profile) +# make sure PROFILE is set to an 'all_examples' string for 'examples' and empty target +ifeq ($(MAKECMDGOALS), $(filter $(MAKECMDGOALS),examples '')) +override PROFILE = 'all_examples' +endif + +# make sure PROFILE is defined for mode-related targets +ifndef PROFILE +ifeq ($(MAKECMDGOALS), $(filter $(MAKECMDGOALS),k8s-profile vm-profile)) +$(error please specify which profile should be generated, e.g. PROFILE=basic. Run 'make help' for more information.) +endif +endif + +examples: k8s-profile vm-profile + +k8s-profile: clean-playbooks + python3 generate/render.py \ + --config generate/profiles_templates/k8s/profiles.yml \ + --host generate/profiles_templates/common/host_vars.j2 \ + --group generate/profiles_templates/common/group_vars.j2 \ + --inventory generate/profiles_templates/k8s/inventory.j2 \ + --output examples/k8s \ + --mode k8s \ + -p $(PROFILE) \ + -a $(ARCH) + +vm-profile: clean-playbooks + python3 generate/render.py \ + --config generate/profiles_templates/vm/vm_host_profiles.yml \ + --vmsconfig generate/profiles_templates/vm/vms_profiles.yml \ + --host generate/profiles_templates/common/host_vars.j2 \ + --group generate/profiles_templates/common/group_vars.j2 \ + --inventory generate/profiles_templates/vm/inventory.j2 \ + --output examples/vm \ + --mode vm \ + -p $(PROFILE) \ + -a $(ARCH) + +clean: clean-playbooks clean-project-root-dir + +clean-backups: + rm -rf backups + +clean-project-root-dir: + rm -rf examples host_vars group_vars inventory.ini + +clean-playbooks: + for d in $(PLAYBOOKS_DIRS) ; do for n in $(PLAYBOOK_NAMES) ; do rm -f $$d/$$n.yml ; done done + +help: + @echo "Cleaning targets:" + @echo " clean - removes examples directory," + @echo " all host_vars and group_vars dirs," + @echo " inventory files and playbooks" + @echo "" + @echo " clean-backups - clean generated backup files." + @echo "" + @echo "Genertare example profiles:" + @echo " make, examples - generate sample files of all available profiles." + @echo "" + @echo "Generating k8s profile:" + @echo " k8s-profile PROFILE= - generate files required for deployment of specific profile in k8s mode." + @echo "" + @echo "Generating VM profile:" + @echo " vm-profile PROFILE= - generate files required for deployment of specific profile in vm mode." + @echo "" + @echo "For more information about:" + @echo " - profiles generation" + @echo " - supported architectures" + @echo " - available profiles" + @echo "please read the docs/generate_profiles.md file." diff --git a/README.md b/README.md index 853b90bd..493dba0b 100644 --- a/README.md +++ b/README.md @@ -12,7 +12,8 @@ The software provided here is for reference only and not intended for production git submodule update --init ``` -2. Decide which configuration profile you want to use and optionally export environmental variable. (> **_NOTE:_** It will be used only to ease execution of the steps listed below.) +2. Decide which configuration profile you want to use and optionally export environmental variable. + > **_NOTE:_** It will be used only to ease execution of the steps listed below. - For **Kubernetes Basic Infrastructure** deployment: ```bash @@ -49,33 +50,43 @@ The software provided here is for reference only and not intended for production export PROFILE=full_nfv ``` + - For **Kubernetes Storage Infrastructure** deployment: + + ```bash + export PROFILE=storage + ``` + 3. Install dependencies ```bash - pip3 install -r profiles/requirements.txt + pip3 install -r requirements.txt ``` -4. Generate example host_vars, group_vars and inventory for BMRA profiles. +4. Generate example host_vars, group_vars and inventory files for Intel Container Experience Kits profiles. + + > **_NOTE:_** It is **highly recommended** to read [this](docs/generate_profiles.md) file before profiles generation. ```bash - make bmra-profiles + make examples ``` - > **_NOTE:_** You can provide the optional `profile` argument to automatically copy files needed for deployment. Then, you can skip both the 4th and the 6th steps. +5. Copy example inventory file to the project root dir. ```bash - make bmra-profiles profile=$PROFILE + cp examples/k8s/${PROFILE}/inventory.ini . ``` -5. Copy example inventory file to the project root dir. + or, for VM case: ```bash - cp examples/${PROFILE}/inventory.ini . + cp examples/vm/${PROFILE}/inventory.ini . ``` 6. Update inventory file with your environment details. - > **_NOTE:_** at this stage you can inspect your target environment by running: + For VM case: update details relevant for vm_host + + > **_NOTE:_** At this stage you can inspect your target environment by running: ```bash ansible -i inventory.ini -m setup all > all_system_facts.txt @@ -86,14 +97,26 @@ The software provided here is for reference only and not intended for production 7. Copy group_vars and host_vars directories to the project root dir. ```bash - cp -r examples/${PROFILE}/group_vars examples/${PROFILE}/host_vars . + cp -r examples/k8s/${PROFILE}/group_vars examples/k8s/${PROFILE}/host_vars . + + or + + For VM case: + cp -r examples/vm/${PROFILE}/group_vars examples/vm/${PROFILE}/host_vars . ``` 8. Update group and host vars to match your desired configuration. Refer to [this section](#configuration) for more details. > **_NOTE:_** Please pay special attention to the `http_proxy`, `https_proxy` and `additional_no_proxy` vars if you're behind proxy. -9. RECOMMENDED: Apply bug fix patch for Kubespray submodule (Required for RHEL 8+). + For VM case: + - update details relevant for vm_host (e.g.: datalane_interfaces, ...) + - update VMs definition in host_vars/host-for-vms-1.yml + - update/create host_vars for all defined VMs (e.g.: host_vars/vm-ctrl-1.yml and host_vars/vm-work-1.yml) + Needed details are at least dataplane_interfaces + For more details see [VM case configuration guide](docs/vm_config_guide.md) + +9. **Recommended:** Apply bug fix patch for Kubespray submodule (Required for RHEL 8+). ```bash ansible-playbook -i inventory.ini playbooks/k8s/patch_kubespray.yml @@ -105,23 +128,34 @@ The software provided here is for reference only and not intended for production ansible-playbook -i inventory.ini playbooks/${PROFILE}.yml ``` + or, for VM case: + + ```bash + ansible-playbook -i inventory.ini playbooks/vm.yml + ``` + + > **_NOTE:_** VMs are accessible from ansible host via ssh vm-ctrl-1 or ssh vm-work-1 + ## Configuration Refer to the documentation linked below to see configuration details for selected capabilities and deployment profiles. - [SRIOV Network Device Plugin and SRIOV CNI plugin](docs/sriov.md) - +- [MinIO Operator](docs/storage.md) +- [VM case configuration guide](docs/vm_config_guide.md) ## Prerequisites and Requirements -- Python present on the target servers depending on the target distribution. Python 3 is highly recommended, but Python 2 is still supported for CentOS 7. -- Ansible 2.9.20 installed on the Ansible host machine (the one you run these playbooks from). +> **_NOTE:_** Packages requirements might be installed in 3rd step. + +- Python present on the target servers depending on the target distribution. Python3 is required. +- Ansible 3.4.0 and ansible-base 2.10.15 installed on the Ansible host machine (the one you run these playbooks from). - python-pip3 installed on the Ansible machine. - python-netaddr installed on the Ansible machine. - SSH keys copied to all Kubernetes cluster nodes (`ssh-copy-id @` command can be used for that). - Internet access on all target servers is mandatory. Proxy is supported. - At least 8GB of RAM on the target servers/VMs for minimal number of functions (some Docker image builds are memory-hungry and may cause OOM kills of Docker registry - observed with 4GB of RAM), more if you plan to run heavy workloads such as NFV applications. -- For the `RHEL`-like OSes `SELinux` must be configured prior to the BMRA deployment and required `SELinux`-related packages should be installed. - `BMRA` itself is keeping initial `SELinux` state but `SELinux`-related packages might be installed during `k8s` cluster deployment as a dependency, for `Docker` engine e.g., +- For the `RHEL`-like OSes `SELinux` must be configured prior to the CEK deployment and required `SELinux`-related packages should be installed. + `CEK` itself is keeping initial `SELinux` state but `SELinux`-related packages might be installed during `k8s` cluster deployment as a dependency, for `Docker` engine e.g., causing OS boot failure or other inconsistencies if `SELinux` is not configured properly. Preferable `SELinux` state is `permissive`. For more details, please, refer to the respective OS documentation. diff --git a/action_plugins/validate_isolcpus.py b/action_plugins/validate_isolcpus.py index 5fcc3069..2a1b858c 100644 --- a/action_plugins/validate_isolcpus.py +++ b/action_plugins/validate_isolcpus.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2020-2021 Intel Corporation. +# Copyright (c) 2020-2022 Intel Corporation. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/docs/generate_profiles.md b/docs/generate_profiles.md new file mode 100644 index 00000000..815856e9 --- /dev/null +++ b/docs/generate_profiles.md @@ -0,0 +1,107 @@ +# Container Experience Kits (CEK) Profiles Generation + +1. [Installing Dependencies](#install-dependencies) +2. [Creating Sample Profiles](#creating-sample-profiles) +3. [Discover Supported Architectures](#discover-supported-architectures) +4. [Discover Supported Profiles](#discover-supported-profiles) +5. [Example Commands](#example-commands) +6. [Playbooks Generation](#playbook-generation) + +--- + +## Install Dependencies: + +```bash +pip3 install -r requirements.txt +``` +## Creating Sample Profiles + +To create sample profiles one of the following commands must be executed: + +```bash +make +``` + +or + +```bash +make examples +``` + +After successful profiles generation, the results might be investigated in the `examples` directory. +The two directories should be visible: + +* k8s - contains profiles deployable on the bare metal machines +* vm - contains profiles deployable on the virtual machines + +These two directories represent available modes of the CEK project. + +## Discover Supported Architectures + +At the moment, Container Experience Kits supports the following machine architectures: + +* `spr` - Sapphire Rapids - '4th Generation Intel(R) Xeon(R) Scalable Processor' +* `icx` - IceLake (default) - '3rd Generation Intel(R) Xeon(R) Scalable Processor' +* `clx` - CascadeLake - '2nd Generation Intel(R) Xeon(R) Scalable Processor' +* `skl` - SkyLake - '1st Generation Intel(R) Xeon(R) Scalable Processor' + +Architecture acronym, e.g. skl, will be passed to the make command via the optional `ARCH` parameter. It is important to be aware of the machine's type before generating profiles as some functionalities are exclusive for specific architectures. + +## Discover Supported Profiles + +At the moment, Container Experience Kits supports the following profiles: + +* access +* basic +* full_nfv +* on_prem +* regional_dc +* remote_fp +* storage + +Profile's name will be passed to the make command via the required `PROFILE` parameter. Each profile includes some specific sort of functionalities. Choose the profile that suits you the most via inspecting the examples generated [here](#creating-sample-profiles). +If you would like to know more about CEK profiles read section 2.2 in [here](https://networkbuilders.intel.com/solutionslibrary/container-bare-metal-for-2nd-3rd-generation-intel-xeon-scalable-processor). + + +## Example Commands + +To generate files needed for deployment of `full_nfv` profile, for `Sapphire Rapids` machines, in `k8s` mode, the following command must be executed: + +```bash +make k8s-profile PROFILE=full_nfv ARCH=spr +``` + +To generate the same profile as above, but for `vm` mode, run: + +```bash +make vm-profile PROFILE=full_nfv ARCH=spr +``` + +The values of both `PROFILE` and `ARCH` parameters are up to you. Please update accordingly. + +If you run both commands from above, you should see backups folder in your project root directory: + +```bash +ls backups/ +``` +> **_NOTE:_** Above command will result in an output similar to this: "container-experience-kits$ backups/full_nfv_20220121_141523/" and within the mentioned folder location "group_vars, host_vars, inventory.ini" files can be found which will be backup of earlier prepared deployments. + +Backups folder is created so that earlier prepared deployments are not lost. That way, you can easily switch between profiles deployment. Each backup contains a unique timestamp. + +To clean files and directories that were created by make commands, please run: + +```bash +make clean +``` + +This command will not remove backup directories. If you would like to remove all generated files and directories, please run: + +```bash +make clean-all +``` + +## Playbook Generation + +Before profiles generation playbooks are not available. To generate playbooks no additional command must be executed. + +They would be created as soon as the first `make` command is executed. diff --git a/docs/power_operator.md b/docs/power_operator.md new file mode 100644 index 00000000..b987d4d9 --- /dev/null +++ b/docs/power_operator.md @@ -0,0 +1,162 @@ +# Intel Power Manager + +1. [Introduction](#introduction) +2. [Check the existence of sample power pods on the cluster](#check-the-existence-of-sample-power-pods-on-the-cluster) +3. [Check the frequencies which will be set by `balance-performance` Power Profile](#check-the-frequencies-which-will-be-set-by-balance-performance-power-profile) +4. [Obtain cores on which `balance-performance` Power Profile is applied](#obtain-cores-on-which-balance-performance-power-profile-is-applied) +5. [Check the frequencies on cores](#check-the-frequencies-on-cores) +6. [The Shared Profile](#the-shared-profile) +7. [Known limitations](#known-limitations) + +--- + +## Introduction + +Intel Power Manager is available for icx, spr, and clx architectures (you can find more about supported architectures in `generate_profiles` docs), and can be enabled in group vars. + +After a successful deployment, the user can utilize special resources to manipulate cores' frequencies. +Sample pods can be deployed by setting `deploy_example_pods: true` in group vars. + +The results of Power Manager work can be obtained in the following way: + +## Check the existence of sample power pods on the cluster + +```bash +kubectl get pods -n intel-power +NAME READY STATUS RESTARTS AGE +balance-performance-power-pod 1/1 Running 0 21m +balance-power-power-pod 1/1 Running 0 21m +controller-manager-f584c9458-682p5 1/1 Running 0 16h +performance-power-pod 1/1 Running 0 21m +power-node-agent-8cxmp 2/2 Running 0 16h +``` + +> NOTE: output may be different depending on the number of nodes and requested Power Profiles. + +Three pods, one for each profile, were deployed. Let's stick to `balance-performance-power-pod`. + +## Check the frequencies which will be set by `balance-performance` Power Profile + +```bash +kubectl get PowerProfiles -n intel-power balance-performance-node1 -o yaml +apiVersion: power.intel.com/v1alpha1 +kind: PowerProfile +metadata: + creationTimestamp: "2022-01-25T17:07:08Z" + generation: 1 + name: balance-performance-node1 + namespace: intel-power + resourceVersion: "17538" + uid: 05599219-d042-4b9c-9bbf-42ef67effd24 +spec: + epp: balance_performance + max: 2700 + min: 2500 + name: balance-performance-node1 +``` + +> NOTE: The max/min frequencies may differ on your machine. + +In `spec` the values max and min represent new frequencies that will be set to specific cores. + +## Obtain cores on which `balance-performance` Power Profile is applied + +```bash +kubectl get PowerWorkloads -n intel-power balance-performance-node1-workload -o yaml +apiVersion: power.intel.com/v1alpha1 +kind: PowerWorkload +metadata: + creationTimestamp: "2022-01-26T10:12:00Z" + generation: 1 + name: balance-performance-node1-workload + namespace: intel-power + resourceVersion: "246287" + uid: f8720a7e-f7b2-4f31-bf4f-2a38ad8a7c07 +spec: + name: balance-performance-node1-workload + nodeInfo: + containers: + - exclusiveCpus: + - 2 + - 66 + id: 495d5547a5211774e605c4a2ebe4b9fbcf44fbd056cc08e0847b68143627700a + name: balance-performance-container + pod: balance-performance-power-pod + powerProfile: balance-performance-node1 + cpuIds: + - 2 + - 66 + name: node1 + powerProfile: balance-performance-node1 +``` + +> > NOTE: The cores may differ on your machine. + +`balance-performance` Power Profile is applied to core numbers 2 and 66 + +You can also check all assigned cores in your Power Nodes with the following command: + +```bash +kubectl get PowerNodes -A -o yaml +``` + +## Check the frequencies on cores + +```bash +cat /sys/devices/system/cpu/cpu2/cpufreq/scaling_max_freq +2700000 +cat /sys/devices/system/cpu/cpu2/cpufreq/scaling_min_freq +2500000 +cat /sys/devices/system/cpu/cpu66/cpufreq/scaling_max_freq +2700000 +cat /sys/devices/system/cpu/cpu66/cpufreq/scaling_min_freq +2500000 +``` + +In comparison, the core that was not obtained by Power Workload has the following values: + +```bash +cat /sys/devices/system/cpu/cpu22/cpufreq/scaling_max_freq +3500000 +cat /sys/devices/system/cpu/cpu22/cpufreq/scaling_min_freq +800000 +``` + +> NOTE: The frequencies may differ on your machine. + +## The Shared Profile + +The Shared Profile is a custom profile that can be defined by the user. It requires deploying the custom Shared Workload as well. +The main purpose of Shared Profile is to allow users to customization of core frequencies. + +The Shared Profile has either a cluster-wide or single node impact. The Shared Workload is deployed per node, so the cores might be scaled. + +**The user is responsible for setting the correct frequency values for Shared Profile. This cannot be checked in preflight!** + +The resources for Shared Profile are not visible in allocatable kubelet resources as cores will be scaled as soon as Shared Workload is deployed. + +## Known limitations + +1. The Performance Power Profile + +The CPUs with `CPU max MHz` lower than 3600 are not capable of successfully applying the `performance` Power Profile. +Despite the fact that Power Profile seems to be deployed and the Power Workload reserves the cores the frequencies will not be applied. + +To check if your machine is capable of using the `performance` Power Profile use the following command: + +```bash +lscpu | grep "CPU max MHz" +CPU max MHz: 3500.0000 +``` + +You can see that this machine is not capable of utilizing the `performance` Power Profile. + +2. The Shared Power Profile + +More than one Shared Power Profile cannot be used on the same node. For example, it is not possible to use a global shared power profile configured in group vars and at the same time scale core with config for a specific node. + +Shared Profile will grab all cores that are not marked as exclusive - please consider not deploying shared profile if special pods will need access to cores scaled via performance, balance-performance, or balance-power profiles. + +Due to strong dependency on AppQoS the list for exclusive CPUs must not be empty even if there are no exclusive CPUs in the kubelet config at the moment. Please put the last core from the machine to the list of exclusive CPUs in host vars in that case. + +Shared Workload **may not** obtain all available cores, but will grab ones from the default pool if other profiles released them. diff --git a/docs/sriov.md b/docs/sriov.md index 3b5e9fca..a984b4c9 100644 --- a/docs/sriov.md +++ b/docs/sriov.md @@ -43,7 +43,7 @@ Next option defines whether the SRIOV CNI plugin will be installed on the target sriov_cni_enabled: true` ``` -If `sriov_net_dp_enabled` is set to `true` in all.yml (group vars), plase adjust and uncomment below configuration in the node host vars file. Below dictionary will be used to prepare and apply SRIOV Network Device Plugin configuration. +If `sriov_net_dp_enabled` is set to `true` in all.yml (group vars), please adjust and uncomment below configuration in the node host vars file. Below dictionary will be used to prepare and apply SRIOV Network Device Plugin configuration. In the example below we use PF names of the interfaces that we enabled SRIOV for in the above example. Then we define driver bindings for each of them. VFs created on `enp175s0f1` and `enp175s0f2` will be attached to the userspace vfio-pci and igb_uio drivers respectively, which will make them available for use with the userspace dataplane applications. This configuration will also cause assignment of VFs to appropriate resource pools in the SRIOV Network Device Plugin: `intel_sriov_netdevice` for the `enp175s0f0` VFs and `intel_sriov_dpdk` for the `enp175s0f1` VFs. ``` sriov_net_dp_config: diff --git a/docs/storage.md b/docs/storage.md new file mode 100644 index 00000000..e603ab38 --- /dev/null +++ b/docs/storage.md @@ -0,0 +1,82 @@ +# MinIO Operator/Console/Tenant + +## Cluster configuration options + +In order to install MinIO Operator/Console set `minio_enabled` value to `true` in your group vars file. Setting it to `false` will disable Minio Operator/Console installation and cause other related options to be ignored. +```yaml +minio_enabled: true +``` + +In order to install MinIO Tenant sample set `minio_tenant_enabled` value to `true` in your group vars file. Setting it to `false` won't install Minio Tenant sample and cause other related options(host vars) to be ignored. +```yaml +minio_tenant_enabled: true +``` + +You can also change the MinIO Operator/Console namespace used for the MinIO Operator/Console deployment - by default it's `minio-operator`. +```yaml +minio_operator_namespace: minio-operator +``` + +You can also change the MinIO Tenant namespace used for the MinIO Tenant sample deployment - by default it's `minio-tenant`. +```yaml +minio_tenant_namespace: minio-tenant +``` +### Access to MinIO Console + +From your controller, get the JWT (Jason Web Token) to the MinIO Console Access +```bash +kubectl get secret $(kubectl get serviceaccount console-sa --namespace minio-operator -o jsonpath="{.secrets[0].name}") --namespace minio-operator -o jsonpath="{.data.token}" | base64 --decode +``` + +Now you will need to forward the port of MinIO Console service. +```bash +kubectl --namespace minio-operator port-forward svc/console 9090:9090 +``` + +Depending on your environment, you can use tunnel to access your console. Here, I used the remote tunnel on x.x.x.x which I can open the MinIO Console UI +```bash +ssh -R 9090:localhost:9090 user_id@x.x.x.x +``` + +You can now open the browser on your laptop, and use the generated JWT to access Console UI at `localhost:9090`. + + +## Worker node specific options + +There's also a set of configuration options that are applied in per-node manner. + +First set of variables enable Persistent Volumes and also this info was used for sample tenant deployment. You would need your own tenant settings. +```yaml +minio_pv: + - name: "mnt-data-1" # PV name will be followed by kube_node name(e.g., mnt-data-1-hostname) + storageClassName: "local-storage" # default storage class name which PVC should match with + accessMode: "ReadWriteOnce" # ReadWriteOnce/ReadOnlyMany/ReadWriteMany/ReadWriteOncePod + persistentVolumeReclaimPolicy: "Retain" # Retain/Recycle/Delete + mountPath: /mnt/data1 # mount path + storage: file # file = file block device, nvme = nvme m.2 SSDs. + device: /dev/nvme0n1 # when storage=nvme, device will be used for block device name. when storage=file, loop devices will be populated with /root/diskimage* automatically. + capacity: 1GiB # size of the PV. support only GiB/TiB +``` + +## Sample Tenants +In order to deploy the sample tenants, 4 or more worker nodes would be ideal for distributed mode. Current sample tenant deployment has two different modes. + +1. Test Mode (`minio_deploy_test_mode: true`) +In an automation environment, this mode is useful and deployed on a special block device called the loop device when no extra storage device, which maps a normal file onto a virtual block device. This allows for the file to be used as a **virtual file system** inside another file. + +2. Distributed Mode (`minio_deploy_test_mode: false`) +In clustering environment, this mode provides object storage service to build high performance infrastructure for machine learning, analytics and application data workloads. This mode uses local storage devices like hard disk, m.2 ssd, etc. + +### Sample Tenant Secret Key +A MinIO user is an identity that includes at minimum credentials consiting of an Access Key and Secret Key. MinIO requires all incoming requests include credentials which match an existing user. You can find default credentials in +```yaml +secrets: + # create a kubernetes secret object with the accessKey and secretKey as defined here. + enabled: true + name: minio1-secret + accessKey: minio + secretKey: minio123 +``` +in ./role/minio_install/templates/minio_tenant_custom_values.yml.j2 + +Please refer to the [MinIO Operator](https://github.com/minio/operator) to get more details and usage examples. diff --git a/docs/vm_config_guide.md b/docs/vm_config_guide.md new file mode 100644 index 00000000..5a9091e8 --- /dev/null +++ b/docs/vm_config_guide.md @@ -0,0 +1,331 @@ +# VM specific configuration for SRIOV NIC and SRIOV QAT + + +In order to deploy VM case, VM specific configuration has to be used. It is generated automatically via `make`. It is generated into `examples/vm/` directory. Configurations for each profile are kept in profile named directories. Group vars for VM case have set `vm_enabled` value to `true`. To deploy standard BM RA the configuration files must be re-generated. It is not sufficient to change the value of `vm_enabled` to `false`. + +``` +vm_enabled: true +``` + + +## vm-host specific options + +There's also a set of configuration options that are applied in per-node manner. + +First set of variables enables SRIOV for selected network adapters and QAT devices. It requires setting `iommu_enabled` as `true`. + +**For SRIOV NIC** it requires passing names of the physical function interfaces together with additional NIC parameters, which contain also an option to define how many virtual functions should be created for each physical function. In below example `dataplane_interfaces` configuration will create 8 VFs for `enp24s0f0` PF interface and attach them to kernel mode `iavf` driver by default and listed vfs (vf_00 and vf_05) attach to selected driver (in our case `vfio-pci` driver). Then it will create 2 VFs for `enp24s0f1` PF interface and attach them to `vfio-pci` driver. + +**For SRIOV QAT** it requires passing qat id of the QAT physical function together with an option to define how many virtual functions should be created for each physical function. In below example `qat_devices` configuration will create 12 VFs for `"0000:3d:00.0"` PF device, 10 VFs for `"0000:3f:00.0"` PF device and 10 VFs for `"0000:da:00.0"` PF device. +**_NOTE:_** Some QAT drivers ignore requested number of VFs and create maximum number of VFs allowed by current QAT device. Where: "0" means no VFs, number between "1" and "max num of VFs" creates max number of VFs and number above "max num of VFs" cause an error. + +This setting will also add IOMMU kernel flags, and as a result will reboot the target vm-host during deployment. +``` +dataplane_interfaces: + - name: enp24s0f0 + bus_info: "18:00.0" + pf_driver: i40e + ddp_profile: "gtp.pkgo" + default_vf_driver: "iavf" + sriov_numvfs: 8 + sriov_vfs: + vf_00: "vfio-pci" + vf_05: "vfio-pci" + + - name: enp24s0f1 + bus_info: "18:00.1" + pf_driver: i40e + ddp_profile: "gtp.pkgo" + default_vf_driver: "vfio-pci" + sriov_numvfs: 2 + +qat_devices: + - qat_id: "0000:3d:00.0" + qat_sriov_numvfs: 12 + + - qat_id: "0000:3f:00.0" + qat_sriov_numvfs: 10 + + - qat_id: "0000:da:00.0" + qat_sriov_numvfs: 10 + +``` + +Next section provides definition of VMs, which will be created during deployment process and which will be used as control and worker nodes there. +The first option defines version of Ubuntu cloud image, which will be used inside VMs. Currently supported versions are: "20.04" and "21.04" +``` +vm_image_version: "21.04"` +``` + +The second option defines the list of VMs. Each VM is defined by following parameters: +`type` defines type of VM and following types are supported: "ctrl" and "work" +`name` defines hostname for the VM, which is assigned to VM. That name have to be used for corresponding host_vars file. e.g.: host_vars/vm-work-1.yml +`cpus` defines, which cpus from the vm-host will be assigned to that VM + +To be able to configure it, we need to get cpu info from target vm-host, where we run `lscpu` command. Following lines from output are relevant for it: + +``` +lscpu +CPU(s): 112 +On-line CPU(s) list: 0-111 +Thread(s) per core: 2 +Core(s) per socket: 28 +Socket(s): 2 +NUMA node(s): 2 +NUMA node0 CPU(s): 0-27,56-83 +NUMA node1 CPU(s): 28-55,84-111 +``` + +In current case we have machine with 112 CPUs. It has 2 sockets with 28 cores. Each core has two threads. It has 2 NUMA nodes one per socket. +The first few cores from NUMA node0 we reserve for system. In our case 8. It means 16 CPUs. Specifically CPUs 0-7 (the first threads from selected cores) and CPUs 56-63 (the corresponding second threads from selected cores) +If we want to assign 8 CPUs to vm-ctrl-1 then we can select next 4 cores, which means CPUs 8-11 and 64-67. We need to ensure that all CPUs comes from single NUMA node. +If we want to assign 16 CPUs to vm-work-1 then we can select next 8 cores from NUMA node0 or select 8 cores from NUMA node1. In example configuration bellow we've selected 8 cores from NUMA node1, which means CPUs 28-35 and 84-91. + +`emu_cpus` defines, which CPUs from CPUs listed in `cpus` will be used for emulator. We select the first assigned core, which means for vm-ctrl-1 CPUs 8 and 64. For vm-work-1 it means CPUs 28 and 84 +`numa` defines NUMA node from which we have selected the cores. For vm-ctrl-1 it means NUMA node0. For vm-work-1 it means NUMA node1. +`cpu_total` defines total number of CPUs assigned to VM +`memory` defines amount of memory assigned to VM in MB +`pci` defines list of PCI devices assigned to VM. It contains PCI ids for SRIOV NIC VFs and SRIOV QAT VFs which are assigned to VM. The list can be empty as well. PCI section is relevant only for VM type `work`. In example configuration bellow we've assigned 4 NIC VFs and 2 QAT VFs. + +To be able to configure PCI ids for VFs we need to know their "naming convention". We need to connect to vm-host and check PCI ids for VFs there. +**For SRIOV NIC VFs:** +To check if VFs exist there run following command: +for the first PF interface enp24s0f0 from `dataplane_interfaces` above: +``` +cat /sys/bus/pci/devices/0000\:18\:00.0/sriov_numvfs +``` + +for the second PF interface enp24s0f1 from `dataplane_interfaces` above: +``` +cat /sys/bus/pci/devices/0000\:18\:00.1/sriov_numvfs +``` +If the commands return "0" then there are no VFs created there and we need to create them temporary. +To create them you need to run following command: +for the first PF interface enp24s0f0 from `dataplane_interfaces` above: +``` +echo "8" > /sys/bus/pci/devices/0000\:18\:00.0/sriov_numvfs +``` + +for the second PF interface enp24s0f1 from `dataplane_interfaces` above: +``` +echo "2" > /sys/bus/pci/devices/0000\:18\:00.1/sriov_numvfs +``` +Number used in those command is number of VFs to be created and it was taken from interface configuration above. Nevertheless we can use higher number as well to see naming convention. + +Now we can check number of created VFs using cat command as before: +for the first PF interface enp24s0f0 from `dataplane_interfaces` above: +``` +cat /sys/bus/pci/devices/0000\:18\:00.0/sriov_numvfs +``` + +for the second PF interface enp24s0f1 from `dataplane_interfaces` above: +``` +cat /sys/bus/pci/devices/0000\:18\:00.1/sriov_numvfs +``` +The commands return number of created VFs. It should be the same number as number used in echo command above. +**_NOTE:_** for some drivers the received number of created VFs can be limited to "max number of VFs" + +To see PCI ids for NICs and created VFs run following command: +``` +lspci |grep -i Ether +``` + +``` +18:00.0 Ethernet controller: Intel Corporation Ethernet Controller XXV710 for 25GbE SFP28 (rev 02) +18:00.1 Ethernet controller: Intel Corporation Ethernet Controller XXV710 for 25GbE SFP28 (rev 02) +18:02.0 Ethernet controller: Intel Corporation Ethernet Virtual Function 700 Series (rev 02) +18:02.1 Ethernet controller: Intel Corporation Ethernet Virtual Function 700 Series (rev 02) +18:02.2 Ethernet controller: Intel Corporation Ethernet Virtual Function 700 Series (rev 02) +18:02.3 Ethernet controller: Intel Corporation Ethernet Virtual Function 700 Series (rev 02) +18:02.4 Ethernet controller: Intel Corporation Ethernet Virtual Function 700 Series (rev 02) +18:02.5 Ethernet controller: Intel Corporation Ethernet Virtual Function 700 Series (rev 02) +18:02.6 Ethernet controller: Intel Corporation Ethernet Virtual Function 700 Series (rev 02) +18:02.7 Ethernet controller: Intel Corporation Ethernet Virtual Function 700 Series (rev 02) +18:0a.0 Ethernet controller: Intel Corporation Ethernet Virtual Function 700 Series (rev 02) +18:0a.1 Ethernet controller: Intel Corporation Ethernet Virtual Function 700 Series (rev 02) +``` + +The first 8 VFs belongs to PF 18:00.0 and the next 2 VFs belongs to PF 18:00.1 +Select PCI ids for VFs to be assigned to VM. In example configuration bellow we've selected 18:02.2, 18:02.3, 18:02.4 and 18:02.5 + + +If we created VFs in steps above then we can delete them again via following command: +for the first PF interface enp24s0f0 from `dataplane_interfaces` above: +``` +echo "0" > /sys/bus/pci/devices/0000\:18\:00.0/sriov_numvfs +``` + +for the second PF interface enp24s0f1 from `dataplane_interfaces` above: +``` +echo "0" > /sys/bus/pci/devices/0000\:18\:00.1/sriov_numvfs +``` + +**For SRIOV QAT VFs:** +To check if VFs exist there run following command: +for the first PF device 0000:3d:00.0 from `qat_devices` above. For other PF devices use the same commands with updated PCI id and VF number. +``` +cat /sys/bus/pci/devices/0000\:3d\:00.0/sriov_numvfs +``` + +If the commands return "0" then there are no VFs created there and we need to create them temporary. +To create them you need to run following command: +``` +echo "12" > /sys/bus/pci/devices/0000\:3d\:00.0/sriov_numvfs +``` + +Number used in this command is number of VFs to be created and it was taken from `qat_devices` configuration above. Nevertheless we can use higher number as well to see naming convention. + +Now we can check number of created VFs using cat command as before: +``` +cat /sys/bus/pci/devices/0000\:3d\:00.0/sriov_numvfs +``` + +The commands return number of created VFs. It should be the same number as number used in echo command above. +**_NOTE:_** for some drivers the received number of created VFs can be limited to "nax number of VFs" + +To see PCI ids for QAT devices run following command: +``` +lspci -nn |grep -i Quick +``` + +``` +3d:00.0 Co-processor [0b40]: Intel Corporation C62x Chipset QuickAssist Technology [8086:37c8] (rev 04) +3f:00.0 Co-processor [0b40]: Intel Corporation C62x Chipset QuickAssist Technology [8086:37c8] (rev 04) +da:00.0 Co-processor [0b40]: Intel Corporation C62x Chipset QuickAssist Technology [8086:37c8] (rev 04) +``` + +To see PCI ids for created QAT VFs run following command: +Device id string to grep is taken from square brackets above. We search for that device is and "device id + 1", which corresponds to VFs +``` +lspci -nn |grep -i "37c[89]" +``` + +``` +3d:00.0 Co-processor [0b40]: Intel Corporation C62x Chipset QuickAssist Technology [8086:37c8] (rev 04) +3d:01.0 Co-processor [0b40]: Intel Corporation Device [8086:37c9] (rev 04) +3d:01.1 Co-processor [0b40]: Intel Corporation Device [8086:37c9] (rev 04) +3d:01.2 Co-processor [0b40]: Intel Corporation Device [8086:37c9] (rev 04) +3d:01.3 Co-processor [0b40]: Intel Corporation Device [8086:37c9] (rev 04) +3d:01.4 Co-processor [0b40]: Intel Corporation Device [8086:37c9] (rev 04) +3d:01.5 Co-processor [0b40]: Intel Corporation Device [8086:37c9] (rev 04) +3d:01.6 Co-processor [0b40]: Intel Corporation Device [8086:37c9] (rev 04) +3d:01.7 Co-processor [0b40]: Intel Corporation Device [8086:37c9] (rev 04) +3d:02.0 Co-processor [0b40]: Intel Corporation Device [8086:37c9] (rev 04) +3d:02.1 Co-processor [0b40]: Intel Corporation Device [8086:37c9] (rev 04) +3d:02.2 Co-processor [0b40]: Intel Corporation Device [8086:37c9] (rev 04) +3d:02.3 Co-processor [0b40]: Intel Corporation Device [8086:37c9] (rev 04) +3d:02.4 Co-processor [0b40]: Intel Corporation Device [8086:37c9] (rev 04) +3d:02.5 Co-processor [0b40]: Intel Corporation Device [8086:37c9] (rev 04) +3d:02.6 Co-processor [0b40]: Intel Corporation Device [8086:37c9] (rev 04) +3d:02.7 Co-processor [0b40]: Intel Corporation Device [8086:37c9] (rev 04) +3f:00.0 Co-processor [0b40]: Intel Corporation C62x Chipset QuickAssist Technology [8086:37c8] (rev 04) +3f:01.0 Co-processor [0b40]: Intel Corporation Device [8086:37c9] (rev 04) +3f:01.1 Co-processor [0b40]: Intel Corporation Device [8086:37c9] (rev 04) +3f:01.2 Co-processor [0b40]: Intel Corporation Device [8086:37c9] (rev 04) +3f:01.3 Co-processor [0b40]: Intel Corporation Device [8086:37c9] (rev 04) +3f:01.4 Co-processor [0b40]: Intel Corporation Device [8086:37c9] (rev 04) +3f:01.5 Co-processor [0b40]: Intel Corporation Device [8086:37c9] (rev 04) +3f:01.6 Co-processor [0b40]: Intel Corporation Device [8086:37c9] (rev 04) +3f:01.7 Co-processor [0b40]: Intel Corporation Device [8086:37c9] (rev 04) +3f:02.0 Co-processor [0b40]: Intel Corporation Device [8086:37c9] (rev 04) +3f:02.1 Co-processor [0b40]: Intel Corporation Device [8086:37c9] (rev 04) +3f:02.2 Co-processor [0b40]: Intel Corporation Device [8086:37c9] (rev 04) +3f:02.3 Co-processor [0b40]: Intel Corporation Device [8086:37c9] (rev 04) +3f:02.4 Co-processor [0b40]: Intel Corporation Device [8086:37c9] (rev 04) +3f:02.5 Co-processor [0b40]: Intel Corporation Device [8086:37c9] (rev 04) +... +``` + +There are 16 VFs created for each QAT PF device +Select PCI ids for VFs to be assigned to VM. In example configuration bellow we've selected 3d:02.3 and 3f:02.3 + + +If we created VFs in steps above then we can delete them again via following command: +``` +echo "0" > /sys/bus/pci/devices/0000\:3d\:00.0/sriov_numvfs +``` + + +Example configuration contains 2 VMs, 1 control and 1 work node. We've tested configuration with 5 VMs, 3 control and 2 work nodes + +``` +vms: + - type: "ctrl" + name: "vm-ctrl-1" + cpus: "8-11,64-67" + emu_cpus: "8,64" + numa: 0 + cpu_total: 8 + memory: 20480 + - type: "work" + name: "vm-work-1" + cpus: "28-35,84-91" + emu_cpus: "28,84" + numa: 1 + cpu_total: 16 + memory: 61440 + pci: + - "18:02.2" + - "18:02.3" + - "18:02.4" + - "18:02.5" + - "3d:02.3" + - "3f:02.3" + +``` + + +## Worker node specific options + +There's also a set of configuration options that are applied in per-node manner in current case for VM type `work`. + +The first set of variables configure assigned SRIOV NIC VFs and SRIOV QAT VFs inside VM. It requires setting `iommu_enabled` as `false`. + +**For SRIOV NIC** it requires passing names of interfaces together with additional NIC parameters. In below example `dataplane_interfaces` configuration contains 4 interfaces, where the first one starting with name enp4s0 and bus_info "04:00.0". Last number in name and PCI id is sequentially increasing. `sriov_numvfs` must be "0" here. We can't create new VFs out of provided VF. +`pf_driver` and `default_vf_driver` are not use at the moment. All interfaces are assigned to kernel mode iavf driver inside VM. +The number of interfaces defined here in `dataplane_interfaces` have to be the same as number of NIC VFs assigned to this VM ! +In our example configuration we've assigned 4 NIC VFs, so we have 4 interfaces defined here. + +**For SRIOV QAT** it requires passing qat id of QAT device. `qat_sriov_numvfs` must be "0" here. We can't create new VFs out of provided VF. In below example `qat_devices` configuration contains 2 QAT devices. `qat_id` continue in numbering from `dataplane_interfaces`. The last bus_info there was `"07:00:0"` so, the first qat_id will be `"0000:08:00.0"`. +The number of QAT devices defined here in `qat_devices` has to be the same as number of QAT VFs assigned to this VM! +In our example configuration we've assigned 2 QAT VFs, so we have 2 devices defined here. + +This setting will add `vfio-pci.disable_denylist=1` kernel flags for kernel >=5.9 or specific RHEL/CentOS versions, and as a result will reboot the target vm-work VM during deployment. +``` +dataplane_interfaces: + - name: enp4s0 + bus_info: "04:00.0" + pf_driver: iavf + sriov_numvfs: 0 + default_vf_driver: "igb_uio" + - name: enp5s0 + bus_info: "05:00.0" + pf_driver: iavf + sriov_numvfs: 0 + default_vf_driver: "igb_uio" + - name: enp6s0 + bus_info: "06:00.0" + pf_driver: iavf + sriov_numvfs: 0 + default_vf_driver: "vfio-pci" + - name: enp7s0 + bus_info: "07:00.0" + pf_driver: vfio-pci + sriov_numvfs: 0 + default_vf_driver: "vfio-pci" + + +qat_devices: + - qat_id: "0000:08:00.0" + qat_sriov_numvfs: 0 + + - qat_id: "0000:09:00.0" + qat_sriov_numvfs: 0 + +``` + +### Once the deployment is finished we can access VMs from ansible_host via VM name: +``` +ssh vm-ctrl-1 +ssh vm-work-1 +``` diff --git a/generate/playbook_templates/infra_playbook.j2 b/generate/playbook_templates/infra_playbook.j2 new file mode 100644 index 00000000..22674a75 --- /dev/null +++ b/generate/playbook_templates/infra_playbook.j2 @@ -0,0 +1,186 @@ +--- +# apply common cluster node configuration +- hosts: k8s_cluster,vm_host + tasks: [] + pre_tasks: + - name: End play for VM host + meta: end_host + when: + - "'vm_host' in group_names" + - on_vms is defined and on_vms + handlers: + - name: reboot server + reboot: { reboot_timeout: 1200 } + roles: + - role: cluster_defaults + - role: bootstrap/configure_proxy + - role: bootstrap/install_packages + - role: bootstrap/update_grub + when: update_kernel | default(false) | bool + - role: bootstrap/configure_additional_grub_parameters + tags: grub-extra-params + - role: bootstrap/configure_security + - role: bootstrap/apply_kubernetes_reqs + - role: bootstrap/configure_dns + when: + - ansible_distribution == "Ubuntu" and dns_disable_stub_listener | default(true) | bool + - not vm_enabled or (on_vms is defined and on_vms) + - role: bootstrap/golang_install + environment: "{{ '{{' }} proxy_env | d({}) {{ '}}' }}" + any_errors_fatal: true + +# apply worker node kernel configuration +- hosts: kube_node,vm_host + tasks: [] + pre_tasks: + - name: End play for VM host + meta: end_host + when: + - "'vm_host' in group_names" + - on_vms is defined and on_vms + handlers: + - name: reboot server + reboot: { reboot_timeout: 1200 } + roles: + - role: cluster_defaults + - role: bootstrap/configure_hugepages + tags: hugepages + when: hugepages_enabled | default(true) | bool + - role: bootstrap/configure_cpu_isolation + when: isolcpus_enabled | default(false) | bool + - role: bootstrap/configure_intel_pstate + when: intel_pstate_enabled is defined and intel_pstate_enabled | default(true) | bool + - role: bootstrap/set_sriov_kernel_flags + tags: setup-sriov + when: iommu_enabled | default(true) | bool or (on_vms is defined and on_vms) + - role: bootstrap/set_rdt_kernel_flags + when: telegraf_enabled | default(true) | bool +{%- if playbook_name in ['full_nfv', 'remote_fp', 'on_prem'] %} + - role: bootstrap/configure_sst + tags: sst + when: + - (ansible_distribution == 'Ubuntu' and ansible_distribution_version >= '20.04') or + (ansible_distribution == "CentOS" and ansible_distribution_version >= '8.3') or + (ansible_distribution == "RedHat" and ansible_distribution_version >= '8.3') + - (sst_bf_configuration_enabled is defined and sst_bf_configuration_enabled | default(false)) or + (sst_cp_configuration_enabled is defined and sst_cp_configuration_enabled | default(false)) or + (sst_tf_configuration_enabled is defined and sst_tf_configuration_enabled | default(false)) or + (sst_pp_configuration_enabled is defined and sst_pp_configuration_enabled | default(false)) + - not vm_enabled or (on_vms is defined and on_vms) +{% endif %} +{%- if playbook_name in ['full_nfv', 'regional_dc'] %} + - role: bootstrap/install_gpu_kernel + when: + - gpu_dp_enabled is defined and gpu_dp_enabled | default(false) + - not vm_enabled or (on_vms is defined and on_vms) +{% endif %} + - role: bootstrap/update_grub + environment: "{{ '{{' }} proxy_env | d({}) {{ '}}' }}" + any_errors_fatal: true + +{%- if playbook_name in ['full_nfv', 'access', 'on_prem', 'remote_fp', 'storage'] %} +# install worker node qat software +- hosts: kube_node,vm_host + tasks: [] + pre_tasks: + - name: End play for VM host + meta: end_host + when: + - "'vm_host' in group_names" + - on_vms is defined and on_vms + handlers: + - name: reboot server + reboot: { reboot_timeout: 1200 } + roles: + - role: cluster_defaults + - role: bootstrap/install_qat_drivers_services + tags: setup-qat + when: + - update_qat_drivers | default(false) | bool + - qat_devices is defined and (qat_devices|length>0) + environment: "{{ '{{' }} proxy_env | d({}) {{ '}}' }}" + any_errors_fatal: true +{% endif %} + +# install worker node network software +- hosts: kube_node,vm_host + tasks: [] + pre_tasks: + - name: End play for VM host + meta: end_host + when: + - "'vm_host' in group_names" + - on_vms is defined and on_vms + roles: + - role: cluster_defaults + - role: bootstrap/update_nic_drivers + when: update_nic_drivers | default(false) | bool + - role: bootstrap/update_nic_firmware + tags: firmware + when: + - dataplane_interfaces is defined and (dataplane_interfaces|length>0) + - update_nic_firmware | default(false) | bool + - update_nic_drivers | default(false) | bool + - role: bootstrap/determine_dataplane_interfaces + when: + - not vm_enabled or (on_vms is defined and on_vms) + environment: "{{ '{{' }} proxy_env | d({}) {{ '}}' }}" + any_errors_fatal: true + +# install DPDK and apply SRIOV configuration +- hosts: kube_node,vm_host + pre_tasks: + - name: End play for VM host + meta: end_host + when: + - "'vm_host' in group_names" + - on_vms is defined and on_vms + roles: + - role: cluster_defaults +{%- if playbook_name not in ['storage'] %} + - role: install_dpdk + when: + - ovs_dpdk_enabled | default(false) | bool or install_dpdk | default(true) | bool + tags: dpdk +{% endif %} +{%- if playbook_name in ['full_nfv', 'remote_fp'] %} + - role: install_ddp_pkgs + when: install_ddp_packages | default(true) | bool +{% endif %} +{%- if playbook_name not in ['storage'] %} + - role: sriov_nic_init + tags: setup-sriov-nic + when: + - install_dpdk is defined + - iommu_enabled | default(true) | bool + - install_dpdk | default(true) | bool + - (kubernetes | default(true) | bool + and not container_runtime_only_deployment | default(false) | bool + and not sriov_network_operator_enabled | default(true) | bool + or (not kubernetes | default(true) | bool + and container_runtime_only_deployment | default(false) | bool)) +{% endif %} +{%- if playbook_name in ['full_nfv', 'access', 'on_prem', 'remote_fp'] %} + - role: bootstrap/configure_qat + tags: setup-sriov-qat + when: + - qat_devices is defined and (qat_devices|length>0) + - iommu_enabled | default(true) | bool + or (on_vms is defined and on_vms) + - update_qat_drivers | default(false) | bool + - role: bootstrap/configure_openssl + when: + - qat_devices is defined and (qat_devices|length>0) + - iommu_enabled | default(true) | bool + or (on_vms is defined and on_vms) + - update_qat_drivers | default(false) | bool + - openssl_install | default(false) | bool +{% endif %} +{%- if playbook_name in ['full_nfv', 'on_prem', 'remote_fp'] %} + - role: bootstrap/configure_sgx + tags: sgx + when: + - configure_sgx is defined and configure_sgx | default(true) | bool +{% endif %} + environment: "{{ '{{' }} proxy_env | d({}) {{ '}}' }}" + any_errors_fatal: true diff --git a/generate/playbook_templates/intel_playbook.j2 b/generate/playbook_templates/intel_playbook.j2 new file mode 100644 index 00000000..d1ce4e27 --- /dev/null +++ b/generate/playbook_templates/intel_playbook.j2 @@ -0,0 +1,128 @@ +--- +- hosts: k8s_cluster + tasks: [] + roles: + - role: cluster_defaults + tags: defaults + - role: nfd_install + tags: nfd + when: nfd_enabled | default(true) | bool +{%- if playbook_name not in ['storage'] %} + - role: sriov_dp_install + tags: sriov-net-dp + when: + - sriov_net_dp_enabled | default(false) | bool + - not sriov_network_operator_enabled | default(true) | bool + - role: sriov_network_operator_install + tags: sriov-network-operator + when: + - sriov_network_operator_enabled | default(true) | bool + - not sriov_net_dp_enabled | default(false) | bool + - not sriov_cni_enabled | default(false) | bool +{% endif %} +{%- if playbook_name in ['access', 'full_nfv', 'on_prem', 'regional_dc', 'remote_fp', 'storage'] %} + - role: intel_dp_operator + tags: dp-operator + when: sgx_dp_enabled is defined and sgx_dp_enabled | default(true) or + gpu_dp_enabled is defined and gpu_dp_enabled | default(false) or + qat_dp_enabled is defined and qat_dp_enabled | default(true) +{% endif %} +{%- if playbook_name in ['full_nfv', 'on_prem', 'remote_fp'] %} + - role: sgx_dp_install + tags: sgx-dp + when: + - sgx_dp_enabled is defined and sgx_dp_enabled | default(true) + - ansible_os_family == "Debian" or (ansible_os_family == "RedHat" and ansible_distribution_version >= '8.3') + - role: kmra_install + tags: kmra + when: + - kmra_enabled is defined and kmra_enabled | default(true) + - (ansible_distribution == "Ubuntu" and ansible_distribution_version >= '20.04') + or (ansible_os_family == "RedHat" and ansible_distribution_version >= '8.3') + - role: intel_power_manager + tags: power-manager + when: intel_power_manager is defined and intel_power_manager.enabled | default(false) | bool +{% endif %} +{%- if playbook_name in ['access', 'full_nfv', 'on_prem', 'remote_fp', 'storage'] %} + - role: qat_dp_install + tags: qat-dp + when: qat_dp_enabled | default(true) | bool + - role: openssl_engine_install + tags: openssl-engine + when: openssl_engine_enabled | default(false) | bool +{% endif %} + - role: kube_prometheus + when: prometheus_operator | default(true) | bool +{%- if playbook_name in ['full_nfv', 'regional_dc'] %} + - role: gpu_dp_install + tags: gpu-dp + when: gpu_dp_enabled is defined and gpu_dp_enabled | default(false) +{% endif %} +{%- if playbook_name in ['full_nfv', 'on_prem', 'regional_dc', 'remote_fp'] %} + - role: platform_aware_scheduling_install + tags: platform-aware-scheduling + when: tas_enabled | default(true) | bool or gas_enabled | default(true) | bool +{% endif %} + - role: collectd_install + tags: monitoring + when: + - collectd_enabled | default(false) | bool + - not (telegraf_enabled | default(true) | bool) + vars: + collectd_profile: {{ playbook_name }} + - role: telegraf_install + when: + - telegraf_enabled | default(true) | bool + - not (collectd_enabled | default(false) | bool) + tags: monitoring + vars: + telegraf_profile: {{ playbook_name }} +{%- if playbook_name in ['access', 'full_nfv', 'on_prem', 'regional_dc', 'remote_fp'] %} + - role: service_mesh_install + tags: service-mesh + when: + - service_mesh.enabled | default(true) | bool +{% endif %} +{%- if playbook_name in ['storage', 'full_nfv'] %} + - role: minio_install + tags: minio + when: + - minio_enabled is defined and minio_enabled | default(false) | bool +{% endif %} + environment: + - "{{ '{{' }} proxy_env | d({}) {{ '}}' }}" + - REGISTRY_AUTH_FILE: "{{ '{{' }} registry_containerd {{ '}}' }}" + any_errors_fatal: true + +- hosts: kube_node + tasks: [] + roles: + - role: cluster_defaults + tags: defaults +{%- if playbook_name not in ['storage'] %} + - role: sriov_cni_install + when: + - sriov_cni_enabled | default(false) | bool + - not sriov_network_operator_enabled | default(true) | bool + tags: sriov-cni +{% endif %} +{%- if playbook_name in ['full_nfv', 'on_prem', 'remote_fp'] %} + - role: bond_cni_install + when: bond_cni_enabled | default(true) | bool + tags: bond-cni +{% endif %} +{%- if playbook_name in ['full_nfv', 'remote_fp'] %} + - role: userspace_cni_install + tags: userspace-cni + when: userspace_cni_enabled | default(true) | bool +{% endif %} + environment: "{{ '{{' }} proxy_env | d({}) {{ '}}' }}" + any_errors_fatal: true + +- hosts: kube_control_plane + tasks: [] + roles: + - role: net_attach_defs_create + tags: net-attach-defs + environment: "{{ '{{' }} proxy_env | d({}) {{ '}}' }}" + any_errors_fatal: true diff --git a/generate/playbook_templates/main_playbook.j2 b/generate/playbook_templates/main_playbook.j2 new file mode 100644 index 00000000..b7587ae6 --- /dev/null +++ b/generate/playbook_templates/main_playbook.j2 @@ -0,0 +1,15 @@ +--- +- name: preflight checks + import_playbook: preflight.yml + when: preflight_enabled | default(true) | bool +- name: configure target hosts OS layer + import_playbook: infra/{{ playbook_name }}.yml +- name: deploy container runtime + import_playbook: infra/container_runtime_deployment.yml + when: not kubernetes | default(true) and container_runtime_only_deployment | default(false) +- name: provision Kubernetes cluster using kubespray + import_playbook: k8s/k8s.yml + when: kubernetes | default(true) +- name: install Intel Container Experience Kit features + import_playbook: intel/{{ playbook_name }}.yml + when: kubernetes | default(true) diff --git a/generate/profiles_templates/common/group_vars.j2 b/generate/profiles_templates/common/group_vars.j2 new file mode 100644 index 00000000..dbf647f3 --- /dev/null +++ b/generate/profiles_templates/common/group_vars.j2 @@ -0,0 +1,377 @@ +--- +## Container Experience Kits (CEK) primary playbook variables ## +# Do not change profile_name and configured_arch here !!! +# To generate vars for different profile/architecture use make command +# generated for profile and arch: +profile_name: {{ name }} +configured_arch: {{ arch }} + +# CEK project directory on all nodes +project_root_dir: /opt/cek/ + +vm_enabled: {% if vm_mode == 'on' %}true{% else %}false{% endif %} +{%- if vm_mode in ['optional'] %} +# vm_mode can't be enabled manualy here +# to enable it, vm specific configuration from examples/vm need to be taken +{%- endif %} + +# Kubernetes version +kubernetes: true +kube_version: v1.22.3 +#kube_version: v1.21.5 +#kube_version: v1.20.6 +# To deploy only container runtime set this variable as "true", and kubernetes as "false" +# set both variables as "false" to perform only host configuration +container_runtime_only_deployment: false + +# Kubernetes container runtime: docker, containerd, crio +container_runtime: docker + +# Preflight will check vars configuration +# It is NOT recommended to disable preflight, unless it is a conscious decision +preflight_enabled: true + +# Run system-wide package update (apt dist-upgrade, yum update, ...) +# Note: enabling this may lead to unexpected results +# Tip: you can set this per host using host_vars +update_all_packages: false +update_kernel: false + +# Add arbitrary parameters to GRUB +additional_grub_parameters_enabled: false +additional_grub_parameters: "" + +# SELinux configuration state: current, enabled, disabled +selinux_state: current +{% if nfd in ['on', 'optional'] %} +# Node Feature Discovery +nfd_enabled: {% if nfd == 'on' %}true{% else %}false{% endif %} +nfd_build_image_locally: false +nfd_namespace: kube-system +nfd_sleep_interval: 60s +{% endif %} + +{%- if native_cpu_manager in ['on', 'optional'] %} +# Native CPU Manager (Kubernetes built-in) +# Setting this option as "true" enables the "static" policy, otherwise the default "none" policy is used. +# The reserved CPU cores settings are individual per each worker node, and therefore are available to configure in the host_vars file +native_cpu_manager_enabled: {% if native_cpu_manager == 'on' %}true{% else %}false{% endif %} +{% endif %} +{% if topology_manager in ['on', 'optional'] -%} +# Enable Kubernetes built-in Topology Manager +topology_manager_enabled: {% if topology_manager == 'on' %}true{% else %}false{% endif %} +# There are four supported policies: none, best-effort, restricted, single-numa-node. +topology_manager_policy: "best-effort" +{% endif %} + +{%- if sriov_operator in ['on', 'optional'] %} +# OpenShift SRIOV Network Operator +sriov_network_operator_enabled: {% if sriov_operator == 'on' %}true{% else %}false{% endif %} +{%- if vm_mode in ['on', 'optional'] %} +# For VM mode sriov_network_operator_enabled has to be false, otherwise VFs +# are not created before VM creation +{%- endif %} +sriov_network_operator_namespace: "sriov-network-operator" +{% endif %} + +{%- if sriov_network_dp in ['on', 'optional'] %} +# Intel SRIOV Network Device Plugin +sriov_net_dp_enabled: {% if sriov_network_dp == 'on' %}true{% else %}false{% endif %} +sriov_net_dp_namespace: kube-system +# whether to build and store image locally or use one from public external registry +sriov_net_dp_build_image_locally: false +# SR-IOV network device plugin configuration. +# For more information on supported configuration refer to: https://github.com/intel/sriov-network-device-plugin#configurations +sriovdp_config_data: | + { + "resourceList": [{ + "resourceName": "intel_sriov_netdevice", + "selectors": { + "vendors": ["8086"], + "devices": ["154c", "10ed", "1889"], + "drivers": ["iavf", "ixgbevf"] + } + }, + { + "resourceName": "intel_sriov_dpdk_700_series", + "selectors": { + "vendors": ["8086"], + "devices": ["154c", "10ed"], + "drivers": ["vfio-pci"] + } + }, + { + "resourceName": "intel_sriov_dpdk_800_series", + "selectors": { + "vendors": ["8086"], + "devices": ["1889"], + "drivers": ["vfio-pci"] + } + {% if name in ['full_nfv', 'access', 'regional_dc'] -%} + }, + { + "resourceName": "intel_fpga", + "deviceType": "accelerator", + "selectors": { + "vendors": ["8086"], + "devices": ["0d90"] + } + } + {%- else -%} + } + {%- endif %} + ] + } +{% endif %} + +{%- if power_manager in ['on', 'optional'] and arch in ['icx', 'clx', 'spr'] %} +# Intel Kubernetes Power Manager +intel_power_manager: + enabled: {% if power_manager == 'on' %}true{% else %}false{% endif %} + # The performance profile is available for nodes that has CPU max MHz > 3500.0000 - use 'lscpu' command to see your node details + power_profiles: [performance, balance-performance, balance-power] # the list of PowerProfiles that will be available on the nodes + # possible PowerProfiles are: performance, balance_performance, balance_power + power_nodes: [] # list of nodes that should be considered during Operator work and profiles deployment + build_image_locally: false + deploy_example_pods: false # deploy example Pods that will utilize special resources + global_shared_profile_enabled: false # deploy custom Power Profile with user defined frequencies that can be applied to all power nodes + # to make use of Shared Profile fill Shared Workload settings in host vars + max_shared_frequency: 1500 # max frequency that will be applied for cores by Shared Workload + min_shared_frequency: 1000 # min frequency that will be applied for cores by Shared Workload +{% endif %} + +{%- if sgx_dp in ['on', 'optional'] and arch in ['icx', 'spr'] or + gpu in ['on', 'optional'] and arch in ['icx', 'clx', 'spr'] or + qat_dp in ['on', 'optional'] %} +# Intel Device Plugin Operator +intel_dp_namespace: kube-system # namespace will be applied for SGX DP, GPU DP and QAT DP +{% endif %} + +{%- if qat_dp in ['on', 'optional'] %} +# Intel QAT Device Plugin for Kubernetes +qat_dp_enabled: {% if qat_dp == 'on' %}true{% else %}false{% endif %} +qat_dp_verbosity: 4 +# Maximum number of QAT devices (minimum is 1) to be provided to the QAT Device Plugin. +# To use all available QAT devices on each node, qat_dp_max_devices must be equal to the highest number of QAT Devices from all nodes +# e.g node1 - 48VFs, node2 - 32VFs, qat_dp_max_devices: 48 +# It is possible to use a subset of QAT devices in QAT DP. E.g by putting 10 here, QAT DP will use just 10VFs on each node +qat_dp_max_num_devices: 32 +qat_dp_build_image_locally: {% if vm_mode == 'on' %}true{% else %}false{% endif %} + +qat_supported_pf_dev_ids: + - "435" + - "37c8" + - "19e2" + - "18ee" + - "6f54" + - "18a0" + - "4940" + +qat_supported_vf_dev_ids: + - "443" + - "37c9" + - "19e3" + - "18ef" + - "6f55" + - "18a1" + - "4941" +{% endif %} + +{%- if openssl in ['on', 'optional'] %} +# This feature will enable OpenSSL*Engine +openssl_engine_enabled: {% if openssl == 'on' and qat == 'on' %}true{% else %}false{% endif %} # To activate OpenSSL*Engine set both install_openssl and update_qat_drivers to 'true' in host_vars +{% endif %} + +{%- if gpu in ['on', 'optional'] and arch in ['icx', 'clx', 'spr'] %} +# Intel GPU Device Plugin for Kubernetes +gpu_dp_enabled: {% if gpu == 'on' %}true{% else %}false{% endif %} +gpu_dp_kernel_version: "5.4.48+" +gpu_dp_verbosity: 4 +gpu_dp_build_image_locally: false + +# Please refer to: https://github.com/intel/intel-device-plugins-for-kubernetes/tree/v0.23.0/cmd/gpu_plugin#configuration-options +# to fully discover the below settings usage +gpu_dp_shared_devices: 10 # number of containers (min. 1) that can share the same GPU device +gpu_dp_monitor_resources: false # enable monitoring all GPU resources on the node +gpu_dp_fractional_manager: false # enable handling of fractional resources for multi-GPU nodes +gpu_dp_prefered_allocation: 'none' # available policies are: ['balanced', 'packed', 'none'] + +# For systems with older kernel drivers or GPUs which do not support reading the GPU memory amount, +# the gpu_dp_max_memory variable value is turned into a GPU memory amount label instead of a read value. +# Please set carefully as this will be set as the maximum available memory of GPU on the nodes! +# If GPU will be capable of manifest its size via the driver this variable will be ignored. +gpu_dp_max_memory: "8 GB" # max memory per card - for Intel SG1 single card has 8 GB of memory +{% endif %} + +{%- if sgx_dp in ['on', 'optional'] and arch in ['icx', 'spr'] %} +# Intel SGX Device Plugin for Kubernetes +sgx_dp_enabled: {% if sgx_dp == 'on' %}true{% else %}false{% endif %} +sgx_dp_verbosity: 4 +sgx_dp_build_image_locally: false +sgx_aesmd_namespace: kube-system +# ProvisionLimit is a number of containers that can share +# the same SGX provision device. +sgx_dp_provision_limit: 20 +# EnclaveLimit is a number of containers that can share the +# same SGX enclave device. +sgx_dp_enclave_limit: 20 +{% endif %} + +{%- if kmra in ['on', 'optional'] and arch in ['icx', 'spr'] %} +# KMRA (Key Management Reference Application) +kmra_enabled: {% if kmra == 'on' %}true{% else %}false{% endif %} +# The PCCS uses this API key to request collaterals from Intel's Provisioning Certificate Service. +# User needs to subscribe first to obtain an API key. +# For how to subscribe to Intel Provisioning Certificate Service and receive an API key, +# goto https://api.portal.trustedservices.intel.com/provisioning-certification and click on 'Subscribe'. +kmra_pccs_api_key: "ffffffffffffffffffffffffffffffff" +# deploy KMRA demo workload (NGINX server) +kmra_deploy_demo_workload: true +{% endif %} + +{%- if service_mesh in ['on', 'optional'] %} +# Service mesh deployment +# https://istio.io/latest/docs/setup/install/istioctl/ + +# for all available options, please, refer to the 'roles/service_mesh_install/vars/main.yml; +service_mesh: + enabled: {% if service_mesh == 'on' %}true{% else %}false{% endif %} + # available profiles are: 'default', 'demo', 'minimal', 'external', 'empty', 'preview' + # if custom profile needs to be deployed, please, place the file named '.yaml' + # into the directory 'roles/service_mesh_install/files/profiles/' + profile: default + tcpip_bypass_ebpf: + enabled: true +{% endif %} + +{%- if tas in ['on', 'optional'] or gas in ['on', 'optional'] %} +# Intel Platform Aware Scheduling (PAS) +pas_namespace: kube-system + +{%- if tas in ['on', 'optional'] %} +# Intel Platform Aware Scheduling - Telemetry Aware Scheduling (TAS) +tas_enabled: {% if tas == 'on' %}true{% else %}false{% endif %} +tas_build_image_locally: false +# create and enable TAS demonstration policy: [true, false] +tas_enable_demo_policy: false +{% endif %} + +{%- if gas in ['on', 'optional'] %} +# Intel Platform Aware Scheduling - GPU Aware Scheduling (GAS) +gas_enabled: {% if gas == 'on' %}true{% else %}false{% endif %} +gas_build_image_locally: false +{%- endif %} +{%- endif %} + +# Telemetry configuration. Collectd and Telegraf variables are mutually exclusive. +collectd_enabled: false +telegraf_enabled: true +collectd_scrap_interval: 30 +telegraf_scrap_interval: 30 +{% if sriov_network_dp in ["on", "optional"] or network_userspace in ["on", "optional"] %} +# Create reference net-attach-def objects +example_net_attach_defs: +{%- if sriov_network_dp in ["on", "optional"] %} + sriov_net_dp: {% if sriov_network_dp == "on" %}true{% else %}false{% endif %} # Update to match host_vars CNI configuration +{%- endif -%} +{%- if network_userspace in ["on", "optional"] %} + userspace_ovs_dpdk: {% if network_userspace == "on" %}true{% else %}false{% endif %} # Update to match host_vars CNI configuration + userspace_vpp: false # Update to match host_vars CNI configuration +{%- endif %} +{%- endif %} +{% if firewall in ['on', 'optional'] %} +firewall_enabled: {% if firewall == "on" %}true{% else %}false{% endif %} +{%- endif %} + +## Proxy configuration ## +#http_proxy: "http://proxy.example.com:1080" +#https_proxy: "http://proxy.example.com:1080" +#additional_no_proxy: ".example.com,mirror_ip" + +# (Ubuntu only) disables DNS stub listener which may cause issues on Ubuntu +dns_disable_stub_listener: true + +# Kubernetes cluster name, also will be used as DNS domain +cluster_name: cluster.local + +## Kubespray variables ## + +kube_controller_manager_bind_address: 127.0.0.1 +kube_proxy_metrics_bind_address: 127.0.0.1 +# supported network plugins(calico, flannel) and kube-proxy configuration +kube_network_plugin: calico +# supported calico backend: [vxlan, bird] +{%- if vm_mode in ['on'] %} +calico_backend: vxlan +# For VM mode calico_backend has to be vxlan, otherwise deployment will fail +{%- else %} +calico_backend: bird +{%- endif %} +wireguard_enabled: {% if wireguard == 'on' %}true{% else %}false{% endif %} +kube_network_plugin_multus: true +kube_pods_subnet: 10.244.0.0/16 +{%- if name in ['regional_dc', 'full_nfv', 'access', 'storage'] -%} +{% set mask = 18 %} +{%- elif name == 'remote_fp' -%} +{% set mask = 19 %} +{%- elif name == 'on_prem' -%} +{% set mask = 21 %} +{%- elif name == 'basic' -%} +{% set mask = 22 %} +{%- endif %} +kube_service_addresses: 10.233.0.0/{{ mask }} +kube_proxy_mode: iptables + +#set both below on true if you want to enable the eBPF dataplane support +calico_bpf_enabled: false +kube_proxy_remove: false + +# Set this var to true if you want to expose calico metrics endpoint +calico_metrics_enabled: false + +# comment this line out if you want to expose k8s services of type nodePort externally. +kube_proxy_nodeport_addresses_cidr: 127.0.0.0/8 + +# local Docker Hub mirror, if it exists +#docker_registry_mirrors: +# - http://mirror_ip:mirror_port +#docker_insecure_registries: +# - http://docker_insecure_registry_ip +#containerd_registries: +# "docker.io": +# - "https://registry-1.docker.io" +# - "http://mirror_ip:mirror_port" +#crio_registries_mirrors: +# - prefix: docker.io +# insecure: false +# blocked: false +# location: registry-1.docker.io +# mirrors: +# - location: mirror_ip:mirror_port +# insecure: false +#crio_insecure_registries: +# - http://crio_insecure_registry_ip + +# Docker registry running on the cluster allows us to store images not avaialble on Docker Hub +# The range of valid ports is 30000-32767 +registry_nodeport: 30500 +registry_local_address: "localhost:{{ '{{' }} registry_nodeport {{ '}}' }}" + +# Enable Pod Security Policy. This option enables PSP admission controller and creates minimal set of rules. +psp_enabled: true + +# Set image pull policy to Always. Pulls images prior to starting containers. Valid credentials must be configured. +always_pull_enabled: false + +{%- if minio in ['on', 'optional'] %} +## MinIO variables ## +# Enable Minio Storage service. +minio_enabled: {% if minio == 'on' %}true{% else %}false{% endif %} + +minio_tenant_enabled: true # Specifies whether to install MinIO Sample Tenant +minio_tenant_servers: 4 # The number of MinIO Tenant nodes +minio_tenant_volumes_per_server: 4 # The number of volumes per servers +minio_deploy_test_mode: true # true (Test Mode) - use a file as loop device when creating storage + # called "virtual block device" which is useful for test or automation purpose + # false (Performance Mode) - use an actual NVME or SSD device when creating storage +{%- endif %} diff --git a/profiles/host_vars.j2 b/generate/profiles_templates/common/host_vars.j2 similarity index 58% rename from profiles/host_vars.j2 rename to generate/profiles_templates/common/host_vars.j2 index a46c42fa..d90e4160 100644 --- a/profiles/host_vars.j2 +++ b/generate/profiles_templates/common/host_vars.j2 @@ -1,21 +1,48 @@ --- # Kubernetes node configuration +# Do not change profile_name and configured_arch here !!! +# To generate vars for different profile/architecture use make command +# generated for profile and arch: +profile_name: {{ name }} +configured_arch: {{ arch }} {% if sriov_network_dp in ['on', 'optional'] or qat in ['on', 'optional'] -%} # Enable IOMMU (required for SR-IOV networking and QAT) -iommu_enabled: {% if sriov_network_dp == 'on' or qat == 'on' %}true{% else %}false{% endif %} +iommu_enabled: {% if (sriov_network_dp == 'on' or qat == 'on') and on_vms != 'on' %}true{% else %}false{% endif %} {% endif %} -{%- if sriov_network_dp in ['on', 'optional'] %} +{%- if sriov_network_dp in ['on', 'optional'] or minio in ['on', 'optional'] %} # dataplane interface configuration list dataplane_interfaces: [] +{%- if on_vms == 'on' %} +# - name: enp4s0 # PF interface name +# bus_info: "04:00.0" # pci bus info +# pf_driver: iavf # driver inside VM +# sriov_numvfs: 0 +# default_vf_driver: "igb_uio" +# - name: enp5s0 # PF interface name +# bus_info: "05:00.0" # pci bus info +# pf_driver: iavf # driver inside VM +# sriov_numvfs: 0 +# default_vf_driver: "igb_uio" +# - name: enp6s0 # PF interface name +# bus_info: "06:00.0" # pci bus info +# pf_driver: iavf # driver inside VM +# sriov_numvfs: 0 +# default_vf_driver: "iavf" +# - name: enp7s0 # PF interface name +# bus_info: "07:00.0" # pci bus info +# pf_driver: iavf # driver inside VM +# sriov_numvfs: 0 +# default_vf_driver: "iavf" +{%- else %} # - name: enp24s0f0 # PF interface name # bus_info: "18:00.0" # pci bus info # pf_driver: ice # PF driver, "i40e", "ice" {%- if ddp in ['on', 'optional'] %} -# ddp_profile: "ice_comms-1.3.30.0.pkg" # DDP package name to be loaded into the NIC +# ddp_profile: "ice_comms-1.3.31.0.pkg" # DDP package name to be loaded into the NIC # For i40e(XV710-*) allowable ddp values are: "ecpri.pkg", "esp-ah.pkg", "ppp-oe-ol2tpv2.pkgo", "mplsogreudp.pkg" and "gtp.pkgo", replace as required - # For ice(E810-*) allowable ddp values are: ice_comms-1.3.[17,20,22,24,28,30].0.pkg such as "ice_comms-1.3.30.0.pkg", replace as required + # For ice(E810-*) allowable ddp values are: ice_comms-1.3.[17,20,22,24,28,30,31].0.pkg such as "ice_comms-1.3.31.0.pkg", replace as required # ddp_profile must be defined for first port of each network device. bifurcated cards will appear as unique devices. {% endif %} # default_vf_driver: "iavf" # default driver to be used with VFs if specific driver is not defined in the "sriov_vfs" section @@ -31,15 +58,21 @@ dataplane_interfaces: [] # bus_info: "18:00.1" # pf_driver: ice {%- if ddp in ['on', 'optional'] %} -# ddp_profile: "ice_comms-1.3.30.0.pkg" +# ddp_profile: "ice_comms-1.3.31.0.pkg" {%- endif %} # default_vf_driver: "vfio-pci" # sriov_numvfs: 4 # sriov_vfs: {} # no VFs with specific driver on this PF or "sriov_vfs" can be omitted for convenience +{%- endif %} sriov_cni_enabled: {% if sriov_network_dp == 'on' %}true{% else %}false{% endif %} {% endif %} +{%- if sriov_operator in ['on', 'optional'] %} +# Custom SriovNetworkNodePolicy manifests local path +# custom_sriov_network_policies_dir: /tmp/sriov +{%- endif %} + {%- if bond_cni in ['on', 'optional'] %} # Bond CNI bond_cni_enabled: {% if bond_cni == 'on' %}true{% else %}false{% endif %} @@ -49,32 +82,29 @@ bond_cni_enabled: {% if bond_cni == 'on' %}true{% else %}false{% endif %} # Install DPDK (required for SR-IOV networking) install_dpdk: {% if dpdk == 'on' %}true{% else %}false{% endif %} # DPDK version (will be in action if install_dpdk: true) -dpdk_version: "21.08" +dpdk_version: "21.11" # Custom DPDK patches local path -# dpdk_local_patches_dir: "/tmp/patches/dpdk-20.11" +#dpdk_local_patches_dir: "/tmp/patches/dpdk" +# It might be necessary to adjust the patch strip parameter, update as required. +#dpdk_local_patches_strip: 0 {%- endif %} {% if network_userspace in ['on', 'optional'] %} # Userspace networking userspace_cni_enabled: {% if network_userspace == 'on' %}true{% else %}false{% endif %} ovs_dpdk_enabled: {% if ovs_dpdk == 'on' %}true{% else %}false{% endif %} # Should be enabled with Userspace CNI, when VPP is set to "false"; 1G hugepages required -ovs_version: "v2.16.0" +ovs_version: "v2.16.2" # CPU mask for OVS-DPDK PMD threads ovs_dpdk_lcore_mask: 0x1 # Huge memory pages allocated by OVS-DPDK per NUMA node in megabytes -# example 1: "256,512" will allocate 256MB from node 0 abd 512MB from node 1 +# example 1: "256,512" will allocate 256MB from node 0 and 512MB from node 1 # example 2: "1024" will allocate 1GB from node 0 on a single socket board, e.g. in a VM ovs_dpdk_socket_mem: "256,0" vpp_enabled: {% if vpp == 'on'%}true{% else %}false{% endif %} # Should be enabled with Userspace CNI, when ovs_dpdk is set to "false"; 2M hugepages required {% endif %} # Set to 'true' to update i40e, ice and iavf kernel modules -update_nic_drivers: true +update_nic_drivers: {% if sriov_network_dp != 'on' and on_vms == 'on' %}false{% else %}true{% endif %} # Set 'true' to update NIC firmware -update_nic_firmware: false -# Additional list of NIC interfaces that the FW update will be executed on -# NOTE: FW update will be also executed on all NICs listed in "dataplane_interfaces[*].name" -firmware_update_nics: [] -# - enp24s0f0 -# - enp24s0f1 +update_nic_firmware: false # FW update will be executed on all NICs listed in "dataplane_interfaces[*].name" {% if sriov_network_dp in ['on', 'optional'] or qat in ['on', 'optional'] or network_userspace in ['on', 'optional'] -%} # Enables hugepages support @@ -82,12 +112,18 @@ hugepages_enabled: {% if sriov_network_dp == 'on' or qat == 'on' or network_user # Hugepage sizes available: 2M, 1G default_hugepage_size: {% if vpp == 'on' %}2M{% else %}1G{% endif %} # Sets how many hugepages should be created -number_of_hugepages: 4 +number_of_hugepages_1G: 4 +number_of_hugepages_2M: 128 {% endif %} {%- if ddp in ['on', 'optional'] %} # install Intel x700 & x800 series NICs DDP packages install_ddp_packages: {% if ddp == "on" %}true{% else %}false{% endif %} +# If following error appears: "Flashing failed: Operation not permitted" +# run deployment with update_nic_firmware: true +# or +# Disable ddp installation via install_ddp_packages: false + # set 'true' to enable custom ddp package to be loaded after reboot enable_ice_systemd_service: {% if ddp == "on" %}true{% else %}false{% endif %} {% endif %} @@ -98,20 +134,22 @@ update_qat_drivers: {% if qat == "on" %}true{% else %}false{% endif %} # qat interface configuration list qat_devices: [] -# - qat_dev: crypto01 # Device name as separate QAT Symmetric Crypto devices on which qat_sriov_numvfs will be distributed -# qat_id: "0000:ab:00.0" # Device QAT id one using DPDK compatible driver for VF devices to be used by vfio-pci kernel driver, replace as required -# qat_pci_type: c6xx # QAT Crypto Poll Mode Pci Driver id supported are dh895xcc,c6xx,c3xxx,d15xx,200xx & c4xxx -# qat_sriov_numvfs: 12 # set how many VFs per qat_id to be created such as c6xxvf support 48 so per device will be 16+16+16=48, replace as required - # Note: If don't want to assign VFs to id leave it as 0 such as qat_sriov_numvfs: 0 -# - qat_dev: crypto02 -# qat_id: "0000:xy:00.0" -# qat_pci_type: c6xx +{%- if on_vms == 'on' %} +# - qat_id: "0000:08:00.0" +# qat_sriov_numvfs: 0 # Have to be set to 0 here to not create any VFs inside VM. + +# - qat_id: "0000:09:00.0" +# qat_sriov_numvfs: 0 # Have to be set to 0 here to not create any VFs inside VM. +{%- else %} +# - qat_id: "0000:ab:00.0" # QAT device id one using DPDK compatible driver for VF devices to be used by vfio-pci kernel driver, replace as required +# qat_sriov_numvfs: 12 # Number of VFs per PF to create - cannot exceed the maximum number of VFs available for the device. Set to 0 to not create any VFs. + # Note: Currently when trying to create fewer virtual functions than the maximum, the maximum number always gets created +# - qat_id: "0000:xy:00.0" # qat_sriov_numvfs: 10 -# - qat_dev: crypto03 -# qat_id: "0000:yz:00.0" -# qat_pci_type: c6xx +# - qat_id: "0000:yz:00.0" # qat_sriov_numvfs: 10 +{%- endif %} {% endif %} {%- if openssl in ['on', 'optional'] %} @@ -119,54 +157,51 @@ qat_devices: [] openssl_install: {% if openssl == 'on' and qat == "on" %}true{% else %}false{% endif %} # This requires update_qat_drivers set to 'true' in host vars {% endif %} -{%- if cmk in ["on", "optional"] or sriov_network_dp in ["on", "optional"] or qat in ["on", "optional"] or network_userspace in ["on", "optional"] %} +{%- if isolcpu in ["on", "optional"] %} # CPU isolation from Linux scheduler -{%- if cmk == "on" or sriov_network_dp == "on" or qat == "on" or network_userspace == "on" %} -isolcpus_enabled: true -{% else %} -isolcpus_enabled: false -{% endif %} -# Disable CMKs autogenerate_isolcpus in group_vars to set custom isolcpus range; Otherwise this range will be automatically generated -# If defining range and using CMK you must account group_vars requested exclusive/shared cores +isolcpus_enabled: {% if isolcpu == 'on' %}true{% else %}false{% endif %} +{%- if on_vms == 'on' %} +isolcpus: "4-15" +{%- else %} isolcpus: "4-11" +{%- endif %} {% endif %} {%- if native_cpu_manager in ["on", "optional"] %} # Native CPU Manager (Kubernetes built-in) -# These settings are relevant only if in group_vars native_cpu_manager_enabled: true +# These settings are relevant only if in group_vars native_cpu_manager_enabled: true # Amount of CPU cores that will be reserved for the housekeeping (2000m = 2000 millicores = 2 cores) native_cpu_manager_system_reserved_cpus: 2000m # Amount of CPU cores that will be reserved for Kubelet native_cpu_manager_kube_reserved_cpus: 1000m -# Explicit list of the CPUs reserved from pods scheduling. -# Note: Supported only with kube_version 1.17 and newer, overrides native_cpu_manager_system_reserved_cpus and native_cpu_manager_kube_reserved_cpus. +# Explicit list of the CPUs reserved for the host level system threads and Kubernetes related threads #native_cpu_manager_reserved_cpus: "0,1,2" # Note: All remaining unreserved CPU cores will be consumed by the workloads. {% endif %} -{%- if pstate in ['on', 'optional'] or (sst in ['on', 'optional'] and arch == "clx") %} +{%- if (pstate in ['on', 'optional'] or sst in ['on', 'optional']) and arch in ['clx', 'icx', 'spr'] %} # Enable/Disable Intel PState scaling driver intel_pstate_enabled: {% if pstate == "on" or sst == "on" %}true{% else %}false{% endif %} # Config options for intel_pstate: disable, passive, force, no_hwp, hwp_only, support_acpi_ppc, per_cpu_perf_limits intel_pstate: {% if pstate == "on" or sst == "on" %}hwp_only{% else %}disable{% endif %} # Enable/Disable Intel Turbo Boost PState attribute -turbo_boost_enabled: true +turbo_boost_enabled: {% if on_vms != 'on' %}true{% else %}false{% endif %} {% endif -%} -{% if sst in ["on", "optional"] %} -{%- if arch == "icx" %} +{% if sst in ['on', 'optional'] %} +{%- if arch in ['icx', 'spr'] %} # Intel(R) SST-PP (perf-profile) configuration # [true] Enable Intel(R) SST-PP (perf-profile) # [false] Disable Intel(R) SST-PP (perf-profile) sst_pp_configuration_enabled: {% if sst == "on" %}true{% else %}false{% endif %} sst_pp_config_list: # "enable" or "disable" list options per SST-PP setup requirements - - sst_bf: "enable" # "enable" or "disable" Intel(R) SST-BF (base-freq) to configure with SST-PP + - sst_bf: "enable" # "enable" or "disable" Intel(R) SST-BF (base-freq) to configure with SST-PP - sst_cp: "enable" # "enable" or "disable" Intel(R) SST-CP (core-power) to configure with SST-PP. - sst_tf: "enable" # "enable" or "disable" Intel(R) SST-TF (turbo-freq) to configure with SST-PP. online_cpus_range: "auto" # "auto" will config turbo-freq for all available online CPUs or else define specific CPUs such as "2,3,5" to prioritize among others. {% endif %} # Intel Speed Select Base-Frequency configuration. +{%- if arch == 'clx' %} sst_bf_configuration_enabled: {% if sst == "on" %}true{% else %}false{% endif %} -{% if arch == "clx" %} # Intel Speed Select Base-Frequency configuration for Cascade Lake (CLX) Platforms. # CLX support of SST-BF requires 'intel_pstate' to be 'enabled' # Option clx_sst_bf_mode requires sst_bf_configuration_enabled to be set to 'true'. @@ -175,8 +210,8 @@ sst_bf_configuration_enabled: {% if sst == "on" %}true{% else %}false{% endif %} # [m] Set P1 on all cores (set min/max to 2300/2300) # [r] Revert cores to min/Turbo (set min/max to 800/3900) clx_sst_bf_mode: s -{% endif %} -{%- if arch == "icx" %} +{%- endif %} +{%- if arch == 'icx' %} # Intel Speed Select Base-Frequency configuration for Ice Lake (ICX) Platforms. # [true] Enable Intel Speed Select Base Frequency (SST-BF) # [false] Disable Intel Speed Select Base Frequency (SST-BF) @@ -217,18 +252,14 @@ sst_cp_cpu_clos: # assign required values to CLOS group after priority type setu # Intel(R) SST-TF (feature turbo-freq) configuration for Ice Lake (ICX) Platforms. # [true] Enable Intel Speed Select Turbo Frequency (SST-TF) -# [false] Disable Intel Speed Select Base Frequency (SST-TF) +# [false] Disable Intel Speed Select Turbo Frequency (SST-TF) sst_tf_configuration_enabled: {% if sst == "on" %}true{% else %}false{% endif %} -{% endif %} +{%- endif %} {% endif %} -{%- if rt_kernel in ['on', 'optional'] %} -# (CentOS 7 only) install real time kernel and related packages for flexran -install_real_time_package: false -{% endif -%} -{% if sgx in ['on', 'optional'] -%} +{%- if sgx in ['on', 'optional'] and arch in ['icx', 'spr'] %} # Intel Software Guard Extensions (SGX) -sgx_enabled: {% if sgx == 'on' %}true{% else %}false{% endif %} +configure_sgx: {% if sgx == 'on' %}true{% else %}false{% endif %} {% endif %} {%- if telemetry in ['on', 'optional'] %} @@ -249,3 +280,88 @@ intel_rdt_plugin_monitored_cores: "" # Additional list of plugins that will be excluded from collectd deployment. exclude_collectd_plugins: [] {% endif %} + +{%- if vm_mode in ['on'] and on_vms != 'on' %} +# VM image version for Ubuntu 20.04 - focal +vm_image_version: "20.04" +vms: + - type: "ctrl" + name: "vm-ctrl-1" + cpus: "8-11,64-67" + emu_cpus: "8,64" + numa: 0 + cpu_total: 8 + memory: 20480 + - type: "work" + name: "vm-work-1" + cpus: "12-19,68-75" + emu_cpus: "12,68" + numa: 0 + cpu_total: 16 + memory: 61440 +{%- if name in ['access', 'full_nfv', 'on_prem', 'remote_fp'] %} + pci: + - "18:02.2" + - "18:02.3" + - "18:02.4" + - "18:02.5" +{%- if qat == "on" %} +# - "3d:01.1" +# - "3f:01.1" +{%- endif %} +{%- else %} + pci: [] +{%- endif %} +{% endif -%} +{%- if power_manager in ['on', 'optional'] and arch in ['icx', 'clx', 'spr'] %} +# Power Operator Shared Profile/Workload settings. +# It is possible to create node-specific Power Profile +local_shared_profile: + enabled: false + node_max_shared_frequency: 2000 + node_min_shared_frequency: 1500 + +# Shared Workload is required to make use of Shared Power Profile +shared_workload: + enabled: false + reserved_cpus: [] # The CPUs in reserved_cpus should match the value of the reserved system CPUs in your Kubelet config file, if none please + # set here a dummy core - the last one to avoid AppQos bug + shared_workload_type: "global" # set to node name to make use of node-specific Power Profile, 'global' means use cluster-specific custom Power Profile +{% endif %} +{%- if minio in ['on', 'optional'] %} +# MinIO storage configuration +minio_pv: [] +# - name: "mnt-data-1" # PV identifier will be used for PVs names followed by node name(e.g., mnt-data-1-hostname) +# storageClassName: "local-storage" # Storage class name to match with PVC +# accessMode: "ReadWriteOnce" # Access mode when mounting a volume, e.g., ReadWriteOnce/ReadOnlyMany/ReadWriteMany/ReadWriteOncePod +# persistentVolumeReclaimPolicy: "Retain" # Recalim policy when a volume is released once it's bound, e.g., Retain/Recycle/Delete +# capacity: 1GiB # Size of the PV. support only GiB/TiB +# mountPath: /mnt/data0 # Mount path of a volume +# device: /dev/nvme0n1 # Target storage device name when creating a volume. + # When group_vars: minio_deploy_test_mode == true, use a file as a loop device for storage + # otherwise, an actual NVME or SSD device for storage on the device name. + +# - name: "mnt-data-2" +# storageClassName: "local-storage" +# accessMode: "ReadWriteOnce" +# persistentVolumeReclaimPolicy: "Retain" +# capacity: 1GiB +# mountPath: /mnt/data1 +# device: /dev/nvme1n1 + +# - name: "mnt-data-3" +# storageClassName: "local-storage" +# accessMode: "ReadWriteOnce" +# persistentVolumeReclaimPolicy: "Retain" +# capacity: 1GiB +# mountPath: /mnt/data2 +# device: /dev/nvme2n1 + +# - name: "mnt-data-4" +# storageClassName: "local-storage" +# accessMode: "ReadWriteOnce" +# persistentVolumeReclaimPolicy: "Retain" +# capacity: 1GiB +# mountPath: /mnt/data3 +# device: /dev/nvme3n1 +{% endif -%} diff --git a/profiles/inventory.j2 b/generate/profiles_templates/k8s/inventory.j2 similarity index 98% rename from profiles/inventory.j2 rename to generate/profiles_templates/k8s/inventory.j2 index 4863aa09..6d8a367d 100644 --- a/profiles/inventory.j2 +++ b/generate/profiles_templates/k8s/inventory.j2 @@ -6,6 +6,8 @@ node1 ansible_host=10.0.0.4 ip=10.0.0.4 ansible_user=USER ansible_password=XXX node2 ansible_host=10.0.0.5 ip=10.0.0.5 ansible_user=USER ansible_password=XXXX localhost ansible_connection=local ansible_python_interpreter=/usr/bin/python3 +[vm_host] + [kube_control_plane] controller1 controller2 @@ -26,4 +28,3 @@ kube_node [all:vars] ansible_python_interpreter=/usr/bin/python3 - diff --git a/profiles/profiles.yml b/generate/profiles_templates/k8s/profiles.yml similarity index 57% rename from profiles/profiles.yml rename to generate/profiles_templates/k8s/profiles.yml index 25ca9cdb..15504ee8 100644 --- a/profiles/profiles.yml +++ b/generate/profiles_templates/k8s/profiles.yml @@ -1,18 +1,3 @@ -## -## Copyright (c) 2020-2021 Intel Corporation. -## -## Licensed under the Apache License, Version 2.0 (the "License"); -## you may not use this file except in compliance with the License. -## You may obtain a copy of the License at -## -## http://www.apache.org/licenses/LICENSE-2.0 -## -## Unless required by applicable law or agreed to in writing, software -## distributed under the License is distributed on an "AS IS" BASIS, -## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -## See the License for the specific language governing permissions and -## limitations under the License. -## # How to use this file: # -------------------- # can be: @@ -20,15 +5,11 @@ # - optional (included in vars but disabled) # - off (not included, might as well drop it from the profile section) # -# except arch: -# - icx (Ice Lake) -# - clx (Cascade Lake) -# - skl (Sky Lake) -# # features: -# - arch +# - vm_mode +# - on_vms # - nfd -# - cmk +# - isolcpu # - native_cpu_manager # - bond_cni # - topology_manager @@ -45,17 +26,19 @@ # - pstate # - sst # - telemetry -# - rt_kernel # supported only on CentOS 7 which is not supported since v21.06 # - kmra -# +# - wireguard +# - power manager +# ovs_dpdk and network_userspace disabled temporary - waiting for new ovs version 2.17 compatible with dpdk 21.11 --- access: - arch: icx name: access + vm_mode: optional + on_vms: optional nfd: on - cmk: on - native_cpu_manager: optional + isolcpu: optional + native_cpu_manager: on topology_manager: on sriov_operator: on sriov_network_dp: optional @@ -65,30 +48,37 @@ access: dpdk: on pstate: optional telemetry: optional - istio: on + service_mesh: on + wireguard: on + firewall: optional basic: - arch: icx name: basic + vm_mode: optional + on_vms: optional nfd: on + isolcpu: optional topology_manager: on sriov_operator: optional sriov_network_dp: optional dpdk: optional telemetry: optional + wireguard: on + firewall: optional full_nfv: - arch: icx name: full_nfv + vm_mode: optional + on_vms: optional nfd: on - cmk: on - native_cpu_manager: optional + isolcpu: optional + native_cpu_manager: on topology_manager: on sriov_operator: on sriov_network_dp: optional bond_cni: on - network_userspace: on - ovs_dpdk: on + network_userspace: optional + ovs_dpdk: optional qat: on qat_dp: on openssl: on @@ -97,19 +87,25 @@ full_nfv: sgx_dp: on kmra: on tas: on + gas: optional ddp: on dpdk: on - pstate: on - sst: on + pstate: optional + sst: optional + power_manager: on telemetry: on - istio: on + service_mesh: on + wireguard: on + firewall: optional + minio: optional on_prem: - arch: icx name: on_prem + vm_mode: optional + on_vms: optional nfd: on - cmk: on - native_cpu_manager: optional + isolcpu: optional + native_cpu_manager: on topology_manager: on sriov_operator: on sriov_network_dp: optional @@ -117,37 +113,45 @@ on_prem: sgx_dp: on qat: on qat_dp: on - gpu: optional openssl: on tas: on dpdk: on bond_cni: optional pstate: optional sst: optional + power_manager: optional telemetry: on kmra: on - istio: on + service_mesh: on + wireguard: on + firewall: optional regional_dc: - arch: icx name: regional_dc + vm_mode: optional + on_vms: optional nfd: on + isolcpu: optional topology_manager: on sriov_operator: optional sriov_network_dp: optional native_cpu_manager: on gpu: on tas: on + gas: on dpdk: optional telemetry: on - istio: on + service_mesh: on + wireguard: on + firewall: optional remote_fp: - arch: icx name: remote_fp + vm_mode: optional + on_vms: optional nfd: on - cmk: on - native_cpu_manager: optional + isolcpu: optional + native_cpu_manager: on topology_manager: on sriov_operator: on sriov_network_dp: optional @@ -156,7 +160,6 @@ remote_fp: sgx_dp: on qat: on qat_dp: optional - gpu: optional openssl: on tas: on ddp: on @@ -164,6 +167,24 @@ remote_fp: dpdk: on pstate: on sst: optional + power_manager: optional telemetry: on kmra: on - istio: on + service_mesh: on + wireguard: on + firewall: optional + +storage: + name: storage + vm_mode: optional + on_vms: optional + nfd: on + native_cpu_manager: on + topology_manager: on + qat: optional + qat_dp: optional + tas: on + telemetry: on + wireguard: on + firewall: optional + minio: on diff --git a/generate/profiles_templates/vm/inventory.j2 b/generate/profiles_templates/vm/inventory.j2 new file mode 100644 index 00000000..84ed2e88 --- /dev/null +++ b/generate/profiles_templates/vm/inventory.j2 @@ -0,0 +1,22 @@ +[all] +host-for-vms-1 ansible_host=10.0.0.1 ip=10.0.0.1 ansible_user=USER ansible_password=XXXX +localhost ansible_connection=local ansible_python_interpreter=/usr/bin/python3 + +[vm_host] +host-for-vms-1 + +[kube_control_plane] +#vm-ctrl-1 + +[etcd] +#vm-ctrl-1 + +[kube_node] +#vm-work-1 + +[k8s_cluster:children] +kube_control_plane +kube_node + +[all:vars] +ansible_python_interpreter=/usr/bin/python3 diff --git a/generate/profiles_templates/vm/vm_host_profiles.yml b/generate/profiles_templates/vm/vm_host_profiles.yml new file mode 100644 index 00000000..6c39e4b4 --- /dev/null +++ b/generate/profiles_templates/vm/vm_host_profiles.yml @@ -0,0 +1,177 @@ +# How to use this file: +# -------------------- +# can be: +# - on (included and enabled) +# - optional (included in vars but disabled) +# - off (not included, might as well drop it from the profile section) +# +# features: +# - vm_mode +# - on_vms +# - nfd +# - isolcpu +# - native_cpu_manager +# - bond_cni +# - topology_manager +# - sriov_operator +# - sriov_network_dp +# - network_userspace +# - sgx +# - qat +# - gpu +# - openssl +# - tas +# - ddp +# - dpdk +# - pstate +# - sst +# - telemetry +# - kmra +# - wireguard + +# sriov_operator is permanently disabled in VM mode +# kmra is temporary disabled in VM mode - needs to be tested +# sst is temporary disabled in VM mode +# ovs_dpdk and network_userspace disabled temporary - waiting for new ovs version 2.17 compatible with dpdk 21.11 +--- +access: + name: access + vm_mode: on + on_vms: optional + nfd: on + isolcpu: optional + native_cpu_manager: on + topology_manager: on + sriov_operator: optional + sriov_network_dp: on + qat: on + qat_dp: on + openssl: on + dpdk: on + pstate: optional + telemetry: optional + service_mesh: on + wireguard: on + firewall: optional + +basic: + name: basic + vm_mode: on + on_vms: optional + nfd: on + isolcpu: optional + topology_manager: on + sriov_operator: optional + sriov_network_dp: optional + dpdk: optional + telemetry: optional + wireguard: on + firewall: optional + +full_nfv: + name: full_nfv + vm_mode: on + on_vms: optional + nfd: on + isolcpu: optional + native_cpu_manager: on + topology_manager: on + sriov_operator: optional + sriov_network_dp: on + bond_cni: on + network_userspace: optional + ovs_dpdk: optional + qat: on + qat_dp: on + openssl: on + gpu: optional + sgx: optional + sgx_dp: optional + kmra: optional + tas: on + gas: optional + ddp: on + dpdk: on + pstate: on + sst: optional + telemetry: on + service_mesh: on + wireguard: on + firewall: optional + +on_prem: + name: on_prem + vm_mode: on + on_vms: optional + nfd: on + isolcpu: optional + native_cpu_manager: on + topology_manager: on + sriov_operator: optional + sriov_network_dp: on + sgx: optional + sgx_dp: optional + qat: on + qat_dp: on + gpu: optional + openssl: on + tas: on + gas: optional + dpdk: on + bond_cni: optional + pstate: optional + sst: optional + telemetry: on + kmra: optional + service_mesh: on + wireguard: on + firewall: optional + +regional_dc: + name: regional_dc + vm_mode: on + on_vms: optional + nfd: on + isolcpu: optional + topology_manager: on + sriov_operator: optional + sriov_network_dp: optional + native_cpu_manager: on + gpu: on + tas: on + gas: on + dpdk: optional + telemetry: on + service_mesh: on + wireguard: on + firewall: optional + +remote_fp: + name: remote_fp + vm_mode: on + on_vms: optional + nfd: on + isolcpu: optional + native_cpu_manager: on + topology_manager: on + sriov_operator: optional + sriov_network_dp: on + network_userspace: optional + sgx: optional + sgx_dp: optional + qat: on + qat_dp: optional + gpu: optional + openssl: on + tas: on + gas: optional + ddp: on + bond_cni: optional + dpdk: on + pstate: on + sst: optional + telemetry: on + kmra: optional + service_mesh: on + wireguard: on + firewall: optional diff --git a/generate/profiles_templates/vm/vms_profiles.yml b/generate/profiles_templates/vm/vms_profiles.yml new file mode 100644 index 00000000..fbf18a28 --- /dev/null +++ b/generate/profiles_templates/vm/vms_profiles.yml @@ -0,0 +1,184 @@ +# How to use this file: +# -------------------- +# can be: +# - on (included and enabled) +# - optional (included in vars but disabled) +# - off (not included, might as well drop it from the profile section) +# +# features: +# - vm_mode +# - on_vms +# - nfd +# - isolcpu +# - native_cpu_manager +# - bond_cni +# - topology_manager +# - sriov_operator +# - sriov_network_dp +# - network_userspace +# - sgx +# - qat +# - gpu +# - openssl +# - tas +# - ddp +# - dpdk +# - pstate +# - sst +# - telemetry +# - kmra +# - wireguard + +# sriov_operator is permanently disabled in VM mode +# kmra is temporary disabled in VM mode - needs to be tested +# sst is temporary disabled in VM mode + +# On VMs +# sriov_network_dp is enabled on vms +# bond_cni is disabled on vms +# ddp is disabled on vms +# +# sgx - disabled on vms due to incompatible kernel version +# ovs_dpdk and network_userspace disabled temporary - waiting for new ovs version 2.17 compatible with dpdk 21.11 +--- +access: + name: access + vm_mode: on + on_vms: on + nfd: on + isolcpu: optional + native_cpu_manager: on + topology_manager: on + sriov_operator: optional + sriov_network_dp: on + qat: on + qat_dp: on + openssl: on + dpdk: on + pstate: optional + telemetry: optional + service_mesh: on + wireguard: on + firewall: optional + +basic: + name: basic + vm_mode: on + on_vms: on + nfd: on + isolcpu: optional + topology_manager: on + sriov_operator: optional + sriov_network_dp: optional + dpdk: optional + telemetry: optional + wireguard: on + firewall: optional + +full_nfv: + name: full_nfv + vm_mode: optional + on_vms: on + nfd: on + isolcpu: optional + native_cpu_manager: on + topology_manager: on + sriov_operator: optional + sriov_network_dp: on + bond_cni: optional + network_userspace: optional + ovs_dpdk: optional + qat: on + qat_dp: on + openssl: on + gpu: optional + sgx: optional + sgx_dp: optional + kmra: optional + tas: on + gas: optional + ddp: optional + dpdk: on + pstate: on + sst: optional + telemetry: on + service_mesh: on + wireguard: on + firewall: optional + +on_prem: + name: on_prem + vm_mode: on + on_vms: on + nfd: on + isolcpu: optional + native_cpu_manager: on + topology_manager: on + sriov_operator: optional + sriov_network_dp: on + sgx: optional + sgx_dp: optional + qat: on + qat_dp: on + gpu: optional + openssl: on + tas: on + gas: optional + dpdk: on + bond_cni: optional + pstate: optional + sst: optional + telemetry: on + kmra: optional + service_mesh: on + wireguard: on + firewall: optional + +regional_dc: + name: regional_dc + vm_mode: on + on_vms: on + nfd: on + isolcpu: optional + topology_manager: on + sriov_operator: optional + sriov_network_dp: optional + native_cpu_manager: on + gpu: on + tas: on + gas: on + dpdk: optional + telemetry: on + service_mesh: on + wireguard: on + firewall: optional + +remote_fp: + name: remote_fp + vm_mode: on + on_vms: on + nfd: on + isolcpu: optional + native_cpu_manager: on + topology_manager: on + sriov_operator: optional + sriov_network_dp: on + network_userspace: optional + sgx: optional + sgx_dp: optional + qat: on + qat_dp: optional + gpu: optional + openssl: on + tas: on + gas: optional + ddp: optional + bond_cni: optional + dpdk: on + pstate: on + sst: optional + telemetry: on + kmra: optional + service_mesh: on + wireguard: on + firewall: optional diff --git a/generate/render.py b/generate/render.py new file mode 100644 index 00000000..c1158a03 --- /dev/null +++ b/generate/render.py @@ -0,0 +1,53 @@ +#!/usr/bin/env python + +# Copyright (c) 2021 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +This module loads Container Experience Kit Profiles configuration file and renders example vars +and inventory files using Jinja templates. +""" + +import argparse +import importlib +from render.common.cli import parse_cli +from render.renderers.playbook import render_playbooks + +def main(): + args = parse_cli() + + # render profiles in given mode + _render_mode(args) + + # render playbooks + render_playbooks(args.profile) + +def _render_mode(args: argparse.Namespace) -> None: + # determine function name based on passed mode + mode_to_render = "render_{}_profiles".format(args.mode) + + # determine file name in which desired function is defined and implemented + renderer = "{}_profiles".format(args.mode) + + # try to import module and then + # obtain and call required method + try: + renderer_module = importlib.import_module("render.renderers.{}".format(renderer), package=None) + method = getattr(renderer_module, mode_to_render) + method(args) + except (ImportError, NameError) as e: + print("The method '{}' is not defined or cannot be imported... \nError: {}".format(method, e)) + +if __name__ == "__main__": + main() diff --git a/generate/render/__init__.py b/generate/render/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/generate/render/common/cli.py b/generate/render/common/cli.py new file mode 100644 index 00000000..3494bffc --- /dev/null +++ b/generate/render/common/cli.py @@ -0,0 +1,43 @@ +#!/usr/bin/env python + +# Copyright (c) 2021 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import argparse + +def parse_cli() -> argparse.Namespace: + """parse_cli creates CLI interface and returns parsed arguments""" + parser = argparse.ArgumentParser() + + # common args + parser.add_argument('--config', '-c', type=str, default="k8s_profiles.yml", + help='path to the profiles configuration file') + parser.add_argument('--output', type=str, default="../examples/k8s", + help='directory where generated example files for k8s mode will be stored') + parser.add_argument('--inventory', type=str, default="k8s_inventory.j2", + help='inventory template filepath') + parser.add_argument('--group', '-g', type=str, default="group_vars.j2", + help='group_vars template filepath') + parser.add_argument('--host', type=str, default="host_vars.j2", + help='host_vars template filepath') + parser.add_argument('--profile', '-p', type=str, default='', + choices={'all_examples', 'access', 'basic', 'full_nfv', 'on_prem', 'regional_dc', 'remote_fp', 'storage'}, # add new profiles here + help='''profile name which files, required in deployment, will be copied to the project root directory''') + parser.add_argument('--arch', '-a', type=str, default='icx', choices={"icx", "clx", "skl", "spr"}) # please add acronyms for new architectures here + parser.add_argument('--mode', type=str, default='k8s', choices={"k8s", "vm"}, help='generate configuration files for selected mode') # please add new modes' name here + + # vm mode specific args + parser.add_argument('--vmsconfig', type=str, default="vms_profiles.yml", help='configuration file for created Virtual Machine') + + return parser.parse_args() diff --git a/generate/render/common/common.py b/generate/render/common/common.py new file mode 100644 index 00000000..c1a295a0 --- /dev/null +++ b/generate/render/common/common.py @@ -0,0 +1,95 @@ +#!/usr/bin/env python + +# Copyright (c) 2021 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +This file contains common functions used to generate profiles in different modes +""" + +import argparse +import os +from datetime import datetime +from shutil import move, copy + +from ruamel.yaml import YAML +from jinja2 import Environment, FileSystemLoader + +def load_config(path: str) -> dict: + """Loads YAML file and returns it as configuration dict.""" + with open(path) as config_file: + yaml = YAML(typ='safe') + profiles = yaml.load(config_file) + return profiles + +def create_dir_idempotent(path: str) -> None: + """Creates directory if not present.""" + if not os.path.exists(path): + os.makedirs(path) + +def render(template_path: str, jinja_vars: dict, target_path: str) -> None: + """Renders Jinja template and writes it to file.""" + file_loader = FileSystemLoader('.') + template = Environment(keep_trailing_newline=True, loader=file_loader, autoescape=True).get_template(template_path) + out = template.render(jinja_vars) + with open(target_path, "w+") as f: + f.write(out) + +def add_arch_parameter(profiles: dict, args: argparse.Namespace) -> None: + """Add architecture information to profiles config""" + for p in profiles.values(): + p['arch'] = args.arch + +def create_backups(src: str, dirs: list=[], files: list=[]) -> None: + """Create backup for given dirs/files""" + # create specific backup dir + previous_profile_name = _get_previous_profile_name() + if previous_profile_name: + backup_dir_name = previous_profile_name + "_" + datetime.now().strftime('%Y%m%d_%H%M%S') + path_to_backup_dir = os.path.join('backups', backup_dir_name) + create_dir_idempotent(path_to_backup_dir) + + for d in dirs: + _backup_dirs(src, path_to_backup_dir, d) + + for f in files: + _backup_files(src, path_to_backup_dir, f) + +# Helper Functions +def _move(path_to_dir: str, path_to_backup_dir: str) -> None: + """Move directory from specific path to backup path""" + move(path_to_dir, path_to_backup_dir) + +def _backup_dirs(src: str, dst: str, name: str) -> None: + path = os.path.join(src, name) + if os.path.exists(path): + _move(path, dst) + +def _backup_files(src: str, dst: str, name: str) -> None: + path = os.path.join(src, name) + if os.path.exists(path): + print(path) + copy(path, dst) + +def _get_previous_profile_name() -> str: + group_vars_path = os.path.join('./', 'group_vars', 'all.yml') + if os.path.exists(group_vars_path): + with open(group_vars_path) as f: + for line in f: + if 'profile_name:' in line: + name = line.split()[1] + return name + + # project root directory is clean + return "" diff --git a/generate/render/renderers/k8s_profiles.py b/generate/render/renderers/k8s_profiles.py new file mode 100644 index 00000000..05f381c4 --- /dev/null +++ b/generate/render/renderers/k8s_profiles.py @@ -0,0 +1,72 @@ +#!/usr/bin/env python + +# Copyright (c) 2021 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +This file contains functions needed for profile generation in k8s mode +""" + +import argparse +import os +from render.common.common import create_dir_idempotent, render, load_config, add_arch_parameter, create_backups + +def render_k8s_profiles(args: argparse.Namespace) -> None: + """Creates example CEK profiles in k8s mode""" + # create backup for already generated profile's files + if 'all_examples' != args.profile: + src = "./" # look for reference in project_root_dir + create_backups(src, ['host_vars', 'group_vars'], ['inventory.ini',]) + + # load config from k8s_profiles.yml + k8s_profiles = load_config(args.config) + + # add architecture information + add_arch_parameter(k8s_profiles, args) + + # create example diretory with all profiles and its files + _create_k8s_examples(k8s_profiles, args) + +# Helper functions +def _create_example(config: dict, vars_path_prefix: str, inventory_path: str, args: argparse.Namespace) -> None: + group_vars_dir_path = os.path.join(vars_path_prefix, "group_vars") + host_vars_dir_path = os.path.join(vars_path_prefix, "host_vars") + create_dir_idempotent(group_vars_dir_path) + create_dir_idempotent(host_vars_dir_path) + + render(args.group, config, os.path.join(group_vars_dir_path, "all.yml")) + render(args.host, config, os.path.join(host_vars_dir_path, "node1.yml")) + + # do not generate inventory file if already there + inventory_path = os.path.join(inventory_path, "inventory.ini") + if not os.path.exists(inventory_path): + render(args.inventory, config, inventory_path) + +def _create_k8s_examples(profiles: dict, args: argparse.Namespace) -> None: + """Creates all sample files for profiles in k8s mode if provided profiles is 'all_examples', otherwise + only files for the specific profile will be generated into project root direcotory""" + if 'all_examples' == args.profile: + for k8s_profile, k8s_config in profiles.items(): + vars_path_prefix = os.path.join(args.output, k8s_profile) + inventory_path = vars_path_prefix + + _create_example(k8s_config, vars_path_prefix, inventory_path, args) + print("\nSample files for all profiles in K8S mode are generated to examples directory.") + else: + k8s_profile = args.profile + vars_path_prefix = './' + inventory_path = vars_path_prefix + + _create_example(profiles[k8s_profile], vars_path_prefix, inventory_path, args) + print("\nFiles needed for the '{}' profile are generated in the project root dir.".format(k8s_profile)) diff --git a/generate/render/renderers/playbook.py b/generate/render/renderers/playbook.py new file mode 100644 index 00000000..919ae3b8 --- /dev/null +++ b/generate/render/renderers/playbook.py @@ -0,0 +1,62 @@ +#!/usr/bin/env python + +# Copyright (c) 2021 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +This file contains functions needed for playbooks generation +""" + +import os +from render.common.common import render + +_available_playbooks = ['basic', 'full_nfv', 'access', 'remote_fp', 'regional_dc', 'on_prem', 'storage'] +_playbook_dir = 'playbooks' + +def render_playbooks(profile: str) -> None: + """Renders playbooks for all CEK profiles""" + # generate playbooks + if profile not in _available_playbooks: + _create_all_playbooks() # profile is set to all_examples + else: + _create_playbooks_for_profile(profile) # specific profile was requested + + # print some usefull information for user at the end + if profile != 'all_examples': + _print_command(profile) + +def _create_playbook(template_name: str, playbook_file: str, jinja_vars: dict, playbook_subdir: str='') -> None: + """Creates one playbook""" + playbook_path = os.path.join(_playbook_dir, playbook_subdir, playbook_file) + template_path = os.path.join("generate/playbook_templates", template_name) + + render(template_path, jinja_vars, playbook_path) + +def _create_all_playbooks() -> None: + """Creates all playbooks files""" + for playbook_name in _available_playbooks: + _create_playbooks_for_profile(playbook_name) + +def _create_playbooks_for_profile(profile: str) -> None: + """Creates playbooks only for specific profile""" + playbook_file = profile + ".yml" + jinja_vars = {"playbook_name": profile} + _create_playbook("main_playbook.j2", playbook_file, jinja_vars) + _create_playbook("infra_playbook.j2", playbook_file, jinja_vars, playbook_subdir="infra") + _create_playbook("intel_playbook.j2", playbook_file, jinja_vars, playbook_subdir="intel") + +def _print_command(profile: str) -> None: + print("""To run your deployment configure host vars and group vars, then use the following command: + + ansible-playbook -i inventory.ini playbooks/{}.yml""".format(profile)) diff --git a/generate/render/renderers/vm_profiles.py b/generate/render/renderers/vm_profiles.py new file mode 100644 index 00000000..85fb16fe --- /dev/null +++ b/generate/render/renderers/vm_profiles.py @@ -0,0 +1,108 @@ +#!/usr/bin/env python + +# Copyright (c) 2021 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +This file contains functions needed for profile generation in VM mode +""" + +import argparse +import os +from render.common.common import create_dir_idempotent, render, load_config, add_arch_parameter, create_backups + +def render_vm_profiles(args: argparse.Namespace) -> None: + """Creates example CEK profiles in VM mode""" + # create backup for already generated profile's hv, gv and inventory + if 'all_examples' != args.profile: + src = "./" # look for reference in project_root_dir + create_backups(src, ['host_vars', 'group_vars'], ['inventory.ini',]) + + # load config from vm_profiles.yml + vm_profiles = load_config(args.vmsconfig) + + # add architecture information + add_arch_parameter(vm_profiles, args) + + # create example diretory with all profiles and its files for VM configuration + _create_vms_examples(vm_profiles, args) + + # load config for VMs' host + host_vm_profiles = load_config(args.config) + + # add architecture information + add_arch_parameter(host_vm_profiles, args) + + # create example diretory with all profiles and its files + _create_host_vm_examples(host_vm_profiles, args) + +# Helper Functions +# creating files needed by the VMs +def _create_vm_example(config: dict, vars_path_prefix: str, args: argparse.Namespace) -> None: + """Create one sample file required by the VM""" + host_vars_dir_path = os.path.join(vars_path_prefix, "host_vars") + create_dir_idempotent(host_vars_dir_path) + + render(args.host, config, os.path.join(host_vars_dir_path, "vm-ctrl-1.yml")) + render(args.host, config, os.path.join(host_vars_dir_path, "vm-work-1.yml")) + +def _create_vms_examples(profiles: dict, args: argparse.Namespace) -> None: + """Creates sample configuration files required by the VMs. If profile is marked as all_examples + then all available examples will be created, otherwise only specific files will be generated into project root directory.""" + if 'all_examples' == args.profile: + for vms_profile, vms_config in profiles.items(): + vars_path_prefix = os.path.join(args.output, vms_profile) + + _create_vm_example(vms_config, vars_path_prefix, args) + else: + vms_profile = args.profile + vms_config = profiles[vms_profile] + vars_path_prefix = './' + + _create_vm_example(vms_config, vars_path_prefix, args) + +# creating files needed by host on top of which VMs will be created +def _create_host_example(config: dict, vars_path_prefix: str, inventory_path: str, args: argparse.Namespace) -> None: + """Create one sample file required by host on top of which VMs will be created""" + host_vars_dir_path = os.path.join(vars_path_prefix, "host_vars") + group_vars_dir_path = os.path.join(vars_path_prefix, "group_vars") + create_dir_idempotent(group_vars_dir_path) + create_dir_idempotent(host_vars_dir_path) + + render(args.host, config, os.path.join(host_vars_dir_path, "host-for-vms-1.yml")) + render(args.group, config, os.path.join(group_vars_dir_path, "all.yml")) + + # do not generate inventory file if already there + inventory_path = os.path.join(inventory_path, "inventory.ini") + if not os.path.exists(inventory_path): + render(args.inventory, config, inventory_path) + +def _create_host_vm_examples(profiles: dict, args: argparse.Namespace) -> None: + """Creates sample configuration files required by host on which the VMs will be created. + If profile is marked as all_examples then all available examples will be created, + otherwise only specific files will be generated into project root directory.""" + if 'all_examples' == args.profile: + for host_vm_profile, host_vm_config in profiles.items(): + vars_path_prefix = os.path.join(args.output, host_vm_profile) + inventory_path = vars_path_prefix # inventory file is supposed to be created only with host-related files + + _create_host_example(host_vm_config, vars_path_prefix, inventory_path, args) + print("\nSample files for all profiles in VM mode are generated to examples directory.") + else: + host_vm_profile = args.profile + vars_path_prefix = './' + inventory_path = vars_path_prefix + + _create_host_example(profiles[host_vm_profile], vars_path_prefix, inventory_path, args) + print("\nFiles needed for the '{}' profile are generated in the project root dir.".format(host_vm_profile)) diff --git a/playbooks/full_nfv.yml b/playbooks/full_nfv.yml deleted file mode 100644 index 6ed76fb7..00000000 --- a/playbooks/full_nfv.yml +++ /dev/null @@ -1,26 +0,0 @@ -## -## Copyright (c) 2020-2021 Intel Corporation. -## -## Licensed under the Apache License, Version 2.0 (the "License"); -## you may not use this file except in compliance with the License. -## You may obtain a copy of the License at -## -## http://www.apache.org/licenses/LICENSE-2.0 -## -## Unless required by applicable law or agreed to in writing, software -## distributed under the License is distributed on an "AS IS" BASIS, -## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -## See the License for the specific language governing permissions and -## limitations under the License. -## ---- -- name: preflight checks - import_playbook: preflight.yml -- name: configure target hosts OS layer - import_playbook: infra/full_nfv.yml -- name: provision Kubernetes cluster using kubespray - import_playbook: k8s/k8s.yml - when: kubernetes | default(true) -- name: install Intel BMRA Container Experience Kit features - import_playbook: intel/full_nfv.yml - when: kubernetes | default(true) diff --git a/playbooks/infra/access.yml b/playbooks/infra/access.yml deleted file mode 100644 index 47f81b90..00000000 --- a/playbooks/infra/access.yml +++ /dev/null @@ -1,126 +0,0 @@ -## -## Copyright (c) 2020-2021 Intel Corporation. -## -## Licensed under the Apache License, Version 2.0 (the "License"); -## you may not use this file except in compliance with the License. -## You may obtain a copy of the License at -## -## http://www.apache.org/licenses/LICENSE-2.0 -## -## Unless required by applicable law or agreed to in writing, software -## distributed under the License is distributed on an "AS IS" BASIS, -## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -## See the License for the specific language governing permissions and -## limitations under the License. -## ---- -- hosts: 127.0.0.1 - connection: local - tasks: [] - roles: - - role: bootstrap/ansible_host - environment: "{{ proxy_env | d({}) }}" - any_errors_fatal: true - -# apply common cluster node configuration -- hosts: k8s_cluster - tasks: [] - handlers: - - name: reboot server - reboot: { reboot_timeout: 1200 } - roles: - - role: cluster_defaults - - role: bootstrap/configure_proxy - - role: bootstrap/install_packages - - role: bootstrap/update_grub - when: update_kernel | default(false) - - role: bootstrap/configure_security - - role: bootstrap/apply_kubernetes_reqs - - role: bootstrap/configure_dns - when: ansible_distribution == "Ubuntu" and dns_disable_stub_listener | default(true) | bool - environment: "{{ proxy_env | d({}) }}" - any_errors_fatal: true - -# apply worker node kernel configuration -- hosts: kube_node - tasks: [] - handlers: - - name: reboot server - reboot: { reboot_timeout: 1200 } - roles: - - role: cluster_defaults - - role: bootstrap/configure_hugepages - when: hugepages_enabled | default(true) - - role: bootstrap/configure_cpu_isolation - when: isolcpus_enabled | default(true) - - role: bootstrap/configure_intel_pstate - when: intel_pstate_enabled | default(false) | bool - - role: bootstrap/set_sriov_kernel_flags - when: iommu_enabled | default(true) - - role: bootstrap/set_rdt_kernel_flags - when: telegraf_enabled | default(true) | bool - - role: bootstrap/install_rt_package - when: - - install_real_time_package | default(false) | bool - - ansible_distribution == 'CentOS' - - ansible_distribution_version < '8' - - role: bootstrap/update_grub - environment: "{{ proxy_env | d({}) }}" - any_errors_fatal: true - -# install worker node qat software -- hosts: kube_node - tasks: [] - handlers: - - name: reboot server - reboot: { reboot_timeout: 1200 } - roles: - - role: cluster_defaults - - role: bootstrap/install_qat_drivers_services - when: - - update_qat_drivers | default(false) - - qat_devices is defined and (qat_devices|length>0) - environment: "{{ proxy_env | d({}) }}" - any_errors_fatal: true - -# install worker node network software -- hosts: kube_node - tasks: [] - roles: - - role: cluster_defaults - - role: bootstrap/update_nic_drivers - when: update_nic_drivers | default(false) | bool - - role: bootstrap/update_nic_firmware - tags: firmware - when: - - update_nic_firmware | default(false) | bool - - update_nic_drivers | default(false) | bool - - role: bootstrap/determine_dataplane_interfaces - environment: "{{ proxy_env | d({}) }}" - any_errors_fatal: true - -# install DPDK and apply SRIOV configuration -- hosts: kube_node - roles: - - role: cluster_defaults - - role: install_dpdk - when: install_dpdk | default(true) - tags: dpdk - - role: sriov_nic_init - when: - - iommu_enabled | default(true) - - install_dpdk | default(true) - - not sriov_network_operator_enabled | default(true) - - role: bootstrap/configure_qat - when: - - qat_devices is defined and (qat_devices|length>0) - - iommu_enabled | default(true) - - update_qat_drivers | default(false) - - role: bootstrap/configure_openssl - when: - - qat_devices is defined and (qat_devices|length>0) - - iommu_enabled | default(true) - - update_qat_drivers | default(false) - - openssl_install | default(false) - environment: "{{ proxy_env | d({}) }}" - any_errors_fatal: true diff --git a/playbooks/infra/basic.yml b/playbooks/infra/basic.yml deleted file mode 100644 index 83a06dc6..00000000 --- a/playbooks/infra/basic.yml +++ /dev/null @@ -1,96 +0,0 @@ -## -## Copyright (c) 2020-2021 Intel Corporation. -## -## Licensed under the Apache License, Version 2.0 (the "License"); -## you may not use this file except in compliance with the License. -## You may obtain a copy of the License at -## -## http://www.apache.org/licenses/LICENSE-2.0 -## -## Unless required by applicable law or agreed to in writing, software -## distributed under the License is distributed on an "AS IS" BASIS, -## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -## See the License for the specific language governing permissions and -## limitations under the License. -## ---- -- hosts: 127.0.0.1 - connection: local - tasks: [] - roles: - - role: bootstrap/ansible_host - environment: "{{ proxy_env | d({}) }}" - any_errors_fatal: true - -# apply common cluster node configuration -- hosts: k8s_cluster - tasks: [] - handlers: - - name: reboot server - reboot: { reboot_timeout: 1200 } - roles: - - role: cluster_defaults - - role: bootstrap/configure_proxy - - role: bootstrap/install_packages - - role: bootstrap/update_grub - when: update_kernel | default(false) - - role: bootstrap/configure_security - - role: bootstrap/apply_kubernetes_reqs - - role: bootstrap/configure_dns - when: ansible_distribution == "Ubuntu" and dns_disable_stub_listener | default(true) | bool - environment: "{{ proxy_env | d({}) }}" - any_errors_fatal: true - -# apply worker node kernel configuration -- hosts: kube_node - tasks: [] - handlers: - - name: reboot server - reboot: { reboot_timeout: 1200 } - roles: - - role: cluster_defaults - - role: bootstrap/configure_hugepages - when: hugepages_enabled | default(false) - - role: bootstrap/configure_cpu_isolation - when: isolcpus_enabled | default(false) - - role: bootstrap/configure_intel_pstate - when: intel_pstate_enabled | default(false) | bool - - role: bootstrap/set_sriov_kernel_flags - when: iommu_enabled | default(false) - - role: bootstrap/set_rdt_kernel_flags - when: telegraf_enabled | default(true) | bool - - role: bootstrap/update_grub - environment: "{{ proxy_env | d({}) }}" - any_errors_fatal: true - -# install worker node network software -- hosts: kube_node - tasks: [] - roles: - - role: cluster_defaults - - role: bootstrap/update_nic_drivers - when: update_nic_drivers | default(false) | bool - - role: bootstrap/update_nic_firmware - tags: firmware - when: - - update_nic_firmware | default(false) | bool - - update_nic_drivers | default(false) | bool - - role: bootstrap/determine_dataplane_interfaces - environment: "{{ proxy_env | d({}) }}" - any_errors_fatal: true - -# install DPDK and apply SRIOV configuration -- hosts: kube_node - roles: - - role: cluster_defaults - - role: install_dpdk - when: install_dpdk | default(false) - tags: dpdk - - role: sriov_nic_init - when: - - iommu_enabled | default(false) - - install_dpdk | default(false) - - not sriov_network_operator_enabled | default(false) - environment: "{{ proxy_env | d({}) }}" - any_errors_fatal: true - diff --git a/playbooks/infra/container_runtime_deployment.yml b/playbooks/infra/container_runtime_deployment.yml new file mode 100644 index 00000000..6f02159e --- /dev/null +++ b/playbooks/infra/container_runtime_deployment.yml @@ -0,0 +1,22 @@ +## +## Copyright (c) 2020-2022 Intel Corporation. +## +## Licensed under the Apache License, Version 2.0 (the "License"); +## you may not use this file except in compliance with the License. +## You may obtain a copy of the License at +## +## http://www.apache.org/licenses/LICENSE-2.0 +## +## Unless required by applicable law or agreed to in writing, software +## distributed under the License is distributed on an "AS IS" BASIS, +## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +## See the License for the specific language governing permissions and +## limitations under the License. +## +--- +- hosts: k8s_cluster + roles: + - role: cluster_defaults + - role: container_engine + environment: "{{ proxy_env | d({}) }}" + any_errors_fatal: true diff --git a/playbooks/infra/full_nfv.yml b/playbooks/infra/full_nfv.yml deleted file mode 100644 index f6a31419..00000000 --- a/playbooks/infra/full_nfv.yml +++ /dev/null @@ -1,146 +0,0 @@ -## -## Copyright (c) 2020-2021 Intel Corporation. -## -## Licensed under the Apache License, Version 2.0 (the "License"); -## you may not use this file except in compliance with the License. -## You may obtain a copy of the License at -## -## http://www.apache.org/licenses/LICENSE-2.0 -## -## Unless required by applicable law or agreed to in writing, software -## distributed under the License is distributed on an "AS IS" BASIS, -## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -## See the License for the specific language governing permissions and -## limitations under the License. -## ---- -- hosts: 127.0.0.1 - connection: local - tasks: [] - roles: - - role: bootstrap/ansible_host - environment: "{{ proxy_env | d({}) }}" - any_errors_fatal: true - -# apply common cluster node configuration -- hosts: k8s_cluster - tasks: [] - handlers: - - name: reboot server - reboot: { reboot_timeout: 1200 } - roles: - - role: cluster_defaults - - role: bootstrap/configure_proxy - - role: bootstrap/install_packages - - role: bootstrap/update_grub - when: update_kernel | default(false) - - role: bootstrap/configure_security - - role: bootstrap/apply_kubernetes_reqs - - role: bootstrap/configure_dns - when: ansible_distribution == "Ubuntu" and dns_disable_stub_listener | default(true) | bool - environment: "{{ proxy_env | d({}) }}" - any_errors_fatal: true - -# apply worker node kernel configuration -- hosts: kube_node - tasks: [] - handlers: - - name: reboot server - reboot: { reboot_timeout: 1200 } - roles: - - role: cluster_defaults - - role: bootstrap/configure_hugepages - when: hugepages_enabled | default(true) - - role: bootstrap/configure_cpu_isolation - when: isolcpus_enabled | default(true) - - role: bootstrap/configure_intel_pstate - when: intel_pstate_enabled | default(true) | bool - - role: bootstrap/set_sriov_kernel_flags - when: iommu_enabled | default(true) - - role: bootstrap/set_rdt_kernel_flags - when: telegraf_enabled | default(true) | bool - - role: bootstrap/configure_sst - tags: sst - when: - - (ansible_distribution == 'Ubuntu' and ansible_distribution_version >= '20.04') or - (ansible_distribution == "CentOS" and ansible_distribution_version >= '8.2') or - (ansible_distribution == "RedHat" and ansible_distribution_version >= '8.3') - - (sst_bf_configuration_enabled | default(true)) or - (sst_cp_configuration_enabled | default(false)) or - (sst_tf_configuration_enabled | default(false)) - - role: bootstrap/install_gpu_kernel - when: gpu_dp_enabled | default (false) - - role: bootstrap/install_rt_package - when: - - install_real_time_package | default(false) | bool - - ansible_distribution == 'CentOS' - - ansible_distribution_version < '8' - - role: bootstrap/update_grub - environment: "{{ proxy_env | d({}) }}" - any_errors_fatal: true - -# install worker node qat software -- hosts: kube_node - tasks: [] - handlers: - - name: reboot server - reboot: { reboot_timeout: 1200 } - roles: - - role: cluster_defaults - - role: bootstrap/install_qat_drivers_services - when: - - update_qat_drivers | default(false) - - qat_devices is defined and (qat_devices|length>0) - environment: "{{ proxy_env | d({}) }}" - any_errors_fatal: true - -# install worker node network software -- hosts: kube_node - tasks: [] - roles: - - role: cluster_defaults - - role: bootstrap/update_nic_drivers - when: update_nic_drivers | default(false) | bool - - role: bootstrap/update_nic_firmware - tags: firmware - when: - - update_nic_firmware | default(false) | bool - - update_nic_drivers | default(false) | bool - - role: bootstrap/determine_dataplane_interfaces - environment: "{{ proxy_env | d({}) }}" - any_errors_fatal: true - -# install DPDK and apply SRIOV configuration -- hosts: kube_node - roles: - - role: cluster_defaults - - role: install_dpdk - when: ovs_dpdk_enabled | default(false) or install_dpdk | default(true) - tags: dpdk - - role: install_ddp_pkgs - when: install_ddp_packages | default(true) | bool - - role: sriov_nic_init - when: - - iommu_enabled | default(true) - - install_dpdk | default(true) - - not sriov_network_operator_enabled | default(true) - - role: bootstrap/configure_qat - when: - - qat_devices is defined and (qat_devices|length>0) - - iommu_enabled | default(true) - - update_qat_drivers | default(false) - - role: bootstrap/configure_openssl - when: - - qat_devices is defined and (qat_devices|length>0) - - iommu_enabled | default(true) - - update_qat_drivers | default(false) - - openssl_install | default(false) - - role: sgx_configure_icx - tags: sgx - when: - - sgx_enabled | default(true) - - (ansible_os_family == "Debian" and ansible_distribution_version != '21.04') - or (ansible_os_family == "RedHat" and ansible_distribution_version != '8.4') - environment: "{{ proxy_env | d({}) }}" - any_errors_fatal: true - diff --git a/playbooks/infra/on_prem.yml b/playbooks/infra/on_prem.yml deleted file mode 100644 index cb585e38..00000000 --- a/playbooks/infra/on_prem.yml +++ /dev/null @@ -1,143 +0,0 @@ -## -## Copyright (c) 2020-2021 Intel Corporation. -## -## Licensed under the Apache License, Version 2.0 (the "License"); -## you may not use this file except in compliance with the License. -## You may obtain a copy of the License at -## -## http://www.apache.org/licenses/LICENSE-2.0 -## -## Unless required by applicable law or agreed to in writing, software -## distributed under the License is distributed on an "AS IS" BASIS, -## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -## See the License for the specific language governing permissions and -## limitations under the License. -## ---- -- hosts: 127.0.0.1 - connection: local - tasks: [] - roles: - - role: bootstrap/ansible_host - environment: "{{ proxy_env | d({}) }}" - any_errors_fatal: true - -# apply common cluster node configuration -- hosts: k8s_cluster - tasks: [] - handlers: - - name: reboot server - reboot: { reboot_timeout: 1200 } - roles: - - role: cluster_defaults - - role: bootstrap/configure_proxy - - role: bootstrap/install_packages - - role: bootstrap/update_grub - when: update_kernel | default(false) - - role: bootstrap/configure_security - - role: bootstrap/apply_kubernetes_reqs - - role: bootstrap/configure_dns - when: ansible_distribution == "Ubuntu" and dns_disable_stub_listener | default(true) | bool - environment: "{{ proxy_env | d({}) }}" - any_errors_fatal: true - -# apply worker node kernel configuration -- hosts: kube_node - tasks: [] - handlers: - - name: reboot server - reboot: { reboot_timeout: 1200 } - roles: - - role: cluster_defaults - - role: bootstrap/configure_hugepages - when: hugepages_enabled | default(true) - - role: bootstrap/configure_cpu_isolation - when: isolcpus_enabled | default(true) - - role: bootstrap/configure_intel_pstate - when: intel_pstate_enabled | default(false) | bool - - role: bootstrap/set_sriov_kernel_flags - when: iommu_enabled | default(true) - - role: bootstrap/set_rdt_kernel_flags - when: telegraf_enabled | default(true) | bool - - role: bootstrap/configure_sst - tags: sst - when: - - (ansible_distribution == 'Ubuntu' and ansible_distribution_version >= '20.04') or - (ansible_distribution == "CentOS" and ansible_distribution_version >= '8.2') or - (ansible_distribution == "RedHat" and ansible_distribution_version >= '8.3') - - (sst_bf_configuration_enabled | default(true)) or - (sst_cp_configuration_enabled | default(false)) or - (sst_tf_configuration_enabled | default(false)) - - role: bootstrap/install_gpu_kernel - when: gpu_dp_enabled | default(false) - - role: bootstrap/install_rt_package - when: - - install_real_time_package | default(false) | bool - - ansible_distribution == 'CentOS' - - ansible_distribution_version < '8' - - role: bootstrap/update_grub - environment: "{{ proxy_env | d({}) }}" - any_errors_fatal: true - -# install worker node qat software -- hosts: kube_node - tasks: [] - handlers: - - name: reboot server - reboot: { reboot_timeout: 1200 } - roles: - - role: cluster_defaults - - role: bootstrap/install_qat_drivers_services - when: - - update_qat_drivers | default(false) - - qat_devices is defined and (qat_devices|length>0) - environment: "{{ proxy_env | d({}) }}" - any_errors_fatal: true - -# install worker node network software -- hosts: kube_node - tasks: [] - roles: - - role: cluster_defaults - - role: bootstrap/update_nic_drivers - when: update_nic_drivers | default(false) | bool - - role: bootstrap/update_nic_firmware - tags: firmware - when: - - update_nic_firmware | default(false) | bool - - update_nic_drivers | default(false) | bool - - role: bootstrap/determine_dataplane_interfaces - environment: "{{ proxy_env | d({}) }}" - any_errors_fatal: true - -# install DPDK and apply SRIOV configuration -- hosts: kube_node - roles: - - role: cluster_defaults - - role: install_dpdk - tags: dpdk - when: install_dpdk | default(true) - - role: sriov_nic_init - when: - - iommu_enabled | default(true) - - install_dpdk | default(true) - - not sriov_network_operator_enabled | default(true) - - role: bootstrap/configure_qat - when: - - qat_devices is defined and (qat_devices|length>0) - - iommu_enabled | default(true) - - update_qat_drivers | default(false) - - role: bootstrap/configure_openssl - when: - - qat_devices is defined and (qat_devices|length>0) - - iommu_enabled | default(true) - - update_qat_drivers | default(false) - - openssl_install | default(false) - - role: sgx_configure_icx - tags: sgx - when: - - sgx_enabled | default(true) - - (ansible_os_family == "Debian" and ansible_distribution_version != '21.04') - or (ansible_os_family == "RedHat" and ansible_distribution_version != '8.4') - environment: "{{ proxy_env | d({}) }}" - any_errors_fatal: true diff --git a/playbooks/infra/prepare_vms.yml b/playbooks/infra/prepare_vms.yml new file mode 100644 index 00000000..90586173 --- /dev/null +++ b/playbooks/infra/prepare_vms.yml @@ -0,0 +1,31 @@ +## +## Copyright (c) 2020-2022 Intel Corporation. +## +## Licensed under the Apache License, Version 2.0 (the "License"); +## you may not use this file except in compliance with the License. +## You may obtain a copy of the License at +## +## http://www.apache.org/licenses/LICENSE-2.0 +## +## Unless required by applicable law or agreed to in writing, software +## distributed under the License is distributed on an "AS IS" BASIS, +## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +## See the License for the specific language governing permissions and +## limitations under the License. +## +--- +- hosts: vm_host + roles: + - { role: "vm/conf_libvirt" } + +- hosts: vm_host + gather_facts: false + roles: + - "vm/manage_imgs" + - "vm/manage_vms" + - "vm/prepare_cek" + +- hosts: k8s_cluster + tasks: + - name: Test updated inventory + ping: diff --git a/playbooks/infra/redeploy_cleanup.yml b/playbooks/infra/redeploy_cleanup.yml index dd3e0a9c..cc941109 100644 --- a/playbooks/infra/redeploy_cleanup.yml +++ b/playbooks/infra/redeploy_cleanup.yml @@ -1,5 +1,5 @@ ## -## Copyright (c) 2020-2021 Intel Corporation. +## Copyright (c) 2020-2022 Intel Corporation. ## ## Licensed under the Apache License, Version 2.0 (the "License"); ## you may not use this file except in compliance with the License. diff --git a/playbooks/infra/regional_dc.yml b/playbooks/infra/regional_dc.yml deleted file mode 100644 index 89313da1..00000000 --- a/playbooks/infra/regional_dc.yml +++ /dev/null @@ -1,98 +0,0 @@ -## -## Copyright (c) 2020-2021 Intel Corporation. -## -## Licensed under the Apache License, Version 2.0 (the "License"); -## you may not use this file except in compliance with the License. -## You may obtain a copy of the License at -## -## http://www.apache.org/licenses/LICENSE-2.0 -## -## Unless required by applicable law or agreed to in writing, software -## distributed under the License is distributed on an "AS IS" BASIS, -## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -## See the License for the specific language governing permissions and -## limitations under the License. -## ---- -- hosts: 127.0.0.1 - connection: local - tasks: [] - roles: - - role: bootstrap/ansible_host - environment: "{{ proxy_env | d({}) }}" - any_errors_fatal: true - -# apply common cluster node configuration -- hosts: k8s_cluster - tasks: [] - handlers: - - name: reboot server - reboot: { reboot_timeout: 1200 } - roles: - - role: cluster_defaults - - role: bootstrap/configure_proxy - - role: bootstrap/install_packages - - role: bootstrap/update_grub - when: update_kernel | default(false) - - role: bootstrap/configure_security - - role: bootstrap/apply_kubernetes_reqs - - role: bootstrap/configure_dns - when: ansible_distribution == "Ubuntu" and dns_disable_stub_listener | default(true) | bool - environment: "{{ proxy_env | d({}) }}" - any_errors_fatal: true - - -# apply worker node kernel configuration -- hosts: kube_node - tasks: [] - handlers: - - name: reboot server - reboot: { reboot_timeout: 1200 } - roles: - - role: cluster_defaults - - role: bootstrap/configure_hugepages - when: hugepages_enabled | default(false) - - role: bootstrap/configure_cpu_isolation - when: isolcpus_enabled | default(false) - - role: bootstrap/configure_intel_pstate - when: intel_pstate_enabled | default(false) | bool - - role: bootstrap/set_sriov_kernel_flags - when: iommu_enabled | default(false) - - role: bootstrap/set_rdt_kernel_flags - when: telegraf_enabled | default(true) | bool - - role: bootstrap/install_gpu_kernel - when: gpu_dp_enabled | default (true) - - role: bootstrap/update_grub - environment: "{{ proxy_env | d({}) }}" - any_errors_fatal: true - -# install worker node network software -- hosts: kube_node - tasks: [] - roles: - - role: cluster_defaults - - role: bootstrap/update_nic_drivers - when: update_nic_drivers | default(false) | bool - - role: bootstrap/update_nic_firmware - tags: firmware - when: - - update_nic_firmware | default(false) | bool - - update_nic_drivers | default(false) | bool - - role: bootstrap/determine_dataplane_interfaces - environment: "{{ proxy_env | d({}) }}" - any_errors_fatal: true - -# install DPDK and apply SRIOV configuration -- hosts: kube_node - roles: - - role: cluster_defaults - - role: install_dpdk - when: install_dpdk | default(true) - tags: dpdk - - role: sriov_nic_init - when: - - iommu_enabled | default(false) - - install_dpdk | default(false) - - not sriov_network_operator_enabled | default(false) - environment: "{{ proxy_env | d({}) }}" - any_errors_fatal: true diff --git a/playbooks/infra/remote_fp.yml b/playbooks/infra/remote_fp.yml deleted file mode 100644 index 9eae02af..00000000 --- a/playbooks/infra/remote_fp.yml +++ /dev/null @@ -1,140 +0,0 @@ -## -## Copyright (c) 2020-2021 Intel Corporation. -## -## Licensed under the Apache License, Version 2.0 (the "License"); -## you may not use this file except in compliance with the License. -## You may obtain a copy of the License at -## -## http://www.apache.org/licenses/LICENSE-2.0 -## -## Unless required by applicable law or agreed to in writing, software -## distributed under the License is distributed on an "AS IS" BASIS, -## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -## See the License for the specific language governing permissions and -## limitations under the License. -## ---- -- hosts: 127.0.0.1 - connection: local - tasks: [] - roles: - - role: bootstrap/ansible_host - environment: "{{ proxy_env | d({}) }}" - any_errors_fatal: true - -# apply common cluster node configuration -- hosts: k8s_cluster - tasks: [] - handlers: - - name: reboot server - reboot: { reboot_timeout: 1200 } - roles: - - role: cluster_defaults - - role: bootstrap/configure_proxy - - role: bootstrap/install_packages - - role: bootstrap/update_grub - when: update_kernel | default(false) - - role: bootstrap/configure_security - - role: bootstrap/apply_kubernetes_reqs - - role: bootstrap/configure_dns - when: ansible_distribution == "Ubuntu" and dns_disable_stub_listener | default(true) | bool - environment: "{{ proxy_env | d({}) }}" - any_errors_fatal: true - -# apply worker node kernel configuration -- hosts: kube_node - tasks: [] - handlers: - - name: reboot server - reboot: { reboot_timeout: 1200 } - roles: - - role: cluster_defaults - - role: bootstrap/configure_hugepages - when: hugepages_enabled | default(true) - - role: bootstrap/configure_cpu_isolation - when: isolcpus_enabled | default(true) - - role: bootstrap/configure_intel_pstate - when: intel_pstate_enabled | default(true) | bool - - role: bootstrap/set_sriov_kernel_flags - when: iommu_enabled | default(true) - - role: bootstrap/set_rdt_kernel_flags - when: telegraf_enabled | default(true) | bool - - role: bootstrap/configure_sst - tags: sst - when: - - (ansible_distribution == 'Ubuntu' and ansible_distribution_version >= '20.04') or - (ansible_distribution == "CentOS" and ansible_distribution_version >= '8.2') or - (ansible_distribution == "RedHat" and ansible_distribution_version >= '8.3') - - (sst_bf_configuration_enabled | default(true)) or - (sst_cp_configuration_enabled | default(false)) or - (sst_tf_configuration_enabled | default(false)) - - role: bootstrap/install_gpu_kernel - when: gpu_dp_enabled | default(false) - - role: bootstrap/update_grub - environment: "{{ proxy_env | d({}) }}" - any_errors_fatal: true - -# install worker node qat software -- hosts: kube_node - tasks: [] - handlers: - - name: reboot server - reboot: { reboot_timeout: 1200 } - roles: - - role: cluster_defaults - - role: bootstrap/install_qat_drivers_services - when: - - update_qat_drivers | default(false) - - qat_devices is defined and (qat_devices|length>0) - environment: "{{ proxy_env | d({}) }}" - any_errors_fatal: true - -# install worker node network software -- hosts: kube_node - tasks: [] - roles: - - role: cluster_defaults - - role: bootstrap/update_nic_drivers - when: update_nic_drivers | default(false) | bool - - role: bootstrap/update_nic_firmware - tags: firmware - when: - - update_nic_firmware | default(false) | bool - - update_nic_drivers | default(false) | bool - - role: bootstrap/determine_dataplane_interfaces - environment: "{{ proxy_env | d({}) }}" - any_errors_fatal: true - -# install DPDK and apply SRIOV configuration -- hosts: kube_node - roles: - - role: cluster_defaults - - role: install_dpdk - tags: dpdk - when: ovs_dpdk_enabled | default(false) or install_dpdk | default(true) - - role: install_ddp_pkgs - when: install_ddp_packages | default(true) | bool - - role: sriov_nic_init - when: - - iommu_enabled | default(true) - - install_dpdk | default(true) - - not sriov_network_operator_enabled | default(true) - - role: bootstrap/configure_qat - when: - - qat_devices is defined and (qat_devices|length>0) - - iommu_enabled | default(true) - - update_qat_drivers | default(false) - - role: bootstrap/configure_openssl - when: - - qat_devices is defined and (qat_devices|length>0) - - iommu_enabled | default(true) - - update_qat_drivers | default(false) - - openssl_install | default(false) - - role: sgx_configure_icx - tags: sgx - when: - - sgx_enabled | default(true) - - (ansible_os_family == "Debian" and ansible_distribution_version != '21.04') - or (ansible_os_family == "RedHat" and ansible_distribution_version != '8.4') - environment: "{{ proxy_env | d({}) }}" - any_errors_fatal: true diff --git a/playbooks/intel/access.yml b/playbooks/intel/access.yml deleted file mode 100644 index f8843cc7..00000000 --- a/playbooks/intel/access.yml +++ /dev/null @@ -1,91 +0,0 @@ -## -## Copyright (c) 2020-2021 Intel Corporation. -## -## Licensed under the Apache License, Version 2.0 (the "License"); -## you may not use this file except in compliance with the License. -## You may obtain a copy of the License at -## -## http://www.apache.org/licenses/LICENSE-2.0 -## -## Unless required by applicable law or agreed to in writing, software -## distributed under the License is distributed on an "AS IS" BASIS, -## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -## See the License for the specific language governing permissions and -## limitations under the License. -## ---- -- hosts: k8s_cluster - tasks: [] - roles: - - role: cluster_defaults - tags: defaults - - role: golang_install - tags: golang - - role: nfd_install - tags: nfd - when: nfd_enabled | default(true) - - role: sriov_dp_install - tags: sriov-net-dp - when: - - sriov_net_dp_enabled | default(false) - - not sriov_network_operator_enabled | default(true) - - role: sriov_network_operator_install - when: - - sriov_network_operator_enabled | default(true) - - not sriov_net_dp_enabled | default(false) - - not sriov_cni_enabled | default(false) - tags: sriov-network-operator - - role: qat_dp_install - tags: qat-dp - when: qat_dp_enabled | default(true) - - role: openssl_engine_install - tags: openssl_engine - when: openssl_engine_enabled | default(false) - - role: cmk_install - tags: cmk - when: cmk_enabled | default(true) - - role: kube_prometheus - when: prometheus_operator | default(true) - - role: collectd_install - tags: monitoring - when: - - collectd_enabled | default(false) - - not (telegraf_enabled | default(true)) - vars: - collectd_profile: access - - role: telegraf_install - when: - - telegraf_enabled | default(true) - - not (collectd_enabled | default(false)) - tags: monitoring - vars: - telegraf_profile: access - - role: istio_operator_install - tags: istio - when: - - istio_enabled | default(true) - environment: - - "{{ proxy_env | d({}) }}" - - REGISTRY_AUTH_FILE: "{{ registry_containerd }}" - any_errors_fatal: true - -- hosts: kube_node - tasks: [] - roles: - - role: cluster_defaults - tags: defaults - - role: sriov_cni_install - when: - - sriov_cni_enabled | default(false) - - not sriov_network_operator_enabled | default(true) - tags: sriov-cni - environment: "{{ proxy_env | d({}) }}" - any_errors_fatal: true - -- hosts: kube_control_plane - tasks: [] - roles: - - role: net_attach_defs_create - tags: net-attach-defs - environment: "{{ proxy_env | d({}) }}" - any_errors_fatal: true diff --git a/playbooks/intel/basic.yml b/playbooks/intel/basic.yml deleted file mode 100644 index eed11b5b..00000000 --- a/playbooks/intel/basic.yml +++ /dev/null @@ -1,78 +0,0 @@ -## -## Copyright (c) 2020-2021 Intel Corporation. -## -## Licensed under the Apache License, Version 2.0 (the "License"); -## you may not use this file except in compliance with the License. -## You may obtain a copy of the License at -## -## http://www.apache.org/licenses/LICENSE-2.0 -## -## Unless required by applicable law or agreed to in writing, software -## distributed under the License is distributed on an "AS IS" BASIS, -## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -## See the License for the specific language governing permissions and -## limitations under the License. -## ---- -- hosts: k8s_cluster - tasks: [] - roles: - - role: cluster_defaults - tags: defaults - - role: golang_install - tags: golang - - role: nfd_install - tags: nfd - when: nfd_enabled | default(true) - - role: sriov_dp_install - tags: sriov-net-dp - when: - - sriov_net_dp_enabled | default(false) - - not sriov_network_operator_enabled | default(false) - - role: sriov_network_operator_install - when: - - sriov_network_operator_enabled | default(false) - - not sriov_net_dp_enabled | default(false) - - not sriov_cni_enabled | default(false) - tags: sriov-network-operator - - role: kube_prometheus - when: prometheus_operator | default(true) - - role: collectd_install - tags: monitoring - when: - - collectd_enabled | default(false) - - not (telegraf_enabled | default(true)) - vars: - collectd_profile: basic - - role: telegraf_install - when: - - telegraf_enabled | default(true) - - not (collectd_enabled | default(false)) - tags: monitoring - vars: - telegraf_profile: basic - environment: - - "{{ proxy_env | d({}) }}" - - REGISTRY_AUTH_FILE: "{{ registry_containerd }}" - any_errors_fatal: true - -- hosts: kube_node - tasks: [] - roles: - - role: cluster_defaults - tags: defaults - - role: sriov_cni_install - when: - - sriov_cni_enabled | default(false) - - not sriov_network_operator_enabled | default(false) - tags: sriov-cni - environment: "{{ proxy_env | d({}) }}" - any_errors_fatal: true - -- hosts: kube_control_plane - tasks: [] - roles: - - role: net_attach_defs_create - tags: net-attach-defs - environment: "{{ proxy_env | d({}) }}" - any_errors_fatal: true diff --git a/playbooks/intel/full_nfv.yml b/playbooks/intel/full_nfv.yml deleted file mode 100644 index dd6b805e..00000000 --- a/playbooks/intel/full_nfv.yml +++ /dev/null @@ -1,118 +0,0 @@ -## -## Copyright (c) 2020-2021 Intel Corporation. -## -## Licensed under the Apache License, Version 2.0 (the "License"); -## you may not use this file except in compliance with the License. -## You may obtain a copy of the License at -## -## http://www.apache.org/licenses/LICENSE-2.0 -## -## Unless required by applicable law or agreed to in writing, software -## distributed under the License is distributed on an "AS IS" BASIS, -## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -## See the License for the specific language governing permissions and -## limitations under the License. -## ---- -- hosts: k8s_cluster - tasks: [] - roles: - - role: cluster_defaults - tags: defaults - - role: golang_install - tags: golang - - role: nfd_install - tags: nfd - when: nfd_enabled | default(true) - - role: sriov_dp_install - tags: sriov-net-dp - when: - - sriov_net_dp_enabled | default(false) - - not sriov_network_operator_enabled | default(true) - - role: sriov_network_operator_install - tags: sriov-network-operator - when: - - sriov_network_operator_enabled | default(true) - - not sriov_net_dp_enabled | default(false) - - not sriov_cni_enabled | default(false) - - role: intel_dp_operator - tags: dp-operator - when: sgx_dp_enabled | default(true) or gpu_dp_enabled | default(false) - - role: sgx_dp_install - tags: sgx-dp - when: - - sgx_dp_enabled | default(true) - - ansible_os_family == "Debian" or (ansible_os_family == "RedHat" and ansible_distribution_version >= '8.2') - - role: kmra_install - tags: kmra - when: - - kmra_enabled | default(true) - - (ansible_distribution == "Ubuntu" and ansible_distribution_version >= '20.04') - or (ansible_os_family == "RedHat" and ansible_distribution_version >= '8.2') - - role: qat_dp_install - tags: qat-dp - when: qat_dp_enabled | default(true) - - role: openssl_engine_install - tags: openssl_engine - when: openssl_engine_enabled | default(false) - - role: gpu_dp_install - tags: gpu-dp - when: gpu_dp_enabled | default(false) - - role: cmk_install - tags: cmk - when: cmk_enabled | default(true) - - role: kube_prometheus - when: prometheus_operator | default(true) - - role: tas_install - tags: tas-install - when: tas_enabled | default(true) - - role: collectd_install - tags: monitoring - when: - - collectd_enabled | default(false) - - not (telegraf_enabled | default(true)) - vars: - collectd_profile: full_nfv - - role: telegraf_install - when: - - telegraf_enabled | default(true) - - not (collectd_enabled | default(false)) - tags: monitoring - vars: - telegraf_profile: full_nfv - - role: istio_operator_install - tags: istio - when: - - istio_enabled | default(true) - - environment: - - "{{ proxy_env | d({}) }}" - - REGISTRY_AUTH_FILE: "{{ registry_containerd }}" - any_errors_fatal: true - -- hosts: kube_node - tasks: [] - roles: - - role: cluster_defaults - tags: defaults - - role: sriov_cni_install - when: - - sriov_cni_enabled | default(false) - - not sriov_network_operator_enabled | default(true) - tags: sriov-cni - - role: bond_cni_install - when: bond_cni_enabled | default(true) - tags: bond-cni - - role: userspace_cni_install - tags: userspace-cni - when: userspace_cni_enabled | default(true) - environment: "{{ proxy_env | d({}) }}" - any_errors_fatal: true - -- hosts: kube_control_plane - tasks: [] - roles: - - role: net_attach_defs_create - tags: net-attach-defs - environment: "{{ proxy_env | d({}) }}" - any_errors_fatal: true diff --git a/playbooks/intel/on_prem.yml b/playbooks/intel/on_prem.yml deleted file mode 100644 index d3f000f3..00000000 --- a/playbooks/intel/on_prem.yml +++ /dev/null @@ -1,114 +0,0 @@ -## -## Copyright (c) 2020-2021 Intel Corporation. -## -## Licensed under the Apache License, Version 2.0 (the "License"); -## you may not use this file except in compliance with the License. -## You may obtain a copy of the License at -## -## http://www.apache.org/licenses/LICENSE-2.0 -## -## Unless required by applicable law or agreed to in writing, software -## distributed under the License is distributed on an "AS IS" BASIS, -## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -## See the License for the specific language governing permissions and -## limitations under the License. -## ---- -- hosts: k8s_cluster - tasks: [] - roles: - - role: cluster_defaults - tags: defaults - - role: golang_install - tags: golang - - role: nfd_install - tags: nfd - when: nfd_enabled | default(true) - - role: sriov_dp_install - tags: sriov-net-dp - when: - - sriov_net_dp_enabled | default(false) - - not sriov_network_operator_enabled | default(true) - - role: sriov_network_operator_install - tags: sriov-network-operator - when: - - sriov_network_operator_enabled | default(true) - - not sriov_net_dp_enabled | default(false) - - not sriov_cni_enabled | default(false) - - role: intel_dp_operator - tags: dp-operator - when: sgx_dp_enabled | default(true) or gpu_dp_enabled | default(false) - - role: sgx_dp_install - tags: sgx-dp - when: - - sgx_dp_enabled | default(true) - - ansible_os_family == "Debian" or (ansible_os_family == "RedHat" and ansible_distribution_version >= '8.2') - - role: kmra_install - tags: kmra - when: - - kmra_enabled | default(true) - - (ansible_distribution == "Ubuntu" and ansible_distribution_version >= '20.04') - or (ansible_os_family == "RedHat" and ansible_distribution_version >= '8.2') - - role: qat_dp_install - tags: qat-dp - when: qat_dp_enabled | default(true) - - role: openssl_engine_install - tags: openssl_engine - when: openssl_engine_enabled | default(false) - - role: cmk_install - tags: cmk - when: cmk_enabled | default(true) - - role: gpu_dp_install - tags: gpu-dp - when: gpu_dp_enabled | default(false) - - role: kube_prometheus - when: prometheus_operator | default(true) - - role: tas_install - tags: tas-install - when: tas_enabled | default(true) - - role: collectd_install - tags: monitoring - when: - - collectd_enabled | default(false) - - not (telegraf_enabled | default(true)) - vars: - collectd_profile: on_prem - - role: telegraf_install - when: - - telegraf_enabled | default(true) - - not (collectd_enabled | default(false)) - tags: monitoring - vars: - telegraf_profile: on_prem - - role: istio_operator_install - tags: istio - when: - - istio_enabled | default(true) - environment: - - "{{ proxy_env | d({}) }}" - - REGISTRY_AUTH_FILE: "{{ registry_containerd }}" - any_errors_fatal: true - -- hosts: kube_node - tasks: [] - roles: - - role: cluster_defaults - tags: defaults - - role: sriov_cni_install - when: - - sriov_cni_enabled | default(false) - - not sriov_network_operator_enabled | default(true) - tags: sriov-cni - - role: bond_cni_install - when: bond_cni_enabled | default(false) - tags: bond-cni - environment: "{{ proxy_env | d({}) }}" - any_errors_fatal: true - -- hosts: kube_control_plane - tasks: [] - roles: - - role: net_attach_defs_create - tags: net-attach-defs - environment: "{{ proxy_env | d({}) }}" - any_errors_fatal: true diff --git a/playbooks/intel/regional_dc.yml b/playbooks/intel/regional_dc.yml deleted file mode 100644 index 211ee768..00000000 --- a/playbooks/intel/regional_dc.yml +++ /dev/null @@ -1,91 +0,0 @@ -## -## Copyright (c) 2020-2021 Intel Corporation. -## -## Licensed under the Apache License, Version 2.0 (the "License"); -## you may not use this file except in compliance with the License. -## You may obtain a copy of the License at -## -## http://www.apache.org/licenses/LICENSE-2.0 -## -## Unless required by applicable law or agreed to in writing, software -## distributed under the License is distributed on an "AS IS" BASIS, -## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -## See the License for the specific language governing permissions and -## limitations under the License. -## ---- -- hosts: k8s_cluster - tasks: [] - roles: - - role: cluster_defaults - tags: defaults - - role: golang_install - tags: golang - - role: nfd_install - tags: nfd - when: nfd_enabled | default(true) - - role: sriov_dp_install - tags: sriov-net-dp - when: - - sriov_net_dp_enabled | default(false) - - not sriov_network_operator_enabled | default(false) - - role: sriov_network_operator_install - tags: sriov-network-operator - when: - - sriov_network_operator_enabled | default(false) - - not sriov_net_dp_enabled | default(false) - - not sriov_cni_enabled | default(false) - - role: intel_dp_operator - tags: dp-operator - when: gpu_dp_enabled | default(true) - - role: gpu_dp_install - tags: gpu-dp - when: gpu_dp_enabled | default(true) - - role: kube_prometheus - when: prometheus_operator | default(true) - - role: tas_install - tags: tas-install - when: tas_enabled | default(true) - - role: collectd_install - tags: monitoring - when: - - collectd_enabled | default(false) - - not (telegraf_enabled | default(true)) - vars: - collectd_profile: regional_dc - - role: telegraf_install - when: - - telegraf_enabled | default(true) - - not (collectd_enabled | default(false)) - tags: monitoring - vars: - telegraf_profile: regional_dc - - role: istio_operator_install - tags: istio - when: - - istio_enabled | default(true) - environment: - - "{{ proxy_env | d({}) }}" - - REGISTRY_AUTH_FILE: "{{ registry_containerd }}" - any_errors_fatal: true - -- hosts: kube_node - tasks: [] - roles: - - role: cluster_defaults - tags: defaults - - role: sriov_cni_install - when: - - sriov_cni_enabled | default(false) - - not sriov_network_operator_enabled | default(false) - tags: sriov-cni - environment: "{{ proxy_env | d({}) }}" - any_errors_fatal: true - -- hosts: kube_control_plane - tasks: [] - roles: - - role: net_attach_defs_create - tags: net-attach-defs - environment: "{{ proxy_env | d({}) }}" - any_errors_fatal: true diff --git a/playbooks/intel/remote_fp.yml b/playbooks/intel/remote_fp.yml deleted file mode 100644 index 6a253406..00000000 --- a/playbooks/intel/remote_fp.yml +++ /dev/null @@ -1,117 +0,0 @@ -## -## Copyright (c) 2020-2021 Intel Corporation. -## -## Licensed under the Apache License, Version 2.0 (the "License"); -## you may not use this file except in compliance with the License. -## You may obtain a copy of the License at -## -## http://www.apache.org/licenses/LICENSE-2.0 -## -## Unless required by applicable law or agreed to in writing, software -## distributed under the License is distributed on an "AS IS" BASIS, -## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -## See the License for the specific language governing permissions and -## limitations under the License. -## ---- -- hosts: k8s_cluster - tasks: [] - roles: - - role: cluster_defaults - tags: defaults - - role: golang_install - tags: golang - - role: nfd_install - tags: nfd - when: nfd_enabled | default(true) - - role: sriov_dp_install - tags: sriov-net-dp - when: - - sriov_net_dp_enabled | default(false) - - not sriov_network_operator_enabled | default(true) - - role: sriov_network_operator_install - tags: sriov-network-operator - when: - - sriov_network_operator_enabled | default(true) - - not sriov_net_dp_enabled | default(false) - - not sriov_cni_enabled | default(false) - - role: intel_dp_operator - tags: dp-operator - when: sgx_dp_enabled | default(true) or gpu_dp_enabled | default(false) - - role: sgx_dp_install - tags: sgx-dp - when: - - sgx_dp_enabled | default(true) - - ansible_os_family == "Debian" or (ansible_os_family == "RedHat" and ansible_distribution_version >= '8.2') - - role: kmra_install - tags: kmra - when: - - kmra_enabled | default(true) - - (ansible_distribution == "Ubuntu" and ansible_distribution_version >= '20.04') - or (ansible_os_family == "RedHat" and ansible_distribution_version >= '8.2') - - role: qat_dp_install - tags: qat-dp - when: qat_dp_enabled | default(true) - - role: openssl_engine_install - tags: openssl_engine - when: openssl_engine_enabled | default(false) - - role: cmk_install - tags: cmk - when: cmk_enabled | default(true) - - role: gpu_dp_install - tags: gpu-dp - when: gpu_dp_enabled | default(false) - - role: kube_prometheus - when: prometheus_operator | default(true) - - role: tas_install - tags: tas_install - when: tas_enabled | default(true) - - role: collectd_install - tags: monitoring - when: - - collectd_enabled | default(false) - - not (telegraf_enabled | default(true)) - vars: - collectd_profile: remote_fp - - role: telegraf_install - when: - - telegraf_enabled | default(true) - - not (collectd_enabled | default(false)) - tags: monitoring - vars: - telegraf_profile: remote_fp - - role: istio_operator_install - tags: istio - when: - - istio_enabled | default(true) - environment: - - "{{ proxy_env | d({}) }}" - - REGISTRY_AUTH_FILE: "{{ registry_containerd }}" - any_errors_fatal: true - -- hosts: kube_node - tasks: [] - roles: - - role: cluster_defaults - tags: defaults - - role: sriov_cni_install - when: - - sriov_cni_enabled | default(false) - - not sriov_network_operator_enabled | default(true) - tags: sriov-cni - - role: bond_cni_install - when: bond_cni_enabled | default(false) - tags: bond-cni - - role: userspace_cni_install - when: userspace_cni_enabled | default(false) - tags: userspace-cni - environment: "{{ proxy_env | d({}) }}" - any_errors_fatal: true - -- hosts: kube_control_plane - tasks: [] - roles: - - role: net_attach_defs_create - tags: net-attach-defs - environment: "{{ proxy_env | d({}) }}" - any_errors_fatal: true diff --git a/playbooks/k8s/k8s.yml b/playbooks/k8s/k8s.yml index 3cd9b411..980e1f92 100644 --- a/playbooks/k8s/k8s.yml +++ b/playbooks/k8s/k8s.yml @@ -1,5 +1,5 @@ ## -## Copyright (c) 2020-2021 Intel Corporation. +## Copyright (c) 2020-2022 Intel Corporation. ## ## Licensed under the Apache License, Version 2.0 (the "License"); ## you may not use this file except in compliance with the License. @@ -14,14 +14,6 @@ ## limitations under the License. ## --- -- hosts: 127.0.0.1 - connection: local - tasks: [] - roles: - - { role: kubespray_install } - environment: "{{ proxy_env | d({}) }}" - any_errors_fatal: true - - hosts: k8s_cluster tasks: - name: regather selinux facts @@ -34,18 +26,18 @@ environment: "{{ proxy_env | d({}) }}" any_errors_fatal: true -- hosts: all +- hosts: k8s_cluster gather_facts: false tasks: - name: prepare additional kubespray facts set_fact: kubelet_node_custom_flags_prepare: >- - {%- if native_cpu_manager_enabled | default(false) and native_cpu_manager_reserved_cpus is defined -%} - --reserved-cpus={{ native_cpu_manager_reserved_cpus }} + {%- if native_cpu_manager_enabled | default(true) and native_cpu_manager_reserved_cpus is defined -%} + reservedSystemCPUs: "{{ native_cpu_manager_reserved_cpus }}" {%- endif -%} enable_admission_plugins_prepare: >- - [EventRateLimit,{% if always_pull_enabled %} AlwaysPullImages,{% endif %} NodeRestriction{% if psp_enabled %}, PodSecurityPolicy{% endif %}] - bmra_docker_version: >- + [EventRateLimit, DefaultStorageClass,{% if always_pull_enabled %} AlwaysPullImages,{% endif %} NodeRestriction{% if psp_enabled %}, PodSecurityPolicy{% endif %}] + cek_docker_version: >- {% if ansible_distribution_version >= '21.04' %}latest{% else %}19.03{%endif %} kube_config_dir: /etc/kubernetes - name: set kube_cert_dir @@ -57,23 +49,35 @@ flannel_backend_type: >- {% if ansible_distribution_version >= '21.04' %}host-gw{% else %}vxlan{%endif %} when: kube_network_plugin == "flannel" - - name: prepare calico CNI facts + - name: prepare calico CNI facts for vxlan backend set_fact: calico_network_backend: vxlan calico_vxlan_mode: 'CrossSubnet' calico_ipip_mode: 'Never' - when: kube_network_plugin == "calico" + when: + - kube_network_plugin == "calico" + - calico_backend == "vxlan" + - name: prepare calico CNI facts for bird backend + set_fact: + calico_ipip_mode: 'Always' + calico_endpoint_to_host_action: "ACCEPT" + calico_wireguard_enabled: "{{ wireguard_enabled | default(true) | bool }}" + epel_enabled: >- + {% if ansible_distribution == 'CentOS' %}true{% else %}false{% endif %} + when: + - kube_network_plugin == "calico" + - calico_backend == "bird" environment: "{{ proxy_env | d({}) }}" any_errors_fatal: true -- hosts: all +- hosts: k8s_cluster tasks: - name: add docker runtime vars set_fact: container_manager: docker docker_iptables_enabled: true docker_dns_servers_strict: false - docker_version: "{{ bmra_docker_version }}" + docker_version: "{{ cek_docker_version }}" when: container_runtime == "docker" - name: add containerd runtime vars set_fact: @@ -98,7 +102,9 @@ kubeadm_enabled: true helm_enabled: true multus_conf_file: /host/etc/cni/net.d/templates/00-multus.conf - nginx_image_tag: 1.21.1 + nginx_image_tag: 1.21.3 + calico_node_livenessprobe_timeout: 15 + calico_node_readinessprobe_timeout: 15 override_system_hostname: false kube_proxy_mode: iptables enable_nodelocaldns: false @@ -107,7 +113,7 @@ system_cpu_reserved: "{{ native_cpu_manager_system_reserved_cpus | default('1000m') }}" kube_cpu_reserved: "{{ native_cpu_manager_kube_reserved_cpus | default('1000m') }}" kubelet_custom_flags: "--cpu-cfs-quota=false" - kubelet_node_custom_flags: "{{ kubelet_node_custom_flags_prepare | from_yaml }}" + kubelet_node_config_extra_args: "{{ kubelet_node_custom_flags_prepare | from_yaml }}" kube_api_anonymous_auth: true kube_feature_gates: - CPUManager=true # feature gate can be enabled by default, default policy is none in Kubernetes @@ -116,10 +122,8 @@ # Kubernetes cluster hardening kubernetes_audit: true audit_log_maxbackups: 10 - kube_controller_manager_bind_address: 127.0.0.1 kube_scheduler_bind_address: 127.0.0.1 kube_proxy_healthz_bind_address: 127.0.0.1 - kube_proxy_metrics_bind_address: 127.0.0.1 kube_read_only_port: 0 kube_override_hostname: "" kube_kubeadm_apiserver_extra_args: @@ -134,7 +138,7 @@ service-account-private-key-file: "{{ kube_cert_dir }}/sa.key" kubelet_config_extra_args: protectKernelDefaults: true - cpuManagerPolicy: "{% if native_cpu_manager_enabled | default(false) %}static{% else %}none{% endif %}" + cpuManagerPolicy: "{% if native_cpu_manager_enabled | default(true) %}static{% else %}none{% endif %}" topologyManagerPolicy: "{{ topology_manager_policy | default('none') }}" eventRecordQPS: 0 kube_apiserver_request_timeout: 60s @@ -153,6 +157,7 @@ - TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 etcd_extra_vars: ETCD_CIPHER_SUITES: "TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384" + any_errors_fatal: true - hosts: k8s_cluster tasks: @@ -176,25 +181,53 @@ when: - kube_network_plugin == "flannel" - flannel_endpoint.stat.exists + + # environment was added here to disable proxy for this health check + # standard no_proxy was not evaluated properly here - name: wait for kube-apiserver to be up uri: url: "https://127.0.0.1:6443/healthz" validate_certs: no + environment: + http_proxy: '' + https_proxy: '' + HTTP_PROXY: '' + HTTPS_PROXY: '' register: results until: results.status == 200 retries: 30 delay: 5 - - name: change Liveness & Readiness probes timeout in calico/node daemonset - # noqa 305 - shell is used intentionally here - shell: >- - set -o pipefail && kubectl get ds calico-node -n kube-system -o yaml | - sed "s/timeoutSeconds: 1/timeoutSeconds: 10/g" | - kubectl replace -f - && sleep 40 - args: - executable: /bin/bash + - name: allow traffic on wireguard interface + block: + - name: allow traffic on wireguard interface on Ubuntu + ufw: + rule: allow + interface: wireguard.cali + direction: in + state: reloaded + when: ansible_os_family == "Debian" + + - name: allow traffic on wireguard interface on RHEL/CentOS + firewalld: + zone: trusted + interface: wireguard.cali + permanent: yes + state: enabled + when: ansible_os_family == "RedHat" when: - - inventory_hostname == groups['kube_control_plane'][0] + - wireguard_enabled | default(true) | bool + - kube_network_plugin == "calico" and calico_backend == "bird" + - firewall_enabled | default(false) | bool + + - name: patch default calico controller configuration to not expose metrics port + command: "/usr/local/bin/calicoctl patch kubeControllersConfiguration default --patch='{ \"spec\": { \"prometheusMetricsPort\": 0 }}'" + when: + - ansible_hostname == groups['kube_control_plane'][0] - kube_network_plugin == "calico" + - not calico_metrics_enabled + changed_when: true + + any_errors_fatal: true - hosts: etcd tasks: @@ -213,6 +246,7 @@ group: etcd mode: '0700' state: directory + any_errors_fatal: true - hosts: k8s_cluster roles: @@ -227,3 +261,4 @@ - name: run certificate generation for mTLS in kubelet import_playbook: kubelet-certificates.yml + when: kubernetes | default(true) diff --git a/playbooks/k8s/kubelet-certificates.yml b/playbooks/k8s/kubelet-certificates.yml index 5a476a34..2dba3135 100644 --- a/playbooks/k8s/kubelet-certificates.yml +++ b/playbooks/k8s/kubelet-certificates.yml @@ -1,5 +1,5 @@ ## -## Copyright (c) 2020-2021 Intel Corporation. +## Copyright (c) 2020-2022 Intel Corporation. ## ## Licensed under the Apache License, Version 2.0 (the "License"); ## you may not use this file except in compliance with the License. @@ -14,9 +14,9 @@ ## limitations under the License. ## --- - - hosts: localhost connection: local + become: no gather_facts: false tasks: - name: create temporary directory for storing certificate material @@ -26,12 +26,9 @@ become: no register: temp_cert_dir_localhost - - hosts: k8s_cluster environment: "{{ proxy_env | d({}) }}" any_errors_fatal: true - roles: - - golang_install tasks: - name: set temporary certificate location set_fact: @@ -58,6 +55,12 @@ node_inventory_name: "{{ inventory_hostname }}" when: inventory_hostname == groups['kube_control_plane'][0] + - name: get GOPATH + command: /usr/local/go/bin/go env GOPATH + register: gopath + changed_when: false + when: inventory_hostname == groups['kube_control_plane'][0] + - name: create certs from CSRs shell: cmd: "{{ gopath.stdout }}/bin/cfssl gencert -ca {{ kube_cert_dir }}/ca.crt \ @@ -113,6 +116,7 @@ - hosts: localhost connection: local + become: no gather_facts: false tasks: - name: remove directory with certs @@ -124,15 +128,15 @@ - hosts: kube_control_plane environment: "{{ proxy_env | d({}) }}" tasks: + - name: set kubelet certificate authority fact + set_fact: + kubelet_cert_authority: "- --kubelet-certificate-authority={{ kube_cert_dir }}/ca.crt" - name: insert kubelet-authority into api-server configuration lineinfile: path: "{{ kube_config_dir }}/manifests/kube-apiserver.yaml" insertbefore: "--tls-cipher-suites" - line: "{{ item }}" + line: "{{ kubelet_cert_authority|indent(4,true) }}" mode: 0600 - loop: - - "{{ '- --kubelet-certificate-authority={{ kube_cert_dir }}/ca.crt'|indent(4,true) }}" - - hosts: k8s_cluster gather_facts: false diff --git a/playbooks/k8s/patch_kubespray.yml b/playbooks/k8s/patch_kubespray.yml index f4d0a363..28d7dcd1 100644 --- a/playbooks/k8s/patch_kubespray.yml +++ b/playbooks/k8s/patch_kubespray.yml @@ -1,5 +1,5 @@ ## -## Copyright (c) 2020-2021 Intel Corporation. +## Copyright (c) 2020-2022 Intel Corporation. ## ## Licensed under the Apache License, Version 2.0 (the "License"); ## you may not use this file except in compliance with the License. @@ -14,7 +14,7 @@ ## limitations under the License. ## --- -- hosts: k8s_cluster +- hosts: k8s_cluster,vm_host tasks: - name: set target vars set_fact: diff --git a/playbooks/on_prem.yml b/playbooks/on_prem.yml deleted file mode 100644 index a7dfdf57..00000000 --- a/playbooks/on_prem.yml +++ /dev/null @@ -1,26 +0,0 @@ -## -## Copyright (c) 2020-2021 Intel Corporation. -## -## Licensed under the Apache License, Version 2.0 (the "License"); -## you may not use this file except in compliance with the License. -## You may obtain a copy of the License at -## -## http://www.apache.org/licenses/LICENSE-2.0 -## -## Unless required by applicable law or agreed to in writing, software -## distributed under the License is distributed on an "AS IS" BASIS, -## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -## See the License for the specific language governing permissions and -## limitations under the License. -## ---- -- name: preflight checks - import_playbook: preflight.yml -- name: configure target hosts OS layer - import_playbook: infra/on_prem.yml -- name: provision Kubernetes cluster using kubespray - import_playbook: k8s/k8s.yml - when: kubernetes | default(true) -- name: install Intel BMRA Container Experience Kit features - import_playbook: intel/on_prem.yml - when: kubernetes | default(true) diff --git a/playbooks/preflight.yml b/playbooks/preflight.yml index 4b250202..b38cefd0 100644 --- a/playbooks/preflight.yml +++ b/playbooks/preflight.yml @@ -1,5 +1,5 @@ ## -## Copyright (c) 2020-2021 Intel Corporation. +## Copyright (c) 2020-2022 Intel Corporation. ## ## Licensed under the Apache License, Version 2.0 (the "License"); ## you may not use this file except in compliance with the License. @@ -25,11 +25,9 @@ # - Check Python version (min) # - Check Group Vars (exist) # - Check Host Vars (exist for each worker node) -# - Check CMK Hosts (valid targets) # On All targets (k8s_cluster): # - Check Linux Distro # - Check Hostnames (match Inventory) -# - Check CMK Config (isolcpus defined) # - Check isolcpus Total (not more than actual) # - Check isolcpus IDs (valid on system) # - Check isolcpus OS Reserved (not 0,1,etc) @@ -47,39 +45,41 @@ # - Warn BIOS VT-d (should be enabled) # - Warn BIOS Hyper-Threading (should be enabled) # - Warn collectd (kernel update needed on old OSs) -# - Check OVS DPDK Version Compatability (for OVS support) +# - Check OVS DPDK Version Compatibility (for OVS support) +# - Check MinIO requirements # additional vars required: -# bmra_supported_ansible: # must be version -# bmra_supported_python: # min version -# bmra_supported_distros: [] # list -# bmra_supported_distros_versions: [] # list +# cek_supported_ansible: # must be version +# cek_supported_python: # min version +# cek_supported_distros: [] # list +# cek_supported_distros_versions: [] # list ################################## # Prerequisites for Ansible Host # ################################## - hosts: 127.0.0.1 + any_errors_fatal: true connection: local vars: - bmra_supported_ansible: 2.9.20 - bmra_supported_python: 2.7 + cek_supported_ansible: 2.10.15 + cek_supported_python: 3 tasks: - debug: msg="Ansible version is {{ ansible_version.string }}" - name: Check Ansible Version assert: - that: (ansible_version.full is version_compare(bmra_supported_ansible, '==')) - msg: "Ansible version must be {{ bmra_supported_ansible }}. Please update" + that: (ansible_version.full is version_compare(cek_supported_ansible, '==')) + msg: "Ansible version must be {{ cek_supported_ansible }}. Please update" - debug: msg="Python version is {{ ansible_python_version }}" - name: Check Python Version assert: - that: (ansible_python_version is version_compare(bmra_supported_python, '>=')) - msg: "Python version must be at least {{ bmra_supported_python }}. Please update" + that: (ansible_python_version is version_compare(cek_supported_python, '>=')) + msg: "Python version must be at least {{ cek_supported_python }}. Please update" - - name: Read Group Vars + - name: read Group Vars stat: path: "{{ inventory_dir }}/group_vars/" register: group_vars_details @@ -89,52 +89,47 @@ that: "group_vars_details.stat.exists and group_vars_details.stat.isdir" msg: "File group_vars/all.yml does NOT exist. Must be created per Guide" - - name: Read Host Vars + - name: read Host Vars stat: path: "{{ inventory_dir }}/host_vars/{{ item }}.yml" register: host_vars_details with_items: "{{ groups['kube_node'] }}" - - name: Check Host Vars + - name: check Host Vars assert: that: "item.stat.exists and item.stat.isreg" msg: "File host_vars/{{ item.item }}.yml does NOT exist. Must be created per Guide" with_items: "{{ host_vars_details.results }}" - - debug: - msg: - - cmk_enabled = {{ cmk_enabled }} (group_vars/all.yml) - - cmk_hosts_list = {{ cmk_hosts_list | default('') }} (group_vars/all.yml) - - all targets = {{ groups['all'] }} (inventory.ini) - when: cmk_enabled is defined # CMK expected true for all profiles except basic - - - name: Check Intel CMK Hosts - assert: - that: "item in groups['all']" - msg: "Hostname '{{ item }}' is NOT a valid target from inventory. Please correct the cmk_hosts_list or disable the CMK feature in group vars" - with_items: "{{ (cmk_hosts_list.split(',') if (cmk_hosts_list is defined and cmk_hosts_list | length > 0) else []) }}" - when: cmk_enabled is defined and cmk_enabled - + - name: show profile name + debug: + msg: "Profile name is: {{ profile_name | default('not generated') }}" ############################################## # Prerequisites for Control and Worker Nodes # ############################################## -- hosts: k8s_cluster +- hosts: k8s_cluster,vm_host + any_errors_fatal: true vars: - bmra_supported_distros: [CentOS, RedHat, Ubuntu] - bmra_supported_distros_versions: ['7.6', '7.8', '7.9', '8.2', '8.3', '8.4', '18.04', '20.04', '21.04'] + cek_supported_distros: [RedHat, Ubuntu] + cek_supported_distros_versions: ['8.5', '20.04', '21.04', '21.10'] isolcpus_ranges: [] isolcpus_discretes: [] tasks: + - name: end play for VM host + meta: end_host + when: + - "'vm_host' in group_names" + - on_vms is defined and on_vms - debug: msg="Linux distribution on target is {{ ansible_distribution }} {{ ansible_distribution_version }} {{ ansible_distribution_release }}" - name: Check Linux Distro and Version assert: - that: "ansible_distribution in bmra_supported_distros and ansible_distribution_version in bmra_supported_distros_versions" + that: "ansible_distribution in cek_supported_distros and ansible_distribution_version in cek_supported_distros_versions" msg: - Linux distribution {{ ansible_distribution }} {{ ansible_distribution_version }} on target '{{ inventory_hostname }}' is NOT supported - - Must be one of {{ bmra_supported_distros }} and version {{ bmra_supported_distros_versions }} + - Must be one of {{ cek_supported_distros }} and version {{ cek_supported_distros_versions }} # - name: Check Linux Across Cluster # TODO ?? Linux OS must be the same on all targets (no mix-n-match) @@ -164,7 +159,7 @@ - "mode: {{ ansible_selinux.mode | default('') }}" - "config_mode: {{ ansible_selinux.config_mode | default('') }}" - - name: check selinux condition possibly causing system boot failure + - name: check SELinux condition possibly causing system boot failure debug: msg: - "Current SELinux setup might cause the system possibly will not boot up on next reboot." @@ -185,14 +180,29 @@ when: - ansible_os_family == "RedHat" -# STORY: "cmk requires isolcpus to be configured" + - name: check kubernetes and container runtime variables + assert: + that: + - (not kubernetes and container_runtime_only_deployment) or + (not kubernetes and not container_runtime_only_deployment) + fail_msg: >- + "For container runtime only deployment set container_runtime_only_deployment as "true" and kubernetes as "false". + Set both variables as "false" to perform only host configuration." + when: + - not kubernetes or container_runtime_only_deployment + + - name: assert that Multus is enabled in the config + assert: + that: + - "kube_network_plugin_multus" + fail_msg: "SRIOV and the Userspace CNI plugin require Multus for a fully functional cluster deployment" + when: sriov_net_dp_enabled is defined and sriov_net_dp_enabled or + sriov_cni_enabled is defined and sriov_cni_enabled or + userspace_cni_enabled is defined and userspace_cni_enabled + - block: - debug: msg: - - cmk_enabled = {{ cmk_enabled }} (group_vars/all.yml) - - cmk_hosts_list = {{ cmk_hosts_list | default('') }} (group_vars/all.yml) - - cmk_shared_num_cores = {{ cmk_shared_num_cores }} (group_vars/all.yml) - - cmk_exclusive_num_cores = {{ cmk_exclusive_num_cores }} (group_vars/all.yml) - isolcpus_enabled = {{ isolcpus_enabled }} (host_vars) - isolcpus = {{ isolcpus }} (host_vars) - ansible_processor_count = {{ ansible_processor_count }} @@ -202,20 +212,20 @@ - CPUs Reserved for OS = 0...{{ ansible_processor_count - 1 }} # - CPUs Reserved for OS = {{ lookup('sequence','0-{{ ansible_processor_count - 1 }}').split(',') }} # [E207] Nested jinja pattern - - name: Check Intel CMK Config + - name: check CPUs isolation assert: - that: ({{ cmk_enabled }} and {{ isolcpus_enabled }} and "{{ isolcpus }}" | length > 0) + that: ( "{{ isolcpus }}" | length > 0 ) msg: - - Incorrect configuration pertaining Intel CMK. Conflicting or improper values detected - - When Intel CMK is enabled, CPUs isolation ('isolcpus') must be set according to the example file for host_vars. Please correct the configuration + - Incorrect configuration. Conflicting or improper values detected + - CPUs isolation ('isolcpus') must be set according to the example file for host_vars. Please correct the configuration - - name: Split isolcpus Groups + - name: split isolcpus Groups set_fact: isolcpus_groups: "{{ isolcpus.split(',') }}" - debug: msg="isolcpus_groups = {{ isolcpus_groups }}" - - name: Filter isolcpus Ranges + - name: filter isolcpus Ranges set_fact: isolcpus_ranges: "{{ isolcpus_ranges + [item] }}" with_items: "{{ isolcpus_groups }}" @@ -223,7 +233,7 @@ - debug: msg="isolcpus_ranges = {{ isolcpus_ranges }}" - - name: Filter isolcpus Discretes + - name: filter isolcpus Discretes set_fact: isolcpus_discretes: "{{ isolcpus_discretes + [item] }}" with_items: "{{ isolcpus_groups }}" @@ -231,61 +241,60 @@ - debug: msg="isolcpus_discretes = {{ isolcpus_discretes }}" - - name: Build isolcpus List + - name: build isolcpus List set_fact: isolcpus_list: "{{ isolcpus_list | default([]) | union(isolcpus_discretes) | union([item]) }}" with_sequence: "{{ isolcpus_ranges }}" - debug: msg="isolcpus_list = {{ isolcpus_list }}" - - name: Check isolcpus Total + - name: check isolcpus Total assert: that: "{{ isolcpus_list | length }} <= ansible_processor_vcpus" msg: - Incorrect configuration pertaining isolcpus. Conflicting or improper values detected - The number of isolcpus {{ isolcpus_list | length }}, exceeds total CPUs on target {{ ansible_processor_vcpus }}. Please correct the configuration - when: isolcpus is defined - - name: Check isolcpus IDs + - name: check isolcpus IDs assert: that: "item | int <= ansible_processor_vcpus" msg: - Incorrect configuration pertaining isolcpus. Conflicting or improper values detected - The CPU ID {{ item }} set for isolcpus is NOT actually present on target. Please correct the configuration with_items: "{{ isolcpus_list }}" - when: isolcpus is defined - - name: Check isolcpus OS Reserved + - name: check isolcpus OS Reserved assert: that: "item not in isolcpus_list" msg: - Incorrect configuration pertaining isolcpus. Conflicting or improper values detected - The CPU ID 0...{{ ansible_processor_count - 1 }} should NOT be set for isolcpus. Please correct the configuration with_items: "{{ lookup('sequence','0-{{ ansible_processor_count - 1 }}').split(',') }}" - when: isolcpus is defined - -#TODO relationship between cmk shared/exclusive cores and isolcpus when: - - cmk_enabled is defined - - (not cmk_hosts_list is defined) or (inventory_hostname in cmk_hosts_list) #CMK expected true for all profiles except basic - + - isolcpus_enabled is defined and isolcpus_enabled | bool #################################### # Prerequisites for Worker Node(s) # #################################### -- hosts: kube_node +- hosts: kube_node,vm_host + any_errors_fatal: true vars: phy_nics_pciids: [] tasks: + - name: end play for VM host + meta: end_host + when: + - "'vm_host' in group_names" + - on_vms is defined and on_vms # STORY: "nic bus info specified is present on system" - debug: msg: "Dataplane (DP) interface(s) defined in host_vars = {{ dataplane_interfaces }}" when: dataplane_interfaces is defined - - name: Check DP Interfaces + - name: check DP Interfaces assert: that: "dataplane_interfaces != []" msg: "Dataplane (DP) interface(s) on target '{{ ansible_hostname }}' must be set in host_vars. Please correct the configuration" @@ -299,7 +308,7 @@ - debug: msg: "Network interfaces present on target '{{ ansible_hostname }}' = {{ ansible_interfaces }}" - - name: Read Physical NICs PCIIDs + - name: read Physical NICs PCIIDs set_fact: phy_nics_pciids: "{{ phy_nics_pciids + [ ansible_facts[item]['pciid'] ] }}" with_items: "{{ ansible_interfaces }}" @@ -307,64 +316,115 @@ - debug: msg="PCI Slots for the NICs on target '{{ ansible_hostname }}' = {{ phy_nics_pciids }}" - - name: Check DP Interfaces Names - assert: - that: ("{{ item.name }}" in {{ ansible_interfaces }}) - msg: "Dataplane interface '{{ item.name }}' defined in host_vars does NOT exist on target. Please correct the configuration" - with_items: "{{ dataplane_interfaces }}" - when: dataplane_interfaces is defined and dataplane_interfaces != [] - ignore_errors: True - - - name: Check DP Interfaces Bus Info + - name: check DP Interfaces Bus Info assert: that: ("{{ item.bus_info }}" in "{{ phy_nics_pciids }}") msg: "Dataplane interface '{{ item.name }}' defined with PCI ID '{{ item.bus_info }}' does NOT exist on target. Please correct the configuration" with_items: "{{ dataplane_interfaces }}" when: dataplane_interfaces is defined and dataplane_interfaces != [] - ignore_errors: True - - -# QAT Devices list is okay to be left empty (default), but if was defined, device(s) must exist on target - - debug: - msg: "QAT device(s) defined in host_vars = {{ qat_devices }}" - when: qat_devices is defined - - - name: Read QAT PCIIDs - shell: lshw -businfo -numeric | grep -i quickassist - register: lshw_qat - ignore_errors: True - when: qat_devices is defined - - - debug: - msg: "QAT devices found on target = {{ lshw_qat.stdout }}" - when: qat_devices is defined - - name: Check QAT Devices' Bus Info - assert: - that: ("{{ item.qat_id }}" in """{{ lshw_qat.stdout }}""") - msg: "QAT device '{{ item.qat_dev }}' defined with PCI ID '{{ item.qat_id }}' does NOT exist on target. Please correct the configuration" - with_items: "{{ qat_devices }}" - when: qat_devices is defined and qat_devices != [] - ignore_errors: True - -# STORY: "qat_sriov_numvfs should not exceed max supported (16) per each dev_ID" - - debug: - msg: - - qat_sriov_numvfs for {{ item.qat_id }} = {{ item.qat_sriov_numvfs }} (host_vars) - - update_qat_drivers = {{ update_qat_drivers }} (host_vars) - with_items: "{{ qat_devices }}" - when: qat_devices is defined and qat_devices != [] # update_qat_drivers expected as 'true' for all profiles except basic - - - name: Check QAT SRIOV VFs - assert: - that: ({{ item.qat_sriov_numvfs }} <= 16) - msg: - - Incorrect configuration pertaining QAT SRIOV. Conflicting or improper values detected - - When SRIOV VFs are set for QAT, max value is 16 for each ID (max 48 total per card). Please correct the configuration - with_items: "{{ qat_devices }}" + - name: check first DP interface driver and DDP package + block: + - name: Check first DP interface driver + assert: + that: "dataplane_interfaces[0].pf_driver == ansible_facts[item]['module']" + msg: >- + "Dataplane interface '{{ dataplane_interfaces[0].name }}' 'pf_driver' is set to '{{ dataplane_interfaces[0].pf_driver }}'. + Please, set it to '{{ ansible_facts[item]['module'] }}'." + loop: "{{ ansible_interfaces }}" + when: + - ansible_facts[item]['pciid'] is defined and ansible_facts[item]['type'] == "ether" + - dataplane_interfaces[0].bus_info in ansible_facts[item]['pciid'] + + - name: check if selected DDP package corresponds PF driver ice + assert: + that: "{{ dataplane_interfaces[0].ddp_profile is regex('^ice_comms*') }}" + msg: "ddp_profile '{{ dataplane_interfaces[0].ddp_profile }}' doesn't correspond pf_driver '{{ dataplane_interfaces[0].pf_driver }}'" + when: + - dataplane_interfaces[0].ddp_profile is defined + - dataplane_interfaces[0].ddp_profile | length > 0 + - dataplane_interfaces[0].pf_driver == "ice" + - install_ddp_packages is defined and install_ddp_packages + + - name: check if selected DDP package corresponds PF driver i40e + assert: + that: "{{ dataplane_interfaces[0].ddp_profile in ddp_profiles_allowed }}" + msg: "ddp_profile '{{ dataplane_interfaces[0].ddp_profile }}' doesn't correspond pf_driver '{{ dataplane_interfaces[0].pf_driver }}'" + when: + - dataplane_interfaces[0].ddp_profile is defined + - dataplane_interfaces[0].ddp_profile | length > 0 + - dataplane_interfaces[0].pf_driver == "i40e" + - install_ddp_packages is defined and install_ddp_packages + vars: + ddp_profiles_allowed: + - ecpri.pkg + - esp-ah.pkg + - ppp-oe-ol2tpv2.pkgo + - mplsogreudp.pkg + - gtp.pkgo + when: + - dataplane_interfaces is defined and dataplane_interfaces | length > 0 + + - name: check QAT Devices list is configured properly + block: + - debug: + msg: "QAT device(s) defined in host_vars = {{ qat_devices }}" + + - name: read QAT PCIIDs + shell: "set -o pipefail && lshw -businfo -numeric | egrep -i '{{ qat_supported_pf_dev_ids | join('|') }}'" + args: + executable: /bin/bash + register: lshw_qat_host + when: + - on_vms is not defined or not on_vms + + - name: read QAT PCIIDs on VMs + shell: "set -o pipefail && lshw -businfo -numeric | egrep -i '{{ (qat_supported_vf_dev_ids + qat_supported_pf_dev_ids) | join('|') }}'" + args: + executable: /bin/bash + register: lshw_qat_vms + when: + - on_vms is defined and on_vms + + - name: store lshw_qat + set_fact: + lshw_qat: "{% if (on_vms is defined and on_vms) %}{{ lshw_qat_vms }}{% else %}{{ lshw_qat_host }}{% endif %}" + + - debug: + msg: "QAT devices found on target = {{ lshw_qat.stdout }}" + + - name: check QAT Devices' Bus Info + assert: + that: ("{{ item.qat_id }}" in """{{ lshw_qat.stdout }}""") + msg: "QAT device defined with PCI ID '{{ item.qat_id }}' does NOT exist on target. Please correct the configuration" + with_items: "{{ qat_devices }}" + # STORY: "qat_sriov_numvfs should not exceed max supported (16) per each dev_ID" + - debug: + msg: + - qat_sriov_numvfs for {{ item.qat_id }} = {{ item.qat_sriov_numvfs }} (host_vars) + - update_qat_drivers = {{ update_qat_drivers }} (host_vars) + with_items: "{{ qat_devices }}" + + - name: check QAT SRIOV VFs + assert: + that: ({{ item.qat_sriov_numvfs }} <= 16) + msg: + - Incorrect configuration pertaining QAT SRIOV. Conflicting or improper values detected + - When SRIOV VFs are set for QAT, max value is 16 for each ID (max 48 total per card). Please correct the configuration + with_items: "{{ qat_devices }}" when: - update_qat_drivers is defined and update_qat_drivers - qat_devices is defined and qat_devices != [] + +# GPU DP Fractional Resources must be passed in correct format + - name: check if the GPU max memory is passed in correct format + assert: + that: + - "{{ gpu_dp_max_memory.split(' ') | length == 2 }}" # value is passed in format + - "{{ gpu_dp_max_memory.split(' ')[1] in ['GB', 'MB', 'KB', 'Kb'] }}" # supported units + fail_msg: "Please provide variable value in correct format. Use format, e.g. '8 GB'" + when: gpu_dp_max_memory is defined + # OpenSSL & OpenSSL*Engine must only be configured / installed when update_qat_drivers is set to 'true' and qat_devices is defined in host vars - name: check OpenSSL and OpenSSL*Engine requirements assert: @@ -383,16 +443,15 @@ success_msg: "KMRA requirements verified" when: kmra_enabled is defined and kmra_enabled - - name: check SGX configuration + - name: check Intel SGX DP configuration assert: that: - - sgx_enabled - fail_msg: "SGX drivers installation requires sgx_enabled set to 'true'" - success_msg: "SGX configuration verified" + - configure_sgx is defined + - configure_sgx + fail_msg: + - "Please set configure_sgx to true in host vars to allow SGX configuration on the system" when: - sgx_dp_enabled is defined and sgx_dp_enabled - - (ansible_distribution == 'Ubuntu' and ansible_distribution_version != '21.04') - or (ansible_os_family == 'RedHat' and ansible_distribution_version != '8.4') - name: check NFD configuration assert: @@ -434,6 +493,27 @@ when: - kmra_enabled is defined and kmra_enabled +# Validate Hugepage settings + - name: validate hugepage settings + block: + - name: validate default hugepage size setting + assert: + that: default_hugepage_size == "2M" or default_hugepage_size == "1G" + fail_msg: "Valid sizes for hugepage are: [2M, 1G], given: {{ default_hugepage_size }}." + + - name: validate that at least one 1G hugepage is requested + assert: + that: number_of_hugepages_1G > 0 + fail_msg: "number_of_hugepages_1G should not be equal to 0. Please update host vars settings." + when: default_hugepage_size == "1G" + + - name: validate that at least one 2M hugepage is requested + assert: + that: number_of_hugepages_2M > 0 + fail_msg: "number_of_hugepages_2M should not be equal to 0. Please update host vars settings." + when: default_hugepage_size == "2M" + when: hugepages_enabled | default(false) | bool + # STORY: "vpp/ovsdpdk require hugepage enabled and configured" - debug: msg: @@ -450,49 +530,65 @@ - userspace_cni_enabled = {{ userspace_cni_enabled }} (host_vars) - hugepages_enabled = {{ hugepages_enabled }} (host_vars) - default_hugepage_size = {{ default_hugepage_size }} (host_vars) - - number_of_hugepages = {{ number_of_hugepages }} (host_vars) - when: vpp_enabled is defined #host_vars + - number_of_hugepages_1G = {{ number_of_hugepages_1G }} (host_vars) + - number_of_hugepages_2M = {{ number_of_hugepages_2M }} (host_vars) + when: vpp_enabled is defined and vpp_enabled # host_vars - - name: Check OVS DPDK Dependencies + - name: check OVS DPDK Dependencies assert: that: >- ({{ ovs_dpdk_enabled }} and not {{ vpp_enabled }} and {{ hugepages_enabled }} and - "{{ default_hugepage_size }}" == "1G" and {{ number_of_hugepages }} >= 0) + "{{ default_hugepage_size }}" == "1G" and {{ number_of_hugepages_1G }} >= 0) or {{ vpp_enabled }} msg: - Incorrect configuration pertaining OVS DPDK. Conflicting or improper values detected - - When OVS DPDK is enabled, VPP must be disabled and Hugepages must be set to 1G according to host_vars example. Please correct the configuration + - When OVS DPDK is enabled, VPP must be disabled and default_hugepage_size must be set to 1G according to host_vars example. + - Also check these conditions in group_vars, example_net_attach_defs['userspace_ovs_dpdk']=true, example_net_attach_defs['userspace_vpp']=false. + - Please correct the configuration when: ovs_dpdk_enabled is defined and ovs_dpdk_enabled - - name: Check VPP Dependencies + - name: check VPP Dependencies assert: that: >- ({{ vpp_enabled }} and not {{ ovs_dpdk_enabled }} and {{ hugepages_enabled }} and - "{{ default_hugepage_size }}" == "2M" and {{ number_of_hugepages }} >= 0) + "{{ default_hugepage_size }}" == "2M" and {{ number_of_hugepages_2M }} >= 0) or {{ ovs_dpdk_enabled }} msg: - Incorrect configuration pertaining VPP. Conflicting or improper values detected - - When VPP is enabled, OVS DPDK must be disabled and Hugepages must be set to 2M according to host_vars example. Please correct the configuration + - When VPP is enabled, OVS DPDK must be disabled and default_hugepage_size must be set to 2M according to host_vars example. + - Also check these conditions in group_vars, example_net_attach_defs['userspace_ovs_dpdk']=false, example_net_attach_defs['userspace_vpp']=true. + - Please correct the configuration. when: vpp_enabled is defined and vpp_enabled # STORY: "cnis require net-attach-defs to be enabled" - - name: Check CNI Config + - name: check CNI Config assert: that: >- ({{ userspace_cni_enabled }} and {{ ovs_dpdk_enabled }} and {{ example_net_attach_defs['userspace_ovs_dpdk'] }} and not {{ vpp_enabled }} and not {{ example_net_attach_defs['userspace_vpp'] }} and {{ hugepages_enabled }} and - "{{ default_hugepage_size }}" == "1G" and {{ number_of_hugepages }} >= 0) + "{{ default_hugepage_size }}" == "1G" and {{ number_of_hugepages_1G }} >= 0) or ({{ userspace_cni_enabled }} and not {{ ovs_dpdk_enabled }} and not {{ example_net_attach_defs['userspace_ovs_dpdk'] }} and {{ vpp_enabled }} and {{ example_net_attach_defs['userspace_vpp'] }} and {{ hugepages_enabled }} and - "{{ default_hugepage_size }}" == "2M" and {{ number_of_hugepages }} >= 0) + "{{ default_hugepage_size }}" == "2M" and {{ number_of_hugepages_2M }} >= 0) msg: - Incorrect configuration pertaining CNI. Conflicting or improper values detected. - - When CNI is enabled, either OVS DPDK either VPP must be enabled and Hugepages must be according to example files. Please correct the configuration + - When Userspace CNI is enabled, either OVS DPDK or VPP must be enabled, default_hugepage_size must also set under these options. + - 1. If VPP is enabled, ovs_dpdk_enabled=false, default_hugepage_size = 2M in host_vars; and + - example_net_attach_defs['userspace_ovs_dpdk']=false, example_net_attach_defs['userspace_vpp']=true in group_vars. + - 2. If OVS DPDK is enabled, default_hugepage_size = 1G in host_vars; and + - vpp_enabled=false, example_net_attach_defs['userspace_ovs_dpdk']=true, example_net_attach_defs['userspace_vpp']=false in group_vars. + - Please correct the configuration. when: userspace_cni_enabled is defined and userspace_cni_enabled # STORY: "If SST enabled, confirm minimum kernel or kernel_update specified" + - name: check platform before SST-PP verification + command: "cat /sys/devices/cpu/caps/pmu_name" + when: sst_pp_configuration_enabled is defined and sst_pp_configuration_enabled + changed_when: true + register: verify_platform_for_sst_pp + - name: check Intel(R) SST-PP (feature perf-profile) requirements assert: that: @@ -505,50 +601,47 @@ - "sst_tf_configuration_enable & sst_pp_configuration_enabled all should be disabled in host_vars" - "Secondly, make sure turbo_boost_enabled in host_vars is set to 'true', as required for turbo-freq configuration" success_msg: "Intel(R) SST-PP (feature perf-profile), verification completed" - when: sst_pp_configuration_enabled is defined and sst_pp_configuration_enabled - - - name: Check SST # see Jira NPF-1545 - assert: - that: (not sst_bf_configuration_enabled) - msg: "SST-BF is NOT supported on {{ ansible_distribution }} {{ ansible_distribution_version }}. Please use a different OS or disable this feature" when: - - sst_bf_configuration_enabled is defined - - (ansible_distribution == "RedHat" and ansible_distribution_version == '8.2') or ansible_distribution_version in ['7.6', '7.8', '7.9', '18.04'] - ignore_errors: True - -# STORY: Intel VT-d should be enabled in BIOS - - name: Check Intel VT-d - shell: dmesg | grep DMAR | grep remapping - register: dmesg_dmar_remap - ignore_errors: True - changed_when: False + - sst_pp_configuration_enabled is defined and sst_pp_configuration_enabled + - "'skylake' not in verify_platform_for_sst_pp.stdout" - - debug: msg="dmesg >> {{ dmesg_dmar_remap.stdout }}" - - - name: Warn about Intel VT-d + - name: Intel(R) SST-PP (feature perf-profile) not available fail: - msg: "Warning: Intel VT-d appears DISABLED on target. Please check BIOS under 'Advanced > Integrated IO Configuration' and Enable if necessary" - when: dmesg_dmar_remap.stdout|length == 0 - ignore_errors: True + msg: + - "SST-PP is not supported on {{ verify_platform_for_sst_pp.stdout }} platform" + - "Make sure sst_pp_configuration_enabled is set to false in host vars" + when: + - sst_pp_configuration_enabled is defined and sst_pp_configuration_enabled + - "'skylake' in verify_platform_for_sst_pp.stdout" +# STORY: Intel VT-d should be enabled in BIOS + - name: check Intel VT-d on BMs + block: + - name: Check Intel VT-d + shell: "set -o pipefail && dmesg | grep DMAR | grep remapping" + args: + executable: /bin/bash + register: dmesg_dmar_remap + changed_when: False + + - debug: msg="dmesg >> {{ dmesg_dmar_remap.stdout }}" + + - name: warn about Intel VT-d + fail: + msg: "Warning: Intel VT-d appears DISABLED on target. Please check BIOS under 'Advanced > Integrated IO Configuration' and Enable if necessary" + when: + - dmesg_dmar_remap.stdout|length == 0 + when: + - on_vms is not defined or not on_vms # STORY: CPU Hyper-Threading should be enabled in BIOS - - name: Warn about Hyper-Threading + - name: warn about Hyper-Threading fail: msg: "Warning: CPU Hyper-Threading is DISABLED on target. Please check BIOS under 'Advanced > Processor Configuration' and Enable if necessary" when: ansible_processor_threads_per_core != 2 - ignore_errors: True - - -# STORY: "check for collectd. See Jira NPF-1687" - - name: Warn about collectd - fail: - msg: "Warning: On {{ ansible_distribution }} {{ ansible_distribution_version }} collectd won't work unless 'update_kernel' is enabled in group_vars" - when: ansible_distribution_version in ['7.6', '18.04'] - ignore_errors: True # STORY: "collectd and telegraf are mutually exclusive" - - name: Fail if collectd and telegraf are both enabled + - name: fail if collectd and telegraf are both enabled assert: that: >- (({{ collectd_enabled | bool }}) and (not ({{ telegraf_enabled | bool }}))) @@ -559,6 +652,21 @@ - collectd_enabled is defined - telegraf_enabled is defined +# STORY: "istio 1.9 requires k8s version 1.20 at most" + - name: fail if istio version is not compatible with current k8s version + assert: + that: + - "{{ kube_version is version('v1.21', '<') }}" + msg: | + "Selected Istio service mesh version: '{{ service_mesh.version }}' is not compatible with selected k8s version: '{{ kube_version }}'" + "Please, refer to the compatibility table at https://istio.io/latest/docs/releases/supported-releases/" + when: + - kubernetes + - not container_runtime_only_deployment + - service_mesh is defined + - service_mesh.version is defined + - service_mesh.version is version("1.10", "<") + # STORY: TEMPORARY: "ovs dpdk version requirements" - debug: msg: @@ -572,23 +680,66 @@ - ovs_version is defined #host_vars - ovs_dpdk_enabled is defined and ovs_dpdk_enabled #host_vars - - name: Check OVS DPDK compatibility + - name: check OVS DPDK compatibility assert: - that: >- - "{{ ovs_version }} == \"v2.15.0\" and {{ dpdk_version }} >= \"20.11\"" - or "{{ ovs_version }} == \"v2.14.2\" and {{ dpdk_version }} == \"19.11.6\"" - or "{{ ovs_version }} == \"v2.14.1\" and {{ dpdk_version }} == \"19.11.6\"" - or "{{ ovs_version }} == \"v2.14.0\" and {{ dpdk_version }} == \"19.11.6\"" - or "{{ ovs_version }} == \"v2.13.3\" and {{ dpdk_version }} == \"19.11.6\"" - or "{{ ovs_version }} == \"v2.13.2\" and {{ dpdk_version }} == \"19.11.6\"" - or "{{ ovs_version }} == \"v2.13.1\" and {{ dpdk_version }} == \"19.11.6\"" - or "{{ ovs_version }} == \"v2.13.0\" and {{ dpdk_version }} == \"19.11.6\"" + that: + ovs_version == 'v2.16.2' and (dpdk_version < '21.11' and dpdk_version >= '21.08') + or (ovs_version < 'v2.16.2' and ovs_version >= 'v2.16.0') and dpdk_version == '21.08' + or ovs_version == 'v2.15.0' and dpdk_version == '20.11' + or ovs_version == 'v2.14.2' and dpdk_version == '19.11.6' + or ovs_version == 'v2.14.1' and dpdk_version == '19.11.6' + or ovs_version == 'v2.14.0' and dpdk_version == '19.11.6' + or ovs_version == 'v2.13.3' and dpdk_version == '19.11.6' + or ovs_version == 'v2.13.2' and dpdk_version == '19.11.6' + or ovs_version == 'v2.13.1' and dpdk_version == '19.11.6' + or ovs_version == 'v2.13.0' and dpdk_version == '19.11.6' msg: "OVS {{ ovs_version }} does not build with DPDK version {{ dpdk_version }}. Please correct the host_vars configuration" when: - dpdk_version is defined #host_vars - ovs_version is defined #host_vars - ovs_dpdk_enabled is defined and ovs_dpdk_enabled #host_vars + - name: check settings for Intel Power Operator + assert: + that: + - intel_power_manager.power_profiles | length > 0 + - intel_power_manager.power_nodes | length > 0 + fail_msg: "Intel Power Operator is enabled, but either Power Profiles or Power Nodes are not specified in group vars." + when: intel_power_manager is defined and intel_power_manager.enabled + + - name: check if Intel Power Operator is only Intel SST feature enalbed + assert: + that: + - not (sst_bf_configuration_enabled is defined and sst_bf_configuration_enabled or + sst_cp_configuration_enabled is defined and sst_cp_configuration_enabled or + sst_tf_configuration_enabled is defined and sst_tf_configuration_enabled or + sst_pp_configuration_enabled is defined and sst_pp_configuration_enabled) + fail_msg: + - "Currently Intel Power Operator and other Intel SST features are mutually exclusive." + - "Please disable the SST-TF, SST-CP and SST-PP in host vars." + when: intel_power_manager is defined and intel_power_manager.enabled + +# STORY: "MinIO requires number of nodes should be more than the minimum number of nodes defined in group_vars/all/minio_tenant_servers" + - name: check MinIO configuration + block: + - name: check MinIO minimum number of nodes + assert: + that: "{{ minio_tenant_servers |int }} <= {{ groups['kube_node']|length |int }}" + msg: | + "Incorrect configuration." + "The number of MinIO tenant servers '{{ minio_tenant_servers |int }}' defined in group vars must be" + "less or equal to the number of nodes '{{ groups['kube_node']|length |int }}'" + + - name: make sure the MinIO tenant volumes per server >= the MiniO PV list + assert: + that: "{{ minio_pv|length |int }} >= {{ minio_tenant_volumes_per_server |int }}" + msg: + - "Incorrect configuration." + - "The number of MinIO Persistent Volumes (PVs) '{{ minio_pv|length |int }}' defined in the host vars must be" + - "equal or more than MinIO Tenant Volumes per Server '{{ minio_tenant_volumes_per_server }}' defined in group vars." + when: + - kubernetes + - minio_enabled is defined and minio_enabled - meta: end_play diff --git a/playbooks/redeploy_cleanup.yml b/playbooks/redeploy_cleanup.yml index ed90032c..37d2dd75 100644 --- a/playbooks/redeploy_cleanup.yml +++ b/playbooks/redeploy_cleanup.yml @@ -1,5 +1,5 @@ ## -## Copyright (c) 2020-2021 Intel Corporation. +## Copyright (c) 2020-2022 Intel Corporation. ## ## Licensed under the Apache License, Version 2.0 (the "License"); ## you may not use this file except in compliance with the License. diff --git a/playbooks/regional_dc.yml b/playbooks/regional_dc.yml deleted file mode 100644 index 6c1ebe94..00000000 --- a/playbooks/regional_dc.yml +++ /dev/null @@ -1,26 +0,0 @@ -## -## Copyright (c) 2020-2021 Intel Corporation. -## -## Licensed under the Apache License, Version 2.0 (the "License"); -## you may not use this file except in compliance with the License. -## You may obtain a copy of the License at -## -## http://www.apache.org/licenses/LICENSE-2.0 -## -## Unless required by applicable law or agreed to in writing, software -## distributed under the License is distributed on an "AS IS" BASIS, -## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -## See the License for the specific language governing permissions and -## limitations under the License. -## ---- -- name: preflight checks - import_playbook: preflight.yml -- name: configure target hosts OS layer - import_playbook: infra/regional_dc.yml -- name: provision Kubernetes cluster using kubespray - import_playbook: k8s/k8s.yml - when: kubernetes | default(true) -- name: install Intel BMRA Container Experience Kit features - import_playbook: intel/regional_dc.yml - when: kubernetes | default(true) diff --git a/playbooks/remote_fp.yml b/playbooks/remote_fp.yml deleted file mode 100644 index c12fb413..00000000 --- a/playbooks/remote_fp.yml +++ /dev/null @@ -1,26 +0,0 @@ -## -## Copyright (c) 2020-2021 Intel Corporation. -## -## Licensed under the Apache License, Version 2.0 (the "License"); -## you may not use this file except in compliance with the License. -## You may obtain a copy of the License at -## -## http://www.apache.org/licenses/LICENSE-2.0 -## -## Unless required by applicable law or agreed to in writing, software -## distributed under the License is distributed on an "AS IS" BASIS, -## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -## See the License for the specific language governing permissions and -## limitations under the License. -## ---- -- name: preflight checks - import_playbook: preflight.yml -- name: configure target hosts OS layer - import_playbook: infra/remote_fp.yml -- name: provision Kubernetes cluster using kubespray - import_playbook: k8s/k8s.yml - when: kubernetes | default(true) -- name: install Intel BMRA Container Experience Kit features - import_playbook: intel/remote_fp.yml - when: kubernetes | default(true) diff --git a/playbooks/test/test_roles.yml b/playbooks/test/test_roles.yml index 62f1a289..cda4d9b6 100644 --- a/playbooks/test/test_roles.yml +++ b/playbooks/test/test_roles.yml @@ -1,5 +1,5 @@ ## -## Copyright (c) 2020-2021 Intel Corporation. +## Copyright (c) 2020-2022 Intel Corporation. ## ## Licensed under the Apache License, Version 2.0 (the "License"); ## you may not use this file except in compliance with the License. diff --git a/playbooks/access.yml b/playbooks/vm.yml similarity index 60% rename from playbooks/access.yml rename to playbooks/vm.yml index e16fe0e8..cd274fe9 100644 --- a/playbooks/access.yml +++ b/playbooks/vm.yml @@ -1,5 +1,5 @@ ## -## Copyright (c) 2020-2021 Intel Corporation. +## Copyright (c) 2020-2022 Intel Corporation. ## ## Licensed under the Apache License, Version 2.0 (the "License"); ## you may not use this file except in compliance with the License. @@ -14,13 +14,16 @@ ## limitations under the License. ## --- +# VM enabled +# If VM is enabled then Virtual Machines are created and CEK is deployed into those VMs + - name: preflight checks import_playbook: preflight.yml - name: configure target hosts OS layer - import_playbook: infra/access.yml -- name: provision Kubernetes cluster using kubespray - import_playbook: k8s/k8s.yml - when: kubernetes | default(true) -- name: install Intel BMRA Container Experience Kit features - import_playbook: intel/access.yml - when: kubernetes | default(true) + import_playbook: infra/{{ lookup('env', 'PROFILE') | default('full_nfv', True) }}.yml +- name: prepare VMs for VM deploymnet + import_playbook: infra/prepare_vms.yml +- name: deploy CEK on VMs + vars: + on_vms: True + import_playbook: "{{ lookup('env', 'PROFILE') | default('full_nfv', True) }}.yml" diff --git a/profiles/group_vars.j2 b/profiles/group_vars.j2 deleted file mode 100644 index 530e9c20..00000000 --- a/profiles/group_vars.j2 +++ /dev/null @@ -1,251 +0,0 @@ ---- -## BMRA primary playbook variables ## - -# Kubernetes version -kubernetes: true -kube_version: v1.21.1 -#kube_version: v1.20.6 -#kube_version: v1.19.8 - -# Kubernetes container runtime: docker, containerd, crio -container_runtime: docker - -# Run system-wide package update (apt dist-upgrade, yum update, ...) -# Note: enabling this may lead to unexpected results -# Tip: you can set this per host using host_vars -update_all_packages: false -update_kernel: false - -# SELinux configuration state: current, enabled, disabled -selinux_state: current -{% if nfd in ['on', 'optional'] %} -# Node Feature Discovery -nfd_enabled: {% if nfd == 'on' %}true{% else %}false{% endif %} -nfd_build_image_locally: false -nfd_namespace: kube-system -nfd_sleep_interval: 60s -{%- endif %} - -{%- if cmk in ['on', 'optional'] %} -# Intel CPU Manager for Kubernetes (CMK) -cmk_enabled: {% if cmk == 'on' %}true{% else %}false{% endif %} -cmk_namespace: kube-system -cmk_hosts_list: node1,node2 # host_vars/node.yml file must exist for all nodes in the list -cmk_shared_num_cores: 2 # number of CPU cores to be assigned to the "shared" pool on each of the nodes -cmk_exclusive_num_cores: 2 # number of CPU cores to be assigned to the "exclusive" pool on each of the nodes -# cmk_shared_mode: packed # choose between: packed, spread, default: packed -# cmk_exclusive_mode: packed # choose between: packed, spread, default: packed -autogenerate_isolcpus: true -{%- endif %} -{% if native_cpu_manager in ['on', 'optional'] %} -# Native CPU Manager (Kubernetes built-in) -# Note: Enabling CMK and built-in Native CPU Manager is NOT recommended. -# Setting this option as "true" enables the "static" policy, otherwise the default "none" policy is used. -# The reserved CPU cores settings are individual per each worker node, and therefore are available to configure in the host_vars file -native_cpu_manager_enabled: {% if native_cpu_manager == 'on' %}true{% else %}false{% endif %} -{% endif %} -{% if topology_manager in ['on', 'optional'] -%} -# Enable Kubernetes built-in Topology Manager -topology_manager_enabled: {% if topology_manager == 'on' %}true{% else %}false{% endif %} -# There are four supported policies: none, best-effort, restricted, single-numa-node. -topology_manager_policy: "best-effort" -{% endif %} - -{%- if sriov_operator in ['on', 'optional'] %} -# OpenShift SRIOV Network Operator -sriov_network_operator_enabled: {% if sriov_operator == 'on' %}true{% else %}false{% endif %} -sriov_network_operator_namespace: "sriov-network-operator" -{% endif %} - -{%- if sriov_network_dp in ['on', 'optional'] %} -# Intel SRIOV Network Device Plugin -sriov_net_dp_enabled: {% if sriov_network_dp == 'on' %}true{% else %}false{% endif %} -sriov_net_dp_namespace: kube-system -# whether to build and store image locally or use one from public external registry -sriov_net_dp_build_image_locally: true -# SR-IOV network device plugin configuration. -# For more information on supported configuration refer to: https://github.com/intel/sriov-network-device-plugin#configurations -sriovdp_config_data: | - { - "resourceList": [{ - "resourceName": "intel_sriov_netdevice", - "selectors": { - "vendors": ["8086"], - "devices": ["154c", "10ed", "1889"], - "drivers": ["iavf", "ixgbevf"] - } - }, - { - "resourceName": "intel_sriov_dpdk_700_series", - "selectors": { - "vendors": ["8086"], - "devices": ["154c", "10ed"], - "drivers": ["vfio-pci"] - } - }, - { - "resourceName": "intel_sriov_dpdk_800_series", - "selectors": { - "vendors": ["8086"], - "devices": ["1889"], - "drivers": ["vfio-pci"] - } - {% if name in ['full_nfv', 'access', 'regional_dc'] -%} - }, - { - "resourceName": "intel_fpga", - "deviceType": "accelerator", - "selectors": { - "vendors": ["8086"], - "devices": ["0d90"] - } - } - {%- else -%} - } - {%- endif %} - ] - } -{% endif %} - -{%- if sgx in ['on', 'optional'] or gpu in ['on', 'optional'] %} -# Intel Device Plugin Operator -intel_dp_namespace: kube-system # namespace will be applied for SGX DP and GPU DP -{% endif %} - -{%- if qat_dp in ['on', 'optional'] %} -# Intel QAT Device Plugin for Kubernetes -qat_dp_enabled: {% if qat_dp == 'on' %}true{% else %}false{% endif %} -qat_dp_namespace: kube-system -qat_dp_build_image_locally: true -{% endif %} - -{%- if openssl in ['on', 'optional'] %} -# This feature will enable OpenSSL*Engine -openssl_engine_enabled: {% if openssl == 'on' and qat == 'on' %}true{% else %}false{% endif %} # To activate OpenSSL*Engine set both install_openssl and update_qat_drivers to ‘true’ in host_vars -{% endif %} - -{%- if gpu in ['on', 'optional'] %} -# Intel GPU Device Plugin for Kubernetes -gpu_dp_enabled: {% if gpu == 'on' %}true{% else %}false{% endif %} -gpu_dp_kernel_version: "5.4.48+" -gpu_dp_build_image_locally: true -{% endif %} - -{%- if sgx_dp in ['on', 'optional'] %} -# Intel SGX Device Plugin for Kubernetes -sgx_dp_enabled: {% if sgx_dp == 'on' %}true{% else %}false{% endif %} -sgx_dp_build_image_locally: true -sgx_aesmd_namespace: kube-system -# ProvisionLimit is a number of containers that can share -# the same SGX provision device. -sgx_dp_provision_limit: 20 -# EnclaveLimit is a number of containers that can share the -# same SGX enclave device. -sgx_dp_enclave_limit: 20 -{% endif %} - -{%- if kmra in ['on', 'optional'] %} -# KMRA (Key Management Reference Application) -kmra_enabled: {% if kmra == 'on' %}true{% else %}false{% endif %} -# The PCCS uses this API key to request collaterals from Intel's Provisioning Certificate Service. -# User needs to subscribe first to obtain an API key. -# For how to subscribe to Intel Provisioning Certificate Service and receive an API key, -# goto https://api.portal.trustedservices.intel.com/provisioning-certification and click on 'Subscribe'. -kmra_pccs_api_key: "ffffffffffffffffffffffffffffffff" -# deploy KMRA demo workload (NGINX server) -kmra_deploy_demo_workload: true -{% endif %} - -{%- if istio in ['on', 'optional'] %} -# Istio operator -# https://istio.io/latest/docs/setup/install/operator/#install -istio_enabled: {% if istio == 'on' %}true{% else %}false{% endif %} -{% endif %} - -{%- if tas in ['on', 'optional'] %} -# Intel Telemetry Aware Scheduling -tas_enabled: {% if tas == 'on' %}true{% else %}false{% endif %} -tas_namespace: monitoring -# create and enable TAS demonstration policy: [true, false] -tas_enable_demo_policy: false -{% endif %} - -# Telemetry configuration. Collectd and Telegraf variables are mutually exclusive. -collectd_enabled: false -telegraf_enabled: true -collectd_scrap_interval: 30 -telegraf_scrap_interval: 30 - -{%- if sriov_network_dp in ["on", "optional"] or network_userspace in ["on", "optional"] %} -# Create reference net-attach-def objects -example_net_attach_defs: -{%- if sriov_network_dp in ["on", "optional"] %} - sriov_net_dp: {% if sriov_network_dp == "on" %}true{% else %}false{% endif %} # Update to match host_vars CNI configuration -{%- endif -%} -{%- if network_userspace in ["on", "optional"] %} - userspace_ovs_dpdk: {% if network_userspace == "on" %}true{% else %}false{% endif %} # Update to match host_vars CNI configuration - userspace_vpp: false # Update to match host_vars CNI configuration -{%- endif %} -{% endif %} -## Proxy configuration ## -#http_proxy: "http://proxy.example.com:1080" -#https_proxy: "http://proxy.example.com:1080" -#additional_no_proxy: ".example.com,mirror_ip" - -# (Ubuntu only) disables DNS stub listener which may cause issues on Ubuntu -dns_disable_stub_listener: true - -# Kubernetes cluster name, also will be used as DNS domain -cluster_name: cluster.local - -## Kubespray variables ## - -# supported network plugins(calico, flannel) and kube-proxy configuration -kube_network_plugin: calico -kube_network_plugin_multus: true -kube_pods_subnet: 10.244.0.0/16 -{%- if name in ['regional_dc', 'full_nfv', 'access'] -%} -{% set mask = 18 %} -{%- elif name == 'remote_fp' -%} -{% set mask = 19 %} -{%- elif name == 'on_prem' -%} -{% set mask = 21 %} -{%- elif name == 'basic' -%} -{% set mask = 22 %} -{%- endif %} -kube_service_addresses: 10.233.0.0/{{ mask }} -kube_proxy_mode: iptables - -# comment this line out if you want to expose k8s services of type nodePort externally. -kube_proxy_nodeport_addresses_cidr: 127.0.0.0/8 - -# local Docker Hub mirror, if it exists -#docker_registry_mirrors: -# - http://mirror_ip:mirror_port -#docker_insecure_registries: -# - http://docker_insecure_registry_ip -#containerd_registries: -# "docker.io": -# - "https://registry-1.docker.io" -# - "https://mirror_ip:mirror_port" -#crio_registries_mirrors: -# - prefix: docker.io -# insecure: false -# blocked: false -# location: registry-1.docker.io -# mirrors: -# - location: mirror_ip:mirror_port -# insecure: false -#crio_insecure_registries: -# - http://crio_insecure_registry_ip - -# Docker registry running on the cluster allows us to store images not avaialble on Docker Hub, e.g. CMK -# The range of valid ports is 30000-32767 -registry_nodeport: 30500 -registry_local_address: "localhost:{{ '{{' }} registry_nodeport {{ '}}' }}" - -# Enable Pod Security Policy. This option enables PSP admission controller and creates minimal set of rules. -psp_enabled: true - -# Set image pull policy to Always. Pulls images prior to starting containers. Valid credentials must be configured. -always_pull_enabled: true diff --git a/profiles/render.py b/profiles/render.py deleted file mode 100755 index de0ecac0..00000000 --- a/profiles/render.py +++ /dev/null @@ -1,138 +0,0 @@ -#!/usr/bin/env python - -# -# Copyright (c) 2020-2021 Intel Corporation. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -"""This module loads BMRA Profiles configuration file and renders example vars -and inventory files using Jinja templates. -""" - -import argparse -import os -from shutil import copytree, copy2, rmtree - -from ruamel.yaml import YAML -from jinja2 import Template - -def load_config(path): - """Loads YAML file and returns it as dict.""" - with open(path) as config_file: - yaml = YAML(typ='safe') - profiles = yaml.load(config_file) - return profiles - -def create_dir_idempotent(path): - """Creates directory if not present.""" - if not os.path.exists(path): - os.makedirs(path) - -def render(template_path, jinja_vars, target_path): - """Renders Jinja template and writes it to file.""" - with open(template_path) as file_: - template = Template(file_.read()) - out = template.render(jinja_vars) - target_file = open(target_path, "w+") - target_file.write(out) - target_file.close() - -def copy_file(path_to_file, dest): - """Copy file to specific destination""" - copy2(path_to_file, dest) - -def copy_dir(src, dst): - """Copy dir to specific destination""" - copytree(src, dst) - -def copy_dirs(src, dst, dirs_to_copy): - """Copy given dirs to specific destination""" - for d in dirs_to_copy: - path_to_dir = os.path.join(src, d) - dest_pat = os.path.join(dst, d) - copy_dir(path_to_dir, dest_pat) - -def remove_dir(path_to_dir): - """Remove directory from specific path""" - rmtree(path_to_dir, ignore_errors=True) # don't fail if dir doesn't exist - -def remove_dirs(src, dirs): - """Remove given dirs from specific destination""" - for d in dirs: - path_to_dir = os.path.join(src, d) - remove_dir(path_to_dir) - -def prepare_profile(output_dir, profile_name): - """Remove old and provide new profile's files/dirs. - Will not override inventory.ini file.""" - src = os.path.join(output_dir, profile_name) - - # determine project root dir location - project_root_dir = '../' if 'profiles' in os.getcwd() else './' - dirs = ['host_vars', 'group_vars'] - inventory = 'inventory.ini' - - # clean project root dir from host_vars and group_vars - remove_dirs(project_root_dir, dirs) - - # obtain newly generated project files - copy_dirs(src, project_root_dir, dirs) - - # do not override invetory.ini if exists already - path = os.path.join(project_root_dir, inventory) - if not os.path.exists(path): - example_inv_path = os.path.join(src, inventory) - copy_file(example_inv_path, project_root_dir) - -def main(): - """Loads configuration and renders inventory and vars templates.""" - parser = argparse.ArgumentParser() - parser.add_argument( - '--config', '-c', type=str, default="profiles.yml", - help='path to the profiles configuration file') - parser.add_argument('--group', '-g', type=str, default="group_vars.j2", - help='group_vars template filepath') - parser.add_argument('--host', type=str, default="host_vars.j2", - help='host_vars template filepath') - parser.add_argument('--inventory', '-i', type=str, default="inventory.j2", - help='inventory template filepath') - parser.add_argument('--output', '-o', type=str, default="../examples", - help='directory where generated files will be stored') - parser.add_argument('--profile', '-p', type=str, default='', - help='''profile, specified as full profile name - e.g. -p full_nfv, which files should be copied to project root dir. - Only one profile can be specified.''') - args = parser.parse_args() - - profiles = load_config(args.config) - - for profile, config in profiles.items(): - group_vars_dir_path = os.path.join(args.output, profile, "group_vars") - host_vars_dir_path = os.path.join(args.output, profile, "host_vars") - - create_dir_idempotent(group_vars_dir_path) - create_dir_idempotent(host_vars_dir_path) - - render(args.group, config, os.path.join(group_vars_dir_path, "all.yml")) - render(args.host, config, os.path.join(host_vars_dir_path, "node1.yml")) - render(args.inventory, config, os.path.join(args.output, profile, "inventory.ini")) - - if args.profile in profiles.keys(): - prepare_profile(args.output, args.profile) - print("Files needed for {} profile are copied to project root dir.".format(args.profile)) - elif args.profile: - print("Specified profile does not exist in profiles.yml file. Is there a typo in profile name?") - -if __name__ == "__main__": - main() diff --git a/profiles/requirements.txt b/profiles/requirements.txt deleted file mode 100644 index c6f2fcfd..00000000 --- a/profiles/requirements.txt +++ /dev/null @@ -1,2 +0,0 @@ -ruamel.yaml>=0.16.12 -jinja2>=2.11.2 diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 00000000..1c320539 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,11 @@ +ansible==3.4.0 +ansible-base==2.10.15 +cryptography==3.3.2 +jinja2==2.11.3 +netaddr==0.7.19 +pbr==5.4.4 +jmespath==0.9.5 +ruamel.yaml==0.16.10 +ruamel.yaml.clib==0.2.4 +MarkupSafe==1.1.1 +ipaddr diff --git a/roles/bond_cni_install/defaults/main.yml b/roles/bond_cni_install/defaults/main.yml index 08dd4872..fb0ae38f 100644 --- a/roles/bond_cni_install/defaults/main.yml +++ b/roles/bond_cni_install/defaults/main.yml @@ -1,5 +1,5 @@ ## -## Copyright (c) 2020-2021 Intel Corporation. +## Copyright (c) 2020-2022 Intel Corporation. ## ## Licensed under the Apache License, Version 2.0 (the "License"); ## you may not use this file except in compliance with the License. @@ -15,6 +15,5 @@ ## --- bond_cni_git_url: "https://github.com/intel/bond-cni.git" -bond_cni_dir: "/usr/src/bond-cni" +bond_cni_dir: "{{ project_root_dir }}/bond-cni" bond_cni_version: "v1.0" -bond_cni_path: "/usr/src/bond-cni/bond" diff --git a/roles/bond_cni_install/tasks/main.yml b/roles/bond_cni_install/tasks/main.yml index 2c99c587..94dcebcf 100644 --- a/roles/bond_cni_install/tasks/main.yml +++ b/roles/bond_cni_install/tasks/main.yml @@ -1,5 +1,5 @@ ## -## Copyright (c) 2020-2021 Intel Corporation. +## Copyright (c) 2020-2022 Intel Corporation. ## ## Licensed under the Apache License, Version 2.0 (the "License"); ## you may not use this file except in compliance with the License. @@ -18,7 +18,7 @@ include_role: name: install_dependencies -- name: clone Bond cni repository +- name: clone Bond CNI repository git: repo: "{{ bond_cni_git_url }}" dest: "{{ bond_cni_dir }}" @@ -38,9 +38,14 @@ recurse: yes mode: 0755 -- name: install Bond cni binary to /opt/cni/bin directory +- name: get GOPATH + command: /usr/local/go/bin/go env GOPATH + register: gopath + changed_when: false + +- name: install Bond CNI binary to /opt/cni/bin directory copy: - src: "{{ bond_cni_path }}" + src: "{{ gopath.stdout }}/bin/bond" dest: "/opt/cni/bin/bond" mode: 0755 remote_src: yes diff --git a/roles/bond_cni_install/vars/main.yml b/roles/bond_cni_install/vars/main.yml index 1819a19d..77dddbc0 100644 --- a/roles/bond_cni_install/vars/main.yml +++ b/roles/bond_cni_install/vars/main.yml @@ -1,5 +1,5 @@ ## -## Copyright (c) 2020-2021 Intel Corporation. +## Copyright (c) 2020-2022 Intel Corporation. ## ## Licensed under the Apache License, Version 2.0 (the "License"); ## you may not use this file except in compliance with the License. diff --git a/roles/bootstrap/ansible_host/tasks/main.yml b/roles/bootstrap/ansible_host/tasks/main.yml deleted file mode 100644 index 441cedc0..00000000 --- a/roles/bootstrap/ansible_host/tasks/main.yml +++ /dev/null @@ -1,53 +0,0 @@ -## -## Copyright (c) 2020-2021 Intel Corporation. -## -## Licensed under the Apache License, Version 2.0 (the "License"); -## you may not use this file except in compliance with the License. -## You may obtain a copy of the License at -## -## http://www.apache.org/licenses/LICENSE-2.0 -## -## Unless required by applicable law or agreed to in writing, software -## distributed under the License is distributed on an "AS IS" BASIS, -## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -## See the License for the specific language governing permissions and -## limitations under the License. -## ---- -- name: install pip on a RedHat Ansible host - package: - name: python-pip - when: - - ansible_distribution in ["RedHat", "CentOS"] - - ansible_distribution_version < '8' - ignore_errors: true - -- name: install pip3 on a RedHat Ansible host - package: - name: python3-pip - when: - - ansible_distribution in ["RedHat", "CentOS"] - - ansible_distribution_version >= '8' - ignore_errors: true - -- name: install pip3 on a Debian Ansible host - apt: - name: python3-pip - when: ansible_os_family == "Debian" - ignore_errors: true - -- name: install dependencies on the Ansible host - pip: - name: - - jmespath==0.9.5 - - netaddr==0.7.19 - - ipaddr - become: yes - -- name: Verify Ansible minimal version requirement - assert: - that: "ansible_version.full is \ - version_compare(2.9, '>=')" - msg: >- - Update Ansible to minimum version 2.9 - to use ansible-playbook. diff --git a/roles/bootstrap/apply_kubernetes_reqs/tasks/main.yml b/roles/bootstrap/apply_kubernetes_reqs/tasks/main.yml index c5d8c2c5..deedc78c 100644 --- a/roles/bootstrap/apply_kubernetes_reqs/tasks/main.yml +++ b/roles/bootstrap/apply_kubernetes_reqs/tasks/main.yml @@ -1,5 +1,5 @@ ## -## Copyright (c) 2020-2021 Intel Corporation. +## Copyright (c) 2020-2022 Intel Corporation. ## ## Licensed under the Apache License, Version 2.0 (the "License"); ## you may not use this file except in compliance with the License. @@ -28,37 +28,33 @@ changed_when: false become: yes -# for CentOS 7.x this is enough +# swap disabling routine - name: disable swap usage command: swapoff -a changed_when: false become: yes -# for other OSes additional actions are required - name: get swap service unit name shell: set -o pipefail | systemctl list-unit-files | grep \.swap | awk '{print $1}' # noqa 303 + args: + executable: /bin/bash register: unit_name changed_when: false become: yes - when: not (ansible_distribution == "CentOS" and ansible_distribution_version < '8') - name: stop swap service systemd: state: stopped name: "{{ unit_name.stdout }}" become: yes - when: - - not (ansible_distribution == "CentOS" and ansible_distribution_version < '8') - - '"swap" in unit_name.stdout' + when: '"swap" in unit_name.stdout' - name: mask swap service to /dev/null systemd: name: "{{ unit_name.stdout }}" masked: yes become: yes - when: - - not (ansible_distribution == "CentOS" and ansible_distribution_version < '8') - - '"swap" in unit_name.stdout' + when: '"swap" in unit_name.stdout' # note: this uses "failed_when: false" for the cases where br_netfilter is built Linux into kernel and not listed as a module # please see here for more information: https://github.com/kubernetes/kubernetes/issues/23385 @@ -92,4 +88,4 @@ - name: apply sysctl.conf command: sysctl -p /etc/sysctl.conf become: yes - changed_when: true \ No newline at end of file + changed_when: true diff --git a/roles/sgx_dp_install/charts/intel-sgx-aesmd/values.yaml b/roles/bootstrap/configure_additional_grub_parameters/defaults/main.yml similarity index 80% rename from roles/sgx_dp_install/charts/intel-sgx-aesmd/values.yaml rename to roles/bootstrap/configure_additional_grub_parameters/defaults/main.yml index c6a1065a..13ddf838 100644 --- a/roles/sgx_dp_install/charts/intel-sgx-aesmd/values.yaml +++ b/roles/bootstrap/configure_additional_grub_parameters/defaults/main.yml @@ -1,5 +1,5 @@ ## -## Copyright (c) 2020-2021 Intel Corporation. +## Copyright (c) 2020-2022 Intel Corporation. ## ## Licensed under the Apache License, Version 2.0 (the "License"); ## you may not use this file except in compliance with the License. @@ -14,7 +14,5 @@ ## limitations under the License. ## --- -sgx_aesmd_image: - repository: intel/sgx-aesmd-demo - tag: 0.21.0 - pullPolicy: IfNotPresent +additional_grub_parameters_enabled: false +additional_grub_parameters: "" diff --git a/roles/bootstrap/configure_additional_grub_parameters/tasks/main.yml b/roles/bootstrap/configure_additional_grub_parameters/tasks/main.yml new file mode 100644 index 00000000..2c034e3c --- /dev/null +++ b/roles/bootstrap/configure_additional_grub_parameters/tasks/main.yml @@ -0,0 +1,36 @@ +## +## Copyright (c) 2020-2022 Intel Corporation. +## +## Licensed under the Apache License, Version 2.0 (the "License"); +## you may not use this file except in compliance with the License. +## You may obtain a copy of the License at +## +## http://www.apache.org/licenses/LICENSE-2.0 +## +## Unless required by applicable law or agreed to in writing, software +## distributed under the License is distributed on an "AS IS" BASIS, +## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +## See the License for the specific language governing permissions and +## limitations under the License. +## +--- +- name: remove additional_grub_parameter settings when setting is disabled + lineinfile: + dest: /etc/default/grub + regexp: '^GRUB_CMDLINE_LINUX=(.*?){{ additional_grub_parameters_marker }}$' + state: absent + register: grub + become: yes + when: not additional_grub_parameters_enabled + +- name: setup additional grub parameters + include_tasks: setup_additional_grub_parameters.yml + when: + - additional_grub_parameters_enabled + +# noqa 503 - because include_role cannot be used in a handler +- name: update grub # noqa 503 - no-handler + include_role: + name: bootstrap/update_grub + when: + - grub.changed diff --git a/roles/bootstrap/configure_additional_grub_parameters/tasks/setup_additional_grub_parameters.yml b/roles/bootstrap/configure_additional_grub_parameters/tasks/setup_additional_grub_parameters.yml new file mode 100644 index 00000000..a4105b9a --- /dev/null +++ b/roles/bootstrap/configure_additional_grub_parameters/tasks/setup_additional_grub_parameters.yml @@ -0,0 +1,29 @@ +## +## Copyright (c) 2020-2022 Intel Corporation. +## +## Licensed under the Apache License, Version 2.0 (the "License"); +## you may not use this file except in compliance with the License. +## You may obtain a copy of the License at +## +## http://www.apache.org/licenses/LICENSE-2.0 +## +## Unless required by applicable law or agreed to in writing, software +## distributed under the License is distributed on an "AS IS" BASIS, +## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +## See the License for the specific language governing permissions and +## limitations under the License. +## +--- +- name: prepare additional grub parameters commandline string + set_fact: + additional_grub_parameters_cmdline: 'GRUB_CMDLINE_LINUX="${GRUB_CMDLINE_LINUX} {{ additional_grub_parameters }}" {{ additional_grub_parameters_marker }}' + +- name: set additional grub parameters in /etc/default/grub + lineinfile: + dest: /etc/default/grub + regexp: '^GRUB_CMDLINE_LINUX="\${GRUB_CMDLINE_LINUX}(.*?)" {{ additional_grub_parameters_marker }}$' + line: '{{ additional_grub_parameters_cmdline }}' + state: present + mode: a=r,u+w,g+w + register: grub + become: yes diff --git a/roles/git2_install/vars/main.yml b/roles/bootstrap/configure_additional_grub_parameters/vars/main.yml similarity index 79% rename from roles/git2_install/vars/main.yml rename to roles/bootstrap/configure_additional_grub_parameters/vars/main.yml index 8e33fc74..662e498e 100644 --- a/roles/git2_install/vars/main.yml +++ b/roles/bootstrap/configure_additional_grub_parameters/vars/main.yml @@ -1,5 +1,5 @@ ## -## Copyright (c) 2020-2021 Intel Corporation. +## Copyright (c) 2020-2022 Intel Corporation. ## ## Licensed under the Apache License, Version 2.0 (the "License"); ## you may not use this file except in compliance with the License. @@ -14,5 +14,4 @@ ## limitations under the License. ## --- -git_version: "2.25.1" -git_tarball_url: "https://github.com/git/git/archive/v{{ git_version }}.tar.gz" \ No newline at end of file +additional_grub_parameters_marker: "# additional_grub_parameters" diff --git a/roles/bootstrap/configure_cpu_isolation/tasks/autogenerate_isolcpus.yml b/roles/bootstrap/configure_cpu_isolation/tasks/autogenerate_isolcpus.yml deleted file mode 100755 index 0238b3ae..00000000 --- a/roles/bootstrap/configure_cpu_isolation/tasks/autogenerate_isolcpus.yml +++ /dev/null @@ -1,68 +0,0 @@ -## -## Copyright (c) 2020-2021 Intel Corporation. -## -## Licensed under the Apache License, Version 2.0 (the "License"); -## you may not use this file except in compliance with the License. -## You may obtain a copy of the License at -## -## http://www.apache.org/licenses/LICENSE-2.0 -## -## Unless required by applicable law or agreed to in writing, software -## distributed under the License is distributed on an "AS IS" BASIS, -## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -## See the License for the specific language governing permissions and -## limitations under the License. -## ---- -#2-27 -- name: autogenerate isolcpu physical range for socket 1 - set_fact: - isolcpus: "{{ cmk_exclusive_num_cores }}\ - -{{ ansible_processor_cores -1 }}" - when: cmk_exclusive_num_cores is defined - -#30-55 -- name: autogenerate isolcpu logical range for socket 1 hyperthread enabled - set_fact: - isolcpus: "{{ isolcpus }}\ - ,{{ ansible_processor_cores + cmk_exclusive_num_cores }}\ - -{{ ansible_processor_cores * ansible_processor_threads_per_core - 1 }}" - when: - - cmk_exclusive_num_cores is defined - - ansible_processor_threads_per_core > 1 - -#30-55 -- name: autogenerate isolcpu physical range for socket 2 hyperthread disabled - set_fact: - isolcpus: "{{ isolcpus }}\ - ,{{ ansible_processor_cores + cmk_exclusive_num_cores }}\ - -{{ ansible_processor_vcpus - 1 }}" - when: - - cmk_exclusive_num_cores is defined - - ansible_processor_count > 1 - - ansible_processor_threads_per_core == 1 - -#58-83 -- name: autogenerate isolcpu physical range for socket 2 hyperthread enabled - set_fact: - isolcpus: "{{ isolcpus }}\ - ,{{ ansible_processor_cores * ansible_processor_threads_per_core + cmk_exclusive_num_cores }}\ - -{{ ansible_processor_vcpus - ansible_processor_cores - 1 }}" - when: - - cmk_exclusive_num_cores is defined - - ansible_processor_count > 1 - - ansible_processor_threads_per_core > 1 - -#86-111 -- name: autogenerate isolcpu logical range for socket 2 hyperthread enabled - set_fact: - isolcpus: "{{ isolcpus }}\ - ,{{ ansible_processor_vcpus - ansible_processor_cores + cmk_exclusive_num_cores }}\ - -{{ ansible_processor_vcpus - 1 }}" - when: - - cmk_exclusive_num_cores is defined - - ansible_processor_count > 1 - - ansible_processor_threads_per_core > 1 - -- debug: - msg: "{{ isolcpus }}" diff --git a/roles/bootstrap/configure_cpu_isolation/tasks/main.yml b/roles/bootstrap/configure_cpu_isolation/tasks/main.yml index 537fdee8..b57a439e 100644 --- a/roles/bootstrap/configure_cpu_isolation/tasks/main.yml +++ b/roles/bootstrap/configure_cpu_isolation/tasks/main.yml @@ -1,5 +1,5 @@ ## -## Copyright (c) 2020-2021 Intel Corporation. +## Copyright (c) 2020-2022 Intel Corporation. ## ## Licensed under the Apache License, Version 2.0 (the "License"); ## you may not use this file except in compliance with the License. @@ -14,23 +14,5 @@ ## limitations under the License. ## --- -- name: ensure that isolcpus settings are disabled when isolcpus_enabled is False - lineinfile: - dest: /etc/default/grub - regexp: '^GRUB_CMDLINE_LINUX=(.*?){{ isolcpus_marker }}$' - state: absent - register: grub - notify: - - reboot server - when: not (isolcpus_enabled | default(false) | bool) - -- name: autogenerate isolcpus - include_tasks: autogenerate_isolcpus.yml - when: - - cmk_enabled is defined and cmk_enabled - - autogenerate_isolcpus is defined and autogenerate_isolcpus - - name: setup CPU isolation include_tasks: setup_isolcpus.yml - when: - - isolcpus_enabled | default(false) diff --git a/roles/bootstrap/configure_cpu_isolation/tasks/setup_isolcpus.yml b/roles/bootstrap/configure_cpu_isolation/tasks/setup_isolcpus.yml index a56bf2c7..28d2d52e 100644 --- a/roles/bootstrap/configure_cpu_isolation/tasks/setup_isolcpus.yml +++ b/roles/bootstrap/configure_cpu_isolation/tasks/setup_isolcpus.yml @@ -1,5 +1,5 @@ ## -## Copyright (c) 2020-2021 Intel Corporation. +## Copyright (c) 2020-2022 Intel Corporation. ## ## Licensed under the Apache License, Version 2.0 (the "License"); ## you may not use this file except in compliance with the License. @@ -36,12 +36,6 @@ set_fact: isolcpus_flags: "isolcpus={{ isolcpus }} rcu_nocbs={{ isolcpus }} nohz_full={{ isolcpus }} nr_cpus={{ ansible_processor_vcpus }}" -- name: set isolcpus flag for realtime kernel - set_fact: - isolcpus_flags: "isolcpus={{ isolcpus }} rcu_nocbs={{ isolcpus }} nohz_full={{ isolcpus }} nr_cpus={{ ansible_processor_vcpus }} idle=poll" - when: - - install_real_time_package | default(false) - - name: prepare CPU isolation grub commandline string set_fact: cpu_isolation_cmdline: 'GRUB_CMDLINE_LINUX="${GRUB_CMDLINE_LINUX} {{ isolcpus_flags }}" {{ isolcpus_marker }}' diff --git a/roles/bootstrap/configure_cpu_isolation/vars/main.yml b/roles/bootstrap/configure_cpu_isolation/vars/main.yml index cf20c1ab..1b9c02aa 100644 --- a/roles/bootstrap/configure_cpu_isolation/vars/main.yml +++ b/roles/bootstrap/configure_cpu_isolation/vars/main.yml @@ -1,5 +1,5 @@ ## -## Copyright (c) 2020-2021 Intel Corporation. +## Copyright (c) 2020-2022 Intel Corporation. ## ## Licensed under the Apache License, Version 2.0 (the "License"); ## you may not use this file except in compliance with the License. diff --git a/roles/bootstrap/configure_dns/tasks/main.yml b/roles/bootstrap/configure_dns/tasks/main.yml index e874d3c3..b26d2140 100644 --- a/roles/bootstrap/configure_dns/tasks/main.yml +++ b/roles/bootstrap/configure_dns/tasks/main.yml @@ -1,5 +1,5 @@ ## -## Copyright (c) 2020-2021 Intel Corporation. +## Copyright (c) 2020-2022 Intel Corporation. ## ## Licensed under the Apache License, Version 2.0 (the "License"); ## you may not use this file except in compliance with the License. @@ -55,4 +55,4 @@ service: name: systemd-resolved state: restarted - become: yes \ No newline at end of file + become: yes diff --git a/roles/bootstrap/configure_hugepages/defaults/main.yml b/roles/bootstrap/configure_hugepages/defaults/main.yml index 7bb79dd6..195ad66e 100644 --- a/roles/bootstrap/configure_hugepages/defaults/main.yml +++ b/roles/bootstrap/configure_hugepages/defaults/main.yml @@ -1,5 +1,5 @@ ## -## Copyright (c) 2020-2021 Intel Corporation. +## Copyright (c) 2020-2022 Intel Corporation. ## ## Licensed under the Apache License, Version 2.0 (the "License"); ## you may not use this file except in compliance with the License. diff --git a/roles/bootstrap/configure_hugepages/tasks/main.yml b/roles/bootstrap/configure_hugepages/tasks/main.yml index ef3252d8..a426cdd9 100644 --- a/roles/bootstrap/configure_hugepages/tasks/main.yml +++ b/roles/bootstrap/configure_hugepages/tasks/main.yml @@ -1,5 +1,5 @@ ## -## Copyright (c) 2020-2021 Intel Corporation. +## Copyright (c) 2020-2022 Intel Corporation. ## ## Licensed under the Apache License, Version 2.0 (the "License"); ## you may not use this file except in compliance with the License. diff --git a/roles/bootstrap/configure_hugepages/tasks/setup_hugepages.yml b/roles/bootstrap/configure_hugepages/tasks/setup_hugepages.yml index 422a65e1..85a2fca4 100644 --- a/roles/bootstrap/configure_hugepages/tasks/setup_hugepages.yml +++ b/roles/bootstrap/configure_hugepages/tasks/setup_hugepages.yml @@ -1,5 +1,5 @@ ## -## Copyright (c) 2020-2021 Intel Corporation. +## Copyright (c) 2020-2022 Intel Corporation. ## ## Licensed under the Apache License, Version 2.0 (the "License"); ## you may not use this file except in compliance with the License. @@ -14,42 +14,32 @@ ## limitations under the License. ## --- -- name: validate default hugepage size setting - assert: { that: default_hugepage_size == "2M" or default_hugepage_size == "1G" } - -- name: validate that any hugepages of default size are requested - assert: { that: number_of_hugepages > 0 } - when: - - default_hugepage_size == "1G" or default_hugepage_size == "2M" - - name: calculate total number of requested hugepages set_fact: - mem_huge_2M: "{{ 2 * number_of_hugepages | default(0) if default_hugepage_size == '2M' else 0 }}" - mem_huge_1G: "{{ 1024 * number_of_hugepages | default(0) if default_hugepage_size == '1G' else 0 }}" + mem_huge_2M: "{{ 2 * number_of_hugepages_2M }}" + mem_huge_1G: "{{ 1024 * number_of_hugepages_1G }}" -- name: assert that there's enough memory on the target system for the requested hugepages +- name: check if there is enough memory on the target system for the requested hugepages assert: that: (mem_huge_2M|int + mem_huge_1G|int) < (ansible_memtotal_mb|int - mem_reserved|int) msg: "Requested {{ mem_huge_2M|int + mem_huge_1G|int }}MB of hugepages, while {{ ansible_memtotal_mb|int - mem_reserved|int }}MB is available." -- name: set hugepages kernel boot flags +- name: prepare kernel boot flags set_fact: hugepages_flags: >- default_hugepagesz={{ default_hugepage_size }} - {% if default_hugepage_size == "1G" %}hugepagesz=1G hugepages={{ number_of_hugepages | int }}{% endif %} - {% if default_hugepage_size == "2M" %}hugepagesz=2M hugepages={{ number_of_hugepages | int }}{% endif %} + {% if number_of_hugepages_1G > 0 %}hugepagesz=1G hugepages={{ number_of_hugepages_1G | int }}{% endif %} + {% if number_of_hugepages_2M > 0 %}hugepagesz=2M hugepages={{ number_of_hugepages_2M | int }}{% endif %} - name: prepare hugepages grub commandline string set_fact: hugepages_cmdline: 'GRUB_CMDLINE_LINUX="${GRUB_CMDLINE_LINUX} {{ hugepages_flags }}" {{ hugepages_marker }}' -- name: set cpu isolation flags in /etc/default/grub +- name: set hugepages flags in /etc/default/grub lineinfile: dest: /etc/default/grub regexp: '^GRUB_CMDLINE_LINUX="\${GRUB_CMDLINE_LINUX}(.*?)" {{ hugepages_marker }}$' line: '{{ hugepages_cmdline }}' state: present mode: 0664 - register: grub - notify: - - reboot server + notify: reboot server diff --git a/roles/bootstrap/configure_hugepages/vars/main.yml b/roles/bootstrap/configure_hugepages/vars/main.yml index 6c91343a..0cd1a7a9 100644 --- a/roles/bootstrap/configure_hugepages/vars/main.yml +++ b/roles/bootstrap/configure_hugepages/vars/main.yml @@ -1,5 +1,5 @@ ## -## Copyright (c) 2020-2021 Intel Corporation. +## Copyright (c) 2020-2022 Intel Corporation. ## ## Licensed under the Apache License, Version 2.0 (the "License"); ## you may not use this file except in compliance with the License. diff --git a/roles/bootstrap/configure_intel_pstate/tasks/main.yml b/roles/bootstrap/configure_intel_pstate/tasks/main.yml index d45f8507..768da76c 100644 --- a/roles/bootstrap/configure_intel_pstate/tasks/main.yml +++ b/roles/bootstrap/configure_intel_pstate/tasks/main.yml @@ -1,5 +1,5 @@ ## -## Copyright (c) 2020-2021 Intel Corporation. +## Copyright (c) 2020-2022 Intel Corporation. ## ## Licensed under the Apache License, Version 2.0 (the "License"); ## you may not use this file except in compliance with the License. @@ -14,20 +14,10 @@ ## limitations under the License. ## --- -- name: ensure that intel_pstate settings are disabled when intel_pstate_enabled is false - lineinfile: - dest: /etc/default/grub - regexp: '^GRUB_CMDLINE_LINUX=(.*?){{ intel_pstate_marker }}$' - state: absent - register: grub - notify: - - reboot server - when: not (intel_pstate_enabled | default(false) | bool) - - name: setup intel_pstate driver include_tasks: setup_intel_pstate.yml when: - - intel_pstate_enabled | default(false) + - intel_pstate_enabled | default(false) | bool - name: setup turbo boost include_tasks: setup_turbo.yml diff --git a/roles/bootstrap/configure_intel_pstate/tasks/setup_intel_pstate.yml b/roles/bootstrap/configure_intel_pstate/tasks/setup_intel_pstate.yml index a2ddf3ab..ecdfe346 100644 --- a/roles/bootstrap/configure_intel_pstate/tasks/setup_intel_pstate.yml +++ b/roles/bootstrap/configure_intel_pstate/tasks/setup_intel_pstate.yml @@ -1,5 +1,5 @@ ## -## Copyright (c) 2020-2021 Intel Corporation. +## Copyright (c) 2020-2022 Intel Corporation. ## ## Licensed under the Apache License, Version 2.0 (the "License"); ## you may not use this file except in compliance with the License. @@ -33,6 +33,5 @@ line: '{{ intel_pstate_cmdline }}' state: present mode: 0664 - register: grub notify: - reboot server diff --git a/roles/bootstrap/configure_intel_pstate/tasks/setup_turbo.yml b/roles/bootstrap/configure_intel_pstate/tasks/setup_turbo.yml index 2293ecf5..714b025e 100644 --- a/roles/bootstrap/configure_intel_pstate/tasks/setup_turbo.yml +++ b/roles/bootstrap/configure_intel_pstate/tasks/setup_turbo.yml @@ -1,5 +1,5 @@ ## -## Copyright (c) 2020-2021 Intel Corporation. +## Copyright (c) 2020-2022 Intel Corporation. ## ## Licensed under the Apache License, Version 2.0 (the "License"); ## you may not use this file except in compliance with the License. @@ -24,68 +24,36 @@ register: turbo_details changed_when: false -- name: Intel Turbo Boost Technology is enabled in CPU/BIOS, set value = True +- name: set Intel Turbo Boost Technology is enabled in CPU/BIOS set_fact: turbo_bios_enabled: True when: '"true" in turbo_details.stdout' -- name: Intel Turbo Boost Technology is disabled in CPU/BIOS, set value = False +- name: set Intel Turbo Boost Technology is disabled in CPU/BIOS set_fact: turbo_bios_enabled: False when: '"false" in turbo_details.stdout' -- name: check if the pstate turbo management file is available - stat: - path: "{{ intel_turbo_path }}" - register: intel_pstate_folder - -- name: pstate turbo management file missing, playbook terminated - fail: - msg: - - "failed to configure sysfs turbo attribute for Intel pstate" - - "pstate turbo management file path is missing, please verify BIOS setting Hardware P-States is enabled" - when: not intel_pstate_folder.stat.exists - -- name: check status of turbo_value on system before configuration - command: "cat {{ intel_turbo_path }}" - register: turbo_status - changed_when: true - -- debug: - msg: "current status of turbo_value on system is = {{ turbo_status.stdout }}" +- name: configure Intel Turbo Boost Technology + block: + # returned 1 indicates turbo is now off and returned 0 indicates turbo is now on + - name: set turbo boost is enabled + set_fact: + turbo_value: 1 + when: not turbo_boost_enabled + + - name: set turbo boost is disabled + set_fact: + turbo_value: 0 + when: turbo_boost_enabled + + - name: configure sysfs turbo attribute for Intel pstate when turbo_value = {{ turbo_value }} + shell: "echo {{ turbo_value }} > {{ intel_turbo_path }}" + args: + executable: /bin/bash + when: turbo_bios_enabled -- name: configuration cannot be continued as turbo_value = 1 (False) +- name: configuration cannot be continued debug: msg: "Intel Turbo Boost Technology is disabled in CPU/BIOS, configuration skipped" when: not turbo_bios_enabled - -# returned 1 indicates turbo is now off and returned 0 indicates turbo is now on -- name: set turbo_value = 1 when turbo_boost_enabled = false in host vars - set_fact: - turbo_value: 1 - when: - - turbo_bios_enabled - - not turbo_boost_enabled - -- name: set turbo_value = 0 when turbo_boost_enabled = true in host vars - set_fact: - turbo_value: 0 - when: - - turbo_bios_enabled - - turbo_boost_enabled - -- name: configure sysfs turbo attribute for Intel pstate when turbo_value = {{ turbo_value }} - shell: "echo {{ turbo_value }} > {{ intel_turbo_path }}" - args: - executable: /bin/bash - when: turbo_bios_enabled - -- name: check status of turbo_value on system after configuration - command: "cat {{ intel_turbo_path }}" - register: turbo_status_recheck - changed_when: true - when: turbo_bios_enabled - -- debug: - msg: "status of turbo_value on system after configuration is = {{ turbo_status_recheck.stdout }}" - when: turbo_bios_enabled \ No newline at end of file diff --git a/roles/bootstrap/configure_intel_pstate/vars/main.yml b/roles/bootstrap/configure_intel_pstate/vars/main.yml index 460a5813..75c411a0 100644 --- a/roles/bootstrap/configure_intel_pstate/vars/main.yml +++ b/roles/bootstrap/configure_intel_pstate/vars/main.yml @@ -1,5 +1,5 @@ ## -## Copyright (c) 2020-2021 Intel Corporation. +## Copyright (c) 2020-2022 Intel Corporation. ## ## Licensed under the Apache License, Version 2.0 (the "License"); ## you may not use this file except in compliance with the License. diff --git a/roles/bootstrap/configure_openssl/defaults/main.yml b/roles/bootstrap/configure_openssl/defaults/main.yml index c466e8e6..85d3fbaa 100644 --- a/roles/bootstrap/configure_openssl/defaults/main.yml +++ b/roles/bootstrap/configure_openssl/defaults/main.yml @@ -1,5 +1,5 @@ ## -## Copyright (c) 2020-2021 Intel Corporation. +## Copyright (c) 2020-2022 Intel Corporation. ## ## Licensed under the Apache License, Version 2.0 (the "License"); ## you may not use this file except in compliance with the License. @@ -15,18 +15,5 @@ ## --- openssl_url: "https://github.com/openssl/openssl.git" -openssl_version: "openssl-3.0.0" -openssl_dir: "/usr/src/OpenSSL" -openssl_ubuntu_libssldev_pkg: "http://archive.ubuntu.com/ubuntu/pool/main/o/openssl/libssl-dev_1.1.1j-1ubuntu3_amd64.deb" - -# CentOS <= 7.9 requirements -openssl_centos7_dir: "/usr/src/centos7_openssl_packages" -perl_dir: "{{ openssl_centos7_dir }}/perl" -perl_url: "https://www.cpan.org/src/5.0/perl-5.28.1.tar.gz" -cmake_dir: "{{ openssl_centos7_dir }}/cmake" -cmake_url: "https://github.com/Kitware/CMake/releases/download/v3.19.8/cmake-3.19.8.tar.gz" -openssl_111d_url: "https://www.openssl.org/source/openssl-1.1.1d.tar.gz" -nasm_url: "https://www.nasm.us/pub/nasm/releasebuilds/2.15.05/nasm-2.15.05.tar.xz" -nasm_dir: "{{ openssl_centos7_dir }}/nasm" -gcc_url: "https://ftp.gnu.org/gnu/gcc/gcc-8.2.0/gcc-8.2.0.tar.gz" -gcc_dir: "{{ openssl_centos7_dir }}/gcc" +openssl_version: "openssl-3.0.1" +openssl_dir: "{{ (project_root_dir, 'openssl') | path_join }}" diff --git a/roles/bootstrap/configure_openssl/tasks/install_openssl_1.1.1_centos7.yml b/roles/bootstrap/configure_openssl/tasks/install_openssl_1.1.1_centos7.yml deleted file mode 100644 index c0fb4c51..00000000 --- a/roles/bootstrap/configure_openssl/tasks/install_openssl_1.1.1_centos7.yml +++ /dev/null @@ -1,127 +0,0 @@ -## -## Copyright (c) 2020-2021 Intel Corporation. -## -## Licensed under the Apache License, Version 2.0 (the "License"); -## you may not use this file except in compliance with the License. -## You may obtain a copy of the License at -## -## http://www.apache.org/licenses/LICENSE-2.0 -## -## Unless required by applicable law or agreed to in writing, software -## distributed under the License is distributed on an "AS IS" BASIS, -## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -## See the License for the specific language governing permissions and -## limitations under the License. -## ---- -# Ref: https://www.hostnextra.com/kb/how-to-install-openssl-1-1-1d-in-centos/ -- name: download and unarchive openssl-1.1.1d.tar.gz - unarchive: - src: "{{ openssl_111d_url }}" - dest: "/usr/local/src/" - remote_src: yes - mode: "u=rwx,g=rx,o=rx" - register: openssl111d_install - until: openssl111d_install is not failed - retries: 5 - -- name: configure openssl-1.1.1d with PATH - command: "./config --prefix=/usr/local/ssl --openssldir=/usr/local/ssl shared zlib" - args: - chdir: "/usr/local/src/openssl-1.1.1d" - register: check_config - changed_when: true - -- debug: - var: check_config.stdout_lines - -- name: make install openssl-1.1.1d - make: - chdir: "/usr/local/src/openssl-1.1.1d" - target: install - -- name: create file for configuration - file: - path: "/etc/ld.so.conf.d/openssl-1.1.1d.conf" - state: touch - owner: root - group: root - mode: '0700' - become: yes - -- name: insert ssl/lib path in the config file - lineinfile: - path: "/etc/ld.so.conf.d/openssl-1.1.1d.conf" - line: "/usr/local/ssl/lib" - owner: root - group: root - mode: '0700' - become: yes - -- name: reload the dynamic link - command: "ldconfig -v" - args: - chdir: "/etc/ld.so.conf.d/" - changed_when: true - -- name: move backup of existed openssl file - command: "mv /bin/openssl /bin/openssl.backup" - args: - chdir: "/etc/ld.so.conf.d/" - changed_when: true - -- name: create new environment file for OpenSSL - file: - path: "/etc/profile.d/openssl.sh" - state: touch - owner: root - group: root - mode: '0700' - become: yes - -- name: insert path in the OpenSSL config file - lineinfile: - path: "/etc/profile.d/openssl.sh" - state: present - line: "{{ item }}" - owner: root - group: root - mode: '0700' - with_items: - - 'OPENSSL_PATH="/usr/local/ssl/bin"' - - 'export OPENSSL_PATH' - - 'PATH=$PATH:$OPENSSL_PATH' - - 'export PATH' - become: yes - -- name: reload the new OpenSSL environment file - command: /bin/bash -c "source /etc/profile.d/openssl.sh" - args: - chdir: "/etc/profile.d" - changed_when: true - -- name: check the default PATH - command: "echo $PATH" - args: - chdir: "/etc/profile.d" - changed_when: true - -- name: check the default PATH after reload - command: "which openssl" - args: - chdir: "/etc/profile.d/" - register: check_path - changed_when: true - -- debug: - var: check_path.stdout_lines - -- name: verify the installation and version of the OpenSSL - command: "openssl version -a" - args: - chdir: "/etc/profile.d/" - register: verify_install - changed_when: true - -- debug: - var: verify_install.stdout_lines[0] diff --git a/roles/bootstrap/configure_openssl/tasks/install_req_packages_centos7.yml b/roles/bootstrap/configure_openssl/tasks/install_req_packages_centos7.yml deleted file mode 100644 index d6a72b84..00000000 --- a/roles/bootstrap/configure_openssl/tasks/install_req_packages_centos7.yml +++ /dev/null @@ -1,134 +0,0 @@ -## -## Copyright (c) 2020-2021 Intel Corporation. -## -## Licensed under the Apache License, Version 2.0 (the "License"); -## you may not use this file except in compliance with the License. -## You may obtain a copy of the License at -## -## http://www.apache.org/licenses/LICENSE-2.0 -## -## Unless required by applicable law or agreed to in writing, software -## distributed under the License is distributed on an "AS IS" BASIS, -## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -## See the License for the specific language governing permissions and -## limitations under the License. -## ---- -- name: create directory for all packages required in CentOS <= 7.9 - file: - path: "{{ openssl_centos7_dir }}" - state: directory - mode: "u=rwx,g=rx,o=rx" - -- name: create directory for perl-5.28.1 - file: - path: "{{ perl_dir }}" - state: directory - mode: "u=rwx,g=rx,o=rx" - -- name: download and unarchive perl-5.28.1 - unarchive: - src: "{{ perl_url }}" - dest: "{{ perl_dir }}" - remote_src: yes - mode: "u=rwx,g=rx,o=rx" - register: perl_install - until: perl_install is not failed - retries: 5 - -- name: install the extracted source - command: "./Configure -de" - args: - chdir: "{{ perl_dir }}/perl-5.28.1" - changed_when: true - -- name: make install perl-5.28.1 - make: - chdir: "{{ perl_dir }}/perl-5.28.1" - target: install - -- name: verify perl version after installation - command: "perl --version" - register: perl_version - changed_when: true - -- debug: - var: perl_version.stdout_lines[1] - -- name: create directory for cmake-3.19.8 - file: - path: "{{ cmake_dir }}" - state: directory - mode: "u=rwx,g=rx,o=rx" - -- name: download and unarchive cmake-3.19.8 - unarchive: - src: "{{ cmake_url }}" - dest: "{{ cmake_dir }}" - remote_src: yes - mode: "u=rwx,g=rx,o=rx" - register: cmake_install - until: cmake_install is not failed - retries: 5 - -- name: install the extracted source by running bootstrap - command: "./bootstrap" - args: - chdir: "{{ cmake_dir }}/cmake-3.19.8" - changed_when: true - -- name: make prepare cmake-3.19.8 - command: "make -j4" - args: - chdir: "{{ cmake_dir }}/cmake-3.19.8" - changed_when: true - -- name: make install cmake-3.19.8 - make: - chdir: "{{ cmake_dir }}/cmake-3.19.8" - target: install - changed_when: true - -- name: verify cmake version after installation - command: "cmake --version" - register: cmake_verify - changed_when: true - -- debug: - var: cmake_verify.stdout_lines[0] - -- name: create directory for nasm-2.15.05 - file: - path: "{{ nasm_dir }}" - state: directory - mode: "u=rwx,g=rx,o=rx" - -- name: download and unarchive nasm-2.15.05 - unarchive: - src: "{{ nasm_url }}" - dest: "{{ nasm_dir }}" - remote_src: yes - mode: "u=rwx,g=rx,o=rx" - register: nasm_install - until: nasm_install is not failed - retries: 5 - -- name: config the extracted source by running bootstrap - command: "./configure --prefix=/usr" - args: - chdir: "{{ nasm_dir }}/nasm-2.15.05" - changed_when: true - -- name: make install nasm - make: - chdir: "{{ nasm_dir }}/nasm-2.15.05" - target: install - changed_when: true - -- name: verify nasm version after installation in CentOS <= 7.9 - command: "nasm --version" - register: nasm_verify - changed_when: true - -- debug: - var: nasm_verify.stdout_lines diff --git a/roles/bootstrap/configure_openssl/tasks/main.yml b/roles/bootstrap/configure_openssl/tasks/main.yml index 0131e2b5..56bd8933 100644 --- a/roles/bootstrap/configure_openssl/tasks/main.yml +++ b/roles/bootstrap/configure_openssl/tasks/main.yml @@ -1,5 +1,5 @@ ## -## Copyright (c) 2020-2021 Intel Corporation. +## Copyright (c) 2020-2022 Intel Corporation. ## ## Licensed under the Apache License, Version 2.0 (the "License"); ## you may not use this file except in compliance with the License. @@ -14,11 +14,6 @@ ## limitations under the License. ## --- -- name: install libssl-dev development files in Ubuntu 21.04 - apt: - deb: "{{ openssl_ubuntu_libssldev_pkg }}" - when: ansible_distribution == 'Ubuntu' and ansible_distribution_version >= '21.04' - - name: install dependencies for OpenSSL include_role: name: install_dependencies @@ -26,7 +21,6 @@ - name: confirm module before OpenSSL installation shell: "set -o pipefail && lsmod | grep qat" args: - chdir: "/usr/src" executable: /bin/bash register: confirm_mod failed_when: '"intel_qat" not in confirm_mod.stdout' @@ -35,7 +29,7 @@ - name: Module not found, playbook terminated fail: - msg: "No QAT module found. Kindly, confirm QAT module on system can also download latest drivers + services as enabling update_qat_drivers: true" + msg: "No QAT module found. Please set update_qat_drivers to true in host vars to resolve issue." when: '"intel_qat" not in confirm_mod.stdout' - debug: @@ -52,28 +46,10 @@ - name: Service not found, playbook terminated fail: - msg: "Failed to start qat_service on system. Kindly, download latest drivers + services this can also be achieved as enabling update_qat_drivers: true" + msg: "Failed to start qat_service on system. Please check if QAT configuration in host vars is correct." when: - "'up' not in service_check.stdout" -- name: check if OpenSSL dir exists in CentOS <= 7.9 - stat: - path: "{{ openssl_centos7_dir }}" - register: centos7_openssl_stat_result - when: ansible_distribution == 'CentOS' and ansible_distribution_version <= '7.9' - changed_when: true - -- name: install required packages in CentOS 7.9 before OpenSSL Configuration - include_tasks: install_req_packages_centos7.yml - when: - - ansible_distribution == 'CentOS' and ansible_distribution_version <= '7.9' - - not centos7_openssl_stat_result.stat.exists - -- name: install required packages in CentOS 7.9 before OpenSSL Configuration - include_tasks: install_openssl_1.1.1_centos7.yml - when: - - ansible_distribution == 'CentOS' and ansible_distribution_version <= '7.9' - - not centos7_openssl_stat_result.stat.exists # OpenSSL build - name: create directory {{ openssl_dir }} for all OpenSSL dependencies file: @@ -96,25 +72,30 @@ changed_when: true ignore_errors: true -- name: playbook terminated. Configuration for OpenSSL is missing requirements +- name: Configuration for OpenSSL is missing requirements, playbook terminated fail: msg: - - "OpenSSL configuration failed. Make sure QAT drivers + services are properly configured as per requirements for OpenSSL*Engine" + - "OpenSSL configuration failed. Please make sure all requirements, described in host vars/group vars, for OpenSSL*Engine are satisfied." when: '"OpenSSL has been successfully configured" not in openssl_config.stdout' -- debug: - var: openssl_config.stdout_lines - when: '"OpenSSL has been successfully configured" in openssl_config.stdout' +- name: detect number of cores on system + command: nproc + register: nproc_out + changed_when: false - name: make depend once OpenSSL has been successfully configured make: chdir: "{{ openssl_dir }}" target: depend + environment: + "MAKEFLAGS": "-j{{ nproc_out.stdout | int }}" - name: make install once OpenSSL has been successfully configured make: chdir: "{{ openssl_dir }}" target: install + environment: + "MAKEFLAGS": "-j{{ nproc_out.stdout | int / 2 }}" # accelerate, but do not break the installation - name: exporting environment variable for OpenSSL dynamic engines at runtime command: /bin/bash -c "export OPENSSL_ENGINES=/usr/local/ssl/lib/engines-1.1" @@ -126,13 +107,3 @@ - name: reload the dynamic linker cache command: "ldconfig" changed_when: true - -- name: verify OpenSSL dir after successful configuration - command: "ls -al" - args: - chdir: "/usr/local/ssl" - register: verify_openssl - changed_when: true - -- debug: - var: verify_openssl.stdout_lines diff --git a/roles/bootstrap/configure_openssl/vars/main.yml b/roles/bootstrap/configure_openssl/vars/main.yml index 63bb5749..51b4f48e 100644 --- a/roles/bootstrap/configure_openssl/vars/main.yml +++ b/roles/bootstrap/configure_openssl/vars/main.yml @@ -1,5 +1,5 @@ ## -## Copyright (c) 2020-2021 Intel Corporation. +## Copyright (c) 2020-2022 Intel Corporation. ## ## Licensed under the Apache License, Version 2.0 (the "License"); ## you may not use this file except in compliance with the License. @@ -27,8 +27,6 @@ install_dependencies: - cpuid - make - nasm - - cpuid - - cmake RedHat: - "@Development Tools" - cmake diff --git a/roles/bootstrap/configure_proxy/tasks/main.yml b/roles/bootstrap/configure_proxy/tasks/main.yml index f823e956..9eb3e1fa 100644 --- a/roles/bootstrap/configure_proxy/tasks/main.yml +++ b/roles/bootstrap/configure_proxy/tasks/main.yml @@ -1,5 +1,5 @@ ## -## Copyright (c) 2020-2021 Intel Corporation. +## Copyright (c) 2020-2022 Intel Corporation. ## ## Licensed under the Apache License, Version 2.0 (the "License"); ## you may not use this file except in compliance with the License. @@ -34,21 +34,28 @@ regexp: '^{{ item.key }}' line: '{{ item.key }}={{ item.value }}' create: yes - owner: root - group: root + owner: "{{ ansible_user | default(ansible_user_id) }}" + group: "{{ ansible_user | default(ansible_user_id) }}" mode: 0644 with_dict: "{{ proxy_env }}" when: '"http_proxy" in proxy_env or "https_proxy" in proxy_env' become: yes - name: ensure Docker config directory exists - file: path={{ ansible_env.HOME }}/.docker state=directory mode=0755 + file: + path: "{{ ansible_env.HOME }}/.docker" + state: "directory" + mode: 0755 + owner: "{{ ansible_user | default(ansible_user_id) }}" + group: "{{ ansible_user | default(ansible_user_id) }}" when: container_runtime == "docker" - name: create Docker config.json file with proxy setttings template: src: docker_config.json.j2 dest: "{{ ansible_env.HOME }}/.docker/config.json" + owner: "{{ ansible_user | default(ansible_user_id) }}" + group: "{{ ansible_user | default(ansible_user_id) }}" force: yes mode: 0755 when: diff --git a/roles/bootstrap/configure_qat/files/bmra_sriov_qat_init b/roles/bootstrap/configure_qat/files/cek_sriov_qat_init similarity index 94% rename from roles/bootstrap/configure_qat/files/bmra_sriov_qat_init rename to roles/bootstrap/configure_qat/files/cek_sriov_qat_init index d81e9c2d..9ae6ad39 100644 --- a/roles/bootstrap/configure_qat/files/bmra_sriov_qat_init +++ b/roles/bootstrap/configure_qat/files/cek_sriov_qat_init @@ -15,7 +15,7 @@ # limitations under the License. # -QAT_SRIOV_NUMVFS_MAPPINGS=${QAT_SRIOV_NUMVFS_MAPPINGS:-"/etc/bmra/bmra_sriov_qat_numvfs"} +QAT_SRIOV_NUMVFS_MAPPINGS=${QAT_SRIOV_NUMVFS_MAPPINGS:-"/etc/cek/cek_sriov_qat_numvfs"} setup_vfs() { echo "Setting up VFs" diff --git a/roles/bootstrap/configure_qat/tasks/asymmetric_crypto_service_on_qat_confg.yml b/roles/bootstrap/configure_qat/tasks/check_qat_service.yml similarity index 53% rename from roles/bootstrap/configure_qat/tasks/asymmetric_crypto_service_on_qat_confg.yml rename to roles/bootstrap/configure_qat/tasks/check_qat_service.yml index 126539f1..2627423f 100644 --- a/roles/bootstrap/configure_qat/tasks/asymmetric_crypto_service_on_qat_confg.yml +++ b/roles/bootstrap/configure_qat/tasks/check_qat_service.yml @@ -1,5 +1,5 @@ ## -## Copyright (c) 2020-2021 Intel Corporation. +## Copyright (c) 2020-2022 Intel Corporation. ## ## Licensed under the Apache License, Version 2.0 (the "License"); ## you may not use this file except in compliance with the License. @@ -14,36 +14,24 @@ ## limitations under the License. ## --- -- name: confirm QAT module +- name: confirm QAT module is loaded shell: "set -o pipefail && lsmod | grep qat" args: - chdir: "/usr/src" executable: /bin/bash register: qat_confirm_mod - failed_when: '"intel_qat" not in qat_confirm_mod.stdout' changed_when: false ignore_errors: true -- name: check if QAT drivers build exists in Ubuntu >= 20.04 - stat: - path: "/usr/src/QAT_drivers/build" - register: qat_drivers_verify - when: - - ansible_distribution == "Ubuntu" and ansible_distribution_version >= '20.04' - - '"intel_qat" in qat_confirm_mod.stdout' +- name: QAT kernel module not found + fail: + msg: "No QAT module found. Please set update_qat_drivers to true in host vars to resolve the issue." + when: '"intel_qat" not in qat_confirm_mod.stdout' -- name: start QAT service, if not started in Ubuntu >= 20.04 +- name: make sure QAT service is started and enabled service: - state: started name: qat_service - when: - - ansible_distribution == "Ubuntu" and ansible_distribution_version >= '20.04' - - qat_drivers_verify.stat.exists - -- name: QAT Kernel Module not found, playbook terminated - fail: - msg: "No QAT module found. Kindly, confirm QAT module on system can also download latest drivers + services as enabling update_qat_drivers: true" - when: '"intel_qat" not in qat_confirm_mod.stdout' + state: started + enabled: yes # ansible_facts.services is not supported currently on Ubuntu 20.04, once sorted will remove and use ansible service module - name: check status of QAT service @@ -51,10 +39,10 @@ args: executable: /bin/bash register: qat_service_check - changed_when: true + changed_when: false ignore_errors: true - name: QAT Service not found, playbook terminated fail: - msg: "Failed to start qat_service on system. Kindly, download latest drivers + services this can also be achieved as enabling update_qat_drivers: true" - when: "'up' not in qat_service_check.stdout" \ No newline at end of file + msg: "Failed to start qat_service on system. Please set update_qat_drivers to true in host vars to resolve the issue." + when: "'up' not in qat_service_check.stdout" diff --git a/roles/bootstrap/configure_qat/tasks/main.yml b/roles/bootstrap/configure_qat/tasks/main.yml index 651ffd1e..7780c39e 100644 --- a/roles/bootstrap/configure_qat/tasks/main.yml +++ b/roles/bootstrap/configure_qat/tasks/main.yml @@ -1,5 +1,5 @@ ## -## Copyright (c) 2020-2021 Intel Corporation. +## Copyright (c) 2020-2022 Intel Corporation. ## ## Licensed under the Apache License, Version 2.0 (the "License"); ## you may not use this file except in compliance with the License. @@ -18,8 +18,8 @@ include_role: name: install_dependencies -- name: configure Asymmetric Crypto service on QAT devices - include_tasks: asymmetric_crypto_service_on_qat_confg.yml +- name: confirm QAT service is correctly installed on the system + include_tasks: check_qat_service.yml - name: create configuration directory if it does not exist file: @@ -30,58 +30,22 @@ - name: remove existing configuration file if it exists file: - path: "{{ sriov_config_path }}/bmra_sriov_qat_numvfs" + path: "{{ sriov_config_path }}/cek_sriov_qat_numvfs" state: absent become: yes - name: create file for QAT ids to create defined VFs file: - path: "{{ sriov_config_path }}/bmra_sriov_qat_numvfs" + path: "{{ sriov_config_path }}/cek_sriov_qat_numvfs" state: touch owner: root group: root mode: '0700' become: yes -- name: fail if requested number of VFs is higher than supported for each QAT id - assert: - that: item.qat_sriov_numvfs | default(0)| int <= 16 - fail_msg: "each qat_sriov_numvfs support max 16 Vfs. Kindly modify qat_sriov_numvfs range <= 16 in host vars for qat_devices list defined per qat_dev" - with_items: "{{ qat_devices }}" - -- name: remove existing calculated VFs file if it exists - file: - path: "{{ sriov_config_path }}/bmra_sriov_qat_total_vfs" - state: absent - become: yes - -- name: create calculated VFs config file - file: - path: "{{ sriov_config_path }}/bmra_sriov_qat_total_vfs" - state: touch - owner: root - group: root - mode: '0700' - become: yes - -- name: calculate total VFs as defined in host vars - set_fact: - total: "{{ total|default(0)|int + item.qat_sriov_numvfs | default(0)|int }}" - with_items: "{{ qat_devices }}" - -- debug: - msg: "calculated VFs = {{ total }}" - -- name: save total calculated VFs = {{ total }} to file - shell: "echo {{ total }} > bmra_sriov_qat_total_vfs" - args: - chdir: "{{ sriov_config_path }}" - executable: /bin/bash - changed_when: true - - name: populate QAT vf template with vfs per bus location lineinfile: - path: "{{ sriov_config_path }}/bmra_sriov_qat_numvfs" + path: "{{ sriov_config_path }}/cek_sriov_qat_numvfs" line: "{{ item.qat_id }} {{ item.qat_sriov_numvfs | default(0) }}" owner: root group: root @@ -91,8 +55,8 @@ - name: copy QAT SRIOV setup script to /usr/local/bin copy: - src: "{{ role_path }}/files/bmra_sriov_qat_init" - dest: /usr/local/bin/bmra_sriov_qat_init + src: "{{ role_path }}/files/cek_sriov_qat_init" + dest: /usr/local/bin/cek_sriov_qat_init owner: root group: root mode: '0700' @@ -100,17 +64,17 @@ - name: create systemd unit file template: - src: bmra_sriov_qat_init.service.j2 - dest: /lib/systemd/system/bmra_sriov_qat_init.service + src: cek_sriov_qat_init.service.j2 + dest: /lib/systemd/system/cek_sriov_qat_init.service owner: root group: root mode: '0644' become: yes -- name: ensure that systemd service is enabled on startup and restarted to apply the configuration +- name: ensure that systemd service is enabled and restarted systemd: - daemon_reload: yes - enabled: yes + name: cek_sriov_qat_init state: restarted - name: bmra_sriov_qat_init + enabled: yes + daemon_reload: yes become: yes diff --git a/roles/bootstrap/configure_qat/templates/bmra_sriov_qat_init.service.j2 b/roles/bootstrap/configure_qat/templates/bmra_sriov_qat_init.service.j2 deleted file mode 100644 index bd61723a..00000000 --- a/roles/bootstrap/configure_qat/templates/bmra_sriov_qat_init.service.j2 +++ /dev/null @@ -1,12 +0,0 @@ -[Unit] -Description=Intel BM RA SR-IOV configuration for QAT devices -AssertPathExists=/usr/local/bin/bmra_sriov_qat_init - -[Service] -Environment=SRIOV_NUMVFS_MAPPINGS={{ sriov_config_path }}/bmra_sriov_qat_numvfs -Type=oneshot -ExecStartPre=/bin/sleep 10 -ExecStart=/usr/local/bin/bmra_sriov_qat_init - -[Install] -WantedBy=multi-user.target diff --git a/roles/bootstrap/configure_qat/templates/cek_sriov_qat_init.service.j2 b/roles/bootstrap/configure_qat/templates/cek_sriov_qat_init.service.j2 new file mode 100644 index 00000000..bf982797 --- /dev/null +++ b/roles/bootstrap/configure_qat/templates/cek_sriov_qat_init.service.j2 @@ -0,0 +1,13 @@ +[Unit] +Description=Intel Container Experience Kits SR-IOV configuration for QAT devices +AssertPathExists=/usr/local/bin/cek_sriov_qat_init +Before=qat.service + +[Service] +Environment=SRIOV_NUMVFS_MAPPINGS={{ sriov_config_path }}/cek_sriov_qat_numvfs +Type=oneshot +ExecStartPre=/bin/sleep 10 +ExecStart=/usr/local/bin/cek_sriov_qat_init + +[Install] +WantedBy=multi-user.target diff --git a/roles/bootstrap/configure_qat/vars/main.yml b/roles/bootstrap/configure_qat/vars/main.yml index 0463fb0a..94f321c8 100644 --- a/roles/bootstrap/configure_qat/vars/main.yml +++ b/roles/bootstrap/configure_qat/vars/main.yml @@ -1,5 +1,5 @@ ## -## Copyright (c) 2020-2021 Intel Corporation. +## Copyright (c) 2020-2022 Intel Corporation. ## ## Licensed under the Apache License, Version 2.0 (the "License"); ## you may not use this file except in compliance with the License. @@ -20,4 +20,4 @@ install_dependencies: RedHat: - pciutils -sriov_config_path: /etc/bmra \ No newline at end of file +sriov_config_path: /etc/cek diff --git a/roles/bootstrap/configure_security/tasks/fw_debian.yaml b/roles/bootstrap/configure_security/tasks/fw_debian.yaml new file mode 100644 index 00000000..ffc7adfe --- /dev/null +++ b/roles/bootstrap/configure_security/tasks/fw_debian.yaml @@ -0,0 +1,63 @@ +## +## Copyright (c) 2020-2022 Intel Corporation. +## +## Licensed under the Apache License, Version 2.0 (the "License"); +## you may not use this file except in compliance with the License. +## You may obtain a copy of the License at +## +## http://www.apache.org/licenses/LICENSE-2.0 +## +## Unless required by applicable law or agreed to in writing, software +## distributed under the License is distributed on an "AS IS" BASIS, +## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +## See the License for the specific language governing permissions and +## limitations under the License. +## +--- +- name: open required ports in the firewall configuration on the controller nodes + command: ufw allow {{ item }} + with_items: "{{ fw_open_ports['controller'] }}" + become: yes + when: inventory_hostname in groups['kube_control_plane'] or + ( 'vm_host' in groups and inventory_hostname in groups['vm_host']) + +- name: open required ports in the firewall configuration on the worker nodes + command: ufw allow {{ item }} + with_items: "{{ fw_open_ports['node'] }}" + become: yes + when: inventory_hostname in groups['kube_node'] + +- name: allow traffic from Kubernetes subnets + command: ufw allow from {{ item }} + become: yes + changed_when: true + with_items: "{{ fw_open_subnets }}" + +- name: configure ufw to work with calico and wireguard + block: + - name: set the default forward policy to ACCEPT + lineinfile: + path: /etc/default/ufw + regexp: '^DEFAULT_FORWARD_POLICY=' + line: 'DEFAULT_FORWARD_POLICY="ACCEPT"' + mode: 0644 + + - name: get the default interface's name + shell: set pipefail -o && route | grep default | awk '{print $8}' # interface name is at the very end of line + args: + executable: /bin/bash + register: default_if + + - name: allow incoming trafiic on default interface + ufw: + rule: allow + direction: in + interface: "{{ default_if.stdout }}" + when: + - calico_backend == "bird" + - kube_network_plugin == "calico" + +- name: enable ufw + ufw: + state: enabled + become: yes diff --git a/roles/bootstrap/configure_security/tasks/fw_redhat.yaml b/roles/bootstrap/configure_security/tasks/fw_redhat.yaml new file mode 100644 index 00000000..bc114648 --- /dev/null +++ b/roles/bootstrap/configure_security/tasks/fw_redhat.yaml @@ -0,0 +1,68 @@ +## +## Copyright (c) 2020-2022 Intel Corporation. +## +## Licensed under the Apache License, Version 2.0 (the "License"); +## you may not use this file except in compliance with the License. +## You may obtain a copy of the License at +## +## http://www.apache.org/licenses/LICENSE-2.0 +## +## Unless required by applicable law or agreed to in writing, software +## distributed under the License is distributed on an "AS IS" BASIS, +## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +## See the License for the specific language governing permissions and +## limitations under the License. +## +--- +- name: open required ports in the firewall configuration on the controller + command: firewall-cmd --zone=public --add-port={{ item | regex_replace(':', '-') }} --permanent + with_items: "{{ fw_open_ports['controller'] }}" + become: yes + when: inventory_hostname in groups['kube_control_plane'] or + ( 'vm_host' in groups and inventory_hostname in groups['vm_host']) + +- name: open required ports in the firewall configuration on the node + command: firewall-cmd --zone=public --add-port={{ item | regex_replace(':', '-') }} --permanent + with_items: "{{ fw_open_ports['node'] }}" + become: yes + when: inventory_hostname in groups['kube_node'] + +- name: add Kubernetes pods and services subnets to the "trusted" zone in firewalld + command: firewall-cmd --zone=trusted --permanent --add-source={{ item }} + changed_when: true + with_items: "{{ fw_open_subnets }}" + +- name: configure firewalld to work with calico and wireguard + block: + - name: add firewall rule for calico + command: firewall-cmd --direct --permanent --add-rule ipv4 filter FORWARD 0 -m mark --mark 0x10000/0x10000 -j ACCEPT + changed_when: true + become: yes + + - name: get the default interface's name + shell: set pipefail -o && route | grep default | awk '{print $8}' # interface name is at the very end of line + args: + executable: /bin/bash + register: default_if + + - name: allow incoming trafiic on default interface + firewalld: + zone: trusted + interface: "{{ default_if.stdout }}" + permanent: yes + state: enabled + when: + - calico_backend == "bird" + - kube_network_plugin == "calico" + +- name: disable zone drifting + lineinfile: + path: /etc/firewalld/firewalld.conf + regexp: '^AllowZoneDrifting=' + line: 'AllowZoneDrifting=no' + mode: 0644 + +- name: apply firewall configuration + command: firewall-cmd --reload + changed_when: true + become: yes diff --git a/roles/bootstrap/configure_security/tasks/main.yml b/roles/bootstrap/configure_security/tasks/main.yml index 0faf22b1..7f7f9e8c 100644 --- a/roles/bootstrap/configure_security/tasks/main.yml +++ b/roles/bootstrap/configure_security/tasks/main.yml @@ -1,5 +1,5 @@ ## -## Copyright (c) 2020-2021 Intel Corporation. +## Copyright (c) 2020-2022 Intel Corporation. ## ## Licensed under the Apache License, Version 2.0 (the "License"); ## you may not use this file except in compliance with the License. @@ -14,101 +14,48 @@ ## limitations under the License. ## --- -- name: check if firewall service is enabled on the target server -# noqa 303 - systemctl is called intentionally here - command: systemctl is-enabled {{ firewall_svc[ansible_distribution] }} - register: fw_service_enabled - failed_when: fw_service_enabled.rc not in [0, 1] - become: yes - vars: - firewall_svc: - CentOS: firewalld - RedHat: firewalld - Ubuntu: ufw - changed_when: false +- name: configure firewall rules + block: + - name: ensure that firewall is running if the service is enabled + systemd: + name: "{{ firewall_svc[ansible_distribution] }}" + state: started + enabled: yes + masked: no + daemon_reload: yes + vars: + firewall_svc: + CentOS: firewalld + RedHat: firewalld + Ubuntu: ufw + become: yes -- name: register firewall service status - set_fact: - fw_enabled: "{{ fw_service_enabled.rc == 0 | default(false) | bool }}" + - name: configure firewall on RedHat distributions + include_tasks: fw_redhat.yaml + when: ansible_os_family == "RedHat" -- name: ensure that firewall is running if the service is enabled + - name: configure firewall on Debian distributions + include_tasks: fw_debian.yaml + when: ansible_os_family == "Debian" + when: firewall_enabled | default(false) | bool + +- name: make sure firewall is disabled systemd: name: "{{ firewall_svc[ansible_distribution] }}" - state: started + state: stopped + enabled: no + masked: yes + daemon_reload: yes vars: firewall_svc: CentOS: firewalld RedHat: firewalld Ubuntu: ufw - when: - - fw_enabled - become: yes - -- name: open required ports in the firewall configuration on the controller - command: firewall-cmd --zone=public --add-port={{ item | regex_replace(':', '-') }} --permanent - with_items: "{{ fw_open_ports['controller'] }}" become: yes - when: - - ansible_distribution in ["CentOS", "RedHat"] - - inventory_hostname in groups['kube_control_plane'] - - fw_enabled + when: not firewall_enabled | default(false) | bool -- name: open required ports in the firewall configuration on the node - command: firewall-cmd --zone=public --add-port={{ item | regex_replace(':', '-') }} --permanent - with_items: "{{ fw_open_ports['node'] }}" - become: yes - when: - - ansible_distribution in ["CentOS", "RedHat"] - - inventory_hostname in groups['kube_node'] - - fw_enabled - -- name: add Kubernetes pods and services subnets to the "trusted" zone in firewalld - command: firewall-cmd --zone=trusted --permanent --add-source={{ item }} - with_items: "{{ fw_open_subnets }}" - when: - - ansible_distribution in ["CentOS", "RedHat"] - - fw_enabled - -- name: apply firewall configuration - command: firewall-cmd --reload - become: yes - when: - - ansible_distribution in ["CentOS", "RedHat"] - - fw_enabled - -- name: open required ports in the firewall configuration on the controller nodes - command: ufw allow {{ item }} - with_items: "{{ fw_open_ports['controller'] }}" - become: yes - when: - - ansible_distribution == "Ubuntu" - - inventory_hostname in groups['kube_control_plane'] - - fw_enabled - -- name: open required ports in the firewall configuration on the worker nodes - command: ufw allow {{ item }} - with_items: "{{ fw_open_ports['node'] }}" - become: yes - when: - - ansible_distribution == "Ubuntu" - - inventory_hostname in groups['kube_node'] - - fw_enabled - -- name: allow traffic from Kubernetes subnets - command: ufw allow from {{ item }} - become: yes - with_items: "{{ fw_open_subnets }}" - when: - - ansible_distribution == "Ubuntu" - - fw_enabled - -- name: enable ufw - ufw: - state: enabled - become: yes - when: - - ansible_distribution == "Ubuntu" - - fw_enabled +- name: configure SELinux + include_tasks: selinux.yaml # fix for file size limit bug on RHEL-based distros, please see https://access.redhat.com/solutions/33993 - name: fix file size limit settings bug @@ -124,22 +71,3 @@ root hard fsize unlimited root soft nofile unlimited root hard nofile unlimited - -- name: enable selinux - selinux: - state: enabled - notify: reboot server - when: - - ansible_os_family == "RedHat" - - ansible_selinux.status != "enabled" - - selinux_state is defined and selinux_state == "enabled" - -- name: disable selinux - selinux: - state: disabled - notify: reboot server - when: - - ansible_os_family == "RedHat" - - ansible_selinux.status != "disabled" - - (selinux_state is defined and selinux_state == "disabled") or - (container_runtime == "docker" and ansible_distribution_version >= '8.4') \ No newline at end of file diff --git a/roles/bootstrap/configure_security/tasks/selinux.yaml b/roles/bootstrap/configure_security/tasks/selinux.yaml new file mode 100644 index 00000000..6d95a01a --- /dev/null +++ b/roles/bootstrap/configure_security/tasks/selinux.yaml @@ -0,0 +1,34 @@ +## +## Copyright (c) 2020-2022 Intel Corporation. +## +## Licensed under the Apache License, Version 2.0 (the "License"); +## you may not use this file except in compliance with the License. +## You may obtain a copy of the License at +## +## http://www.apache.org/licenses/LICENSE-2.0 +## +## Unless required by applicable law or agreed to in writing, software +## distributed under the License is distributed on an "AS IS" BASIS, +## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +## See the License for the specific language governing permissions and +## limitations under the License. +## +--- +- name: enable selinux + selinux: + state: enabled + notify: reboot server + when: + - ansible_os_family == "RedHat" + - ansible_selinux.status != "enabled" + - selinux_state is defined and selinux_state == "enabled" + +- name: disable selinux + selinux: + state: disabled + notify: reboot server + when: + - ansible_os_family == "RedHat" + - ansible_selinux.status != "disabled" + - (selinux_state is defined and selinux_state == "disabled") or + (container_runtime == "docker" and ansible_distribution_version >= '8.4') diff --git a/roles/bootstrap/configure_security/vars/main.yml b/roles/bootstrap/configure_security/vars/main.yml index de5c0d1e..46b4b4b1 100644 --- a/roles/bootstrap/configure_security/vars/main.yml +++ b/roles/bootstrap/configure_security/vars/main.yml @@ -1,5 +1,5 @@ ## -## Copyright (c) 2020-2021 Intel Corporation. +## Copyright (c) 2020-2022 Intel Corporation. ## ## Licensed under the Apache License, Version 2.0 (the "License"); ## you may not use this file except in compliance with the License. @@ -61,6 +61,7 @@ fw_open_ports: - 8285/udp - 8472/udp + fw_open_subnets: - "{{ kube_pods_subnet }}" - "{{ kube_service_addresses }}" diff --git a/roles/bootstrap/configure_sgx/defaults/main.yml b/roles/bootstrap/configure_sgx/defaults/main.yml new file mode 100644 index 00000000..d7f5baa0 --- /dev/null +++ b/roles/bootstrap/configure_sgx/defaults/main.yml @@ -0,0 +1,69 @@ +## +## Copyright (c) 2020-2022 Intel Corporation. +## +## Licensed under the Apache License, Version 2.0 (the "License"); +## you may not use this file except in compliance with the License. +## You may obtain a copy of the License at +## +## http://www.apache.org/licenses/LICENSE-2.0 +## +## Unless required by applicable law or agreed to in writing, software +## distributed under the License is distributed on an "AS IS" BASIS, +## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +## See the License for the specific language governing permissions and +## limitations under the License. +## +--- +# Intel SGX-DCAP drivers module for Ubuntu 20.04 +dcap_driver_series_ubuntu_20: "1.41" +dcap_driver_version_ubuntu_20: "sgx_linux_x64_driver_{{ dcap_driver_series_ubuntu_20 }}.bin" +dcap_driver_url_ubuntu_20: "https://download.01.org/intel-sgx/sgx-dcap/1.12.1/linux/distro/ubuntu20.04-server/{{ dcap_driver_version_ubuntu_20 }}" +dcap_driver_checksum_ubuntu_20: "sha256:f279228259629103b12de41aba0326ed3d77f4e7bf647484d4abad2f29f355c3" +sgx_folder_check_ubuntu_20: "{{ project_root_dir }}/sgx-{{ dcap_driver_series_ubuntu_20 }}" + +sgx_sdk_version_ubuntu_20: "sgx_linux_x64_sdk_2.15.101.1.bin" +sgx_sdk_url_ubuntu_20: "https://download.01.org/intel-sgx/sgx-dcap/1.12.1/linux/distro/ubuntu20.04-server/{{ sgx_sdk_version_ubuntu_20 }}" +sgx_sdk_checksum_ubuntu_20: "sha256:21c633c271eafb3a1d15d604526ec2f827a4606ef91bac78a756bae7bed73dac" + +# Intel SGX-SGX Key configuration for Ubuntu >= 18.04.4 +sgx_apt_source_list: "intel-sgx" +sgx_apt_repo_url: "https://download.01.org/intel-sgx/sgx_repo/ubuntu" +sgx_apt_repo_key: "{{ sgx_apt_repo_url }}/intel-sgx-deb.key" + +# Intel SGX-DCAP drivers module for <= CentOS 8.3 +dcap_driver_series_centos: "1.41" +dcap_driver_version_centos: "sgx_linux_x64_driver_{{ dcap_driver_series_centos }}.bin" +dcap_driver_url_centos: "https://download.01.org/intel-sgx/sgx-dcap/1.12.1/linux/distro/centos8.2-server/{{ dcap_driver_version_centos }}" +dcap_driver_checksum_centos: "sha256:63d1e11943284a75322d50755e57ae25b154c46bcbbbb04583e9e48b8ec357fe" +sgx_folder_check_centos: "{{ project_root_dir }}/sgx-{{ dcap_driver_series_centos }}" + +sgx_sdk_version_centos: "sgx_linux_x64_sdk_2.15.101.1.bin" +sgx_sdk_url_centos: "https://download.01.org/intel-sgx/sgx-dcap/1.12.1/linux/distro/centos8.2-server/{{ sgx_sdk_version_centos }}" +sgx_sdk_checksum_centos: "sha256:b8db5d1e8730474b1d96bf9aac8aea070a2f65f31624d074383d906dfa34ef7a" + +# Intel SGX-DCAP drivers module for <= RHEL 8.3 +dcap_driver_series_rhel: "1.41" +dcap_driver_version_rhel: "sgx_linux_x64_driver_{{ dcap_driver_series_rhel }}.bin" +dcap_driver_url_rhel: "https://download.01.org/intel-sgx/sgx-dcap/1.12.1/linux/distro/rhel8.2-server/{{ dcap_driver_version_rhel }}" +dcap_driver_checksum_rhel: "sha256:efd41d8a8ef515b91a64077ee95d983450eb3e0abf54675f1e9446a270c408e1" +sgx_folder_check_rhel: "{{ project_root_dir }}/sgx-{{ dcap_driver_series_centos }}" + +sgx_sdk_version_rhel: "sgx_linux_x64_sdk_2.15.101.1.bin" +sgx_sdk_url_rhel: "https://download.01.org/intel-sgx/sgx-dcap/1.12.1/linux/distro/rhel8.2-server/{{ sgx_sdk_version_rhel }}" +sgx_sdk_checksum_rhel: "sha256:632c4306439a968590b658c1f83a9279cdded5d215d1627127709e6c07d320ce" + +# Intel SGX RPM local repository for CentOS <= 8.3 +sgx_rpm_local_repo_version_centos: "sgx_rpm_local_repo.tgz" +sgx_rpm_local_repo_url_centos: "https://download.01.org/intel-sgx/sgx-dcap/1.12.1/linux/distro/centos8.2-server/{{ sgx_rpm_local_repo_version_centos }}" +sgx_rpm_local_repo_checksum_centos: "sha256:aa78100ff58dd1dc8f763ae209d07ae7709587f56bebd90c8a0ea60cbe4fb715" + +# Intel SGX RPM local repository for RHEL <= 8.3 +sgx_rpm_local_repo_version_rhel: "sgx_rpm_local_repo.tgz" +sgx_rpm_local_repo_url_rhel: "https://download.01.org/intel-sgx/sgx-dcap/1.12.1/linux/distro/rhel8.2-server/{{ sgx_rpm_local_repo_version_rhel }}" +sgx_rpm_local_repo_checksum_rhel: "sha256:7f85db7b44721c03c361c3855cd890fe5198dcf19bc12bba8a8a6908e1f87163" + +sgx_config_dir: "{{ project_root_dir }}" +sgx_rpm_directory: "{{ project_root_dir }}/sgx_rpm_local_repo" + +sgx_pkg_version: "2.15.101.1" +sgx_pkg_dcap_version: "1.12.101.1" diff --git a/roles/sgx_configure_icx/tasks/centos.yml b/roles/bootstrap/configure_sgx/tasks/centos.yml similarity index 77% rename from roles/sgx_configure_icx/tasks/centos.yml rename to roles/bootstrap/configure_sgx/tasks/centos.yml index 3347ba46..d82c5d70 100644 --- a/roles/sgx_configure_icx/tasks/centos.yml +++ b/roles/bootstrap/configure_sgx/tasks/centos.yml @@ -1,5 +1,5 @@ ## -## Copyright (c) 2020-2021 Intel Corporation. +## Copyright (c) 2020-2022 Intel Corporation. ## ## Licensed under the Apache License, Version 2.0 (the "License"); ## you may not use this file except in compliance with the License. @@ -18,52 +18,39 @@ include_role: name: install_dependencies -- name: check CPU/BIOS is enabled for SGX - shell: - cmd: cpuid | grep -i sgx | grep -v ENCL - register: cpuid_output - failed_when: '"false" in cpuid_output.stdout' - changed_when: false - - name: download DCAP drivers get_url: url: "{{ dcap_driver_url_centos }}" - dest: "/usr/src/" + dest: "{{ project_root_dir }}" mode: u=rwx,g=rx,o=rx checksum: "{{ dcap_driver_checksum_centos }}" - when: ansible_distribution == "CentOS" - name: check if sgx-{{ dcap_driver_series_centos }} folder is present stat: path: "{{ sgx_folder_check_centos }}" register: folder_check - when: ansible_distribution == "CentOS" changed_when: true - debug: msg: "sgx-{{ dcap_driver_series_centos }} is already installed because folder exists, re-installation skipped as it can cause kernel module load issues" - when: - - ansible_distribution == "CentOS" - - folder_check.stat.exists + when: folder_check.stat.exists changed_when: true - name: install DCAP driver # noqa 305 - shell is used intentionally here shell: "./{{ dcap_driver_version_centos }}" args: - chdir: "/usr/src" + chdir: "{{ project_root_dir }}" executable: /bin/bash register: dcap_output_centos failed_when: '"Installation is successful!" not in dcap_output_centos.stdout' changed_when: '"Installation is successful!" in dcap_output_centos.stdout' - when: - - ansible_distribution == "CentOS" - - not folder_check.stat.exists + when: not folder_check.stat.exists - name: confirm DCAP driver module is installed shell: "set -o pipefail && lsmod | grep sgx" args: - chdir: "/usr/src" + chdir: "{{ project_root_dir }}" executable: /bin/bash register: lsmod_output failed_when: '"intel_sgx" not in lsmod_output.stdout' @@ -75,7 +62,7 @@ - name: download SGX RPM local repository get_url: url: "{{ sgx_rpm_local_repo_url_centos }}" - dest: "/usr/src/" + dest: "{{ project_root_dir }}" mode: u=rwx,g=rx,o=rx checksum: "{{ sgx_rpm_local_repo_checksum_centos }}" @@ -87,24 +74,22 @@ - name: unpack SGX RPM local repository unarchive: - src: "/usr/src/{{ sgx_rpm_local_repo_version_centos }}" + src: "{{ project_root_dir }}/{{ sgx_rpm_local_repo_version_centos }}" dest: "{{ sgx_rpm_directory }}" extra_opts: "--strip-components=1" mode: u=rwx,g=rx,o=rx remote_src: yes - when: ansible_distribution == "CentOS" - name: add SGX RPM local repository command: "yum-config-manager --add-repo={{ sgx_rpm_directory }}" - when: ansible_distribution == "CentOS" + changed_when: true - name: disable gpg check in local sgx repo lineinfile: - dest: "/etc/yum.repos.d/usr_src_sgx_rpm_local_repo.repo" + dest: "/etc/yum.repos.d/opt_cek_sgx_rpm_local_repo.repo" insertafter: 'EOF' line: 'gpgcheck=0' mode: '0644' - when: ansible_distribution == "CentOS" - name: install sgx platform sw package: @@ -120,7 +105,6 @@ - sgx-aesm-service - libsgx-dcap-ql state: present - when: ansible_distribution == "CentOS" - name: start aesmd service, if not started service: @@ -130,37 +114,33 @@ - name: get aesmd service facts service_facts: register: service_info - when: ansible_distribution == "CentOS" - debug: var: service_info.ansible_facts.services['aesmd.service'].state failed_when: "'running' not in service_info.ansible_facts.services['aesmd.service'].state" - when: ansible_distribution == "CentOS" - name: download sgx sdk get_url: url: "{{ sgx_sdk_url_centos }}" - dest: "/usr/src/" + dest: "{{ project_root_dir }}" mode: u=rwx,g=rx,o=rx checksum: "{{ sgx_sdk_checksum_centos }}" - when: ansible_distribution == "CentOS" - name: install sgx sdk shell: "set -o pipefail && echo 'yes' | ./{{ sgx_sdk_version_centos }}" args: - chdir: "/usr/src" + chdir: "{{ project_root_dir }}" executable: /bin/bash register: sdk_output failed_when: '"Installation is successful!" not in sdk_output.stdout' changed_when: '"Installation is successful!" in sdk_output.stdout' - when: ansible_distribution == "CentOS" - name: setup environment & make code samples in hardware mode to ensure package works well shell: > - source /usr/src/sgxsdk/environment && + source {{ project_root_dir }}/sgxsdk/environment && make args: - chdir: "/usr/src/sgxsdk/SampleCode/LocalAttestation" + chdir: "{{ project_root_dir }}/sgxsdk/SampleCode/LocalAttestation" executable: /bin/bash register: make_confirm failed_when: '"The project has been built in hardware debug mode." not in make_confirm.stdout' @@ -169,7 +149,7 @@ - name: ensure that sgx driver / psw is installed in hardware mode with code samples command: ./app args: - chdir: /usr/src/sgxsdk/SampleCode/LocalAttestation/bin + chdir: "{{ project_root_dir }}/sgxsdk/SampleCode/LocalAttestation/bin" register: psw_confirm changed_when: '"Succeed" in psw_confirm.stdout' diff --git a/roles/bootstrap/configure_sgx/tasks/main.yml b/roles/bootstrap/configure_sgx/tasks/main.yml new file mode 100644 index 00000000..523c93ae --- /dev/null +++ b/roles/bootstrap/configure_sgx/tasks/main.yml @@ -0,0 +1,68 @@ +## +## Copyright (c) 2020-2022 Intel Corporation. +## +## Licensed under the Apache License, Version 2.0 (the "License"); +## you may not use this file except in compliance with the License. +## You may obtain a copy of the License at +## +## http://www.apache.org/licenses/LICENSE-2.0 +## +## Unless required by applicable law or agreed to in writing, software +## distributed under the License is distributed on an "AS IS" BASIS, +## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +## See the License for the specific language governing permissions and +## limitations under the License. +## +--- +- name: determine machine type + include_role: + name: check_machine_type + +- name: install dependencies - cpuid + package: + name: cpuid + state: present + when: is_icx or is_spr + +- name: check CPU/BIOS is enabled for SGX + shell: set -o pipefail && cpuid | grep -i sgx | grep -v ENCL + args: + executable: /bin/bash + register: cpuid_output + when: is_icx or is_spr + changed_when: false + +- name: SGX is not enabled in BIOS + fail: + msg: + - "Please enable all required options for Intel SGX in BIOS." + - "If failure persists, check with your system vendor." + when: + - is_icx or is_spr + - "'false' in cpuid_output.stdout" + +- name: configure SGX on Ubuntu distribution + include_tasks: ubuntu.yml + when: + - ansible_distribution == 'Ubuntu' and ansible_distribution_version < '21.04' + - is_icx or is_spr + +- name: configure SGX on CentOS distribution + include_tasks: centos.yml + when: + - ansible_distribution == "CentOS" and ansible_distribution_version < '8.4' + - is_icx or is_spr + +- name: configure SGX on RHEL distribution + include_tasks: rhel.yml + when: + - ansible_distribution == 'RedHat' and ansible_distribution_version < '8.4' + - is_icx or is_spr + +- name: SGX configuration is successful + debug: + msg: + - "The BIOS check passed..." + - "The system is properly configured..." + - "Intel SGX Device Plugin may be deployed now!" + when: is_icx or is_spr diff --git a/roles/sgx_configure_icx/tasks/rhel.yml b/roles/bootstrap/configure_sgx/tasks/rhel.yml similarity index 78% rename from roles/sgx_configure_icx/tasks/rhel.yml rename to roles/bootstrap/configure_sgx/tasks/rhel.yml index 9fbf2aeb..24b11332 100644 --- a/roles/sgx_configure_icx/tasks/rhel.yml +++ b/roles/bootstrap/configure_sgx/tasks/rhel.yml @@ -1,5 +1,5 @@ ## -## Copyright (c) 2020-2021 Intel Corporation. +## Copyright (c) 2020-2022 Intel Corporation. ## ## Licensed under the Apache License, Version 2.0 (the "License"); ## you may not use this file except in compliance with the License. @@ -18,26 +18,17 @@ include_role: name: install_dependencies -- name: check CPU/BIOS is enabled for SGX - shell: - cmd: cpuid | grep -i sgx | grep -v ENCL - register: cpuid_output - failed_when: '"false" in cpuid_output.stdout' - changed_when: false - - name: download DCAP drivers get_url: url: "{{ dcap_driver_url_rhel }}" - dest: "/usr/src/" + dest: "{{ project_root_dir }}" mode: u=rwx,g=rx,o=rx checksum: "{{ dcap_driver_checksum_rhel }}" - when: ansible_distribution == "RedHat" - name: check if sgx-{{ dcap_driver_series_rhel }} folder is present stat: path: "{{ sgx_folder_check_rhel }}" register: folder_check - when: ansible_distribution == "RedHat" changed_when: true - debug: @@ -51,19 +42,17 @@ # noqa 305 - shell is used intentionally here shell: "./{{ dcap_driver_version_rhel }}" args: - chdir: "/usr/src" + chdir: "{{ project_root_dir }}" executable: /bin/bash register: dcap_output_rhel failed_when: '"Installation is successful!" not in dcap_output_rhel.stdout' changed_when: '"Installation is successful!" in dcap_output_rhel.stdout' - when: - - ansible_distribution == "RedHat" - - not folder_check.stat.exists + when: not folder_check.stat.exists - name: confirm DCAP driver module is installed shell: "set -o pipefail && lsmod | grep sgx" args: - chdir: "/usr/src" + chdir: "{{ project_root_dir }}" executable: /bin/bash register: lsmod_output failed_when: '"intel_sgx" not in lsmod_output.stdout' @@ -75,7 +64,7 @@ - name: download SGX RPM local repository get_url: url: "{{ sgx_rpm_local_repo_url_rhel }}" - dest: "/usr/src/" + dest: "{{ project_root_dir }}" mode: u=rwx,g=rx,o=rx checksum: "{{ sgx_rpm_local_repo_checksum_rhel }}" @@ -87,24 +76,22 @@ - name: unpack SGX RPM local repository unarchive: - src: "/usr/src/{{ sgx_rpm_local_repo_version_rhel }}" + src: "{{ project_root_dir }}/{{ sgx_rpm_local_repo_version_rhel }}" dest: "{{ sgx_rpm_directory }}" extra_opts: "--strip-components=1" mode: u=rwx,g=rx,o=rx remote_src: yes - when: ansible_distribution == "RedHat" - name: add SGX RPM local repository command: "yum-config-manager --add-repo={{ sgx_rpm_directory }}" - when: ansible_distribution == "RedHat" + changed_when: true - name: disable gpg check in local sgx repo lineinfile: - dest: "/etc/yum.repos.d/usr_src_sgx_rpm_local_repo.repo" + dest: "/etc/yum.repos.d/opt_cek_sgx_rpm_local_repo.repo" insertafter: 'EOF' line: 'gpgcheck=0' mode: '0644' - when: ansible_distribution == "RedHat" - name: install sgx platform sw package: @@ -120,7 +107,6 @@ - sgx-aesm-service - libsgx-dcap-ql state: present - when: ansible_distribution == "RedHat" - name: start aesmd service, if not started service: @@ -130,37 +116,33 @@ - name: get aesmd service facts service_facts: register: service_info - when: ansible_distribution == "RedHat" - debug: var: service_info.ansible_facts.services['aesmd.service'].state failed_when: "'running' not in service_info.ansible_facts.services['aesmd.service'].state" - when: ansible_distribution == "RedHat" - name: download sgx sdk get_url: url: "{{ sgx_sdk_url_rhel }}" - dest: "/usr/src/" + dest: "{{ project_root_dir }}" mode: u=rwx,g=rx,o=rx checksum: "{{ sgx_sdk_checksum_rhel }}" - when: ansible_distribution == "RedHat" - name: install sgx sdk shell: "set -o pipefail && echo 'yes' | ./{{ sgx_sdk_version_rhel }}" args: - chdir: "/usr/src" + chdir: "{{ project_root_dir }}" executable: /bin/bash register: sdk_output failed_when: '"Installation is successful!" not in sdk_output.stdout' changed_when: '"Installation is successful!" in sdk_output.stdout' - when: ansible_distribution == "RedHat" - name: setup environment & make code samples in hardware mode to ensure package works well shell: > - source /usr/src/sgxsdk/environment && + source {{ project_root_dir }}/sgxsdk/environment && make args: - chdir: "/usr/src/sgxsdk/SampleCode/LocalAttestation" + chdir: "{{ project_root_dir }}/sgxsdk/SampleCode/LocalAttestation" executable: /bin/bash register: make_confirm failed_when: '"The project has been built in hardware debug mode." not in make_confirm.stdout' @@ -169,7 +151,7 @@ - name: ensure that sgx driver / psw is installed in hardware mode with code samples command: ./app args: - chdir: /usr/src/sgxsdk/SampleCode/LocalAttestation/bin + chdir: "{{ project_root_dir }}/sgxsdk/SampleCode/LocalAttestation/bin" register: psw_confirm changed_when: '"Succeed" in psw_confirm.stdout' diff --git a/roles/sgx_configure_icx/tasks/ubuntu.yml b/roles/bootstrap/configure_sgx/tasks/ubuntu.yml similarity index 64% rename from roles/sgx_configure_icx/tasks/ubuntu.yml rename to roles/bootstrap/configure_sgx/tasks/ubuntu.yml index 73196174..09c8e1e3 100644 --- a/roles/sgx_configure_icx/tasks/ubuntu.yml +++ b/roles/bootstrap/configure_sgx/tasks/ubuntu.yml @@ -1,5 +1,5 @@ ## -## Copyright (c) 2020-2021 Intel Corporation. +## Copyright (c) 2020-2022 Intel Corporation. ## ## Licensed under the Apache License, Version 2.0 (the "License"); ## you may not use this file except in compliance with the License. @@ -18,13 +18,6 @@ include_role: name: install_dependencies -- name: check CPU/BIOS is enabled for SGX - shell: - cmd: cpuid | grep -i sgx | grep -v ENCL - register: cpuid_output - failed_when: '"false" in cpuid_output.stdout' - changed_when: false - - name: create configuration directory if it does not exist file: path: "{{ sgx_config_dir }}" @@ -32,54 +25,17 @@ mode: '0700' become: yes -- name: download DCAP drivers on Ubuntu <= 18.04.4 - get_url: - url: "{{ dcap_driver_url_ubuntu_18 }}" - dest: "{{ sgx_config_dir }}" - mode: u=rwx,g=rx,o=rx - checksum: "{{ dcap_driver_checksum_ubuntu_18 }}" - when: ansible_distribution == "Ubuntu" and ansible_distribution_version <= '18.04.4' - -- name: check if sgx-{{ dcap_driver_series_ubuntu_18 }} folder present on Ubuntu <= 18.04.4 - stat: - path: "{{ sgx_folder_check_ubuntu_18 }}" - register: folder_check - when: ansible_distribution == "Ubuntu" and ansible_distribution_version <= '18.04.4' - changed_when: true - -- debug: - msg: "sgx-{{ dcap_driver_series_ubuntu_18 }} is already installed because folder exists, re-installation skipped as it can cause kernel module load issues" - when: - - ansible_distribution == "Ubuntu" and ansible_distribution_version <= '18.04.4' - - folder_check.stat.exists - changed_when: true - -- name: install DCAP driver on Ubuntu <= 18.04.4 -# noqa 305 - shell is used intentionally here - shell: "./{{ dcap_driver_version_ubuntu_18 }}" - args: - chdir: "{{ sgx_config_dir }}" - executable: /bin/bash - register: dcap_output_ubuntu_18 - failed_when: '"Installation is successful!" not in dcap_output_ubuntu_18.stdout' - changed_when: '"Installation is successful!" in dcap_output_ubuntu_18.stdout' - when: - - ansible_distribution == "Ubuntu" and ansible_distribution_version <= '18.04.4' - - not folder_check.stat.exists - - name: download DCAP drivers on Ubuntu 20.04 get_url: url: "{{ dcap_driver_url_ubuntu_20 }}" dest: "{{ sgx_config_dir }}" mode: u=rwx,g=rx,o=rx checksum: "{{ dcap_driver_checksum_ubuntu_20 }}" - when: ansible_distribution == "Ubuntu" and ansible_distribution_version == '20.04' - name: check if sgx-{{ dcap_driver_series_ubuntu_20 }} folder present on Ubuntu 20.04 stat: path: "{{ sgx_folder_check_ubuntu_20 }}" register: folder_re_check - when: ansible_distribution == "Ubuntu" and ansible_distribution_version == '20.04' changed_when: true - debug: @@ -122,13 +78,11 @@ - name: add {{ sgx_apt_source_list }} repo key apt_key: url: "{{ sgx_apt_repo_key }}" - when: ansible_distribution == "Ubuntu" - name: add sgx apt repository apt_repository: repo: "deb {{ sgx_apt_repo_url }} {{ ansible_distribution_release }} main" filename: "{{ sgx_apt_source_list }}" - when: ansible_distribution == "Ubuntu" - name: install sgx platform sw apt: @@ -151,54 +105,22 @@ - libsgx-dcap-default-qpl-dev={{ sgx_pkg_dcap_version }}-{{ ansible_distribution_release }}1 - libsgx-dcap-quote-verify={{ sgx_pkg_dcap_version }}-{{ ansible_distribution_release }}1 - libsgx-dcap-quote-verify-dev={{ sgx_pkg_dcap_version }}-{{ ansible_distribution_release }}1 - when: - - ansible_distribution == "Ubuntu" - name: start aesmd service, if not started service: state: started name: aesmd -- name: get aesmd service facts - service_facts: - register: service_info - when: ansible_distribution == "Ubuntu" and ansible_distribution_version <= '18.04.4' - -- debug: - var: service_info.ansible_facts.services['aesmd.service'].state - failed_when: "'running' not in service_info.ansible_facts.services['aesmd.service'].state" - when: ansible_distribution == "Ubuntu" and ansible_distribution_version <= '18.04.4' - # ansible_facts.services is not supported currently on Ubuntu 20.04, once sorted will remove when conditions and merge code as one task. - name: check status of aesmd service after started on Ubuntu 20.04 command: systemctl status aesmd.service args: warn: false register: aesmd_enabled - when: ansible_distribution == "Ubuntu" and ansible_distribution_version == '20.04' changed_when: true - debug: var: aesmd_enabled.stdout_lines - when: ansible_distribution == "Ubuntu" and ansible_distribution_version == '20.04' - -- name: download sgx sdk on Ubuntu <= 18.04.4 - get_url: - url: "{{ sgx_sdk_url_ubuntu_18 }}" - dest: "{{ sgx_config_dir }}" - mode: u=rwx,g=rx,o=rx - checksum: "{{ sgx_sdk_checksum_ubuntu_18 }}" - when: ansible_distribution == "Ubuntu" and ansible_distribution_version <= '18.04.4' - -- name: install sgx sdk on Ubuntu <= 18.04.4 - shell: "set -o pipefail && echo 'yes' | ./{{ sgx_sdk_version_ubuntu_18 }}" - args: - chdir: "{{ sgx_config_dir }}" - executable: /bin/bash - register: sdk_output - failed_when: '"Installation is successful!" not in sdk_output.stdout' - changed_when: '"Installation is successful!" in sdk_output.stdout' - when: ansible_distribution == "Ubuntu" and ansible_distribution_version <= '18.04.4' - name: download sgx sdk on Ubuntu 20.04 get_url: @@ -206,7 +128,6 @@ dest: "{{ sgx_config_dir }}" mode: u=rwx,g=rx,o=rx checksum: "{{ sgx_sdk_checksum_ubuntu_20 }}" - when: ansible_distribution == "Ubuntu" and ansible_distribution_version == '20.04' - name: install sgx sdk on Ubuntu 20.04 shell: "set -o pipefail && echo 'yes' | ./{{ sgx_sdk_version_ubuntu_20 }}" @@ -216,7 +137,6 @@ register: sdk_output failed_when: '"Installation is successful!" not in sdk_output.stdout' changed_when: '"Installation is successful!" in sdk_output.stdout' - when: ansible_distribution == "Ubuntu" and ansible_distribution_version == '20.04' - name: setup environment & make code samples in hardware mode to ensure package works well shell: > diff --git a/roles/sgx_configure_icx/vars/main.yml b/roles/bootstrap/configure_sgx/vars/main.yml similarity index 94% rename from roles/sgx_configure_icx/vars/main.yml rename to roles/bootstrap/configure_sgx/vars/main.yml index 4ce78848..10e38b34 100644 --- a/roles/sgx_configure_icx/vars/main.yml +++ b/roles/bootstrap/configure_sgx/vars/main.yml @@ -1,5 +1,5 @@ ## -## Copyright (c) 2020-2021 Intel Corporation. +## Copyright (c) 2020-2022 Intel Corporation. ## ## Licensed under the Apache License, Version 2.0 (the "License"); ## you may not use this file except in compliance with the License. @@ -16,7 +16,6 @@ --- install_dependencies: Debian: - - cpuid - ocaml - automake - autoconf @@ -41,7 +40,6 @@ install_dependencies: - git - cmake - perl - - cpuid - "@Development Tools" - createrepo - libcurl-devel diff --git a/roles/bootstrap/configure_sst/defaults/main.yml b/roles/bootstrap/configure_sst/defaults/main.yml index d97e533d..5c4ff677 100644 --- a/roles/bootstrap/configure_sst/defaults/main.yml +++ b/roles/bootstrap/configure_sst/defaults/main.yml @@ -1,5 +1,5 @@ ## -## Copyright (c) 2020-2021 Intel Corporation. +## Copyright (c) 2020-2022 Intel Corporation. ## ## Licensed under the Apache License, Version 2.0 (the "License"); ## you may not use this file except in compliance with the License. @@ -14,15 +14,9 @@ ## limitations under the License. ## --- -sst_bf_configuration_enabled: true -sst_cp_configuration_enabled: true -sst_tf_configuration_enabled: true - -icx_centos8_2_load_isst_modules: true - clx_sst_bf_git_url: "https://github.com/intel/CommsPowerManagement.git" clx_sst_bf_commit_hash: "a3a1869fd88eff5b2b872f447ca69b866e3d318e" -clx_sst_bf_dir: "/usr/src/CommsPowerManagement" +clx_sst_bf_dir: "{{ project_root_dir }}/CommsPowerManagement" clx_sst_bf_exec: "/usr/local/bin/sst_bf.py" sst_cp_priority_type: "1" # 0 - Proportional, 1 - Ordered @@ -49,5 +43,6 @@ sst_cp_cpu_clos: # assign required values to CLOS group after priority type setu - clos: 1 cpus: 3,7 -source_rpm_directory: "/usr/src/rpms" -rpmbuild_dir: "{{ source_rpm_directory }}/rpmbuild" +isst_tool_git_url: https://github.com/torvalds/linux.git +isst_tool_git_version: v5.16 +isst_tool_src_dir: "{{ (project_root_dir, 'speedselect') | path_join }}" diff --git a/roles/bootstrap/configure_sst/tasks/centos8_2_install_isst_interface.yml b/roles/bootstrap/configure_sst/tasks/centos8_2_install_isst_interface.yml deleted file mode 100644 index c898fd62..00000000 --- a/roles/bootstrap/configure_sst/tasks/centos8_2_install_isst_interface.yml +++ /dev/null @@ -1,158 +0,0 @@ -## -## Copyright (c) 2020-2021 Intel Corporation. -## -## Licensed under the Apache License, Version 2.0 (the "License"); -## you may not use this file except in compliance with the License. -## You may obtain a copy of the License at -## -## http://www.apache.org/licenses/LICENSE-2.0 -## -## Unless required by applicable law or agreed to in writing, software -## distributed under the License is distributed on an "AS IS" BASIS, -## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -## See the License for the specific language governing permissions and -## limitations under the License. -## ---- -- name: install dependencies - package: - name: - - ncurses-devel - - hmaccalc - - zlib-devel - - binutils-devel - - elfutils-libelf-devel - - audit-libs-devel - - bc - - elfutils-devel - - java-devel - - kabi-dw - - libbpf-devel - - libcap-devel - - libcap-ng-devel - - llvm-toolset - - newt-devel - - openssl-devel - - pciutils-devel - - perl(ExtUtils::Embed) - - perl-devel - - python3-devel - - python3-docutils - - xmlto - - xz-devel - - rsync - - "@Development tools" - state: present - -- name: install dwarves needed for kernel requirements - command: "dnf --enablerepo=PowerTools install dwarves -y" - changed_when: true - -- name: install libbpf-devel needed for kernel requirements - command: "dnf --enablerepo=PowerTools install libbpf-devel -y" - changed_when: true - -- name: create directories for kernel compilation - file: - path: "{{ rpmbuild_dir }}/{{ item }}" - state: directory - mode: '0755' - owner: root - group: root - loop: - - BUILD - - BUILDROOT - - RPMS - - SOURCES - - SPECS - - SRPMS - -- name: create rpmmacros file - copy: - content: "%_topdir {{ source_rpm_directory }}/rpmbuild\n" - dest: "{{ ansible_env.HOME }}/.rpmmacros" - mode: '0755' - owner: root - group: root - -- name: read kernel version - set_fact: - kernel_version: "{{ ansible_kernel | regex_replace('.x86_64', '') }}" - -# NOTE(skonefal): dnf module refuse to install source package. -- name: pull and unpack kernel sources - command: - cmd: "rpm -ivh http://vault.centos.org/8.2.2004/BaseOS/Source/SPackages/kernel-{{ kernel_version }}.src.rpm" - warn: false - args: - creates: "{{ source_rpm_directory }}/{{ kernel_version }}-source-installed" - -- name: rpmbuild kernel.spec - command: - cmd: rpmbuild -bp --target=x86_64 kernel.spec - args: - chdir: "{{ rpmbuild_dir }}/SPECS" - creates: "{{ source_rpm_directory }}/{{ kernel_version }}-rpmbuild-done" - -- name: set kernel source dir fact - set_fact: - kernel_source_code_dir: "{{ rpmbuild_dir }}/BUILD/kernel-{{ kernel_version }}/linux-{{ ansible_kernel }}" - -- name: copy current kernel config - make: - chdir: "{{ kernel_source_code_dir }}" - target: oldconfig - -- name: enable INTEL_SPEED_SELECT_INTERFACE as a kernel module - lineinfile: - path: "{{ kernel_source_code_dir }}/.config" - state: present - regexp: '# CONFIG_INTEL_SPEED_SELECT_INTERFACE is not set' - line: 'CONFIG_INTEL_SPEED_SELECT_INTERFACE=m' - mode: '0755' - owner: root - group: root - -- name: make prepare - make: - chdir: "{{ kernel_source_code_dir }}" - target: prepare - -- name: make modules prepare - make: - chdir: "{{ kernel_source_code_dir }}" - target: modules_prepare - -- name: compile ISST module - make: - chdir: "{{ kernel_source_code_dir }}" - params: - M: "drivers/platform/x86/intel_speed_select_if" - -- name: install the isst-load-modules-icx systemd service - template: - src: isst-load-modules-icx.service.j2 - dest: /lib/systemd/system/isst-load-modules-icx.service - owner: root - group: root - mode: u=rwx,g=r,o=r - -- name: check if isst-load-modules-icx systemd service is present # noqa 303 - shell: - cmd: systemctl list-units --all --type=service | grep "isst-load-modules-icx.service" - register: isst_svc_check - changed_when: false - failed_when: false - -- debug: - msg: "isst-load-modules-icx systemd service is present, skipping kernel module loading" - when: '"isst-load-modules-icx.service" in isst_svc_check.stdout' - changed_when: true - -- name: run isst-load-modules-icx service on boot - systemd: - daemon_reload: yes - state: started - name: isst-load-modules-icx - enabled: yes - when: '"isst-load-modules-icx.service" not in isst_svc_check.stdout' diff --git a/roles/bootstrap/configure_sst/tasks/clx_setup_sst_bf.yml b/roles/bootstrap/configure_sst/tasks/clx_setup_sst_bf.yml index 83805fda..5976a4ab 100644 --- a/roles/bootstrap/configure_sst/tasks/clx_setup_sst_bf.yml +++ b/roles/bootstrap/configure_sst/tasks/clx_setup_sst_bf.yml @@ -1,5 +1,5 @@ ## -## Copyright (c) 2020-2021 Intel Corporation. +## Copyright (c) 2020-2022 Intel Corporation. ## ## Licensed under the Apache License, Version 2.0 (the "License"); ## you may not use this file except in compliance with the License. diff --git a/roles/bootstrap/configure_sst/tasks/icx_sst_bf.yml b/roles/bootstrap/configure_sst/tasks/icx_sst_bf.yml index 6f2f065f..2eb759df 100644 --- a/roles/bootstrap/configure_sst/tasks/icx_sst_bf.yml +++ b/roles/bootstrap/configure_sst/tasks/icx_sst_bf.yml @@ -1,5 +1,5 @@ ## -## Copyright (c) 2020-2021 Intel Corporation. +## Copyright (c) 2020-2022 Intel Corporation. ## ## Licensed under the Apache License, Version 2.0 (the "License"); ## you may not use this file except in compliance with the License. diff --git a/roles/bootstrap/configure_sst/tasks/icx_sst_cp.yml b/roles/bootstrap/configure_sst/tasks/icx_sst_cp.yml index 29bcf88a..0e61db09 100644 --- a/roles/bootstrap/configure_sst/tasks/icx_sst_cp.yml +++ b/roles/bootstrap/configure_sst/tasks/icx_sst_cp.yml @@ -1,5 +1,5 @@ ## -## Copyright (c) 2020-2021 Intel Corporation. +## Copyright (c) 2020-2022 Intel Corporation. ## ## Licensed under the Apache License, Version 2.0 (the "License"); ## you may not use this file except in compliance with the License. diff --git a/roles/bootstrap/configure_sst/tasks/icx_sst_tf.yml b/roles/bootstrap/configure_sst/tasks/icx_sst_tf.yml index bfe9ea7c..43c4dd40 100644 --- a/roles/bootstrap/configure_sst/tasks/icx_sst_tf.yml +++ b/roles/bootstrap/configure_sst/tasks/icx_sst_tf.yml @@ -1,5 +1,5 @@ ## -## Copyright (c) 2020-2021 Intel Corporation. +## Copyright (c) 2020-2022 Intel Corporation. ## ## Licensed under the Apache License, Version 2.0 (the "License"); ## you may not use this file except in compliance with the License. diff --git a/roles/bootstrap/configure_sst/tasks/main.yml b/roles/bootstrap/configure_sst/tasks/main.yml index e504188b..5a404896 100644 --- a/roles/bootstrap/configure_sst/tasks/main.yml +++ b/roles/bootstrap/configure_sst/tasks/main.yml @@ -1,5 +1,5 @@ ## -## Copyright (c) 2020-2021 Intel Corporation. +## Copyright (c) 2020-2022 Intel Corporation. ## ## Licensed under the Apache License, Version 2.0 (the "License"); ## you may not use this file except in compliance with the License. @@ -14,9 +14,9 @@ ## limitations under the License. ## --- -- name: install dependencies +- name: determine machine type include_role: - name: install_dependencies + name: check_machine_type # Common part for both ICX and CLX platform - name: install libraries utility required for CentOS 8.3+ @@ -25,10 +25,6 @@ state: present when: ansible_distribution == 'CentOS' and ansible_distribution_version >= '8.3' -- name: determine machine type - include_role: - name: check_machine_type - # Configuartion for Intel(R) Speed Select Technology "SST-BF,SST-CP,SST-TF and SST-PP" - name: configure Intel Speed Select Technology (ISST) include_tasks: sst_bf_cp_tf_pp_setup.yml @@ -39,4 +35,4 @@ include_tasks: clx_setup_sst_bf.yml # for now only sst bf is supported on CLX when: - is_clx and sst_bf_configuration_enabled - - not (ansible_distribution == 'RedHat' and ansible_distribution_version < '8.3') + - ansible_distribution == 'RedHat' and ansible_distribution_version >= '8.3' diff --git a/roles/bootstrap/configure_sst/tasks/sst_bf_cp_tf_pp_setup.yml b/roles/bootstrap/configure_sst/tasks/sst_bf_cp_tf_pp_setup.yml index a3258030..3bb6d4a7 100644 --- a/roles/bootstrap/configure_sst/tasks/sst_bf_cp_tf_pp_setup.yml +++ b/roles/bootstrap/configure_sst/tasks/sst_bf_cp_tf_pp_setup.yml @@ -1,5 +1,5 @@ ## -## Copyright (c) 2020-2021 Intel Corporation. +## Copyright (c) 2020-2022 Intel Corporation. ## ## Licensed under the Apache License, Version 2.0 (the "License"); ## you may not use this file except in compliance with the License. @@ -18,12 +18,6 @@ include_tasks: ubuntu_install_sst_tool.yml when: ansible_distribution == 'Ubuntu' and ansible_distribution_version > '18.04' -- name: compile and load isst module on CentOS 8.2 on ICX Platform - include_tasks: centos8_2_install_isst_interface.yml - when: - - sst_bf_configuration_enabled and icx_centos8_2_load_isst_modules - - ansible_distribution == 'CentOS' and ansible_distribution_version == '8.2' - - name: Intel(R)-Speed-Select-Technology (ISST) verification command: "intel-speed-select --info" register: isst_verify @@ -42,21 +36,21 @@ command: "intel-speed-select base-freq enable -a" register: sst_bf_verify when: - - sst_bf_configuration_enabled + - sst_bf_configuration_enabled is defined and sst_bf_configuration_enabled - '"get-config-levels:0" in sst_pp_verify.stderr' - name: SST-CP verification command: "intel-speed-select core-power enable -a" register: sst_cp_verify when: - - sst_cp_configuration_enabled + - sst_cp_configuration_enabled is defined and sst_cp_configuration_enabled - '"get-config-levels:0" in sst_pp_verify.stderr' - name: SST-TF verification command: "intel-speed-select turbo-freq enable -a" register: sst_tf_verify when: - - sst_tf_configuration_enabled + - sst_tf_configuration_enabled is defined and sst_tf_configuration_enabled - '"get-config-levels:0" in sst_pp_verify.stderr' - name: update the dynamic linker cache before ISST features configuration @@ -68,54 +62,53 @@ - name: configure SST-BF include_tasks: icx_sst_bf.yml when: - - sst_bf_configuration_enabled + - sst_bf_configuration_enabled is defined and sst_bf_configuration_enabled - '"enable:succes" in sst_bf_verify.stderr' - debug: msg: "Intel(R) SST-BF (feature base-freq) is not supported on platform or already been configured, SST-BF enablement skipped" when: - - sst_bf_configuration_enabled + - sst_bf_configuration_enabled is defined and sst_bf_configuration_enabled - '"enable:succes" not in sst_bf_verify.stderr' changed_when: true - name: configure SST-CP include_tasks: icx_sst_cp.yml when: - - sst_cp_configuration_enabled + - sst_cp_configuration_enabled is defined and sst_cp_configuration_enabled - '"enable:succes" in sst_cp_verify.stderr' - debug: msg: "Intel(R) SST-CP (feature core-power) is not supported on platform or already been configured, SST-CP enablement skipped" when: - - sst_cp_configuration_enabled + - sst_cp_configuration_enabled is defined and sst_cp_configuration_enabled - '"enable:succes" not in sst_cp_verify.stderr' changed_when: true - name: configure SST-TF include_tasks: icx_sst_tf.yml when: - - sst_tf_configuration_enabled + - sst_tf_configuration_enabled is defined and sst_tf_configuration_enabled - '"enable:succes" in sst_tf_verify.stderr' - debug: msg: "Intel(R) SST-TF (feature turbo-freq) is not supported on platform or already been configured, SST-TF enablement skipped" when: - - sst_tf_configuration_enabled + - sst_tf_configuration_enabled is defined and sst_tf_configuration_enabled - '"enable:succes" not in sst_tf_verify.stderr' changed_when: true when: - '"get-config-levels:0" in sst_pp_verify.stderr' - ansible_distribution == 'Ubuntu' or - (ansible_distribution == 'CentOS' and ansible_distribution_version == '8.2' and icx_centos8_2_load_isst_modules) or (ansible_distribution in ['RedHat', 'CentOS'] and ansible_distribution_version >= '8.3') # Intel(R) SST-PP (feature perf-profile) configuration - name: check if SST-PP dir exists stat: - path: "/usr/src/SST_PP_config" + path: "{{ project_root_dir }}/sst_pp_config" register: check_sst_pp_dir - name: configure Intel(R) SST-PP (perf-profile) include_tasks: sst_pp.yml when: - - sst_pp_configuration_enabled + - sst_pp_configuration_enabled is defined and sst_pp_configuration_enabled - not check_sst_pp_dir.stat.exists - '"Intel(R) SST-PP (feature perf-profile) is supported" in isst_verify.stderr_lines' - ansible_distribution == 'Ubuntu' and ansible_distribution_version >= '20.04' or @@ -124,7 +117,7 @@ - debug: msg: "Intel(R) SST-PP (perf-profile) is not supported on platform, SST-PP deployment skipped" when: - - sst_pp_configuration_enabled + - sst_pp_configuration_enabled is defined and sst_pp_configuration_enabled - '"Intel(R) SST-PP (feature perf-profile) is supported" not in isst_verify.stderr_lines' changed_when: true @@ -132,6 +125,6 @@ debug: msg: "Intel(R) SST-PP (perf-profile) already been configured, SST-PP enablement skipped" when: - - sst_pp_configuration_enabled + - sst_pp_configuration_enabled is defined and sst_pp_configuration_enabled - check_sst_pp_dir.stat.exists changed_when: true diff --git a/roles/bootstrap/configure_sst/tasks/sst_pp.yml b/roles/bootstrap/configure_sst/tasks/sst_pp.yml index ec1b941f..29607d00 100644 --- a/roles/bootstrap/configure_sst/tasks/sst_pp.yml +++ b/roles/bootstrap/configure_sst/tasks/sst_pp.yml @@ -1,5 +1,5 @@ ## -## Copyright (c) 2020-2021 Intel Corporation. +## Copyright (c) 2020-2022 Intel Corporation. ## ## Licensed under the Apache License, Version 2.0 (the "License"); ## you may not use this file except in compliance with the License. @@ -56,7 +56,7 @@ command: "intel-speed-select turbo-freq {{ sst_tf_config }} -a" changed_when: true -- name: set SST-BF status status before SST-PP configuration +- name: set SST-BF status before SST-PP configuration command: "intel-speed-select base-freq {{ sst_bf_config }} -a" changed_when: true @@ -109,13 +109,13 @@ # create dir for SST-PP - name: remove existing SST-PP dir if exists file: - path: "/usr/src/SST_PP_config" + path: "{{ project_root_dir }}/sst_pp_config" state: absent become: yes -- name: create directory /usr/src/SST_PP_config to save details +- name: create directory sst_pp_config to save details file: - path: /usr/src/SST_PP_config + path: "{{ project_root_dir }}/sst_pp_config" state: directory mode: "u=rwx,g=rx,o=rx" when: '"auto" in sst_tf_online_cpus' @@ -123,8 +123,17 @@ # Config SST-TP turbo-freq When online_cpus_range is set to "auto" in host vars - name: get available online CPUs shell: - cmd: set -o pipefail && intel-speed-select -d perf-profile set-config-level -l {{ sst_pp_level_set.stdout }} -o 2>&1 \ - | grep -i 'online cpu' | grep -Eo '[0-9]{1,4}' | tr "\n" " " | tr ' ' ',' | sed '$ s/,$//g' + cmd: | + set -o pipefail && \ + intel-speed-select \ + -d perf-profile set-config-level \ + -l {{ sst_pp_level_set.stdout }} \ + -o 2>&1 | \ + grep -i 'online cpu' | \ + grep -Eo '[0-9]{1,4}' | \ + tr "\n" " " | \ + tr ' ' ',' | \ + sed '$ s/,$//g' args: executable: /bin/bash register: online_cpus_range @@ -132,8 +141,7 @@ - name: check vailable online CPUs values for auto config shell: - cmd: "turbostat -c {{ online_cpus_range.stdout }} --show Package,Core,CPU,Bzy_MHz -i 1 \ - | head -n 59" + cmd: "turbostat -c {{ online_cpus_range.stdout }} --show Package,Core,CPU,Bzy_MHz -i 1 | head -n 59" args: executable: /bin/bash register: read_turbostat_output @@ -151,7 +159,7 @@ cmd: "turbostat -c {{ online_cpus_range.stdout }} --show Package,Core,CPU,Bzy_MHz -i 1 | head -n 59 > sst_pp_turbostat_output_when_auto.txt" args: executable: /bin/bash - chdir: "/usr/src/SST_PP_config" + chdir: "{{ project_root_dir }}/sst_pp_config" when: - '"enable" in sst_tf_config' - '"auto" in sst_tf_online_cpus' @@ -166,11 +174,10 @@ # Config SST-TF turbo-freq when all are disabled - name: check available online CPUs values when SST-BF,SST-CP and SST-TF are disabled shell: - cmd: turbostat -c {{ online_cpus_range.stdout }} --show Package,Core,CPU,Bzy_MHz -i 1\ - | head -n 59 + cmd: turbostat -c {{ online_cpus_range.stdout }} --show Package,Core,CPU,Bzy_MHz -i 1 | head -n 59 args: executable: /bin/bash - chdir: "/usr/src/SST_PP_config" + chdir: "{{ project_root_dir }}/sst_pp_config" register: turbostat_output_for_disabled when: - '"disable" in sst_bf_config' @@ -190,7 +197,7 @@ cmd: "turbostat -c {{ online_cpus_range.stdout }} --show Package,Core,CPU,Bzy_MHz -i 1 | head -n 59 > sst_pp_turbostat_output_when_disabled.txt" args: executable: /bin/bash - chdir: "/usr/src/SST_PP_config" + chdir: "{{ project_root_dir }}/sst_pp_config" when: - '"disable" in sst_bf_config' - '"disable" in sst_cp_config' @@ -199,20 +206,27 @@ - name: save perf-profile info to SST-PP dir path shell: - cmd: set -o pipefail && intel-speed-select perf-profile info -l 0 2>&1 \ - | grep -E '(core-count:|bucket|max-turbo|high-priority|low-priority)' > sst_pp_perf_profile.txt + cmd: | + set -o pipefail && \ + intel-speed-select perf-profile info \ + -l 0 2>&1 | \ + grep -E '(core-count:|bucket|max-turbo|high-priority|low-priority)' > sst_pp_perf_profile.txt args: executable: /bin/bash - chdir: "/usr/src/SST_PP_config" + chdir: "{{ project_root_dir }}/sst_pp_config" changed_when: true - name: save information of config-level perf-profile to SST-PP dir path shell: - cmd: set -o pipefail && intel-speed-select -d perf-profile set-config-level -l {{ sst_pp_level_set.stdout }} -o \ - | grep -E '(set_tdp_level|online|offline|logical)' > sst_pp_config_details.txt + cmd: | + set -o pipefail && \ + intel-speed-select \ + -d perf-profile set-config-level \ + -l {{ sst_pp_level_set.stdout }} -o | \ + grep -E '(set_tdp_level|online|offline|logical)' > sst_pp_config_details.txt args: executable: /bin/bash - chdir: "/usr/src/SST_PP_config" + chdir: "{{ project_root_dir }}/sst_pp_config" changed_when: true - name: update the dynamic linker cache after SST-PP successful configuration diff --git a/roles/bootstrap/configure_sst/tasks/sst_pp_user_defined_setup.yml b/roles/bootstrap/configure_sst/tasks/sst_pp_user_defined_setup.yml index 1c9d66fc..04e0082a 100644 --- a/roles/bootstrap/configure_sst/tasks/sst_pp_user_defined_setup.yml +++ b/roles/bootstrap/configure_sst/tasks/sst_pp_user_defined_setup.yml @@ -1,5 +1,5 @@ ## -## Copyright (c) 2020-2021 Intel Corporation. +## Copyright (c) 2020-2022 Intel Corporation. ## ## Licensed under the Apache License, Version 2.0 (the "License"); ## you may not use this file except in compliance with the License. @@ -16,8 +16,17 @@ --- - name: validate host_vars CPUs range with online CPUs range in system shell: - cmd: set -o pipefail && intel-speed-select -d perf-profile set-config-level -l {{ sst_pp_level_set.stdout }} -o 2>&1 \ - | grep -i 'online cpu' | grep -Eo '[0-9]{1,4}' | tr "\n" " " | tr ' ' ',' | grep -Eo '{{ sst_tf_online_cpus }}' + cmd: | + set -o pipefail && \ + intel-speed-select \ + -d perf-profile set-config-level \ + -l {{ sst_pp_level_set.stdout }} \ + -o 2>&1 | \ + grep -i 'online cpu' | \ + grep -Eo '[0-9]{1,4}' | \ + tr "\n" " " | \ + tr ' ' ',' | \ + grep -Eo '{{ sst_tf_online_cpus }}' args: executable: /bin/bash register: validate_hostvars_cpus @@ -38,8 +47,7 @@ - name: set user defined online CPUs values for turbo-freq configuration shell: - cmd: "turbostat -c {{ sst_tf_online_cpus }} --show Package,Core,CPU,Bzy_MHz -i 1 \ - | head -n 59" + cmd: "turbostat -c {{ sst_tf_online_cpus }} --show Package,Core,CPU,Bzy_MHz -i 1 | head -n 59" args: executable: /bin/bash register: read_turbostat_values @@ -49,9 +57,9 @@ debug: msg: "{{ read_turbostat_values.stdout|replace('\\t',' ') }}" -- name: create directory /usr/src/SST_PP_config to save details +- name: create directory sst_pp_config to save details file: - path: /usr/src/SST_PP_config + path: "{{ project_root_dir }}/sst_pp_config" state: directory mode: "u=rwx,g=rx,o=rx" @@ -60,5 +68,5 @@ cmd: "turbostat -c {{ sst_tf_online_cpus }} --show Package,Core,CPU,Bzy_MHz -i 1 2>&1 | head -n 59 > sst_pp_user_defined_turbostat_output.txt" args: executable: /bin/bash - chdir: "/usr/src/SST_PP_config" - changed_when: true \ No newline at end of file + chdir: "{{ project_root_dir }}/sst_pp_config" + changed_when: true diff --git a/roles/bootstrap/configure_sst/tasks/ubuntu_install_sst_tool.yml b/roles/bootstrap/configure_sst/tasks/ubuntu_install_sst_tool.yml index a7c0c341..8bb623a8 100644 --- a/roles/bootstrap/configure_sst/tasks/ubuntu_install_sst_tool.yml +++ b/roles/bootstrap/configure_sst/tasks/ubuntu_install_sst_tool.yml @@ -1,5 +1,5 @@ ## -## Copyright (c) 2020-2021 Intel Corporation. +## Copyright (c) 2020-2022 Intel Corporation. ## ## Licensed under the Apache License, Version 2.0 (the "License"); ## you may not use this file except in compliance with the License. @@ -14,28 +14,22 @@ ## limitations under the License. ## --- -- name: clone git repository from torvalds/linux to compile ISST +- name: clone git repository to compile ISST git: - repo: https://github.com/torvalds/linux.git - dest: speedselect - clone: yes - version: v5.14 + repo: "{{ isst_tool_git_url }}" + dest: "{{ isst_tool_src_dir }}" + depth: 1 + force: true + version: "{{ isst_tool_git_version }}" -- name: compile ISST +- name: compile and install ISST make: - chdir: speedselect/tools/power/x86/intel-speed-select + chdir: "{{ (isst_tool_src_dir, 'tools', 'power', 'x86', 'intel-speed-select') | path_join }}" + target: install params: NUM_THREADS: "{{ ansible_processor_vcpus }}" -- name: copy compiled binary to folder - copy: - src: speedselect/tools/power/x86/intel-speed-select/intel-speed-select - dest: /usr/bin/intel-speed-select - remote_src: yes - mode: u+rwx,g+rx,o+rx - become: yes - - name: remove repository file: - path: speedselect + path: "{{ isst_tool_src_dir }}" state: absent diff --git a/roles/bootstrap/configure_sst/templates/isst-load-modules-icx.service.j2 b/roles/bootstrap/configure_sst/templates/isst-load-modules-icx.service.j2 deleted file mode 100644 index 7e2b40c4..00000000 --- a/roles/bootstrap/configure_sst/templates/isst-load-modules-icx.service.j2 +++ /dev/null @@ -1,14 +0,0 @@ -[Unit] -Description=Load ISST modules -AssertPathExists={{ kernel_source_code_dir }}/drivers/platform/x86/intel_speed_select_if/isst_if_common.ko -AssertPathExists={{ kernel_source_code_dir }}/drivers/platform/x86/intel_speed_select_if/isst_if_mbox_pci.ko -AssertPathExists={{ kernel_source_code_dir }}/drivers/platform/x86/intel_speed_select_if/isst_if_mmio.ko - -[Service] -Type=oneshot -ExecStart=insmod {{ kernel_source_code_dir }}/drivers/platform/x86/intel_speed_select_if/isst_if_common.ko -ExecStart=insmod {{ kernel_source_code_dir }}/drivers/platform/x86/intel_speed_select_if/isst_if_mbox_pci.ko -ExecStart=insmod {{ kernel_source_code_dir }}/drivers/platform/x86/intel_speed_select_if/isst_if_mmio.ko - -[Install] -WantedBy=multi-user.target diff --git a/roles/bootstrap/determine_dataplane_interfaces/defaults/main.yml b/roles/bootstrap/determine_dataplane_interfaces/defaults/main.yml index 7f002fd1..c341e5ba 100644 --- a/roles/bootstrap/determine_dataplane_interfaces/defaults/main.yml +++ b/roles/bootstrap/determine_dataplane_interfaces/defaults/main.yml @@ -1,5 +1,5 @@ ## -## Copyright (c) 2020-2021 Intel Corporation. +## Copyright (c) 2020-2022 Intel Corporation. ## ## Licensed under the Apache License, Version 2.0 (the "License"); ## you may not use this file except in compliance with the License. diff --git a/roles/bootstrap/determine_dataplane_interfaces/tasks/dataplane-interfaces.yml b/roles/bootstrap/determine_dataplane_interfaces/tasks/dataplane-interfaces.yml index 7ca4aab9..83135acb 100644 --- a/roles/bootstrap/determine_dataplane_interfaces/tasks/dataplane-interfaces.yml +++ b/roles/bootstrap/determine_dataplane_interfaces/tasks/dataplane-interfaces.yml @@ -1,5 +1,5 @@ ## -## Copyright (c) 2020-2021 Intel Corporation. +## Copyright (c) 2020-2022 Intel Corporation. ## ## Licensed under the Apache License, Version 2.0 (the "License"); ## you may not use this file except in compliance with the License. @@ -15,7 +15,7 @@ ## --- - name: get interface of dataplane device - shell: "set -o pipefail && hwinfo --network | grep -A 5 {{ nic.bus_info }}" + shell: "set -o pipefail && hwinfo --network | grep -A 5 {{ nic.bus_info }}" args: executable: /bin/bash register: dataplane_interface_list diff --git a/roles/bootstrap/determine_dataplane_interfaces/tasks/main.yml b/roles/bootstrap/determine_dataplane_interfaces/tasks/main.yml index 4db24dcc..4d324782 100644 --- a/roles/bootstrap/determine_dataplane_interfaces/tasks/main.yml +++ b/roles/bootstrap/determine_dataplane_interfaces/tasks/main.yml @@ -1,5 +1,5 @@ ## -## Copyright (c) 2020-2021 Intel Corporation. +## Copyright (c) 2020-2022 Intel Corporation. ## ## Licensed under the Apache License, Version 2.0 (the "License"); ## you may not use this file except in compliance with the License. @@ -21,7 +21,7 @@ include_role: name: install_dependencies -- name: Create temporary nic configuration file +- name: create temporary NIC configuration file tempfile: state: file prefix: "dataplane_interfaces_{{ inventory_hostname }}_" @@ -44,7 +44,7 @@ loop_control: loop_var: nic -- name: include discovered nics configuration file +- name: include discovered NICs configuration file include_vars: file: "{{ dataplane_interface_configuration_file.path }}" diff --git a/roles/golang_install/defaults/main.yml b/roles/bootstrap/golang_install/defaults/main.yml similarity index 79% rename from roles/golang_install/defaults/main.yml rename to roles/bootstrap/golang_install/defaults/main.yml index 8fdb4e45..ead2dc13 100644 --- a/roles/golang_install/defaults/main.yml +++ b/roles/bootstrap/golang_install/defaults/main.yml @@ -1,5 +1,5 @@ ## -## Copyright (c) 2020-2021 Intel Corporation. +## Copyright (c) 2020-2022 Intel Corporation. ## ## Licensed under the Apache License, Version 2.0 (the "License"); ## you may not use this file except in compliance with the License. @@ -14,6 +14,6 @@ ## limitations under the License. ## --- -golang_version: 1.16.5 +golang_version: 1.17.6 golang_download_url: "https://dl.google.com/go/go{{ golang_version }}.linux-amd64.tar.gz" -golang_download_checksum: "sha256:b12c23023b68de22f74c0524f10b753e7b08b1504cb7e417eccebdd3fae49061" +golang_download_checksum: "sha256:231654bbf2dab3d86c1619ce799e77b03d96f9b50770297c8f4dff8836fc8ca2" diff --git a/roles/bootstrap/golang_install/tasks/main.yml b/roles/bootstrap/golang_install/tasks/main.yml new file mode 100644 index 00000000..6a032cb1 --- /dev/null +++ b/roles/bootstrap/golang_install/tasks/main.yml @@ -0,0 +1,103 @@ +## +## Copyright (c) 2020-2022 Intel Corporation. +## +## Licensed under the Apache License, Version 2.0 (the "License"); +## you may not use this file except in compliance with the License. +## You may obtain a copy of the License at +## +## http://www.apache.org/licenses/LICENSE-2.0 +## +## Unless required by applicable law or agreed to in writing, software +## distributed under the License is distributed on an "AS IS" BASIS, +## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +## See the License for the specific language governing permissions and +## limitations under the License. +## +--- +- name: check current golang version + shell: /usr/local/go/bin/go version|sed -e 's/go version go//g'|cut -d' ' -f1 + failed_when: false + changed_when: false + register: go_version + +- debug: + msg: | + "installed golang version = {{ go_version.stdout if go_version.stdout else 'None' }}" + "required golang version = {{ golang_version }}" + +- name: start procedure to install golang in required version + block: + - name: uninstall existing golang + file: + path: "{{ item }}" + state: absent + with_items: + - "/usr/local/go" + - "$HOME/go" + + - name: download golang tarball + get_url: + url: "{{ golang_download_url }}" + checksum: "{{ golang_download_checksum }}" + dest: "{{ project_root_dir }}" + register: golang_download + until: golang_download is not failed + retries: 5 + + - name: untar downloaded golang tarball + unarchive: + src: "{{ golang_download.dest }}" + dest: /usr/local + copy: no + mode: 0755 + + - name: set GOPATH env and add golang bin to PATH for all users + copy: + content: | + export GOROOT=/usr/local/go + export GOPATH=$HOME/go + export PATH=$GOPATH/bin:$GOROOT/bin:$PATH + dest: /etc/profile.d/golang.sh + mode: 0755 + + - name: create symlinks to golang binaries + file: + state: link + src: /usr/local/go/bin/{{ item }} + dest: /usr/bin/{{ item }} + mode: 0755 + with_items: + - go + - gofmt + when: golang_version != go_version.stdout + +# golang is successfully installed in required version +# add GOPATH to ansible facts +- name: get GOPATH + command: go env GOPATH + register: gopath + changed_when: false + +- name: start procedure to install cfssl and cfssljson in required versions + block: + - name: check current cfssl version + shell: go version -m $(which cfssl) | grep mod | awk '{print $3}' + failed_when: false + register: cfssl_current_version + + - name: check latest cfssl version + shell: go list -m -versions github.com/cloudflare/cfssl | awk '{print $7}' # $7 should be latest varsion + failed_when: false + register: cfssl_latest_version + + - name: install cfssl in latest version + command: go install github.com/cloudflare/cfssl/cmd/cfssl@latest + when: cfssl_current_version.stdout != cfssl_latest_version.stdout + + # NOTE(pklimowx): cfssljson doesn't return useful version information + # we have to lose 1s here + - name: install cfssljson in latest version + command: go install github.com/cloudflare/cfssl/cmd/cfssljson@latest + when: + - groups['kube_control_plane'] | length > 0 + - inventory_hostname == groups['kube_control_plane'][0] diff --git a/roles/bootstrap/install_gpu_kernel/defaults/main.yml b/roles/bootstrap/install_gpu_kernel/defaults/main.yml index 9f6b8918..d13d7ea7 100644 --- a/roles/bootstrap/install_gpu_kernel/defaults/main.yml +++ b/roles/bootstrap/install_gpu_kernel/defaults/main.yml @@ -1,5 +1,5 @@ ## -## Copyright (c) 2020-2021 Intel Corporation. +## Copyright (c) 2020-2022 Intel Corporation. ## ## Licensed under the Apache License, Version 2.0 (the "License"); ## you may not use this file except in compliance with the License. @@ -15,10 +15,6 @@ ## --- gpu_kernel_git_url: "https://github.com/intel-gpu/kernel" -gpu_kernel_src_dst: "/usr/src/kernels/gpu_kernel/" +gpu_kernel_src_dst: "{{ project_root_dir }}/kernels/gpu_kernel/" gpu_kernel_flags: "i915.force_probe=* modprobe.blacklist=ast,snd_hda_intel i915.enable_guc=2" linux_dg1_firmware_url: "https://repositories.intel.com/graphics/firmware/linux-firmware-dg1_2020.43.tgz" - -dracut_version: "055" -dracut_url: "https://mirrors.edge.kernel.org/pub/linux/utils/boot/dracut/dracut-{{ dracut_version }}.tar.gz" -dracut_checksum: sha1:a2aac601767f6d1dda0c09ac03fc49e062a4aa96 diff --git a/roles/bootstrap/install_gpu_kernel/tasks/centos82_dracut_install.yml b/roles/bootstrap/install_gpu_kernel/tasks/centos82_dracut_install.yml deleted file mode 100644 index 90549c3d..00000000 --- a/roles/bootstrap/install_gpu_kernel/tasks/centos82_dracut_install.yml +++ /dev/null @@ -1,44 +0,0 @@ -## -## Copyright (c) 2020-2021 Intel Corporation. -## -## Licensed under the Apache License, Version 2.0 (the "License"); -## you may not use this file except in compliance with the License. -## You may obtain a copy of the License at -## -## http://www.apache.org/licenses/LICENSE-2.0 -## -## Unless required by applicable law or agreed to in writing, software -## distributed under the License is distributed on an "AS IS" BASIS, -## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -## See the License for the specific language governing permissions and -## limitations under the License. -## ---- -- name: download dracut tar.gz archive - get_url: - url: "{{ dracut_url }}" - dest: "{{ gpu_kernel_src_dst }}" - checksum: "{{ dracut_checksum }}" - changed_when: false - -- name: untar dracut archive - unarchive: - src: "{{ gpu_kernel_src_dst }}/dracut-{{ dracut_version }}.tar.gz" - dest: "{{ gpu_kernel_src_dst }}" - mode: 0755 - remote_src: yes - -- name: configure dracut - command: ./configure - args: - chdir: "{{ gpu_kernel_src_dst }}/dracut-{{ dracut_version }}" - changed_when: false - -- name: prepare dracut - make: - chdir: "{{ gpu_kernel_src_dst }}/dracut-{{ dracut_version }}" - -- name: install dracut - make: - chdir: "{{ gpu_kernel_src_dst }}/dracut-{{ dracut_version }}" - target: install diff --git a/roles/bootstrap/install_gpu_kernel/tasks/main.yml b/roles/bootstrap/install_gpu_kernel/tasks/main.yml index aab7d777..faab32d8 100644 --- a/roles/bootstrap/install_gpu_kernel/tasks/main.yml +++ b/roles/bootstrap/install_gpu_kernel/tasks/main.yml @@ -1,5 +1,5 @@ ## -## Copyright (c) 2020-2021 Intel Corporation. +## Copyright (c) 2020-2022 Intel Corporation. ## ## Licensed under the Apache License, Version 2.0 (the "License"); ## you may not use this file except in compliance with the License. @@ -86,6 +86,12 @@ dest: "{{ gpu_kernel_src_dst }}" changed_when: false +- name: ensure destination directory exists + file: + path: /lib/firmware/i915 + state: directory + mode: 0755 + - name: untar dg1 tar.gz archive unarchive: src: "{{ gpu_kernel_src_dst }}/linux-firmware-dg1_2020.43.tgz" @@ -116,10 +122,6 @@ changed_when: false become: yes -- name: install dracut-{{ dracut_version }} - include_tasks: centos82_dracut_install.yml - when: ansible_distribution == 'CentOS' and ansible_distribution_version == '8.2' - - name: install custom kernel command: >- installkernel {{ gpu_dp_kernel_version }} diff --git a/roles/bootstrap/install_gpu_kernel/tasks/prepare_grub.yml b/roles/bootstrap/install_gpu_kernel/tasks/prepare_grub.yml index 46f87d34..6328233d 100644 --- a/roles/bootstrap/install_gpu_kernel/tasks/prepare_grub.yml +++ b/roles/bootstrap/install_gpu_kernel/tasks/prepare_grub.yml @@ -1,5 +1,5 @@ ## -## Copyright (c) 2020-2021 Intel Corporation. +## Copyright (c) 2020-2022 Intel Corporation. ## ## Licensed under the Apache License, Version 2.0 (the "License"); ## you may not use this file except in compliance with the License. diff --git a/roles/bootstrap/install_gpu_kernel/tasks/update_kernel_config.yml b/roles/bootstrap/install_gpu_kernel/tasks/update_kernel_config.yml index e3e98412..d62a7cd3 100644 --- a/roles/bootstrap/install_gpu_kernel/tasks/update_kernel_config.yml +++ b/roles/bootstrap/install_gpu_kernel/tasks/update_kernel_config.yml @@ -1,5 +1,5 @@ ## -## Copyright (c) 2020-2021 Intel Corporation. +## Copyright (c) 2020-2022 Intel Corporation. ## ## Licensed under the Apache License, Version 2.0 (the "License"); ## you may not use this file except in compliance with the License. @@ -57,17 +57,6 @@ mode: 0644 when: ansible_distribution in ["CentOS", "RedHat"] -- name: change Intel Speed Select Interface setting - lineinfile: - path: "{{ gpu_kernel_src_dst }}/.config" - regexp: '^# CONFIG_INTEL_SPEED_SELECT_INTERFACE is not set' - line: 'CONFIG_INTEL_SPEED_SELECT_INTERFACE=m' - state: present - mode: 0644 - when: - - ansible_distribution == 'CentOS' and ansible_distribution_version == '8.2' - - sst_bf_configuration_enabled is defined and sst_bf_configuration_enabled - - name: modify kernel config while created from Ubuntu 21.04 block: - name: change trusted keys setting @@ -85,7 +74,7 @@ line: '#CONFIG_DEBUG_INFO_BTF' state: present mode: 0644 - when: ansible_distribution == "Ubuntu" and ansible_distribution_version == '21.04' + when: ansible_distribution == "Ubuntu" and ansible_distribution_version >= '20.04' - name: allow force probe of i915 module lineinfile: diff --git a/roles/bootstrap/install_gpu_kernel/vars/main.yml b/roles/bootstrap/install_gpu_kernel/vars/main.yml index 69a4abe8..ca1a5378 100644 --- a/roles/bootstrap/install_gpu_kernel/vars/main.yml +++ b/roles/bootstrap/install_gpu_kernel/vars/main.yml @@ -1,5 +1,5 @@ ## -## Copyright (c) 2020-2021 Intel Corporation. +## Copyright (c) 2020-2022 Intel Corporation. ## ## Licensed under the Apache License, Version 2.0 (the "License"); ## you may not use this file except in compliance with the License. diff --git a/roles/bootstrap/install_packages/tasks/debian.yml b/roles/bootstrap/install_packages/tasks/debian.yml index 6008bb7e..e89ce0fc 100644 --- a/roles/bootstrap/install_packages/tasks/debian.yml +++ b/roles/bootstrap/install_packages/tasks/debian.yml @@ -1,5 +1,5 @@ ## -## Copyright (c) 2020-2021 Intel Corporation. +## Copyright (c) 2020-2022 Intel Corporation. ## ## Licensed under the Apache License, Version 2.0 (the "License"); ## you may not use this file except in compliance with the License. @@ -48,22 +48,17 @@ apt_repository: repo: "{{ item }}" with_items: - - "deb http://archive.ubuntu.com/ubuntu {{ ansible_distribution_release }} main universe" - - "deb http://archive.ubuntu.com/ubuntu {{ ansible_distribution_release }} main restricted" - - "deb http://archive.ubuntu.com/ubuntu {{ ansible_distribution_release }}-updates main restricted" - - "deb http://archive.ubuntu.com/ubuntu {{ ansible_distribution_release }} universe" - - "deb http://archive.ubuntu.com/ubuntu {{ ansible_distribution_release }}-updates universe" - - "deb http://archive.ubuntu.com/ubuntu {{ ansible_distribution_release }} multiverse" - - "deb http://archive.ubuntu.com/ubuntu {{ ansible_distribution_release }}-updates multiverse" + - "deb http://archive.ubuntu.com/ubuntu {{ ansible_distribution_release }} main restricted universe multiverse" + - "deb http://archive.ubuntu.com/ubuntu {{ ansible_distribution_release }}-updates main restricted universe multiverse" - "deb http://archive.ubuntu.com/ubuntu {{ ansible_distribution_release }}-backports main restricted universe multiverse" - - "deb http://security.ubuntu.com/ubuntu {{ ansible_distribution_release }}-security main restricted" - - "deb http://security.ubuntu.com/ubuntu {{ ansible_distribution_release }}-security universe" - - "deb http://security.ubuntu.com/ubuntu {{ ansible_distribution_release }}-security multiverse" + - "deb http://security.ubuntu.com/ubuntu {{ ansible_distribution_release }}-security main restricted universe multiverse" when: ansible_distribution == "Ubuntu" - name: Add kubic apt repo key apt_key: - url: "https://download.opensuse.org/repositories/devel:kubic:libcontainers:stable/xUbuntu_{{ ansible_distribution_version }}/Release.key" + url: >- + https://download.opensuse.org/repositories/devel:kubic:libcontainers:stable/xUbuntu_ + {%- if ansible_distribution_version >= '21.04' -%}21.04{%- else -%}20.04{%- endif -%}/Release.key state: present register: apt_key_download until: apt_key_download is succeeded @@ -74,7 +69,9 @@ - name: Add kubic apt repo apt_repository: - repo: "deb http://download.opensuse.org/repositories/devel:/kubic:/libcontainers:/stable/xUbuntu_{{ ansible_distribution_version }}/ /" + repo: >- + deb http://download.opensuse.org/repositories/devel:/kubic:/libcontainers:/stable/xUbuntu_ + {%- if ansible_distribution_version >= '21.04' -%}21.04{%- else -%}20.04{%- endif -%}/ / state: present filename: devel-kubic-libcontainers-stable when: @@ -146,14 +143,18 @@ - jq when: ansible_distribution == "Ubuntu" -- name: update to the latest kernel and kernel headers on the Ubuntu OS +# hirsute (21.04) package for (image & headers) is 20.04. (Note: ansible_distribution_version will not be returned as the correct version) +# Depending on the needs, we can split tasks for future Ubuntu releases if necessary. +# Ref: https://launchpad.net/ubuntu/hirsute/+package/linux-image-generic-hwe-20.04 +# https://launchpad.net/ubuntu/hirsute/+package/linux-headers-generic-hwe-20.04 +- name: Update Ubuntu to the latest kernel and kernel headers apt: name: - - linux-image-generic-hwe-{{ ansible_distribution_version }} - - linux-headers-generic-hwe-{{ ansible_distribution_version }} + - linux-image-generic-hwe-20.04 + - linux-headers-generic-hwe-20.04 state: latest # noqa 403 notify: - reboot server when: - - ansible_distribution == "Ubuntu" - update_kernel | default(false) + - ansible_distribution == 'Ubuntu' diff --git a/roles/bootstrap/install_packages/tasks/main.yml b/roles/bootstrap/install_packages/tasks/main.yml index 61e2c373..e057aee3 100644 --- a/roles/bootstrap/install_packages/tasks/main.yml +++ b/roles/bootstrap/install_packages/tasks/main.yml @@ -1,5 +1,5 @@ ## -## Copyright (c) 2020-2021 Intel Corporation. +## Copyright (c) 2020-2022 Intel Corporation. ## ## Licensed under the Apache License, Version 2.0 (the "License"); ## you may not use this file except in compliance with the License. @@ -22,7 +22,7 @@ include_tasks: debian.yml when: ansible_os_family == "Debian" -#net_attach_def, docker registry, cmk reqs +#net_attach_def, docker registry - name: upgrade Python wheel and setuptools pip: name: @@ -39,7 +39,8 @@ - cachetools==4.2.1 - markupsafe==1.1.1 - jinja2==2.11.3 - - openshift==0.11.2 + - openshift==0.12.1 + - kubernetes==12.0.1 - six>=1.15.0 - websocket-client==0.58.0 - oauthlib==3.1.0 diff --git a/roles/bootstrap/install_packages/tasks/rhel.yml b/roles/bootstrap/install_packages/tasks/rhel.yml index 177919e3..a78a2bd2 100644 --- a/roles/bootstrap/install_packages/tasks/rhel.yml +++ b/roles/bootstrap/install_packages/tasks/rhel.yml @@ -1,5 +1,5 @@ ## -## Copyright (c) 2020-2021 Intel Corporation. +## Copyright (c) 2020-2022 Intel Corporation. ## ## Licensed under the Apache License, Version 2.0 (the "License"); ## you may not use this file except in compliance with the License. @@ -14,13 +14,6 @@ ## limitations under the License. ## --- -- name: enable PowerTools repository on CentOS >= 8 and < 8.3 -# noqa 303 - yum is called intenionallly here - command: yum config-manager --set-enabled PowerTools - when: - - ansible_distribution == "CentOS" - - ansible_distribution_version >= '8' and ansible_distribution_version < '8.3' - - name: enable powertools repository on CentOS >= 8.3 # noqa 303 - yum is called intenionallly here command: yum config-manager --set-enabled powertools @@ -65,20 +58,6 @@ set_fact: full_dist_version: "{{ release.stdout | regex_replace('.*(\\d+.\\d+.\\d\\d\\d\\d).*', '\\1') }}" -- name: update CentOS Vault yum repository on CentOS 7 - yum_repository: - name: C{{ full_dist_version }}-base - description: CentOS-{{ full_dist_version }} - Base - file: CentOS-Vault - baseurl: http://vault.centos.org/{{ full_dist_version }}/os/$basearch/ - gpgcheck: yes - gpgkey: file:///etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-{{ ansible_distribution_major_version }} - enabled: yes - when: - - ansible_distribution == "CentOS" - - ansible_distribution_version < '7.9' - - not update_kernel - #- name: update CentOS Vault yum repository on CentOS 8 # yum_repository: # name: C{{ full_dist_version }}-base @@ -95,22 +74,6 @@ # - not update_kernel # CentOS-Vault repo not working for CentOS 8, so install kernel headers directly -- name: pull matching kernel headers on CentOS 8.2 - package: - name: "{{ item }}" - state: present - retries: 5 - delay: 10 - register: source_status - until: source_status is not failed - with_items: - - "https://vault.centos.org/8.2.2004/BaseOS/x86_64/os/Packages/kernel-headers-4.18.0-193.el8.x86_64.rpm" - - "https://vault.centos.org/8.2.2004/BaseOS/x86_64/os/Packages/kernel-devel-4.18.0-193.el8.x86_64.rpm" - when: - - ansible_distribution == "CentOS" - - ansible_distribution_version == '8.2' - - not update_kernel - - name: pull matching kernel headers on CentOS 8.3 package: name: "{{ item }}" @@ -127,23 +90,40 @@ - ansible_distribution_version == '8.3' - not update_kernel -- name: pull matching kernel headers on CentOS 8.4 - package: - name: "{{ item }}" - state: present - retries: 5 - delay: 10 - register: source_status - until: source_status is not failed - with_items: - - "http://mirror.centos.org/centos/8/BaseOS/x86_64/os/Packages/kernel-headers-4.18.0-305.3.1.el8.x86_64.rpm" - - "http://mirror.centos.org/centos/8/BaseOS/x86_64/os/Packages/kernel-devel-4.18.0-305.3.1.el8.x86_64.rpm" -# - "https://vault.centos.org/8.4.2105/BaseOS/x86_64/os/Packages/kernel-headers-4.18.0-305.el8.x86_64.rpm" -# - "https://vault.centos.org/8.4.2105/BaseOS/x86_64/os/Packages/kernel-devel-4.18.0-305.el8.x86_64.rpm" +- name: configuration block for CentOS 8.4 + block: + - name: pull matching kernel headers on CentOS 8.4 + package: + name: "{{ item }}" + state: present + retries: 5 + delay: 10 + register: source_status + until: source_status is not failed + with_items: + - "http://mirror.centos.org/centos/8/BaseOS/x86_64/os/Packages/kernel-headers-4.18.0-348.2.1.el8_5.x86_64.rpm" + - "http://mirror.centos.org/centos/8/BaseOS/x86_64/os/Packages/kernel-devel-4.18.0-348.2.1.el8_5.x86_64.rpm" + # - "https://vault.centos.org/8.4.2105/BaseOS/x86_64/os/Packages/kernel-headers-4.18.0-305.el8.x86_64.rpm" + # - "https://vault.centos.org/8.4.2105/BaseOS/x86_64/os/Packages/kernel-devel-4.18.0-305.el8.x86_64.rpm" + when: not update_kernel + + - name: removing unrequired dependencies in CentOS 8.4 + package: + name: + - glibc-lib32 + - ltp-ddt + state: absent + + - name: install required dependencies before update all packages in CentOS 8.4 + package: + name: + - python3-libstoragemgmt + - kmod-kvdo + state: latest + when: update_all_packages when: - ansible_distribution == "CentOS" - ansible_distribution_version == '8.4' - - not update_kernel # pull the matching kernel headers if kernel is not updated - name: pull matching kernel headers from configured repos @@ -157,7 +137,8 @@ until: kernel_source is success when: - not source_status.changed - - ansible_os_family == "RedHat" + - (ansible_distribution == "RedHat") or + (ansible_distribution == "CentOS" and ansible_distribution_version != '8.4') - not update_kernel - name: install the 'Development tools' package group @@ -166,14 +147,7 @@ when: - ansible_os_family == "RedHat" -- name: install pip - package: - name: python-pip - when: - - ansible_distribution in ["RedHat", "CentOS"] - - ansible_distribution_version < '8' - -- name: install pip +- name: install pip3 package: name: python3-pip when: @@ -201,24 +175,6 @@ - ansible_os_family == "RedHat" - update_kernel | default(false) -#note(przemeklal): fixes issue with missing selinux in packet.net CentOS 7 images -- name: ensure selinux is installed on CentOS/RHEL 7 - package: - name: - - policycoreutils - - policycoreutils-python - - selinux-policy - - selinux-policy-targeted - - libselinux-utils - - setools - - setools-console - - shtool - - lshw - state: present - when: - - ansible_distribution in ["RedHat", "CentOS"] - - ansible_distribution_version < '8' - - name: set python is python3 alternatives: name: python @@ -237,33 +193,38 @@ state: present when: ansible_distribution in ["RedHat", "CentOS"] -- name: Add kubic yum repo and install updated version of buildah +- name: add kubic yum repo and install updated version of buildah block: - - name: disable container-tools module - # noqa 305 - shell is used intentionally here - shell: dnf -y module disable container-tools - - name: enable rhcontainerbot/container-selinux repository - # noqa 305 - shell is used intentionally here - shell: dnf -y copr enable rhcontainerbot/container-selinux - - name: Add kubic yum repo - yum_repository: - name: devel_kubic_libcontainers_stable - description: Stable Releases of Upstream github.com/containers packages (CentOS_$releasever) - baseurl: https://download.opensuse.org/repositories/devel:/kubic:/libcontainers:/stable/CentOS_$releasever/ - gpgcheck: yes - gpgkey: https://download.opensuse.org/repositories/devel:/kubic:/libcontainers:/stable/CentOS_$releasever/repodata/repomd.xml.key - keepcache: '0' - - name: install crun - dnf: - name: crun - state: present - - name: install buildah package - package: - name: buildah - state: present - - name: enable container-tools module - # noqa 305 - shell is used intentionally here - shell: dnf -y module enable container-tools + - name: disable container-tools module + # noqa 305 - shell is used intentionally here + shell: dnf -y module disable container-tools + + - name: enable rhcontainerbot/container-selinux repository + # noqa 305 - shell is used intentionally here + shell: dnf -y copr enable rhcontainerbot/container-selinux + + - name: add kubic yum repo + yum_repository: + name: devel_kubic_libcontainers_stable + description: Stable Releases of Upstream github.com/containers packages (CentOS_$releasever) + baseurl: https://download.opensuse.org/repositories/devel:/kubic:/libcontainers:/stable/CentOS_$releasever/ + gpgcheck: yes + gpgkey: https://download.opensuse.org/repositories/devel:/kubic:/libcontainers:/stable/CentOS_$releasever/repodata/repomd.xml.key + keepcache: '0' + + - name: install crun + dnf: + name: crun + state: present + + - name: install buildah package + package: + name: buildah + state: present + + - name: enable container-tools module + # noqa 305 - shell is used intentionally here + shell: dnf -y module enable container-tools when: - ansible_os_family == "RedHat" and ansible_distribution_version >= '8.2' - '"docker" not in container_runtime' diff --git a/roles/bootstrap/install_qat_drivers_services/defaults/main.yml b/roles/bootstrap/install_qat_drivers_services/defaults/main.yml index da51e5d9..df88570c 100644 --- a/roles/bootstrap/install_qat_drivers_services/defaults/main.yml +++ b/roles/bootstrap/install_qat_drivers_services/defaults/main.yml @@ -1,5 +1,5 @@ ## -## Copyright (c) 2020-2021 Intel Corporation. +## Copyright (c) 2020-2022 Intel Corporation. ## ## Licensed under the Apache License, Version 2.0 (the "License"); ## you may not use this file except in compliance with the License. @@ -14,8 +14,9 @@ ## limitations under the License. ## --- -qat_drivers_version: '1.7.l.4.14.0-00031' -qat_drivers_download_url: 'https://downloadmirror.intel.com/30178/eng/qat{{qat_drivers_version }}.tar.gz' -qat_drivers_pkg_checksum: 'sha1:7D38020917B4D488F875E66ADC4CFF207732CF03' +qat_drivers_version: 'QAT.L.4.16.0-00017' +qat_drivers_download_url: 'https://downloadmirror.intel.com/710050/{{qat_drivers_version }}.tar.gz' +qat_drivers_pkg_checksum: 'sha1:ACD86FD58AE0C96DBAD1F7992829B4B8ACBA9940' +qat_drivers_dir: "{{ (project_root_dir, 'qat_drivers') | path_join }}" libudev_version: '247.3-3ubuntu3' diff --git a/roles/bootstrap/install_qat_drivers_services/tasks/main.yml b/roles/bootstrap/install_qat_drivers_services/tasks/main.yml index 763b65fe..d2340747 100644 --- a/roles/bootstrap/install_qat_drivers_services/tasks/main.yml +++ b/roles/bootstrap/install_qat_drivers_services/tasks/main.yml @@ -1,5 +1,5 @@ ## -## Copyright (c) 2020-2021 Intel Corporation. +## Copyright (c) 2020-2022 Intel Corporation. ## ## Licensed under the Apache License, Version 2.0 (the "License"); ## you may not use this file except in compliance with the License. @@ -22,15 +22,19 @@ apt: name: >- {%- if ansible_distribution_version == '21.04' -%} - libudev-dev={{ libudev_version }} + {%- if on_vms is defined and on_vms -%} + ['libudev-dev={{ libudev_version }}*'] + {%- else -%} + ['libudev-dev={{ libudev_version }}'] + {%- endif -%} {%- else -%} libudev-dev {%- endif -%} when: ansible_distribution == "Ubuntu" -- name: create directory /usr/src/QAT_drivers for all QAT dependencies +- name: create directory {{ qat_drivers_dir }} for all QAT dependencies file: - path: /usr/src/QAT_drivers + path: "{{ qat_drivers_dir }}" state: directory mode: "u=rwx,g=rx,o=rx" @@ -38,22 +42,22 @@ get_url: url: "{{ qat_drivers_download_url }}" checksum: "{{ qat_drivers_pkg_checksum }}" - dest: /usr/src/QAT_drivers + dest: "{{ qat_drivers_dir }}" register: qat_driver until: qat_driver is not failed retries: 5 - name: unarchive QAT drivers package unarchive: - src: /usr/src/QAT_drivers/qat{{ qat_drivers_version }}.tar.gz - dest: /usr/src/QAT_drivers + src: "{{ qat_drivers_dir }}/{{ qat_drivers_version }}.tar.gz" + dest: "{{ qat_drivers_dir }}" remote_src: yes mode: 0755 - name: check all packages are present for QAT drivers installation command: ./configure args: - chdir: /usr/src/QAT_drivers + chdir: "{{ qat_drivers_dir }}" register: qat_requirements ignore_errors: true changed_when: true @@ -67,7 +71,7 @@ - name: make install QAT drivers make: - chdir: /usr/src/QAT_drivers + chdir: "{{ qat_drivers_dir }}" target: install register: qat_make_install become: yes @@ -80,35 +84,31 @@ - "Purpose for failure might be an already intel_qat module set up on server. Recommended is to have clean image of OS without pre-installed QAT module." - "make uninstall && make clean && make distclean commands can roll-back QAT drivers package {{ qat_drivers_version }} already carried out this point" when: - - ansible_distribution == "CentOS" - - ansible_distribution_version > '8' + - ansible_distribution == "CentOS" and ansible_distribution_version >= '8.3' - "'ERROR:' in qat_make_install.stderr" - name: confirm QAT module installed shell: "set -o pipefail && lsmod | grep qat" args: - chdir: "/usr/src/QAT_drivers" executable: /bin/bash register: qat_confirm failed_when: '"intel_qat" not in qat_confirm.stdout' changed_when: false -- debug: - var: qat_confirm.stdout_lines +- name: enable SRIOV QAT devices on VMs + lineinfile: + path: "/etc/default/qat" + line: "SRIOV_ENABLE=1" + regexp: "^#SRIOV_ENABLE=1" + create: yes + owner: root + group: root + mode: '0644' + when: + - on_vms is defined and on_vms -- name: start QAT service, if not started +- name: make sure QAT service is started and enabled service: - state: started + state: restarted name: qat_service - register: qat_service_state - -# ansible_facts.services is not supported currently with latest QAT drivers -- name: check status of QAT service after started - command: service qat_service status - args: - warn: false - register: qat_enabled - changed_when: true - -- debug: - var: qat_enabled.stdout_lines + enabled: yes diff --git a/roles/bootstrap/install_qat_drivers_services/vars/main.yml b/roles/bootstrap/install_qat_drivers_services/vars/main.yml index 7498646a..fe476190 100644 --- a/roles/bootstrap/install_qat_drivers_services/vars/main.yml +++ b/roles/bootstrap/install_qat_drivers_services/vars/main.yml @@ -1,5 +1,5 @@ ## -## Copyright (c) 2020-2021 Intel Corporation. +## Copyright (c) 2020-2022 Intel Corporation. ## ## Licensed under the Apache License, Version 2.0 (the "License"); ## you may not use this file except in compliance with the License. diff --git a/roles/bootstrap/install_rt_package/tasks/main.yml b/roles/bootstrap/install_rt_package/tasks/main.yml deleted file mode 100644 index 66f08397..00000000 --- a/roles/bootstrap/install_rt_package/tasks/main.yml +++ /dev/null @@ -1,85 +0,0 @@ -## -## Copyright (c) 2020-2021 Intel Corporation. -## -## Licensed under the Apache License, Version 2.0 (the "License"); -## you may not use this file except in compliance with the License. -## You may obtain a copy of the License at -## -## http://www.apache.org/licenses/LICENSE-2.0 -## -## Unless required by applicable law or agreed to in writing, software -## distributed under the License is distributed on an "AS IS" BASIS, -## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -## See the License for the specific language governing permissions and -## limitations under the License. -## ---- -- name: install dependencies - include_role: - name: install_dependencies - -- name: add CentOS-RT repo - template: - src: CentOS-RT.repo - dest: /etc/yum.repos.d/CentOS-RT.repo - mode: preserve - -- name: import cern repository GPG key to RPM - rpm_key: - key: http://linuxsoft.cern.ch/cern/centos/7/os/x86_64/RPM-GPG-KEY-cern - state: present - -- name: download tuned noarch package - get_url: - url: "{{ tuned_noarch_url }}" - dest: /usr/src - register: tuned_download - -- name: remove vanilla tuned.noarch package - package: - name: tuned.noarch - state: absent - -- name: install tuned.noarch package - package: - name: "{{ tuned_download.dest }}" - state: present - -- name: install RT kernel on Red Hat - package: - name: - - kernel-rt - - kernel-rt-devel - - kernel-rt-kvm - - rtctl - - rt-setup - - rt-tests - - tuna - - tuned-profiles-nfv - - tuned-profiles-nfv-host - - tuned-profiles-nfv-guest - - qemu-kvm-tools-ev - state: present - -- name: set default isolated cores flags values for tuned - set_fact: - isolated_cores: "" - -- name: validate isolcpus settings - assert: - that: isolcpus | length > 0 - fail_msg: "CPU isolation enabled, but list of CPUs to isolate is not defined, make sure that 'isolcpus' variable is set" - when: isolcpus_enabled - -- name: set isolated_cores flag - set_fact: - isolated_cores: "isolated_cores={{ isolcpus }}" - when: isolcpus_enabled | default(false) - -- name: activate Real-Time Profile - shell: echo "{{ isolated_cores }}" > /etc/tuned/realtime-virtual-host-variables.conf && tuned-adm profile realtime-virtual-host - changed_when: true - -- name: reboot after kernel update - reboot: - reboot_timeout: 1200 # wait up to 20 minutes diff --git a/roles/bootstrap/install_rt_package/templates/CentOS-RT.repo b/roles/bootstrap/install_rt_package/templates/CentOS-RT.repo deleted file mode 100644 index 9865d988..00000000 --- a/roles/bootstrap/install_rt_package/templates/CentOS-RT.repo +++ /dev/null @@ -1,64 +0,0 @@ -# -# CERN CentOS 7 RealTime repository at http://linuxsoft.cern.ch/ -# - -[rt] -name=CentOS-$releasever - RealTime -baseurl=http://linuxsoft.cern.ch/cern/centos/$releasever/rt/$basearch/ -gpgcheck=1 -enabled=1 -protect=1 -priority=10 -gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-cern - -[rt-debug] -name=CentOS-$releasever - RealTime - Debuginfo -baseurl=http://linuxsoft.cern.ch/cern/centos/$releasever/rt/Debug/$basearch/ -gpgcheck=1 -enabled=0 -protect=1 -priority=10 -gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-cern - -[rt-source] -name=CentOS-$releasever - RealTime - Sources -baseurl=http://linuxsoft.cern.ch/cern/centos/$releasever/rt/Sources/ -gpgcheck=1 -enabled=0 -protect=1 -priority=10 -gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-cern - -[rt-testing] -name=CentOS-$releasever - RealTime Testing -baseurl=http://linuxsoft.cern.ch/cern/centos/$releasever/rt-testing/$basearch/ -gpgcheck=1 -enabled=0 -protect=1 -priority=10 -gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-cern - -[rt-testing-debug] -name=CentOS-$releasever - RealTime Testing - Debuginfo -baseurl=http://linuxsoft.cern.ch/cern/centos/$releasever/rt-testing/Debug/$basearch/ -gpgcheck=1 -enabled=0 -protect=1 -priority=10 -gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-cern - -[rt-testing-source] -name=CentOS-$releasever - RealTime Testing - Sources -baseurl=http://linuxsoft.cern.ch/cern/centos/$releasever/rt-testing/Sources/ -gpgcheck=1 -enabled=0 -protect=1 -priority=10 -gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-cern - -[kvm] -name=CentOS-kvm-tool -baseurl=http://mirror.centos.org/centos-$releasever/$releasever/virt/$basearch/kvm-common/ -gpgcheck=0 -gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-7 -enabled=1 diff --git a/roles/bootstrap/set_rdt_kernel_flags/defaults/main.yml b/roles/bootstrap/set_rdt_kernel_flags/defaults/main.yml index adf7301b..6548a67c 100644 --- a/roles/bootstrap/set_rdt_kernel_flags/defaults/main.yml +++ b/roles/bootstrap/set_rdt_kernel_flags/defaults/main.yml @@ -1,5 +1,5 @@ ## -## Copyright (c) 2020-2021 Intel Corporation. +## Copyright (c) 2020-2022 Intel Corporation. ## ## Licensed under the Apache License, Version 2.0 (the "License"); ## you may not use this file except in compliance with the License. diff --git a/roles/bootstrap/set_rdt_kernel_flags/tasks/main.yml b/roles/bootstrap/set_rdt_kernel_flags/tasks/main.yml index 4e460a0c..410bda7e 100644 --- a/roles/bootstrap/set_rdt_kernel_flags/tasks/main.yml +++ b/roles/bootstrap/set_rdt_kernel_flags/tasks/main.yml @@ -1,5 +1,5 @@ ## -## Copyright (c) 2020-2021 Intel Corporation. +## Copyright (c) 2020-2022 Intel Corporation. ## ## Licensed under the Apache License, Version 2.0 (the "License"); ## you may not use this file except in compliance with the License. diff --git a/roles/bootstrap/set_sriov_kernel_flags/tasks/main.yml b/roles/bootstrap/set_sriov_kernel_flags/tasks/main.yml index ca0a1b83..0ce60461 100644 --- a/roles/bootstrap/set_sriov_kernel_flags/tasks/main.yml +++ b/roles/bootstrap/set_sriov_kernel_flags/tasks/main.yml @@ -1,5 +1,5 @@ ## -## Copyright (c) 2020-2021 Intel Corporation. +## Copyright (c) 2020-2022 Intel Corporation. ## ## Licensed under the Apache License, Version 2.0 (the "License"); ## you may not use this file except in compliance with the License. @@ -27,4 +27,4 @@ - name: setup sriov grub commandline parameters include_tasks: setup_sriov_kernel_flags.yml when: - - iommu_enabled | default(false) + - iommu_enabled | default(false) | bool or (on_vms is defined and on_vms) diff --git a/roles/bootstrap/set_sriov_kernel_flags/tasks/setup_sriov_kernel_flags.yml b/roles/bootstrap/set_sriov_kernel_flags/tasks/setup_sriov_kernel_flags.yml index 953df90a..69591245 100644 --- a/roles/bootstrap/set_sriov_kernel_flags/tasks/setup_sriov_kernel_flags.yml +++ b/roles/bootstrap/set_sriov_kernel_flags/tasks/setup_sriov_kernel_flags.yml @@ -1,5 +1,5 @@ ## -## Copyright (c) 2020-2021 Intel Corporation. +## Copyright (c) 2020-2022 Intel Corporation. ## ## Licensed under the Apache License, Version 2.0 (the "License"); ## you may not use this file except in compliance with the License. @@ -18,25 +18,35 @@ set_fact: vfio_cmdline: "" -- name: set vfio kernel flags for kernel >=5.9 in Ubuntu and for kernel >=4.8 in RHEL +- name: set vfio kernel flags for kernel >=5.9 or specific RHEL/CentOS versions set_fact: vfio_cmdline: " vfio-pci.disable_denylist=1" when: - qat_devices is defined and (qat_devices|length>0) - - iommu_enabled | default(false) - install_dpdk | default(false) - ansible_distribution == "Ubuntu" and ansible_distribution_version >= '21.04' or - (ansible_distribution == "RedHat" and ansible_distribution_version >= '8.4') + (ansible_distribution == "RedHat" and ansible_distribution_version >= '8.4') or + (ansible_distribution == "CentOS" and ansible_distribution_version >= '8.5') + +- name: set noiommu default kernel flags + set_fact: + iommu_cmdline: "" + +- name: set iommu default kernel flags + set_fact: + iommu_cmdline: " intel_iommu=on iommu=pt" + when: + - iommu_enabled | default(false) | bool - name: set sriov kernel flags set_fact: - iommu_cmdline: 'GRUB_CMDLINE_LINUX="${GRUB_CMDLINE_LINUX} intel_iommu=on iommu=pt{{ vfio_cmdline }}" {{ sriov_marker }}' + sriov_cmdline: 'GRUB_CMDLINE_LINUX="${GRUB_CMDLINE_LINUX}{{ iommu_cmdline }}{{ vfio_cmdline }}" {{ sriov_marker }}' - name: set sriov kernel flags in /etc/default/grub lineinfile: dest: /etc/default/grub regexp: '^GRUB_CMDLINE_LINUX="\${GRUB_CMDLINE_LINUX}(.*?)" {{ sriov_marker }}$' - line: '{{ iommu_cmdline }}' + line: '{{ sriov_cmdline }}' state: present mode: 0664 register: grub diff --git a/roles/bootstrap/set_sriov_kernel_flags/vars/main.yml b/roles/bootstrap/set_sriov_kernel_flags/vars/main.yml index 62582a04..faecf1c5 100644 --- a/roles/bootstrap/set_sriov_kernel_flags/vars/main.yml +++ b/roles/bootstrap/set_sriov_kernel_flags/vars/main.yml @@ -1,5 +1,5 @@ ## -## Copyright (c) 2020-2021 Intel Corporation. +## Copyright (c) 2020-2022 Intel Corporation. ## ## Licensed under the Apache License, Version 2.0 (the "License"); ## you may not use this file except in compliance with the License. diff --git a/roles/bootstrap/update_grub/tasks/main.yml b/roles/bootstrap/update_grub/tasks/main.yml index 5c7b83c6..5b44815e 100644 --- a/roles/bootstrap/update_grub/tasks/main.yml +++ b/roles/bootstrap/update_grub/tasks/main.yml @@ -1,5 +1,5 @@ ## -## Copyright (c) 2020-2021 Intel Corporation. +## Copyright (c) 2020-2022 Intel Corporation. ## ## Licensed under the Apache License, Version 2.0 (the "License"); ## you may not use this file except in compliance with the License. diff --git a/roles/bootstrap/update_nic_drivers/defaults/main.yml b/roles/bootstrap/update_nic_drivers/defaults/main.yml index 16ef33d6..8d1dc793 100644 --- a/roles/bootstrap/update_nic_drivers/defaults/main.yml +++ b/roles/bootstrap/update_nic_drivers/defaults/main.yml @@ -1,5 +1,5 @@ ## -## Copyright (c) 2020-2021 Intel Corporation. +## Copyright (c) 2020-2022 Intel Corporation. ## ## Licensed under the Apache License, Version 2.0 (the "License"); ## you may not use this file except in compliance with the License. @@ -16,18 +16,18 @@ --- # i40e i40e_driver_name: i40e -i40e_driver_version: 2.16.11 +i40e_driver_version: 2.17.15 i40e_driver_url: https://sourceforge.net/projects/e1000/files/i40e%20stable/{{ i40e_driver_version }}/i40e-{{ i40e_driver_version }}.tar.gz -i40e_driver_checksum: sha1:8fbfb9d0bf8feec0c74a5dc150613b430921fdcd +i40e_driver_checksum: sha1:e329cbb71f08c2902f8da1a54cba29c761199f61 # ice ice_driver_name: ice -ice_driver_version: 1.6.4 +ice_driver_version: 1.7.16 ice_driver_url: https://sourceforge.net/projects/e1000/files/ice%20stable/{{ ice_driver_version }}/ice-{{ ice_driver_version }}.tar.gz -ice_driver_checksum: sha1:9e860bf3cafcabd1d4897e87e749334f73828bad +ice_driver_checksum: sha1:8e7333c33fba3dd47048b8236875ca66d40e97b5 # iavf iavf_driver_name: iavf -iavf_driver_version: 4.2.7 +iavf_driver_version: 4.3.19 iavf_driver_url: https://sourceforge.net/projects/e1000/files/iavf%20stable/{{ iavf_driver_version }}/iavf-{{ iavf_driver_version }}.tar.gz -iavf_driver_checksum: sha1:5b0f144a60bdfcc5928f78691dc42cb85c2ed734 +iavf_driver_checksum: sha1:085ba1f0a1d99161e685bf66664a36dc335f91fa diff --git a/roles/bootstrap/update_nic_drivers/tasks/i40e.yml b/roles/bootstrap/update_nic_drivers/tasks/i40e.yml index c5b02a6e..904e448a 100644 --- a/roles/bootstrap/update_nic_drivers/tasks/i40e.yml +++ b/roles/bootstrap/update_nic_drivers/tasks/i40e.yml @@ -1,5 +1,5 @@ ## -## Copyright (c) 2020-2021 Intel Corporation. +## Copyright (c) 2020-2022 Intel Corporation. ## ## Licensed under the Apache License, Version 2.0 (the "License"); ## you may not use this file except in compliance with the License. @@ -23,17 +23,6 @@ - debug: msg: "Currently installed i40e version: {{ i40e_installed_version.stdout }}" -- name: download i40e driver - get_url: - url: "{{ i40e_driver_url }}" - dest: /usr/src - checksum: "{{ i40e_driver_checksum }}" - timeout: 60 - register: i40e_download - until: i40e_download is not failed - retries: 5 - when: i40e_installed_version.stdout != i40e_driver_version - - name: unload i40e module modprobe: name: i40e @@ -42,25 +31,36 @@ - i40e_installed_version.stdout != i40e_driver_version - mgmt_interface_driver.stdout != i40e_driver_name -- name: untar i40e driver sources - unarchive: - src: "{{ i40e_download.dest }}" - dest: /usr/src - list_files: yes - remote_src: yes - mode: 0644 - become: yes - register: i40e_untar - when: i40e_installed_version.stdout != i40e_driver_version +- name: update i40e driver + block: + - name: download i40e driver + get_url: + url: "{{ i40e_driver_url }}" + dest: "{{ project_root_dir }}" + checksum: "{{ i40e_driver_checksum }}" + timeout: 60 + register: i40e_download + until: i40e_download is not failed + retries: 5 -- name: build and install i40e driver - make: - chdir: "{{ i40e_untar.dest }}/{{ i40e_untar.files[0] }}/src" - target: "{{ item }}" - become: yes - with_items: - - clean - - install + - name: untar i40e driver sources + unarchive: + src: "{{ i40e_download.dest }}" + dest: "{{ project_root_dir }}" + list_files: yes + remote_src: yes + mode: 0774 + become: yes + register: i40e_untar + + - name: build and install i40e driver + make: + chdir: "{{ i40e_untar.dest }}/{{ i40e_untar.files[0] }}/src" + target: "{{ item }}" + become: yes + with_items: + - clean + - install when: i40e_installed_version.stdout != i40e_driver_version - name: reboot if driver is used by management interface diff --git a/roles/bootstrap/update_nic_drivers/tasks/iavf.yml b/roles/bootstrap/update_nic_drivers/tasks/iavf.yml index 79165d9c..9292c9df 100644 --- a/roles/bootstrap/update_nic_drivers/tasks/iavf.yml +++ b/roles/bootstrap/update_nic_drivers/tasks/iavf.yml @@ -1,5 +1,5 @@ ## -## Copyright (c) 2020-2021 Intel Corporation. +## Copyright (c) 2020-2022 Intel Corporation. ## ## Licensed under the Apache License, Version 2.0 (the "License"); ## you may not use this file except in compliance with the License. @@ -23,17 +23,6 @@ - debug: msg: "Currently installed iavf version: {{ iavf_installed_version.stdout }}" -- name: download iavf driver - get_url: - url: "{{ iavf_driver_url }}" - dest: /usr/src - checksum: "{{ iavf_driver_checksum }}" - timeout: 60 - register: iavf_download - until: iavf_download is not failed - retries: 5 - when: iavf_installed_version.stdout != iavf_driver_version - - name: unload iavf module modprobe: name: iavf @@ -42,25 +31,36 @@ - iavf_installed_version.stdout != iavf_driver_version - mgmt_interface_driver.stdout != iavf_driver_name -- name: untar iavf driver sources - unarchive: - src: "{{ iavf_download.dest }}" - dest: /usr/src - remote_src: yes - list_files: yes - mode: 0644 - become: yes - register: iavf_untar - when: iavf_installed_version.stdout != iavf_driver_version +- name: update iavf driver + block: + - name: download iavf driver + get_url: + url: "{{ iavf_driver_url }}" + dest: "{{ project_root_dir }}" + checksum: "{{ iavf_driver_checksum }}" + timeout: 60 + register: iavf_download + until: iavf_download is not failed + retries: 5 -- name: build and install iavf driver - make: - chdir: "{{ iavf_untar.dest }}/{{ iavf_untar.files[0] }}/src" - target: "{{ item }}" - become: yes - with_items: - - clean - - install + - name: untar iavf driver sources + unarchive: + src: "{{ iavf_download.dest }}" + dest: "{{ project_root_dir }}" + remote_src: yes + list_files: yes + mode: 0774 + become: yes + register: iavf_untar + + - name: build and install iavf driver + make: + chdir: "{{ iavf_untar.dest }}/{{ iavf_untar.files[0] }}/src" + target: "{{ item }}" + become: yes + with_items: + - clean + - install when: iavf_installed_version.stdout != iavf_driver_version - name: reboot if driver is used by management interface diff --git a/roles/bootstrap/update_nic_drivers/tasks/ice.yml b/roles/bootstrap/update_nic_drivers/tasks/ice.yml index 3b3af4f4..0d2c182e 100644 --- a/roles/bootstrap/update_nic_drivers/tasks/ice.yml +++ b/roles/bootstrap/update_nic_drivers/tasks/ice.yml @@ -1,5 +1,5 @@ ## -## Copyright (c) 2020-2021 Intel Corporation. +## Copyright (c) 2020-2022 Intel Corporation. ## ## Licensed under the Apache License, Version 2.0 (the "License"); ## you may not use this file except in compliance with the License. @@ -23,17 +23,6 @@ - debug: msg: "Currently installed ice version: {{ ice_installed_version.stdout }}" -- name: download ice driver - become: yes - get_url: - url: "{{ ice_driver_url }}" - dest: /usr/src - checksum: "{{ ice_driver_checksum }}" - timeout: 60 - register: ice_download - until: ice_download is not failed - retries: 5 - - name: unload ice module modprobe: name: ice @@ -42,24 +31,37 @@ - ice_installed_version.stdout != ice_driver_version - mgmt_interface_driver.stdout != ice_driver_name -- name: untar ice driver sources - unarchive: - src: "{{ ice_download.dest }}" - dest: /usr/src - list_files: yes - remote_src: yes - mode: 0644 - become: yes - register: ice_untar +- name: update ice driver + block: + - name: download ice driver + become: yes + get_url: + url: "{{ ice_driver_url }}" + dest: "{{ project_root_dir }}" + checksum: "{{ ice_driver_checksum }}" + timeout: 60 + register: ice_download + until: ice_download is not failed + retries: 5 -- name: build and install ice driver - make: - chdir: "{{ ice_untar.dest }}/{{ ice_untar.files[0] }}/src" - target: "{{ item }}" - become: yes - loop: - - clean - - install + - name: untar ice driver sources + unarchive: + src: "{{ ice_download.dest }}" + dest: "{{ project_root_dir }}" + list_files: yes + remote_src: yes + mode: 0774 + become: yes + register: ice_untar + + - name: build and install ice driver + make: + chdir: "{{ ice_untar.dest }}/{{ ice_untar.files[0] }}/src" + target: "{{ item }}" + become: yes + loop: + - clean + - install when: ice_installed_version.stdout != ice_driver_version - name: reboot if driver is used by management interface @@ -75,15 +77,3 @@ name: ice state: present -- name: get dataplane_interface Device IDs - shell: "set -o pipefail && lshw -numeric -businfo -c network | grep -Poi '^.*?{{ item.bus_info }}.*?8086:\\K[0-9a-fA-F]*'" - args: - executable: /bin/bash - register: device_id - changed_when: false - with_items: "{{ dataplane_interfaces }}" - -- name: wait 5 min for CPK devices to initialize - pause: - minutes: 5 - when: '"124c" in device_id.results[0].stdout' diff --git a/roles/bootstrap/update_nic_drivers/tasks/main.yml b/roles/bootstrap/update_nic_drivers/tasks/main.yml index 8452523c..6071ce5b 100644 --- a/roles/bootstrap/update_nic_drivers/tasks/main.yml +++ b/roles/bootstrap/update_nic_drivers/tasks/main.yml @@ -1,5 +1,5 @@ ## -## Copyright (c) 2020-2021 Intel Corporation. +## Copyright (c) 2020-2022 Intel Corporation. ## ## Licensed under the Apache License, Version 2.0 (the "License"); ## you may not use this file except in compliance with the License. @@ -42,7 +42,7 @@ - name: determine custom kernel source dir find: - path: /usr/src/kernels + path: "{{ project_root_dir }}/kernels" file_type: directory patterns: 'gpu_kernel' recurse: yes diff --git a/roles/bootstrap/update_nic_drivers/vars/main.yml b/roles/bootstrap/update_nic_drivers/vars/main.yml index d9754c46..3e3c25e5 100644 --- a/roles/bootstrap/update_nic_drivers/vars/main.yml +++ b/roles/bootstrap/update_nic_drivers/vars/main.yml @@ -1,5 +1,5 @@ ## -## Copyright (c) 2020-2021 Intel Corporation. +## Copyright (c) 2020-2022 Intel Corporation. ## ## Licensed under the Apache License, Version 2.0 (the "License"); ## you may not use this file except in compliance with the License. diff --git a/roles/bootstrap/update_nic_firmware/defaults/main.yml b/roles/bootstrap/update_nic_firmware/defaults/main.yml new file mode 100644 index 00000000..fb8176e1 --- /dev/null +++ b/roles/bootstrap/update_nic_firmware/defaults/main.yml @@ -0,0 +1,33 @@ +## +## Copyright (c) 2020-2022 Intel Corporation. +## +## Licensed under the Apache License, Version 2.0 (the "License"); +## you may not use this file except in compliance with the License. +## You may obtain a copy of the License at +## +## http://www.apache.org/licenses/LICENSE-2.0 +## +## Unless required by applicable law or agreed to in writing, software +## distributed under the License is distributed on an "AS IS" BASIS, +## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +## See the License for the specific language governing permissions and +## limitations under the License. +## +--- +# nvmupdate package URLs, checksums and minimum target FW version required (e.g. to enable DDP profiles loading) +nvmupdate: + # 700 Series + i40e: + nvmupdate_pkg_url: https://downloadmirror.intel.com/682041/700Series_NVMUpdatePackage_v8_50_Linux.tar.gz + nvmupdate_pkg_checksum: sha1:C668839B1D55470D0A889468EC026CE79A60450D + min_required_fw_version: 8.50 + min_updatable_fw_version: 5.02 + # 800 Series (CVL) + ice: + nvmupdate_pkg_url: https://downloadmirror.intel.com/29738/eng/e810_nvmupdatepackage_v3_00_linux.tar_.gz + nvmupdate_pkg_checksum: sha1:E03C3F9CFB042EAE5C1C16BED352E09FCDEBD0E6 + min_required_fw_version: 3.00 + min_updatable_fw_version: 0.70 + +nvmupdate_result: + stdout: "" diff --git a/roles/bootstrap/update_nic_firmware/tasks/main.yml b/roles/bootstrap/update_nic_firmware/tasks/main.yml index 48f4761d..2e1d678f 100644 --- a/roles/bootstrap/update_nic_firmware/tasks/main.yml +++ b/roles/bootstrap/update_nic_firmware/tasks/main.yml @@ -1,5 +1,5 @@ ## -## Copyright (c) 2020-2021 Intel Corporation. +## Copyright (c) 2020-2022 Intel Corporation. ## ## Licensed under the Apache License, Version 2.0 (the "License"); ## you may not use this file except in compliance with the License. @@ -14,6 +14,18 @@ ## limitations under the License. ## --- +- name: check interfaces in system + command: "inxi -n" + register: check_nics_in_system + changed_when: true + +- name: validate host vars interface names with system interface names before FW update + assert: + that: "item.name in check_nics_in_system.stdout" + fail_msg: "host vars defined dataplane interface name != interface name found in system. Kindly, select correct interface name" + success_msg: "host vars defined dataplane interface name = interface name found in system, verification completed" + with_items: "{{ dataplane_interfaces }}" + - name: update NIC firmware include_tasks: update.yml vars: @@ -23,5 +35,5 @@ # "i40e" (FVL) or "ice" (CVL) - ansible_facts[item]['module'] in ['i40e', 'ice'] - update_nic_firmware | default(false) | bool - # update all dataplane_interfaces and all additional firmware_update_nics (uesful in basic profile where SRIOV is not used) - with_items: "{{ dataplane_interfaces | map(attribute='name') | list + firmware_update_nics | unique }}" + # update all dataplane_interfaces + with_items: "{{ dataplane_interfaces | map(attribute='name') | list | unique }}" diff --git a/roles/bootstrap/update_nic_firmware/tasks/update.yml b/roles/bootstrap/update_nic_firmware/tasks/update.yml index fabc7712..5037ed93 100644 --- a/roles/bootstrap/update_nic_firmware/tasks/update.yml +++ b/roles/bootstrap/update_nic_firmware/tasks/update.yml @@ -1,5 +1,5 @@ ## -## Copyright (c) 2020-2021 Intel Corporation. +## Copyright (c) 2020-2022 Intel Corporation. ## ## Licensed under the Apache License, Version 2.0 (the "License"); ## you may not use this file except in compliance with the License. @@ -13,6 +13,7 @@ ## See the License for the specific language governing permissions and ## limitations under the License. ## + # Usage: include this tasks file and pass vars listed below # Variables: # interface_name: interface name of the PF (mandatory, string) diff --git a/roles/bootstrap/update_nic_firmware/vars/main.yml b/roles/bootstrap/update_nic_firmware/vars/main.yml index 76946a2f..543176ba 100644 --- a/roles/bootstrap/update_nic_firmware/vars/main.yml +++ b/roles/bootstrap/update_nic_firmware/vars/main.yml @@ -1,5 +1,5 @@ ## -## Copyright (c) 2020-2021 Intel Corporation. +## Copyright (c) 2020-2022 Intel Corporation. ## ## Licensed under the Apache License, Version 2.0 (the "License"); ## you may not use this file except in compliance with the License. @@ -14,24 +14,6 @@ ## limitations under the License. ## --- -# nvmupdate package URLs, checksums and minimum target FW version required (e.g. to enable DDP profiles loading) -nvmupdate: - # 700 Series - i40e: - nvmupdate_pkg_url: https://downloadmirror.intel.com/25791/eng/700series_nvmupdatepackage_v8_40_linux.tar.gz - nvmupdate_pkg_checksum: sha1:D9DF143BF7B1EAF172C5BFA70216FFD7EBCBF980 - min_required_fw_version: 8.40 - min_updatable_fw_version: 5.51 - # 800 Series (CVL) - ice: - nvmupdate_pkg_url: https://downloadmirror.intel.com/29738/eng/e810_nvmupdatepackage_v3_00_linux.tar_.gz - nvmupdate_pkg_checksum: sha1:E03C3F9CFB042EAE5C1C16BED352E09FCDEBD0E6 - min_required_fw_version: 3.00 - min_updatable_fw_version: 0.70 - -nvmupdate_result: - stdout: "" - # dependencies install_dependencies: Debian: diff --git a/roles/cert_manager_install/defaults/main.yml b/roles/cert_manager_install/defaults/main.yml index 454f39a0..27383a33 100644 --- a/roles/cert_manager_install/defaults/main.yml +++ b/roles/cert_manager_install/defaults/main.yml @@ -1,5 +1,5 @@ ## -## Copyright (c) 2020-2021 Intel Corporation. +## Copyright (c) 2020-2022 Intel Corporation. ## ## Licensed under the Apache License, Version 2.0 (the "License"); ## you may not use this file except in compliance with the License. @@ -17,4 +17,3 @@ cert_manager_namespace: "cert-manager" cert_manager_version: "v1.5.1" cert_manager_repo: "https://charts.jetstack.io" - diff --git a/roles/cert_manager_install/tasks/main.yml b/roles/cert_manager_install/tasks/main.yml index 35132b01..9970a68a 100644 --- a/roles/cert_manager_install/tasks/main.yml +++ b/roles/cert_manager_install/tasks/main.yml @@ -1,5 +1,5 @@ ## -## Copyright (c) 2020-2021 Intel Corporation. +## Copyright (c) 2020-2022 Intel Corporation. ## ## Licensed under the Apache License, Version 2.0 (the "License"); ## you may not use this file except in compliance with the License. @@ -43,4 +43,3 @@ - name: wait for cert-manager to become fully available pause: seconds: 30 - diff --git a/roles/check_machine_type/tasks/main.yml b/roles/check_machine_type/tasks/main.yml index 3714ec93..01f3e5d7 100644 --- a/roles/check_machine_type/tasks/main.yml +++ b/roles/check_machine_type/tasks/main.yml @@ -1,5 +1,5 @@ ## -## Copyright (c) 2020-2021 Intel Corporation. +## Copyright (c) 2020-2022 Intel Corporation. ## ## Licensed under the Apache License, Version 2.0 (the "License"); ## you may not use this file except in compliance with the License. @@ -39,3 +39,13 @@ is_icx: True when: item in cpu_model.stdout loop: "{{ supported_icx_skus }}" + +- name: set is_spr to false + set_fact: + is_spr: False + +- name: check if spr mode + set_fact: + is_spr: True + when: item in cpu_model.stdout + loop: "{{ supported_spr_skus }}" diff --git a/roles/check_machine_type/vars/main.yml b/roles/check_machine_type/vars/main.yml index 0ac21931..d928b004 100644 --- a/roles/check_machine_type/vars/main.yml +++ b/roles/check_machine_type/vars/main.yml @@ -1,5 +1,5 @@ ## -## Copyright (c) 2020-2021 Intel Corporation. +## Copyright (c) 2020-2022 Intel Corporation. ## ## Licensed under the Apache License, Version 2.0 (the "License"); ## you may not use this file except in compliance with the License. @@ -14,6 +14,7 @@ ## limitations under the License. ## --- +# NOTE(pklimowx): these lists should be updated with complete set of machine SKUs supported_clx_skus: - "6252N" - "6230N" @@ -25,3 +26,7 @@ supported_icx_skus: - "6338N" - "5318N" - "06/6c" + - "6348" + +supported_spr_skus: + - "0000%@" diff --git a/roles/cluster_defaults/defaults/main.yml b/roles/cluster_defaults/defaults/main.yml index 9b626821..6b15069d 100644 --- a/roles/cluster_defaults/defaults/main.yml +++ b/roles/cluster_defaults/defaults/main.yml @@ -1,5 +1,5 @@ ## -## Copyright (c) 2020-2021 Intel Corporation. +## Copyright (c) 2020-2022 Intel Corporation. ## ## Licensed under the Apache License, Version 2.0 (the "License"); ## you may not use this file except in compliance with the License. diff --git a/roles/cluster_defaults/tasks/main.yml b/roles/cluster_defaults/tasks/main.yml index a152e526..ab436b2b 100644 --- a/roles/cluster_defaults/tasks/main.yml +++ b/roles/cluster_defaults/tasks/main.yml @@ -1,5 +1,5 @@ ## -## Copyright (c) 2020-2021 Intel Corporation. +## Copyright (c) 2020-2022 Intel Corporation. ## ## Licensed under the Apache License, Version 2.0 (the "License"); ## you may not use this file except in compliance with the License. @@ -17,3 +17,9 @@ - name: Set defaults debug: msg: "Setting defaults from roles/cluster_defaults/defaults/main.yml" + +- name: Create Container Experience Kit directory if needed + file: + state: directory + path: "{{ project_root_dir }}" + mode: 0644 diff --git a/roles/cmk_install/charts/cpu-manager-for-kubernetes/templates/NOTES.txt b/roles/cmk_install/charts/cpu-manager-for-kubernetes/templates/NOTES.txt deleted file mode 100644 index cf07c1b8..00000000 --- a/roles/cmk_install/charts/cpu-manager-for-kubernetes/templates/NOTES.txt +++ /dev/null @@ -1,10 +0,0 @@ -{{ .Chart.Name }} was installed . - -Your release is named {{ .Release.Name }}. - -To learn more about the release, try: - - $ helm status {{ .Release.Name }} - $ helm get {{ .Release.Name }} - - diff --git a/roles/cmk_install/charts/cpu-manager-for-kubernetes/templates/_helpers.tpl b/roles/cmk_install/charts/cpu-manager-for-kubernetes/templates/_helpers.tpl deleted file mode 100644 index 39a63b21..00000000 --- a/roles/cmk_install/charts/cpu-manager-for-kubernetes/templates/_helpers.tpl +++ /dev/null @@ -1,32 +0,0 @@ -{{/* vim: set filetype=mustache: */}} -{{/* -Expand the name of the chart. -*/}} -{{- define "cmk.name" -}} -{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}} -{{- end -}} - -{{/* -Create a default fully qualified app name. -We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). -If release name contains chart name it will be used as a full name. -*/}} -{{- define "cmk.fullname" -}} -{{- if .Values.fullnameOverride -}} -{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}} -{{- else -}} -{{- $name := default .Chart.Name .Values.nameOverride -}} -{{- if contains $name .Release.Name -}} -{{- .Release.Name | trunc 63 | trimSuffix "-" -}} -{{- else -}} -{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}} -{{- end -}} -{{- end -}} -{{- end -}} - -{{/* -Create chart name and version as used by the chart label. -*/}} -{{- define "cmk.chart" -}} -{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" -}} -{{- end -}} diff --git a/roles/cmk_install/charts/cpu-manager-for-kubernetes/templates/daemonset.yml b/roles/cmk_install/charts/cpu-manager-for-kubernetes/templates/daemonset.yml deleted file mode 100644 index ceac1a92..00000000 --- a/roles/cmk_install/charts/cpu-manager-for-kubernetes/templates/daemonset.yml +++ /dev/null @@ -1,79 +0,0 @@ ---- -apiVersion: apps/v1 -kind: DaemonSet -metadata: - name: {{ include "cmk.fullname" . }} - labels: - app: {{ include "cmk.fullname" . }} -spec: - selector: - matchLabels: - name: {{ include "cmk.fullname" . }} - template: - metadata: - labels: - name: {{ include "cmk.fullname" . }} - spec: - {{- if not .Values.hosts.all }} - affinity: - nodeAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - nodeSelectorTerms: - {{- range splitList "," .Values.hosts.list }} - - matchFields: - - key: metadata.name - operator: In - values: - - {{ . }} - {{- end }} - {{- end }} - serviceAccountName: {{ include "cmk.fullname" . }} - tolerations: - - key: cmk - operator: Exists - containers: - - name: reconcile - image: {{ .Values.image.repository }}:{{ .Values.image.tag }} - imagePullPolicy: {{ .Values.image.pullPolicy }} - env: - - name: CMK_RECONCILE_SLEEP_TIME - value: '60' - - name: CMK_PROC_FS - value: /host/proc - - name: NODE_NAME - valueFrom: - fieldRef: - apiVersion: v1 - fieldPath: spec.nodeName - command: ["/bin/bash", "-c"] - args: - - "/cmk/cmk.py isolate --pool=infra /cmk/cmk.py -- reconcile --interval=$CMK_RECONCILE_SLEEP_TIME --publish" - volumeMounts: - - mountPath: /host/proc - name: host-proc - readOnly: true - - name: nodereport - image: {{ .Values.image.repository }}:{{ .Values.image.tag }} - imagePullPolicy: {{ .Values.image.pullPolicy }} - env: - - name: CMK_NODE_REPORT_SLEEP_TIME - value: '60' - - name: CMK_PROC_FS - value: /host/proc - - name: NODE_NAME - valueFrom: - fieldRef: - apiVersion: v1 - fieldPath: spec.nodeName - command: ["/bin/bash", "-c"] - args: - - "/cmk/cmk.py isolate --pool=infra /cmk/cmk.py -- node-report --interval=$CMK_NODE_REPORT_SLEEP_TIME --publish" - volumeMounts: - - mountPath: /host/proc - name: host-proc - readOnly: true - volumes: - - hostPath: - path: /proc - type: "" - name: host-proc diff --git a/roles/cmk_install/charts/cpu-manager-for-kubernetes/templates/job.yml b/roles/cmk_install/charts/cpu-manager-for-kubernetes/templates/job.yml deleted file mode 100644 index 9655e48e..00000000 --- a/roles/cmk_install/charts/cpu-manager-for-kubernetes/templates/job.yml +++ /dev/null @@ -1,85 +0,0 @@ -{{- $fullName := include "cmk.fullname" . -}} -{{- $exclusiveMode := .Values.exclusiveMode -}} -{{- $numExclusiveCores := .Values.numExclusiveCores -}} -{{- $sharedMode := .Values.sharedMode -}} -{{- $numSharedCores := .Values.numSharedCores -}} -{{- $pullPolicy := .Values.image.pullPolicy -}} -{{- $image := .Values.image.repository -}} -{{- $tag := .Values.image.tag -}} -{{- $noTaint := .Values.noTaint }} -{{ range splitList "," .Values.hosts.list }} ---- -apiVersion: batch/v1 -kind: Job -metadata: - annotations: - helm.sh/hook: pre-install,pre-upgrade,pre-rollback - helm.sh/hook-weight: "10" - labels: - app: {{ $fullName }}-init-discover-{{ . }} - name: {{ $fullName }}-init-discover-{{ . }} -spec: - template: - spec: - nodeName: {{ . }} - serviceAccountName: {{ $fullName }} - restartPolicy: Never - tolerations: - - key: cmk - operator: Exists - containers: - - name: install - image: {{ $image }}:{{ $tag }} - imagePullPolicy: {{ $pullPolicy }} - command: ["/bin/bash", "-c"] - args: - - "/cmk/cmk.py install" - volumeMounts: - - mountPath: /opt/bin - name: cmk-install-dir - - name: init - image: {{ $image }}:{{ $tag }} - imagePullPolicy: {{ $pullPolicy }} - env: - - name: CMK_PROC_FS - value: "/host/proc" - - name: NODE_NAME - valueFrom: - fieldRef: - apiVersion: v1 - fieldPath: spec.nodeName - command: ["/bin/bash", "-c"] - args: - - "/cmk/cmk.py init --exclusive-mode={{ $exclusiveMode }} --num-exclusive-cores={{ $numExclusiveCores }} --shared-mode={{ $sharedMode }} --num-shared-cores={{ $numSharedCores }}" - volumeMounts: - - mountPath: /host/proc - name: host-proc - readOnly: true - - name: discover - image: {{ $image }}:{{ $tag }} - imagePullPolicy: {{ $pullPolicy }} - env: - - name: CMK_PROC_FS - value: /host/proc - - name: NODE_NAME - valueFrom: - fieldRef: - apiVersion: v1 - fieldPath: spec.nodeName - command: ["/bin/bash", "-c"] - args: - - "/cmk/cmk.py discover {{ if $noTaint }}--no-taint{{ end }}" - volumeMounts: - - mountPath: /host/proc - name: host-proc - readOnly: true - volumes: - - hostPath: - path: /proc - type: Directory - name: host-proc - - hostPath: - path: /opt/bin - type: DirectoryOrCreate - name: cmk-install-dir -{{ end }} diff --git a/roles/cmk_install/charts/cpu-manager-for-kubernetes/templates/rbac.yml b/roles/cmk_install/charts/cpu-manager-for-kubernetes/templates/rbac.yml deleted file mode 100644 index 7cbe043e..00000000 --- a/roles/cmk_install/charts/cpu-manager-for-kubernetes/templates/rbac.yml +++ /dev/null @@ -1,162 +0,0 @@ ---- -kind: ClusterRole -apiVersion: rbac.authorization.k8s.io/v1beta1 -metadata: - annotations: - helm.sh/hook: pre-install,pre-upgrade,pre-rollback - helm.sh/hook-weight: "5" - name: {{ include "cmk.fullname" . }}-third-party-resource-controller -rules: -- apiGroups: ["cmk.intel.com"] - resources: ["*"] - verbs: ["*"] -- apiGroups: ["extensions"] - resources: ["thirdpartyresources", "thirdpartyresources.extensions"] - verbs: ["*"] ---- -kind: ClusterRole -apiVersion: rbac.authorization.k8s.io/v1beta1 -metadata: - annotations: - helm.sh/hook: pre-install,pre-upgrade,pre-rollback - helm.sh/hook-weight: "5" - name: {{ include "cmk.fullname" . }}-custom-resource-definition-controller -rules: -- apiGroups: ["intel.com"] - resources: ["*"] - verbs: ["*"] -- apiGroups: ["apiextensions.k8s.io"] - resources: ["customresourcedefinitions", "customresourcedefinitions.extensions"] - verbs: ["*"] ---- -kind: ClusterRole -apiVersion: rbac.authorization.k8s.io/v1beta1 -metadata: - annotations: - helm.sh/hook: pre-install,pre-upgrade,pre-rollback - helm.sh/hook-weight: "5" - name: {{ include "cmk.fullname" . }}-daemonset-controller -rules: -- apiGroups: ["extensions"] - resources: ["daemonsets", "daemonsets.extensions"] - verbs: ["*"] ---- -kind: ClusterRole -apiVersion: rbac.authorization.k8s.io/v1beta1 -metadata: - annotations: - helm.sh/hook: pre-install,pre-upgrade,pre-rollback - helm.sh/hook-weight: "5" - name: {{ include "cmk.fullname" . }}-version-controller -rules: - - nonResourceURLs: ["*"] - verbs: - - get ---- -kind: ClusterRole -apiVersion: rbac.authorization.k8s.io/v1beta1 -metadata: - annotations: - helm.sh/hook: pre-install,pre-upgrade,pre-rollback - helm.sh/hook-weight: "5" - name: {{ include "cmk.fullname" . }}-webhook-installer -rules: -- apiGroups: ["", "apps", "extensions", "admissionregistration.k8s.io"] - resources: ["secrets", "configmaps", "deployments", "services", "mutatingwebhookconfigurations"] - verbs: ["*"] ---- -apiVersion: rbac.authorization.k8s.io/v1beta1 -kind: ClusterRoleBinding -metadata: - annotations: - helm.sh/hook: pre-install,pre-upgrade,pre-rollback - helm.sh/hook-weight: "5" - name: {{ include "cmk.fullname" . }}-role-binding-daemonset -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: {{ include "cmk.fullname" . }}-daemonset-controller -subjects: -- kind: ServiceAccount - name: {{ include "cmk.fullname" . }} - namespace: {{ .Release.Namespace }} ---- -apiVersion: rbac.authorization.k8s.io/v1beta1 -kind: ClusterRoleBinding -metadata: - annotations: - helm.sh/hook: pre-install,pre-upgrade,pre-rollback - helm.sh/hook-weight: "5" - name: {{ include "cmk.fullname" . }}-role-binding-node -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: system:node -subjects: -- kind: ServiceAccount - name: {{ include "cmk.fullname" . }} - namespace: {{ .Release.Namespace }} ---- -apiVersion: rbac.authorization.k8s.io/v1beta1 -kind: ClusterRoleBinding -metadata: - annotations: - helm.sh/hook: pre-install,pre-upgrade,pre-rollback - helm.sh/hook-weight: "5" - name: {{ include "cmk.fullname" . }}-role-binding-tpr -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: {{ include "cmk.fullname" . }}-third-party-resource-controller -subjects: -- kind: ServiceAccount - name: {{ include "cmk.fullname" . }} - namespace: {{ .Release.Namespace }} ---- -apiVersion: rbac.authorization.k8s.io/v1beta1 -kind: ClusterRoleBinding -metadata: - annotations: - helm.sh/hook: pre-install,pre-upgrade,pre-rollback - helm.sh/hook-weight: "5" - name: {{ include "cmk.fullname" . }}-role-binding-crd -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: {{ include "cmk.fullname" . }}-custom-resource-definition-controller -subjects: -- kind: ServiceAccount - name: {{ include "cmk.fullname" . }} - namespace: {{ .Release.Namespace }} ---- -apiVersion: rbac.authorization.k8s.io/v1beta1 -kind: ClusterRoleBinding -metadata: - annotations: - helm.sh/hook: pre-install,pre-upgrade,pre-rollback - helm.sh/hook-weight: "5" - name: {{ include "cmk.fullname" . }}-role-binding-version -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: {{ include "cmk.fullname" . }}-version-controller -subjects: -- kind: ServiceAccount - name: {{ include "cmk.fullname" . }} - namespace: {{ .Release.Namespace }} ---- -apiVersion: rbac.authorization.k8s.io/v1beta1 -kind: ClusterRoleBinding -metadata: - annotations: - helm.sh/hook: pre-install,pre-upgrade,pre-rollback - helm.sh/hook-weight: "5" - name: {{ include "cmk.fullname" . }}-role-binding-webhook-installer -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: {{ include "cmk.fullname" . }}-webhook-installer -subjects: -- kind: ServiceAccount - name: {{ include "cmk.fullname" . }} - namespace: {{ .Release.Namespace }} diff --git a/roles/cmk_install/charts/cpu-manager-for-kubernetes/templates/serviceaccount.yml b/roles/cmk_install/charts/cpu-manager-for-kubernetes/templates/serviceaccount.yml deleted file mode 100644 index 94dd81bb..00000000 --- a/roles/cmk_install/charts/cpu-manager-for-kubernetes/templates/serviceaccount.yml +++ /dev/null @@ -1,8 +0,0 @@ ---- -apiVersion: v1 -kind: ServiceAccount -metadata: - annotations: - helm.sh/hook: pre-install,pre-upgrade,pre-rollback - helm.sh/hook-weight: "0" - name: {{ include "cmk.fullname" . }} diff --git a/roles/cmk_install/charts/cpu-manager-for-kubernetes/templates/webhook.yml b/roles/cmk_install/charts/cpu-manager-for-kubernetes/templates/webhook.yml deleted file mode 100644 index 42ce95ae..00000000 --- a/roles/cmk_install/charts/cpu-manager-for-kubernetes/templates/webhook.yml +++ /dev/null @@ -1,131 +0,0 @@ -{{ $ca := genCA "cmk-webhook-ca" 365 }} -{{ $altNames := list ( printf "%s-webhook.%s" (include "cmk.fullname" .) .Release.Namespace ) ( printf "%s-webhook.%s.svc" (include "cmk.fullname" .) .Release.Namespace ) }} -{{ $cert := genSignedCert ( printf "%s-webhook" (include "cmk.fullname" .) ) nil $altNames 365 $ca }} ---- -apiVersion: v1 -kind: Secret -metadata: - name: {{ include "cmk.fullname" . }}-webhook -type: Opaque -data: - {{ if and .Values.webhook.cert .Values.webhook.key -}} - cert.pem: {{ .Values.webhook.cert }} - key.pem: {{ .Values.webhook.key }} - {{ else -}} - cert.pem: {{ $cert.Cert | b64enc }} - key.pem: {{ $cert.Key | b64enc }} - {{ end }} ---- -apiVersion: v1 -kind: ConfigMap -metadata: - name: {{ include "cmk.fullname" . }}-webhook -data: - server.yaml: | - server: - binding-address: "0.0.0.0" - port: 443 - cert: "/etc/ssl/cert.pem" - key: "/etc/ssl/key.pem" - mutations: "/etc/webhook/mutations.yaml" - mutations.yaml: | - mutations: - perPod: - metadata: - annotations: - cmk.intel.com/resources-injected: "true" - spec: - tolerations: - - key: cmk - operator: Exists - volumes: - - name: cmk-host-proc - hostPath: - path: "/proc" - perContainer: - env: - - name: CMK_PROC_FS - value: "/host/proc" - volumeMounts: - - name: cmk-host-proc - mountPath: /host/proc - readOnly: true ---- -apiVersion: admissionregistration.k8s.io/v1beta1 -kind: MutatingWebhookConfiguration -metadata: - labels: - app: {{ include "cmk.fullname" . }}-webhook - name: {{ include "cmk.fullname" . }}-webhook -webhooks: -- clientConfig: - caBundle: {{ .Values.webhook.caBundle }} - service: - name: {{ include "cmk.fullname" . }}-webhook - namespace: {{ .Release.Namespace }} - path: /mutate - failurePolicy: Ignore - name: cmk.intel.com - rules: - - apiGroups: - - "" - apiVersions: - - v1 - operations: - - CREATE - resources: - - pods ---- -apiVersion: v1 -kind: Service -metadata: - name: {{ include "cmk.fullname" . }}-webhook - labels: - app: {{ include "cmk.fullname" . }}-webhook -spec: - ports: - - port: 443 - targetPort: 443 - selector: - app: {{ include "cmk.fullname" . }}-webhook ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - labels: - app: {{ include "cmk.fullname" . }}-webhook - name: {{ include "cmk.fullname" . }}-webhook -spec: - replicas: 1 - selector: - matchLabels: - app: {{ include "cmk.fullname" . }}-webhook - template: - metadata: - labels: - app: {{ include "cmk.fullname" . }}-webhook - spec: - tolerations: - - key: cmk - operator: "Exists" - containers: - - args: - - "/cmk/cmk.py webhook --conf-file /etc/webhook/server.yaml --cafile /var/run/secrets/kubernetes.io/serviceaccount/ca.crt" - command: - - "/bin/bash" - - "-c" - image: {{ .Values.image.repository }}:{{ .Values.image.tag }} - name: cmk-webhook - volumeMounts: - - mountPath: /etc/webhook - name: cmk-webhook-configmap - - mountPath: /etc/ssl - name: cmk-webhook-certs - readOnly: True - volumes: - - name: cmk-webhook-configmap - configMap: - name: {{ include "cmk.fullname" . }}-webhook - - name: cmk-webhook-certs - secret: - secretName: {{ include "cmk.fullname" . }}-webhook diff --git a/roles/cmk_install/tasks/main.yml b/roles/cmk_install/tasks/main.yml deleted file mode 100644 index 13aecfd0..00000000 --- a/roles/cmk_install/tasks/main.yml +++ /dev/null @@ -1,364 +0,0 @@ -## -## Copyright (c) 2020-2021 Intel Corporation. -## -## Licensed under the Apache License, Version 2.0 (the "License"); -## you may not use this file except in compliance with the License. -## You may obtain a copy of the License at -## -## http://www.apache.org/licenses/LICENSE-2.0 -## -## Unless required by applicable law or agreed to in writing, software -## distributed under the License is distributed on an "AS IS" BASIS, -## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -## See the License for the specific language governing permissions and -## limitations under the License. -## ---- -- name: install dependencies - include_role: - name: install_dependencies - -- name: clone CMK repository - git: - repo: "{{ cmk_git_url }}" - dest: "{{ cmk_dir }}" - version: "{{ cmk_version }}" - force: yes - -# NOTE(krystianmlynek): more information here https://github.com/intel/CPU-Manager-for-Kubernetes/pull/291 -- name: patch isolate.py - replace: - path: "{{ cmk_dir }}/intel/isolate.py" - regexp: 'c = config\.Config\(configmap_name, pod_name\)' - replace: 'c = config.Config(configmap_name, pod_name, namespace)' - mode: '0700' - -- name: build CMK image - make: - chdir: "{{ cmk_dir }}" - when: container_runtime == "docker" - -# NOTE(przemeklal): this fixes problem in CMK with ImagePullPolicy hardcoded to Never and the pod is scheduled on controller node -- name: tag CMK image - command: docker tag cmk:{{ cmk_img_version }} {{ registry_local_address }}/cmk:{{ cmk_img_version }} - changed_when: true - when: container_runtime == "docker" - -- name: push CMK image to local registry - command: docker push {{ registry_local_address }}/cmk:{{ cmk_img_version }} - changed_when: true - when: - - container_runtime == "docker" - - inventory_hostname == groups['kube_node'][0] - -- name: build and tag CMK image - command: buildah bud -f Dockerfile -t {{ registry_local_address }}/cmk:{{ cmk_img_version }} - args: - chdir: "{{ cmk_dir }}" - changed_when: true - when: '"docker" not in container_runtime' - -- name: push CMK image to local registry - command: buildah push {{ registry_local_address }}/cmk:{{ cmk_img_version }} - changed_when: true - when: - - inventory_hostname == groups['kube_node'][0] - - '"docker" not in container_runtime' - -- name: clean up any pre-existing certs/key/CSR files - file: path=/etc/ssl/cmk state=absent - when: inventory_hostname == groups['kube_control_plane'][0] - failed_when: false - become: yes - -- name: delete any pre-existing certs/key/CSR from Kubernetes - command: kubectl delete csr cmk-webhook-{{ item }}.{{ cmk_namespace }} - when: inventory_hostname == groups['kube_control_plane'][0] - failed_when: false - with_items: - - "client" - - "server" - -- name: create directory for CMK cert and key generation - become: yes - file: - path: /etc/ssl/cmk - state: directory - mode: 0700 - owner: root - group: root - -- name: populate CMK CSR template - template: - src: "webhook_{{ item }}_csr.json.j2" - dest: "/etc/ssl/cmk/cmk-webhook-{{ item }}-csr.json" - force: yes - mode: preserve - become: yes - with_items: - - "client" - - "server" - when: - - inventory_hostname == groups['kube_control_plane'][0] - -- name: get GOPATH - command: go env GOPATH - register: gopath - when: - - inventory_hostname == groups['kube_control_plane'][0] - -- name: generate key and CSR - shell: "set -o pipefail \ - && {{ gopath.stdout }}/bin/cfssl genkey cmk-webhook-{{ item }}-csr.json | {{ gopath.stdout }}/bin/cfssljson -bare cmk-webhook-{{ item }}" - args: - chdir: "/etc/ssl/cmk/" - executable: /bin/bash - with_items: - - "client" - - "server" - when: - - inventory_hostname == groups['kube_control_plane'][0] - become: yes - -- name: read generated server key - command: cat cmk-webhook-server-key.pem - args: - chdir: "/etc/ssl/cmk/" - register: server_key - when: - - inventory_hostname == groups['kube_control_plane'][0] - -- name: read generated client key - command: cat cmk-webhook-client-key.pem - args: - chdir: "/etc/ssl/cmk/" - register: client_key - when: - - inventory_hostname == groups['kube_control_plane'][0] - -- name: load generated server key - set_fact: - cmk_webhook_server_key: "{{ server_key.stdout | b64encode }}" - when: - - inventory_hostname == groups['kube_control_plane'][0] - -- name: load generated client key - set_fact: - cmk_webhook_client_key: "{{ client_key.stdout | b64encode }}" - when: - - inventory_hostname == groups['kube_control_plane'][0] - -- name: read generated client csr - command: cat cmk-webhook-client.csr - args: - chdir: "/etc/ssl/cmk/" - register: client_csr - when: - - inventory_hostname == groups['kube_control_plane'][0] - -- name: load generated client csr - set_fact: - cmk_webhook_client_csr: "{{ client_csr.stdout | b64encode }}" - when: - - inventory_hostname == groups['kube_control_plane'][0] - -- name: read generated server csr - command: cat cmk-webhook-server.csr - args: - chdir: "/etc/ssl/cmk/" - register: server_csr - when: - - inventory_hostname == groups['kube_control_plane'][0] - -- name: load generated server csr - set_fact: - cmk_webhook_server_csr: "{{ server_csr.stdout | b64encode }}" - when: - - inventory_hostname == groups['kube_control_plane'][0] - -- name: populate CMK Kubernetes CA CSR template - template: - src: "kube_{{ item }}_csr.yml.j2" - dest: "/etc/ssl/cmk/cmk-webhook-kube-{{ item }}-csr.yml" - force: yes - mode: preserve - with_items: - - "client" - - "server" - when: - - inventory_hostname == groups['kube_control_plane'][0] - -- name: send CSR to the Kubernetes API Server - command: kubectl apply -f /etc/ssl/cmk/cmk-webhook-kube-{{ item }}-csr.yml - with_items: - - "client" - - "server" - when: - - inventory_hostname == groups['kube_control_plane'][0] - -- name: approve request - command: kubectl certificate approve cmk-webhook-{{ item }}.{{ cmk_namespace }} - with_items: - - "client" - - "server" - when: - - inventory_hostname == groups['kube_control_plane'][0] - -- name: get approved server certificate - shell: kubectl get csr cmk-webhook-server.{{ cmk_namespace }} -o jsonpath='{.status.certificate}' - args: - chdir: "/etc/ssl/cmk/" - register: server_cert - when: - - inventory_hostname == groups['kube_control_plane'][0] - retries: 30 - delay: 1 - until: server_cert.rc == 0 - -- name: get approved client certificate - shell: kubectl get csr cmk-webhook-client.{{ cmk_namespace }} -o jsonpath='{.status.certificate}' - args: - chdir: "/etc/ssl/cmk/" - register: client_cert - when: - - inventory_hostname == groups['kube_control_plane'][0] - retries: 30 - delay: 1 - until: client_cert.rc == 0 - -- name: load generated server cert - set_fact: - cmk_webhook_server_cert: "{{ server_cert.stdout }}" - when: - - inventory_hostname == groups['kube_control_plane'][0] - -- name: load generated client cert - set_fact: - cmk_webhook_client_cert: "{{ client_cert.stdout }}" - when: - - inventory_hostname == groups['kube_control_plane'][0] - -- name: populate cmk-webhook.conf file - template: - src: "cmk-webhook.conf.j2" - dest: "/etc/kubernetes/admission-control/cmk-webhook.conf" - force: yes - mode: preserve - when: - - inventory_hostname == groups['kube_control_plane'][0] - -- name: add MutatingAdmissionWebhook to AdmissionConfiguration - blockinfile: - path: /etc/kubernetes/admission-control/config.yaml - insertafter: "plugins:" - block: | - - name: MutatingAdmissionWebhook - configuration: - apiVersion: apiserver.config.k8s.io/v1 - kind: WebhookAdmissionConfiguration - kubeConfigFile: /etc/kubernetes/admission-control/cmk-webhook.conf - when: - - inventory_hostname == groups['kube_control_plane'][0] - - -- name: restart kube-apiserver after updating admission control configuration - when: inventory_hostname == groups['kube_control_plane'][0] - block: - - name: remove kube-apiserver container - # noqa 305 - shell is used intentionally here - shell: >- - {{ (container_runtime == 'docker') | ternary('docker ps -af name=k8s_kube-apiserver* -q | - xargs --no-run-if-empty docker rm -f', - 'crictl ps -a --name=kube-apiserver* -q | - xargs --no-run-if-empty crictl rm -f') }} - args: - executable: /bin/bash - register: remove_apiserver_container - retries: 10 - until: remove_apiserver_container.rc == 0 - delay: 1 - - name: wait for kube-apiserver to be up - uri: - url: "https://127.0.0.1:6443/healthz" - client_cert: "/etc/kubernetes/ssl/ca.crt" - client_key: "/etc/kubernetes/ssl/ca.key" - validate_certs: no - register: result - until: result.status == 200 - retries: 120 - delay: 1 - -- name: create Helm charts directory if needed - file: - path: /usr/src/charts - state: directory - mode: 0755 - when: - - inventory_hostname == groups['kube_control_plane'][0] - -- name: copy CMK Helm chart to the controller node - copy: - src: "{{ role_path }}/charts/cpu-manager-for-kubernetes" - dest: "/usr/src/charts/" - mode: 0755 - when: - - inventory_hostname == groups['kube_control_plane'][0] - -- name: set fact if controller node in the cmk_hosts - set_fact: - cmk_use_all_hosts: true - when: - - inventory_hostname in groups['kube_control_plane'] - -- name: set values for CMK Helm chart values - set_fact: - cmk_image: "{{ registry_local_address }}/cmk" - cmk_tag: "{{ cmk_img_version }}" - when: - - inventory_hostname == groups['kube_control_plane'][0] - -- name: read ca cert - command: cat ca.crt - args: - chdir: "/etc/kubernetes/ssl/" - register: ca_cert - when: - - inventory_hostname == groups['kube_control_plane'][0] - -- name: load ca cert - set_fact: - caBundle_cert: "{{ ca_cert.stdout | b64encode }}" - when: - - inventory_hostname == groups['kube_control_plane'][0] - -- name: populate CMK Helm chart values template and push to controller node - template: - src: "helm_values.yml.j2" - dest: "/usr/src/charts/cmk-values.yml" - force: yes - mode: preserve - when: - - inventory_hostname == groups['kube_control_plane'][0] - -# remove any pre-existing configmaps before cmk redeployment -- name: remove any pre-existing configmaps before CMK deployment - command: kubectl delete cm cmk-config-{{ inventory_hostname }} - when: - - inventory_hostname in (cmk_hosts_list.split(',') if (cmk_hosts_list is defined and cmk_hosts_list | length > 0) else []) - delegate_to: "{{ groups['kube_control_plane']|first }}" - failed_when: false - -- name: update dynamic linker cache before install CMK helm chart - command: "ldconfig" - changed_when: true - -- name: install CMK helm chart - command: helm upgrade --install cmk --namespace {{ cmk_namespace }} -f /usr/src/charts/cmk-values.yml /usr/src/charts/cpu-manager-for-kubernetes - when: - - inventory_hostname == groups['kube_control_plane'][0] - -- name: clean up any certs/key/CSR files - file: path=/etc/ssl/cmk state=absent - when: inventory_hostname == groups['kube_control_plane'][0] - failed_when: false - become: yes diff --git a/roles/cmk_install/tasks/reset.yml b/roles/cmk_install/tasks/reset.yml deleted file mode 100644 index b7f45f78..00000000 --- a/roles/cmk_install/tasks/reset.yml +++ /dev/null @@ -1,72 +0,0 @@ -## -## Copyright (c) 2020-2021 Intel Corporation. -## -## Licensed under the Apache License, Version 2.0 (the "License"); -## you may not use this file except in compliance with the License. -## You may obtain a copy of the License at -## -## http://www.apache.org/licenses/LICENSE-2.0 -## -## Unless required by applicable law or agreed to in writing, software -## distributed under the License is distributed on an "AS IS" BASIS, -## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -## See the License for the specific language governing permissions and -## limitations under the License. -## ---- -#NOTE(przemeklal): for the remove-node use case we don't need to care about setting anti-affinity in the ds spec as -# kubespray will drain that node first, we only need to clean up CMK reports, labels and files - -- name: delete Helm deployment - command: helm delete --purge cmk - failed_when: false - register: helm_result - changed_when: "helm_result.rc == 0" - delegate_to: "{{ groups['kube_control_plane'] | first }}" - when: inventory_hostname == groups['kube_control_plane'][0] - -- name: untaint nodes - command: kubectl taint node "{{ inventory_hostname }}" cmk- - failed_when: false - register: untaint_result - changed_when: "untaint_result.rc == 0" - delegate_to: "{{ groups['kube_control_plane']|first }}" - -- name: unlabel nodes - command: kubectl label node "{{ inventory_hostname }}" cmk.intel.com/cmk-node- - failed_when: false - register: unlabel_result - changed_when: "unlabel_result.rc == 0" - delegate_to: "{{ groups['kube_control_plane']|first }}" - -- name: remove CMK reports - command: kubectl delete "{{ item }}" - with_items: - - "cmk-nodereports.intel.com/{{ inventory_hostname }}" - - "cmk-reconcilereports.intel.com/{{ inventory_hostname }}" - failed_when: false - register: delete_report_result - changed_when: "delete_report_result.rc == 0" - delegate_to: "{{ groups['kube_control_plane']|first }}" - -- name: remove CRDs - command: kubectl delete crd "{{ item }}" - with_items: - - "cmk-nodereports.intel.com" - - "cmk-reconcilereports.intel.com" - failed_when: false - register: delete_crd_result - changed_when: "delete_crd_result.rc == 0" - delegate_to: "{{ groups['kube_control_plane']|first }}" - when: inventory_hostname == groups['kube_control_plane'][0] - -- name: remove files - file: - state: absent - path: "{{ item }}" - with_items: - - "/etc/cmk" - - "/opt/bin/cmk" - - "{{ cmk_dir }}" - - "/usr/src/charts/cmk-values.yml" - - "/usr/src/charts/cpu-manager-for-kubernetes" diff --git a/roles/cmk_install/templates/cmk-webhook.conf.j2 b/roles/cmk_install/templates/cmk-webhook.conf.j2 deleted file mode 100644 index bc24c006..00000000 --- a/roles/cmk_install/templates/cmk-webhook.conf.j2 +++ /dev/null @@ -1,7 +0,0 @@ -apiVersion: v1 -kind: Config -users: -- name: cmk-webhook.{{ cmk_namespace }}.svc - user: - client-certificate-data: {{ cmk_webhook_client_cert }} - client-key-data: {{ cmk_webhook_client_key }} \ No newline at end of file diff --git a/roles/cmk_install/templates/helm_values.yml.j2 b/roles/cmk_install/templates/helm_values.yml.j2 deleted file mode 100644 index 3c40a31e..00000000 --- a/roles/cmk_install/templates/helm_values.yml.j2 +++ /dev/null @@ -1,21 +0,0 @@ ---- -image: - repository: {{ cmk_image | default("localhost:30500/cmk") }} - tag: {{ cmk_tag | default("v1.5.2") }} - pullPolicy: IfNotPresent - -hosts: - all: {{ cmk_use_all_hosts | default(false) }} - list: {{ ((cmk_hosts_list.split(',')) | map('extract', hostvars, 'ansible_hostname') | list | join(',')) | default("node1") }} - -sharedMode: {{ cmk_shared_mode | default("packed") }} -numSharedCores: {{ cmk_shared_num_cores | default(1) }} -exclusiveMode: {{ cmk_exclusive_mode | default("packed") }} -numExclusiveCores: {{ cmk_exclusive_num_cores | default(1) }} - -noTaint: {{ cmk_untaint_nodes | bool | default(true) }} - -webhook: - cert: {{ cmk_webhook_server_cert }} - key: {{ cmk_webhook_server_key }} - caBundle: {{ caBundle_cert }} diff --git a/roles/cmk_install/templates/kube_client_csr.yml.j2 b/roles/cmk_install/templates/kube_client_csr.yml.j2 deleted file mode 100644 index 20aba61c..00000000 --- a/roles/cmk_install/templates/kube_client_csr.yml.j2 +++ /dev/null @@ -1,10 +0,0 @@ -apiVersion: certificates.k8s.io/v1beta1 -kind: CertificateSigningRequest -metadata: - name: cmk-webhook-client.{{ cmk_namespace }} -spec: - request: {{ cmk_webhook_client_csr }} - usages: - - digital signature - - key encipherment - - client auth diff --git a/roles/cmk_install/templates/kube_server_csr.yml.j2 b/roles/cmk_install/templates/kube_server_csr.yml.j2 deleted file mode 100644 index 7254afcb..00000000 --- a/roles/cmk_install/templates/kube_server_csr.yml.j2 +++ /dev/null @@ -1,10 +0,0 @@ -apiVersion: certificates.k8s.io/v1beta1 -kind: CertificateSigningRequest -metadata: - name: cmk-webhook-server.{{ cmk_namespace }} -spec: - request: {{ cmk_webhook_server_csr }} - usages: - - digital signature - - key encipherment - - server auth diff --git a/roles/cmk_install/templates/webhook_client_csr.json.j2 b/roles/cmk_install/templates/webhook_client_csr.json.j2 deleted file mode 100644 index e4f6d839..00000000 --- a/roles/cmk_install/templates/webhook_client_csr.json.j2 +++ /dev/null @@ -1,10 +0,0 @@ -{ - "hosts": [ - "cmk-webhook-client" - ], - "CN": "cmk-webhook-client", - "key": { - "algo": "rsa", - "size": 2048 - } -} diff --git a/roles/cmk_install/templates/webhook_server_csr.json.j2 b/roles/cmk_install/templates/webhook_server_csr.json.j2 deleted file mode 100644 index 83b254aa..00000000 --- a/roles/cmk_install/templates/webhook_server_csr.json.j2 +++ /dev/null @@ -1,12 +0,0 @@ -{ - "hosts": [ - "cmk-webhook.{{ cmk_namespace }}.svc.{{ cluster_name }}", - "cmk-webhook.{{ cmk_namespace }}.svc", - "cmk-webhook.{{ cmk_namespace }}" - ], - "CN": "cmk-webhook.{{ cmk_namespace }}.svc.{{ cluster_name }}", - "key": { - "algo": "rsa", - "size": 2048 - } -} diff --git a/roles/collectd_install/defaults/main.yml b/roles/collectd_install/defaults/main.yml index bc1fdf12..7acaf3af 100644 --- a/roles/collectd_install/defaults/main.yml +++ b/roles/collectd_install/defaults/main.yml @@ -1,5 +1,5 @@ ## -## Copyright (c) 2020-2021 Intel Corporation. +## Copyright (c) 2020-2022 Intel Corporation. ## ## Licensed under the Apache License, Version 2.0 (the "License"); ## you may not use this file except in compliance with the License. @@ -14,8 +14,9 @@ ## limitations under the License. ## --- -barometer_collectd_dir: "/usr/src/barometer" -collectd_deployment_dir: "/usr/src/k8s/collectd/" +collectd_configuration_files_dir: "{{ host_collectd_folder }}/collectd.conf.d" +barometer_collectd_dir: "{{ project_root_dir }}/barometer" +collectd_deployment_dir: "{{ project_root_dir }}/k8s/collectd/" collectd_scrap_interval: 30 collectd_write_threads: 25 @@ -36,9 +37,6 @@ enable_intel_rdt_plugin: true # Please refer to https://collectd.org/wiki/index.php/Plugin:IntelRDT for configuration details. intel_rdt_plugin_monitored_cores: "" -# pkgpower plugin collects platform power telemetry data -enable_pkgpower_plugin: true - # unixsock plugin host socket base dir unixsock_host_socket_dir: /var/run/collectd/ @@ -47,7 +45,7 @@ enable_custom_types_db: false image_collectd: repository: "opnfv/barometer-collectd" - digest: sha256:f30e965aa6195e6ac4ca2410f5a15e3704c92e4afa5208178ca22a7911975d66 + digest: sha256:a7cea43d9d2f67c38fbf0407786edbe660ee9072945f7bb272b55fd255e8eaca pullPolicy: Always image_collectd_exporter: repository: prom/collectd-exporter @@ -62,7 +60,7 @@ psp_enabled: true collectd_namespace: monitoring host_collectd_folder: /opt/collect.d pkgpower_repo_url: "https://github.com/intel/CommsPowerManagement.git" -pkgpower_dir: "/usr/src/CommsPowerManagement" +pkgpower_dir: "{{ project_root_dir }}/commspowermanagement" # currently excluded plugins were not delivered with latest stable # opnfv/barometer-collectd image (digest sha256:ed5c574f653e) @@ -81,6 +79,7 @@ collectd_plugins: - pkgpower - unixsock - network + - turbostat # - write_http # - smart on_prem: @@ -97,6 +96,7 @@ collectd_plugins: - pkgpower - unixsock - network + - turbostat # - write_http # - smart access: @@ -115,6 +115,7 @@ collectd_plugins: - hugepages - unixsock - network + - turbostat # - write_http # - smart remote_fp: @@ -131,6 +132,7 @@ collectd_plugins: - pkgpower - unixsock - network + - turbostat # - write_http # - smart regional_dc: @@ -147,6 +149,7 @@ collectd_plugins: - pkgpower - unixsock - network + - turbostat # - write_http # - smart full_nfv: @@ -168,6 +171,7 @@ collectd_plugins: - ovs_stats - unixsock - network + - turbostat # - write_http # - smart diff --git a/roles/collectd_install/tasks/collectd.yml b/roles/collectd_install/tasks/collectd.yml index efd382d8..8c243b19 100644 --- a/roles/collectd_install/tasks/collectd.yml +++ b/roles/collectd_install/tasks/collectd.yml @@ -1,5 +1,5 @@ ## -## Copyright (c) 2020-2021 Intel Corporation. +## Copyright (c) 2020-2022 Intel Corporation. ## ## Licensed under the Apache License, Version 2.0 (the "License"); ## you may not use this file except in compliance with the License. diff --git a/roles/collectd_install/tasks/copy-configs.yml b/roles/collectd_install/tasks/copy-configs.yml index 9fd2966e..351b56f5 100644 --- a/roles/collectd_install/tasks/copy-configs.yml +++ b/roles/collectd_install/tasks/copy-configs.yml @@ -1,5 +1,5 @@ ## -## Copyright (c) 2020-2021 Intel Corporation. +## Copyright (c) 2020-2022 Intel Corporation. ## ## Licensed under the Apache License, Version 2.0 (the "License"); ## you may not use this file except in compliance with the License. @@ -24,19 +24,6 @@ exclude_collectd_plugins: "{{ exclude_collectd_plugins + [ 'rdt' ] }}" when: not enable_intel_rdt_plugin -- name: read CPU type - shell: - cmd: lscpu | grep "Model name:" - register: cpu_model - changed_when: false - -- name: disable pkgppower plugin - set_fact: - exclude_collectd_plugins: "{{ exclude_collectd_plugins + [ 'pkgpower' ] }}" - enable_pkgpower_plugin: false - when: item in cpu_model.stdout - loop: '{{ unsupported_power_skus }}' - - name: prepare list of plugins to be deployed set_fact: plugins: "{{ collectd_plugins[collectd_profile|default('basic')] | difference(exclude_collectd_plugins) }}" diff --git a/roles/collectd_install/tasks/main.yml b/roles/collectd_install/tasks/main.yml index 1f4bfee2..b0a82a40 100644 --- a/roles/collectd_install/tasks/main.yml +++ b/roles/collectd_install/tasks/main.yml @@ -1,5 +1,5 @@ ## -## Copyright (c) 2020-2021 Intel Corporation. +## Copyright (c) 2020-2022 Intel Corporation. ## ## Licensed under the Apache License, Version 2.0 (the "License"); ## you may not use this file except in compliance with the License. diff --git a/roles/collectd_install/tasks/msr-config.yml b/roles/collectd_install/tasks/msr-config.yml index c486b538..b29f8e55 100644 --- a/roles/collectd_install/tasks/msr-config.yml +++ b/roles/collectd_install/tasks/msr-config.yml @@ -1,5 +1,5 @@ ## -## Copyright (c) 2020-2021 Intel Corporation. +## Copyright (c) 2020-2022 Intel Corporation. ## ## Licensed under the Apache License, Version 2.0 (the "License"); ## you may not use this file except in compliance with the License. @@ -25,4 +25,4 @@ line: msr create: yes mode: 0644 - become: yes \ No newline at end of file + become: yes diff --git a/roles/collectd_install/templates/daemonset.yml b/roles/collectd_install/templates/daemonset.yml index 6318353e..3c9681cd 100644 --- a/roles/collectd_install/templates/daemonset.yml +++ b/roles/collectd_install/templates/daemonset.yml @@ -38,11 +38,9 @@ spec: mountPath: "/var/run/.client" - name: dpdksocketpath mountPath: "/var/run/dpdk/rte/telemetry" -{% if enable_pkgpower_plugin %} - name: pkgpower mountPath: "/sys/devices/virtual/powercap/intel-rapl/" readOnly: true -{% endif %} - name: hugepages mountPath: "/sys/kernel/mm/hugepages" readOnly: true @@ -76,11 +74,6 @@ spec: - "--tls-cipher-suites={{ rbac_proxy_tls_cipher_suites | join(',') }}" - "--secure-listen-address=0.0.0.0:9104" - "--upstream=http://127.0.0.1:9103/" - tolerations: - - key: cmk - operator: Equal - effect: NoSchedule - value: "true" volumes: - name: ssl secret: diff --git a/roles/collectd_install/templates/plugin_configuration/turbostat.conf b/roles/collectd_install/templates/plugin_configuration/turbostat.conf new file mode 100644 index 00000000..49f94ea5 --- /dev/null +++ b/roles/collectd_install/templates/plugin_configuration/turbostat.conf @@ -0,0 +1 @@ +LoadPlugin turbostat diff --git a/roles/collectd_install/templates/psp.yml.j2 b/roles/collectd_install/templates/psp.yml.j2 index c6ccad2a..15cb0190 100644 --- a/roles/collectd_install/templates/psp.yml.j2 +++ b/roles/collectd_install/templates/psp.yml.j2 @@ -28,9 +28,7 @@ spec: - pathPrefix: "/usr/local/var/run/openvswitch/" - pathPrefix: "/var/run/.client" - pathPrefix: "/var/run/dpdk/rte/telemetry" -{% if enable_pkgpower_plugin %} - pathPrefix: "/sys/devices/virtual/powercap/intel-rapl/" -{% endif %} - pathPrefix: "/sys/kernel/mm/hugepages" - pathPrefix: "/sys/devices/system/cpu" - pathPrefix: "{{ unixsock_host_socket_dir }}" diff --git a/roles/container_engine/containerd/defaults/main.yml b/roles/container_engine/containerd/defaults/main.yml new file mode 100644 index 00000000..1e1ed72b --- /dev/null +++ b/roles/container_engine/containerd/defaults/main.yml @@ -0,0 +1,79 @@ +## +## Copyright (c) 2020-2022 Intel Corporation. +## +## Licensed under the Apache License, Version 2.0 (the "License"); +## you may not use this file except in compliance with the License. +## You may obtain a copy of the License at +## +## http://www.apache.org/licenses/LICENSE-2.0 +## +## Unless required by applicable law or agreed to in writing, software +## distributed under the License is distributed on an "AS IS" BASIS, +## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +## See the License for the specific language governing permissions and +## limitations under the License. +## +--- +containerd_storage_dir: "/var/lib/containerd" +containerd_state_dir: "/run/containerd" +containerd_oom_score: 0 + +containerd_sandbox_image: "k8s.gcr.io/pause:3.3" +containerd_default_runtime: "runc" +# containerd_snapshotter: "native" + +containerd_runtimes: + - name: runc + type: "io.containerd.runc.v2" + engine: "" + root: "" + options: + systemdCgroup: "true" + +containerd_grpc_max_recv_message_size: 16777216 +containerd_grpc_max_send_message_size: 16777216 + +containerd_debug_level: "info" + +containerd_metrics_address: "" + +containerd_metrics_grpc_histogram: false + +containerd_registries: + "docker.io": "https://registry-1.docker.io" + +containerd_max_container_log_line_size: -1 + +containerd_cfg_dir: /etc/containerd + +containerd_bin_dir: /usr/bin + +# Path to runc binary +runc_binary: /usr/bin/runc + +yum_repo_dir: /etc/yum.repos.d + +# Optional values for containerd apt repo +containerd_package_info: + pkgs: + +containerd_repo_key_info: + repo_keys: + +containerd_repo_info: + repos: + +# Ubuntu docker-ce repo +containerd_ubuntu_repo_base_url: "https://download.docker.com/linux/ubuntu" +containerd_ubuntu_repo_gpgkey: "https://download.docker.com/linux/ubuntu/gpg" +containerd_ubuntu_repo_repokey: "9DC858229FC7DD38854AE2D88D81803C0EBFCD88" +containerd_ubuntu_repo_component: "stable" + +# Extra config to be put in {{ containerd_cfg_dir }}/config.toml literally +containerd_extra_args: '' + +# Configure registry auth (if applicable to secure/insecure registries) +containerd_registry_auth: [] +# - registry: 10.0.0.2:5000 +# username: user +# password: pass diff --git a/roles/qat_dp_install/charts/intel-qat-plugin/values.yaml b/roles/container_engine/containerd/handlers/main.yml similarity index 54% rename from roles/qat_dp_install/charts/intel-qat-plugin/values.yaml rename to roles/container_engine/containerd/handlers/main.yml index f714ffc0..746dd62a 100644 --- a/roles/qat_dp_install/charts/intel-qat-plugin/values.yaml +++ b/roles/container_engine/containerd/handlers/main.yml @@ -1,5 +1,5 @@ ## -## Copyright (c) 2020-2021 Intel Corporation. +## Copyright (c) 2020-2022 Intel Corporation. ## ## Licensed under the Apache License, Version 2.0 (the "License"); ## you may not use this file except in compliance with the License. @@ -14,18 +14,22 @@ ## limitations under the License. ## --- -image: - repository: docker.io/intel/intel-qat-plugin - tag: 0.21.0 - pullPolicy: IfNotPresent +- name: restart containerd + command: /bin/true + notify: + - containerd | restart containerd + - containerd | wait for containerd -drivers: - dpdk: "vfio-pci" - kernel: "dh895xccvf,c6xxvf,c3xxxvf,d15xxvf,200xxvf,c4xxxvf" +- name: containerd | restart containerd + systemd: + name: containerd + state: restarted + enabled: yes + daemon-reload: yes -maxNumDevices: "" - -# sets annotation for deployment on apparmor enabled systems -apparmor: - enabled: true - profile: unconfined # unconfinded will not use any of the existing profiles, if you have a targetted profile, please enter its name here +- name: containerd | wait for containerd + command: "{{ containerd_bin_dir }}/ctr images ls -q" + register: containerd_ready + retries: 8 + delay: 4 + until: containerd_ready.rc == 0 diff --git a/roles/tas_install/tasks/main.yml b/roles/container_engine/containerd/meta/main.yml similarity index 78% rename from roles/tas_install/tasks/main.yml rename to roles/container_engine/containerd/meta/main.yml index ceef87c2..4c5cc977 100644 --- a/roles/tas_install/tasks/main.yml +++ b/roles/container_engine/containerd/meta/main.yml @@ -1,5 +1,5 @@ ## -## Copyright (c) 2020-2021 Intel Corporation. +## Copyright (c) 2020-2022 Intel Corporation. ## ## Licensed under the Apache License, Version 2.0 (the "License"); ## you may not use this file except in compliance with the License. @@ -14,7 +14,5 @@ ## limitations under the License. ## --- -- name: install TAS - import_tasks: tas.yml - when: - - inventory_hostname == groups['kube_control_plane'][0] +dependencies: # noqa 701 + - role: container_engine/containerd_common diff --git a/roles/container_engine/containerd/tasks/containerd_repo.yml b/roles/container_engine/containerd/tasks/containerd_repo.yml new file mode 100644 index 00000000..593ea017 --- /dev/null +++ b/roles/container_engine/containerd/tasks/containerd_repo.yml @@ -0,0 +1,41 @@ +## +## Copyright (c) 2020-2022 Intel Corporation. +## +## Licensed under the Apache License, Version 2.0 (the "License"); +## you may not use this file except in compliance with the License. +## You may obtain a copy of the License at +## +## http://www.apache.org/licenses/LICENSE-2.0 +## +## Unless required by applicable law or agreed to in writing, software +## distributed under the License is distributed on an "AS IS" BASIS, +## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +## See the License for the specific language governing permissions and +## limitations under the License. +## +--- +- name: ensure containerd repository public key is installed + apt_key: + id: "{{ item }}" + url: "{{ containerd_repo_key_info.url }}" + state: present + register: keyserver_task_result + until: keyserver_task_result is succeeded + retries: 4 + delay: 3 + with_items: "{{ containerd_repo_key_info.repo_keys }}" + when: ansible_distribution == "Ubuntu" + +- name: ensure containerd repository is enabled + apt_repository: + repo: "{{ item }}" + state: present + with_items: "{{ containerd_repo_info.repos }}" + when: ansible_distribution == "Ubuntu" + +- name: Configure containerd repository on RedHat/CentOS + template: + src: "rh_containerd.repo.j2" + dest: "{{ yum_repo_dir }}/containerd.repo" + mode: 0644 + when: ansible_os_family == "RedHat" diff --git a/roles/container_engine/containerd/tasks/main.yml b/roles/container_engine/containerd/tasks/main.yml new file mode 100644 index 00000000..fef5d00a --- /dev/null +++ b/roles/container_engine/containerd/tasks/main.yml @@ -0,0 +1,87 @@ +## +## Copyright (c) 2020-2022 Intel Corporation. +## +## Licensed under the Apache License, Version 2.0 (the "License"); +## you may not use this file except in compliance with the License. +## You may obtain a copy of the License at +## +## http://www.apache.org/licenses/LICENSE-2.0 +## +## Unless required by applicable law or agreed to in writing, software +## distributed under the License is distributed on an "AS IS" BASIS, +## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +## See the License for the specific language governing permissions and +## limitations under the License. +## +--- +- name: load a variable file based on the OS type + include_vars: "{{ lookup('first_found', params) }}" + vars: + params: + files: + - "{{ ansible_distribution|lower }}.yml" + - "{{ ansible_os_family|lower }}.yml" + paths: + - ../vars + +- include_tasks: containerd_repo.yml + +- name: create containerd service systemd directory if it doesn't exist + file: + path: /etc/systemd/system/containerd.service.d + state: directory + mode: 0755 + +- name: write containerd proxy drop-in + template: + src: http_proxy.conf.j2 + dest: /etc/systemd/system/containerd.service.d/http-proxy.conf + mode: 0644 + notify: restart containerd + when: http_proxy is defined or https_proxy is defined + +- name: ensure containerd config directory + file: + dest: "{{ containerd_cfg_dir }}" + state: directory + mode: 0755 + owner: root + group: root + +- name: copy containerd config file + template: + src: config.toml.j2 + dest: "{{ containerd_cfg_dir }}/config.toml" + owner: "root" + mode: 0640 + notify: restart containerd + +- name: ensure containerd packages are installed + package: + name: "{{ containerd_package_info.pkgs }}" + state: present + module_defaults: + apt: + update_cache: true + yum: + enablerepo: "{{ containerd_package_info.enablerepo | default(omit) }}" + dnf: + enablerepo: "{{ containerd_package_info.enablerepo | default(omit) }}" + register: containerd_task_result + until: containerd_task_result is succeeded + retries: 4 + delay: 3 + notify: restart containerd + when: containerd_package_info.pkgs|length > 0 + +- include_role: # noqa unnamed-task + name: container_engine/crictl + +- name: flush handlers + meta: flush_handlers + +- name: ensure containerd is started and enabled + service: + name: containerd + enabled: yes + state: started diff --git a/roles/container_engine/containerd/templates/config.toml.j2 b/roles/container_engine/containerd/templates/config.toml.j2 new file mode 100644 index 00000000..e0a4f1b3 --- /dev/null +++ b/roles/container_engine/containerd/templates/config.toml.j2 @@ -0,0 +1,55 @@ +version = 2 +root = "{{ containerd_storage_dir }}" +state = "{{ containerd_state_dir }}" +oom_score = {{ containerd_oom_score }} + +[grpc] + max_recv_message_size = {{ containerd_grpc_max_recv_message_size | default(16777216) }} + max_send_message_size = {{ containerd_grpc_max_send_message_size | default(16777216) }} + +[debug] + level = "{{ containerd_debug_level | default('info') }}" + +[metrics] + address = "{{ containerd_metrics_address | default('') }}" + grpc_histogram = {{ containerd_metrics_grpc_histogram | default(false) | lower }} + +[plugins] + [plugins."io.containerd.grpc.v1.cri"] + sandbox_image = "{{ containerd_sandbox_image }}" + max_container_log_line_size = {{ containerd_max_container_log_line_size }} + [plugins."io.containerd.grpc.v1.cri".containerd] + default_runtime_name = "{{ containerd_default_runtime | default('runc') }}" + snapshotter = "{{ containerd_snapshotter | default('overlayfs') }}" + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes] +{% for runtime in containerd_runtimes %} + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.{{ runtime.name }}] + runtime_type = "{{ runtime.type }}" + runtime_engine = "{{ runtime.engine }}" + runtime_root = "{{ runtime.root }}" + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.{{ runtime.name }}.options] +{% for key, value in runtime.options.items() %} + {{ key }} = {{ value }} +{% endfor %} +{% endfor %} + [plugins."io.containerd.grpc.v1.cri".registry] + [plugins."io.containerd.grpc.v1.cri".registry.mirrors] +{% for registry, addr in containerd_registries.items() %} + [plugins."io.containerd.grpc.v1.cri".registry.mirrors."{{ registry }}"] + endpoint = ["{{ ([ addr ] | flatten ) | join('","') }}"] +{% endfor %} +{% for registry in containerd_registry_auth if registry['registry'] is defined %} +{% if (registry['username'] is defined and registry['password'] is defined) or registry['auth'] is defined %} + [plugins."io.containerd.grpc.v1.cri".registry.configs."{{ registry['registry'] }}".auth] +{% if registry['username'] is defined and registry['password'] is defined %} + password = "{{ registry['password'] }}" + username = "{{ registry['username'] }}" +{% else %} + auth = "{{ registry['auth'] }}" +{% endif %} +{% endif %} +{% endfor %} + +{% if containerd_extra_args is defined %} +{{ containerd_extra_args }} +{% endif %} diff --git a/roles/container_engine/containerd/templates/http_proxy.conf.j2 b/roles/container_engine/containerd/templates/http_proxy.conf.j2 new file mode 100644 index 00000000..212f30f9 --- /dev/null +++ b/roles/container_engine/containerd/templates/http_proxy.conf.j2 @@ -0,0 +1,2 @@ +[Service] +Environment={% if http_proxy is defined %}"HTTP_PROXY={{ http_proxy }}"{% endif %} {% if https_proxy is defined %}"HTTPS_PROXY={{ https_proxy }}"{% endif %} {% if no_proxy is defined %}"NO_PROXY={{ no_proxy }}"{% endif %} diff --git a/roles/container_engine/containerd/templates/rh_containerd.repo.j2 b/roles/container_engine/containerd/templates/rh_containerd.repo.j2 new file mode 100644 index 00000000..178bbc2c --- /dev/null +++ b/roles/container_engine/containerd/templates/rh_containerd.repo.j2 @@ -0,0 +1,10 @@ +[docker-ce] +name=Docker-CE Repository +baseurl={{ docker_rh_repo_base_url }} +enabled=0 +gpgcheck={{ '1' if docker_rh_repo_gpgkey else '0' }} +keepcache={{ docker_rpm_keepcache | default('1') }} +gpgkey={{ docker_rh_repo_gpgkey }} +{% if http_proxy is defined %} +proxy={{ http_proxy }} +{% endif %} diff --git a/roles/container_engine/containerd/vars/redhat.yml b/roles/container_engine/containerd/vars/redhat.yml new file mode 100644 index 00000000..2bb0bd19 --- /dev/null +++ b/roles/container_engine/containerd/vars/redhat.yml @@ -0,0 +1,20 @@ +## +## Copyright (c) 2020-2022 Intel Corporation. +## +## Licensed under the Apache License, Version 2.0 (the "License"); +## you may not use this file except in compliance with the License. +## You may obtain a copy of the License at +## +## http://www.apache.org/licenses/LICENSE-2.0 +## +## Unless required by applicable law or agreed to in writing, software +## distributed under the License is distributed on an "AS IS" BASIS, +## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +## See the License for the specific language governing permissions and +## limitations under the License. +## +--- +containerd_package_info: + enablerepo: "docker-ce" + pkgs: + - "{{ containerd_versioned_pkg[containerd_version | string] }}" diff --git a/roles/container_engine/containerd/vars/ubuntu.yml b/roles/container_engine/containerd/vars/ubuntu.yml new file mode 100644 index 00000000..dce76c6c --- /dev/null +++ b/roles/container_engine/containerd/vars/ubuntu.yml @@ -0,0 +1,31 @@ +## +## Copyright (c) 2020-2022 Intel Corporation. +## +## Licensed under the Apache License, Version 2.0 (the "License"); +## you may not use this file except in compliance with the License. +## You may obtain a copy of the License at +## +## http://www.apache.org/licenses/LICENSE-2.0 +## +## Unless required by applicable law or agreed to in writing, software +## distributed under the License is distributed on an "AS IS" BASIS, +## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +## See the License for the specific language governing permissions and +## limitations under the License. +## +--- +containerd_package_info: + pkgs: + - "{{ containerd_versioned_pkg[containerd_version | string] }}" + +containerd_repo_key_info: + url: '{{ containerd_ubuntu_repo_gpgkey }}' + repo_keys: + - '{{ containerd_ubuntu_repo_repokey }}' + +containerd_repo_info: + repos: + - > + deb {{ containerd_ubuntu_repo_base_url }} + {{ ansible_distribution_release|lower }} + {{ containerd_ubuntu_repo_component }} diff --git a/roles/bootstrap/install_rt_package/vars/main.yml b/roles/container_engine/containerd_common/defaults/main.yml similarity index 50% rename from roles/bootstrap/install_rt_package/vars/main.yml rename to roles/container_engine/containerd_common/defaults/main.yml index 099dfa58..ad3b22e5 100644 --- a/roles/bootstrap/install_rt_package/vars/main.yml +++ b/roles/container_engine/containerd_common/defaults/main.yml @@ -1,5 +1,5 @@ ## -## Copyright (c) 2020-2021 Intel Corporation. +## Copyright (c) 2020-2022 Intel Corporation. ## ## Licensed under the Apache License, Version 2.0 (the "License"); ## you may not use this file except in compliance with the License. @@ -14,30 +14,13 @@ ## limitations under the License. ## --- -install_dependencies: - RedHat: - - git - - gcc - - make - - bc - - wget - - numactl-devel - - libhugetlbfs-devel - - kernel-devel - - "@Development tools" - - elfutils-libelf-devel - - gcc-c++ - - libhugetlbfs* - - libstdc++* - - numa* - - mlocate - - cmake - - ncurses-devel - - hmaccalc - - zlib-devel - - binutils-devel - - net-tools - - libcurl-devel - Debian: [] +containerd_package: 'containerd.io' +containerd_version: 1.4.6 -tuned_noarch_url: http://linuxsoft.cern.ch/cern/centos/7.5.1804/os/x86_64/Packages/tuned-2.9.0-1.el7.noarch.rpm +# CentOS/RedHat docker-ce repo +docker_rh_repo_base_url: 'https://download.docker.com/linux/centos/{{ ansible_distribution_major_version }}/$basearch/stable' +docker_rh_repo_gpgkey: 'https://download.docker.com/linux/centos/gpg' +# Ubuntu docker-ce repo +docker_ubuntu_repo_base_url: "https://download.docker.com/linux/ubuntu" +docker_ubuntu_repo_gpgkey: 'https://download.docker.com/linux/ubuntu/gpg' +docker_ubuntu_repo_repokey: '9DC858229FC7DD38854AE2D88D81803C0EBFCD88' diff --git a/roles/container_engine/containerd_common/tasks/main.yml b/roles/container_engine/containerd_common/tasks/main.yml new file mode 100644 index 00000000..2b383566 --- /dev/null +++ b/roles/container_engine/containerd_common/tasks/main.yml @@ -0,0 +1,25 @@ +## +## Copyright (c) 2020-2022 Intel Corporation. +## +## Licensed under the Apache License, Version 2.0 (the "License"); +## you may not use this file except in compliance with the License. +## You may obtain a copy of the License at +## +## http://www.apache.org/licenses/LICENSE-2.0 +## +## Unless required by applicable law or agreed to in writing, software +## distributed under the License is distributed on an "AS IS" BASIS, +## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +## See the License for the specific language governing permissions and +## limitations under the License. +## +--- +- name: load a variable file based on the OS type + include_vars: "{{ lookup('first_found', params) }}" + vars: + params: + files: + - "{{ ansible_distribution|lower }}.yml" + - "{{ ansible_os_family|lower }}.yml" + paths: + - ../vars diff --git a/roles/container_engine/containerd_common/vars/redhat.yml b/roles/container_engine/containerd_common/vars/redhat.yml new file mode 100644 index 00000000..c7317ff9 --- /dev/null +++ b/roles/container_engine/containerd_common/vars/redhat.yml @@ -0,0 +1,25 @@ +## +## Copyright (c) 2020-2022 Intel Corporation. +## +## Licensed under the Apache License, Version 2.0 (the "License"); +## you may not use this file except in compliance with the License. +## You may obtain a copy of the License at +## +## http://www.apache.org/licenses/LICENSE-2.0 +## +## Unless required by applicable law or agreed to in writing, software +## distributed under the License is distributed on an "AS IS" BASIS, +## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +## See the License for the specific language governing permissions and +## limitations under the License. +## +--- +containerd_versioned_pkg: + 'latest': "{{ containerd_package }}" + '1.3.7': "{{ containerd_package }}-1.3.7-3.1.el{{ ansible_distribution_major_version }}" + '1.3.9': "{{ containerd_package }}-1.3.9-3.1.el{{ ansible_distribution_major_version }}" + '1.4.3': "{{ containerd_package }}-1.4.3-3.2.el{{ ansible_distribution_major_version }}" + '1.4.4': "{{ containerd_package }}-1.4.4-3.1.el{{ ansible_distribution_major_version }}" + '1.4.6': "{{ containerd_package }}-1.4.6-3.1.el{{ ansible_distribution_major_version }}" + 'stable': "{{ containerd_package }}-1.4.6-3.1.el{{ ansible_distribution_major_version }}" + 'edge': "{{ containerd_package }}-1.4.6-3.1.el{{ ansible_distribution_major_version }}" diff --git a/playbooks/basic.yml b/roles/container_engine/containerd_common/vars/ubuntu.yml similarity index 57% rename from playbooks/basic.yml rename to roles/container_engine/containerd_common/vars/ubuntu.yml index 4983af4d..c50fcea1 100644 --- a/playbooks/basic.yml +++ b/roles/container_engine/containerd_common/vars/ubuntu.yml @@ -1,5 +1,5 @@ ## -## Copyright (c) 2020-2021 Intel Corporation. +## Copyright (c) 2020-2022 Intel Corporation. ## ## Licensed under the Apache License, Version 2.0 (the "License"); ## you may not use this file except in compliance with the License. @@ -14,13 +14,12 @@ ## limitations under the License. ## --- -- name: preflight checks - import_playbook: preflight.yml -- name: configure target hosts OS layer - import_playbook: infra/basic.yml -- name: provision Kubernetes cluster using kubespray - import_playbook: k8s/k8s.yml - when: kubernetes | default(true) -- name: install Intel BMRA Container Experience Kit features - import_playbook: intel/basic.yml - when: kubernetes | default(true) +containerd_versioned_pkg: + 'latest': "{{ containerd_package }}" + '1.3.7': "{{ containerd_package }}=1.3.7-1" + '1.3.9': "{{ containerd_package }}=1.3.9-1" + '1.4.3': "{{ containerd_package }}=1.4.3-2" + '1.4.4': "{{ containerd_package }}=1.4.4-1" + '1.4.6': "{{ containerd_package }}=1.4.6-1" + 'stable': "{{ containerd_package }}=1.4.6-1" + 'edge': "{{ containerd_package }}=1.4.6-1" diff --git a/roles/container_engine/crictl/defaults/main.yml b/roles/container_engine/crictl/defaults/main.yml new file mode 100644 index 00000000..11315cfc --- /dev/null +++ b/roles/container_engine/crictl/defaults/main.yml @@ -0,0 +1,40 @@ +## +## Copyright (c) 2020-2022 Intel Corporation. +## +## Licensed under the Apache License, Version 2.0 (the "License"); +## you may not use this file except in compliance with the License. +## You may obtain a copy of the License at +## +## http://www.apache.org/licenses/LICENSE-2.0 +## +## Unless required by applicable law or agreed to in writing, software +## distributed under the License is distributed on an "AS IS" BASIS, +## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +## See the License for the specific language governing permissions and +## limitations under the License. +## +crictl_version: "v1.21.0" + +image_arch: "amd64" + +crictl_download_url: "https://github.com/kubernetes-sigs/cri-tools/releases/download/{{ crictl_version }}/crictl-{{ crictl_version }}-{{ ansible_system | lower }}-{{ image_arch }}.tar.gz" + +crictl_checksums: + amd64: + v1.22.0: 45e0556c42616af60ebe93bf4691056338b3ea0001c0201a6a8ff8b1dbc0652a + v1.21.0: 85c78a35584971625bf1c3bcd46e5404a90396f979d7586f18b11119cb623e24 + v1.20.0: 44d5f550ef3f41f9b53155906e0229ffdbee4b19452b4df540265e29572b899c + v1.19.0: 87d8ef70b61f2fe3d8b4a48f6f712fd798c6e293ed3723c1e4bbb5052098f0ae + +crictl_binary_checksum: "{{ crictl_checksums[image_arch][crictl_version] }}" + +local_release_dir: "/tmp/releases" +bin_dir: /usr/local/bin + +crictl: + version: "{{ crictl_version }}" + dest: "{{ local_release_dir }}/crictl-{{ crictl_version }}-linux-{{ image_arch }}.tar.gz" + sha256: "{{ crictl_binary_checksum }}" + url: "{{ crictl_download_url }}" + owner: "root" + mode: "0755" diff --git a/roles/container_engine/crictl/handlers/main.yml b/roles/container_engine/crictl/handlers/main.yml new file mode 100644 index 00000000..815ba58e --- /dev/null +++ b/roles/container_engine/crictl/handlers/main.yml @@ -0,0 +1,27 @@ +## +## Copyright (c) 2020-2022 Intel Corporation. +## +## Licensed under the Apache License, Version 2.0 (the "License"); +## you may not use this file except in compliance with the License. +## You may obtain a copy of the License at +## +## http://www.apache.org/licenses/LICENSE-2.0 +## +## Unless required by applicable law or agreed to in writing, software +## distributed under the License is distributed on an "AS IS" BASIS, +## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +## See the License for the specific language governing permissions and +## limitations under the License. +## +--- +- name: get crictl completion + command: "{{ bin_dir }}/crictl completion" + changed_when: False + register: cri_completion + check_mode: false + +- name: install crictl completion + copy: + dest: /etc/bash_completion.d/crictl + content: "{{ cri_completion.stdout }}" + mode: 0644 diff --git a/roles/container_engine/crictl/tasks/crictl.yml b/roles/container_engine/crictl/tasks/crictl.yml new file mode 100644 index 00000000..f74f52a1 --- /dev/null +++ b/roles/container_engine/crictl/tasks/crictl.yml @@ -0,0 +1,71 @@ +## +## Copyright (c) 2020-2022 Intel Corporation. +## +## Licensed under the Apache License, Version 2.0 (the "License"); +## you may not use this file except in compliance with the License. +## You may obtain a copy of the License at +## +## http://www.apache.org/licenses/LICENSE-2.0 +## +## Unless required by applicable law or agreed to in writing, software +## distributed under the License is distributed on an "AS IS" BASIS, +## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +## See the License for the specific language governing permissions and +## limitations under the License. +## +--- +- block: + - name: starting download of file + debug: + msg: "{{ crictl.url }}" + + - name: create dest directory on node + file: + path: "{{ crictl.dest | dirname }}" + owner: "{{ crictl.owner }}" + mode: 0755 + state: directory + recurse: yes + + - name: download item + get_url: + url: "{{ crictl.url }}" + dest: "{{ crictl.dest }}" + owner: "{{ crictl.owner }}" + mode: "{{ crictl.mode }}" + checksum: "{{ 'sha256:' + crictl.sha256 }}" + delegate_to: "{{ inventory_hostname }}" + register: get_url_result + until: "'OK' in get_url_result.msg or 'file already exists' in get_url_result.msg" + retries: 4 + delay: 5 + no_log: true + + - name: unpack archive + unarchive: + src: "{{ crictl.dest }}" + dest: "{{ crictl.dest | dirname }}" + owner: "{{ crictl.owner }}" + mode: "{{ crictl.mode }}" + copy: no + +- name: install crictl config + template: + src: crictl.yml.j2 + dest: /etc/crictl.yaml + owner: root + mode: 0644 + +- name: copy crictl binary from download dir + copy: + src: "{{ local_release_dir }}/crictl" + dest: "{{ bin_dir }}/crictl" + mode: 0755 + remote_src: true + notify: + - get crictl completion + - install crictl completion + +- name: Set fact crictl_installed + set_fact: + crictl_installed: true diff --git a/roles/container_engine/crictl/tasks/main.yml b/roles/container_engine/crictl/tasks/main.yml new file mode 100644 index 00000000..0a9e7ab0 --- /dev/null +++ b/roles/container_engine/crictl/tasks/main.yml @@ -0,0 +1,26 @@ +## +## Copyright (c) 2020-2022 Intel Corporation. +## +## Licensed under the Apache License, Version 2.0 (the "License"); +## you may not use this file except in compliance with the License. +## You may obtain a copy of the License at +## +## http://www.apache.org/licenses/LICENSE-2.0 +## +## Unless required by applicable law or agreed to in writing, software +## distributed under the License is distributed on an "AS IS" BASIS, +## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +## See the License for the specific language governing permissions and +## limitations under the License. +## +--- +- name: install crictl + include_tasks: crictl.yml + vars: + cri_socket: >- + {%- if container_runtime == 'crio' -%} + /var/run/crio/crio.sock + {%- else -%} + /var/run/containerd/containerd.sock + {%- endif -%} + when: not crictl_installed | default(false) diff --git a/roles/container_engine/crictl/templates/crictl.yml.j2 b/roles/container_engine/crictl/templates/crictl.yml.j2 new file mode 100644 index 00000000..fbf691f8 --- /dev/null +++ b/roles/container_engine/crictl/templates/crictl.yml.j2 @@ -0,0 +1,4 @@ +runtime-endpoint: unix://{{ cri_socket }} +image-endpoint: unix://{{ cri_socket }} +timeout: 30 +debug: false diff --git a/roles/container_engine/crio/defaults/main.yml b/roles/container_engine/crio/defaults/main.yml new file mode 100644 index 00000000..d6392885 --- /dev/null +++ b/roles/container_engine/crio/defaults/main.yml @@ -0,0 +1,83 @@ +## +## Copyright (c) 2020-2022 Intel Corporation. +## +## Licensed under the Apache License, Version 2.0 (the "License"); +## you may not use this file except in compliance with the License. +## You may obtain a copy of the License at +## +## http://www.apache.org/licenses/LICENSE-2.0 +## +## Unless required by applicable law or agreed to in writing, software +## distributed under the License is distributed on an "AS IS" BASIS, +## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +## See the License for the specific language governing permissions and +## limitations under the License. +## +--- +crio_cgroup_manager: "systemd" +crio_default_runtime: "runc" +crio_conmon: "/usr/bin/conmon" +crio_enable_metrics: false +crio_log_level: "info" +crio_metrics_port: "9090" +crio_pause_image: "k8s.gcr.io/pause:3.3" + +crio_download_base: "download.opensuse.org/repositories/devel:kubic:libcontainers:stable" +crio_download_crio: "http://{{ crio_download_base }}:/cri-o:/" + +cri_crio_socket: "/var/run/crio/crio.sock" + +crun_enabled: false + +# Trusted registries to pull unqualified images (e.g. alpine:latest) from +# By default unqualified images are not allowed for security reasons +crio_registries: [] + +# Configure insecure registries. +crio_insecure_registries: [] + +# Configure registry auth (if applicable to secure/insecure registries) +crio_registry_auth: [] +# - registry: 10.0.0.2:5000 +# username: user +# password: pass + +# Define registiries mirror +crio_registries_mirrors: [] +# - prefix: docker.io +# insecure: false +# blocked: false +# location: registry-1.docker.io +# mirrors: +# - location: 172.20.100.52:5000 +# insecure: true +# - location: mirror.gcr.io +# insecure: false + +crio_seccomp_profile: "" +crio_selinux: false + +# Override system default for storage driver +# crio_storage_driver: "overlay" + +crio_stream_port: "10010" + +crio_version: "1.21" + +# The crio_runtimes variable defines a list of OCI compatible runtimes. +crio_runtimes: + - name: runc + path: /usr/bin/runc + type: oci + root: /run/runc + +# crun is a fast and low-memory footprint OCI Container Runtime fully written in C. +crun_runtime: + name: crun + path: /usr/bin/crun + type: oci + root: /run/crun + +# Configure the cri-o pids limit, increase this for heavily multi-threaded workloads +# see https://github.com/cri-o/cri-o/issues/1921 +crio_pids_limit: 1024 diff --git a/roles/container_engine/crio/files/mounts.conf b/roles/container_engine/crio/files/mounts.conf new file mode 100644 index 00000000..b7cde9d8 --- /dev/null +++ b/roles/container_engine/crio/files/mounts.conf @@ -0,0 +1 @@ +/usr/share/rhel/secrets:/run/secrets diff --git a/roles/container_engine/crio/handlers/main.yml b/roles/container_engine/crio/handlers/main.yml new file mode 100644 index 00000000..8f29b2d3 --- /dev/null +++ b/roles/container_engine/crio/handlers/main.yml @@ -0,0 +1,31 @@ +## +## Copyright (c) 2020-2022 Intel Corporation. +## +## Licensed under the Apache License, Version 2.0 (the "License"); +## you may not use this file except in compliance with the License. +## You may obtain a copy of the License at +## +## http://www.apache.org/licenses/LICENSE-2.0 +## +## Unless required by applicable law or agreed to in writing, software +## distributed under the License is distributed on an "AS IS" BASIS, +## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +## See the License for the specific language governing permissions and +## limitations under the License. +## +--- +- name: restart crio + command: /bin/true + notify: + - reload systemd + - reload crio + +- name: reload systemd + systemd: + daemon_reload: true + +- name: reload crio + service: + name: crio + state: restarted + enabled: yes diff --git a/roles/container_engine/crio/tasks/crio_repo.yml b/roles/container_engine/crio/tasks/crio_repo.yml new file mode 100644 index 00000000..94e4257b --- /dev/null +++ b/roles/container_engine/crio/tasks/crio_repo.yml @@ -0,0 +1,66 @@ +## +## Copyright (c) 2020-2022 Intel Corporation. +## +## Licensed under the Apache License, Version 2.0 (the "License"); +## you may not use this file except in compliance with the License. +## You may obtain a copy of the License at +## +## http://www.apache.org/licenses/LICENSE-2.0 +## +## Unless required by applicable law or agreed to in writing, software +## distributed under the License is distributed on an "AS IS" BASIS, +## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +## See the License for the specific language governing permissions and +## limitations under the License. +## +--- +- name: Set CRI-O kubic repo name for Ubuntu + set_fact: + crio_kubic_ubuntu_repo_name: "{{ ((ansible_distribution == 'Ubuntu') | ternary('x','')) ~ ansible_distribution ~ '_' ~ ansible_distribution_version }}" + when: ansible_distribution == "Ubuntu" + +- name: Add CRI-O kubic apt repo key + apt_key: + url: "https://{{ crio_download_base }}/{{ crio_kubic_ubuntu_repo_name }}/Release.key" + state: present + when: crio_kubic_ubuntu_repo_name is defined + register: apt_key_download + until: apt_key_download is succeeded + retries: 4 + delay: 3 + environment: "{{ proxy_env }}" + +- name: Add CRI-O kubic apt repo + apt_repository: + repo: "deb http://{{ crio_download_base }}/{{ crio_kubic_ubuntu_repo_name }}/ /" + state: present + filename: devel-kubic-libcontainers-stable + when: crio_kubic_ubuntu_repo_name is defined + +- name: add CRI-O kubic cri-o apt repo + apt_repository: + repo: "deb {{ crio_download_crio }}{{ crio_version }}/{{ crio_kubic_ubuntu_repo_name }}/ /" + state: present + filename: devel-kubic-libcontainers-stable-cri-o + when: crio_kubic_ubuntu_repo_name is defined + +- name: add CRI-O kubic yum repo + yum_repository: + name: devel_kubic_libcontainers_stable + description: Stable Releases of Upstream github.com/containers packages (CentOS_$releasever) + baseurl: http://{{ crio_download_base }}/CentOS_{{ ansible_distribution_major_version }}/ + gpgcheck: yes + gpgkey: http://{{ crio_download_base }}/CentOS_{{ ansible_distribution_major_version }}/repodata/repomd.xml.key + keepcache: '0' + when: + - ansible_os_family == "RedHat" + +- name: add CRI-O kubic yum repo + yum_repository: + name: "devel_kubic_libcontainers_stable_cri-o_{{ crio_version }}" + description: "CRI-O {{ crio_version }} (CentOS_$releasever)" + baseurl: "{{ crio_download_crio }}{{ crio_version }}/CentOS_{{ ansible_distribution_major_version }}/" + gpgcheck: yes + gpgkey: "{{ crio_download_crio }}{{ crio_version }}/CentOS_{{ ansible_distribution_major_version }}/repodata/repomd.xml.key" + when: + - ansible_os_family == "RedHat" diff --git a/roles/container_engine/crio/tasks/main.yml b/roles/container_engine/crio/tasks/main.yml new file mode 100644 index 00000000..02b29d67 --- /dev/null +++ b/roles/container_engine/crio/tasks/main.yml @@ -0,0 +1,154 @@ +## +## Copyright (c) 2020-2022 Intel Corporation. +## +## Licensed under the Apache License, Version 2.0 (the "License"); +## you may not use this file except in compliance with the License. +## You may obtain a copy of the License at +## +## http://www.apache.org/licenses/LICENSE-2.0 +## +## Unless required by applicable law or agreed to in writing, software +## distributed under the License is distributed on an "AS IS" BASIS, +## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +## See the License for the specific language governing permissions and +## limitations under the License. +## +--- +- name: load a variable file based on the OS type + include_vars: "{{ lookup('first_found', params) }}" + vars: + params: + files: + - "{{ ansible_distribution|lower }}-{{ ansible_distribution_major_version }}.yml" + - "{{ ansible_distribution|lower }}.yml" + - "{{ ansible_os_family|lower }}.yml" + paths: + - ../vars + +- name: import crio repo + import_tasks: "crio_repo.yml" + +- include_role: # noqa unnamed-task + name: container_engine/crictl + +- name: build a list of crio runtimes with crun runtime + set_fact: + crio_runtimes: "{{ crio_runtimes + [crun_runtime] }}" + when: + - crun_enabled + +- name: make sure needed folders exist in the system + with_items: + - /etc/crio + - /etc/containers + - /etc/systemd/system/crio.service.d + file: + path: "{{ item }}" + state: directory + mode: 0755 + +- name: install cri-o config + template: + src: crio.conf.j2 + dest: /etc/crio/crio.conf + mode: 0644 + register: config_install + +- name: install config.json + template: + src: config.json.j2 + dest: /etc/crio/config.json + mode: 0644 + register: reg_auth_install + +- name: install cri-o packages + package: + name: "{{ item }}" + state: present + with_items: "{{ crio_packages }}" + register: package_install + until: package_install is succeeded + retries: 4 + delay: 3 + +- name: remove example CNI configs + file: + path: "/etc/cni/net.d/{{ item }}" + state: absent + loop: + - 100-crio-bridge.conf + - 200-loopback.conf + +- name: copy mounts.conf + copy: + src: mounts.conf + dest: /etc/containers/mounts.conf + mode: 0644 + when: + - ansible_os_family == 'RedHat' + notify: restart crio + +- name: create directory for oci hooks + file: + path: /etc/containers/oci/hooks.d + state: directory + owner: root + mode: 0755 + +# metacopy=on is available since 4.19 and was backported to RHEL 4.18 kernel +- name: set metacopy mount options correctly + ini_file: + dest: /etc/containers/storage.conf + section: storage.options.overlay + option: mountopt + value: '{{ ''"nodev"'' if ansible_kernel is version_compare(("4.18" if ansible_os_family == "RedHat" else "4.19"), "<") else ''"nodev,metacopy=on"'' }}' + mode: 0644 + +- name: create directory registries configs + file: + path: /etc/containers/registries.conf.d + state: directory + owner: root + mode: 0755 + +- name: write registries mirror configs + template: + src: registry_mirror.conf.j2 + dest: "/etc/containers/registries.conf.d/{{ item.prefix }}.conf" + mode: 0644 + loop: "{{ crio_registries_mirrors }}" + notify: restart crio + +- name: write cri-o proxy drop-in + template: + src: http_proxy.conf.j2 + dest: /etc/systemd/system/crio.service.d/http-proxy.conf + mode: 0644 + notify: restart crio + when: http_proxy is defined or https_proxy is defined + +- name: ensure crio service is started and enabled + service: + name: crio + daemon_reload: true + enabled: true + state: started + register: service_start + +- name: trigger service restart only when needed + service: # noqa 503 + name: crio + state: restarted + when: + - config_install.changed + - reg_auth_install.changed + - not package_install.changed + - not service_start.changed + +- name: verify that crio is running + command: "crio-status info" + register: get_crio_info + until: get_crio_info is succeeded + changed_when: false + retries: 5 + delay: 15 diff --git a/roles/container_engine/crio/templates/config.json.j2 b/roles/container_engine/crio/templates/config.json.j2 new file mode 100644 index 00000000..4afd49f6 --- /dev/null +++ b/roles/container_engine/crio/templates/config.json.j2 @@ -0,0 +1,17 @@ +{% if crio_registry_auth is defined and crio_registry_auth|length %} +{ +{% for reg in crio_registry_auth %} + "auths": { + "{{ reg.registry }}": { + "auth": "{{ (reg.username + ':' + reg.password) | string | b64encode }}" + } +{% if not loop.last %} + }, +{% else %} + } +{% endif %} +{% endfor %} +} +{% else %} +{} +{% endif %} diff --git a/roles/container_engine/crio/templates/crictl.yml.j2 b/roles/container_engine/crio/templates/crictl.yml.j2 new file mode 100644 index 00000000..8112d903 --- /dev/null +++ b/roles/container_engine/crio/templates/crictl.yml.j2 @@ -0,0 +1,4 @@ +runtime-endpoint: unix://{{ cri_crio_socket }} +image-endpoint: unix://{{ cri_crio_socket }} +timeout: 30 +debug: false diff --git a/roles/container_engine/crio/templates/crio.conf.j2 b/roles/container_engine/crio/templates/crio.conf.j2 new file mode 100644 index 00000000..d6a7a468 --- /dev/null +++ b/roles/container_engine/crio/templates/crio.conf.j2 @@ -0,0 +1,385 @@ +# The CRI-O configuration file specifies all of the available configuration +# options and command-line flags for the crio(8) OCI Kubernetes Container Runtime +# daemon, but in a TOML format that can be more easily modified and versioned. +# +# Please refer to crio.conf(5) for details of all configuration options. + +# CRI-O supports partial configuration reload during runtime, which can be +# done by sending SIGHUP to the running process. Currently supported options +# are explicitly mentioned with: 'This option supports live configuration +# reload'. + +# CRI-O reads its storage defaults from the containers-storage.conf(5) file +# located at /etc/containers/storage.conf. Modify this storage configuration if +# you want to change the system's defaults. If you want to modify storage just +# for CRI-O, you can change the storage configuration options here. +[crio] + +# Path to the "root directory". CRI-O stores all of its data, including +# containers images, in this directory. +#root = "/var/lib/containers/storage" + +# Path to the "run directory". CRI-O stores all of its state in this directory. +#runroot = "/var/run/containers/storage" + +# Storage driver used to manage the storage of images and containers. Please +# refer to containers-storage.conf(5) to see all available storage drivers. +{% if crio_storage_driver is defined %} +storage_driver = "{{ crio_storage_driver }}" +{% endif %} + +# List to pass options to the storage driver. Please refer to +# containers-storage.conf(5) to see all available storage options. +#storage_option = [ +#] + +# The default log directory where all logs will go unless directly specified by +# the kubelet. The log directory specified must be an absolute directory. +log_dir = "/var/log/crio/pods" + +# Location for CRI-O to lay down the temporary version file. +# It is used to check if crio wipe should wipe containers, which should +# always happen on a node reboot +version_file = "/var/run/crio/version" + +# Location for CRI-O to lay down the persistent version file. +# It is used to check if crio wipe should wipe images, which should +# only happen when CRI-O has been upgraded +version_file_persist = "/var/lib/crio/version" + +# The crio.api table contains settings for the kubelet/gRPC interface. +[crio.api] + +# Path to AF_LOCAL socket on which CRI-O will listen. +listen = "/var/run/crio/crio.sock" + +# IP address on which the stream server will listen. +stream_address = "127.0.0.1" + +# The port on which the stream server will listen. If the port is set to "0", then +# CRI-O will allocate a random free port number. +stream_port = "{{ crio_stream_port }}" + +# Enable encrypted TLS transport of the stream server. +stream_enable_tls = false + +# Path to the x509 certificate file used to serve the encrypted stream. This +# file can change, and CRI-O will automatically pick up the changes within 5 +# minutes. +stream_tls_cert = "" + +# Path to the key file used to serve the encrypted stream. This file can +# change and CRI-O will automatically pick up the changes within 5 minutes. +stream_tls_key = "" + +# Path to the x509 CA(s) file used to verify and authenticate client +# communication with the encrypted stream. This file can change and CRI-O will +# automatically pick up the changes within 5 minutes. +stream_tls_ca = "" + +# Maximum grpc send message size in bytes. If not set or <=0, then CRI-O will default to 16 * 1024 * 1024. +grpc_max_send_msg_size = 16777216 + +# Maximum grpc receive message size. If not set or <= 0, then CRI-O will default to 16 * 1024 * 1024. +grpc_max_recv_msg_size = 16777216 + +# The crio.runtime table contains settings pertaining to the OCI runtime used +# and options for how to set up and manage the OCI runtime. +[crio.runtime] + +# A list of ulimits to be set in containers by default, specified as +# "=:", for example: +# "nofile=1024:2048" +# If nothing is set here, settings will be inherited from the CRI-O daemon +#default_ulimits = [ +#] + +# default_runtime is the _name_ of the OCI runtime to be used as the default. +# The name is matched against the runtimes map below. +default_runtime = "{{ crio_default_runtime }}" + +# If true, the runtime will not use pivot_root, but instead use MS_MOVE. +no_pivot = false + +# decryption_keys_path is the path where the keys required for +# image decryption are stored. This option supports live configuration reload. +decryption_keys_path = "/etc/crio/keys/" + +# Path to the conmon binary, used for monitoring the OCI runtime. +# Will be searched for using $PATH if empty. +conmon = "{{ crio_conmon }}" + +# Cgroup setting for conmon +{% if crio_cgroup_manager == "cgroupfs" %} +conmon_cgroup = "pod" +{% else %} +conmon_cgroup = "system.slice" +{% endif %} + +# Environment variable list for the conmon process, used for passing necessary +# environment variables to conmon or the runtime. +conmon_env = [ + "PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin", +] + +# Additional environment variables to set for all the +# containers. These are overridden if set in the +# container image spec or in the container runtime configuration. +default_env = [ +] + +# If true, SELinux will be used for pod separation on the host. +selinux = {{ crio_selinux | lower }} + +# Path to the seccomp.json profile which is used as the default seccomp profile +# for the runtime. If not specified, then the internal default seccomp profile +# will be used. This option supports live configuration reload. +seccomp_profile = "{{ crio_seccomp_profile }}" + +# Used to change the name of the default AppArmor profile of CRI-O. The default +# profile name is "crio-default". This profile only takes effect if the user +# does not specify a profile via the Kubernetes Pod's metadata annotation. If +# the profile is set to "unconfined", then this equals to disabling AppArmor. +# This option supports live configuration reload. +# apparmor_profile = "crio-default" + +# Cgroup management implementation used for the runtime. +cgroup_manager = "{{ crio_cgroup_manager }}" + +# List of default capabilities for containers. If it is empty or commented out, +# only the capabilities defined in the containers json file by the user/kube +# will be added. +default_capabilities = [ + "CHOWN", + "DAC_OVERRIDE", + "FSETID", + "FOWNER", + "NET_RAW", + "SETGID", + "SETUID", + "SETPCAP", + "NET_BIND_SERVICE", + "SYS_CHROOT", + "KILL", +] + +# List of default sysctls. If it is empty or commented out, only the sysctls +# defined in the container json file by the user/kube will be added. +default_sysctls = [ +] + +# List of additional devices. specified as +# "::", for example: "--device=/dev/sdc:/dev/xvdc:rwm". +#If it is empty or commented out, only the devices +# defined in the container json file by the user/kube will be added. +additional_devices = [ +] + +# Path to OCI hooks directories for automatically executed hooks. If one of the +# directories does not exist, then CRI-O will automatically skip them. +hooks_dir = [ + "/usr/share/containers/oci/hooks.d", +] + +# List of default mounts for each container. **Deprecated:** this option will +# be removed in future versions in favor of default_mounts_file. +default_mounts = [ +] + +# Path to the file specifying the defaults mounts for each container. The +# format of the config is /SRC:/DST, one mount per line. Notice that CRI-O reads +# its default mounts from the following two files: +# +# 1) /etc/containers/mounts.conf (i.e., default_mounts_file): This is the +# override file, where users can either add in their own default mounts, or +# override the default mounts shipped with the package. +# +# 2) /usr/share/containers/mounts.conf: This is the default file read for +# mounts. If you want CRI-O to read from a different, specific mounts file, +# you can change the default_mounts_file. Note, if this is done, CRI-O will +# only add mounts it finds in this file. +# +#default_mounts_file = "" + +# Maximum number of processes allowed in a container. +pids_limit = {{ crio_pids_limit }} + +# Maximum sized allowed for the container log file. Negative numbers indicate +# that no size limit is imposed. If it is positive, it must be >= 8192 to +# match/exceed conmon's read buffer. The file is truncated and re-opened so the +# limit is never exceeded. +log_size_max = -1 + +# Whether container output should be logged to journald in addition to the kuberentes log file +log_to_journald = false + +# Path to directory in which container exit files are written to by conmon. +container_exits_dir = "/var/run/crio/exits" + +# Path to directory for container attach sockets. +container_attach_socket_dir = "/var/run/crio" + +# The prefix to use for the source of the bind mounts. +bind_mount_prefix = "" + +# If set to true, all containers will run in read-only mode. +read_only = false + +# Changes the verbosity of the logs based on the level it is set to. Options +# are fatal, panic, error, warn, info, debug and trace. This option supports +# live configuration reload. +log_level = "{{ crio_log_level }}" + +# Filter the log messages by the provided regular expression. +# This option supports live configuration reload. +log_filter = "" + +# The UID mappings for the user namespace of each container. A range is +# specified in the form containerUID:HostUID:Size. Multiple ranges must be +# separated by comma. +uid_mappings = "" + +# The GID mappings for the user namespace of each container. A range is +# specified in the form containerGID:HostGID:Size. Multiple ranges must be +# separated by comma. +gid_mappings = "" + +# The minimal amount of time in seconds to wait before issuing a timeout +# regarding the proper termination of the container. The lowest possible +# value is 30s, whereas lower values are not considered by CRI-O. +ctr_stop_timeout = 30 + +# **DEPRECATED** this option is being replaced by manage_ns_lifecycle, which is described below. +# manage_network_ns_lifecycle = false + +# manage_ns_lifecycle determines whether we pin and remove namespaces +# and manage their lifecycle +manage_ns_lifecycle = false + +# The directory where the state of the managed namespaces gets tracked. +# Only used when manage_ns_lifecycle is true. +namespaces_dir = "/var/run" + +# pinns_path is the path to find the pinns binary, which is needed to manage namespace lifecycle +pinns_path = "" + +# The "crio.runtime.runtimes" table defines a list of OCI compatible runtimes. +# The runtime to use is picked based on the runtime_handler provided by the CRI. +# If no runtime_handler is provided, the runtime will be picked based on the level +# of trust of the workload. Each entry in the table should follow the format: +# +#[crio.runtime.runtimes.runtime-handler] +# runtime_path = "/path/to/the/executable" +# runtime_type = "oci" +# runtime_root = "/path/to/the/root" +# +# Where: +# - runtime-handler: name used to identify the runtime +# - runtime_path (optional, string): absolute path to the runtime executable in +# the host filesystem. If omitted, the runtime-handler identifier should match +# the runtime executable name, and the runtime executable should be placed +# in $PATH. +# - runtime_type (optional, string): type of runtime, one of: "oci", "vm". If +# omitted, an "oci" runtime is assumed. +# - runtime_root (optional, string): root directory for storage of containers +# state. + +{% for runtime in crio_runtimes %} +[crio.runtime.runtimes.{{ runtime.name }}] +runtime_path = "{{ runtime.path }}" +runtime_type = "{{ runtime.type }}" +runtime_root = "{{ runtime.root }}" +privileged_without_host_devices = {{ runtime.privileged_without_host_devices|default(false)|lower }} +{% endfor %} + +# Kata Containers with the Firecracker VMM +#[crio.runtime.runtimes.kata-fc] + +# The crio.image table contains settings pertaining to the management of OCI images. +# +# CRI-O reads its configured registries defaults from the system wide +# containers-registries.conf(5) located in /etc/containers/registries.conf. If +# you want to modify just CRI-O, you can change the registries configuration in +# this file. Otherwise, leave insecure_registries and registries commented out to +# use the system's defaults from /etc/containers/registries.conf. +[crio.image] + +# Default transport for pulling images from a remote container storage. +default_transport = "docker://" + +# The path to a file containing credentials necessary for pulling images from +# secure registries. The file is similar to that of /var/lib/kubelet/config.json +global_auth_file = "/etc/crio/config.json" + +# The image used to instantiate infra containers. +# This option supports live configuration reload. +pause_image = "{{ crio_pause_image }}" + +# The path to a file containing credentials specific for pulling the pause_image from +# above. The file is similar to that of /var/lib/kubelet/config.json +# This option supports live configuration reload. +pause_image_auth_file = "" + +# The command to run to have a container stay in the paused state. +# When explicitly set to "", it will fallback to the entrypoint and command +# specified in the pause image. When commented out, it will fallback to the +# default: "/pause". This option supports live configuration reload. +pause_command = "/pause" + +# Path to the file which decides what sort of policy we use when deciding +# whether or not to trust an image that we've pulled. It is not recommended that +# this option be used, as the default behavior of using the system-wide default +# policy (i.e., /etc/containers/policy.json) is most often preferred. Please +# refer to containers-policy.json(5) for more details. +signature_policy = "" + +# List of registries to skip TLS verification for pulling images. Please +# consider configuring the registries via /etc/containers/registries.conf before +# changing them here. +insecure_registries = [ + {% for insecure_registry in crio_insecure_registries %} + "{{ insecure_registry }}", + {% endfor %} +] + +# Controls how image volumes are handled. The valid values are mkdir, bind and +# ignore; the latter will ignore volumes entirely. +image_volumes = "mkdir" + +# List of registries to be used when pulling an unqualified image (e.g., +# "alpine:latest"). By default, registries is set to "docker.io" for +# compatibility reasons. Depending on your workload and usecase you may add more +# registries (e.g., "quay.io", "registry.fedoraproject.org", +# "registry.opensuse.org", etc.). +registries = [ + {% for registry in crio_registries %} + "{{ registry }}", + {% endfor %} +] + + +# The crio.network table containers settings pertaining to the management of +# CNI plugins. +[crio.network] + +# The default CNI network name to be selected. If not set or "", then +# CRI-O will pick-up the first one found in network_dir. +# cni_default_network = "" + +# Path to the directory where CNI configuration files are located. +network_dir = "/etc/cni/net.d/" + +# Paths to directories where CNI plugin binaries are located. +plugin_dirs = [ + "/opt/cni/bin", + "/usr/libexec/cni", +] + +# A necessary configuration for Prometheus based metrics retrieval +[crio.metrics] + +# Globally enable or disable metrics support. +enable_metrics = {{ crio_enable_metrics | bool | lower }} + +# The port on which the metrics server will listen. +metrics_port = {{ crio_metrics_port }} diff --git a/roles/container_engine/crio/templates/http_proxy.conf.j2 b/roles/container_engine/crio/templates/http_proxy.conf.j2 new file mode 100644 index 00000000..212f30f9 --- /dev/null +++ b/roles/container_engine/crio/templates/http_proxy.conf.j2 @@ -0,0 +1,2 @@ +[Service] +Environment={% if http_proxy is defined %}"HTTP_PROXY={{ http_proxy }}"{% endif %} {% if https_proxy is defined %}"HTTPS_PROXY={{ https_proxy }}"{% endif %} {% if no_proxy is defined %}"NO_PROXY={{ no_proxy }}"{% endif %} diff --git a/roles/container_engine/crio/templates/registry_mirror.conf.j2 b/roles/container_engine/crio/templates/registry_mirror.conf.j2 new file mode 100644 index 00000000..3c55026e --- /dev/null +++ b/roles/container_engine/crio/templates/registry_mirror.conf.j2 @@ -0,0 +1,11 @@ +[[registry]] +prefix = "{{ item.prefix }}" +insecure = {{ item.insecure | d('false') | string | lower }} +blocked = {{ item.blocked | d('false') | string | lower }} +location = "{{ item.location | d(item.prefix) }}" +{% for mirror in item.mirrors %} + +[[registry.mirror]] +location = "{{ mirror.location }}" +insecure = {{ mirror.insecure | d ('false') | string | lower }} +{% endfor %} diff --git a/roles/cmk_install/defaults/main.yml b/roles/container_engine/crio/vars/centos-8.yml similarity index 73% rename from roles/cmk_install/defaults/main.yml rename to roles/container_engine/crio/vars/centos-8.yml index f8735000..44a03952 100644 --- a/roles/cmk_install/defaults/main.yml +++ b/roles/container_engine/crio/vars/centos-8.yml @@ -1,5 +1,5 @@ ## -## Copyright (c) 2020-2021 Intel Corporation. +## Copyright (c) 2020-2022 Intel Corporation. ## ## Licensed under the Apache License, Version 2.0 (the "License"); ## you may not use this file except in compliance with the License. @@ -14,9 +14,12 @@ ## limitations under the License. ## --- -cmk_git_url: "https://github.com/intel/CPU-Manager-for-Kubernetes" -cmk_version: "v1.5.2" -cmk_img_version: "v1.5.2" -cmk_dir: "/usr/src/cmk" +crio_versioned_pkg: + "1.21": + - "cri-o-1.21.*" + "1.20": + - "cri-o-1.20.*" + "1.19": + - "cri-o-1.19.*" -cmk_untaint_nodes: true +crio_packages: "{{ crio_versioned_pkg[crio_version] }}" diff --git a/roles/container_engine/crio/vars/redhat.yml b/roles/container_engine/crio/vars/redhat.yml new file mode 100644 index 00000000..3427d117 --- /dev/null +++ b/roles/container_engine/crio/vars/redhat.yml @@ -0,0 +1,19 @@ +## +## Copyright (c) 2020-2022 Intel Corporation. +## +## Licensed under the Apache License, Version 2.0 (the "License"); +## you may not use this file except in compliance with the License. +## You may obtain a copy of the License at +## +## http://www.apache.org/licenses/LICENSE-2.0 +## +## Unless required by applicable law or agreed to in writing, software +## distributed under the License is distributed on an "AS IS" BASIS, +## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +## See the License for the specific language governing permissions and +## limitations under the License. +## +--- +crio_packages: + - cri-o + - oci-systemd-hook diff --git a/roles/kubespray_install/tasks/main.yml b/roles/container_engine/crio/vars/ubuntu.yml similarity index 57% rename from roles/kubespray_install/tasks/main.yml rename to roles/container_engine/crio/vars/ubuntu.yml index ec348523..0b9f4841 100644 --- a/roles/kubespray_install/tasks/main.yml +++ b/roles/container_engine/crio/vars/ubuntu.yml @@ -1,5 +1,5 @@ ## -## Copyright (c) 2020-2021 Intel Corporation. +## Copyright (c) 2020-2022 Intel Corporation. ## ## Licensed under the Apache License, Version 2.0 (the "License"); ## you may not use this file except in compliance with the License. @@ -14,12 +14,22 @@ ## limitations under the License. ## --- -- name: assert that Multus is enabled in the config - assert: - that: - - "kube_network_plugin_multus" - fail_msg: "Multus must be enabled to have fully functional cluster deployment" +crio_versioned_pkg: + "1.21": + - "cri-o=1.21*" + - cri-o-runc + "1.20": + - "cri-o=1.20*" + - cri-o-runc + "1.19": + - "cri-o=1.19*" + - cri-o-runc -- name: install kubespray python requirements - pip: - requirements: "{{ playbook_dir }}/kubespray/requirements.txt" +crio_packages: "{{ crio_versioned_pkg[crio_version] }}" + +# The crio_runtimes variable defines a list of OCI compatible runtimes. +crio_runtimes: + - name: runc + path: /usr/sbin/runc + type: oci + root: /run/runc diff --git a/roles/container_engine/docker/defaults/main.yml b/roles/container_engine/docker/defaults/main.yml new file mode 100644 index 00000000..93471c01 --- /dev/null +++ b/roles/container_engine/docker/defaults/main.yml @@ -0,0 +1,55 @@ +## +## Copyright (c) 2020-2022 Intel Corporation. +## +## Licensed under the Apache License, Version 2.0 (the "License"); +## you may not use this file except in compliance with the License. +## You may obtain a copy of the License at +## +## http://www.apache.org/licenses/LICENSE-2.0 +## +## Unless required by applicable law or agreed to in writing, software +## distributed under the License is distributed on an "AS IS" BASIS, +## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +## See the License for the specific language governing permissions and +## limitations under the License. +## +--- +docker_version: '20.10' +docker_cli_version: "{{ docker_version }}" + +# A string of extra options to pass to the docker daemon. +# This string should be exactly as you wish it to appear. +# docker_options: "" + +# define docker bin_dir +docker_bin_dir: "/usr/bin" + +# Used to set docker daemon iptables options to true +docker_iptables_enabled: true + +docker_registry_mirrors: [] +docker_insecure_registries: [] + +# Docker Cgroup driver (native.cgroupdriver) +# Valid options are systemd or cgroupfs +docker_cgroup_driver: systemd + +# Path used to store Docker data +docker_daemon_graph: "/var/lib/docker" + +# Docker log options +# Rotate container stderr/stdout logs at 50m and keep last 5 +docker_log_opts: "--log-opt max-size=50m --log-opt max-file=5" + +## If non-empty will override default system MountFlags value. +## This option takes a mount propagation flag: shared, slave +## or private, which control whether mounts in the file system +## namespace set up for docker will receive or propagate mounts +## and unmounts. Leave empty for system default +# docker_mount_flags: + +# keep docker packages after installation; speeds up repeated ansible provisioning runs when '1' +# docker_rpm_keepcache: 1 + +yum_repo_dir: /etc/yum.repos.d + diff --git a/roles/container_engine/docker/handlers/main.yml b/roles/container_engine/docker/handlers/main.yml new file mode 100644 index 00000000..8db7031b --- /dev/null +++ b/roles/container_engine/docker/handlers/main.yml @@ -0,0 +1,38 @@ +## +## Copyright (c) 2020-2022 Intel Corporation. +## +## Licensed under the Apache License, Version 2.0 (the "License"); +## you may not use this file except in compliance with the License. +## You may obtain a copy of the License at +## +## http://www.apache.org/licenses/LICENSE-2.0 +## +## Unless required by applicable law or agreed to in writing, software +## distributed under the License is distributed on an "AS IS" BASIS, +## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +## See the License for the specific language governing permissions and +## limitations under the License. +## +--- +- name: restart docker + command: /bin/true + notify: + - reload systemd + - reload docker + - wait for docker + +- name: reload systemd + systemd: + daemon_reload: true + +- name: reload docker + service: + name: docker + state: restarted + +- name: wait for docker + command: "{{ docker_bin_dir }}/docker images" + register: docker_ready + retries: 20 + delay: 1 + until: docker_ready.rc == 0 diff --git a/roles/container_engine/docker/meta/main.yml b/roles/container_engine/docker/meta/main.yml new file mode 100644 index 00000000..4c5cc977 --- /dev/null +++ b/roles/container_engine/docker/meta/main.yml @@ -0,0 +1,18 @@ +## +## Copyright (c) 2020-2022 Intel Corporation. +## +## Licensed under the Apache License, Version 2.0 (the "License"); +## you may not use this file except in compliance with the License. +## You may obtain a copy of the License at +## +## http://www.apache.org/licenses/LICENSE-2.0 +## +## Unless required by applicable law or agreed to in writing, software +## distributed under the License is distributed on an "AS IS" BASIS, +## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +## See the License for the specific language governing permissions and +## limitations under the License. +## +--- +dependencies: # noqa 701 + - role: container_engine/containerd_common diff --git a/roles/container_engine/docker/tasks/main.yml b/roles/container_engine/docker/tasks/main.yml new file mode 100644 index 00000000..b9dc60c2 --- /dev/null +++ b/roles/container_engine/docker/tasks/main.yml @@ -0,0 +1,113 @@ +## +## Copyright (c) 2020-2022 Intel Corporation. +## +## Licensed under the Apache License, Version 2.0 (the "License"); +## you may not use this file except in compliance with the License. +## You may obtain a copy of the License at +## +## http://www.apache.org/licenses/LICENSE-2.0 +## +## Unless required by applicable law or agreed to in writing, software +## distributed under the License is distributed on an "AS IS" BASIS, +## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +## See the License for the specific language governing permissions and +## limitations under the License. +## +--- +- name: load a variable file based on the OS type + include_vars: "{{ lookup('first_found', params) }}" + vars: + params: + files: + - "{{ ansible_distribution|lower }}.yml" + - "{{ ansible_os_family|lower }}.yml" + paths: + - ../vars + +- name: ensure docker-ce repository public key is installed + apt_key: + id: "{{ item }}" + url: "{{ docker_repo_key_info.url }}" + state: present + register: keyserver_task_result + until: keyserver_task_result is succeeded + retries: 4 + delay: 3 + with_items: "{{ docker_repo_key_info.repo_keys }}" + when: ansible_distribution == 'Ubuntu' + +- name: ensure docker-ce repository is enabled + apt_repository: + repo: "{{ item }}" + state: present + with_items: "{{ docker_repo_info.repos }}" + when: ansible_distribution == 'Ubuntu' + +- name: Configure docker repository on RedHat/CentOS + template: + src: "rh_docker.repo.j2" + dest: "{{ yum_repo_dir }}/docker-ce.repo" + mode: 0644 + when: + - ansible_os_family == "RedHat" + +- name: Remove dpkg hold + dpkg_selections: + name: "{{ item }}" + selection: install + when: ansible_distribution == 'Ubuntu' + changed_when: false + with_items: + - "{{ containerd_package }}" + - docker-ce + - docker-ce-cli + +- name: ensure docker packages are installed + package: + name: "{{ docker_package_info.pkgs }}" + state: "{{ docker_package_info.state | default('present') }}" + module_defaults: + apt: + update_cache: true + dnf: + enablerepo: "{{ docker_package_info.enablerepo | default(omit) }}" + yum: + enablerepo: "{{ docker_package_info.enablerepo | default(omit) }}" + register: docker_task_result + until: docker_task_result is succeeded + retries: 4 + delay: 3 + notify: restart docker + when: + - docker_package_info.pkgs|length > 0 + +- name: ensure docker started, remove config if docker start failed and try again + block: + - name: ensure service is started if docker packages are already present + service: + name: docker + state: started + when: docker_task_result is not changed + rescue: + - debug: # noqa unnamed-task + msg: "Docker start failed. Try to remove config" + - name: remove config + file: + path: "{{ item }}" + state: absent + with_items: + - /etc/systemd/system/docker.service.d/http-proxy.conf + - /etc/systemd/system/docker.service.d/docker-options.conf + notify: restart docker + +- name: flush handlers so we can wait for docker to come up + meta: flush_handlers + +- name: Set docker systemd config + import_tasks: systemd.yml + +- name: ensure docker service is started and enabled + service: + name: docker + enabled: yes + state: started diff --git a/roles/container_engine/docker/tasks/systemd.yml b/roles/container_engine/docker/tasks/systemd.yml new file mode 100644 index 00000000..2edeb106 --- /dev/null +++ b/roles/container_engine/docker/tasks/systemd.yml @@ -0,0 +1,56 @@ +## +## Copyright (c) 2020-2022 Intel Corporation. +## +## Licensed under the Apache License, Version 2.0 (the "License"); +## you may not use this file except in compliance with the License. +## You may obtain a copy of the License at +## +## http://www.apache.org/licenses/LICENSE-2.0 +## +## Unless required by applicable law or agreed to in writing, software +## distributed under the License is distributed on an "AS IS" BASIS, +## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +## See the License for the specific language governing permissions and +## limitations under the License. +## +--- +- name: create docker service systemd directory if it doesn't exist + file: + path: /etc/systemd/system/docker.service.d + state: directory + mode: 0755 + +- name: write docker proxy drop-in + template: + src: http_proxy.conf.j2 + dest: /etc/systemd/system/docker.service.d/http-proxy.conf + mode: 0644 + notify: restart docker + when: http_proxy is defined or https_proxy is defined + +- name: get systemd version + # noqa 303 - systemctl is called intentionally here + shell: set -o pipefail && systemctl --version | head -n 1 | cut -d " " -f 2 + args: + executable: /bin/bash + register: systemd_version + changed_when: false + check_mode: false + +- name: write docker.service systemd file + template: + src: docker.service.j2 + dest: /etc/systemd/system/docker.service + mode: 0644 + register: docker_service_file + notify: restart docker + +- name: write docker options systemd drop-in + template: + src: docker_options.conf.j2 + dest: "/etc/systemd/system/docker.service.d/docker-options.conf" + mode: 0644 + notify: restart docker + +- name: Flush handlers + meta: flush_handlers diff --git a/roles/container_engine/docker/templates/docker.service.j2 b/roles/container_engine/docker/templates/docker.service.j2 new file mode 100644 index 00000000..6ca78f2e --- /dev/null +++ b/roles/container_engine/docker/templates/docker.service.j2 @@ -0,0 +1,34 @@ +[Unit] +Description=Docker Application Container Engine +Documentation=http://docs.docker.com +{% if ansible_os_family == "RedHat" %} +After=network.target containerd.service +BindsTo=containerd.service +{% elif ansible_distribution == "Ubuntu" %} +After=network.target docker.socket containerd.service +BindsTo=containerd.service +Wants=docker.socket +{% endif %} + +[Service] +Type=notify +Environment=GOTRACEBACK=crash +ExecReload=/bin/kill -s HUP $MAINPID +Delegate=yes +KillMode=process +ExecStart={{ docker_bin_dir }}/dockerd \ + $DOCKER_OPTS +{% if systemd_version.stdout|int >= 226 %} +TasksMax=infinity +{% endif %} +LimitNOFILE=1048576 +LimitNPROC=1048576 +LimitCORE=infinity +TimeoutStartSec=1min +# restart the docker process if it exits prematurely +Restart=on-failure +StartLimitBurst=3 +StartLimitInterval=60s + +[Install] +WantedBy=multi-user.target diff --git a/roles/container_engine/docker/templates/docker_options.conf.j2 b/roles/container_engine/docker/templates/docker_options.conf.j2 new file mode 100644 index 00000000..dd7edcea --- /dev/null +++ b/roles/container_engine/docker/templates/docker_options.conf.j2 @@ -0,0 +1,11 @@ +[Service] +Environment="DOCKER_OPTS={{ docker_options|default('') }} --iptables={{ docker_iptables_enabled }} \ +--exec-opt native.cgroupdriver={{ docker_cgroup_driver }} \ +{% for i in docker_insecure_registries %}--insecure-registry={{ i }} {% endfor %} \ +{% for i in docker_registry_mirrors %}--registry-mirror={{ i }} {% endfor %} \ +--data-root={{ docker_daemon_graph }} \ +{{ docker_log_opts }} \ +" +{% if docker_mount_flags is defined and docker_mount_flags != "" %} +MountFlags={{ docker_mount_flags }} +{% endif %} diff --git a/roles/container_engine/docker/templates/http_proxy.conf.j2 b/roles/container_engine/docker/templates/http_proxy.conf.j2 new file mode 100644 index 00000000..212f30f9 --- /dev/null +++ b/roles/container_engine/docker/templates/http_proxy.conf.j2 @@ -0,0 +1,2 @@ +[Service] +Environment={% if http_proxy is defined %}"HTTP_PROXY={{ http_proxy }}"{% endif %} {% if https_proxy is defined %}"HTTPS_PROXY={{ https_proxy }}"{% endif %} {% if no_proxy is defined %}"NO_PROXY={{ no_proxy }}"{% endif %} diff --git a/roles/container_engine/docker/templates/rh_docker.repo.j2 b/roles/container_engine/docker/templates/rh_docker.repo.j2 new file mode 100644 index 00000000..178bbc2c --- /dev/null +++ b/roles/container_engine/docker/templates/rh_docker.repo.j2 @@ -0,0 +1,10 @@ +[docker-ce] +name=Docker-CE Repository +baseurl={{ docker_rh_repo_base_url }} +enabled=0 +gpgcheck={{ '1' if docker_rh_repo_gpgkey else '0' }} +keepcache={{ docker_rpm_keepcache | default('1') }} +gpgkey={{ docker_rh_repo_gpgkey }} +{% if http_proxy is defined %} +proxy={{ http_proxy }} +{% endif %} diff --git a/roles/container_engine/docker/vars/redhat.yml b/roles/container_engine/docker/vars/redhat.yml new file mode 100644 index 00000000..93bf3379 --- /dev/null +++ b/roles/container_engine/docker/vars/redhat.yml @@ -0,0 +1,39 @@ +## +## Copyright (c) 2020-2022 Intel Corporation. +## +## Licensed under the Apache License, Version 2.0 (the "License"); +## you may not use this file except in compliance with the License. +## You may obtain a copy of the License at +## +## http://www.apache.org/licenses/LICENSE-2.0 +## +## Unless required by applicable law or agreed to in writing, software +## distributed under the License is distributed on an "AS IS" BASIS, +## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +## See the License for the specific language governing permissions and +## limitations under the License. +## +--- +# https://docs.docker.com/engine/installation/linux/centos/#install-from-a-package +# https://download.docker.com/linux/centos/>/x86_64/stable/Packages/ +# or do 'yum --showduplicates list docker-engine' +docker_versioned_pkg: + 'latest': docker-ce + '18.09': docker-ce-18.09.9-3.el7 + '19.03': docker-ce-19.03.15-3.el{{ ansible_distribution_major_version }} + '20.10': docker-ce-20.10.8-3.el{{ ansible_distribution_major_version }} + 'stable': docker-ce-20.10.8-3.el{{ ansible_distribution_major_version }} + 'edge': docker-ce-20.10.8-3.el{{ ansible_distribution_major_version }} + +docker_cli_versioned_pkg: + 'latest': docker-ce-cli + '18.09': docker-ce-cli-18.09.9-3.el7 + '19.03': docker-ce-cli-19.03.15-3.el{{ ansible_distribution_major_version }} + '20.10': docker-ce-cli-20.10.8-3.el{{ ansible_distribution_major_version }} + +docker_package_info: + enablerepo: "docker-ce" + pkgs: + - "{{ containerd_versioned_pkg[containerd_version | string] }}" + - "{{ docker_cli_versioned_pkg[docker_cli_version | string] }}" + - "{{ docker_versioned_pkg[docker_version | string] }}" diff --git a/roles/container_engine/docker/vars/ubuntu.yml b/roles/container_engine/docker/vars/ubuntu.yml new file mode 100644 index 00000000..1b0fa365 --- /dev/null +++ b/roles/container_engine/docker/vars/ubuntu.yml @@ -0,0 +1,48 @@ +## +## Copyright (c) 2020-2022 Intel Corporation. +## +## Licensed under the Apache License, Version 2.0 (the "License"); +## you may not use this file except in compliance with the License. +## You may obtain a copy of the License at +## +## http://www.apache.org/licenses/LICENSE-2.0 +## +## Unless required by applicable law or agreed to in writing, software +## distributed under the License is distributed on an "AS IS" BASIS, +## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +## See the License for the specific language governing permissions and +## limitations under the License. +## +--- +# https://download.docker.com/linux/ubuntu/ +docker_versioned_pkg: + 'latest': docker-ce + '18.09': docker-ce=5:18.09.9~3-0~ubuntu-{{ ansible_distribution_release|lower }} + '19.03': docker-ce=5:19.03.15~3-0~ubuntu-{{ ansible_distribution_release|lower }} + '20.10': docker-ce=5:20.10.8~3-0~ubuntu-{{ ansible_distribution_release|lower }} + 'stable': docker-ce=5:20.10.8~3-0~ubuntu-{{ ansible_distribution_release|lower }} + 'edge': docker-ce=5:20.10.8~3-0~ubuntu-{{ ansible_distribution_release|lower }} + +docker_cli_versioned_pkg: + 'latest': docker-ce-cli + '18.09': docker-ce-cli=5:18.09.9~3-0~ubuntu-{{ ansible_distribution_release|lower }} + '19.03': docker-ce-cli=5:19.03.15~3-0~ubuntu-{{ ansible_distribution_release|lower }} + '20.10': docker-ce-cli=5:20.10.8~3-0~ubuntu-{{ ansible_distribution_release|lower }} + +docker_package_info: + pkgs: + - "{{ containerd_versioned_pkg[containerd_version | string] }}" + - "{{ docker_cli_versioned_pkg[docker_cli_version | string] }}" + - "{{ docker_versioned_pkg[docker_version | string] }}" + +docker_repo_key_info: + url: '{{ docker_ubuntu_repo_gpgkey }}' + repo_keys: + - '{{ docker_ubuntu_repo_repokey }}' + +docker_repo_info: + repos: + - > + deb [arch=amd64] {{ docker_ubuntu_repo_base_url }} + {{ ansible_distribution_release|lower }} + stable diff --git a/roles/container_engine/meta/main.yml b/roles/container_engine/meta/main.yml new file mode 100644 index 00000000..e68b856d --- /dev/null +++ b/roles/container_engine/meta/main.yml @@ -0,0 +1,28 @@ +## +## Copyright (c) 2020-2022 Intel Corporation. +## +## Licensed under the Apache License, Version 2.0 (the "License"); +## you may not use this file except in compliance with the License. +## You may obtain a copy of the License at +## +## http://www.apache.org/licenses/LICENSE-2.0 +## +## Unless required by applicable law or agreed to in writing, software +## distributed under the License is distributed on an "AS IS" BASIS, +## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +## See the License for the specific language governing permissions and +## limitations under the License. +## +--- +dependencies: # noqa 701 + - role: container_engine/crio + when: + - container_runtime == 'crio' + + - role: container_engine/containerd + when: + - container_runtime == 'containerd' + + - role: container_engine/docker + when: + - container_runtime == 'docker' diff --git a/roles/container_registry/defaults/main.yml b/roles/container_registry/defaults/main.yml index 9965c20f..628a18b1 100644 --- a/roles/container_registry/defaults/main.yml +++ b/roles/container_registry/defaults/main.yml @@ -1,5 +1,5 @@ ## -## Copyright (c) 2020-2021 Intel Corporation. +## Copyright (c) 2020-2022 Intel Corporation. ## ## Licensed under the Apache License, Version 2.0 (the "License"); ## you may not use this file except in compliance with the License. @@ -14,7 +14,7 @@ ## limitations under the License. ## --- -registry_secret_name: "docker-registry-tls" +registry_secret_name: "container-registry-tls" registry_namespace: "kube-system" registry_proxy: 5043 registry_port: 5000 @@ -30,9 +30,3 @@ release_name: container-registry registry_auth: "/var/lib/kubelet/config.json" registry_auth_env: "REGISTRY_AUTH_FILE=/var/lib/kubelet/config.json" - -install_dependencies: - Debian: - - apache2-utils - RedHat: - - httpd-tools diff --git a/roles/container_registry/tasks/main.yml b/roles/container_registry/tasks/main.yml index cbd796c6..d4120e34 100644 --- a/roles/container_registry/tasks/main.yml +++ b/roles/container_registry/tasks/main.yml @@ -1,5 +1,5 @@ ## -## Copyright (c) 2020-2021 Intel Corporation. +## Copyright (c) 2020-2022 Intel Corporation. ## ## Licensed under the Apache License, Version 2.0 (the "License"); ## you may not use this file except in compliance with the License. @@ -19,22 +19,6 @@ name: install_dependencies when: inventory_hostname == groups['kube_control_plane'][0] -- name: install required Python google-auth module for CentOS 7 - pip: - name: google-auth==1.24.0 - when: - - inventory_hostname == groups['kube_control_plane'][0] - - ansible_distribution == "CentOS" - - ansible_distribution_version < '8' - -- name: install Podman - package: - name: podman - when: - - ansible_distribution == "CentOS" - - ansible_distribution_version < '8' - - '"docker" not in container_runtime' - - name: wait for kube-apiserver to be up uri: url: "https://127.0.0.1:6443/healthz" @@ -57,6 +41,7 @@ - name: clean up temporary files file: path=registry_htpasswd state=absent delegate_to: localhost + become: no run_once: yes when: inventory_hostname == groups['kube_control_plane'][0] @@ -65,14 +50,7 @@ register: htpasswd when: inventory_hostname == groups['kube_control_plane'][0] -- name: create Helm charts directory if needed - file: - path: /usr/src/charts - state: directory - mode: 0755 - when: inventory_hostname == groups['kube_control_plane'][0] - -- name: create Helm registry charts directory if needed +- name: create container registry directory file: path: "{{ container_registry_path }}" state: directory @@ -167,18 +145,6 @@ name: - docker==4.3.1 -# Ubuntu 18.04 does not support metacopy option -- name: remove metacopy from storage.conf - lineinfile: - path: "/etc/containers/storage.conf" - regexp: '^mountopt =' - line: 'mountopt = "nodev"' - state: present - mode: 0644 - when: - - ansible_distribution == "Ubuntu" and ansible_distribution_version == '18.04' - - '"docker" not in container_runtime' - - name: grant access to the registry to Docker on all nodes docker_login: username: docker diff --git a/roles/container_registry/tasks/tls.yml b/roles/container_registry/tasks/tls.yml index 03ac3e5c..4e9b0166 100644 --- a/roles/container_registry/tasks/tls.yml +++ b/roles/container_registry/tasks/tls.yml @@ -1,5 +1,5 @@ ## -## Copyright (c) 2020-2021 Intel Corporation. +## Copyright (c) 2020-2022 Intel Corporation. ## ## Licensed under the Apache License, Version 2.0 (the "License"); ## you may not use this file except in compliance with the License. @@ -14,128 +14,108 @@ ## limitations under the License. ## --- -# dependencies -- name: install Golang - import_role: - name: golang_install - when: inventory_hostname == groups['kube_control_plane'][0] - - # server -- name: clean up any preexisting certs/key/CSR files - file: path=/etc/ssl/registry state=absent - failed_when: false - become: yes - when: inventory_hostname == groups['kube_control_plane'][0] - -- name: create registry SSL directory - become: yes - file: - path: /etc/ssl/registry - state: directory - mode: '0700' - owner: root - group: root - when: - inventory_hostname == groups['kube_control_plane'][0] - -- name: delete any preexisting certs/key/CSR from Kubernetes - command: kubectl delete csr registry.{{ registry_namespace }} - when: inventory_hostname == groups['kube_control_plane'][0] - failed_when: false - -- name: delete any preexisting secrets from Kubernetes - command: kubectl delete secret -n {{ registry_namespace }} {{ registry_secret_name }} - when: inventory_hostname == groups['kube_control_plane'][0] - failed_when: false - -- name: populate registry CSR template - template: - src: "registry_csr.json.j2" - dest: "/etc/ssl/registry/registry-csr.json" - force: yes - mode: preserve - become: yes - when: inventory_hostname == groups['kube_control_plane'][0] - -- name: get GOPATH - command: /usr/local/go/bin/go env GOPATH - register: gopath - when: inventory_hostname == groups['kube_control_plane'][0] - -- name: generate key and CSR - shell: set -o pipefail && {{ gopath.stdout }}/bin/cfssl genkey registry-csr.json | {{ gopath.stdout }}/bin/cfssljson -bare registry - args: - chdir: "/etc/ssl/registry/" - executable: /bin/bash - when: inventory_hostname == groups['kube_control_plane'][0] - become: yes - -- name: read generated key - command: cat registry-key.pem - args: - chdir: "/etc/ssl/registry/" - register: key - when: inventory_hostname == groups['kube_control_plane'][0] - -- name: load generated key - set_fact: - registry_key: "{{ key.stdout }}" - when: inventory_hostname == groups['kube_control_plane'][0] - -- name: read generated csr - command: cat registry.csr - args: - chdir: "/etc/ssl/registry/" - register: csr - when: inventory_hostname == groups['kube_control_plane'][0] - -- name: load generated csr - set_fact: - registry_csr: "{{ csr.stdout | b64encode }}" - when: inventory_hostname == groups['kube_control_plane'][0] - -- name: populate registry Kubernetes CA CSR template - template: - src: "kube_registry_csr.yml.j2" - dest: "/etc/ssl/registry/kube-registry-csr.yml" - force: yes - mode: preserve - when: inventory_hostname == groups['kube_control_plane'][0] - -- name: send CSR to the Kubernetes API Server - command: kubectl apply -f /etc/ssl/registry/kube-registry-csr.yml - when: inventory_hostname == groups['kube_control_plane'][0] - -- name: approve request - command: kubectl certificate approve registry.kube-system - when: inventory_hostname == groups['kube_control_plane'][0] - -- name: get approved certificate - shell: kubectl get csr registry.kube-system -o jsonpath='{.status.certificate}' - args: - chdir: "/etc/ssl/registry" - register: cert - when: inventory_hostname == groups['kube_control_plane'][0] - retries: 30 - delay: 1 - until: cert.stdout | length > 0 - -- name: load generated cert - set_fact: - registry_cert: "{{ cert.stdout | b64decode }}" - when: inventory_hostname == groups['kube_control_plane'][0] - -- name: create TLS secret for registry - command: >- - kubectl create -n {{ registry_namespace }} secret generic {{ registry_secret_name }} - --from-literal=tls.crt='{{ registry_cert }}' - --from-literal=tls.key='{{ registry_key }}' - when: inventory_hostname == groups['kube_control_plane'][0] - -- name: clean up - file: path=/etc/ssl/registry state=absent - failed_when: false - become: yes +# server +- name: configure master node + block: + - name: clean up any preexisting certs/key/CSR files + file: path=/etc/ssl/registry state=absent + failed_when: false + become: yes + + - name: create registry SSL directory + become: yes + file: + path: /etc/ssl/registry + state: directory + mode: '0700' + owner: root + group: root + + - name: delete any preexisting certs/key/CSR from Kubernetes + command: kubectl delete csr registry.{{ registry_namespace }} + failed_when: false + + - name: delete any preexisting secrets from Kubernetes + command: kubectl delete secret -n {{ registry_namespace }} {{ registry_secret_name }} + failed_when: false + + - name: populate registry CSR template + template: + src: "registry_csr.json.j2" + dest: "/etc/ssl/registry/registry-csr.json" + force: yes + mode: preserve + become: yes + + - name: get GOPATH + command: /usr/local/go/bin/go env GOPATH + register: gopath + changed_when: false + + - name: generate key and CSR + shell: >- + set -o pipefail && {{ gopath.stdout }}/bin/cfssl genkey registry-csr.json | {{ gopath.stdout }}/bin/cfssljson -bare registry + args: + chdir: "/etc/ssl/registry/" + executable: /bin/bash + become: yes + + - name: read generated key + command: cat registry-key.pem + args: + chdir: "/etc/ssl/registry/" + register: key + + - name: load generated key + set_fact: + registry_key: "{{ key.stdout }}" + + - name: read generated csr + command: cat registry.csr + args: + chdir: "/etc/ssl/registry/" + register: csr + + - name: load generated csr + set_fact: + registry_csr: "{{ csr.stdout | b64encode }}" + + - name: populate registry Kubernetes CA CSR template + template: + src: "kube_registry_csr.yml.j2" + dest: "/etc/ssl/registry/kube-registry-csr.yml" + force: yes + mode: preserve + + - name: send CSR to the Kubernetes API Server + command: kubectl apply -f /etc/ssl/registry/kube-registry-csr.yml + + - name: approve request + command: kubectl certificate approve registry.kube-system + + - name: get approved certificate + shell: kubectl get csr registry.kube-system -o jsonpath='{.status.certificate}' + args: + chdir: "/etc/ssl/registry" + register: cert + retries: 30 + delay: 1 + until: cert.stdout | length > 0 + + - name: load generated cert + set_fact: + registry_cert: "{{ cert.stdout | b64decode }}" + + - name: create TLS secret for registry + command: >- + kubectl create -n {{ registry_namespace }} secret generic {{ registry_secret_name }} + --from-literal=tls.crt='{{ registry_cert }}' + --from-literal=tls.key='{{ registry_key }}' + + - name: clean up + file: path=/etc/ssl/registry state=absent + failed_when: false + become: yes when: inventory_hostname == groups['kube_control_plane'][0] # copy CA file so that registry clients can validate its certificate @@ -182,4 +162,4 @@ mode: '0600' owner: root group: root - when: '"docker" not in container_runtime' \ No newline at end of file + when: '"docker" not in container_runtime' diff --git a/roles/container_registry/templates/container-registry/configmap.yaml.j2 b/roles/container_registry/templates/container-registry/configmap.yaml.j2 index 3480accf..5030d376 100644 --- a/roles/container_registry/templates/container-registry/configmap.yaml.j2 +++ b/roles/container_registry/templates/container-registry/configmap.yaml.j2 @@ -1,5 +1,5 @@ --- -# Source: docker-registry/templates/configmap.yaml +# Source: container-registry/templates/configmap.yaml apiVersion: v1 kind: ConfigMap metadata: diff --git a/roles/container_registry/templates/container-registry/deployment.yaml.j2 b/roles/container_registry/templates/container-registry/deployment.yaml.j2 index 396df63e..5911de7e 100644 --- a/roles/container_registry/templates/container-registry/deployment.yaml.j2 +++ b/roles/container_registry/templates/container-registry/deployment.yaml.j2 @@ -1,5 +1,5 @@ --- -# Source: docker-registry/templates/deployment.yaml +# Source: container-registry/templates/deployment.yaml apiVersion: apps/v1 kind: Deployment metadata: @@ -37,13 +37,13 @@ spec: mountPath: /etc/nginx - name: htpasswd mountPath: "/etc/nginx/conf.d/auth" - - name: docker-registry + - name: container-registry image: "{{ registry_image }}" imagePullPolicy: IfNotPresent command: - /bin/registry - serve - - /etc/docker/registry/config.yml + - /etc/container/registry/config.yml livenessProbe: exec: command: @@ -71,7 +71,7 @@ spec: - name: data mountPath: /var/lib/registry/ - name: "{{ release_name }}-config" - mountPath: "/etc/docker/registry" + mountPath: "/etc/container/registry" - name: probe mountPath: /tmp/probe.sh diff --git a/roles/container_registry/templates/container-registry/nginx-configmap.yaml.j2 b/roles/container_registry/templates/container-registry/nginx-configmap.yaml.j2 index c2bdacff..f4bd3459 100644 --- a/roles/container_registry/templates/container-registry/nginx-configmap.yaml.j2 +++ b/roles/container_registry/templates/container-registry/nginx-configmap.yaml.j2 @@ -71,5 +71,3 @@ data: } } } - - diff --git a/roles/container_registry/templates/container-registry/secret.yaml.j2 b/roles/container_registry/templates/container-registry/secret.yaml.j2 index 7b68185d..b4271577 100644 --- a/roles/container_registry/templates/container-registry/secret.yaml.j2 +++ b/roles/container_registry/templates/container-registry/secret.yaml.j2 @@ -1,5 +1,5 @@ --- -# Source: docker-registry/templates/secret.yaml +# Source: container-registry/templates/secret.yaml apiVersion: v1 kind: Secret metadata: diff --git a/roles/container_registry/templates/container-registry/service.yaml.j2 b/roles/container_registry/templates/container-registry/service.yaml.j2 index cabb75cc..5643c6fe 100644 --- a/roles/container_registry/templates/container-registry/service.yaml.j2 +++ b/roles/container_registry/templates/container-registry/service.yaml.j2 @@ -1,5 +1,5 @@ --- -# Source: docker-registry/templates/service.yaml +# Source: container-registry/templates/service.yaml apiVersion: v1 kind: Service metadata: diff --git a/roles/container_registry/templates/kube_registry_csr.yml.j2 b/roles/container_registry/templates/kube_registry_csr.yml.j2 index d0ad43ae..19fc83ed 100644 --- a/roles/container_registry/templates/kube_registry_csr.yml.j2 +++ b/roles/container_registry/templates/kube_registry_csr.yml.j2 @@ -1,8 +1,9 @@ -apiVersion: certificates.k8s.io/v1beta1 +apiVersion: certificates.k8s.io/v1 kind: CertificateSigningRequest metadata: name: registry.{{ registry_namespace }} spec: + signerName: kubernetes.io/kubelet-serving request: {{ registry_csr }} usages: - digital signature diff --git a/roles/container_registry/templates/registry_csr.json.j2 b/roles/container_registry/templates/registry_csr.json.j2 index a93f96ce..b03a04ce 100644 --- a/roles/container_registry/templates/registry_csr.json.j2 +++ b/roles/container_registry/templates/registry_csr.json.j2 @@ -1,14 +1,17 @@ { "hosts": [ - "docker-registry.kube-system.svc.{{ cluster_name }}", - "docker-registry.kube-system.svc", - "docker-registry.kube-system", + "container-registry.kube-system.svc.{{ cluster_name }}", + "container-registry.kube-system.svc", + "container-registry.kube-system", "localhost", "127.0.0.1" ], - "CN": "localhost", + "CN": "system:node:localhost", "key": { "algo": "rsa", "size": 2048 - } + }, + "names": [{ + "O": "system:nodes" + }] } diff --git a/roles/container_registry/vars/main.yml b/roles/container_registry/vars/main.yml index e78aed93..20c81a6d 100644 --- a/roles/container_registry/vars/main.yml +++ b/roles/container_registry/vars/main.yml @@ -1,5 +1,5 @@ ## -## Copyright (c) 2020-2021 Intel Corporation. +## Copyright (c) 2020-2022 Intel Corporation. ## ## Licensed under the Apache License, Version 2.0 (the "License"); ## you may not use this file except in compliance with the License. @@ -14,4 +14,10 @@ ## limitations under the License. ## --- -container_registry_path: /usr/src/container_registry +container_registry_path: "{{ project_root_dir }}/container_registry" + +install_dependencies: + Debian: + - apache2-utils + RedHat: + - httpd-tools diff --git a/roles/create_signed_k8s_certs/defaults/main.yml b/roles/create_signed_k8s_certs/defaults/main.yml index 81b6ea05..79064bbc 100644 --- a/roles/create_signed_k8s_certs/defaults/main.yml +++ b/roles/create_signed_k8s_certs/defaults/main.yml @@ -1,5 +1,5 @@ ## -## Copyright (c) 2020-2021 Intel Corporation. +## Copyright (c) 2020-2022 Intel Corporation. ## ## Licensed under the Apache License, Version 2.0 (the "License"); ## you may not use this file except in compliance with the License. diff --git a/roles/create_signed_k8s_certs/tasks/create_signed_k8s_certs.yml b/roles/create_signed_k8s_certs/tasks/create_signed_k8s_certs.yml index c684a46a..7c7db628 100644 --- a/roles/create_signed_k8s_certs/tasks/create_signed_k8s_certs.yml +++ b/roles/create_signed_k8s_certs/tasks/create_signed_k8s_certs.yml @@ -1,5 +1,5 @@ ## -## Copyright (c) 2020-2021 Intel Corporation. +## Copyright (c) 2020-2022 Intel Corporation. ## ## Licensed under the Apache License, Version 2.0 (the "License"); ## you may not use this file except in compliance with the License. @@ -63,12 +63,14 @@ mode: preserve - name: get GOPATH - command: go env GOPATH + command: /usr/local/go/bin/go env GOPATH register: gopath changed_when: false - name: generate key and CSR - shell: set -o pipefail && {{ gopath.stdout }}/bin/cfssl genkey {{ secret_name }}-csr.json | {{ gopath.stdout }}/bin/cfssljson -bare {{ key_pair_name }} + shell: >- + set -o pipefail && + {{ gopath.stdout }}/bin/cfssl genkey {{ secret_name }}-csr.json | {{ gopath.stdout }}/bin/cfssljson -bare {{ key_pair_name }} args: chdir: "{{ host_secrets_folder }}" executable: /bin/bash diff --git a/roles/create_signed_k8s_certs/tasks/main.yml b/roles/create_signed_k8s_certs/tasks/main.yml index 46bd42bc..68f030ad 100644 --- a/roles/create_signed_k8s_certs/tasks/main.yml +++ b/roles/create_signed_k8s_certs/tasks/main.yml @@ -1,5 +1,5 @@ ## -## Copyright (c) 2020-2021 Intel Corporation. +## Copyright (c) 2020-2022 Intel Corporation. ## ## Licensed under the Apache License, Version 2.0 (the "License"); ## you may not use this file except in compliance with the License. diff --git a/roles/create_signed_k8s_certs/templates/csr.json.j2 b/roles/create_signed_k8s_certs/templates/csr.json.j2 index dfd2eaed..cf7c65da 100644 --- a/roles/create_signed_k8s_certs/templates/csr.json.j2 +++ b/roles/create_signed_k8s_certs/templates/csr.json.j2 @@ -5,9 +5,12 @@ "{{ service_name }}.{{ k8s_namespace }}", "{{ service_name }}" ], - "CN": "{{ service_name }}.{{ k8s_namespace }}.svc", + "CN": "system:node:{{ service_name }}.{{ k8s_namespace }}.svc", "key": { "algo": "rsa", "size": 2048 - } + }, + "names": [{ + "O": "system:nodes" + }] } diff --git a/roles/create_signed_k8s_certs/templates/kube_csr.yml.j2 b/roles/create_signed_k8s_certs/templates/kube_csr.yml.j2 index 18a27a89..e23fadc9 100644 --- a/roles/create_signed_k8s_certs/templates/kube_csr.yml.j2 +++ b/roles/create_signed_k8s_certs/templates/kube_csr.yml.j2 @@ -1,9 +1,10 @@ -apiVersion: certificates.k8s.io/v1beta1 +apiVersion: certificates.k8s.io/v1 kind: CertificateSigningRequest metadata: name: {{ secret_name }}.{{ k8s_namespace }} namespace: {{ k8s_namespace }} spec: + signerName: kubernetes.io/kubelet-serving request: {{ csr }} usages: - digital signature diff --git a/roles/create_signed_k8s_certs/vars/main.yml b/roles/create_signed_k8s_certs/vars/main.yml index 9dde6519..0bd518d2 100644 --- a/roles/create_signed_k8s_certs/vars/main.yml +++ b/roles/create_signed_k8s_certs/vars/main.yml @@ -1,5 +1,5 @@ ## -## Copyright (c) 2020-2021 Intel Corporation. +## Copyright (c) 2020-2022 Intel Corporation. ## ## Licensed under the Apache License, Version 2.0 (the "License"); ## you may not use this file except in compliance with the License. diff --git a/roles/git2_install/tasks/main.yml b/roles/git2_install/tasks/main.yml deleted file mode 100644 index be49363b..00000000 --- a/roles/git2_install/tasks/main.yml +++ /dev/null @@ -1,56 +0,0 @@ -## -## Copyright (c) 2020-2021 Intel Corporation. -## -## Licensed under the Apache License, Version 2.0 (the "License"); -## you may not use this file except in compliance with the License. -## You may obtain a copy of the License at -## -## http://www.apache.org/licenses/LICENSE-2.0 -## -## Unless required by applicable law or agreed to in writing, software -## distributed under the License is distributed on an "AS IS" BASIS, -## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -## See the License for the specific language governing permissions and -## limitations under the License. -## ---- -# Role for updating git on RHEL7-based distros to resolve 'go get' issues on some projects - -# install deps -- name: install deps - package: - name: - - gettext-devel - - openssl-devel - - perl-CPAN - - perl-devel - - zlib-devel - - curl-devel - state: present - -- name: download git tarball - unarchive: - src: "{{ git_tarball_url }}" - dest: /usr/src/ - remote_src: yes - mode: 0755 - -- name: make configure - make: - target: configure - chdir: "/usr/src/git-{{ git_version }}" - -- name: configure - command: ./configure --prefix=/usr/local - args: - chdir: "/usr/src/git-{{ git_version }}" - changed_when: true - -- name: build - make: - chdir: "/usr/src/git-{{ git_version }}" - -- name: install - make: - target: install - chdir: "/usr/src/git-{{ git_version }}" diff --git a/roles/golang_install/tasks/main.yml b/roles/golang_install/tasks/main.yml deleted file mode 100644 index 870ff0ad..00000000 --- a/roles/golang_install/tasks/main.yml +++ /dev/null @@ -1,82 +0,0 @@ -## -## Copyright (c) 2020-2021 Intel Corporation. -## -## Licensed under the Apache License, Version 2.0 (the "License"); -## you may not use this file except in compliance with the License. -## You may obtain a copy of the License at -## -## http://www.apache.org/licenses/LICENSE-2.0 -## -## Unless required by applicable law or agreed to in writing, software -## distributed under the License is distributed on an "AS IS" BASIS, -## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -## See the License for the specific language governing permissions and -## limitations under the License. -## ---- -- name: uninstall existing golang - file: - path: "{{ item }}" - state: absent - with_items: - - "/usr/local/go" - - "$HOME/go" - -- name: download golang tarball - get_url: - url: "{{ golang_download_url }}" - checksum: "{{ golang_download_checksum }}" - dest: /usr/src - register: golang_download - until: golang_download is not failed - retries: 5 - -- name: untar downloaded golang tarball - unarchive: - src: "{{ golang_download.dest }}" - dest: /usr/local - copy: no - mode: 0755 - -- name: set GOPATH env and add golang bin to PATH for all users - copy: - content: | - export GOROOT=/usr/local/go - export GOPATH=$HOME/go - export PATH=$GOPATH/bin:$GOROOT/bin:$PATH - dest: /etc/profile.d/golang.sh - mode: 0755 - -- name: install godoc if go version >= 1.13 - shell: source /etc/profile.d/golang.sh && go get golang.org/x/tools/cmd/godoc - args: - executable: /bin/bash - register: godoc_download - until: godoc_download is not failed - retries: 5 - when: golang_version is version_compare('1.13', '>=') - -- name: create symlinks to golang binaries - file: - state: link - src: /usr/local/go/bin/{{ item }} - dest: /usr/bin/{{ item }} - mode: 0755 - with_items: - - go - - gofmt - -- name: get GOPATH - command: go env GOPATH - register: gopath - changed_when: false - -- name: install cfssl - command: go get github.com/cloudflare/cfssl/cmd/cfssl - when: - - inventory_hostname == groups['kube_control_plane'][0] - -- name: install cfssljson - command: go get github.com/cloudflare/cfssl/cmd/cfssljson - when: - - inventory_hostname == groups['kube_control_plane'][0] diff --git a/roles/gpu_dp_install/defaults/main.yml b/roles/gpu_dp_install/defaults/main.yml index 94545f32..fd6fa5f7 100644 --- a/roles/gpu_dp_install/defaults/main.yml +++ b/roles/gpu_dp_install/defaults/main.yml @@ -1,5 +1,5 @@ ## -## Copyright (c) 2020-2021 Intel Corporation. +## Copyright (c) 2020-2022 Intel Corporation. ## ## Licensed under the Apache License, Version 2.0 (the "License"); ## you may not use this file except in compliance with the License. @@ -15,9 +15,12 @@ ## --- intel_gpu_dp_git_url: "https://github.com/intel/intel-device-plugins-for-kubernetes.git" -intel_gpu_dp_git_ref: "v0.21.0" -intel_gpu_dp_version: "0.21.0" -intel_gpu_dp_dir: "/usr/src/intel-device-plugins-for-kubernetes-gpu" +intel_gpu_dp_git_ref: "v0.23.0" +intel_gpu_dp_version: "0.23.0" +intel_gpu_dp_dir: "{{ (project_root_dir, 'intel-gpu-dp') | path_join }}" -gpu_shared_devices: 10 -gpu_dp_build_image_locally: true +gpu_dp_shared_devices: 10 +gpu_dp_verbosity: 4 +gpu_dp_monitor_resources: false +gpu_dp_fractional_manager: false +gpu_dp_build_image_locally: false diff --git a/roles/gpu_dp_install/tasks/main.yml b/roles/gpu_dp_install/tasks/main.yml index 19bf41ae..7cd2007c 100644 --- a/roles/gpu_dp_install/tasks/main.yml +++ b/roles/gpu_dp_install/tasks/main.yml @@ -1,5 +1,5 @@ ## -## Copyright (c) 2020-2021 Intel Corporation. +## Copyright (c) 2020-2022 Intel Corporation. ## ## Licensed under the Apache License, Version 2.0 (the "License"); ## you may not use this file except in compliance with the License. @@ -78,6 +78,7 @@ - gpu_dp_build_image_locally - '"docker" not in container_runtime' +# start deployment of GPU DP - name: prepare and deploy GPU Device Plugin block: - name: set values @@ -87,15 +88,21 @@ gpu_dp_version: "{{ intel_gpu_dp_version }}" when: gpu_dp_build_image_locally + - name: create directory for GPU DP files + file: + state: directory + path: "{{ project_root_dir }}/gpu_dp/" + mode: 0644 + - name: populate Intel GPU Device Plugin yaml file and push to controller node template: src: "intel-gpu-plugin.yml.j2" - dest: "/usr/src/intel-gpu-plugin.yml" + dest: "{{ project_root_dir }}/gpu_dp/intel-gpu-plugin.yml" force: yes mode: preserve - name: deploy Intel GPU Device Plugin with the Intel Device Plugin Operator k8s: state: present - src: "/usr/src/intel-gpu-plugin.yml" + src: "{{ project_root_dir }}/gpu_dp/intel-gpu-plugin.yml" when: inventory_hostname == groups['kube_control_plane'][0] diff --git a/roles/gpu_dp_install/templates/intel-gpu-plugin.yml.j2 b/roles/gpu_dp_install/templates/intel-gpu-plugin.yml.j2 index c23ac2f7..19e89945 100644 --- a/roles/gpu_dp_install/templates/intel-gpu-plugin.yml.j2 +++ b/roles/gpu_dp_install/templates/intel-gpu-plugin.yml.j2 @@ -4,9 +4,12 @@ kind: GpuDevicePlugin metadata: name: intel-gpu-plugin spec: - image: {{ gpu_dp_image | default("docker.io/intel/intel-gpu-plugin") }}:{{ gpu_dp_version | default("0.21.0") }} - initImage: {{ gpu_dp_init_image | default("docker.io/intel/intel-gpu-initcontainer") }}:{{ gpu_dp_version | default("0.21.0") }} - sharedDevNum: {{ gpu_shared_devices }} - logLevel: 4 + image: {{ gpu_dp_image | default("docker.io/intel/intel-gpu-plugin") }}:{{ gpu_dp_version | default("0.23.0") }} + initImage: {{ gpu_dp_init_image | default("docker.io/intel/intel-gpu-initcontainer") }}:{{ gpu_dp_version | default("0.23.0") }} + sharedDevNum: {{ gpu_dp_shared_devices | default(10) }} + logLevel: {{ gpu_dp_verbosity | default(4) }} + enableMonitoring: {{ gpu_dp_monitor_resources | default(false) }} + resourceManager: {{ gpu_dp_fractional_manager | default(false) }} + preferredAllocationPolicy: {{ gpu_dp_prefered_allocation | default('none') }} nodeSelector: feature.node.kubernetes.io/pci-0300_8086.present: 'true' diff --git a/roles/gpu_dp_install/vars/main.yml b/roles/gpu_dp_install/vars/main.yml index 9f60069c..170d32e5 100644 --- a/roles/gpu_dp_install/vars/main.yml +++ b/roles/gpu_dp_install/vars/main.yml @@ -1,5 +1,5 @@ ## -## Copyright (c) 2020-2021 Intel Corporation. +## Copyright (c) 2020-2022 Intel Corporation. ## ## Licensed under the Apache License, Version 2.0 (the "License"); ## you may not use this file except in compliance with the License. diff --git a/roles/install_ddp_pkgs/vars/main.yml b/roles/install_ddp_pkgs/defaults/main.yml similarity index 92% rename from roles/install_ddp_pkgs/vars/main.yml rename to roles/install_ddp_pkgs/defaults/main.yml index 1cc1c475..506f0fc0 100644 --- a/roles/install_ddp_pkgs/vars/main.yml +++ b/roles/install_ddp_pkgs/defaults/main.yml @@ -1,5 +1,5 @@ ## -## Copyright (c) 2020-2021 Intel Corporation. +## Copyright (c) 2020-2022 Intel Corporation. ## ## Licensed under the Apache License, Version 2.0 (the "License"); ## you may not use this file except in compliance with the License. @@ -33,4 +33,4 @@ ddp_pkgs: - "https://downloadmirror.intel.com/30467/eng/800%20series%20comms%20binary%20package%201.3.28.0.zip" - "https://downloadmirror.intel.com/30590/eng/800%20series%20comms%20binary%20package%201.3.30.0.zip" - "https://downloadmirror.intel.com/29889/eng/800%20series%20comms%20binary%20package%201.3.30.0_rev1.1.zip" - + - "https://downloadmirror.intel.com/713853/800%20Series%20DDP%20Comms%20Package%201.3.31.0.zip" diff --git a/roles/install_ddp_pkgs/tasks/install_a_pkg.yml b/roles/install_ddp_pkgs/tasks/install_a_pkg.yml index b940384a..636e17c7 100644 --- a/roles/install_ddp_pkgs/tasks/install_a_pkg.yml +++ b/roles/install_ddp_pkgs/tasks/install_a_pkg.yml @@ -1,5 +1,5 @@ ## -## Copyright (c) 2020-2021 Intel Corporation. +## Copyright (c) 2020-2022 Intel Corporation. ## ## Licensed under the Apache License, Version 2.0 (the "License"); ## you may not use this file except in compliance with the License. @@ -53,6 +53,14 @@ mode: 0644 when: '"1.3.30.0" in pkgurl' +- name: unarchive DDP package subfolder excluding from list of URLs + unarchive: + src: "{{ temp_ddp_path }}/ice_comms-1.3.31.0.zip" + dest: "{{ temp_ddp_path }}" + remote_src: yes + mode: 0644 + when: '"1.3.31.0" in pkgurl' + - name: find PKG files find: paths: "{{ temp_ddp_path }}" @@ -63,8 +71,8 @@ - name: copy DDP packages into installation directory copy: - src: "{{ pkgfile.path }}" - dest: "{{ install_dir }}" + src: "{{ pkgfile.path }}" + dest: "{{ install_dir }}" remote_src: yes mode: 0755 loop: "{{ pkgfiles.files }}" diff --git a/roles/install_ddp_pkgs/tasks/install_pkgs.yml b/roles/install_ddp_pkgs/tasks/install_pkgs.yml index e8bdc41e..df16f573 100644 --- a/roles/install_ddp_pkgs/tasks/install_pkgs.yml +++ b/roles/install_ddp_pkgs/tasks/install_pkgs.yml @@ -1,5 +1,5 @@ ## -## Copyright (c) 2020-2021 Intel Corporation. +## Copyright (c) 2020-2022 Intel Corporation. ## ## Licensed under the Apache License, Version 2.0 (the "License"); ## you may not use this file except in compliance with the License. diff --git a/roles/install_ddp_pkgs/tasks/main.yml b/roles/install_ddp_pkgs/tasks/main.yml index 1c3fae0f..08053a51 100644 --- a/roles/install_ddp_pkgs/tasks/main.yml +++ b/roles/install_ddp_pkgs/tasks/main.yml @@ -1,5 +1,5 @@ ## -## Copyright (c) 2020-2021 Intel Corporation. +## Copyright (c) 2020-2022 Intel Corporation. ## ## Licensed under the Apache License, Version 2.0 (the "License"); ## you may not use this file except in compliance with the License. @@ -68,4 +68,3 @@ file: path: "{{ tempdir_ddp.path }}" state: absent - diff --git a/roles/install_dependencies/tasks/main.yml b/roles/install_dependencies/tasks/main.yml index 982e31db..564a3890 100644 --- a/roles/install_dependencies/tasks/main.yml +++ b/roles/install_dependencies/tasks/main.yml @@ -1,5 +1,5 @@ ## -## Copyright (c) 2020-2021 Intel Corporation. +## Copyright (c) 2020-2022 Intel Corporation. ## ## Licensed under the Apache License, Version 2.0 (the "License"); ## you may not use this file except in compliance with the License. diff --git a/roles/install_dpdk/defaults/main.yml b/roles/install_dpdk/defaults/main.yml new file mode 100644 index 00000000..f702eb4e --- /dev/null +++ b/roles/install_dpdk/defaults/main.yml @@ -0,0 +1,22 @@ +## +## Copyright (c) 2020-2022 Intel Corporation. +## +## Licensed under the Apache License, Version 2.0 (the "License"); +## you may not use this file except in compliance with the License. +## You may obtain a copy of the License at +## +## http://www.apache.org/licenses/LICENSE-2.0 +## +## Unless required by applicable law or agreed to in writing, software +## distributed under the License is distributed on an "AS IS" BASIS, +## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +## See the License for the specific language governing permissions and +## limitations under the License. +## +--- +dpdk_url: "https://fast.dpdk.org/rel/dpdk-{{ dpdk_version }}.tar.xz" +dpdk_target: x86_64-native-linuxapp-gcc +dpdk_kmods_repo: "http://dpdk.org/git/dpdk-kmods" +dpdk_kmods_dest: "{{ (project_root_dir, '/dpdk-kmods') | path_join }}" + +dpdk_local_patches_strip: 0 diff --git a/roles/install_dpdk/tasks/install_dpdk_make.yml b/roles/install_dpdk/tasks/install_dpdk_make.yml index 38a6b72f..d0fa7e14 100644 --- a/roles/install_dpdk/tasks/install_dpdk_make.yml +++ b/roles/install_dpdk/tasks/install_dpdk_make.yml @@ -1,5 +1,5 @@ ## -## Copyright (c) 2020-2021 Intel Corporation. +## Copyright (c) 2020-2022 Intel Corporation. ## ## Licensed under the Apache License, Version 2.0 (the "License"); ## you may not use this file except in compliance with the License. diff --git a/roles/install_dpdk/tasks/install_dpdk_meson.yml b/roles/install_dpdk/tasks/install_dpdk_meson.yml index 1366d1b3..46e233d1 100644 --- a/roles/install_dpdk/tasks/install_dpdk_meson.yml +++ b/roles/install_dpdk/tasks/install_dpdk_meson.yml @@ -1,5 +1,5 @@ ## -## Copyright (c) 2020-2021 Intel Corporation. +## Copyright (c) 2020-2022 Intel Corporation. ## ## Licensed under the Apache License, Version 2.0 (the "License"); ## you may not use this file except in compliance with the License. @@ -14,18 +14,6 @@ ## limitations under the License. ## --- -- name: install Python3 from epel-release - package: - name: - - python36 - - python36-devel - - libselinux-python - state: present - enablerepo: "epel" - when: - - ansible_distribution == 'RedHat' or ansible_distribution == 'CentOS' - - ansible_distribution_version < '8' - - name: install dpdk-devel required for libraries enablement in RHEL / CentOS >= 8.2 dnf: name: dpdk-devel @@ -37,7 +25,7 @@ pip: name: - selinux - - meson>=0.53.2 + - meson>=0.53.2,<0.60.0 - ninja>=1.10.0 - pyelftools>=0.26 state: present diff --git a/roles/install_dpdk/tasks/main.yml b/roles/install_dpdk/tasks/main.yml index f46dec97..6fdb775f 100644 --- a/roles/install_dpdk/tasks/main.yml +++ b/roles/install_dpdk/tasks/main.yml @@ -1,5 +1,5 @@ ## -## Copyright (c) 2020-2021 Intel Corporation. +## Copyright (c) 2020-2022 Intel Corporation. ## ## Licensed under the Apache License, Version 2.0 (the "License"); ## you may not use this file except in compliance with the License. @@ -27,7 +27,7 @@ - name: download DPDK unarchive: src: "{{ dpdk_url }}" - dest: "/usr/src" + dest: "{{ project_root_dir }}" remote_src: yes list_files: yes mode: 0755 @@ -38,33 +38,48 @@ - name: set local dpdk directory path set_fact: - dpdk_dir: "{{ dpdk_download.dest }}/{{ dpdk_download.files[0] }}" + dpdk_dir: "{{ (dpdk_download.dest, dpdk_download.files[0]) | path_join }}" dpdk_base_name: "{{ dpdk_download.files[0] | regex_replace('\\/$', '') }}" -- name: "Apply DPDK patches if provided" +- name: Apply DPDK patches if provided block: - name: check if local DPDK patches dir exists stat: path: "{{ dpdk_local_patches_dir }}" - delegate_to: "127.0.0.1" + delegate_to: "localhost" register: dpdk_patches_dir_exists + become: false - - name: find DPDK patches available + - name: find DPDK patches find: - paths: "{{ dpdk_local_patches_dir }}/{{ dpdk_base_name }}" + paths: "{{ (dpdk_local_patches_dir, dpdk_base_name) | path_join }}" recurse: yes - when: dpdk_patches_dir_exists.stat.exists|bool and dpdk_patches_dir_exists.stat.isdir|bool + when: + - dpdk_patches_dir_exists.stat.exists | bool + - dpdk_patches_dir_exists.stat.isdir | bool + delegate_to: "localhost" register: patches_found + become: false - name: apply DPDK patches patch: remote_src: no src: "{{ item.path }}" basedir: "{{ dpdk_dir }}" - strip: 2 + strip: "{{ dpdk_local_patches_strip }}" state: present with_items: "{{ patches_found.files }}" - when: patches_found.matched > 0 + when: + - not patches_found.skipped | default(false) + - patches_found.matched > 0 + + - name: warn user if patches were not found + fail: + msg: "[WARNING] DPDK patches were not found, no patches been applied." + when: + - patches_found.skipped | default(false) or patches_found.matched == 0 + ignore_errors: true + when: dpdk_local_patches_dir is defined - name: include install_dpdk task for DPDK <= 20.08 versions @@ -73,4 +88,4 @@ - name: include install_dpdk task for DPDK >= 20.08 versions include: "install_dpdk_meson.yml" - when: dpdk_version is version("20.08", ">=") \ No newline at end of file + when: dpdk_version is version("20.08", ">=") diff --git a/roles/install_dpdk/vars/main.yml b/roles/install_dpdk/vars/main.yml index 033427b6..013d1f44 100644 --- a/roles/install_dpdk/vars/main.yml +++ b/roles/install_dpdk/vars/main.yml @@ -1,5 +1,5 @@ ## -## Copyright (c) 2020-2021 Intel Corporation. +## Copyright (c) 2020-2022 Intel Corporation. ## ## Licensed under the Apache License, Version 2.0 (the "License"); ## you may not use this file except in compliance with the License. @@ -14,11 +14,6 @@ ## limitations under the License. ## --- -dpdk_url: "http://fast.dpdk.org/rel/dpdk-{{ dpdk_version }}.tar.xz" -dpdk_target: x86_64-native-linuxapp-gcc -dpdk_kmods_repo: "http://dpdk.org/git/dpdk-kmods" -dpdk_kmods_dest: "/usr/src/dpdk-kmods" - install_dependencies: Debian: - build-essential diff --git a/roles/intel_dp_operator/defaults/main.yml b/roles/intel_dp_operator/defaults/main.yml index d42b67cb..e50a5c18 100644 --- a/roles/intel_dp_operator/defaults/main.yml +++ b/roles/intel_dp_operator/defaults/main.yml @@ -1,5 +1,5 @@ ## -## Copyright (c) 2020-2021 Intel Corporation. +## Copyright (c) 2020-2022 Intel Corporation. ## ## Licensed under the Apache License, Version 2.0 (the "License"); ## you may not use this file except in compliance with the License. @@ -15,7 +15,7 @@ ## --- intel_dp_operator_git_url: "https://github.com/intel/intel-device-plugins-for-kubernetes.git" -intel_dp_operator_git_ref: "v0.21.0" -intel_dp_operator_version: "0.21.0" -intel_dp_operator_dir: "/usr/src/intel-device-plugins-for-kubernetes-operator" +intel_dp_operator_git_ref: "v0.23.0" +intel_dp_operator_version: "0.23.0" +intel_dp_operator_dir: "{{ (project_root_dir, 'intel-dp-operator') | path_join }}" intel_dp_namespace: kube-system diff --git a/roles/intel_dp_operator/tasks/main.yml b/roles/intel_dp_operator/tasks/main.yml index 19bd1cf7..7a212759 100644 --- a/roles/intel_dp_operator/tasks/main.yml +++ b/roles/intel_dp_operator/tasks/main.yml @@ -1,5 +1,5 @@ ## -## Copyright (c) 2020-2021 Intel Corporation. +## Copyright (c) 2020-2022 Intel Corporation. ## ## Licensed under the Apache License, Version 2.0 (the "License"); ## you may not use this file except in compliance with the License. diff --git a/roles/intel_dp_operator/vars/main.yml b/roles/intel_dp_operator/vars/main.yml index 9f60069c..170d32e5 100644 --- a/roles/intel_dp_operator/vars/main.yml +++ b/roles/intel_dp_operator/vars/main.yml @@ -1,5 +1,5 @@ ## -## Copyright (c) 2020-2021 Intel Corporation. +## Copyright (c) 2020-2022 Intel Corporation. ## ## Licensed under the Apache License, Version 2.0 (the "License"); ## you may not use this file except in compliance with the License. diff --git a/roles/intel_power_manager/defaults/main.yml b/roles/intel_power_manager/defaults/main.yml new file mode 100644 index 00000000..be79d98e --- /dev/null +++ b/roles/intel_power_manager/defaults/main.yml @@ -0,0 +1,25 @@ +## +## Copyright (c) 2020-2022 Intel Corporation. +## +## Licensed under the Apache License, Version 2.0 (the "License"); +## you may not use this file except in compliance with the License. +## You may obtain a copy of the License at +## +## http://www.apache.org/licenses/LICENSE-2.0 +## +## Unless required by applicable law or agreed to in writing, software +## distributed under the License is distributed on an "AS IS" BASIS, +## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +## See the License for the specific language governing permissions and +## limitations under the License. +## +--- +intel_power_manager_git_url: "https://github.com/intel/kubernetes-power-manager.git" +intel_power_manager_git_ref: "v1.0.2" # project is consistent with git ref and image version +intel_power_manager_dir: "{{ (project_root_dir, 'intel-power-manager') | path_join }}" + +intel_appqos_git_url: "https://github.com/intel/intel-cmt-cat.git" +intel_appqos_git_ref: "v4.3.0" +intel_appqos_version: "v4.3.0" +intel_appqos_dir: "{{ (project_root_dir, 'intel-appqos') | path_join }}" +intel_appqos_cert_dir: "/etc/certs/public" diff --git a/roles/intel_power_manager/files/appqos.conf b/roles/intel_power_manager/files/appqos.conf new file mode 100644 index 00000000..0aea96db --- /dev/null +++ b/roles/intel_power_manager/files/appqos.conf @@ -0,0 +1,8 @@ +{ + "apps": [], + "sstbf": { + "configured": false + }, + "pools": [], + "power_profiles_expert_mode": true +} diff --git a/roles/intel_power_manager/files/rbac.patch b/roles/intel_power_manager/files/rbac.patch new file mode 100644 index 00000000..fd2446fc --- /dev/null +++ b/roles/intel_power_manager/files/rbac.patch @@ -0,0 +1,77 @@ +diff --git a/config/rbac/rbac.yaml b/config/rbac/rbac.yaml +index 4b2700a..a551cee 100644 +--- a/config/rbac/rbac.yaml ++++ b/config/rbac/rbac.yaml +@@ -50,7 +50,11 @@ rules: + - apiGroups: ["", "power.intel.com", "apps"] + resources: ["nodes", "nodes/status", "configmaps", "powerconfigs", "powerconfigs/status", "powerprofiles", "powerprofiles/status", "powerworkloads", "powerworkloads/status", "powernodes", "powernodes/status", "events", "daemonsets"] + verbs: ["*"] +- ++- apiGroups: ['policy'] ++ resources: ['podsecuritypolicies'] ++ verbs: ['use'] ++ resourceNames: ++ - power-manager-psp + --- + + apiVersion: rbac.authorization.k8s.io/v1 +@@ -76,7 +80,11 @@ rules: + - apiGroups: ["", "power.intel.com"] + resources: ["nodes", "nodes/status", "pods", "pods/status", "powerprofiles", "powerprofiles/status", "powerworkloads", "powerworkloads/status", "powernodes", "powernodes/status"] + verbs: ["*"] +- ++- apiGroups: ['policy'] ++ resources: ['podsecuritypolicies'] ++ verbs: ['use'] ++ resourceNames: ++ - node-agent-psp + --- + + apiVersion: rbac.authorization.k8s.io/v1 +@@ -93,3 +101,46 @@ roleRef: + apiGroup: rbac.authorization.k8s.io + + --- ++apiVersion: policy/v1beta1 ++kind: PodSecurityPolicy ++metadata: ++ name: power-manager-psp ++spec: ++ privileged: true ++ allowPrivilegeEscalation: true ++ allowedCapabilities: ++ - '*' ++ allowedUnsafeSysctls: ++ - '*' ++ fsGroup: ++ rule: RunAsAny ++ runAsUser: ++ rule: RunAsAny ++ seLinux: ++ rule: RunAsAny ++ supplementalGroups: ++ rule: RunAsAny ++ volumes: ++ - '*' ++--- ++apiVersion: policy/v1beta1 ++kind: PodSecurityPolicy ++metadata: ++ name: node-agent-psp ++spec: ++ privileged: true ++ allowPrivilegeEscalation: true ++ allowedCapabilities: ++ - '*' ++ allowedUnsafeSysctls: ++ - '*' ++ fsGroup: ++ rule: RunAsAny ++ runAsUser: ++ rule: RunAsAny ++ seLinux: ++ rule: RunAsAny ++ supplementalGroups: ++ rule: RunAsAny ++ volumes: ++ - '*' diff --git a/roles/intel_power_manager/tasks/app_qos.yml b/roles/intel_power_manager/tasks/app_qos.yml new file mode 100644 index 00000000..9859c8c6 --- /dev/null +++ b/roles/intel_power_manager/tasks/app_qos.yml @@ -0,0 +1,103 @@ +## +## Copyright (c) 2020-2022 Intel Corporation. +## +## Licensed under the Apache License, Version 2.0 (the "License"); +## you may not use this file except in compliance with the License. +## You may obtain a copy of the License at +## +## http://www.apache.org/licenses/LICENSE-2.0 +## +## Unless required by applicable law or agreed to in writing, software +## distributed under the License is distributed on an "AS IS" BASIS, +## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +## See the License for the specific language governing permissions and +## limitations under the License. +## +--- +- name: clone Intel CMT CAT repository + git: + repo: "{{ intel_appqos_git_url }}" + version: "{{ intel_appqos_git_ref }}" + dest: "{{ intel_appqos_dir }}" + force: yes + when: inventory_hostname in groups['kube_node'] + +# NOTE(pklimowx): since AppQoS image is not available on docker hub +# and public images of the Power Operator use `appqos:latest` image, +# we have to build AppQoS image on each node, and push it to localregistry +# only once. +# +# docker runtime is in use +- name: prepare image for Application Quality of Service + block: + # NOTE(pklimowx): w/a for error "ModuleNotFoundError: No module named 'pqos'" + - name: copy pqos module to appqos working dir + copy: + remote_src: yes + src: "{{ (intel_appqos_dir, 'lib', 'python', 'pqos') | path_join }}" + dest: "{{ (intel_appqos_dir, 'appqos') | path_join }}" + mode: 0755 + + - name: build image of App QoS + command: docker build --no-cache -t appqos -f Dockerfile ../../ + args: + chdir: "{{ (intel_appqos_dir, 'appqos', 'docker') | path_join }}" + + - name: tag App QoS image + command: docker tag appqos:latest {{ registry_local_address }}/appqos:{{ intel_appqos_version }} + when: inventory_hostname == groups['kube_node'][0] + + - name: push App QoS image to local registry + command: docker push {{ registry_local_address }}/appqos:{{ intel_appqos_version }} + when: inventory_hostname == groups['kube_node'][0] + when: + - container_runtime == "docker" + - inventory_hostname in groups['kube_node'] + +# crio/containerd runtime is in use +- name: prepare image for Application Quality of Service + block: + - name: build and tag App QoS image + command: buildah bud -f Dockerfile -t {{ registry_local_address }}/appqos:{{ intel_appqos_version }} + args: + chdir: "{{ (intel_appqos_dir, 'appqos', 'docker') | path_join }}" + + - name: push App QoS image to local registry + command: buildah push {{ registry_local_address }}/appqos:{{ intel_appqos_version }} + changed_when: true + when: inventory_hostname == groups['kube_node'][0] + when: + - container_runtime in ["crio", "containerd"] + - inventory_hostname in groups['kube_node'] + +- name: generate App QoS certificates + block: + - name: create directory for App QoS certs + file: + state: directory + path: "{{ intel_appqos_cert_dir }}" + owner: "{{ ansible_user }}" + group: "{{ ansible_user }}" + mode: 0755 + recurse: yes + + - name: generate certificates + command: "{{ item }}" + args: + chdir: "{{ intel_appqos_cert_dir }}" + with_items: + - openssl req -nodes -x509 -newkey rsa:4096 -keyout ca.key -out ca.crt -days 365 -subj "/O=AppQoS/OU=root/CN=localhost" + - openssl req -nodes -newkey rsa:3072 -keyout appqos.key -out appqos.csr -subj "/O=AppQoS/OU=AppQoS Server/CN=localhost" + - openssl x509 -req -in appqos.csr -CA ca.crt -CAkey ca.key -CAcreateserial -out appqos.crt + + - name: copy example App QoS config to /etc/certs/public + copy: + src: appqos.conf + dest: "{{ intel_appqos_cert_dir }}" + owner: "{{ ansible_user }}" + group: "{{ ansible_user }}" + mode: 0644 + +- name: set facts for Intel App QoS templates + set_fact: + app_qos_image: "{{ registry_local_address }}/appqos:{{ intel_appqos_version }}" diff --git a/roles/intel_power_manager/tasks/deploy_sample_pods.yml b/roles/intel_power_manager/tasks/deploy_sample_pods.yml new file mode 100644 index 00000000..8ea25eed --- /dev/null +++ b/roles/intel_power_manager/tasks/deploy_sample_pods.yml @@ -0,0 +1,44 @@ +## +## Copyright (c) 2020-2022 Intel Corporation. +## +## Licensed under the Apache License, Version 2.0 (the "License"); +## you may not use this file except in compliance with the License. +## You may obtain a copy of the License at +## +## http://www.apache.org/licenses/LICENSE-2.0 +## +## Unless required by applicable law or agreed to in writing, software +## distributed under the License is distributed on an "AS IS" BASIS, +## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +## See the License for the specific language governing permissions and +## limitations under the License. +## +--- +- name: make sure directory for sample power pods exists + file: + state: directory + mode: 0755 + path: "{{ (intel_power_manager_dir, 'sample_power_pods') | path_join }}" + +# NOTE(pklimowx): +# this task will generate yaml files for each PowerProfile from +# intel_power_manager.power_profiles list for each node in +# intel_power_manager.power_nodes list +- name: generate templates for each available profile for the node + include_tasks: power_pod_template_helper.yml + loop: "{{ intel_power_manager.power_profiles }}" + loop_control: + loop_var: profile_name + +- name: get yaml files to deploy + find: + path: "{{ (intel_power_manager_dir, 'sample_power_pods') | path_join }}" + file_type: file + patterns: "*.yml" + register: sample_pod_files + +- name: deploy sample power pods + k8s: + state: present + src: "{{ item.path }}" + loop: "{{ sample_pod_files.files }}" diff --git a/roles/intel_power_manager/tasks/deploy_shared_resources.yml b/roles/intel_power_manager/tasks/deploy_shared_resources.yml new file mode 100644 index 00000000..1233e422 --- /dev/null +++ b/roles/intel_power_manager/tasks/deploy_shared_resources.yml @@ -0,0 +1,69 @@ +## +## Copyright (c) 2020-2022 Intel Corporation. +## +## Licensed under the Apache License, Version 2.0 (the "License"); +## you may not use this file except in compliance with the License. +## You may obtain a copy of the License at +## +## http://www.apache.org/licenses/LICENSE-2.0 +## +## Unless required by applicable law or agreed to in writing, software +## distributed under the License is distributed on an "AS IS" BASIS, +## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +## See the License for the specific language governing permissions and +## limitations under the License. +## +--- +- name: prepare and deploy node-specific Shared Power Profiles + block: + - name: make sure that direcotry for node-specific Shared Power Profiles exists + file: + state: directory + path: "{{ (intel_power_manager_dir, 'local_shared_power_profiles') | path_join }}" + mode: 0755 + + - name: obtain variables needed for deployment of node-specific Shared Power Profile + set_fact: + node_name: "{{ node_name }}" + max_frequency: "{{ hostvars[node_name]['local_shared_profile']['node_max_shared_frequency'] }}" + min_frequency: "{{ hostvars[node_name]['local_shared_profile']['node_min_shared_frequency'] }}" + + - name: populate template for node-specific Shared Power Profile + template: + src: local_shared_profile.yml.j2 + dest: "{{ (intel_power_manager_dir, 'local_shared_power_profiles', node_name + '_local_shared_profile.yml') | path_join }}" + mode: preserve + force: yes + + - name: deploy node-specific Shared Power Profile + k8s: + state: present + src: "{{ (intel_power_manager_dir, 'local_shared_power_profiles', node_name + '_local_shared_profile.yml') | path_join }}" + when: hostvars[node_name]['local_shared_profile']['enabled'] + +- name: prepare and deploy node-specific Shared Power Workload + block: + - name: make sure that direcotry for node-specific Shared Power Workloads exists + file: + state: directory + path: "{{ (intel_power_manager_dir, 'shared_power_workloads') | path_join }}" + mode: 0755 + + - name: obtain variables needed for deployment of Shared Power Workloads + set_fact: + node_name: "{{ node_name }}" + reserved_cpus: "{{ hostvars[node_name]['shared_workload']['reserved_cpus'] | list }}" + shared_workload_type: "{{ hostvars[node_name]['shared_workload']['shared_workload_type'] }}" + + - name: populate template for Shared Power Workload + template: + src: shared_workload.yml.j2 + dest: "{{ (intel_power_manager_dir, 'shared_power_workloads', node_name + '_shared_workload.yml') | path_join }}" + mode: preserve + force: yes + + - name: deploy node-specific Shared Power Profile + k8s: + state: present + src: "{{ (intel_power_manager_dir, 'shared_power_workloads', node_name + '_shared_workload.yml') | path_join }}" + when: hostvars[node_name]['shared_workload']['enabled'] diff --git a/roles/intel_power_manager/tasks/main.yml b/roles/intel_power_manager/tasks/main.yml new file mode 100644 index 00000000..3f6478e6 --- /dev/null +++ b/roles/intel_power_manager/tasks/main.yml @@ -0,0 +1,68 @@ +## +## Copyright (c) 2020-2022 Intel Corporation. +## +## Licensed under the Apache License, Version 2.0 (the "License"); +## you may not use this file except in compliance with the License. +## You may obtain a copy of the License at +## +## http://www.apache.org/licenses/LICENSE-2.0 +## +## Unless required by applicable law or agreed to in writing, software +## distributed under the License is distributed on an "AS IS" BASIS, +## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +## See the License for the specific language governing permissions and +## limitations under the License. +## +--- +- name: install dependencies for Power Operator + include_role: + name: install_dependencies + +- name: add labels for Power Nodes + command: kubectl label nodes {{ item }} intel.power.node=true --overwrite + loop: "{{ intel_power_manager.power_nodes }}" + when: inventory_hostname == groups['kube_control_plane'][0] + +- name: prepare App QoS + include_tasks: app_qos.yml + +- name: prepare Intel Kubernetes Power Manager + include_tasks: power_manager.yml + +- name: wait for Power Operator and Power Node Agent to be up and running + pause: + prompt: "Waiting for Power Operator pods to be up and running..." + minutes: 1 + +- name: deploy example power pods + include_tasks: deploy_sample_pods.yml + when: + - intel_power_manager.deploy_example_pods + - inventory_hostname == groups['kube_control_plane'][0] + +# The Shared Profiles and Workloads deployment starts here +- name: prepare and deploy Global Shared Power Profile + block: + - name: populate Global Shared Profile template to the controller node + template: + src: global_shared_profile.yml.j2 + dest: "{{ (intel_power_manager_dir, 'global_shared_profile.yml') | path_join }}" + force: yes + mode: preserve + + - name: deploy Global Shared Profile + k8s: + src: "{{ (intel_power_manager_dir, 'global_shared_profile.yml') | path_join }}" + state: present + when: + - intel_power_manager.global_shared_profile_enabled + - inventory_hostname == groups['kube_control_plane'][0] + +- name: prepare and deploy Local Shared Power Profiles/Workloads + block: + - name: deploy shared resources + include_tasks: deploy_shared_resources.yml + loop: "{{ groups['kube_node'] }}" + loop_control: + loop_var: node_name + when: inventory_hostname == groups['kube_control_plane'][0] diff --git a/roles/intel_power_manager/tasks/power_manager.yml b/roles/intel_power_manager/tasks/power_manager.yml new file mode 100644 index 00000000..53d567de --- /dev/null +++ b/roles/intel_power_manager/tasks/power_manager.yml @@ -0,0 +1,150 @@ +## +## Copyright (c) 2020-2022 Intel Corporation. +## +## Licensed under the Apache License, Version 2.0 (the "License"); +## you may not use this file except in compliance with the License. +## You may obtain a copy of the License at +## +## http://www.apache.org/licenses/LICENSE-2.0 +## +## Unless required by applicable law or agreed to in writing, software +## distributed under the License is distributed on an "AS IS" BASIS, +## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +## See the License for the specific language governing permissions and +## limitations under the License. +## +--- +# NOTE(pklimowx): repo must be cloned to the controller and, if we want to build +# images locally, to the first node as well. +- name: clone Intel Kubernetes Power Manager repository + git: + repo: "{{ intel_power_manager_git_url }}" + version: "{{ intel_power_manager_git_ref }}" + dest: "{{ intel_power_manager_dir }}" + force: yes + when: + - inventory_hostname == groups['kube_control_plane'][0] or + (inventory_hostname == groups['kube_node'][0] and intel_power_manager.build_image_locally | default(false) | bool) + +- name: set facts for Intel Kubernetes Power Manager templates + set_fact: + power_operator_image: "{{ registry_local_address }}/intel-power-operator" + power_operator_image_version: "{{ intel_power_manager_git_ref }}" + node_agent_image: "{{ registry_local_address }}/intel-power-node-agent" + node_agent_image_version: "{{ intel_power_manager_git_ref }}" + when: + - intel_power_manager.build_image_locally | default(false) | bool + +# NOTE(pklimowx): node-agent DS is deployed automatically via Power Operator after providing +# PowerProfile. The yaml file needs to be patched before building image to provide correct source for it. +# Both images depend on intel_power_manager* variable as there is no public image for AppQoS +- name: patch Node Agent DaemonSet yaml + block: + - name: use node-agent image from local registry + lineinfile: + path: "{{ intel_power_manager_dir }}/build/manifests/power-node-agent-ds.yaml" + regexp: "^ - image: 'intel-power-node-agent:latest'" + line: " - image: '{{ node_agent_image }}:{{ node_agent_image_version }}'" + + - name: use appqos image from local registry + lineinfile: + path: "{{ intel_power_manager_dir }}/build/manifests/power-node-agent-ds.yaml" + regexp: "^ - image: 'appqos:latest'" + line: " - image: '{{ app_qos_image }}'" + when: + - intel_power_manager.build_image_locally | default(false) | bool + - inventory_hostname == groups['kube_node'][0] + +# docker runtime is in use +- name: prepare images for Intel Kubernetes Power Manager + block: + - name: build images for Intel Kubernetes Power Manager + command: docker build -f build/{{ item.file }} -t {{ item.name }}:latest . + args: + chdir: "{{ intel_power_manager_dir }}" + with_items: + - { file: Dockerfile, name: intel-power-operator } + - { file: Dockerfile.nodeagent, name: intel-power-node-agent } + + - name: tag Intel Kubernetes Power Manager images + command: docker tag {{ item }}:latest {{ registry_local_address }}/{{ item }}:{{ intel_power_manager_git_ref }} + with_items: + - intel-power-operator + - intel-power-node-agent + + - name: push Intel Kubernetes Power Manager images to local registry + command: docker push {{ registry_local_address }}/{{ item }}:{{ intel_power_manager_git_ref }} + with_items: + - intel-power-operator + - intel-power-node-agent + when: + - container_runtime == "docker" + - intel_power_manager.build_image_locally | default(false) | bool + - inventory_hostname == groups['kube_node'][0] + +# crio/containerd runtime is in use +- name: prepare images for Intel Kubernetes Power Manager + block: + - name: build and tag images for Intel Kubernetes Power Manager + command: buildah bud -f build/{{ item.file }} -t {{ registry_local_address }}/{{ item.name }}:{{ intel_power_manager_git_ref }} + args: + chdir: "{{ intel_power_manager_dir }}" + with_items: + - { file: Dockerfile, name: intel-power-operator } + - { file: Dockerfile.nodeagent, name: intel-power-node-agent } + + - name: push Intel Kubernetes Power Manager images to local registry + command: buildah push {{ registry_local_address }}/{{ item }}:{{ intel_power_manager_git_ref }} + changed_when: true + with_items: + - intel-power-operator + - intel-power-node-agent + when: + - container_runtime in ["crio", "containerd"] + - intel_power_manager.build_image_locally | default(false) | bool + - inventory_hostname == groups['kube_node'][0] + +- name: prepare and deploy Intel Power Operator + block: + - name: apply rbac patch to allow PSP + patch: + src: "{{ (role_path, 'files', 'rbac.patch') | path_join }}" + dest: "{{ (intel_power_manager_dir, 'config', 'rbac', 'rbac.yaml') | path_join }}" + when: psp_enabled | default(true) | bool + + - name: apply k8s prerequisites + k8s: + state: present + src: "{{ (intel_power_manager_dir, 'config', 'rbac', item + '.yaml') | path_join }}" + with_items: + - namespace + - rbac + + - name: create and install Intel Power Operator CRDs + make: + chdir: "{{ intel_power_manager_dir }}" + + - name: populate Intel Kubernetes Power Manager Controller Manager template + template: + src: controller_manager.yml.j2 + dest: "{{ (intel_power_manager_dir, 'controller_manager.yml') | path_join }}" + force: yes + mode: preserve + + - name: deploy Intel Kubernetes Power Manager Controller Manager + k8s: + state: present + src: "{{ (intel_power_manager_dir, 'controller_manager.yml') | path_join }}" + + - name: populate Power Config template + template: + src: power_config.yml.j2 + dest: "{{ (intel_power_manager_dir, 'power_config.yml') | path_join }}" + force: yes + mode: preserve + + - name: apply Power Config + k8s: + state: present + src: "{{ (intel_power_manager_dir, 'power_config.yml') | path_join }}" + when: inventory_hostname == groups['kube_control_plane'][0] diff --git a/roles/intel_power_manager/tasks/power_pod_template_helper.yml b/roles/intel_power_manager/tasks/power_pod_template_helper.yml new file mode 100644 index 00000000..8c2e63c7 --- /dev/null +++ b/roles/intel_power_manager/tasks/power_pod_template_helper.yml @@ -0,0 +1,25 @@ +## +## Copyright (c) 2020-2022 Intel Corporation. +## +## Licensed under the Apache License, Version 2.0 (the "License"); +## you may not use this file except in compliance with the License. +## You may obtain a copy of the License at +## +## http://www.apache.org/licenses/LICENSE-2.0 +## +## Unless required by applicable law or agreed to in writing, software +## distributed under the License is distributed on an "AS IS" BASIS, +## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +## See the License for the specific language governing permissions and +## limitations under the License. +## +--- +- name: populate sample power pods templates + template: + src: "sample_power_pod.yml.j2" + dest: "{{ (intel_power_manager_dir, 'sample_power_pods', profile_name + '_power_pod.yml') | path_join }}" + force: yes + mode: preserve + loop: "{{ intel_power_manager.power_nodes }}" + loop_control: + loop_var: node_name diff --git a/roles/intel_power_manager/templates/controller_manager.yml.j2 b/roles/intel_power_manager/templates/controller_manager.yml.j2 new file mode 100644 index 00000000..84186199 --- /dev/null +++ b/roles/intel_power_manager/templates/controller_manager.yml.j2 @@ -0,0 +1,48 @@ +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: controller-manager + namespace: intel-power + labels: + control-plane: controller-manager +spec: + selector: + matchLabels: + control-plane: controller-manager + replicas: 1 + template: + metadata: + labels: + control-plane: controller-manager + spec: + serviceAccountName: intel-power-operator + containers: + - command: + - /manager + args: + - --enable-leader-election + imagePullPolicy: IfNotPresent + image: {{ power_operator_image | default("docker.io/intel/power-operator") }}:{{ power_operator_image_version | default("v1.0.2") }} + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: ["ALL"] + name: manager + resources: + limits: + cpu: 100m + memory: 30Mi + requests: + cpu: 100m + memory: 20Mi + volumeMounts: + - mountPath: /sys/fs + name: cgroup + mountPropagation: HostToContainer + readOnly: true + terminationGracePeriodSeconds: 10 + volumes: + - name: cgroup + hostPath: + path: /sys/fs diff --git a/roles/intel_power_manager/templates/global_shared_profile.yml.j2 b/roles/intel_power_manager/templates/global_shared_profile.yml.j2 new file mode 100644 index 00000000..d95f47c3 --- /dev/null +++ b/roles/intel_power_manager/templates/global_shared_profile.yml.j2 @@ -0,0 +1,11 @@ +--- +apiVersion: "power.intel.com/v1alpha1" +kind: PowerProfile +metadata: + name: shared-global + namespace: intel-power +spec: + name: "shared-global" + max: {{ intel_power_manager.max_shared_frequency }} + min: {{ intel_power_manager.min_shared_frequency }} + epp: "power" diff --git a/roles/intel_power_manager/templates/local_shared_profile.yml.j2 b/roles/intel_power_manager/templates/local_shared_profile.yml.j2 new file mode 100644 index 00000000..81e5d633 --- /dev/null +++ b/roles/intel_power_manager/templates/local_shared_profile.yml.j2 @@ -0,0 +1,11 @@ +--- +apiVersion: "power.intel.com/v1alpha1" +kind: PowerProfile +metadata: + name: shared-{{ node_name }} + namespace: intel-power +spec: + name: "shared-{{ node_name }}" + max: {{ max_frequency }} + min: {{ min_frequency }} + epp: "power" diff --git a/roles/intel_power_manager/templates/power_config.yml.j2 b/roles/intel_power_manager/templates/power_config.yml.j2 new file mode 100644 index 00000000..211bd0f5 --- /dev/null +++ b/roles/intel_power_manager/templates/power_config.yml.j2 @@ -0,0 +1,16 @@ +--- +apiVersion: "power.intel.com/v1alpha1" +kind: PowerConfig +metadata: + name: power-config + namespace: intel-power +spec: + powerImage: {{ app_qos_image }} + powerNodeSelector: + # Add labels here for the Nodes you want the PowerNodeAgent to be applied to + intel.power.node: "true" + # Add wanted PowerProfiles here; valid entries are as follows: + # performance + # balance-performance + # balance-power + powerProfiles: {{ intel_power_manager.power_profiles }} diff --git a/roles/intel_power_manager/templates/sample_power_pod.yml.j2 b/roles/intel_power_manager/templates/sample_power_pod.yml.j2 new file mode 100644 index 00000000..3a6ad2ee --- /dev/null +++ b/roles/intel_power_manager/templates/sample_power_pod.yml.j2 @@ -0,0 +1,23 @@ +# Do not change the name of this file +--- +apiVersion: v1 +kind: Pod +metadata: + name: {{ profile_name }}-power-pod + namespace: intel-power +spec: + containers: + - name: {{ profile_name }}-container + image: busybox + command: ["/bin/sh"] + args: ["-c", "sleep 15000"] + resources: + # IMPORTANT: The amount of the Power Cores have to be the same as the amount of requested CPUs + requests: + memory: "200Mi" + cpu: "2" + power.intel.com/{{ profile_name }}-{{ node_name }}: "2" + limits: + memory: "200Mi" + cpu: "2" + power.intel.com/{{ profile_name }}-{{ node_name }}: "2" diff --git a/roles/intel_power_manager/templates/shared_workload.yml.j2 b/roles/intel_power_manager/templates/shared_workload.yml.j2 new file mode 100644 index 00000000..abb5f665 --- /dev/null +++ b/roles/intel_power_manager/templates/shared_workload.yml.j2 @@ -0,0 +1,14 @@ +--- +apiVersion: "power.intel.com/v1alpha1" +kind: PowerWorkload +metadata: + name: shared-{{ node_name }}-workload + namespace: intel-power +spec: + name: "shared-{{ node_name }}-workload" + allCores: true + # IMPORTANT: The CPUs in reservedCPUs should match the value of the reserved system CPUs in your Kubelet config file + reservedCPUs: {{ reserved_cpus }} + powerNodeSelector: + kubernetes.io/hostname: {{ node_name }} + powerProfile: shared-{{ shared_workload_type }} diff --git a/roles/cmk_install/vars/main.yml b/roles/intel_power_manager/vars/main.yml similarity index 93% rename from roles/cmk_install/vars/main.yml rename to roles/intel_power_manager/vars/main.yml index 9f60069c..170d32e5 100644 --- a/roles/cmk_install/vars/main.yml +++ b/roles/intel_power_manager/vars/main.yml @@ -1,5 +1,5 @@ ## -## Copyright (c) 2020-2021 Intel Corporation. +## Copyright (c) 2020-2022 Intel Corporation. ## ## Licensed under the Apache License, Version 2.0 (the "License"); ## you may not use this file except in compliance with the License. diff --git a/roles/istio_operator_install/defaults/main.yml b/roles/istio_operator_install/defaults/main.yml deleted file mode 100644 index 0d720beb..00000000 --- a/roles/istio_operator_install/defaults/main.yml +++ /dev/null @@ -1,47 +0,0 @@ -## -## Copyright (c) 2020-2021 Intel Corporation. -## -## Licensed under the Apache License, Version 2.0 (the "License"); -## you may not use this file except in compliance with the License. -## You may obtain a copy of the License at -## -## http://www.apache.org/licenses/LICENSE-2.0 -## -## Unless required by applicable law or agreed to in writing, software -## distributed under the License is distributed on an "AS IS" BASIS, -## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -## See the License for the specific language governing permissions and -## limitations under the License. -## ---- -# please, visit https://github.com/istio/istio/releases for release info -istio_release_version: 1.11.1 - -istio_download_url: "https://github.com/istio/istio/releases/download/{{ istio_release_version }}/istio-{{ istio_release_version }}-linux-amd64.tar.gz" - -istio_dir: /usr/local/istio -istio_profiles_dir: /usr/src/charts/istio_profiles -istio_deployment_name: bmra-istio -# available profiles are: 'empty', 'minimal', 'default', 'external' and 'none' (no profile will be deployed) -# please, refer to https://github.com/istio/istio/tree//manifests/profiles for details -istio_profile: default - -# please, refer to https://istio.io/latest/docs/reference/commands/istioctl/#istioctl-operator-init -# and https://istio.io/latest/docs/reference/commands/istioctl/#istioctl-operator-remove -# for detailed explanation - -istio_context: '' -istio_filename: '' -istio_hub: '' -istio_image_pull_secrets: '' -istio_config_namespace: '' -istio_kubeconfig: '' -istio_tag: '' -istio_watched_namespaces: '' -istio_vklog: '' -istio_revision: '' -istio_manifests: '' -istio_dry_run: no -istio_system_namespace: 'istio-system' -istio_operator_namespace: 'istio-operator' -istio_force_removal: yes diff --git a/roles/istio_operator_install/tasks/cleanup.yml b/roles/istio_operator_install/tasks/cleanup.yml deleted file mode 100644 index fd6f8b26..00000000 --- a/roles/istio_operator_install/tasks/cleanup.yml +++ /dev/null @@ -1,66 +0,0 @@ -## -## Copyright (c) 2020-2021 Intel Corporation. -## -## Licensed under the Apache License, Version 2.0 (the "License"); -## you may not use this file except in compliance with the License. -## You may obtain a copy of the License at -## -## http://www.apache.org/licenses/LICENSE-2.0 -## -## Unless required by applicable law or agreed to in writing, software -## distributed under the License is distributed on an "AS IS" BASIS, -## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -## See the License for the specific language governing permissions and -## limitations under the License. -## ---- -- name: remove istio "{{ istio_profile }}" profile first - command: "kubectl delete -f {{ istio_profiles_dir }}/{{ istio_profile }}.yaml" - changed_when: true - failed_when: false - -- name: evaluate parameters to be used to delete istio-operator - template: - src: "istioctl-operator-remove-options.yml.j2" - dest: "{{ role_path }}/tasks/istioctl-operator-remove-options.yml" - force: yes - mode: preserve - delegate_to: localhost - become: false - -- name: include temporary file contained operator cmd with options provided - include_vars: "{{ role_path }}/tasks/istioctl-operator-remove-options.yml" - -- name: show command for operator removal - debug: - msg: | - Following command will be executed for istio operator removal: - {{ argv | join(' ') }} - delegate_to: localhost - become: false - -- name: remove istio-operator - command: - argv: "{{ argv }}" - failed_when: false - changed_when: true - -- name: remove existing istio resources - # noqa 305 - shell is used intentionally here - shell: "set -o pipefail && {{ istio_dir }}/bin/istioctl manifest generate | kubectl delete -f -" - failed_when: false - changed_when: true - -- name: remove temporary operator file - file: - path: "{{ role_path }}/tasks/istioctl-operator-remove-options.yml" - state: absent - delegate_to: localhost - become: false - -- name: remove istio release - file: - path: "{{ istio_dir }}" - state: absent - failed_when: false - diff --git a/roles/istio_operator_install/tasks/install.yml b/roles/istio_operator_install/tasks/install.yml deleted file mode 100644 index 3baee103..00000000 --- a/roles/istio_operator_install/tasks/install.yml +++ /dev/null @@ -1,68 +0,0 @@ -## -## Copyright (c) 2020-2021 Intel Corporation. -## -## Licensed under the Apache License, Version 2.0 (the "License"); -## you may not use this file except in compliance with the License. -## You may obtain a copy of the License at -## -## http://www.apache.org/licenses/LICENSE-2.0 -## -## Unless required by applicable law or agreed to in writing, software -## distributed under the License is distributed on an "AS IS" BASIS, -## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -## See the License for the specific language governing permissions and -## limitations under the License. -## ---- -- name: download and unarchive istio release - unarchive: - src: "{{ istio_download_url }}" - dest: /usr/local - remote_src: yes - list_files: yes - mode: 0755 - extra_opts: - - "--transform" - - "s/^istio-{{ istio_release_version }}/istio/" - register: istio_download - until: istio_download is not failed - retries: 5 - -- name: add istio binaries to the $PATH - copy: - content: | - export PATH={{ istio_dir }}/bin:$PATH - dest: /etc/profile.d/istioctl.sh - mode: 0755 - -- name: evaluate parameters to be used to init istio-operator - template: - src: "istioctl-operator-init-options.yml.j2" - dest: "{{ role_path }}/tasks/istioctl-operator-init-options.yml" - force: yes - mode: preserve - delegate_to: localhost - become: false - -- name: include temporary file contained operator cmd with options provided - include_vars: "{{ role_path }}/tasks/istioctl-operator-init-options.yml" - -- name: show command for operator init - debug: - msg: | - Following command will be executed for istio operator initialization: - {{ argv | join(' ') }} - delegate_to: localhost - become: false - -- name: init istio-operator - command: - argv: "{{ argv }}" - changed_when: true - -- name: remove temporary operator file - file: - path: "{{ role_path }}/tasks/istioctl-operator-init-options.yml" - state: absent - delegate_to: localhost - become: false \ No newline at end of file diff --git a/roles/istio_operator_install/templates/istioctl-operator-init-options.yml.j2 b/roles/istio_operator_install/templates/istioctl-operator-init-options.yml.j2 deleted file mode 100644 index d52d6d82..00000000 --- a/roles/istio_operator_install/templates/istioctl-operator-init-options.yml.j2 +++ /dev/null @@ -1,63 +0,0 @@ -argv: - - {{ istio_dir }}/bin/istioctl - - operator - - init -{% if istio_dry_run is defined and istio_dry_run | bool %} - - --dry-run -{% endif -%} -{% if istio_charts is defined and istio_charts != '' %} - - --charts - - {{ istio_charts }} -{% endif -%} -{% if istio_context is defined and istio_context != '' %} - - --context - - {{ istio_context }} -{% endif -%} -{% if istio_filename is defined and istio_filename != '' %} - - --filename - - {{ istio_filename }} -{% endif -%} -{% if istio_hub is defined and istio_hub != '' %} - - --hub - - {{ istio_hub }} -{% endif -%} -{% if istio_image_pull_secrets is defined and istio_image_pull_secrets != '' %} - - --imagePullSecrets - - {{ istio_image_pull_secrets }} -{% endif -%} -{% if istio_system_namespace is defined and istio_system_namespace != '' %} - - --istioNamespace - - {{ istio_system_namespace }} -{% endif -%} -{% if istio_config_namespace is defined and istio_config_namespace != '' %} - - --namespace - - {{ istio_config_namespace }} -{% endif -%} -{% if istio_operator_namespace is defined and istio_operator_namespace != '' %} - - --operatorNamespace - - {{ istio_operator_namespace }} -{% endif -%} -{% if istio_kubeconfig is defined and istio_kubeconfig != '' %} - - --kubeconfig - - {{ istio_kubeconfig }} -{% endif -%} -{% if istio_manifest is defined and istio_manifest != '' %} - - --manifests - - {{ istio_manifest }} -{% endif -%} -{% if istio_revision is defined and istio_revision != '' %} - - --revision - - {{ istio_revision }} -{% endif -%} -{% if istio_tag is defined and istio_tag != '' %} - - --tag - - {{ istio_tag }} -{% endif -%} -{% if istio_vklog is defined and istio_vklog != '' %} - - --vklog - - {{ istio_vklog }} -{% endif -%} -{% if istio_watched_namespaces is defined and istio_watched_namespaces != '' %} - - --watchedNamespaces - - {{ istio_watched_namespaces }} -{% endif -%} diff --git a/roles/istio_operator_install/templates/istioctl-operator-remove-options.yml.j2 b/roles/istio_operator_install/templates/istioctl-operator-remove-options.yml.j2 deleted file mode 100644 index e96536ef..00000000 --- a/roles/istio_operator_install/templates/istioctl-operator-remove-options.yml.j2 +++ /dev/null @@ -1,42 +0,0 @@ -argv: - - {{ istio_dir }}/bin/istioctl - - operator - - remove -{% if istio_dry_run is defined and istio_dry_run | bool %} - - --dry-run -{% endif -%} -{% if istio_charts is defined and istio_charts != '' %} - - --charts - - {{ istio_charts }} -{% endif -%} -{% if istio_context is defined and istio_context != '' %} - - --context - - {{ istio_context }} -{% endif -%} -{% if istio_system_namespace is defined and istio_system_namespace != '' %} - - --istioNamespace - - {{ istio_system_namespace }} -{% endif -%} -{% if istio_config_namespace is defined and istio_config_namespace != '' %} - - --namespace - - {{ istio_config_namespace }} -{% endif -%} -{% if istio_operator_namespace is defined and istio_operator_namespace != '' %} - - --operatorNamespace - - {{ istio_operator_namespace }} -{% endif -%} -{% if istio_kubeconfig is defined and istio_kubeconfig != '' %} - - --kubeconfig - - {{ istio_kubeconfig }} -{% endif -%} -{% if istio_revision is defined and istio_revision != '' %} - - --revision - - {{ istio_revision }} -{% endif -%} -{% if istio_vklog is defined and istio_vklog != '' %} - - --vklog - - {{ istio_vklog }} -{% endif -%} -{% if istio_force_removal is defined and istio_force_removal | bool %} - - --force -{% endif -%} diff --git a/roles/istio_operator_install/templates/profiles/default.yaml.j2 b/roles/istio_operator_install/templates/profiles/default.yaml.j2 deleted file mode 100644 index 9fc1c44f..00000000 --- a/roles/istio_operator_install/templates/profiles/default.yaml.j2 +++ /dev/null @@ -1,185 +0,0 @@ -apiVersion: install.istio.io/v1alpha1 -kind: IstioOperator -metadata: - namespace: {{ istio_system_namespace }} - name: {{ istio_deployment_name }} - labels: - version: {{ istio_release_version }} -spec: - hub: gcr.io/istio-testing - tag: latest - - # You may override parts of meshconfig by uncommenting the following lines. - meshConfig: - defaultConfig: - proxyMetadata: {} - enablePrometheusMerge: true - # Opt-out of global http2 upgrades. - # Destination rule is used to opt-in. - # h2_upgrade_policy: DO_NOT_UPGRADE - - # Traffic management feature - components: - base: - enabled: true - pilot: - enabled: true - - # Istio Gateway feature - ingressGateways: - - name: istio-ingressgateway - enabled: true - egressGateways: - - name: istio-egressgateway - enabled: false - - # Istio CNI feature - cni: - enabled: false - - # Remote and config cluster configuration for an external istiod - istiodRemote: - enabled: false - - # Global values passed through to helm global.yaml. - # Please keep this in sync with manifests/charts/global.yaml - values: - global: - istioNamespace: {{ istio_system_namespace }} - istiod: - enableAnalysis: false - logging: - level: "default:info" - logAsJson: false - pilotCertProvider: istiod - jwtPolicy: third-party-jwt - proxy: - image: proxyv2 - clusterDomain: "cluster.local" - resources: - requests: - cpu: 100m - memory: 128Mi - limits: - cpu: 2000m - memory: 1024Mi - logLevel: warning - componentLogLevel: "misc:error" - privileged: false - enableCoreDump: false - statusPort: 15020 - readinessInitialDelaySeconds: 1 - readinessPeriodSeconds: 2 - readinessFailureThreshold: 30 - includeIPRanges: "*" - excludeIPRanges: "" - excludeOutboundPorts: "" - excludeInboundPorts: "" - autoInject: enabled - tracer: "zipkin" - proxy_init: - image: proxyv2 - resources: - limits: - cpu: 2000m - memory: 1024Mi - requests: - cpu: 10m - memory: 10Mi - # Specify image pull policy if default behavior isn't desired. - # Default behavior: latest images will be Always else IfNotPresent. - imagePullPolicy: "" - operatorManageWebhooks: false - tracer: - lightstep: {} - zipkin: {} - datadog: {} - stackdriver: {} - imagePullSecrets: [] - oneNamespace: false - defaultNodeSelector: {} - configValidation: true - multiCluster: - enabled: false - clusterName: "" - omitSidecarInjectorConfigMap: false - network: "" - defaultResources: - requests: - cpu: 10m - defaultPodDisruptionBudget: - enabled: true - priorityClassName: "" - useMCP: false - sds: - token: - aud: istio-ca - sts: - servicePort: 0 - meshNetworks: {} - mountMtlsCerts: false - base: - enableCRDTemplates: false - validationURL: "" - pilot: - autoscaleEnabled: true - autoscaleMin: 1 - autoscaleMax: 5 - replicaCount: 1 - image: pilot - traceSampling: 1.0 - env: {} - cpu: - targetAverageUtilization: 80 - nodeSelector: {} - keepaliveMaxServerConnectionAge: 30m - enableProtocolSniffingForOutbound: true - enableProtocolSniffingForInbound: true - deploymentLabels: - configMap: true - - telemetry: - enabled: true - v2: - enabled: true - metadataExchange: - wasmEnabled: false - prometheus: - wasmEnabled: false - enabled: true - stackdriver: - enabled: false - logging: false - monitoring: false - topology: false - configOverride: {} - - istiodRemote: - injectionURL: "" - - gateways: - istio-egressgateway: - env: {} - autoscaleEnabled: true - type: ClusterIP - name: istio-egressgateway - secretVolumes: - - name: egressgateway-certs - secretName: istio-egressgateway-certs - mountPath: /etc/istio/egressgateway-certs - - name: egressgateway-ca-certs - secretName: istio-egressgateway-ca-certs - mountPath: /etc/istio/egressgateway-ca-certs - - istio-ingressgateway: - autoscaleEnabled: true - type: LoadBalancer - name: istio-ingressgateway - env: {} - secretVolumes: - - name: ingressgateway-certs - secretName: istio-ingressgateway-certs - mountPath: /etc/istio/ingressgateway-certs - - name: ingressgateway-ca-certs - secretName: istio-ingressgateway-ca-certs - mountPath: /etc/istio/ingressgateway-ca-certs \ No newline at end of file diff --git a/roles/istio_operator_install/templates/profiles/empty.yaml.j2 b/roles/istio_operator_install/templates/profiles/empty.yaml.j2 deleted file mode 100644 index 4642473e..00000000 --- a/roles/istio_operator_install/templates/profiles/empty.yaml.j2 +++ /dev/null @@ -1,17 +0,0 @@ -apiVersion: install.istio.io/v1alpha1 -kind: IstioOperator -metadata: - namespace: {{ istio_system_namespace }} - name: {{ istio_deployment_name }} - labels: - version: {{ istio_release_version }} -spec: - components: - base: - enabled: false - pilot: - enabled: false - ingressGateways: - - name: istio-ingressgateway - enabled: false - diff --git a/roles/istio_operator_install/templates/profiles/external.yaml.j2 b/roles/istio_operator_install/templates/profiles/external.yaml.j2 deleted file mode 100644 index 52d670aa..00000000 --- a/roles/istio_operator_install/templates/profiles/external.yaml.j2 +++ /dev/null @@ -1,25 +0,0 @@ -apiVersion: install.istio.io/v1alpha1 -kind: IstioOperator -metadata: - namespace: {{ istio_system_namespace }} - name: {{ istio_deployment_name }} - labels: - version: {{ istio_release_version }} -spec: - components: - base: - enabled: false - pilot: - enabled: false - ingressGateways: - - name: istio-ingressgateway - enabled: false - istiodRemote: - enabled: true - values: - global: - externalIstiod: true - omitSidecarInjectorConfigMap: true - configCluster: false - pilot: - configMap: false diff --git a/roles/istio_operator_install/templates/profiles/minimal.yaml.j2 b/roles/istio_operator_install/templates/profiles/minimal.yaml.j2 deleted file mode 100644 index 77597c53..00000000 --- a/roles/istio_operator_install/templates/profiles/minimal.yaml.j2 +++ /dev/null @@ -1,12 +0,0 @@ -apiVersion: install.istio.io/v1alpha1 -kind: IstioOperator -metadata: - namespace: {{ istio_system_namespace }} - name: {{ istio_deployment_name }} - labels: - version: {{ istio_release_version }} -spec: - components: - ingressGateways: - - name: istio-ingressgateway - enabled: false diff --git a/roles/kmra_install/charts/kmra-apphsm/Chart.yaml b/roles/kmra_install/charts/kmra-apphsm/Chart.yaml index a38f264f..0c7a2a35 100644 --- a/roles/kmra_install/charts/kmra-apphsm/Chart.yaml +++ b/roles/kmra_install/charts/kmra-apphsm/Chart.yaml @@ -1,5 +1,5 @@ ## -## Copyright (c) 2020-2021 Intel Corporation. +## Copyright (c) 2020-2022 Intel Corporation. ## ## Licensed under the Apache License, Version 2.0 (the "License"); ## you may not use this file except in compliance with the License. @@ -17,5 +17,5 @@ apiVersion: v1 description: Key Management Reference Application name: kmra -version: 1.2 -appVersion: '1.2' +version: 1.4 +appVersion: '1.4' diff --git a/roles/kmra_install/charts/kmra-apphsm/templates/kmra-apphsm-configmap.yml b/roles/kmra_install/charts/kmra-apphsm/templates/kmra-apphsm-configmap.yml index 0baba1dc..6c7d361a 100644 --- a/roles/kmra_install/charts/kmra-apphsm/templates/kmra-apphsm-configmap.yml +++ b/roles/kmra_install/charts/kmra-apphsm/templates/kmra-apphsm-configmap.yml @@ -10,5 +10,10 @@ data: no_proxy: {{ .Values.no_proxy | default "" | quote }} PCCS_PORT: {{ .Values.apphsm.pccs_port | quote }} PCCS_HOSTNAME: {{ .Values.apphsm.pccs_hostname | quote }} - SGX_PRV_GID: {{ .Values.apphsm.sgx_prv_gid | quote }} - APPHSM_PORT: {{ .Values.apphsm.main.port | quote }} \ No newline at end of file + APPHSM_PORT: {{ .Values.apphsm.main.port | quote }} + APPHSM_KEY_IN_TOKEN_NAME: {{ .Values.apphsm.apphsm_key_in_token_name | quote }} + APPHSM_TOKEN_NAME: {{ .Values.apphsm.apphsm_token_name | quote }} + TEST_CTK_LOADKEY_CERT_USER_ID: {{ .Values.apphsm.test_ctk_loadkey_cert_user_id | quote }} + TEST_UNIQUE_UID: {{ .Values.apphsm.test_unique_uid | quote }} + DEFAULT_USER_PIN: {{ .Values.apphsm.default_user_pin | quote }} + DEFAULT_SO_PIN: {{ .Values.apphsm.default_so_pin | quote }} diff --git a/roles/kmra_install/charts/kmra-apphsm/templates/kmra-apphsm-deployment.yml b/roles/kmra_install/charts/kmra-apphsm/templates/kmra-apphsm-deployment.yml index d5c35e8a..27579601 100644 --- a/roles/kmra_install/charts/kmra-apphsm/templates/kmra-apphsm-deployment.yml +++ b/roles/kmra_install/charts/kmra-apphsm/templates/kmra-apphsm-deployment.yml @@ -18,6 +18,10 @@ spec: sgx.intel.com/quote-provider: {{ .Release.Name }} spec: serviceAccountName: {{ .Release.Name }} + initContainers: + - name: init-tmpfs + image: "{{ .Values.apphsm.init.image.repo }}/{{ .Values.apphsm.init.image.name }}:{{ .Values.apphsm.init.image.tag }}" + command: ['sh', '-c', "rm -rf /var/lib/softhsm/tokens/*"] containers: - name: {{ .Release.Name }} image: "{{ .Values.apphsm.main.image.repo }}/{{ .Values.apphsm.main.image.name }}:{{ .Values.apphsm.main.image.tag }}" @@ -29,6 +33,16 @@ spec: - name: mtls mountPath: /opt/intel/ca readOnly: true + - name: sgx-qcnl-conf + mountPath: /etc/sgx_default_qcnl.conf + subPath: sgx_default_qcnl.conf + readOnly: true + - name: tmpfs + mountPath: /tmp + subPath: tmp + - name: tmpfs + mountPath: /var/lib/softhsm/tokens + subPath: tokens resources: limits: cpu: 500m @@ -39,6 +53,8 @@ spec: memory: 200Mi securityContext: runAsUser: 1000 + runAsGroup: {{ .Values.apphsm.sgx_prv_gid }} + readOnlyRootFilesystem: true - name: apphsm-rproxy image: "{{ .Values.apphsm.rproxy.image.repo }}/{{ .Values.apphsm.rproxy.image.name }}:{{ .Values.apphsm.rproxy.image.tag }}" imagePullPolicy: {{ .Values.apphsm.rproxy.image.pullPolicy }} @@ -79,9 +95,16 @@ spec: path: apphsm.crt - key: ca.cert path: ca.crt + - name: sgx-qcnl-conf + configMap: + name: {{ .Release.Name }}-qcnl-conf - name: nginx-conf configMap: name: {{ .Release.Name }}-rproxy-conf items: - key: nginx.conf - path: nginx.conf \ No newline at end of file + path: nginx.conf + - name: tmpfs + emptyDir: + medium: Memory + sizeLimit: 64Mi diff --git a/roles/kmra_install/charts/kmra-apphsm/templates/kmra-apphsm-qcnl-configmap.yaml b/roles/kmra_install/charts/kmra-apphsm/templates/kmra-apphsm-qcnl-configmap.yaml new file mode 100644 index 00000000..2367fe77 --- /dev/null +++ b/roles/kmra_install/charts/kmra-apphsm/templates/kmra-apphsm-qcnl-configmap.yaml @@ -0,0 +1,11 @@ +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ .Release.Name }}-qcnl-conf + namespace: {{ .Release.Namespace }} +data: + sgx_default_qcnl.conf: | + PCCS_URL=https://{{ .Values.apphsm.pccs_hostname }}:{{ .Values.apphsm.pccs_port }}/sgx/certification/v3/ + # To accept insecure HTTPS cert, set this option to FALSE + USE_SECURE_CERT={{ (upper .Values.apphsm.use_secure_cert) }} diff --git a/roles/kmra_install/charts/kmra-ctk/Chart.yaml b/roles/kmra_install/charts/kmra-ctk/Chart.yaml index a38f264f..0c7a2a35 100644 --- a/roles/kmra_install/charts/kmra-ctk/Chart.yaml +++ b/roles/kmra_install/charts/kmra-ctk/Chart.yaml @@ -1,5 +1,5 @@ ## -## Copyright (c) 2020-2021 Intel Corporation. +## Copyright (c) 2020-2022 Intel Corporation. ## ## Licensed under the Apache License, Version 2.0 (the "License"); ## you may not use this file except in compliance with the License. @@ -17,5 +17,5 @@ apiVersion: v1 description: Key Management Reference Application name: kmra -version: 1.2 -appVersion: '1.2' +version: 1.4 +appVersion: '1.4' diff --git a/roles/kmra_install/charts/kmra-ctk/templates/kmra-ctk-loadkey-configmap.yml b/roles/kmra_install/charts/kmra-ctk/templates/kmra-ctk-loadkey-configmap.yml index 8901495d..a9e27f6d 100644 --- a/roles/kmra_install/charts/kmra-ctk/templates/kmra-ctk-loadkey-configmap.yml +++ b/roles/kmra_install/charts/kmra-ctk/templates/kmra-ctk-loadkey-configmap.yml @@ -12,6 +12,10 @@ data: PCCS_HOSTNAME: {{ .Values.ctk_loadkey.pccs_hostname | quote }} APPHSM_PORT: {{ .Values.ctk_loadkey.apphsm_port | quote }} APPHSM_HOSTNAME: {{ .Values.ctk_loadkey.apphsm_hostname | quote }} - SGX_PRV_GID: {{ .Values.ctk_loadkey.sgx_prv_gid | quote }} NGINX_HOSTNAME: {{ .Values.ctk_loadkey.main.hostname | quote }} NGINX_PORT: {{ .Values.ctk_loadkey.main.port | quote }} + CLIENT_TOKEN: {{ .Values.ctk_loadkey.client_token | quote }} + CLIENT_KEY_LABEL: {{ .Values.ctk_loadkey.client_key_label | quote }} + TEST_UNIQUE_UID: {{ .Values.ctk_loadkey.test_unique_uid | quote }} + DEFAULT_USER_PIN: {{ .Values.ctk_loadkey.default_user_pin | quote }} + DEFAULT_SO_PIN: {{ .Values.ctk_loadkey.default_so_pin | quote }} diff --git a/roles/kmra_install/charts/kmra-ctk/templates/kmra-ctk-loadkey-deployment.yml b/roles/kmra_install/charts/kmra-ctk/templates/kmra-ctk-loadkey-deployment.yml index bdf0213c..6e70a5b6 100644 --- a/roles/kmra_install/charts/kmra-ctk/templates/kmra-ctk-loadkey-deployment.yml +++ b/roles/kmra_install/charts/kmra-ctk/templates/kmra-ctk-loadkey-deployment.yml @@ -17,6 +17,10 @@ spec: sgx.intel.com/quote-provider: {{ .Release.Name }} spec: serviceAccountName: {{ .Release.Name }} + initContainers: + - name: init-tmpfs + image: "{{ .Values.ctk_loadkey.init.image.repo }}/{{ .Values.ctk_loadkey.init.image.name }}:{{ .Values.ctk_loadkey.init.image.tag }}" + command: ['sh', '-c', "rm -rf /opt/intel/cryptoapitoolkit/tokens/*"] containers: - name: {{ .Release.Name }} image: "{{ .Values.ctk_loadkey.main.image.repo }}/{{ .Values.ctk_loadkey.main.image.name }}:{{ .Values.ctk_loadkey.main.image.tag }}" @@ -31,6 +35,16 @@ spec: - name: mtls mountPath: /opt/intel/ca readOnly: true + - name: sgx-qcnl-conf + mountPath: /etc/sgx_default_qcnl.conf + subPath: sgx_default_qcnl.conf + readOnly: true + - name: tmpfs + mountPath: /tmp + subPath: tmp + - name: tmpfs + mountPath: /opt/intel/cryptoapitoolkit/tokens + subPath: tokens resources: limits: cpu: 500m @@ -41,6 +55,8 @@ spec: memory: 200Mi securityContext: runAsUser: 1000 + runAsGroup: {{ .Values.ctk_loadkey.sgx_prv_gid }} + readOnlyRootFilesystem: true affinity: nodeAffinity: preferredDuringSchedulingIgnoredDuringExecution: @@ -62,3 +78,11 @@ spec: path: ctk_loadkey.crt - key: ca.cert path: ca.crt + - name: sgx-qcnl-conf + configMap: + name: {{ .Release.Name }}-qcnl-conf + - name: tmpfs + emptyDir: + medium: Memory + sizeLimit: 64Mi + diff --git a/roles/kmra_install/charts/kmra-ctk/templates/kmra-ctk-loadkey-qcnl-configmap.yaml b/roles/kmra_install/charts/kmra-ctk/templates/kmra-ctk-loadkey-qcnl-configmap.yaml new file mode 100644 index 00000000..1a816077 --- /dev/null +++ b/roles/kmra_install/charts/kmra-ctk/templates/kmra-ctk-loadkey-qcnl-configmap.yaml @@ -0,0 +1,11 @@ +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ .Release.Name }}-qcnl-conf + namespace: {{ .Release.Namespace }} +data: + sgx_default_qcnl.conf: | + PCCS_URL=https://{{ .Values.ctk_loadkey.pccs_hostname }}:{{ .Values.ctk_loadkey.pccs_port }}/sgx/certification/v3/ + # To accept insecure HTTPS cert, set this option to FALSE + USE_SECURE_CERT={{ (upper .Values.ctk_loadkey.use_secure_cert) }} diff --git a/roles/kmra_install/charts/kmra-pccs/Chart.yaml b/roles/kmra_install/charts/kmra-pccs/Chart.yaml index a38f264f..0c7a2a35 100644 --- a/roles/kmra_install/charts/kmra-pccs/Chart.yaml +++ b/roles/kmra_install/charts/kmra-pccs/Chart.yaml @@ -1,5 +1,5 @@ ## -## Copyright (c) 2020-2021 Intel Corporation. +## Copyright (c) 2020-2022 Intel Corporation. ## ## Licensed under the Apache License, Version 2.0 (the "License"); ## you may not use this file except in compliance with the License. @@ -17,5 +17,5 @@ apiVersion: v1 description: Key Management Reference Application name: kmra -version: 1.2 -appVersion: '1.2' +version: 1.4 +appVersion: '1.4' diff --git a/roles/kmra_install/charts/kmra-pccs/templates/kmra-pccs-configmap.yml b/roles/kmra_install/charts/kmra-pccs/templates/kmra-pccs-configmap.yml index b54c1c80..d636c4d7 100644 --- a/roles/kmra_install/charts/kmra-pccs/templates/kmra-pccs-configmap.yml +++ b/roles/kmra_install/charts/kmra-pccs/templates/kmra-pccs-configmap.yml @@ -35,7 +35,7 @@ data: "freezeTableName": true }, "logging" : false, - "storage": "pckcache.db" + "storage": "/tmp/pckcache.db" } }, "myql" : { diff --git a/roles/kmra_install/charts/kmra-pccs/templates/kmra-pccs-deployment.yml b/roles/kmra_install/charts/kmra-pccs/templates/kmra-pccs-deployment.yml index 1f66bf58..9bca44a0 100644 --- a/roles/kmra_install/charts/kmra-pccs/templates/kmra-pccs-deployment.yml +++ b/roles/kmra_install/charts/kmra-pccs/templates/kmra-pccs-deployment.yml @@ -26,6 +26,12 @@ spec: - name: pccs-tls mountPath: /opt/intel/pccs/ssl_key readOnly: true + - name: tmpfs + mountPath: /tmp + subPath: tmp + - name: tmpfs + mountPath: /opt/intel/pccs/logs + subPath: logs resources: limits: cpu: 500m @@ -35,6 +41,7 @@ spec: memory: 200Mi securityContext: runAsUser: 65333 + readOnlyRootFilesystem: true - name: pccs-rproxy image: "{{ .Values.pccs.rproxy.image.repo }}/{{ .Values.pccs.rproxy.image.name }}:{{ .Values.pccs.rproxy.image.tag }}" imagePullPolicy: {{ .Values.pccs.rproxy.image.pullPolicy }} @@ -85,4 +92,8 @@ spec: path: private.pem - key: tls.cert path: file.crt + - name: tmpfs + emptyDir: + medium: Memory + sizeLimit: 64Mi diff --git a/roles/kmra_install/defaults/main.yml b/roles/kmra_install/defaults/main.yml index 8867d447..4bdb4f0c 100644 --- a/roles/kmra_install/defaults/main.yml +++ b/roles/kmra_install/defaults/main.yml @@ -1,5 +1,5 @@ ## -## Copyright (c) 2020-2021 Intel Corporation. +## Copyright (c) 2020-2022 Intel Corporation. ## ## Licensed under the Apache License, Version 2.0 (the "License"); ## you may not use this file except in compliance with the License. @@ -20,42 +20,66 @@ kmra_apphsm_release_name: "kmra-apphsm" kmra_ctk_release_name: "kmra-ctk" kmra_pccs_release_name: "kmra-pccs" -kmra_apphsm_helm_values_file: "/usr/src/charts/kmra-apphsm-values.yml" -kmra_ctk_helm_values_file: "/usr/src/charts/kmra-ctk-values.yml" -kmra_pccs_helm_values_file: "/usr/src/charts/kmra-pccs-values.yml" +kmra_apphsm_helm_values_file: "{{ project_root_dir }}/charts/kmra-apphsm-values.yml" +kmra_ctk_helm_values_file: "{{ project_root_dir }}/charts/kmra-ctk-values.yml" +kmra_pccs_helm_values_file: "{{ project_root_dir }}/charts/kmra-pccs-values.yml" -kmra_apphsm_chart_path: "/usr/src/charts/kmra-apphsm" -kmra_ctk_chart_path: "/usr/src/charts/kmra-ctk" -kmra_pccs_chart_path: "/usr/src/charts/kmra-pccs" +kmra_apphsm_chart_path: "{{ project_root_dir }}/charts/kmra-apphsm" +kmra_ctk_chart_path: "{{ project_root_dir }}/charts/kmra-ctk" +kmra_pccs_chart_path: "{{ project_root_dir }}/charts/kmra-pccs" -kmra_pccs_image_repo: "intel" -kmra_pccs_image_name: "kmra" -kmra_pccs_image_tag: "pccs" +kmra_pccs_image_repo: "docker.io" +kmra_pccs_image_name: "intel/pccs" +kmra_pccs_image_tag: "v1" kmra_pccs_rproxy_image_repo: "docker.io" kmra_pccs_rproxy_image_name: "nginx" -kmra_pccs_rproxy_tag: "1.20.1-alpine" +kmra_pccs_rproxy_tag: "1.21.5-alpine" + kmra_pccs_upstream_port: 8082 kmra_pccs_service_port: 8081 -kmra_apphsm_image_repo: "intel" -kmra_apphsm_image_name: "kmra" -kmra_apphsm_image_tag: "apphsm" +kmra_apphsm_image_repo: "docker.io" +kmra_apphsm_image_name: "intel/apphsm" +kmra_apphsm_image_tag: "v1" + +kmra_apphsm_init_image_repo: "docker.io" +kmra_apphsm_init_image_name: "busybox" +kmra_apphsm_init_image_tag: "1.35" kmra_apphsm_upstream_port: 5001 kmra_apphsm_service_port: 5000 +kmra_apphsm_key_in_token_name: "key_1" +kmra_apphsm_token_name: "token_1" +kmra_apphsm_test_ctk_loadkey_cert_user_id: "ctk_loadkey_user_id_01234" +kmra_apphsm_test_unique_uid: "unique_id_1234" +kmra_apphsm_default_user_pin: "1234" +kmra_apphsm_default_so_pin: "12345678" +kmra_apphsm_use_secure_cert: false + kmra_apphsm_rproxy_image_repo: "docker.io" kmra_apphsm_rproxy_image_name: "nginx" -kmra_apphsm_rproxy_tag: "1.20.1-alpine" +kmra_apphsm_rproxy_tag: "1.21.5-alpine" + +kmra_ctk_loadkey_image_repo: "docker.io" +kmra_ctk_loadkey_image_name: "intel/ctk_loadkey" +kmra_ctk_loadkey_image_tag: "v1" + +kmra_ctk_loadkey_init_image_repo: "docker.io" +kmra_ctk_loadkey_init_image_name: "busybox" +kmra_ctk_loadkey_init_image_tag: "1.35" -kmra_ctk_loadkey_image_repo: "intel" -kmra_ctk_loadkey_image_name: "kmra" -kmra_ctk_loadkey_image_tag: "ctk_loadkey" kmra_ctk_loadkey_upstream_port: 8082 kmra_ctk_loadkey_upstream_server_name: 0.0.0.0 kmra_ctk_loadkey_node_port: 30003 +kmra_ctk_loadkey_default_user_pin: "4321" +kmra_ctk_loadkey_default_so_pin: "87654321" +kmra_ctk_loadkey_client_token: "client_token" +kmra_ctk_loadkey_client_key_label: "client_key_priv" +kmra_ctk_loadkey_test_unique_uid: "{{ kmra_apphsm_test_unique_uid }}" +kmra_ctk_loadkey_use_secure_cert: "{{ kmra_apphsm_use_secure_cert }}" kmra_pccs_crt_subj: O: "SGX-PCCS" @@ -73,4 +97,4 @@ kmra_apphsm_crt_subj: kmra_ctk_crt_subj: O: "AppHSM" - OU: "ctk_loadkey_user_id_01234" + OU: "{{ kmra_apphsm_test_ctk_loadkey_cert_user_id }}" diff --git a/roles/kmra_install/tasks/cleanup.yml b/roles/kmra_install/tasks/cleanup.yml index cb7674fc..a6cffcdf 100644 --- a/roles/kmra_install/tasks/cleanup.yml +++ b/roles/kmra_install/tasks/cleanup.yml @@ -1,5 +1,5 @@ ## -## Copyright (c) 2020-2021 Intel Corporation. +## Copyright (c) 2020-2022 Intel Corporation. ## ## Licensed under the Apache License, Version 2.0 (the "License"); ## you may not use this file except in compliance with the License. diff --git a/roles/kmra_install/tasks/create_tls_secrets.yml b/roles/kmra_install/tasks/create_tls_secrets.yml index bccc57b6..7dd5dff5 100644 --- a/roles/kmra_install/tasks/create_tls_secrets.yml +++ b/roles/kmra_install/tasks/create_tls_secrets.yml @@ -1,5 +1,5 @@ ## -## Copyright (c) 2020-2021 Intel Corporation. +## Copyright (c) 2020-2022 Intel Corporation. ## ## Licensed under the Apache License, Version 2.0 (the "License"); ## you may not use this file except in compliance with the License. @@ -55,12 +55,15 @@ - item.deploy - name: create secrets for kmra apps - command: >- - kubectl create secret generic {{ item.name }}-tls + shell: >- + set -o pipefail && kubectl create secret generic {{ item.name }}-tls --from-file=tls.cert={{ mtls_tmp_dir.path }}/{{ item.name }}.crt --from-file=tls.key={{ mtls_tmp_dir.path }}/{{ item.name }}.key --from-file=ca.cert={{ mtls_tmp_dir.path }}/ca.crt -n {{ kmra_namespace }} + -o yaml --dry-run=client | kubectl apply -f - + args: + executable: /bin/bash loop: "{{ kmra_apps }}" changed_when: true when: diff --git a/roles/kmra_install/tasks/main.yml b/roles/kmra_install/tasks/main.yml index cdd25d07..5cd82b6e 100644 --- a/roles/kmra_install/tasks/main.yml +++ b/roles/kmra_install/tasks/main.yml @@ -1,5 +1,5 @@ ## -## Copyright (c) 2020-2021 Intel Corporation. +## Copyright (c) 2020-2022 Intel Corporation. ## ## Licensed under the Apache License, Version 2.0 (the "License"); ## you may not use this file except in compliance with the License. @@ -17,191 +17,167 @@ - name: determine machine type include_role: name: check_machine_type - -- name: create sgx_prv group - group: - name: sgx_prv - state: present - when: - - inventory_hostname == groups['kube_node'][0] - - is_icx | default(false) | bool - - (ansible_distribution == "Ubuntu" and ansible_distribution_version == '21.04') - or (ansible_distribution == "RedHat" and ansible_distribution_version == '8.4') - -- name: add user to sgx_prv group - user: - name: "{{ ansible_user_id }}" - groups: sgx_prv - append: yes - when: - - inventory_hostname == groups['kube_node'][0] - - is_icx | default(false) | bool - - (ansible_distribution == "Ubuntu" and ansible_distribution_version == '21.04') - or (ansible_distribution == "RedHat" and ansible_distribution_version == '8.4') - -- name: create udev rules - blockinfile: - path: /etc/udev/rules.d/10-sgx.rules - create: yes - mode: '0644' - block: | - SUBSYSTEM=="misc",KERNEL=="enclave",MODE="0666" - SUBSYSTEM=="misc",KERNEL=="provision",GROUP="sgx_prv",MODE="0660" - SUBSYSTEM=="sgx",KERNEL=="sgx/enclave",MODE="0666" - SUBSYSTEM=="sgx",KERNEL=="sgx/provision",MODE="0660" - SUBSYSTEM=="misc",KERNEL=="sgx_enclave",MODE="0666",SYMLINK+="sgx/enclave" - SUBSYSTEM=="misc",KERNEL=="sgx_provision",GROUP="sgx_prv",MODE="0660",SYMLINK+="sgx/provision" when: - inventory_hostname == groups['kube_node'][0] - - is_icx | default(false) | bool - - (ansible_distribution == "Ubuntu" and ansible_distribution_version == '21.04') - or (ansible_distribution == "RedHat" and ansible_distribution_version == '8.4') -- name: load udev rules - # noqa 305 - shell is used intentionally here - shell: udevadm trigger - when: - - inventory_hostname == groups['kube_node'][0] - - is_icx | default(false) | bool - - (ansible_distribution == "Ubuntu" and ansible_distribution_version == '21.04') - or (ansible_distribution == "RedHat" and ansible_distribution_version == '8.4') - -- name: determine sgx_prv GID - getent: - database: group - key: sgx_prv +- name: prepare worker node + block: + - name: create sgx_prv group + group: + name: sgx_prv + state: present + when: + - (ansible_distribution == "Ubuntu" and ansible_distribution_version == '21.04') + or (ansible_os_family == "RedHat" and ansible_distribution_version >= '8.4') + + - name: add user to sgx_prv group + user: + name: "{{ ansible_user_id }}" + groups: sgx_prv + append: yes + when: + - (ansible_distribution == "Ubuntu" and ansible_distribution_version == '21.04') + or (ansible_os_family == "RedHat" and ansible_distribution_version >= '8.4') + + - name: create udev rules + blockinfile: + path: /etc/udev/rules.d/10-sgx.rules + create: yes + mode: '0644' + block: | + SUBSYSTEM=="misc",KERNEL=="enclave",MODE="0666" + SUBSYSTEM=="misc",KERNEL=="provision",GROUP="sgx_prv",MODE="0660" + SUBSYSTEM=="sgx",KERNEL=="sgx/enclave",MODE="0666" + SUBSYSTEM=="sgx",KERNEL=="sgx/provision",MODE="0660" + SUBSYSTEM=="misc",KERNEL=="sgx_enclave",MODE="0666",SYMLINK+="sgx/enclave" + SUBSYSTEM=="misc",KERNEL=="sgx_provision",GROUP="sgx_prv",MODE="0660",SYMLINK+="sgx/provision" + when: + - (ansible_distribution == "Ubuntu" and ansible_distribution_version == '21.04') + or (ansible_os_family == "RedHat" and ansible_distribution_version >= '8.4') + + - name: load udev rules + # noqa 305 - shell is used intentionally here + shell: udevadm trigger + when: + - (ansible_distribution == "Ubuntu" and ansible_distribution_version == '21.04') + or (ansible_os_family == "RedHat" and ansible_distribution_version >= '8.4') + + - name: determine sgx_prv GID + getent: + database: group + key: sgx_prv when: - inventory_hostname == groups['kube_node'][0] - - is_icx | default(false) | bool - -- name: install dependencies - include_role: - name: install_dependencies - when: is_icx | default(false) | bool - -- name: cleanup existing artifacts - include: cleanup.yml - when: - - inventory_hostname == groups['kube_control_plane'][0] - - is_icx | default(false) | bool - -- name: label worker node with KMRA label - command: kubectl label nodes {{ groups['kube_node'][0] }} app=kmra --overwrite - when: - - inventory_hostname == groups['kube_control_plane'][0] - - is_icx | default(false) | bool - -- name: create kmra ns if not existing - command: "kubectl create ns {{ kmra_namespace }}" - changed_when: true - failed_when: false - when: - - inventory_hostname == groups['kube_control_plane'][0] - - is_icx | default(false) | bool - -- name: create k8s tls secrets for apphsm and ctk apps - include: create_tls_secrets.yml - vars: - kmra_apps: - - { name: "{{ kmra_pccs_release_name }}", subj: "{{ kmra_pccs_crt_subj }}", deploy: true } - - { name: "{{ kmra_apphsm_release_name }}", subj: "{{ kmra_apphsm_crt_subj }}", deploy: true } - - { name: "{{ kmra_ctk_release_name }}", subj: "{{ kmra_ctk_crt_subj }}", deploy: "{{ kmra_deploy_demo_workload | default(false) }}" } - when: - - inventory_hostname == groups['kube_control_plane'][0] - - is_icx | default(false) | bool - -- name: create Helm charts directory if needed - file: - path: /usr/src/charts - state: directory - mode: 0755 - when: - - inventory_hostname == groups['kube_control_plane'][0] - - is_icx | default(false) | bool - -- name: copy KMRA Helm chart to the controller node - copy: - src: "{{ role_path }}/charts/{{ item.chart }}" - dest: "/usr/src/charts/" - mode: 0755 - loop: - - { chart: 'kmra-pccs', deploy: true } - - { chart: 'kmra-apphsm', deploy: true } - - { chart: 'kmra-ctk', deploy: "{{ kmra_deploy_demo_workload | default(false) }}" } - when: - - inventory_hostname == groups['kube_control_plane'][0] - - is_icx | default(false) | bool - - item.deploy - -- name: populate KMRA Helm charts values templates and push to controller node - template: - src: "{{ item.src }}" - dest: "{{ item.dest }}" - force: yes - mode: preserve - loop: - - { - src: 'kmra-pccs-values.yaml.j2', - dest: '/usr/src/charts/kmra-pccs-values.yml', - deploy: true - } - - { - src: 'kmra-pccs-rbac-cluster-role.yml.j2', - dest: "{{ kmra_pccs_chart_path }}/templates/kmra-pccs-rbac-cluster-role.yml", - deploy: true - } - - { - src: 'kmra-apphsm-values.yaml.j2', - dest: '/usr/src/charts/kmra-apphsm-values.yml', - deploy: true - } - - { - src: 'kmra-apphsm-rbac-cluster-role.yml.j2', - dest: "{{ kmra_apphsm_chart_path }}/templates/kmra-apphsm-rbac-cluster-role.yml", - deploy: true - } - - { - src: 'kmra-ctk-values.yaml.j2', - dest: '/usr/src/charts/kmra-ctk-values.yml', - deploy: "{{ kmra_deploy_demo_workload | default(false) }}" - } - - { - src: 'kmra-ctk-loadkey-rbac-cluster-role.yml.j2', - dest: "{{ kmra_ctk_chart_path }}/templates/kmra-ctk-loadkey-rbac-cluster-role.yml", - deploy: "{{ kmra_deploy_demo_workload | default(false) }}" - } - when: - - inventory_hostname == groups['kube_control_plane'][0] - - is_icx | default(false) | bool - - item.deploy - -- name: install KMRA pccs helm chart - command: >- - helm upgrade -i {{ kmra_pccs_release_name }} - --namespace {{ kmra_namespace }} - -f {{ kmra_pccs_helm_values_file }} - {{ kmra_pccs_chart_path }} - when: - - inventory_hostname == groups['kube_control_plane'][0] - - is_icx | default(false) | bool - -- name: install KMRA AppHSM helm chart - command: >- - helm upgrade -i {{ kmra_apphsm_release_name }} - --namespace {{ kmra_namespace }} - -f {{ kmra_apphsm_helm_values_file }} - {{ kmra_apphsm_chart_path }} - when: - - inventory_hostname == groups['kube_control_plane'][0] - - is_icx | default(false) | bool - -- name: install KMRA Ctk loadkey helm chart - command: >- - helm upgrade -i {{ kmra_ctk_release_name }} - --namespace {{ kmra_namespace }} - -f {{ kmra_ctk_helm_values_file }} - {{ kmra_ctk_chart_path }} + - is_icx | default(false) | bool or + is_spr | default(false) | bool + +- name: prepare and deploy kmra + block: + - name: install dependencies + include_role: + name: install_dependencies + + - name: cleanup existing artifacts + include: cleanup.yml + + - name: label worker node with KMRA label + command: kubectl label nodes {{ groups['kube_node'][0] }} app=kmra --overwrite + + - name: create kmra ns if not existing + shell: "set -o pipefail && kubectl create ns {{ kmra_namespace }} -o yaml --dry-run=client | kubectl apply -f -" + args: + executable: /bin/bash + changed_when: true + + - name: create k8s tls secrets for apphsm and ctk apps + include: create_tls_secrets.yml + vars: + kmra_apps: + - { name: "{{ kmra_pccs_release_name }}", subj: "{{ kmra_pccs_crt_subj }}", deploy: true } + - { name: "{{ kmra_apphsm_release_name }}", subj: "{{ kmra_apphsm_crt_subj }}", deploy: true } + - { name: "{{ kmra_ctk_release_name }}", subj: "{{ kmra_ctk_crt_subj }}", deploy: "{{ kmra_deploy_demo_workload | default(false) }}" } + + - name: create Helm charts directory if needed + file: + path: "{{ project_root_dir }}/charts" + state: directory + mode: 0755 + + - name: copy KMRA Helm chart to the controller node + copy: + src: "{{ role_path }}/charts/{{ item.chart }}" + dest: "{{ project_root_dir }}/charts" + mode: 0755 + loop: + - { chart: 'kmra-pccs', deploy: true } + - { chart: 'kmra-apphsm', deploy: true } + - { chart: 'kmra-ctk', deploy: "{{ kmra_deploy_demo_workload | default(false) }}" } + when: + - item.deploy + + - name: populate KMRA Helm charts values templates and push to controller node + template: + src: "{{ item.src }}" + dest: "{{ item.dest }}" + force: yes + mode: preserve + loop: + - { + src: 'kmra-pccs-values.yaml.j2', + dest: '{{ project_root_dir }}/charts/kmra-pccs-values.yml', + deploy: true + } + - { + src: 'kmra-pccs-rbac-cluster-role.yml.j2', + dest: "{{ kmra_pccs_chart_path }}/templates/kmra-pccs-rbac-cluster-role.yml", + deploy: true + } + - { + src: 'kmra-apphsm-values.yaml.j2', + dest: '{{ project_root_dir }}/charts/kmra-apphsm-values.yml', + deploy: true + } + - { + src: 'kmra-apphsm-rbac-cluster-role.yml.j2', + dest: "{{ kmra_apphsm_chart_path }}/templates/kmra-apphsm-rbac-cluster-role.yml", + deploy: true + } + - { + src: 'kmra-ctk-values.yaml.j2', + dest: '{{ project_root_dir }}/charts/kmra-ctk-values.yml', + deploy: "{{ kmra_deploy_demo_workload | default(false) }}" + } + - { + src: 'kmra-ctk-loadkey-rbac-cluster-role.yml.j2', + dest: "{{ kmra_ctk_chart_path }}/templates/kmra-ctk-loadkey-rbac-cluster-role.yml", + deploy: "{{ kmra_deploy_demo_workload | default(false) }}" + } + when: + - item.deploy + + - name: install KMRA pccs helm chart + command: >- + helm upgrade -i {{ kmra_pccs_release_name }} + --namespace {{ kmra_namespace }} + -f {{ kmra_pccs_helm_values_file }} + {{ kmra_pccs_chart_path }} + + - name: install KMRA AppHSM helm chart + command: >- + helm upgrade -i {{ kmra_apphsm_release_name }} + --namespace {{ kmra_namespace }} + -f {{ kmra_apphsm_helm_values_file }} + {{ kmra_apphsm_chart_path }} + + - name: install KMRA Ctk loadkey helm chart + command: >- + helm upgrade -i {{ kmra_ctk_release_name }} + --namespace {{ kmra_namespace }} + -f {{ kmra_ctk_helm_values_file }} + {{ kmra_ctk_chart_path }} + when: + - kmra_deploy_demo_workload | default(false) when: - inventory_hostname == groups['kube_control_plane'][0] - - is_icx | default(false) | bool - - kmra_deploy_demo_workload | default(false) + - hostvars[groups['kube_node'][0]]['is_icx'] or + hostvars[groups['kube_node'][0]]['is_spr'] diff --git a/roles/kmra_install/templates/kmra-apphsm-values.yaml.j2 b/roles/kmra_install/templates/kmra-apphsm-values.yaml.j2 index 3bc43464..2aa5bc44 100644 --- a/roles/kmra_install/templates/kmra-apphsm-values.yaml.j2 +++ b/roles/kmra_install/templates/kmra-apphsm-values.yaml.j2 @@ -24,6 +24,20 @@ apphsm: tag: "{{ kmra_apphsm_rproxy_tag}}" pullPolicy: "IfNotPresent" port: "{{ kmra_apphsm_service_port }}" + init: + image: + repo: "{{ kmra_apphsm_init_image_repo }}" + name: "{{ kmra_apphsm_init_image_name }}" + tag: "{{ kmra_apphsm_init_image_tag }}" + pullPolicy: "IfNotPresent" pccs_port: "{{ kmra_pccs_service_port }}" pccs_hostname: "{{ kmra_pccs_release_name }}.{{ kmra_namespace }}.svc.{{ cluster_name }}" sgx_prv_gid: "{{ hostvars[groups['kube_node'][0]]['getent_group']['sgx_prv'][1] | default('1002')}}" + use_secure_cert: "{{ kmra_apphsm_use_secure_cert | quote }}" + apphsm_key_in_token_name: "{{ kmra_apphsm_key_in_token_name }}" + apphsm_token_name: "{{ kmra_apphsm_token_name }}" + test_ctk_loadkey_cert_user_id: "{{ kmra_apphsm_test_ctk_loadkey_cert_user_id }}" + test_unique_uid: "{{ kmra_apphsm_test_unique_uid }}" + default_user_pin: "{{ kmra_apphsm_default_user_pin }}" + default_so_pin: "{{ kmra_apphsm_default_so_pin }}" + diff --git a/roles/kmra_install/templates/kmra-ctk-values.yaml.j2 b/roles/kmra_install/templates/kmra-ctk-values.yaml.j2 index 4555a928..4d53b40e 100644 --- a/roles/kmra_install/templates/kmra-ctk-values.yaml.j2 +++ b/roles/kmra_install/templates/kmra-ctk-values.yaml.j2 @@ -18,9 +18,21 @@ ctk_loadkey: pullPolicy: IfNotPresent port: "{{ kmra_ctk_loadkey_upstream_port }}" hostname: "{{ kmra_ctk_loadkey_upstream_server_name | default('0.0.0.0')}}" + init: + image: + repo: "{{ kmra_ctk_loadkey_init_image_repo }}" + name: "{{ kmra_ctk_loadkey_init_image_name }}" + tag: "{{ kmra_ctk_loadkey_init_image_tag }}" + pullPolicy: IfNotPresent node_port: "{{ kmra_ctk_loadkey_node_port }}" pccs_port: "{{ kmra_pccs_service_port }}" pccs_hostname: "{{ kmra_pccs_release_name }}.{{ kmra_namespace }}.svc.{{ cluster_name }}" apphsm_port: "{{ kmra_apphsm_service_port }}" apphsm_hostname: "{{ kmra_apphsm_release_name }}.{{ kmra_namespace }}.svc.{{ cluster_name }}" sgx_prv_gid: "{{ hostvars[groups['kube_node'][0]]['getent_group']['sgx_prv'][1] | default('1002')}}" + use_secure_cert: "{{ kmra_ctk_loadkey_use_secure_cert | quote }}" + client_token: "{{ kmra_ctk_loadkey_client_token }}" + client_key_label: "{{kmra_ctk_loadkey_client_key_label }}" + test_unique_uid: "{{ kmra_ctk_loadkey_test_unique_uid }}" + default_user_pin: "{{ kmra_ctk_loadkey_default_user_pin }}" + default_so_pin: "{{ kmra_ctk_loadkey_default_so_pin }}" diff --git a/roles/kmra_install/vars/main.yml b/roles/kmra_install/vars/main.yml index 1d9eedca..cf5312f5 100644 --- a/roles/kmra_install/vars/main.yml +++ b/roles/kmra_install/vars/main.yml @@ -1,5 +1,5 @@ ## -## Copyright (c) 2020-2021 Intel Corporation. +## Copyright (c) 2020-2022 Intel Corporation. ## ## Licensed under the Apache License, Version 2.0 (the "License"); ## you may not use this file except in compliance with the License. diff --git a/roles/kube_prometheus/defaults/main.yml b/roles/kube_prometheus/defaults/main.yml index 2cf3ffd3..9ca83bb8 100644 --- a/roles/kube_prometheus/defaults/main.yml +++ b/roles/kube_prometheus/defaults/main.yml @@ -1,5 +1,5 @@ ## -## Copyright (c) 2020-2021 Intel Corporation. +## Copyright (c) 2020-2022 Intel Corporation. ## ## Licensed under the Apache License, Version 2.0 (the "License"); ## you may not use this file except in compliance with the License. @@ -24,7 +24,7 @@ prometheus_srv_expose: false prometheus_srv_proxy_port: 9443 prometheus_srv_node_port: 30443 prometheus_srv_address: 127.0.0.1 -prometheus_srv_nginx_image: "docker.io/library/nginx:1.20.1-alpine" +prometheus_srv_nginx_image: "docker.io/library/nginx:1.21.5-alpine" prometheus_srv_nginx_ssl_ciphers: "AES128-CCM-SHA256:CHACHA20-POLY1305-SHA256:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES128-GCM-SHA256" prometheus_srv_nginx_ssl_protocols: "TLSv1.2 TLSv1.3" prometheus_srv_location_exposed: "/prometheus/" diff --git a/roles/kube_prometheus/files/dashboards/grafana-dashboardDefinitions.yaml b/roles/kube_prometheus/files/dashboards/grafana-dashboardDefinitions.yaml index 363d7e9c..2a143d1e 100644 --- a/roles/kube_prometheus/files/dashboards/grafana-dashboardDefinitions.yaml +++ b/roles/kube_prometheus/files/dashboards/grafana-dashboardDefinitions.yaml @@ -1737,7 +1737,7 @@ items: app.kubernetes.io/component: grafana app.kubernetes.io/name: grafana app.kubernetes.io/part-of: kube-prometheus - app.kubernetes.io/version: 8.1.2 + app.kubernetes.io/version: 8.3.4 - apiVersion: v1 data: cluster-total.json: |- @@ -3608,7 +3608,7 @@ items: app.kubernetes.io/component: grafana app.kubernetes.io/name: grafana app.kubernetes.io/part-of: kube-prometheus - app.kubernetes.io/version: 8.1.2 + app.kubernetes.io/version: 8.3.4 - apiVersion: v1 data: controller-manager.json: |- @@ -4748,7 +4748,7 @@ items: app.kubernetes.io/component: grafana app.kubernetes.io/name: grafana app.kubernetes.io/part-of: kube-prometheus - app.kubernetes.io/version: 8.1.2 + app.kubernetes.io/version: 8.3.4 - apiVersion: v1 data: k8s-resources-cluster.json: |- @@ -7319,7 +7319,7 @@ items: app.kubernetes.io/component: grafana app.kubernetes.io/name: grafana app.kubernetes.io/part-of: kube-prometheus - app.kubernetes.io/version: 8.1.2 + app.kubernetes.io/version: 8.3.4 - apiVersion: v1 data: k8s-resources-namespace.json: |- @@ -9594,7 +9594,7 @@ items: app.kubernetes.io/component: grafana app.kubernetes.io/name: grafana app.kubernetes.io/part-of: kube-prometheus - app.kubernetes.io/version: 8.1.2 + app.kubernetes.io/version: 8.3.4 - apiVersion: v1 data: k8s-resources-node.json: |- @@ -10561,7 +10561,7 @@ items: app.kubernetes.io/component: grafana app.kubernetes.io/name: grafana app.kubernetes.io/part-of: kube-prometheus - app.kubernetes.io/version: 8.1.2 + app.kubernetes.io/version: 8.3.4 - apiVersion: v1 data: k8s-resources-pod.json: |- @@ -12322,7 +12322,7 @@ items: app.kubernetes.io/component: grafana app.kubernetes.io/name: grafana app.kubernetes.io/part-of: kube-prometheus - app.kubernetes.io/version: 8.1.2 + app.kubernetes.io/version: 8.3.4 - apiVersion: v1 data: k8s-resources-workload.json: |- @@ -14345,7 +14345,7 @@ items: app.kubernetes.io/component: grafana app.kubernetes.io/name: grafana app.kubernetes.io/part-of: kube-prometheus - app.kubernetes.io/version: 8.1.2 + app.kubernetes.io/version: 8.3.4 - apiVersion: v1 data: k8s-resources-workloads-namespace.json: |- @@ -16529,7 +16529,7 @@ items: app.kubernetes.io/component: grafana app.kubernetes.io/name: grafana app.kubernetes.io/part-of: kube-prometheus - app.kubernetes.io/version: 8.1.2 + app.kubernetes.io/version: 8.3.4 - apiVersion: v1 data: kubelet.json: |- @@ -19051,7 +19051,7 @@ items: app.kubernetes.io/component: grafana app.kubernetes.io/name: grafana app.kubernetes.io/part-of: kube-prometheus - app.kubernetes.io/version: 8.1.2 + app.kubernetes.io/version: 8.3.4 - apiVersion: v1 data: namespace-by-pod.json: |- @@ -20504,7 +20504,7 @@ items: app.kubernetes.io/component: grafana app.kubernetes.io/name: grafana app.kubernetes.io/part-of: kube-prometheus - app.kubernetes.io/version: 8.1.2 + app.kubernetes.io/version: 8.3.4 - apiVersion: v1 data: namespace-by-workload.json: |- @@ -22229,7 +22229,7 @@ items: app.kubernetes.io/component: grafana app.kubernetes.io/name: grafana app.kubernetes.io/part-of: kube-prometheus - app.kubernetes.io/version: 8.1.2 + app.kubernetes.io/version: 8.3.4 - apiVersion: v1 data: node-cluster-rsrc-use.json: |- @@ -23182,7 +23182,7 @@ items: app.kubernetes.io/component: grafana app.kubernetes.io/name: grafana app.kubernetes.io/part-of: kube-prometheus - app.kubernetes.io/version: 8.1.2 + app.kubernetes.io/version: 8.3.4 - apiVersion: v1 data: node-rsrc-use.json: |- @@ -24162,7 +24162,7 @@ items: app.kubernetes.io/component: grafana app.kubernetes.io/name: grafana app.kubernetes.io/part-of: kube-prometheus - app.kubernetes.io/version: 8.1.2 + app.kubernetes.io/version: 8.3.4 - apiVersion: v1 data: nodes.json: |- @@ -25152,7 +25152,7 @@ items: app.kubernetes.io/component: grafana app.kubernetes.io/name: grafana app.kubernetes.io/part-of: kube-prometheus - app.kubernetes.io/version: 8.1.2 + app.kubernetes.io/version: 8.3.4 - apiVersion: v1 data: persistentvolumesusage.json: |- @@ -25718,7 +25718,7 @@ items: app.kubernetes.io/component: grafana app.kubernetes.io/name: grafana app.kubernetes.io/part-of: kube-prometheus - app.kubernetes.io/version: 8.1.2 + app.kubernetes.io/version: 8.3.4 - apiVersion: v1 data: pod-total.json: |- @@ -26935,7 +26935,7 @@ items: app.kubernetes.io/component: grafana app.kubernetes.io/name: grafana app.kubernetes.io/part-of: kube-prometheus - app.kubernetes.io/version: 8.1.2 + app.kubernetes.io/version: 8.3.4 - apiVersion: v1 data: prometheus-remote-write.json: |- @@ -28594,7 +28594,7 @@ items: app.kubernetes.io/component: grafana app.kubernetes.io/name: grafana app.kubernetes.io/part-of: kube-prometheus - app.kubernetes.io/version: 8.1.2 + app.kubernetes.io/version: 8.3.4 - apiVersion: v1 data: prometheus.json: |- @@ -29810,7 +29810,7 @@ items: app.kubernetes.io/component: grafana app.kubernetes.io/name: grafana app.kubernetes.io/part-of: kube-prometheus - app.kubernetes.io/version: 8.1.2 + app.kubernetes.io/version: 8.3.4 - apiVersion: v1 data: proxy.json: |- @@ -31030,7 +31030,7 @@ items: app.kubernetes.io/component: grafana app.kubernetes.io/name: grafana app.kubernetes.io/part-of: kube-prometheus - app.kubernetes.io/version: 8.1.2 + app.kubernetes.io/version: 8.3.4 - apiVersion: v1 data: scheduler.json: |- @@ -32093,7 +32093,7 @@ items: app.kubernetes.io/component: grafana app.kubernetes.io/name: grafana app.kubernetes.io/part-of: kube-prometheus - app.kubernetes.io/version: 8.1.2 + app.kubernetes.io/version: 8.3.4 - apiVersion: v1 data: statefulset.json: |- @@ -33010,7 +33010,7 @@ items: app.kubernetes.io/component: grafana app.kubernetes.io/name: grafana app.kubernetes.io/part-of: kube-prometheus - app.kubernetes.io/version: 8.1.2 + app.kubernetes.io/version: 8.3.4 - apiVersion: v1 data: workload-total.json: |- @@ -34437,7 +34437,7 @@ items: app.kubernetes.io/component: grafana app.kubernetes.io/name: grafana app.kubernetes.io/part-of: kube-prometheus - app.kubernetes.io/version: 8.1.2 + app.kubernetes.io/version: 8.3.4 - apiVersion: v1 data: collectd-cpu.json: |- @@ -35531,7 +35531,7 @@ items: app.kubernetes.io/component: grafana app.kubernetes.io/name: grafana app.kubernetes.io/part-of: kube-prometheus - app.kubernetes.io/version: 8.1.2 + app.kubernetes.io/version: 8.3.4 - apiVersion: v1 data: collectd-disk.json: |- @@ -36621,7 +36621,7 @@ items: app.kubernetes.io/component: grafana app.kubernetes.io/name: grafana app.kubernetes.io/part-of: kube-prometheus - app.kubernetes.io/version: 8.1.2 + app.kubernetes.io/version: 8.3.4 - apiVersion: v1 data: collectd-intel.json: |- @@ -37119,7 +37119,7 @@ items: app.kubernetes.io/component: grafana app.kubernetes.io/name: grafana app.kubernetes.io/part-of: kube-prometheus - app.kubernetes.io/version: 8.1.2 + app.kubernetes.io/version: 8.3.4 - apiVersion: v1 data: collectd-ipmi.json: |- @@ -37562,7 +37562,7 @@ items: app.kubernetes.io/component: grafana app.kubernetes.io/name: grafana app.kubernetes.io/part-of: kube-prometheus - app.kubernetes.io/version: 8.1.2 + app.kubernetes.io/version: 8.3.4 - apiVersion: v1 data: collectd-netlink.json: |- @@ -39115,7 +39115,7 @@ items: app.kubernetes.io/component: grafana app.kubernetes.io/name: grafana app.kubernetes.io/part-of: kube-prometheus - app.kubernetes.io/version: 8.1.2 + app.kubernetes.io/version: 8.3.4 - apiVersion: v1 data: collectd-ovs.json: |- @@ -40335,7 +40335,7 @@ items: app.kubernetes.io/component: grafana app.kubernetes.io/name: grafana app.kubernetes.io/part-of: kube-prometheus - app.kubernetes.io/version: 8.1.2 + app.kubernetes.io/version: 8.3.4 - apiVersion: v1 data: collectd-power.json: |- @@ -40889,7 +40889,7 @@ items: app.kubernetes.io/component: grafana app.kubernetes.io/name: grafana app.kubernetes.io/part-of: kube-prometheus - app.kubernetes.io/version: 8.1.2 + app.kubernetes.io/version: 8.3.4 - apiVersion: v1 data: collectd-numa.json: |- @@ -41665,7 +41665,7 @@ items: app.kubernetes.io/component: grafana app.kubernetes.io/name: grafana app.kubernetes.io/part-of: kube-prometheus - app.kubernetes.io/version: 8.1.2 + app.kubernetes.io/version: 8.3.4 - apiVersion: v1 data: collectd-hugepages.json: |- @@ -42001,7 +42001,7 @@ items: app.kubernetes.io/component: grafana app.kubernetes.io/name: grafana app.kubernetes.io/part-of: kube-prometheus - app.kubernetes.io/version: 8.1.2 + app.kubernetes.io/version: 8.3.4 - apiVersion: v1 data: collectd-ethstats.json: |- @@ -42777,5 +42777,5 @@ items: app.kubernetes.io/component: grafana app.kubernetes.io/name: grafana app.kubernetes.io/part-of: kube-prometheus - app.kubernetes.io/version: 8.1.2 + app.kubernetes.io/version: 8.3.4 kind: ConfigMapList diff --git a/roles/kube_prometheus/files/dashboards/grafana-telegraf-dashboardDefinitions.yaml b/roles/kube_prometheus/files/dashboards/grafana-telegraf-dashboardDefinitions.yaml index 6fd76782..61407291 100644 --- a/roles/kube_prometheus/files/dashboards/grafana-telegraf-dashboardDefinitions.yaml +++ b/roles/kube_prometheus/files/dashboards/grafana-telegraf-dashboardDefinitions.yaml @@ -1737,7 +1737,7 @@ items: app.kubernetes.io/component: grafana app.kubernetes.io/name: grafana app.kubernetes.io/part-of: kube-prometheus - app.kubernetes.io/version: 8.1.2 + app.kubernetes.io/version: 8.3.4 - apiVersion: v1 data: cluster-total.json: |- @@ -3608,7 +3608,7 @@ items: app.kubernetes.io/component: grafana app.kubernetes.io/name: grafana app.kubernetes.io/part-of: kube-prometheus - app.kubernetes.io/version: 8.1.2 + app.kubernetes.io/version: 8.3.4 - apiVersion: v1 data: controller-manager.json: |- @@ -4748,7 +4748,7 @@ items: app.kubernetes.io/component: grafana app.kubernetes.io/name: grafana app.kubernetes.io/part-of: kube-prometheus - app.kubernetes.io/version: 8.1.2 + app.kubernetes.io/version: 8.3.4 - apiVersion: v1 data: k8s-resources-cluster.json: |- @@ -7319,7 +7319,7 @@ items: app.kubernetes.io/component: grafana app.kubernetes.io/name: grafana app.kubernetes.io/part-of: kube-prometheus - app.kubernetes.io/version: 8.1.2 + app.kubernetes.io/version: 8.3.4 - apiVersion: v1 data: k8s-resources-namespace.json: |- @@ -9594,7 +9594,7 @@ items: app.kubernetes.io/component: grafana app.kubernetes.io/name: grafana app.kubernetes.io/part-of: kube-prometheus - app.kubernetes.io/version: 8.1.2 + app.kubernetes.io/version: 8.3.4 - apiVersion: v1 data: k8s-resources-node.json: |- @@ -10561,7 +10561,7 @@ items: app.kubernetes.io/component: grafana app.kubernetes.io/name: grafana app.kubernetes.io/part-of: kube-prometheus - app.kubernetes.io/version: 8.1.2 + app.kubernetes.io/version: 8.3.4 - apiVersion: v1 data: k8s-resources-pod.json: |- @@ -12322,7 +12322,7 @@ items: app.kubernetes.io/component: grafana app.kubernetes.io/name: grafana app.kubernetes.io/part-of: kube-prometheus - app.kubernetes.io/version: 8.1.2 + app.kubernetes.io/version: 8.3.4 - apiVersion: v1 data: k8s-resources-workload.json: |- @@ -14345,7 +14345,7 @@ items: app.kubernetes.io/component: grafana app.kubernetes.io/name: grafana app.kubernetes.io/part-of: kube-prometheus - app.kubernetes.io/version: 8.1.2 + app.kubernetes.io/version: 8.3.4 - apiVersion: v1 data: k8s-resources-workloads-namespace.json: |- @@ -16529,7 +16529,7 @@ items: app.kubernetes.io/component: grafana app.kubernetes.io/name: grafana app.kubernetes.io/part-of: kube-prometheus - app.kubernetes.io/version: 8.1.2 + app.kubernetes.io/version: 8.3.4 - apiVersion: v1 data: kubelet.json: |- @@ -19051,7 +19051,7 @@ items: app.kubernetes.io/component: grafana app.kubernetes.io/name: grafana app.kubernetes.io/part-of: kube-prometheus - app.kubernetes.io/version: 8.1.2 + app.kubernetes.io/version: 8.3.4 - apiVersion: v1 data: namespace-by-pod.json: |- @@ -20504,7 +20504,7 @@ items: app.kubernetes.io/component: grafana app.kubernetes.io/name: grafana app.kubernetes.io/part-of: kube-prometheus - app.kubernetes.io/version: 8.1.2 + app.kubernetes.io/version: 8.3.4 - apiVersion: v1 data: namespace-by-workload.json: |- @@ -22229,7 +22229,7 @@ items: app.kubernetes.io/component: grafana app.kubernetes.io/name: grafana app.kubernetes.io/part-of: kube-prometheus - app.kubernetes.io/version: 8.1.2 + app.kubernetes.io/version: 8.3.4 - apiVersion: v1 data: node-cluster-rsrc-use.json: |- @@ -23182,7 +23182,7 @@ items: app.kubernetes.io/component: grafana app.kubernetes.io/name: grafana app.kubernetes.io/part-of: kube-prometheus - app.kubernetes.io/version: 8.1.2 + app.kubernetes.io/version: 8.3.4 - apiVersion: v1 data: node-rsrc-use.json: |- @@ -24162,7 +24162,7 @@ items: app.kubernetes.io/component: grafana app.kubernetes.io/name: grafana app.kubernetes.io/part-of: kube-prometheus - app.kubernetes.io/version: 8.1.2 + app.kubernetes.io/version: 8.3.4 - apiVersion: v1 data: nodes.json: |- @@ -25152,7 +25152,7 @@ items: app.kubernetes.io/component: grafana app.kubernetes.io/name: grafana app.kubernetes.io/part-of: kube-prometheus - app.kubernetes.io/version: 8.1.2 + app.kubernetes.io/version: 8.3.4 - apiVersion: v1 data: persistentvolumesusage.json: |- @@ -25718,7 +25718,7 @@ items: app.kubernetes.io/component: grafana app.kubernetes.io/name: grafana app.kubernetes.io/part-of: kube-prometheus - app.kubernetes.io/version: 8.1.2 + app.kubernetes.io/version: 8.3.4 - apiVersion: v1 data: pod-total.json: |- @@ -26935,7 +26935,7 @@ items: app.kubernetes.io/component: grafana app.kubernetes.io/name: grafana app.kubernetes.io/part-of: kube-prometheus - app.kubernetes.io/version: 8.1.2 + app.kubernetes.io/version: 8.3.4 - apiVersion: v1 data: prometheus-remote-write.json: |- @@ -28594,7 +28594,7 @@ items: app.kubernetes.io/component: grafana app.kubernetes.io/name: grafana app.kubernetes.io/part-of: kube-prometheus - app.kubernetes.io/version: 8.1.2 + app.kubernetes.io/version: 8.3.4 - apiVersion: v1 data: prometheus.json: |- @@ -29810,7 +29810,7 @@ items: app.kubernetes.io/component: grafana app.kubernetes.io/name: grafana app.kubernetes.io/part-of: kube-prometheus - app.kubernetes.io/version: 8.1.2 + app.kubernetes.io/version: 8.3.4 - apiVersion: v1 data: proxy.json: |- @@ -31030,7 +31030,7 @@ items: app.kubernetes.io/component: grafana app.kubernetes.io/name: grafana app.kubernetes.io/part-of: kube-prometheus - app.kubernetes.io/version: 8.1.2 + app.kubernetes.io/version: 8.3.4 - apiVersion: v1 data: scheduler.json: |- @@ -32093,7 +32093,7 @@ items: app.kubernetes.io/component: grafana app.kubernetes.io/name: grafana app.kubernetes.io/part-of: kube-prometheus - app.kubernetes.io/version: 8.1.2 + app.kubernetes.io/version: 8.3.4 - apiVersion: v1 data: statefulset.json: |- @@ -33010,7 +33010,7 @@ items: app.kubernetes.io/component: grafana app.kubernetes.io/name: grafana app.kubernetes.io/part-of: kube-prometheus - app.kubernetes.io/version: 8.1.2 + app.kubernetes.io/version: 8.3.4 - apiVersion: v1 data: workload-total.json: |- @@ -34437,7 +34437,7 @@ items: app.kubernetes.io/component: grafana app.kubernetes.io/name: grafana app.kubernetes.io/part-of: kube-prometheus - app.kubernetes.io/version: 8.1.2 + app.kubernetes.io/version: 8.3.4 - apiVersion: v1 data: collectd-cpu.json: |- @@ -36386,7 +36386,7 @@ items: app.kubernetes.io/component: grafana app.kubernetes.io/name: grafana app.kubernetes.io/part-of: kube-prometheus - app.kubernetes.io/version: 8.1.2 + app.kubernetes.io/version: 8.3.4 - apiVersion: v1 data: collectd-disk.json: |- @@ -37476,7 +37476,7 @@ items: app.kubernetes.io/component: grafana app.kubernetes.io/name: grafana app.kubernetes.io/part-of: kube-prometheus - app.kubernetes.io/version: 8.1.2 + app.kubernetes.io/version: 8.3.4 - apiVersion: v1 data: collectd-intel.json: |- @@ -37974,7 +37974,7 @@ items: app.kubernetes.io/component: grafana app.kubernetes.io/name: grafana app.kubernetes.io/part-of: kube-prometheus - app.kubernetes.io/version: 8.1.2 + app.kubernetes.io/version: 8.3.4 - apiVersion: v1 data: collectd-ipmi.json: |- @@ -38417,7 +38417,7 @@ items: app.kubernetes.io/component: grafana app.kubernetes.io/name: grafana app.kubernetes.io/part-of: kube-prometheus - app.kubernetes.io/version: 8.1.2 + app.kubernetes.io/version: 8.3.4 - apiVersion: v1 data: collectd-netlink.json: |- @@ -38645,7 +38645,7 @@ items: "yaxes": [ { "format": "bytes", - "label": null, + "label": "Bytes", "logBase": 1, "max": null, "min": null, @@ -38866,8 +38866,8 @@ items: }, "yaxes": [ { - "format": "pps", - "label": "Packets/s", + "format": "none", + "label": "Packets", "logBase": 1, "max": null, "min": null, @@ -39415,7 +39415,7 @@ items: app.kubernetes.io/component: grafana app.kubernetes.io/name: grafana app.kubernetes.io/part-of: kube-prometheus - app.kubernetes.io/version: 8.1.2 + app.kubernetes.io/version: 8.3.4 - apiVersion: v1 data: collectd-power.json: |- @@ -39858,7 +39858,7 @@ items: app.kubernetes.io/component: grafana app.kubernetes.io/name: grafana app.kubernetes.io/part-of: kube-prometheus - app.kubernetes.io/version: 8.1.2 + app.kubernetes.io/version: 8.3.4 - apiVersion: v1 data: collectd-numa.json: |- @@ -41189,7 +41189,7 @@ items: app.kubernetes.io/component: grafana app.kubernetes.io/name: grafana app.kubernetes.io/part-of: kube-prometheus - app.kubernetes.io/version: 8.1.2 + app.kubernetes.io/version: 8.3.4 - apiVersion: v1 data: collectd-hugepages.json: |- @@ -41525,7 +41525,7 @@ items: app.kubernetes.io/component: grafana app.kubernetes.io/name: grafana app.kubernetes.io/part-of: kube-prometheus - app.kubernetes.io/version: 8.1.2 + app.kubernetes.io/version: 8.3.4 - apiVersion: v1 data: collectd-ethstats.json: |- @@ -42301,5 +42301,5 @@ items: app.kubernetes.io/component: grafana app.kubernetes.io/name: grafana app.kubernetes.io/part-of: kube-prometheus - app.kubernetes.io/version: 8.1.2 + app.kubernetes.io/version: 8.3.4 kind: ConfigMapList diff --git a/roles/kube_prometheus/files/kube-prometheus/grafana-dashboardDatasources.yaml b/roles/kube_prometheus/files/kube-prometheus/grafana-dashboardDatasources.yaml index 884210de..cdd6e694 100644 --- a/roles/kube_prometheus/files/kube-prometheus/grafana-dashboardDatasources.yaml +++ b/roles/kube_prometheus/files/kube-prometheus/grafana-dashboardDatasources.yaml @@ -13,5 +13,5 @@ metadata: app.kubernetes.io/component: grafana app.kubernetes.io/name: grafana app.kubernetes.io/part-of: kube-prometheus - app.kubernetes.io/version: 8.1.2 + app.kubernetes.io/version: 8.3.4 type: Opaque diff --git a/roles/kube_prometheus/files/kube-prometheus/grafana-dashboardSources.yaml b/roles/kube_prometheus/files/kube-prometheus/grafana-dashboardSources.yaml index 24f43356..425be93d 100644 --- a/roles/kube_prometheus/files/kube-prometheus/grafana-dashboardSources.yaml +++ b/roles/kube_prometheus/files/kube-prometheus/grafana-dashboardSources.yaml @@ -27,4 +27,4 @@ metadata: app.kubernetes.io/component: grafana app.kubernetes.io/name: grafana app.kubernetes.io/part-of: kube-prometheus - app.kubernetes.io/version: 8.1.2 + app.kubernetes.io/version: 8.3.4 diff --git a/roles/kube_prometheus/files/kube-prometheus/node-exporter-clusterRoleBinding.yaml b/roles/kube_prometheus/files/kube-prometheus/node-exporter-clusterRoleBinding.yaml index 24b81179..788c8df1 100644 --- a/roles/kube_prometheus/files/kube-prometheus/node-exporter-clusterRoleBinding.yaml +++ b/roles/kube_prometheus/files/kube-prometheus/node-exporter-clusterRoleBinding.yaml @@ -10,7 +10,7 @@ metadata: app.kubernetes.io/component: exporter app.kubernetes.io/name: node-exporter app.kubernetes.io/part-of: kube-prometheus - app.kubernetes.io/version: 1.2.2 + app.kubernetes.io/version: 1.3.1 roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole diff --git a/roles/kube_prometheus/files/kube-prometheus/node-exporter-service.yaml b/roles/kube_prometheus/files/kube-prometheus/node-exporter-service.yaml index 3b48225a..f7fddb53 100644 --- a/roles/kube_prometheus/files/kube-prometheus/node-exporter-service.yaml +++ b/roles/kube_prometheus/files/kube-prometheus/node-exporter-service.yaml @@ -9,7 +9,7 @@ metadata: app.kubernetes.io/component: exporter app.kubernetes.io/name: node-exporter app.kubernetes.io/part-of: kube-prometheus - app.kubernetes.io/version: 1.2.2 + app.kubernetes.io/version: 1.3.1 name: node-exporter namespace: monitoring spec: diff --git a/roles/kube_prometheus/files/kube-prometheus/node-exporter-serviceAccount.yaml b/roles/kube_prometheus/files/kube-prometheus/node-exporter-serviceAccount.yaml index 8a602054..9f05db00 100644 --- a/roles/kube_prometheus/files/kube-prometheus/node-exporter-serviceAccount.yaml +++ b/roles/kube_prometheus/files/kube-prometheus/node-exporter-serviceAccount.yaml @@ -11,4 +11,4 @@ metadata: app.kubernetes.io/component: exporter app.kubernetes.io/name: node-exporter app.kubernetes.io/part-of: kube-prometheus - app.kubernetes.io/version: 1.2.2 + app.kubernetes.io/version: 1.3.1 diff --git a/roles/kube_prometheus/files/kube-prometheus/node-exporter-serviceMonitor.yaml b/roles/kube_prometheus/files/kube-prometheus/node-exporter-serviceMonitor.yaml index 86e7bd56..c3403017 100644 --- a/roles/kube_prometheus/files/kube-prometheus/node-exporter-serviceMonitor.yaml +++ b/roles/kube_prometheus/files/kube-prometheus/node-exporter-serviceMonitor.yaml @@ -9,7 +9,7 @@ metadata: app.kubernetes.io/component: exporter app.kubernetes.io/name: node-exporter app.kubernetes.io/part-of: kube-prometheus - app.kubernetes.io/version: 1.2.2 + app.kubernetes.io/version: 1.3.1 name: node-exporter namespace: monitoring spec: diff --git a/roles/kube_prometheus/files/kube-prometheus/persistent-volume-claim-grafana.yaml b/roles/kube_prometheus/files/kube-prometheus/persistent-volume-claim-grafana.yaml index 6c13144a..a45ca8bf 100644 --- a/roles/kube_prometheus/files/kube-prometheus/persistent-volume-claim-grafana.yaml +++ b/roles/kube_prometheus/files/kube-prometheus/persistent-volume-claim-grafana.yaml @@ -11,7 +11,7 @@ metadata: app.kubernetes.io/component: grafana app.kubernetes.io/name: grafana app.kubernetes.io/part-of: kube-prometheus - app.kubernetes.io/version: 8.1.2 + app.kubernetes.io/version: 8.3.4 spec: storageClassName: kube-prometheus selector: diff --git a/roles/kube_prometheus/files/kube-prometheus/prometheus-rules.yaml b/roles/kube_prometheus/files/kube-prometheus/prometheus-rules.yaml index 76953fc6..4ebe554e 100644 --- a/roles/kube_prometheus/files/kube-prometheus/prometheus-rules.yaml +++ b/roles/kube_prometheus/files/kube-prometheus/prometheus-rules.yaml @@ -11,7 +11,7 @@ metadata: app.kubernetes.io/component: exporter app.kubernetes.io/name: node-exporter app.kubernetes.io/part-of: kube-prometheus - app.kubernetes.io/version: 1.2.2 + app.kubernetes.io/version: 1.3.1 name: prometheus-k8s-rules namespace: monitoring spec: diff --git a/roles/kube_prometheus/tasks/create-tas-demo-policy.yml b/roles/kube_prometheus/tasks/create-tas-demo-policy.yml index 19b1a52c..ddd91365 100644 --- a/roles/kube_prometheus/tasks/create-tas-demo-policy.yml +++ b/roles/kube_prometheus/tasks/create-tas-demo-policy.yml @@ -1,5 +1,5 @@ ## -## Copyright (c) 2020-2021 Intel Corporation. +## Copyright (c) 2020-2022 Intel Corporation. ## ## Licensed under the Apache License, Version 2.0 (the "License"); ## you may not use this file except in compliance with the License. diff --git a/roles/kube_prometheus/tasks/kube-prometheus.yml b/roles/kube_prometheus/tasks/kube-prometheus.yml index 9090ac5a..6e15236f 100644 --- a/roles/kube_prometheus/tasks/kube-prometheus.yml +++ b/roles/kube_prometheus/tasks/kube-prometheus.yml @@ -1,5 +1,5 @@ ## -## Copyright (c) 2020-2021 Intel Corporation. +## Copyright (c) 2020-2022 Intel Corporation. ## ## Licensed under the Apache License, Version 2.0 (the "License"); ## you may not use this file except in compliance with the License. @@ -14,7 +14,6 @@ ## limitations under the License. ## --- - - name: delete previous kube-prometheus manifests command: "kubectl delete -f {{ kube_prometheus_path }}" changed_when: true @@ -35,7 +34,7 @@ - name: copy dashboards yamls copy: - src: "dashboards/{{ 'grafana-telegraf-dashboardDefinitions.yaml' if telegraf_enabled else 'grafana-dashboardDefinitions.yaml' }}" + src: "dashboards/{{ 'grafana-telegraf-dashboardDefinitions.yaml' if telegraf_enabled | default(true) else 'grafana-dashboardDefinitions.yaml' }}" dest: "{{ kube_prometheus_path }}" owner: root group: root diff --git a/roles/kube_prometheus/tasks/main.yml b/roles/kube_prometheus/tasks/main.yml index dea07fe8..824cf2e8 100644 --- a/roles/kube_prometheus/tasks/main.yml +++ b/roles/kube_prometheus/tasks/main.yml @@ -1,5 +1,5 @@ ## -## Copyright (c) 2020-2021 Intel Corporation. +## Copyright (c) 2020-2022 Intel Corporation. ## ## Licensed under the Apache License, Version 2.0 (the "License"); ## you may not use this file except in compliance with the License. diff --git a/roles/kube_prometheus/templates/node-exporter-clusterRole.yaml.j2 b/roles/kube_prometheus/templates/node-exporter-clusterRole.yaml.j2 index 2d77a9fd..22b58bb0 100644 --- a/roles/kube_prometheus/templates/node-exporter-clusterRole.yaml.j2 +++ b/roles/kube_prometheus/templates/node-exporter-clusterRole.yaml.j2 @@ -10,7 +10,7 @@ metadata: app.kubernetes.io/component: exporter app.kubernetes.io/name: node-exporter app.kubernetes.io/part-of: kube-prometheus - app.kubernetes.io/version: 1.2.2 + app.kubernetes.io/version: 1.3.1 rules: - apiGroups: ["authentication.k8s.io"] resources: diff --git a/roles/kube_prometheus/templates/node-exporter-daemonset.yaml.j2 b/roles/kube_prometheus/templates/node-exporter-daemonset.yaml.j2 index 5bd92f92..0c753b25 100644 --- a/roles/kube_prometheus/templates/node-exporter-daemonset.yaml.j2 +++ b/roles/kube_prometheus/templates/node-exporter-daemonset.yaml.j2 @@ -7,7 +7,7 @@ kind: DaemonSet metadata: labels: app.kubernetes.io/name: node-exporter - app.kubernetes.io/version: v1.2.2 + app.kubernetes.io/version: 1.3.1 name: node-exporter namespace: monitoring spec: @@ -18,7 +18,7 @@ spec: metadata: labels: app.kubernetes.io/name: node-exporter - app.kubernetes.io/version: v1.2.2 + app.kubernetes.io/version: 1.3.1 spec: containers: - args: @@ -31,7 +31,7 @@ spec: {% if install_tas_demo_policy %} - --collector.textfile.directory=/host/tas-demo-policy {% endif %} - image: quay.io/prometheus/node-exporter:v1.2.2 + image: quay.io/prometheus/node-exporter:v1.3.1 name: node-exporter resources: limits: @@ -98,10 +98,6 @@ spec: - key: node-role.kubernetes.io/master effect: NoSchedule operator: Exists - - key: cmk - operator: Equal - effect: NoSchedule - value: "true" volumes: - hostPath: path: /proc diff --git a/roles/kube_prometheus/templates/persistent-volume-grafana.yaml.j2 b/roles/kube_prometheus/templates/persistent-volume-grafana.yaml.j2 index 58a6b532..0b47e5d4 100644 --- a/roles/kube_prometheus/templates/persistent-volume-grafana.yaml.j2 +++ b/roles/kube_prometheus/templates/persistent-volume-grafana.yaml.j2 @@ -8,7 +8,7 @@ metadata: app.kubernetes.io/component: grafana app.kubernetes.io/name: grafana app.kubernetes.io/part-of: kube-prometheus - app.kubernetes.io/version: 8.1.2 + app.kubernetes.io/version: 8.3.4 spec: capacity: storage: 1Gi diff --git a/roles/kube_prometheus/templates/prometheus-prometheus.yaml.j2 b/roles/kube_prometheus/templates/prometheus-prometheus.yaml.j2 index 400ed951..55d65283 100644 --- a/roles/kube_prometheus/templates/prometheus-prometheus.yaml.j2 +++ b/roles/kube_prometheus/templates/prometheus-prometheus.yaml.j2 @@ -14,11 +14,6 @@ metadata: name: k8s namespace: monitoring spec: - tolerations: - - key: cmk - operator: Equal - effect: NoSchedule - value: "true" containers: - name: custom-metrics-apiserver image: directxman12/k8s-prometheus-adapter:v0.8.4 @@ -77,7 +72,7 @@ spec: scheme: HTTPS timeoutSeconds: 3 - name: grafana - image: grafana/grafana:8.1.2 + image: grafana/grafana:8.3.4 env: - name: GF_SERVER_PROTOCOL value: https diff --git a/roles/kube_prometheus/vars/main.yml b/roles/kube_prometheus/vars/main.yml index caaf2802..5601e228 100644 --- a/roles/kube_prometheus/vars/main.yml +++ b/roles/kube_prometheus/vars/main.yml @@ -1,5 +1,5 @@ ## -## Copyright (c) 2020-2021 Intel Corporation. +## Copyright (c) 2020-2022 Intel Corporation. ## ## Licensed under the Apache License, Version 2.0 (the "License"); ## you may not use this file except in compliance with the License. @@ -14,7 +14,9 @@ ## limitations under the License. ## --- -kube_prometheus_path: /usr/src/kube-prometheus +# variables in this file are not intended to be modified by the user directly +# please use defaults/main.yml instead +kube_prometheus_path: "{{ project_root_dir }}/kube-prometheus" telemetry_namespace: monitoring # this is not templated everywhere yet. tas_demo_policy_dir: /opt/intel/tas-demo-policy diff --git a/roles/kubespray_patch/defaults/main.yml b/roles/kubespray_patch/defaults/main.yml index aca72666..54f04d95 100644 --- a/roles/kubespray_patch/defaults/main.yml +++ b/roles/kubespray_patch/defaults/main.yml @@ -1,5 +1,5 @@ ## -## Copyright (c) 2020-2021 Intel Corporation. +## Copyright (c) 2020-2022 Intel Corporation. ## ## Licensed under the Apache License, Version 2.0 (the "License"); ## you may not use this file except in compliance with the License. diff --git a/roles/kubespray_patch/tasks/main.yml b/roles/kubespray_patch/tasks/main.yml index 97000767..5e4e88d0 100644 --- a/roles/kubespray_patch/tasks/main.yml +++ b/roles/kubespray_patch/tasks/main.yml @@ -1,5 +1,5 @@ ## -## Copyright (c) 2020-2021 Intel Corporation. +## Copyright (c) 2020-2022 Intel Corporation. ## ## Licensed under the Apache License, Version 2.0 (the "License"); ## you may not use this file except in compliance with the License. diff --git a/roles/kubespray_target_setup/tasks/main.yml b/roles/kubespray_target_setup/tasks/main.yml index 0a9c47e7..0bf8c05b 100644 --- a/roles/kubespray_target_setup/tasks/main.yml +++ b/roles/kubespray_target_setup/tasks/main.yml @@ -1,5 +1,5 @@ ## -## Copyright (c) 2020-2021 Intel Corporation. +## Copyright (c) 2020-2022 Intel Corporation. ## ## Licensed under the Apache License, Version 2.0 (the "License"); ## you may not use this file except in compliance with the License. @@ -42,11 +42,11 @@ - name: copy patched Multus config template: dest: /etc/cni/net.d/templates/00-multus.conf - src: roles/kubespray_install/templates/multus.conf.j2 + src: multus.conf.j2 owner: root group: root mode: 0644 - when: kube_network_plugin_multus | default(true) + when: kube_network_plugin_multus | default(true) | bool - name: create Kubernetes admission control config dir file: @@ -58,7 +58,7 @@ - name: copy admission control configuration file template: dest: "{{ kube_config_dir }}/admission-control/config.yaml" - src: config.yaml + src: config.yaml.j2 owner: root group: root mode: 0644 diff --git a/roles/kubespray_target_setup/templates/config.yaml b/roles/kubespray_target_setup/templates/config.yaml.j2 similarity index 51% rename from roles/kubespray_target_setup/templates/config.yaml rename to roles/kubespray_target_setup/templates/config.yaml.j2 index e945b746..6f0819f5 100644 --- a/roles/kubespray_target_setup/templates/config.yaml +++ b/roles/kubespray_target_setup/templates/config.yaml.j2 @@ -1,8 +1,4 @@ -{% if kube_version is version_compare('v1.17', '>=') %} apiVersion: apiserver.config.k8s.io/v1 -{% else %} -apiVersion: apiserver.k8s.io/v1alpha1 -{% endif %} kind: AdmissionConfiguration plugins: - name: EventRateLimit diff --git a/roles/kubespray_install/templates/multus.conf.j2 b/roles/kubespray_target_setup/templates/multus.conf.j2 similarity index 99% rename from roles/kubespray_install/templates/multus.conf.j2 rename to roles/kubespray_target_setup/templates/multus.conf.j2 index 71eda9f9..6b05d1f7 100644 --- a/roles/kubespray_install/templates/multus.conf.j2 +++ b/roles/kubespray_target_setup/templates/multus.conf.j2 @@ -46,4 +46,4 @@ } ], "kubeconfig": "/etc/cni/net.d/multus.d/multus.kubeconfig" -} \ No newline at end of file +} diff --git a/roles/kubespray_target_setup/vars/main.yml b/roles/kubespray_target_setup/vars/main.yml index ad6cc876..c2aab645 100644 --- a/roles/kubespray_target_setup/vars/main.yml +++ b/roles/kubespray_target_setup/vars/main.yml @@ -1,5 +1,5 @@ ## -## Copyright (c) 2020-2021 Intel Corporation. +## Copyright (c) 2020-2022 Intel Corporation. ## ## Licensed under the Apache License, Version 2.0 (the "License"); ## you may not use this file except in compliance with the License. diff --git a/roles/load_ddp/defaults/main.yml b/roles/load_ddp/defaults/main.yml index 3390f1ad..5ff3d8e6 100644 --- a/roles/load_ddp/defaults/main.yml +++ b/roles/load_ddp/defaults/main.yml @@ -1,5 +1,5 @@ ## -## Copyright (c) 2020-2021 Intel Corporation. +## Copyright (c) 2020-2022 Intel Corporation. ## ## Licensed under the Apache License, Version 2.0 (the "License"); ## you may not use this file except in compliance with the License. diff --git a/roles/load_ddp/tasks/load_i40e_ddp.yml b/roles/load_ddp/tasks/load_i40e_ddp.yml index 2e1ee8fd..457edc15 100644 --- a/roles/load_ddp/tasks/load_i40e_ddp.yml +++ b/roles/load_ddp/tasks/load_i40e_ddp.yml @@ -1,5 +1,5 @@ ## -## Copyright (c) 2020-2021 Intel Corporation. +## Copyright (c) 2020-2022 Intel Corporation. ## ## Licensed under the Apache License, Version 2.0 (the "License"); ## you may not use this file except in compliance with the License. diff --git a/roles/load_ddp/tasks/load_ice_ddp.yml b/roles/load_ddp/tasks/load_ice_ddp.yml index 8ad6ef62..d620fce5 100644 --- a/roles/load_ddp/tasks/load_ice_ddp.yml +++ b/roles/load_ddp/tasks/load_ice_ddp.yml @@ -1,5 +1,5 @@ ## -## Copyright (c) 2020-2021 Intel Corporation. +## Copyright (c) 2020-2022 Intel Corporation. ## ## Licensed under the Apache License, Version 2.0 (the "License"); ## you may not use this file except in compliance with the License. diff --git a/roles/load_ddp/tasks/main.yml b/roles/load_ddp/tasks/main.yml index 646738d6..9be6d1e8 100644 --- a/roles/load_ddp/tasks/main.yml +++ b/roles/load_ddp/tasks/main.yml @@ -1,5 +1,5 @@ ## -## Copyright (c) 2020-2021 Intel Corporation. +## Copyright (c) 2020-2022 Intel Corporation. ## ## Licensed under the Apache License, Version 2.0 (the "License"); ## you may not use this file except in compliance with the License. diff --git a/roles/load_ddp/tasks/update_network_card.yml b/roles/load_ddp/tasks/update_network_card.yml index b8ca3897..a34d5f89 100644 --- a/roles/load_ddp/tasks/update_network_card.yml +++ b/roles/load_ddp/tasks/update_network_card.yml @@ -1,5 +1,5 @@ ## -## Copyright (c) 2020-2021 Intel Corporation. +## Copyright (c) 2020-2022 Intel Corporation. ## ## Licensed under the Apache License, Version 2.0 (the "License"); ## you may not use this file except in compliance with the License. diff --git a/roles/load_ddp/vars/main.yml b/roles/load_ddp/vars/main.yml index 955a09f1..fb3caa2e 100644 --- a/roles/load_ddp/vars/main.yml +++ b/roles/load_ddp/vars/main.yml @@ -1,5 +1,5 @@ ## -## Copyright (c) 2020-2021 Intel Corporation. +## Copyright (c) 2020-2022 Intel Corporation. ## ## Licensed under the Apache License, Version 2.0 (the "License"); ## you may not use this file except in compliance with the License. diff --git a/roles/minio_install/charts/operator/Chart.yaml b/roles/minio_install/charts/operator/Chart.yaml new file mode 100644 index 00000000..db3a00f0 --- /dev/null +++ b/roles/minio_install/charts/operator/Chart.yaml @@ -0,0 +1,32 @@ +## +## Copyright (c) 2020-2022 Intel Corporation. +## +## Licensed under the Apache License, Version 2.0 (the "License"); +## you may not use this file except in compliance with the License. +## You may obtain a copy of the License at +## +## http://www.apache.org/licenses/LICENSE-2.0 +## +## Unless required by applicable law or agreed to in writing, software +## distributed under the License is distributed on an "AS IS" BASIS, +## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +## See the License for the specific language governing permissions and +## limitations under the License. +## +apiVersion: v2 +description: A Helm chart for MinIO Operator +name: operator +version: 4.4.1 +appVersion: v4.4.1 +keywords: + - storage + - object-storage + - S3 +home: https://min.io +icon: https://min.io/resources/img/logo/MINIO_wordmark.png +sources: + - https://github.com/minio/operator +maintainers: + - name: MinIO, Inc + email: dev@minio.io +type: application diff --git a/roles/minio_install/charts/operator/crds/minio.min.io_tenants.yaml b/roles/minio_install/charts/operator/crds/minio.min.io_tenants.yaml new file mode 100755 index 00000000..e6bf9073 --- /dev/null +++ b/roles/minio_install/charts/operator/crds/minio.min.io_tenants.yaml @@ -0,0 +1,8232 @@ +## +## Copyright (c) 2020-2022 Intel Corporation. +## +## Licensed under the Apache License, Version 2.0 (the "License"); +## you may not use this file except in compliance with the License. +## You may obtain a copy of the License at +## +## http://www.apache.org/licenses/LICENSE-2.0 +## +## Unless required by applicable law or agreed to in writing, software +## distributed under the License is distributed on an "AS IS" BASIS, +## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +## See the License for the specific language governing permissions and +## limitations under the License. +## +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.4.7 + name: tenants.minio.min.io +spec: + conversion: + strategy: Webhook + webhook: + clientConfig: + service: + name: operator + namespace: minio-operator + path: /webhook/v1/crd-conversion + port: 4222 + conversionReviewVersions: + - v1 + - v2 + group: minio.min.io + names: + kind: Tenant + listKind: TenantList + plural: tenants + shortNames: + - tenant + singular: tenant + preserveUnknownFields: false + scope: Namespaced + versions: + - additionalPrinterColumns: + - jsonPath: .status.currentState + name: State + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1 + schema: + openAPIV3Schema: + properties: + apiVersion: + type: string + kind: + type: string + metadata: + type: object + scheduler: + properties: + name: + type: string + required: + - name + type: object + spec: + properties: + certConfig: + properties: + commonName: + type: string + dnsNames: + items: + type: string + type: array + organizationName: + items: + type: string + type: array + type: object + configuration: + properties: + name: + type: string + type: object + credsSecret: + properties: + name: + type: string + type: object + env: + items: + properties: + name: + type: string + value: + type: string + valueFrom: + properties: + configMapKeyRef: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + fieldRef: + properties: + apiVersion: + type: string + fieldPath: + type: string + required: + - fieldPath + type: object + resourceFieldRef: + properties: + containerName: + type: string + divisor: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + type: string + required: + - resource + type: object + secretKeyRef: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + type: object + required: + - name + type: object + type: array + exposeServices: + properties: + console: + type: boolean + minio: + type: boolean + type: object + externalCaCertSecret: + items: + properties: + name: + type: string + type: + type: string + required: + - name + type: object + type: array + externalCertSecret: + items: + properties: + name: + type: string + type: + type: string + required: + - name + type: object + type: array + externalClientCertSecret: + properties: + name: + type: string + type: + type: string + required: + - name + type: object + image: + type: string + imagePullPolicy: + type: string + imagePullSecret: + properties: + name: + type: string + type: object + kes: + properties: + affinity: + properties: + nodeAffinity: + properties: + preferredDuringSchedulingIgnoredDuringExecution: + items: + properties: + preference: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchFields: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + type: object + weight: + format: int32 + type: integer + required: + - preference + - weight + type: object + type: array + requiredDuringSchedulingIgnoredDuringExecution: + properties: + nodeSelectorTerms: + items: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchFields: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + type: object + type: array + required: + - nodeSelectorTerms + type: object + type: object + podAffinity: + properties: + preferredDuringSchedulingIgnoredDuringExecution: + items: + properties: + podAffinityTerm: + properties: + labelSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + namespaces: + items: + type: string + type: array + topologyKey: + type: string + required: + - topologyKey + type: object + weight: + format: int32 + type: integer + required: + - podAffinityTerm + - weight + type: object + type: array + requiredDuringSchedulingIgnoredDuringExecution: + items: + properties: + labelSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + namespaces: + items: + type: string + type: array + topologyKey: + type: string + required: + - topologyKey + type: object + type: array + type: object + podAntiAffinity: + properties: + preferredDuringSchedulingIgnoredDuringExecution: + items: + properties: + podAffinityTerm: + properties: + labelSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + namespaces: + items: + type: string + type: array + topologyKey: + type: string + required: + - topologyKey + type: object + weight: + format: int32 + type: integer + required: + - podAffinityTerm + - weight + type: object + type: array + requiredDuringSchedulingIgnoredDuringExecution: + items: + properties: + labelSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + namespaces: + items: + type: string + type: array + topologyKey: + type: string + required: + - topologyKey + type: object + type: array + type: object + type: object + annotations: + additionalProperties: + type: string + type: object + clientCertSecret: + properties: + name: + type: string + type: + type: string + required: + - name + type: object + externalCertSecret: + properties: + name: + type: string + type: + type: string + required: + - name + type: object + image: + type: string + imagePullPolicy: + type: string + kesSecret: + properties: + name: + type: string + type: object + keyName: + type: string + labels: + additionalProperties: + type: string + type: object + nodeSelector: + additionalProperties: + type: string + type: object + replicas: + format: int32 + type: integer + resources: + properties: + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + type: object + securityContext: + properties: + fsGroup: + format: int64 + type: integer + fsGroupChangePolicy: + type: string + runAsGroup: + format: int64 + type: integer + runAsNonRoot: + type: boolean + runAsUser: + format: int64 + type: integer + seLinuxOptions: + properties: + level: + type: string + role: + type: string + type: + type: string + user: + type: string + type: object + seccompProfile: + properties: + localhostProfile: + type: string + type: + type: string + required: + - type + type: object + supplementalGroups: + items: + format: int64 + type: integer + type: array + sysctls: + items: + properties: + name: + type: string + value: + type: string + required: + - name + - value + type: object + type: array + windowsOptions: + properties: + gmsaCredentialSpec: + type: string + gmsaCredentialSpecName: + type: string + runAsUserName: + type: string + type: object + type: object + serviceAccountName: + type: string + tolerations: + items: + properties: + effect: + type: string + key: + type: string + operator: + type: string + tolerationSeconds: + format: int64 + type: integer + value: + type: string + type: object + type: array + topologySpreadConstraints: + items: + properties: + labelSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + maxSkew: + format: int32 + type: integer + topologyKey: + type: string + whenUnsatisfiable: + type: string + required: + - maxSkew + - topologyKey + - whenUnsatisfiable + type: object + type: array + required: + - kesSecret + type: object + log: + properties: + affinity: + properties: + nodeAffinity: + properties: + preferredDuringSchedulingIgnoredDuringExecution: + items: + properties: + preference: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchFields: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + type: object + weight: + format: int32 + type: integer + required: + - preference + - weight + type: object + type: array + requiredDuringSchedulingIgnoredDuringExecution: + properties: + nodeSelectorTerms: + items: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchFields: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + type: object + type: array + required: + - nodeSelectorTerms + type: object + type: object + podAffinity: + properties: + preferredDuringSchedulingIgnoredDuringExecution: + items: + properties: + podAffinityTerm: + properties: + labelSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + namespaces: + items: + type: string + type: array + topologyKey: + type: string + required: + - topologyKey + type: object + weight: + format: int32 + type: integer + required: + - podAffinityTerm + - weight + type: object + type: array + requiredDuringSchedulingIgnoredDuringExecution: + items: + properties: + labelSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + namespaces: + items: + type: string + type: array + topologyKey: + type: string + required: + - topologyKey + type: object + type: array + type: object + podAntiAffinity: + properties: + preferredDuringSchedulingIgnoredDuringExecution: + items: + properties: + podAffinityTerm: + properties: + labelSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + namespaces: + items: + type: string + type: array + topologyKey: + type: string + required: + - topologyKey + type: object + weight: + format: int32 + type: integer + required: + - podAffinityTerm + - weight + type: object + type: array + requiredDuringSchedulingIgnoredDuringExecution: + items: + properties: + labelSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + namespaces: + items: + type: string + type: array + topologyKey: + type: string + required: + - topologyKey + type: object + type: array + type: object + type: object + annotations: + additionalProperties: + type: string + type: object + audit: + properties: + diskCapacityGB: + type: integer + type: object + db: + properties: + affinity: + properties: + nodeAffinity: + properties: + preferredDuringSchedulingIgnoredDuringExecution: + items: + properties: + preference: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchFields: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + type: object + weight: + format: int32 + type: integer + required: + - preference + - weight + type: object + type: array + requiredDuringSchedulingIgnoredDuringExecution: + properties: + nodeSelectorTerms: + items: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchFields: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + type: object + type: array + required: + - nodeSelectorTerms + type: object + type: object + podAffinity: + properties: + preferredDuringSchedulingIgnoredDuringExecution: + items: + properties: + podAffinityTerm: + properties: + labelSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + namespaces: + items: + type: string + type: array + topologyKey: + type: string + required: + - topologyKey + type: object + weight: + format: int32 + type: integer + required: + - podAffinityTerm + - weight + type: object + type: array + requiredDuringSchedulingIgnoredDuringExecution: + items: + properties: + labelSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + namespaces: + items: + type: string + type: array + topologyKey: + type: string + required: + - topologyKey + type: object + type: array + type: object + podAntiAffinity: + properties: + preferredDuringSchedulingIgnoredDuringExecution: + items: + properties: + podAffinityTerm: + properties: + labelSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + namespaces: + items: + type: string + type: array + topologyKey: + type: string + required: + - topologyKey + type: object + weight: + format: int32 + type: integer + required: + - podAffinityTerm + - weight + type: object + type: array + requiredDuringSchedulingIgnoredDuringExecution: + items: + properties: + labelSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + namespaces: + items: + type: string + type: array + topologyKey: + type: string + required: + - topologyKey + type: object + type: array + type: object + type: object + annotations: + additionalProperties: + type: string + type: object + image: + type: string + initimage: + type: string + labels: + additionalProperties: + type: string + type: object + nodeSelector: + additionalProperties: + type: string + type: object + resources: + properties: + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + type: object + securityContext: + properties: + fsGroup: + format: int64 + type: integer + fsGroupChangePolicy: + type: string + runAsGroup: + format: int64 + type: integer + runAsNonRoot: + type: boolean + runAsUser: + format: int64 + type: integer + seLinuxOptions: + properties: + level: + type: string + role: + type: string + type: + type: string + user: + type: string + type: object + seccompProfile: + properties: + localhostProfile: + type: string + type: + type: string + required: + - type + type: object + supplementalGroups: + items: + format: int64 + type: integer + type: array + sysctls: + items: + properties: + name: + type: string + value: + type: string + required: + - name + - value + type: object + type: array + windowsOptions: + properties: + gmsaCredentialSpec: + type: string + gmsaCredentialSpecName: + type: string + runAsUserName: + type: string + type: object + type: object + serviceAccountName: + type: string + tolerations: + items: + properties: + effect: + type: string + key: + type: string + operator: + type: string + tolerationSeconds: + format: int64 + type: integer + value: + type: string + type: object + type: array + topologySpreadConstraints: + items: + properties: + labelSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + maxSkew: + format: int32 + type: integer + topologyKey: + type: string + whenUnsatisfiable: + type: string + required: + - maxSkew + - topologyKey + - whenUnsatisfiable + type: object + type: array + volumeClaimTemplate: + properties: + apiVersion: + type: string + kind: + type: string + metadata: + properties: + annotations: + additionalProperties: + type: string + type: object + finalizers: + items: + type: string + type: array + labels: + additionalProperties: + type: string + type: object + name: + type: string + namespace: + type: string + type: object + spec: + properties: + accessModes: + items: + type: string + type: array + dataSource: + properties: + apiGroup: + type: string + kind: + type: string + name: + type: string + required: + - kind + - name + type: object + resources: + properties: + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + type: object + selector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + storageClassName: + type: string + volumeMode: + type: string + volumeName: + type: string + type: object + status: + properties: + accessModes: + items: + type: string + type: array + capacity: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + conditions: + items: + properties: + lastProbeTime: + format: date-time + type: string + lastTransitionTime: + format: date-time + type: string + message: + type: string + reason: + type: string + status: + type: string + type: + type: string + required: + - status + - type + type: object + type: array + phase: + type: string + type: object + type: object + required: + - volumeClaimTemplate + type: object + image: + type: string + labels: + additionalProperties: + type: string + type: object + nodeSelector: + additionalProperties: + type: string + type: object + resources: + properties: + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + type: object + securityContext: + properties: + fsGroup: + format: int64 + type: integer + fsGroupChangePolicy: + type: string + runAsGroup: + format: int64 + type: integer + runAsNonRoot: + type: boolean + runAsUser: + format: int64 + type: integer + seLinuxOptions: + properties: + level: + type: string + role: + type: string + type: + type: string + user: + type: string + type: object + seccompProfile: + properties: + localhostProfile: + type: string + type: + type: string + required: + - type + type: object + supplementalGroups: + items: + format: int64 + type: integer + type: array + sysctls: + items: + properties: + name: + type: string + value: + type: string + required: + - name + - value + type: object + type: array + windowsOptions: + properties: + gmsaCredentialSpec: + type: string + gmsaCredentialSpecName: + type: string + runAsUserName: + type: string + type: object + type: object + serviceAccountName: + type: string + tolerations: + items: + properties: + effect: + type: string + key: + type: string + operator: + type: string + tolerationSeconds: + format: int64 + type: integer + value: + type: string + type: object + type: array + topologySpreadConstraints: + items: + properties: + labelSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + maxSkew: + format: int32 + type: integer + topologyKey: + type: string + whenUnsatisfiable: + type: string + required: + - maxSkew + - topologyKey + - whenUnsatisfiable + type: object + type: array + type: object + mountPath: + type: string + podManagementPolicy: + type: string + priorityClassName: + type: string + prometheus: + properties: + affinity: + properties: + nodeAffinity: + properties: + preferredDuringSchedulingIgnoredDuringExecution: + items: + properties: + preference: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchFields: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + type: object + weight: + format: int32 + type: integer + required: + - preference + - weight + type: object + type: array + requiredDuringSchedulingIgnoredDuringExecution: + properties: + nodeSelectorTerms: + items: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchFields: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + type: object + type: array + required: + - nodeSelectorTerms + type: object + type: object + podAffinity: + properties: + preferredDuringSchedulingIgnoredDuringExecution: + items: + properties: + podAffinityTerm: + properties: + labelSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + namespaces: + items: + type: string + type: array + topologyKey: + type: string + required: + - topologyKey + type: object + weight: + format: int32 + type: integer + required: + - podAffinityTerm + - weight + type: object + type: array + requiredDuringSchedulingIgnoredDuringExecution: + items: + properties: + labelSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + namespaces: + items: + type: string + type: array + topologyKey: + type: string + required: + - topologyKey + type: object + type: array + type: object + podAntiAffinity: + properties: + preferredDuringSchedulingIgnoredDuringExecution: + items: + properties: + podAffinityTerm: + properties: + labelSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + namespaces: + items: + type: string + type: array + topologyKey: + type: string + required: + - topologyKey + type: object + weight: + format: int32 + type: integer + required: + - podAffinityTerm + - weight + type: object + type: array + requiredDuringSchedulingIgnoredDuringExecution: + items: + properties: + labelSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + namespaces: + items: + type: string + type: array + topologyKey: + type: string + required: + - topologyKey + type: object + type: array + type: object + type: object + annotations: + additionalProperties: + type: string + type: object + diskCapacityGB: + type: integer + image: + type: string + initimage: + type: string + labels: + additionalProperties: + type: string + type: object + nodeSelector: + additionalProperties: + type: string + type: object + resources: + properties: + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + type: object + securityContext: + properties: + fsGroup: + format: int64 + type: integer + fsGroupChangePolicy: + type: string + runAsGroup: + format: int64 + type: integer + runAsNonRoot: + type: boolean + runAsUser: + format: int64 + type: integer + seLinuxOptions: + properties: + level: + type: string + role: + type: string + type: + type: string + user: + type: string + type: object + seccompProfile: + properties: + localhostProfile: + type: string + type: + type: string + required: + - type + type: object + supplementalGroups: + items: + format: int64 + type: integer + type: array + sysctls: + items: + properties: + name: + type: string + value: + type: string + required: + - name + - value + type: object + type: array + windowsOptions: + properties: + gmsaCredentialSpec: + type: string + gmsaCredentialSpecName: + type: string + runAsUserName: + type: string + type: object + type: object + serviceAccountName: + type: string + sidecarimage: + type: string + storageClassName: + type: string + topologySpreadConstraints: + items: + properties: + labelSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + maxSkew: + format: int32 + type: integer + topologyKey: + type: string + whenUnsatisfiable: + type: string + required: + - maxSkew + - topologyKey + - whenUnsatisfiable + type: object + type: array + type: object + requestAutoCert: + type: boolean + s3: + properties: + bucketDNS: + type: boolean + required: + - bucketDNS + type: object + securityContext: + properties: + fsGroup: + format: int64 + type: integer + fsGroupChangePolicy: + type: string + runAsGroup: + format: int64 + type: integer + runAsNonRoot: + type: boolean + runAsUser: + format: int64 + type: integer + seLinuxOptions: + properties: + level: + type: string + role: + type: string + type: + type: string + user: + type: string + type: object + seccompProfile: + properties: + localhostProfile: + type: string + type: + type: string + required: + - type + type: object + supplementalGroups: + items: + format: int64 + type: integer + type: array + sysctls: + items: + properties: + name: + type: string + value: + type: string + required: + - name + - value + type: object + type: array + windowsOptions: + properties: + gmsaCredentialSpec: + type: string + gmsaCredentialSpecName: + type: string + runAsUserName: + type: string + type: object + type: object + serviceAccountName: + type: string + sideCars: + properties: + containers: + items: + properties: + args: + items: + type: string + type: array + command: + items: + type: string + type: array + env: + items: + properties: + name: + type: string + value: + type: string + valueFrom: + properties: + configMapKeyRef: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + fieldRef: + properties: + apiVersion: + type: string + fieldPath: + type: string + required: + - fieldPath + type: object + resourceFieldRef: + properties: + containerName: + type: string + divisor: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + type: string + required: + - resource + type: object + secretKeyRef: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + type: object + required: + - name + type: object + type: array + envFrom: + items: + properties: + configMapRef: + properties: + name: + type: string + optional: + type: boolean + type: object + prefix: + type: string + secretRef: + properties: + name: + type: string + optional: + type: boolean + type: object + type: object + type: array + image: + type: string + imagePullPolicy: + type: string + lifecycle: + properties: + postStart: + properties: + exec: + properties: + command: + items: + type: string + type: array + type: object + httpGet: + properties: + host: + type: string + httpHeaders: + items: + properties: + name: + type: string + value: + type: string + required: + - name + - value + type: object + type: array + path: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + scheme: + type: string + required: + - port + type: object + tcpSocket: + properties: + host: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + required: + - port + type: object + type: object + preStop: + properties: + exec: + properties: + command: + items: + type: string + type: array + type: object + httpGet: + properties: + host: + type: string + httpHeaders: + items: + properties: + name: + type: string + value: + type: string + required: + - name + - value + type: object + type: array + path: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + scheme: + type: string + required: + - port + type: object + tcpSocket: + properties: + host: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + required: + - port + type: object + type: object + type: object + livenessProbe: + properties: + exec: + properties: + command: + items: + type: string + type: array + type: object + failureThreshold: + format: int32 + type: integer + httpGet: + properties: + host: + type: string + httpHeaders: + items: + properties: + name: + type: string + value: + type: string + required: + - name + - value + type: object + type: array + path: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + scheme: + type: string + required: + - port + type: object + initialDelaySeconds: + format: int32 + type: integer + periodSeconds: + format: int32 + type: integer + successThreshold: + format: int32 + type: integer + tcpSocket: + properties: + host: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + required: + - port + type: object + timeoutSeconds: + format: int32 + type: integer + type: object + name: + type: string + ports: + items: + properties: + containerPort: + format: int32 + type: integer + hostIP: + type: string + hostPort: + format: int32 + type: integer + name: + type: string + protocol: + default: TCP + type: string + required: + - containerPort + type: object + type: array + x-kubernetes-list-map-keys: + - containerPort + - protocol + x-kubernetes-list-type: map + readinessProbe: + properties: + exec: + properties: + command: + items: + type: string + type: array + type: object + failureThreshold: + format: int32 + type: integer + httpGet: + properties: + host: + type: string + httpHeaders: + items: + properties: + name: + type: string + value: + type: string + required: + - name + - value + type: object + type: array + path: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + scheme: + type: string + required: + - port + type: object + initialDelaySeconds: + format: int32 + type: integer + periodSeconds: + format: int32 + type: integer + successThreshold: + format: int32 + type: integer + tcpSocket: + properties: + host: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + required: + - port + type: object + timeoutSeconds: + format: int32 + type: integer + type: object + resources: + properties: + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + type: object + securityContext: + properties: + allowPrivilegeEscalation: + type: boolean + capabilities: + properties: + add: + items: + type: string + type: array + drop: + items: + type: string + type: array + type: object + privileged: + type: boolean + procMount: + type: string + readOnlyRootFilesystem: + type: boolean + runAsGroup: + format: int64 + type: integer + runAsNonRoot: + type: boolean + runAsUser: + format: int64 + type: integer + seLinuxOptions: + properties: + level: + type: string + role: + type: string + type: + type: string + user: + type: string + type: object + seccompProfile: + properties: + localhostProfile: + type: string + type: + type: string + required: + - type + type: object + windowsOptions: + properties: + gmsaCredentialSpec: + type: string + gmsaCredentialSpecName: + type: string + runAsUserName: + type: string + type: object + type: object + startupProbe: + properties: + exec: + properties: + command: + items: + type: string + type: array + type: object + failureThreshold: + format: int32 + type: integer + httpGet: + properties: + host: + type: string + httpHeaders: + items: + properties: + name: + type: string + value: + type: string + required: + - name + - value + type: object + type: array + path: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + scheme: + type: string + required: + - port + type: object + initialDelaySeconds: + format: int32 + type: integer + periodSeconds: + format: int32 + type: integer + successThreshold: + format: int32 + type: integer + tcpSocket: + properties: + host: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + required: + - port + type: object + timeoutSeconds: + format: int32 + type: integer + type: object + stdin: + type: boolean + stdinOnce: + type: boolean + terminationMessagePath: + type: string + terminationMessagePolicy: + type: string + tty: + type: boolean + volumeDevices: + items: + properties: + devicePath: + type: string + name: + type: string + required: + - devicePath + - name + type: object + type: array + volumeMounts: + items: + properties: + mountPath: + type: string + mountPropagation: + type: string + name: + type: string + readOnly: + type: boolean + subPath: + type: string + subPathExpr: + type: string + required: + - mountPath + - name + type: object + type: array + workingDir: + type: string + required: + - name + type: object + type: array + volumeClaimTemplates: + items: + properties: + apiVersion: + type: string + kind: + type: string + metadata: + properties: + annotations: + additionalProperties: + type: string + type: object + finalizers: + items: + type: string + type: array + labels: + additionalProperties: + type: string + type: object + name: + type: string + namespace: + type: string + type: object + spec: + properties: + accessModes: + items: + type: string + type: array + dataSource: + properties: + apiGroup: + type: string + kind: + type: string + name: + type: string + required: + - kind + - name + type: object + resources: + properties: + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + type: object + selector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + storageClassName: + type: string + volumeMode: + type: string + volumeName: + type: string + type: object + status: + properties: + accessModes: + items: + type: string + type: array + capacity: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + conditions: + items: + properties: + lastProbeTime: + format: date-time + type: string + lastTransitionTime: + format: date-time + type: string + message: + type: string + reason: + type: string + status: + type: string + type: + type: string + required: + - status + - type + type: object + type: array + phase: + type: string + type: object + type: object + type: array + volumes: + items: + properties: + awsElasticBlockStore: + properties: + fsType: + type: string + partition: + format: int32 + type: integer + readOnly: + type: boolean + volumeID: + type: string + required: + - volumeID + type: object + azureDisk: + properties: + cachingMode: + type: string + diskName: + type: string + diskURI: + type: string + fsType: + type: string + kind: + type: string + readOnly: + type: boolean + required: + - diskName + - diskURI + type: object + azureFile: + properties: + readOnly: + type: boolean + secretName: + type: string + shareName: + type: string + required: + - secretName + - shareName + type: object + cephfs: + properties: + monitors: + items: + type: string + type: array + path: + type: string + readOnly: + type: boolean + secretFile: + type: string + secretRef: + properties: + name: + type: string + type: object + user: + type: string + required: + - monitors + type: object + cinder: + properties: + fsType: + type: string + readOnly: + type: boolean + secretRef: + properties: + name: + type: string + type: object + volumeID: + type: string + required: + - volumeID + type: object + configMap: + properties: + defaultMode: + format: int32 + type: integer + items: + items: + properties: + key: + type: string + mode: + format: int32 + type: integer + path: + type: string + required: + - key + - path + type: object + type: array + name: + type: string + optional: + type: boolean + type: object + csi: + properties: + driver: + type: string + fsType: + type: string + nodePublishSecretRef: + properties: + name: + type: string + type: object + readOnly: + type: boolean + volumeAttributes: + additionalProperties: + type: string + type: object + required: + - driver + type: object + downwardAPI: + properties: + defaultMode: + format: int32 + type: integer + items: + items: + properties: + fieldRef: + properties: + apiVersion: + type: string + fieldPath: + type: string + required: + - fieldPath + type: object + mode: + format: int32 + type: integer + path: + type: string + resourceFieldRef: + properties: + containerName: + type: string + divisor: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + type: string + required: + - resource + type: object + required: + - path + type: object + type: array + type: object + emptyDir: + properties: + medium: + type: string + sizeLimit: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + ephemeral: + properties: + readOnly: + type: boolean + volumeClaimTemplate: + properties: + metadata: + properties: + annotations: + additionalProperties: + type: string + type: object + finalizers: + items: + type: string + type: array + labels: + additionalProperties: + type: string + type: object + name: + type: string + namespace: + type: string + type: object + spec: + properties: + accessModes: + items: + type: string + type: array + dataSource: + properties: + apiGroup: + type: string + kind: + type: string + name: + type: string + required: + - kind + - name + type: object + resources: + properties: + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + type: object + selector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + storageClassName: + type: string + volumeMode: + type: string + volumeName: + type: string + type: object + required: + - spec + type: object + type: object + fc: + properties: + fsType: + type: string + lun: + format: int32 + type: integer + readOnly: + type: boolean + targetWWNs: + items: + type: string + type: array + wwids: + items: + type: string + type: array + type: object + flexVolume: + properties: + driver: + type: string + fsType: + type: string + options: + additionalProperties: + type: string + type: object + readOnly: + type: boolean + secretRef: + properties: + name: + type: string + type: object + required: + - driver + type: object + flocker: + properties: + datasetName: + type: string + datasetUUID: + type: string + type: object + gcePersistentDisk: + properties: + fsType: + type: string + partition: + format: int32 + type: integer + pdName: + type: string + readOnly: + type: boolean + required: + - pdName + type: object + gitRepo: + properties: + directory: + type: string + repository: + type: string + revision: + type: string + required: + - repository + type: object + glusterfs: + properties: + endpoints: + type: string + path: + type: string + readOnly: + type: boolean + required: + - endpoints + - path + type: object + hostPath: + properties: + path: + type: string + type: + type: string + required: + - path + type: object + iscsi: + properties: + chapAuthDiscovery: + type: boolean + chapAuthSession: + type: boolean + fsType: + type: string + initiatorName: + type: string + iqn: + type: string + iscsiInterface: + type: string + lun: + format: int32 + type: integer + portals: + items: + type: string + type: array + readOnly: + type: boolean + secretRef: + properties: + name: + type: string + type: object + targetPortal: + type: string + required: + - iqn + - lun + - targetPortal + type: object + name: + type: string + nfs: + properties: + path: + type: string + readOnly: + type: boolean + server: + type: string + required: + - path + - server + type: object + persistentVolumeClaim: + properties: + claimName: + type: string + readOnly: + type: boolean + required: + - claimName + type: object + photonPersistentDisk: + properties: + fsType: + type: string + pdID: + type: string + required: + - pdID + type: object + portworxVolume: + properties: + fsType: + type: string + readOnly: + type: boolean + volumeID: + type: string + required: + - volumeID + type: object + projected: + properties: + defaultMode: + format: int32 + type: integer + sources: + items: + properties: + configMap: + properties: + items: + items: + properties: + key: + type: string + mode: + format: int32 + type: integer + path: + type: string + required: + - key + - path + type: object + type: array + name: + type: string + optional: + type: boolean + type: object + downwardAPI: + properties: + items: + items: + properties: + fieldRef: + properties: + apiVersion: + type: string + fieldPath: + type: string + required: + - fieldPath + type: object + mode: + format: int32 + type: integer + path: + type: string + resourceFieldRef: + properties: + containerName: + type: string + divisor: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + type: string + required: + - resource + type: object + required: + - path + type: object + type: array + type: object + secret: + properties: + items: + items: + properties: + key: + type: string + mode: + format: int32 + type: integer + path: + type: string + required: + - key + - path + type: object + type: array + name: + type: string + optional: + type: boolean + type: object + serviceAccountToken: + properties: + audience: + type: string + expirationSeconds: + format: int64 + type: integer + path: + type: string + required: + - path + type: object + type: object + type: array + type: object + quobyte: + properties: + group: + type: string + readOnly: + type: boolean + registry: + type: string + tenant: + type: string + user: + type: string + volume: + type: string + required: + - registry + - volume + type: object + rbd: + properties: + fsType: + type: string + image: + type: string + keyring: + type: string + monitors: + items: + type: string + type: array + pool: + type: string + readOnly: + type: boolean + secretRef: + properties: + name: + type: string + type: object + user: + type: string + required: + - image + - monitors + type: object + scaleIO: + properties: + fsType: + type: string + gateway: + type: string + protectionDomain: + type: string + readOnly: + type: boolean + secretRef: + properties: + name: + type: string + type: object + sslEnabled: + type: boolean + storageMode: + type: string + storagePool: + type: string + system: + type: string + volumeName: + type: string + required: + - gateway + - secretRef + - system + type: object + secret: + properties: + defaultMode: + format: int32 + type: integer + items: + items: + properties: + key: + type: string + mode: + format: int32 + type: integer + path: + type: string + required: + - key + - path + type: object + type: array + optional: + type: boolean + secretName: + type: string + type: object + storageos: + properties: + fsType: + type: string + readOnly: + type: boolean + secretRef: + properties: + name: + type: string + type: object + volumeName: + type: string + volumeNamespace: + type: string + type: object + vsphereVolume: + properties: + fsType: + type: string + storagePolicyID: + type: string + storagePolicyName: + type: string + volumePath: + type: string + required: + - volumePath + type: object + required: + - name + type: object + type: array + required: + - containers + type: object + subPath: + type: string + users: + items: + properties: + name: + type: string + type: object + type: array + zones: + items: + properties: + affinity: + properties: + nodeAffinity: + properties: + preferredDuringSchedulingIgnoredDuringExecution: + items: + properties: + preference: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchFields: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + type: object + weight: + format: int32 + type: integer + required: + - preference + - weight + type: object + type: array + requiredDuringSchedulingIgnoredDuringExecution: + properties: + nodeSelectorTerms: + items: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchFields: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + type: object + type: array + required: + - nodeSelectorTerms + type: object + type: object + podAffinity: + properties: + preferredDuringSchedulingIgnoredDuringExecution: + items: + properties: + podAffinityTerm: + properties: + labelSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + namespaces: + items: + type: string + type: array + topologyKey: + type: string + required: + - topologyKey + type: object + weight: + format: int32 + type: integer + required: + - podAffinityTerm + - weight + type: object + type: array + requiredDuringSchedulingIgnoredDuringExecution: + items: + properties: + labelSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + namespaces: + items: + type: string + type: array + topologyKey: + type: string + required: + - topologyKey + type: object + type: array + type: object + podAntiAffinity: + properties: + preferredDuringSchedulingIgnoredDuringExecution: + items: + properties: + podAffinityTerm: + properties: + labelSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + namespaces: + items: + type: string + type: array + topologyKey: + type: string + required: + - topologyKey + type: object + weight: + format: int32 + type: integer + required: + - podAffinityTerm + - weight + type: object + type: array + requiredDuringSchedulingIgnoredDuringExecution: + items: + properties: + labelSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + namespaces: + items: + type: string + type: array + topologyKey: + type: string + required: + - topologyKey + type: object + type: array + type: object + type: object + name: + type: string + nodeSelector: + additionalProperties: + type: string + type: object + resources: + properties: + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + type: object + servers: + format: int32 + type: integer + tolerations: + items: + properties: + effect: + type: string + key: + type: string + operator: + type: string + tolerationSeconds: + format: int64 + type: integer + value: + type: string + type: object + type: array + volumeClaimTemplate: + properties: + apiVersion: + type: string + kind: + type: string + metadata: + properties: + annotations: + additionalProperties: + type: string + type: object + finalizers: + items: + type: string + type: array + labels: + additionalProperties: + type: string + type: object + name: + type: string + namespace: + type: string + type: object + spec: + properties: + accessModes: + items: + type: string + type: array + dataSource: + properties: + apiGroup: + type: string + kind: + type: string + name: + type: string + required: + - kind + - name + type: object + resources: + properties: + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + type: object + selector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + storageClassName: + type: string + volumeMode: + type: string + volumeName: + type: string + type: object + status: + properties: + accessModes: + items: + type: string + type: array + capacity: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + conditions: + items: + properties: + lastProbeTime: + format: date-time + type: string + lastTransitionTime: + format: date-time + type: string + message: + type: string + reason: + type: string + status: + type: string + type: + type: string + required: + - status + - type + type: object + type: array + phase: + type: string + type: object + type: object + volumesPerServer: + format: int32 + type: integer + required: + - servers + - volumeClaimTemplate + - volumesPerServer + type: object + type: array + required: + - zones + type: object + status: + properties: + availableReplicas: + format: int32 + type: integer + certificates: + nullable: true + properties: + autoCertEnabled: + nullable: true + type: boolean + type: object + currentState: + type: string + drivesHealing: + format: int32 + type: integer + drivesOffline: + format: int32 + type: integer + drivesOnline: + format: int32 + type: integer + healthMessage: + type: string + healthStatus: + type: string + pools: + items: + properties: + legacySecurityContext: + type: boolean + ssName: + type: string + state: + type: string + required: + - ssName + - state + type: object + nullable: true + type: array + provisionedUsers: + type: boolean + revision: + format: int32 + type: integer + syncVersion: + type: string + usage: + properties: + capacity: + format: int64 + type: integer + rawCapacity: + format: int64 + type: integer + rawUsage: + format: int64 + type: integer + usage: + format: int64 + type: integer + type: object + waitingOnReady: + format: date-time + type: string + writeQuorum: + format: int32 + type: integer + required: + - availableReplicas + - certificates + - currentState + - pools + - revision + - syncVersion + type: object + required: + - spec + type: object + served: true + storage: false + subresources: + status: {} + - additionalPrinterColumns: + - jsonPath: .status.currentState + name: State + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v2 + schema: + openAPIV3Schema: + properties: + apiVersion: + type: string + kind: + type: string + metadata: + type: object + scheduler: + properties: + name: + type: string + required: + - name + type: object + spec: + properties: + certConfig: + properties: + commonName: + type: string + dnsNames: + items: + type: string + type: array + organizationName: + items: + type: string + type: array + type: object + configuration: + properties: + name: + type: string + type: object + credsSecret: + properties: + name: + type: string + type: object + env: + items: + properties: + name: + type: string + value: + type: string + valueFrom: + properties: + configMapKeyRef: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + fieldRef: + properties: + apiVersion: + type: string + fieldPath: + type: string + required: + - fieldPath + type: object + resourceFieldRef: + properties: + containerName: + type: string + divisor: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + type: string + required: + - resource + type: object + secretKeyRef: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + type: object + required: + - name + type: object + type: array + exposeServices: + properties: + console: + type: boolean + minio: + type: boolean + type: object + externalCaCertSecret: + items: + properties: + name: + type: string + type: + type: string + required: + - name + type: object + type: array + externalCertSecret: + items: + properties: + name: + type: string + type: + type: string + required: + - name + type: object + type: array + externalClientCertSecret: + properties: + name: + type: string + type: + type: string + required: + - name + type: object + image: + type: string + imagePullPolicy: + type: string + imagePullSecret: + properties: + name: + type: string + type: object + kes: + properties: + affinity: + properties: + nodeAffinity: + properties: + preferredDuringSchedulingIgnoredDuringExecution: + items: + properties: + preference: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchFields: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + type: object + weight: + format: int32 + type: integer + required: + - preference + - weight + type: object + type: array + requiredDuringSchedulingIgnoredDuringExecution: + properties: + nodeSelectorTerms: + items: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchFields: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + type: object + type: array + required: + - nodeSelectorTerms + type: object + type: object + podAffinity: + properties: + preferredDuringSchedulingIgnoredDuringExecution: + items: + properties: + podAffinityTerm: + properties: + labelSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + namespaces: + items: + type: string + type: array + topologyKey: + type: string + required: + - topologyKey + type: object + weight: + format: int32 + type: integer + required: + - podAffinityTerm + - weight + type: object + type: array + requiredDuringSchedulingIgnoredDuringExecution: + items: + properties: + labelSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + namespaces: + items: + type: string + type: array + topologyKey: + type: string + required: + - topologyKey + type: object + type: array + type: object + podAntiAffinity: + properties: + preferredDuringSchedulingIgnoredDuringExecution: + items: + properties: + podAffinityTerm: + properties: + labelSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + namespaces: + items: + type: string + type: array + topologyKey: + type: string + required: + - topologyKey + type: object + weight: + format: int32 + type: integer + required: + - podAffinityTerm + - weight + type: object + type: array + requiredDuringSchedulingIgnoredDuringExecution: + items: + properties: + labelSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + namespaces: + items: + type: string + type: array + topologyKey: + type: string + required: + - topologyKey + type: object + type: array + type: object + type: object + annotations: + additionalProperties: + type: string + type: object + clientCertSecret: + properties: + name: + type: string + type: + type: string + required: + - name + type: object + externalCertSecret: + properties: + name: + type: string + type: + type: string + required: + - name + type: object + image: + type: string + imagePullPolicy: + type: string + kesSecret: + properties: + name: + type: string + type: object + keyName: + type: string + labels: + additionalProperties: + type: string + type: object + nodeSelector: + additionalProperties: + type: string + type: object + replicas: + format: int32 + type: integer + resources: + properties: + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + type: object + securityContext: + properties: + fsGroup: + format: int64 + type: integer + fsGroupChangePolicy: + type: string + runAsGroup: + format: int64 + type: integer + runAsNonRoot: + type: boolean + runAsUser: + format: int64 + type: integer + seLinuxOptions: + properties: + level: + type: string + role: + type: string + type: + type: string + user: + type: string + type: object + seccompProfile: + properties: + localhostProfile: + type: string + type: + type: string + required: + - type + type: object + supplementalGroups: + items: + format: int64 + type: integer + type: array + sysctls: + items: + properties: + name: + type: string + value: + type: string + required: + - name + - value + type: object + type: array + windowsOptions: + properties: + gmsaCredentialSpec: + type: string + gmsaCredentialSpecName: + type: string + runAsUserName: + type: string + type: object + type: object + serviceAccountName: + type: string + tolerations: + items: + properties: + effect: + type: string + key: + type: string + operator: + type: string + tolerationSeconds: + format: int64 + type: integer + value: + type: string + type: object + type: array + topologySpreadConstraints: + items: + properties: + labelSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + maxSkew: + format: int32 + type: integer + topologyKey: + type: string + whenUnsatisfiable: + type: string + required: + - maxSkew + - topologyKey + - whenUnsatisfiable + type: object + type: array + required: + - kesSecret + type: object + liveness: + properties: + exec: + properties: + command: + items: + type: string + type: array + type: object + failureThreshold: + format: int32 + type: integer + httpGet: + properties: + host: + type: string + httpHeaders: + items: + properties: + name: + type: string + value: + type: string + required: + - name + - value + type: object + type: array + path: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + scheme: + type: string + required: + - port + type: object + initialDelaySeconds: + format: int32 + type: integer + periodSeconds: + format: int32 + type: integer + successThreshold: + format: int32 + type: integer + tcpSocket: + properties: + host: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + required: + - port + type: object + timeoutSeconds: + format: int32 + type: integer + type: object + log: + properties: + affinity: + properties: + nodeAffinity: + properties: + preferredDuringSchedulingIgnoredDuringExecution: + items: + properties: + preference: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchFields: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + type: object + weight: + format: int32 + type: integer + required: + - preference + - weight + type: object + type: array + requiredDuringSchedulingIgnoredDuringExecution: + properties: + nodeSelectorTerms: + items: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchFields: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + type: object + type: array + required: + - nodeSelectorTerms + type: object + type: object + podAffinity: + properties: + preferredDuringSchedulingIgnoredDuringExecution: + items: + properties: + podAffinityTerm: + properties: + labelSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + namespaces: + items: + type: string + type: array + topologyKey: + type: string + required: + - topologyKey + type: object + weight: + format: int32 + type: integer + required: + - podAffinityTerm + - weight + type: object + type: array + requiredDuringSchedulingIgnoredDuringExecution: + items: + properties: + labelSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + namespaces: + items: + type: string + type: array + topologyKey: + type: string + required: + - topologyKey + type: object + type: array + type: object + podAntiAffinity: + properties: + preferredDuringSchedulingIgnoredDuringExecution: + items: + properties: + podAffinityTerm: + properties: + labelSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + namespaces: + items: + type: string + type: array + topologyKey: + type: string + required: + - topologyKey + type: object + weight: + format: int32 + type: integer + required: + - podAffinityTerm + - weight + type: object + type: array + requiredDuringSchedulingIgnoredDuringExecution: + items: + properties: + labelSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + namespaces: + items: + type: string + type: array + topologyKey: + type: string + required: + - topologyKey + type: object + type: array + type: object + type: object + annotations: + additionalProperties: + type: string + type: object + audit: + properties: + diskCapacityGB: + type: integer + type: object + db: + properties: + affinity: + properties: + nodeAffinity: + properties: + preferredDuringSchedulingIgnoredDuringExecution: + items: + properties: + preference: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchFields: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + type: object + weight: + format: int32 + type: integer + required: + - preference + - weight + type: object + type: array + requiredDuringSchedulingIgnoredDuringExecution: + properties: + nodeSelectorTerms: + items: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchFields: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + type: object + type: array + required: + - nodeSelectorTerms + type: object + type: object + podAffinity: + properties: + preferredDuringSchedulingIgnoredDuringExecution: + items: + properties: + podAffinityTerm: + properties: + labelSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + namespaces: + items: + type: string + type: array + topologyKey: + type: string + required: + - topologyKey + type: object + weight: + format: int32 + type: integer + required: + - podAffinityTerm + - weight + type: object + type: array + requiredDuringSchedulingIgnoredDuringExecution: + items: + properties: + labelSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + namespaces: + items: + type: string + type: array + topologyKey: + type: string + required: + - topologyKey + type: object + type: array + type: object + podAntiAffinity: + properties: + preferredDuringSchedulingIgnoredDuringExecution: + items: + properties: + podAffinityTerm: + properties: + labelSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + namespaces: + items: + type: string + type: array + topologyKey: + type: string + required: + - topologyKey + type: object + weight: + format: int32 + type: integer + required: + - podAffinityTerm + - weight + type: object + type: array + requiredDuringSchedulingIgnoredDuringExecution: + items: + properties: + labelSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + namespaces: + items: + type: string + type: array + topologyKey: + type: string + required: + - topologyKey + type: object + type: array + type: object + type: object + annotations: + additionalProperties: + type: string + type: object + image: + type: string + initimage: + type: string + labels: + additionalProperties: + type: string + type: object + nodeSelector: + additionalProperties: + type: string + type: object + resources: + properties: + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + type: object + securityContext: + properties: + fsGroup: + format: int64 + type: integer + fsGroupChangePolicy: + type: string + runAsGroup: + format: int64 + type: integer + runAsNonRoot: + type: boolean + runAsUser: + format: int64 + type: integer + seLinuxOptions: + properties: + level: + type: string + role: + type: string + type: + type: string + user: + type: string + type: object + seccompProfile: + properties: + localhostProfile: + type: string + type: + type: string + required: + - type + type: object + supplementalGroups: + items: + format: int64 + type: integer + type: array + sysctls: + items: + properties: + name: + type: string + value: + type: string + required: + - name + - value + type: object + type: array + windowsOptions: + properties: + gmsaCredentialSpec: + type: string + gmsaCredentialSpecName: + type: string + runAsUserName: + type: string + type: object + type: object + serviceAccountName: + type: string + tolerations: + items: + properties: + effect: + type: string + key: + type: string + operator: + type: string + tolerationSeconds: + format: int64 + type: integer + value: + type: string + type: object + type: array + topologySpreadConstraints: + items: + properties: + labelSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + maxSkew: + format: int32 + type: integer + topologyKey: + type: string + whenUnsatisfiable: + type: string + required: + - maxSkew + - topologyKey + - whenUnsatisfiable + type: object + type: array + volumeClaimTemplate: + properties: + apiVersion: + type: string + kind: + type: string + metadata: + properties: + annotations: + additionalProperties: + type: string + type: object + finalizers: + items: + type: string + type: array + labels: + additionalProperties: + type: string + type: object + name: + type: string + namespace: + type: string + type: object + spec: + properties: + accessModes: + items: + type: string + type: array + dataSource: + properties: + apiGroup: + type: string + kind: + type: string + name: + type: string + required: + - kind + - name + type: object + resources: + properties: + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + type: object + selector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + storageClassName: + type: string + volumeMode: + type: string + volumeName: + type: string + type: object + status: + properties: + accessModes: + items: + type: string + type: array + capacity: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + conditions: + items: + properties: + lastProbeTime: + format: date-time + type: string + lastTransitionTime: + format: date-time + type: string + message: + type: string + reason: + type: string + status: + type: string + type: + type: string + required: + - status + - type + type: object + type: array + phase: + type: string + type: object + type: object + required: + - volumeClaimTemplate + type: object + image: + type: string + labels: + additionalProperties: + type: string + type: object + nodeSelector: + additionalProperties: + type: string + type: object + resources: + properties: + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + type: object + securityContext: + properties: + fsGroup: + format: int64 + type: integer + fsGroupChangePolicy: + type: string + runAsGroup: + format: int64 + type: integer + runAsNonRoot: + type: boolean + runAsUser: + format: int64 + type: integer + seLinuxOptions: + properties: + level: + type: string + role: + type: string + type: + type: string + user: + type: string + type: object + seccompProfile: + properties: + localhostProfile: + type: string + type: + type: string + required: + - type + type: object + supplementalGroups: + items: + format: int64 + type: integer + type: array + sysctls: + items: + properties: + name: + type: string + value: + type: string + required: + - name + - value + type: object + type: array + windowsOptions: + properties: + gmsaCredentialSpec: + type: string + gmsaCredentialSpecName: + type: string + runAsUserName: + type: string + type: object + type: object + serviceAccountName: + type: string + tolerations: + items: + properties: + effect: + type: string + key: + type: string + operator: + type: string + tolerationSeconds: + format: int64 + type: integer + value: + type: string + type: object + type: array + topologySpreadConstraints: + items: + properties: + labelSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + maxSkew: + format: int32 + type: integer + topologyKey: + type: string + whenUnsatisfiable: + type: string + required: + - maxSkew + - topologyKey + - whenUnsatisfiable + type: object + type: array + type: object + logging: + properties: + anonymous: + type: boolean + json: + type: boolean + quiet: + type: boolean + type: object + mountPath: + type: string + podManagementPolicy: + type: string + pools: + items: + properties: + affinity: + properties: + nodeAffinity: + properties: + preferredDuringSchedulingIgnoredDuringExecution: + items: + properties: + preference: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchFields: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + type: object + weight: + format: int32 + type: integer + required: + - preference + - weight + type: object + type: array + requiredDuringSchedulingIgnoredDuringExecution: + properties: + nodeSelectorTerms: + items: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchFields: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + type: object + type: array + required: + - nodeSelectorTerms + type: object + type: object + podAffinity: + properties: + preferredDuringSchedulingIgnoredDuringExecution: + items: + properties: + podAffinityTerm: + properties: + labelSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + namespaces: + items: + type: string + type: array + topologyKey: + type: string + required: + - topologyKey + type: object + weight: + format: int32 + type: integer + required: + - podAffinityTerm + - weight + type: object + type: array + requiredDuringSchedulingIgnoredDuringExecution: + items: + properties: + labelSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + namespaces: + items: + type: string + type: array + topologyKey: + type: string + required: + - topologyKey + type: object + type: array + type: object + podAntiAffinity: + properties: + preferredDuringSchedulingIgnoredDuringExecution: + items: + properties: + podAffinityTerm: + properties: + labelSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + namespaces: + items: + type: string + type: array + topologyKey: + type: string + required: + - topologyKey + type: object + weight: + format: int32 + type: integer + required: + - podAffinityTerm + - weight + type: object + type: array + requiredDuringSchedulingIgnoredDuringExecution: + items: + properties: + labelSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + namespaces: + items: + type: string + type: array + topologyKey: + type: string + required: + - topologyKey + type: object + type: array + type: object + type: object + annotations: + additionalProperties: + type: string + type: object + labels: + additionalProperties: + type: string + type: object + name: + type: string + nodeSelector: + additionalProperties: + type: string + type: object + resources: + properties: + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + type: object + securityContext: + properties: + fsGroup: + format: int64 + type: integer + fsGroupChangePolicy: + type: string + runAsGroup: + format: int64 + type: integer + runAsNonRoot: + type: boolean + runAsUser: + format: int64 + type: integer + seLinuxOptions: + properties: + level: + type: string + role: + type: string + type: + type: string + user: + type: string + type: object + seccompProfile: + properties: + localhostProfile: + type: string + type: + type: string + required: + - type + type: object + supplementalGroups: + items: + format: int64 + type: integer + type: array + sysctls: + items: + properties: + name: + type: string + value: + type: string + required: + - name + - value + type: object + type: array + windowsOptions: + properties: + gmsaCredentialSpec: + type: string + gmsaCredentialSpecName: + type: string + runAsUserName: + type: string + type: object + type: object + servers: + format: int32 + type: integer + tolerations: + items: + properties: + effect: + type: string + key: + type: string + operator: + type: string + tolerationSeconds: + format: int64 + type: integer + value: + type: string + type: object + type: array + topologySpreadConstraints: + items: + properties: + labelSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + maxSkew: + format: int32 + type: integer + topologyKey: + type: string + whenUnsatisfiable: + type: string + required: + - maxSkew + - topologyKey + - whenUnsatisfiable + type: object + type: array + volumeClaimTemplate: + properties: + apiVersion: + type: string + kind: + type: string + metadata: + properties: + annotations: + additionalProperties: + type: string + type: object + finalizers: + items: + type: string + type: array + labels: + additionalProperties: + type: string + type: object + name: + type: string + namespace: + type: string + type: object + spec: + properties: + accessModes: + items: + type: string + type: array + dataSource: + properties: + apiGroup: + type: string + kind: + type: string + name: + type: string + required: + - kind + - name + type: object + resources: + properties: + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + type: object + selector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + storageClassName: + type: string + volumeMode: + type: string + volumeName: + type: string + type: object + status: + properties: + accessModes: + items: + type: string + type: array + capacity: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + conditions: + items: + properties: + lastProbeTime: + format: date-time + type: string + lastTransitionTime: + format: date-time + type: string + message: + type: string + reason: + type: string + status: + type: string + type: + type: string + required: + - status + - type + type: object + type: array + phase: + type: string + type: object + type: object + volumesPerServer: + format: int32 + type: integer + required: + - servers + - volumeClaimTemplate + - volumesPerServer + type: object + type: array + priorityClassName: + type: string + prometheus: + properties: + affinity: + properties: + nodeAffinity: + properties: + preferredDuringSchedulingIgnoredDuringExecution: + items: + properties: + preference: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchFields: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + type: object + weight: + format: int32 + type: integer + required: + - preference + - weight + type: object + type: array + requiredDuringSchedulingIgnoredDuringExecution: + properties: + nodeSelectorTerms: + items: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchFields: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + type: object + type: array + required: + - nodeSelectorTerms + type: object + type: object + podAffinity: + properties: + preferredDuringSchedulingIgnoredDuringExecution: + items: + properties: + podAffinityTerm: + properties: + labelSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + namespaces: + items: + type: string + type: array + topologyKey: + type: string + required: + - topologyKey + type: object + weight: + format: int32 + type: integer + required: + - podAffinityTerm + - weight + type: object + type: array + requiredDuringSchedulingIgnoredDuringExecution: + items: + properties: + labelSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + namespaces: + items: + type: string + type: array + topologyKey: + type: string + required: + - topologyKey + type: object + type: array + type: object + podAntiAffinity: + properties: + preferredDuringSchedulingIgnoredDuringExecution: + items: + properties: + podAffinityTerm: + properties: + labelSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + namespaces: + items: + type: string + type: array + topologyKey: + type: string + required: + - topologyKey + type: object + weight: + format: int32 + type: integer + required: + - podAffinityTerm + - weight + type: object + type: array + requiredDuringSchedulingIgnoredDuringExecution: + items: + properties: + labelSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + namespaces: + items: + type: string + type: array + topologyKey: + type: string + required: + - topologyKey + type: object + type: array + type: object + type: object + annotations: + additionalProperties: + type: string + type: object + diskCapacityGB: + type: integer + image: + type: string + initimage: + type: string + labels: + additionalProperties: + type: string + type: object + nodeSelector: + additionalProperties: + type: string + type: object + resources: + properties: + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + type: object + securityContext: + properties: + fsGroup: + format: int64 + type: integer + fsGroupChangePolicy: + type: string + runAsGroup: + format: int64 + type: integer + runAsNonRoot: + type: boolean + runAsUser: + format: int64 + type: integer + seLinuxOptions: + properties: + level: + type: string + role: + type: string + type: + type: string + user: + type: string + type: object + seccompProfile: + properties: + localhostProfile: + type: string + type: + type: string + required: + - type + type: object + supplementalGroups: + items: + format: int64 + type: integer + type: array + sysctls: + items: + properties: + name: + type: string + value: + type: string + required: + - name + - value + type: object + type: array + windowsOptions: + properties: + gmsaCredentialSpec: + type: string + gmsaCredentialSpecName: + type: string + runAsUserName: + type: string + type: object + type: object + serviceAccountName: + type: string + sidecarimage: + type: string + storageClassName: + type: string + topologySpreadConstraints: + items: + properties: + labelSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + maxSkew: + format: int32 + type: integer + topologyKey: + type: string + whenUnsatisfiable: + type: string + required: + - maxSkew + - topologyKey + - whenUnsatisfiable + type: object + type: array + type: object + prometheusOperator: + type: boolean + readiness: + properties: + exec: + properties: + command: + items: + type: string + type: array + type: object + failureThreshold: + format: int32 + type: integer + httpGet: + properties: + host: + type: string + httpHeaders: + items: + properties: + name: + type: string + value: + type: string + required: + - name + - value + type: object + type: array + path: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + scheme: + type: string + required: + - port + type: object + initialDelaySeconds: + format: int32 + type: integer + periodSeconds: + format: int32 + type: integer + successThreshold: + format: int32 + type: integer + tcpSocket: + properties: + host: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + required: + - port + type: object + timeoutSeconds: + format: int32 + type: integer + type: object + requestAutoCert: + type: boolean + s3: + properties: + bucketDNS: + type: boolean + required: + - bucketDNS + type: object + serviceAccountName: + type: string + serviceMetadata: + properties: + consoleServiceAnnotations: + additionalProperties: + type: string + type: object + consoleServiceLabels: + additionalProperties: + type: string + type: object + minioServiceAnnotations: + additionalProperties: + type: string + type: object + minioServiceLabels: + additionalProperties: + type: string + type: object + type: object + sideCars: + properties: + containers: + items: + properties: + args: + items: + type: string + type: array + command: + items: + type: string + type: array + env: + items: + properties: + name: + type: string + value: + type: string + valueFrom: + properties: + configMapKeyRef: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + fieldRef: + properties: + apiVersion: + type: string + fieldPath: + type: string + required: + - fieldPath + type: object + resourceFieldRef: + properties: + containerName: + type: string + divisor: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + type: string + required: + - resource + type: object + secretKeyRef: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + type: object + required: + - name + type: object + type: array + envFrom: + items: + properties: + configMapRef: + properties: + name: + type: string + optional: + type: boolean + type: object + prefix: + type: string + secretRef: + properties: + name: + type: string + optional: + type: boolean + type: object + type: object + type: array + image: + type: string + imagePullPolicy: + type: string + lifecycle: + properties: + postStart: + properties: + exec: + properties: + command: + items: + type: string + type: array + type: object + httpGet: + properties: + host: + type: string + httpHeaders: + items: + properties: + name: + type: string + value: + type: string + required: + - name + - value + type: object + type: array + path: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + scheme: + type: string + required: + - port + type: object + tcpSocket: + properties: + host: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + required: + - port + type: object + type: object + preStop: + properties: + exec: + properties: + command: + items: + type: string + type: array + type: object + httpGet: + properties: + host: + type: string + httpHeaders: + items: + properties: + name: + type: string + value: + type: string + required: + - name + - value + type: object + type: array + path: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + scheme: + type: string + required: + - port + type: object + tcpSocket: + properties: + host: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + required: + - port + type: object + type: object + type: object + livenessProbe: + properties: + exec: + properties: + command: + items: + type: string + type: array + type: object + failureThreshold: + format: int32 + type: integer + httpGet: + properties: + host: + type: string + httpHeaders: + items: + properties: + name: + type: string + value: + type: string + required: + - name + - value + type: object + type: array + path: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + scheme: + type: string + required: + - port + type: object + initialDelaySeconds: + format: int32 + type: integer + periodSeconds: + format: int32 + type: integer + successThreshold: + format: int32 + type: integer + tcpSocket: + properties: + host: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + required: + - port + type: object + timeoutSeconds: + format: int32 + type: integer + type: object + name: + type: string + ports: + items: + properties: + containerPort: + format: int32 + type: integer + hostIP: + type: string + hostPort: + format: int32 + type: integer + name: + type: string + protocol: + default: TCP + type: string + required: + - containerPort + type: object + type: array + x-kubernetes-list-map-keys: + - containerPort + - protocol + x-kubernetes-list-type: map + readinessProbe: + properties: + exec: + properties: + command: + items: + type: string + type: array + type: object + failureThreshold: + format: int32 + type: integer + httpGet: + properties: + host: + type: string + httpHeaders: + items: + properties: + name: + type: string + value: + type: string + required: + - name + - value + type: object + type: array + path: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + scheme: + type: string + required: + - port + type: object + initialDelaySeconds: + format: int32 + type: integer + periodSeconds: + format: int32 + type: integer + successThreshold: + format: int32 + type: integer + tcpSocket: + properties: + host: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + required: + - port + type: object + timeoutSeconds: + format: int32 + type: integer + type: object + resources: + properties: + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + type: object + securityContext: + properties: + allowPrivilegeEscalation: + type: boolean + capabilities: + properties: + add: + items: + type: string + type: array + drop: + items: + type: string + type: array + type: object + privileged: + type: boolean + procMount: + type: string + readOnlyRootFilesystem: + type: boolean + runAsGroup: + format: int64 + type: integer + runAsNonRoot: + type: boolean + runAsUser: + format: int64 + type: integer + seLinuxOptions: + properties: + level: + type: string + role: + type: string + type: + type: string + user: + type: string + type: object + seccompProfile: + properties: + localhostProfile: + type: string + type: + type: string + required: + - type + type: object + windowsOptions: + properties: + gmsaCredentialSpec: + type: string + gmsaCredentialSpecName: + type: string + runAsUserName: + type: string + type: object + type: object + startupProbe: + properties: + exec: + properties: + command: + items: + type: string + type: array + type: object + failureThreshold: + format: int32 + type: integer + httpGet: + properties: + host: + type: string + httpHeaders: + items: + properties: + name: + type: string + value: + type: string + required: + - name + - value + type: object + type: array + path: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + scheme: + type: string + required: + - port + type: object + initialDelaySeconds: + format: int32 + type: integer + periodSeconds: + format: int32 + type: integer + successThreshold: + format: int32 + type: integer + tcpSocket: + properties: + host: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + required: + - port + type: object + timeoutSeconds: + format: int32 + type: integer + type: object + stdin: + type: boolean + stdinOnce: + type: boolean + terminationMessagePath: + type: string + terminationMessagePolicy: + type: string + tty: + type: boolean + volumeDevices: + items: + properties: + devicePath: + type: string + name: + type: string + required: + - devicePath + - name + type: object + type: array + volumeMounts: + items: + properties: + mountPath: + type: string + mountPropagation: + type: string + name: + type: string + readOnly: + type: boolean + subPath: + type: string + subPathExpr: + type: string + required: + - mountPath + - name + type: object + type: array + workingDir: + type: string + required: + - name + type: object + type: array + volumeClaimTemplates: + items: + properties: + apiVersion: + type: string + kind: + type: string + metadata: + properties: + annotations: + additionalProperties: + type: string + type: object + finalizers: + items: + type: string + type: array + labels: + additionalProperties: + type: string + type: object + name: + type: string + namespace: + type: string + type: object + spec: + properties: + accessModes: + items: + type: string + type: array + dataSource: + properties: + apiGroup: + type: string + kind: + type: string + name: + type: string + required: + - kind + - name + type: object + resources: + properties: + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + type: object + selector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + storageClassName: + type: string + volumeMode: + type: string + volumeName: + type: string + type: object + status: + properties: + accessModes: + items: + type: string + type: array + capacity: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + conditions: + items: + properties: + lastProbeTime: + format: date-time + type: string + lastTransitionTime: + format: date-time + type: string + message: + type: string + reason: + type: string + status: + type: string + type: + type: string + required: + - status + - type + type: object + type: array + phase: + type: string + type: object + type: object + type: array + volumes: + items: + properties: + awsElasticBlockStore: + properties: + fsType: + type: string + partition: + format: int32 + type: integer + readOnly: + type: boolean + volumeID: + type: string + required: + - volumeID + type: object + azureDisk: + properties: + cachingMode: + type: string + diskName: + type: string + diskURI: + type: string + fsType: + type: string + kind: + type: string + readOnly: + type: boolean + required: + - diskName + - diskURI + type: object + azureFile: + properties: + readOnly: + type: boolean + secretName: + type: string + shareName: + type: string + required: + - secretName + - shareName + type: object + cephfs: + properties: + monitors: + items: + type: string + type: array + path: + type: string + readOnly: + type: boolean + secretFile: + type: string + secretRef: + properties: + name: + type: string + type: object + user: + type: string + required: + - monitors + type: object + cinder: + properties: + fsType: + type: string + readOnly: + type: boolean + secretRef: + properties: + name: + type: string + type: object + volumeID: + type: string + required: + - volumeID + type: object + configMap: + properties: + defaultMode: + format: int32 + type: integer + items: + items: + properties: + key: + type: string + mode: + format: int32 + type: integer + path: + type: string + required: + - key + - path + type: object + type: array + name: + type: string + optional: + type: boolean + type: object + csi: + properties: + driver: + type: string + fsType: + type: string + nodePublishSecretRef: + properties: + name: + type: string + type: object + readOnly: + type: boolean + volumeAttributes: + additionalProperties: + type: string + type: object + required: + - driver + type: object + downwardAPI: + properties: + defaultMode: + format: int32 + type: integer + items: + items: + properties: + fieldRef: + properties: + apiVersion: + type: string + fieldPath: + type: string + required: + - fieldPath + type: object + mode: + format: int32 + type: integer + path: + type: string + resourceFieldRef: + properties: + containerName: + type: string + divisor: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + type: string + required: + - resource + type: object + required: + - path + type: object + type: array + type: object + emptyDir: + properties: + medium: + type: string + sizeLimit: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + ephemeral: + properties: + readOnly: + type: boolean + volumeClaimTemplate: + properties: + metadata: + properties: + annotations: + additionalProperties: + type: string + type: object + finalizers: + items: + type: string + type: array + labels: + additionalProperties: + type: string + type: object + name: + type: string + namespace: + type: string + type: object + spec: + properties: + accessModes: + items: + type: string + type: array + dataSource: + properties: + apiGroup: + type: string + kind: + type: string + name: + type: string + required: + - kind + - name + type: object + resources: + properties: + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + type: object + selector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + storageClassName: + type: string + volumeMode: + type: string + volumeName: + type: string + type: object + required: + - spec + type: object + type: object + fc: + properties: + fsType: + type: string + lun: + format: int32 + type: integer + readOnly: + type: boolean + targetWWNs: + items: + type: string + type: array + wwids: + items: + type: string + type: array + type: object + flexVolume: + properties: + driver: + type: string + fsType: + type: string + options: + additionalProperties: + type: string + type: object + readOnly: + type: boolean + secretRef: + properties: + name: + type: string + type: object + required: + - driver + type: object + flocker: + properties: + datasetName: + type: string + datasetUUID: + type: string + type: object + gcePersistentDisk: + properties: + fsType: + type: string + partition: + format: int32 + type: integer + pdName: + type: string + readOnly: + type: boolean + required: + - pdName + type: object + gitRepo: + properties: + directory: + type: string + repository: + type: string + revision: + type: string + required: + - repository + type: object + glusterfs: + properties: + endpoints: + type: string + path: + type: string + readOnly: + type: boolean + required: + - endpoints + - path + type: object + hostPath: + properties: + path: + type: string + type: + type: string + required: + - path + type: object + iscsi: + properties: + chapAuthDiscovery: + type: boolean + chapAuthSession: + type: boolean + fsType: + type: string + initiatorName: + type: string + iqn: + type: string + iscsiInterface: + type: string + lun: + format: int32 + type: integer + portals: + items: + type: string + type: array + readOnly: + type: boolean + secretRef: + properties: + name: + type: string + type: object + targetPortal: + type: string + required: + - iqn + - lun + - targetPortal + type: object + name: + type: string + nfs: + properties: + path: + type: string + readOnly: + type: boolean + server: + type: string + required: + - path + - server + type: object + persistentVolumeClaim: + properties: + claimName: + type: string + readOnly: + type: boolean + required: + - claimName + type: object + photonPersistentDisk: + properties: + fsType: + type: string + pdID: + type: string + required: + - pdID + type: object + portworxVolume: + properties: + fsType: + type: string + readOnly: + type: boolean + volumeID: + type: string + required: + - volumeID + type: object + projected: + properties: + defaultMode: + format: int32 + type: integer + sources: + items: + properties: + configMap: + properties: + items: + items: + properties: + key: + type: string + mode: + format: int32 + type: integer + path: + type: string + required: + - key + - path + type: object + type: array + name: + type: string + optional: + type: boolean + type: object + downwardAPI: + properties: + items: + items: + properties: + fieldRef: + properties: + apiVersion: + type: string + fieldPath: + type: string + required: + - fieldPath + type: object + mode: + format: int32 + type: integer + path: + type: string + resourceFieldRef: + properties: + containerName: + type: string + divisor: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + type: string + required: + - resource + type: object + required: + - path + type: object + type: array + type: object + secret: + properties: + items: + items: + properties: + key: + type: string + mode: + format: int32 + type: integer + path: + type: string + required: + - key + - path + type: object + type: array + name: + type: string + optional: + type: boolean + type: object + serviceAccountToken: + properties: + audience: + type: string + expirationSeconds: + format: int64 + type: integer + path: + type: string + required: + - path + type: object + type: object + type: array + type: object + quobyte: + properties: + group: + type: string + readOnly: + type: boolean + registry: + type: string + tenant: + type: string + user: + type: string + volume: + type: string + required: + - registry + - volume + type: object + rbd: + properties: + fsType: + type: string + image: + type: string + keyring: + type: string + monitors: + items: + type: string + type: array + pool: + type: string + readOnly: + type: boolean + secretRef: + properties: + name: + type: string + type: object + user: + type: string + required: + - image + - monitors + type: object + scaleIO: + properties: + fsType: + type: string + gateway: + type: string + protectionDomain: + type: string + readOnly: + type: boolean + secretRef: + properties: + name: + type: string + type: object + sslEnabled: + type: boolean + storageMode: + type: string + storagePool: + type: string + system: + type: string + volumeName: + type: string + required: + - gateway + - secretRef + - system + type: object + secret: + properties: + defaultMode: + format: int32 + type: integer + items: + items: + properties: + key: + type: string + mode: + format: int32 + type: integer + path: + type: string + required: + - key + - path + type: object + type: array + optional: + type: boolean + secretName: + type: string + type: object + storageos: + properties: + fsType: + type: string + readOnly: + type: boolean + secretRef: + properties: + name: + type: string + type: object + volumeName: + type: string + volumeNamespace: + type: string + type: object + vsphereVolume: + properties: + fsType: + type: string + storagePolicyID: + type: string + storagePolicyName: + type: string + volumePath: + type: string + required: + - volumePath + type: object + required: + - name + type: object + type: array + required: + - containers + type: object + subPath: + type: string + users: + items: + properties: + name: + type: string + type: object + type: array + required: + - pools + type: object + status: + properties: + availableReplicas: + format: int32 + type: integer + certificates: + nullable: true + properties: + autoCertEnabled: + nullable: true + type: boolean + type: object + currentState: + type: string + drivesHealing: + format: int32 + type: integer + drivesOffline: + format: int32 + type: integer + drivesOnline: + format: int32 + type: integer + healthMessage: + type: string + healthStatus: + type: string + pools: + items: + properties: + legacySecurityContext: + type: boolean + ssName: + type: string + state: + type: string + required: + - ssName + - state + type: object + nullable: true + type: array + provisionedUsers: + type: boolean + revision: + format: int32 + type: integer + syncVersion: + type: string + usage: + properties: + capacity: + format: int64 + type: integer + rawCapacity: + format: int64 + type: integer + rawUsage: + format: int64 + type: integer + usage: + format: int64 + type: integer + type: object + waitingOnReady: + format: date-time + type: string + writeQuorum: + format: int32 + type: integer + required: + - availableReplicas + - certificates + - currentState + - pools + - revision + - syncVersion + type: object + required: + - spec + type: object + served: true + storage: true + subresources: + status: {} +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] diff --git a/roles/minio_install/charts/operator/templates/NOTES.txt b/roles/minio_install/charts/operator/templates/NOTES.txt new file mode 100755 index 00000000..d2fd6ae4 --- /dev/null +++ b/roles/minio_install/charts/operator/templates/NOTES.txt @@ -0,0 +1,6 @@ +1. Get the JWT for logging in to the console: + kubectl get secret $(kubectl get serviceaccount console-sa --namespace {{ .Release.Namespace }} -o jsonpath="{.secrets[0].name}") --namespace {{ .Release.Namespace }} -o jsonpath="{.data.token}" | base64 --decode + +2. Get the Operator Console URL by running these commands: + kubectl --namespace {{ .Release.Namespace }} port-forward svc/console 9090:9090 + echo "Visit the Operator Console at http://127.0.0.1:9090" diff --git a/roles/minio_install/charts/operator/templates/_helpers.tpl b/roles/minio_install/charts/operator/templates/_helpers.tpl new file mode 100755 index 00000000..5a969452 --- /dev/null +++ b/roles/minio_install/charts/operator/templates/_helpers.tpl @@ -0,0 +1,84 @@ +{{/* vim: set filetype=mustache: */}} +{{/* +Expand the name of the chart. +*/}} +{{- define "minio-operator.name" -}} +{{- default .Chart.Name | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "minio-operator.fullname" -}} +{{- $name := default .Chart.Name -}} +{{- if contains $name .Release.Name -}} +{{- .Release.Name | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}} +{{- end -}} +{{- end -}} + +{{/* +Expand the name of the Operator Console. +*/}} +{{- define "minio-operator.console-name" -}} +{{- printf "%s-%s" .Chart.Name "console" | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Create a default fully qualified console name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "minio-operator.console-fullname" -}} +{{- printf "%s-%s" .Release.Name "console" | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "minio-operator.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Common labels for operator +*/}} +{{- define "minio-operator.labels" -}} +helm.sh/chart: {{ include "minio-operator.chart" . }} +{{ include "minio-operator.selectorLabels" . }} +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end -}} + +{{/* +Selector labels Operator +*/}} +{{- define "minio-operator.selectorLabels" -}} +app.kubernetes.io/name: {{ include "minio-operator.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end -}} + +{{/* +Common labels for console +*/}} +{{- define "minio-operator.console-labels" -}} +helm.sh/chart: {{ include "minio-operator.chart" . }} +{{ include "minio-operator.console-selectorLabels" . }} +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end -}} + +{{/* +Selector labels Operator +*/}} +{{- define "minio-operator.console-selectorLabels" -}} +app.kubernetes.io/name: {{ include "minio-operator.name" . }} +app.kubernetes.io/instance: {{ printf "%s-%s" .Release.Name "console" }} +{{- end -}} diff --git a/roles/minio_install/charts/operator/templates/cluster-role-binding.yaml b/roles/minio_install/charts/operator/templates/cluster-role-binding.yaml new file mode 100755 index 00000000..b991fe86 --- /dev/null +++ b/roles/minio_install/charts/operator/templates/cluster-role-binding.yaml @@ -0,0 +1,12 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: minio-operator-binding +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: minio-operator-role +subjects: + - kind: ServiceAccount + name: minio-operator + namespace: {{ .Release.Namespace }} diff --git a/roles/minio_install/charts/operator/templates/cluster-role.yaml b/roles/minio_install/charts/operator/templates/cluster-role.yaml new file mode 100755 index 00000000..239181ca --- /dev/null +++ b/roles/minio_install/charts/operator/templates/cluster-role.yaml @@ -0,0 +1,131 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: minio-operator-role +rules: + - apiGroups: + - "apiextensions.k8s.io" + resources: + - customresourcedefinitions + verbs: + - get + - update + - apiGroups: + - "" + resources: + - persistentvolumeclaims + verbs: + - get + - update + - list + - apiGroups: + - "" + resources: + - namespaces + verbs: + - get + - watch + - list + - apiGroups: + - "" + resources: + - pods + - services + - events + - configmaps + verbs: + - get + - watch + - patch + - create + - list + - delete + - deletecollection + - update + - apiGroups: + - "" + resources: + - secrets + verbs: + - get + - watch + - create + - update + - list + - delete + - deletecollection + - apiGroups: + - apps + resources: + - statefulsets + - deployments + verbs: + - get + - create + - list + - patch + - watch + - update + - delete + - apiGroups: + - batch + resources: + - jobs + verbs: + - get + - create + - list + - patch + - watch + - update + - delete + - apiGroups: + - "certificates.k8s.io" + resources: + - "certificatesigningrequests" + - "certificatesigningrequests/approval" + - "certificatesigningrequests/status" + verbs: + - update + - create + - get + - delete + - apiGroups: + - certificates.k8s.io + resourceNames: + - kubernetes.io/legacy-unknown + - kubernetes.io/kube-apiserver-client + - kubernetes.io/kubelet-serving + resources: + - signers + verbs: + - approve + - sign + - apiGroups: + - minio.min.io + resources: + - "*" + verbs: + - "*" + - apiGroups: + - min.io + resources: + - "*" + verbs: + - "*" + - apiGroups: + - monitoring.coreos.com + resources: + - servicemonitors + verbs: + - get + - create + - list + - apiGroups: + - "coordination.k8s.io" + resources: + - leases + verbs: + - get + - update + - create diff --git a/roles/minio_install/charts/operator/templates/console-deployment.yaml b/roles/minio_install/charts/operator/templates/console-deployment.yaml new file mode 100755 index 00000000..0f1b4cfb --- /dev/null +++ b/roles/minio_install/charts/operator/templates/console-deployment.yaml @@ -0,0 +1,65 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: "console" + namespace: {{ .Release.Namespace }} + labels: + {{- include "minio-operator.labels" . | nindent 4 }} +spec: + replicas: {{ .Values.console.replicaCount }} + selector: + matchLabels: + {{- include "minio-operator.console-selectorLabels" . | nindent 6 }} + template: + metadata: + labels: + {{- include "minio-operator.console-selectorLabels" . | nindent 8 }} + spec: + {{- with .Values.operator.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + serviceAccountName: console-sa + {{- with .Values.console.securityContext }} + securityContext: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.console.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.console.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.console.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.console.topologySpreadConstraints }} + topologySpreadConstraints: + {{- toYaml . | nindent 8 }} + {{- end }} + containers: + - name: {{ .Chart.Name }} + image: "{{ .Values.console.image.repository }}:{{ .Values.console.image.tag }}" + imagePullPolicy: {{ .Values.operator.image.pullPolicy }} + ports: + - containerPort: 9090 + name: http + - containerPort: 9443 + name: https + args: + - server + env: + - name: CONSOLE_OPERATOR_MODE + value: "on" + {{- with .Values.console.env }} + {{ toYaml . | nindent 12 }} + {{- end }} + resources: + {{- toYaml .Values.console.resources | nindent 12 }} + {{- with .Values.console.initContainers }} + initContainers: + {{- toYaml . | nindent 8 }} + {{- end}} diff --git a/roles/minio_install/charts/operator/templates/console-ingress.yaml b/roles/minio_install/charts/operator/templates/console-ingress.yaml new file mode 100755 index 00000000..c573882e --- /dev/null +++ b/roles/minio_install/charts/operator/templates/console-ingress.yaml @@ -0,0 +1,50 @@ +{{- if .Values.console.ingress.enabled }} +{{- if .Capabilities.APIVersions.Has "networking.k8s.io/v1/Ingress" -}} +apiVersion: networking.k8s.io/v1 +{{- else if .Capabilities.APIVersions.Has "networking.k8s.io/v1beta1/Ingress" -}} +apiVersion: networking.k8s.io/v1beta1 +{{- else -}} +apiVersion: extensions/v1beta1 +{{- end }} +kind: Ingress +metadata: + name: {{ include "minio-operator.console-fullname" . }} + namespace: {{ .Release.Namespace }} + {{- with .Values.console.ingress.labels }} + labels: {{ toYaml . | nindent 4 }} + {{- end }} + {{- with .Values.console.ingress.annotations }} + annotations: {{ toYaml . | nindent 4 }} + {{- end }} +spec: + {{- if .Values.console.ingress.ingressClassName }} + ingressClassName: {{ .Values.console.ingress.ingressClassName }} + {{- end }} + {{- if .Values.console.ingress.tls }} + tls: + {{- range .Values.console.ingress.tls }} + - hosts: + {{- range .hosts }} + - {{ . | quote }} + {{- end }} + secretName: {{ .secretName }} + {{- end }} + {{- end }} + rules: + - host: {{ .Values.console.ingress.host }} + http: + paths: + - path: {{ .Values.console.ingress.path }} + {{- if $.Capabilities.APIVersions.Has "networking.k8s.io/v1/Ingress" }} + pathType: Prefix + backend: + service: + name: "console" + port: + name: http + {{- else }} + backend: + serviceName: "console" + servicePort: http + {{ end }} +{{ end }} diff --git a/roles/minio_install/charts/operator/templates/console-service.yaml b/roles/minio_install/charts/operator/templates/console-service.yaml new file mode 100755 index 00000000..1677612b --- /dev/null +++ b/roles/minio_install/charts/operator/templates/console-service.yaml @@ -0,0 +1,15 @@ +apiVersion: v1 +kind: Service +metadata: + name: "console" + namespace: {{ .Release.Namespace }} + labels: + {{- include "minio-operator.labels" . | nindent 4 }} +spec: + ports: + - name: http + port: 9090 + - name: https + port: 9443 + selector: + {{- include "minio-operator.console-selectorLabels" . | nindent 4 }} diff --git a/roles/minio_install/charts/operator/templates/console-ui.yaml b/roles/minio_install/charts/operator/templates/console-ui.yaml new file mode 100755 index 00000000..72eb6b0c --- /dev/null +++ b/roles/minio_install/charts/operator/templates/console-ui.yaml @@ -0,0 +1,262 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: console-sa + namespace: {{ .Release.Namespace }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: console-sa-role +rules: + - apiGroups: + - "" + resources: + - secrets + verbs: + - get + - watch + - create + - list + - patch + - update + - deletecollection + - apiGroups: + - "" + resources: + - namespaces + - services + - events + - resourcequotas + - nodes + verbs: + - get + - watch + - create + - list + - patch + - apiGroups: + - "" + resources: + - pods + verbs: + - get + - watch + - create + - list + - patch + - delete + - deletecollection + - apiGroups: + - "" + resources: + - persistentvolumeclaims + verbs: + - deletecollection + - list + - get + - watch + - update + - apiGroups: + - storage.k8s.io + resources: + - storageclasses + verbs: + - get + - watch + - create + - list + - patch + - apiGroups: + - apps + resources: + - statefulsets + - deployments + verbs: + - get + - create + - list + - patch + - watch + - update + - delete + - apiGroups: + - batch + resources: + - jobs + verbs: + - get + - create + - list + - patch + - watch + - update + - delete + - apiGroups: + - certificates.k8s.io + resources: + - certificatesigningrequests + - certificatesigningrequests/approval + - certificatesigningrequests/status + verbs: + - update + - create + - get + - apiGroups: + - minio.min.io + resources: + - '*' + verbs: + - '*' + - apiGroups: + - min.io + resources: + - '*' + verbs: + - '*' + - apiGroups: + - "" + resources: + - persistentvolumes + verbs: + - get + - list + - watch + - create + - delete + - apiGroups: + - "" + resources: + - persistentvolumeclaims + verbs: + - get + - list + - watch + - update + - apiGroups: + - "" + resources: + - events + verbs: + - create + - list + - watch + - update + - patch + - apiGroups: + - snapshot.storage.k8s.io + resources: + - volumesnapshots + verbs: + - get + - list + - apiGroups: + - snapshot.storage.k8s.io + resources: + - volumesnapshotcontents + verbs: + - get + - list + - apiGroups: + - storage.k8s.io + resources: + - csinodes + verbs: + - get + - list + - watch + - apiGroups: + - storage.k8s.io + resources: + - volumeattachments + verbs: + - get + - list + - watch + - apiGroups: + - "" + resources: + - endpoints + verbs: + - get + - list + - watch + - create + - update + - delete + - apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: + - get + - list + - watch + - create + - update + - delete + - apiGroups: + - direct.csi.min.io + resources: + - volumes + verbs: + - get + - list + - watch + - create + - update + - delete + - apiGroups: + - apiextensions.k8s.io + resources: + - customresourcedefinitions + verbs: + - get + - list + - watch + - create + - update + - delete + - apiGroups: + - direct.csi.min.io + resources: + - directcsidrives + - directcsivolumes + verbs: + - get + - list + - watch + - create + - update + - delete + - apiGroups: + - "" + resources: + - pod + - pods/log + verbs: + - get + - list + - watch +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: console-sa-binding +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: console-sa-role +subjects: + - kind: ServiceAccount + name: console-sa + namespace: {{ .Release.Namespace }} +--- +apiVersion: v1 +data: + CONSOLE_PORT: "9090" + CONSOLE_TLS_PORT: "9443" +kind: ConfigMap +metadata: + name: console-env + namespace: {{ .Release.Namespace }} diff --git a/roles/minio_install/charts/operator/templates/operator-deployment.yaml b/roles/minio_install/charts/operator/templates/operator-deployment.yaml new file mode 100755 index 00000000..c8bd8ba7 --- /dev/null +++ b/roles/minio_install/charts/operator/templates/operator-deployment.yaml @@ -0,0 +1,66 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: "minio-operator" + namespace: {{ .Release.Namespace }} + labels: + {{- include "minio-operator.labels" . | nindent 4 }} +spec: + replicas: {{ .Values.operator.replicaCount }} + selector: + matchLabels: + {{- include "minio-operator.selectorLabels" . | nindent 6 }} + template: + metadata: + labels: + {{- include "minio-operator.selectorLabels" . | nindent 8 }} + spec: + {{- with .Values.operator.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + serviceAccountName: minio-operator + {{- with .Values.operator.securityContext }} + securityContext: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.operator.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.operator.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.operator.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.operator.topologySpreadConstraints }} + topologySpreadConstraints: + {{- toYaml . | nindent 8 }} + {{- end }} + containers: + - name: {{ .Chart.Name }} + image: "{{ .Values.operator.image.repository }}:{{ .Values.operator.image.tag }}" + imagePullPolicy: {{ .Values.operator.image.pullPolicy }} + {{- if .Values.operator.env }} + env: + {{ toYaml . | nindent 10 }} + {{- end }} + resources: + {{- toYaml .Values.operator.resources | nindent 12 }} + {{- with .Values.operator.initContainers }} + initContainers: + {{- toYaml . | nindent 8 }} + {{- end}} + affinity: + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchExpressions: + - key: name + operator: In + values: + - minio-operator + topologyKey: kubernetes.io/hostname diff --git a/roles/minio_install/charts/operator/templates/operator-service.yaml b/roles/minio_install/charts/operator/templates/operator-service.yaml new file mode 100755 index 00000000..1a527333 --- /dev/null +++ b/roles/minio_install/charts/operator/templates/operator-service.yaml @@ -0,0 +1,15 @@ +apiVersion: v1 +kind: Service +metadata: + name: "operator" + namespace: {{ .Release.Namespace }} + labels: + {{- include "minio-operator.labels" . | nindent 4 }} +spec: + type: ClusterIP + ports: + - port: 4222 + name: https + selector: + operator: leader + {{- include "minio-operator.selectorLabels" . | nindent 4 }} diff --git a/roles/minio_install/charts/operator/templates/serviceaccount.yaml b/roles/minio_install/charts/operator/templates/serviceaccount.yaml new file mode 100755 index 00000000..a8e840cd --- /dev/null +++ b/roles/minio_install/charts/operator/templates/serviceaccount.yaml @@ -0,0 +1,7 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: minio-operator + namespace: {{ .Release.Namespace }} + labels: +{{ include "minio-operator.labels" . | nindent 4 }} diff --git a/roles/minio_install/charts/operator/values.yaml b/roles/minio_install/charts/operator/values.yaml new file mode 100644 index 00000000..3c645e3a --- /dev/null +++ b/roles/minio_install/charts/operator/values.yaml @@ -0,0 +1,74 @@ +## +## Copyright (c) 2020-2022 Intel Corporation. +## +## Licensed under the Apache License, Version 2.0 (the "License"); +## you may not use this file except in compliance with the License. +## You may obtain a copy of the License at +## +## http://www.apache.org/licenses/LICENSE-2.0 +## +## Unless required by applicable law or agreed to in writing, software +## distributed under the License is distributed on an "AS IS" BASIS, +## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +## See the License for the specific language governing permissions and +## limitations under the License. +## + +# Default values for minio-operator. + +operator: + ## Setup environment variables for the Operator + # env: + # - name: MINIO_OPERATOR_TLS_ENABLE + # value: "off" + # - name: CLUSTER_DOMAIN + # value: "cluster.domain" + # - name: WATCHED_NAMESPACE + # value: "" + image: + repository: minio/operator + tag: v4.4.1 + pullPolicy: IfNotPresent + imagePullSecrets: [ ] + initcontainers: [ ] + replicaCount: 2 + securityContext: + runAsUser: 1000 + runAsGroup: 1000 + runAsNonRoot: true + fsGroup: 1000 + nodeSelector: { } + affinity: { } + tolerations: [ ] + topologySpreadConstraints: [ ] + resources: + requests: + cpu: 200m + memory: 256Mi + ephemeral-storage: 500Mi + +console: + image: + repository: minio/console + tag: v0.13.2 + pullPolicy: IfNotPresent + imagePullSecrets: [ ] + initcontainers: [ ] + replicaCount: 1 + nodeSelector: { } + affinity: { } + tolerations: [ ] + topologySpreadConstraints: [ ] + resources: { } + securityContext: + runAsUser: 1000 + runAsNonRoot: true + ingress: + enabled: false + ingressClassName: "" + labels: { } + annotations: { } + tls: [ ] + host: console.local + path: / + diff --git a/roles/qat_dp_install/charts/intel-qat-plugin/Chart.yaml b/roles/minio_install/charts/tenant/Chart.yaml old mode 100644 new mode 100755 similarity index 59% rename from roles/qat_dp_install/charts/intel-qat-plugin/Chart.yaml rename to roles/minio_install/charts/tenant/Chart.yaml index 955501f0..36199756 --- a/roles/qat_dp_install/charts/intel-qat-plugin/Chart.yaml +++ b/roles/minio_install/charts/tenant/Chart.yaml @@ -1,5 +1,5 @@ ## -## Copyright (c) 2020-2021 Intel Corporation. +## Copyright (c) 2020-2022 Intel Corporation. ## ## Licensed under the Apache License, Version 2.0 (the "License"); ## you may not use this file except in compliance with the License. @@ -13,10 +13,20 @@ ## See the License for the specific language governing permissions and ## limitations under the License. ## -apiVersion: v1 -description: Intel® QuickAssist Technology (QAT) device plugin for Kubernetes -name: intel-qat-plugin -version: 0.21.0 +apiVersion: v2 +description: A Helm chart for MinIO Operator +name: tenant +version: 4.4.1 +appVersion: v4.4.1 +keywords: + - storage + - object-storage + - S3 +home: https://min.io +icon: https://min.io/resources/img/logo/MINIO_wordmark.png sources: -- https://github.com/intel/intel-device-plugins-for-kubernetes.git -appVersion: 0.21.0 + - https://github.com/minio/operator +maintainers: + - name: MinIO, Inc + email: dev@minio.io +type: application diff --git a/roles/minio_install/charts/tenant/templates/NOTES.txt b/roles/minio_install/charts/tenant/templates/NOTES.txt new file mode 100755 index 00000000..1dc4b5b4 --- /dev/null +++ b/roles/minio_install/charts/tenant/templates/NOTES.txt @@ -0,0 +1,13 @@ +{{ range .Values.tenants }} + To connect to the {{.name}} tenant if it doesn't have a service exposed, you can port-forward to it by running: + {{- if dig "certificate" "requestAutoCert" false .}} + + kubectl --namespace {{ .namespace }} port-forward svc/{{ .name }}-console 9443:9443 + + Then visit the MinIO Console at https://127.0.0.1:9443 + {{ else }} + kubectl --namespace {{ .namespace }} port-forward svc/{{ .name }}-console 9090:9090 + + Then visit the MinIO Console at http://127.0.0.1:9090 + {{ end }} +{{ end }} diff --git a/roles/minio_install/charts/tenant/templates/_helpers.tpl b/roles/minio_install/charts/tenant/templates/_helpers.tpl new file mode 100755 index 00000000..5a969452 --- /dev/null +++ b/roles/minio_install/charts/tenant/templates/_helpers.tpl @@ -0,0 +1,84 @@ +{{/* vim: set filetype=mustache: */}} +{{/* +Expand the name of the chart. +*/}} +{{- define "minio-operator.name" -}} +{{- default .Chart.Name | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "minio-operator.fullname" -}} +{{- $name := default .Chart.Name -}} +{{- if contains $name .Release.Name -}} +{{- .Release.Name | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}} +{{- end -}} +{{- end -}} + +{{/* +Expand the name of the Operator Console. +*/}} +{{- define "minio-operator.console-name" -}} +{{- printf "%s-%s" .Chart.Name "console" | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Create a default fully qualified console name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "minio-operator.console-fullname" -}} +{{- printf "%s-%s" .Release.Name "console" | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "minio-operator.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Common labels for operator +*/}} +{{- define "minio-operator.labels" -}} +helm.sh/chart: {{ include "minio-operator.chart" . }} +{{ include "minio-operator.selectorLabels" . }} +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end -}} + +{{/* +Selector labels Operator +*/}} +{{- define "minio-operator.selectorLabels" -}} +app.kubernetes.io/name: {{ include "minio-operator.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end -}} + +{{/* +Common labels for console +*/}} +{{- define "minio-operator.console-labels" -}} +helm.sh/chart: {{ include "minio-operator.chart" . }} +{{ include "minio-operator.console-selectorLabels" . }} +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end -}} + +{{/* +Selector labels Operator +*/}} +{{- define "minio-operator.console-selectorLabels" -}} +app.kubernetes.io/name: {{ include "minio-operator.name" . }} +app.kubernetes.io/instance: {{ printf "%s-%s" .Release.Name "console" }} +{{- end -}} diff --git a/roles/minio_install/charts/tenant/templates/tenant-secret.yaml b/roles/minio_install/charts/tenant/templates/tenant-secret.yaml new file mode 100755 index 00000000..7d7ac1ad --- /dev/null +++ b/roles/minio_install/charts/tenant/templates/tenant-secret.yaml @@ -0,0 +1,16 @@ +{{ range .Values.tenants }} + {{- if dig "secrets" "enabled" false . }} +--- +apiVersion: v1 +kind: Secret +metadata: + name: {{ dig "secrets" "name" "" . }} + namespace: {{ .namespace }} +type: Opaque +data: + ## Access Key for MinIO Tenant + accesskey: {{ dig "secrets" "accessKey" "" . | b64enc }} + ## Secret Key for MinIO Tenant + secretkey: {{ dig "secrets" "secretKey" "" . | b64enc }} + {{- end }} + {{ end }} diff --git a/roles/minio_install/charts/tenant/templates/tenant.yaml b/roles/minio_install/charts/tenant/templates/tenant.yaml new file mode 100755 index 00000000..6a5c76f5 --- /dev/null +++ b/roles/minio_install/charts/tenant/templates/tenant.yaml @@ -0,0 +1,122 @@ +{{ range .Values.tenants }} +--- +apiVersion: minio.min.io/v2 +kind: Tenant +metadata: + name: {{ .name }} + namespace: {{ .namespace }} + ## Optionally pass labels to be applied to the statefulset pods + labels: + app: minio + {{ if dig "metrics" "enabled" false . }} + ## Annotations for MinIO Tenant Pods + annotations: + prometheus.io/path: /minio/v2/metrics/cluster + prometheus.io/port: {{ dig "metrics" "port" 9000 . | quote }} + prometheus.io/scrape: "true" + {{ end }} + {{ if dig "scheduler" "name" "" . }} + scheduler: + name: {{ dig "scheduler" "name" "" . }} + {{ end }} +spec: + image: {{ dig "image" "repository" "minio/minio" . }}:{{ dig "image" "tag" "RELEASE.2022-01-04T07-41-07Z" . }} + imagePullPolicy: {{ dig "image" "pullPolicy" "IfNotPresent" . }} + {{ if dig "imagePullSecret" "name" "" . }} + imagePullSecret: + name: {{ dig "imagePullSecret" "name" "" . }} + {{ end }} + ## Secret with credentials to be used by MinIO Tenant. + {{ if dig "secrets" "enabled" false . }} + credsSecret: + name: {{ dig "secrets" "name" "" . }} + {{ end }} + pools: + {{ range (dig "pools" (list) .) }} + - servers: {{ dig "servers" 4 . }} + volumesPerServer: {{ dig "volumesPerServer" 4 . }} + volumeClaimTemplate: + metadata: + name: data + spec: + storageClassName: {{ dig "storageClassName" "" . }} + accessModes: + - ReadWriteOnce + resources: + requests: + storage: {{ dig "size" "10Gi" . }} + {{- with (dig "tolerations" (list) .) }} + tolerations: + {{ toYaml . | nindent 8 }} + {{ end }} + {{- with (dig "nodeSelector" (dict) .) }} + nodeSelector: + {{ toYaml . | nindent 8 }} + {{ end }} + {{- with (dig "affinity" (dict) .) }} + affinity: + {{ toYaml . | nindent 8 }} + {{ end }} + {{- with (dig "resources" (dict) .) }} + resources: + {{ toYaml . | nindent 8 }} + {{ end }} + {{- with (dig "securityContext" (dict) .) }} + securityContext: + {{ toYaml . | nindent 8 }} + {{ end }} + {{- with (dig "topologySpreadConstraints" (list) .) }} + topologySpreadConstraints: + {{ toYaml . | nindent 8 }} + {{ end }} + {{ end }} + mountPath: {{ dig "mountPath" "/export" . }} + subPath: {{ dig "subPath" "/data" . }} + {{- with (dig "certificate" "externalCaCertSecret" (dict) .) }} + externalCaCertSecret: + {{ toYaml . | nindent 6 }} + {{ end }} + {{- with (dig "certificate" "externalCertSecret" (dict) .) }} + externalCertSecret: + {{ toYaml . | nindent 6 }} + {{ end }} + requestAutoCert: {{ dig "certificate" "requestAutoCert" false . }} + s3: + bucketDNS: {{ dig "s3" "bucketDNS" false . }} + {{- with (dig "certificate" "certConfig" (dict) .) }} + certConfig: + {{ toYaml . | nindent 4 }} + {{- end }} + podManagementPolicy: {{ dig "podManagementPolicy" "Parallel" . }} + # {{- with (dig "readiness" (dict) .) }} + # readiness: + # {{ toYaml . | nindent 4 }} + # {{- end }} + # {{- with (dig "liveness" (dict) .) }} + # liveness: + # {{ toYaml . | nindent 4 }} + # {{- end }} + # {{- with (dig "exposeServices" (dict) .) }} + # exposeServices: + # {{ toYaml . | nindent 4 }} + # {{- end }} + # {{ if dig "serviceAccountName" "" . }} + # serviceAccountName: {{ dig "serviceAccountName" "" . }} + # {{ end }} + # prometheusOperator: {{ dig "prometheusOperator" "false" . }} + # {{- with (dig "logging" (dict) .) }} + # logging: + # {{ toYaml . | nindent 4 }} + # {{- end }} + # {{- with (dig "serviceMetadata" (dict) .) }} + serviceMetadata: + {{ toYaml . | nindent 4 }} + {{- end }} + {{- with (dig "env" (dict) .) }} + env: + {{ toYaml . | nindent 4 }} + {{- end }} + {{ if dig "priorityClassName" "" . }} + priorityClassName: {{ dig "priorityClassName" "" . }} + {{ end }} +{{ end }} diff --git a/roles/minio_install/charts/tenant/values.yaml b/roles/minio_install/charts/tenant/values.yaml new file mode 100755 index 00000000..42021d82 --- /dev/null +++ b/roles/minio_install/charts/tenant/values.yaml @@ -0,0 +1,127 @@ +## +## Copyright (c) 2020-2022 Intel Corporation. +## +## Licensed under the Apache License, Version 2.0 (the "License"); +## you may not use this file except in compliance with the License. +## You may obtain a copy of the License at +## +## http://www.apache.org/licenses/LICENSE-2.0 +## +## Unless required by applicable law or agreed to in writing, software +## distributed under the License is distributed on an "AS IS" BASIS, +## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +## See the License for the specific language governing permissions and +## limitations under the License. +## + +## MinIO Tenant Definition +tenants: + # Tenant name + - name: minio1 + ## Registry location and Tag to download MinIO Server image + image: + repository: quay.io/minio/minio + tag: RELEASE.2022-01-04T07-41-07Z + pullPolicy: IfNotPresent + ## Customize namespace for tenant deployment + namespace: default + ## Customize any private registry image pull secret. + ## currently only one secret registry is supported + imagePullSecret: { } + ## If a scheduler is specified here, Tenant pods will be dispatched by specified scheduler. + ## If not specified, the Tenant pods will be dispatched by default scheduler. + scheduler: { } + ## Specification for MinIO Pool(s) in this Tenant. + pools: + ## Servers specifies the number of MinIO Tenant Pods / Servers in this pool. + ## For standalone mode, supply 1. For distributed mode, supply 4 or more. + ## Note that the operator does not support upgrading from standalone to distributed mode. + - servers: 4 + ## volumesPerServer specifies the number of volumes attached per MinIO Tenant Pod / Server. + volumesPerServer: 4 + ## size specifies the capacity per volume + size: 1Gi + ## storageClass specifies the storage class name to be used for this pool + storageClassName: standard + ## Used to specify a toleration for a pod + tolerations: { } + ## nodeSelector parameters for MinIO Pods. It specifies a map of key-value pairs. For the pod to be + ## eligible to run on a node, the node must have each of the + ## indicated key-value pairs as labels. + ## Read more here: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/ + nodeSelector: { } + ## Affinity settings for MinIO pods. Read more about affinity + ## here: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#affinity-and-anti-affinity. + affinity: { } + ## Configure resource requests and limits for MinIO containers + resources: { } + ## Configure security context + securityContext: { } + ## Configure topology constraints + topologySpreadConstraints: [ ] + ## Mount path where PV will be mounted inside container(s). + mountPath: /export + ## Sub path inside Mount path where MinIO stores data. + subPath: /data + # pool secrets + secrets: + # create a kubernetes secret object with the accessKey and secretKey as defined here. + enabled: true + name: minio1-secret + accessKey: minio + secretKey: minio123 + # pool metrics to be read by Prometheus + metrics: + enabled: false + port: 9000 + certificate: + ## Use this field to provide one or more external CA certificates. This is used by MinIO + ## to verify TLS connections with other applications: + ## https://github.com/minio/minio/tree/master/docs/tls/kubernetes#2-create-kubernetes-secret + externalCaCertSecret: { } + ## Use this field to provide a list of Secrets with external certificates. This can be used to to configure + ## TLS for MinIO Tenant pods. Create secrets as explained here: + ## https://github.com/minio/minio/tree/master/docs/tls/kubernetes#2-create-kubernetes-secret + externalCertSecret: { } + ## Enable automatic Kubernetes based certificate generation and signing as explained in + ## https://kubernetes.io/docs/tasks/tls/managing-tls-in-a-cluster + requestAutoCert: true + ## This field is used only when "requestAutoCert" is set to true. Use this field to set CommonName + ## for the auto-generated certificate. Internal DNS name for the pod will be used if CommonName is + ## not provided. DNS name format is *.minio.default.svc.cluster.local + certConfig: { } + ## Enable S3 specific features such as Bucket DNS which would allow `buckets` to be + ## accessible as DNS entries of form `.minio.default.svc.cluster.local` + s3: + ## This feature is turned off by default + bucketDNS: false + ## PodManagement policy for MinIO Tenant Pods. Can be "OrderedReady" or "Parallel" + ## Refer https://kubernetes.io/docs/tutorials/stateful-application/basic-stateful-set/#pod-management-policy + ## for details. + podManagementPolicy: Parallel + # Liveness Probe for container liveness. Container will be restarted if the probe fails. + # Refer https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-startup-probes. + liveness: { } + # Readiness Probe for container readiness. Container will be removed from service endpoints if the probe fails. + # Refer https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-startup-probes/ + readiness: { } + ## exposeServices defines the exposure of the MinIO object storage and Console services. + ## service is exposed as a loadbalancer in k8s service. + exposeServices: { } + # kubernetes service account associated with a specific tenant + # https://kubernetes.io/docs/tasks/configure-pod-container/configure-service-account/ + serviceAccountName: "" + # Tenant scrape configuration will be added to prometheus managed by the prometheus-operator. + prometheusOperator: false + # Enable JSON, Anonymous logging for MinIO tenants. + # Refer https://github.com/minio/operator/blob/master/pkg/apis/minio.min.io/v2/types.go#L303 + logging: { } + ## serviceMetadata allows passing additional labels and annotations to MinIO and Console specific + ## services created by the operator. + serviceMetadata: { } + ## Add environment variables to be set in MinIO container (https://github.com/minio/minio/tree/master/docs/config) + env: { } + ## PriorityClassName indicates the Pod priority and hence importance of a Pod relative to other Pods. + ## This is applied to MinIO pods only. + ## Refer Kubernetes documentation for details https://kubernetes.io/docs/concepts/configuration/pod-priority-preemption/#priorityclass/ + priorityClassName: "" diff --git a/roles/sgx_configure_icx/tasks/main.yml b/roles/minio_install/defaults/main.yaml old mode 100644 new mode 100755 similarity index 53% rename from roles/sgx_configure_icx/tasks/main.yml rename to roles/minio_install/defaults/main.yaml index bbde721b..79a6fdc3 --- a/roles/sgx_configure_icx/tasks/main.yml +++ b/roles/minio_install/defaults/main.yaml @@ -1,5 +1,5 @@ ## -## Copyright (c) 2020-2021 Intel Corporation. +## Copyright (c) 2020-2022 Intel Corporation. ## ## Licensed under the Apache License, Version 2.0 (the "License"); ## you may not use this file except in compliance with the License. @@ -14,24 +14,11 @@ ## limitations under the License. ## --- -- name: determine machine type - include_role: - name: check_machine_type +minio_operator_namespace: "minio-operator" # MinIO Operator/Console namespace +minio_tenant_namespace: "minio-tenant" # MinIO Sample Tenant namespace -- name: configure sgx on Ubuntu distribution - include_tasks: ubuntu.yml - when: - - ansible_distribution == "Ubuntu" - - is_icx +minio_operator_release_name: "minio-operator" # MinIO Operator/Console Helm Charts release name +minio_tenant_release_name: "minio-tenant" # MinIO Tenant Helm Charts release name -- name: configure sgx on CentOS distribution - include_tasks: centos.yml - when: - - ansible_distribution == "CentOS" - - is_icx - -- name: configure sgx on RHEL distribution - include_tasks: rhel.yml - when: - - ansible_distribution == "RedHat" - - is_icx +minio_storage_worker_key: storage # MinIO worker key +minio_storage_worker_value: minio # MinIO worker value diff --git a/roles/minio_install/tasks/create_blockdevicefiles.yml b/roles/minio_install/tasks/create_blockdevicefiles.yml new file mode 100644 index 00000000..eb06ebc9 --- /dev/null +++ b/roles/minio_install/tasks/create_blockdevicefiles.yml @@ -0,0 +1,36 @@ +## +## Copyright (c) 2020-2022 Intel Corporation. +## +## Licensed under the Apache License, Version 2.0 (the "License"); +## you may not use this file except in compliance with the License. +## You may obtain a copy of the License at +## +## http://www.apache.org/licenses/LICENSE-2.0 +## +## Unless required by applicable law or agreed to in writing, software +## distributed under the License is distributed on an "AS IS" BASIS, +## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +## See the License for the specific language governing permissions and +## limitations under the License. +## +--- +- name: create file block devices + command: >- + dd + if=/dev/zero of=/tmp/diskimage{{ ansible_loop.index }} + bs=1M + count= + {%- if hostvars[inventory_hostname]['minio_pv'][ansible_loop.index0].capacity.endswith('GiB') -%} + "{{ hostvars[inventory_hostname]['minio_pv'][ansible_loop.index0].capacity[:-3] |int * 1024 }}" + {%- elif hostvars[inventory_hostname]['minio_pv'][ansible_loop.index0].capacity.endswith('TiB') -%} + "{{ hostvars[inventory_hostname]['minio_pv'][ansible_loop.index0].capacity[:-3]|int * 1024 * 1024 }}" + {%- else -%} + "{{ hostvars[inventory_hostname]['minio_pv'][ansible_loop.index0].capacity }}" + {%- endif -%} + changed_when: true + +- name: create mount point for the file block devices + file: + path: "{{ hostvars[inventory_hostname]['minio_pv'][ansible_loop.index0].mountPath }}" + state: directory + mode: 0755 diff --git a/roles/minio_install/tasks/create_nvme_partition.yml b/roles/minio_install/tasks/create_nvme_partition.yml new file mode 100644 index 00000000..2c9ee119 --- /dev/null +++ b/roles/minio_install/tasks/create_nvme_partition.yml @@ -0,0 +1,46 @@ +## +## Copyright (c) 2020-2022 Intel Corporation. +## +## Licensed under the Apache License, Version 2.0 (the "License"); +## you may not use this file except in compliance with the License. +## You may obtain a copy of the License at +## +## http://www.apache.org/licenses/LICENSE-2.0 +## +## Unless required by applicable law or agreed to in writing, software +## distributed under the License is distributed on an "AS IS" BASIS, +## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +## See the License for the specific language governing permissions and +## limitations under the License. +## +--- +- name: create a new xfs primary partition + parted: + name: "{{ hostvars[inventory_hostname]['minio_pv'][ansible_loop.index0].name }}" + device: "{{ hostvars[inventory_hostname]['minio_pv'][ansible_loop.index0].device }}" + label: gpt + number: 1 + part_type: primary + part_end: "{{ hostvars[inventory_hostname]['minio_pv'][ansible_loop.index0].capacity }}" + flags: [ lvm ] + state: present + +- name: format the partition + filesystem: + fstype: xfs + dev: "{{ hostvars[inventory_hostname]['minio_pv'][ansible_loop.index0].device }}p1" + force: yes + state: present + +- name: create mount point for the file block devices + file: + path: "{{ hostvars[inventory_hostname]['minio_pv'][ansible_loop.index0].mountPath }}" + state: directory + mode: 0755 + +- name: mount the parition + mount: + fstype: xfs + src: "{{ hostvars[inventory_hostname]['minio_pv'][ansible_loop.index0].device }}p1" + path: "{{ hostvars[inventory_hostname]['minio_pv'][ansible_loop.index0].mountPath }}" + state: mounted diff --git a/roles/minio_install/tasks/create_persistentvolumes.yml b/roles/minio_install/tasks/create_persistentvolumes.yml new file mode 100644 index 00000000..636fba8c --- /dev/null +++ b/roles/minio_install/tasks/create_persistentvolumes.yml @@ -0,0 +1,38 @@ +## +## Copyright (c) 2020-2022 Intel Corporation. +## +## Licensed under the Apache License, Version 2.0 (the "License"); +## you may not use this file except in compliance with the License. +## You may obtain a copy of the License at +## +## http://www.apache.org/licenses/LICENSE-2.0 +## +## Unless required by applicable law or agreed to in writing, software +## distributed under the License is distributed on an "AS IS" BASIS, +## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +## See the License for the specific language governing permissions and +## limitations under the License. +## +--- +- name: populate MinIO Tenant Helm chart values template and push to controller node + template: + src: "minio_tenant_localpersistentvolumes.yml.j2" + dest: "{{ (project_root_dir, 'charts', 'operator', 'temp', 'minio-tenant-persistentvolumes' ~ ansible_loop.index ~ '.yml') | path_join }}" + force: yes + mode: preserve + when: + - hostvars[item]['minio_pv'] | default([]) | length > 0 + loop: "{{ groups['kube_node'] }}" + loop_control: + extended: yes + +- name: apply MinIO Tenant persistent volumes + k8s: + state: present + src: "{{ (project_root_dir, 'charts', 'operator', 'temp', 'minio-tenant-persistentvolumes' ~ ansible_loop.index ~ '.yml') | path_join }}" + when: + - hostvars[item]['minio_pv'] | default([]) | length > 0 + loop: "{{ groups['kube_node'] }}" + loop_control: + extended: yes + loop_var: item diff --git a/roles/istio_operator_install/tasks/deploy_istio_profile.yml b/roles/minio_install/tasks/create_storageclass.yml similarity index 55% rename from roles/istio_operator_install/tasks/deploy_istio_profile.yml rename to roles/minio_install/tasks/create_storageclass.yml index 1549d908..88190a66 100644 --- a/roles/istio_operator_install/tasks/deploy_istio_profile.yml +++ b/roles/minio_install/tasks/create_storageclass.yml @@ -1,5 +1,5 @@ ## -## Copyright (c) 2020-2021 Intel Corporation. +## Copyright (c) 2020-2022 Intel Corporation. ## ## Licensed under the Apache License, Version 2.0 (the "License"); ## you may not use this file except in compliance with the License. @@ -14,20 +14,14 @@ ## limitations under the License. ## --- -- name: Create istio profiles dir if does not exist - file: - path: "{{ istio_profiles_dir }}" - state: directory - owner: root - mode: 0755 - -- name: populate istio crd template with values +- name: create default MinIO Tenant local storage class template: - src: "profiles/{{ istio_profile }}.yaml.j2" - dest: "{{ istio_profiles_dir }}/{{ istio_profile }}.yaml" + src: "minio_tenant_storageclass.yml.j2" + dest: "{{ (project_root_dir, 'charts', 'operator', 'temp', 'minio-tenant-storageclass.yml') | path_join }}" force: yes mode: preserve -- name: deploly istio "{{ istio_profile }}" profile - command: "kubectl apply -f {{ istio_profiles_dir }}/{{ istio_profile }}.yaml" - changed_when: true +- name: apply default MinIO Tenant local storage class + k8s: + state: present + src: "{{ (project_root_dir, 'charts', 'operator', 'temp', 'minio-tenant-storageclass.yml') | path_join }}" diff --git a/roles/minio_install/tasks/file_blockdevice.yml b/roles/minio_install/tasks/file_blockdevice.yml new file mode 100644 index 00000000..97bd6b7f --- /dev/null +++ b/roles/minio_install/tasks/file_blockdevice.yml @@ -0,0 +1,44 @@ +## +## Copyright (c) 2020-2022 Intel Corporation. +## +## Licensed under the Apache License, Version 2.0 (the "License"); +## you may not use this file except in compliance with the License. +## You may obtain a copy of the License at +## +## http://www.apache.org/licenses/LICENSE-2.0 +## +## Unless required by applicable law or agreed to in writing, software +## distributed under the License is distributed on an "AS IS" BASIS, +## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +## See the License for the specific language governing permissions and +## limitations under the License. +## +--- +- name: set iteration number + set_fact: + num: 1 + req_num: "{{ hostvars[inventory_hostname]['minio_pv'] |length }}" + +- name: create local file block device + include_tasks: create_blockdevicefiles.yml + loop: "{{ range(num, req_num|int + 1)|list }}" + loop_control: + extended: yes + +- name: format with xfs file block devices + include_tasks: format_blockdevicefiles.yml + loop: "{{ range(num, req_num|int + 1)|list }}" + loop_control: + extended: yes + +- name: setup the loop devices + include_tasks: setup_loopdevices.yml + loop: "{{ range(num, req_num|int + 1)|list }}" + loop_control: + extended: yes + +- name: mount the loop devices + include_tasks: mount_loopdevices.yml + loop: "{{ range(num, req_num|int + 1)|list }}" + loop_control: + extended: yes diff --git a/roles/minio_install/tasks/format_blockdevicefiles.yml b/roles/minio_install/tasks/format_blockdevicefiles.yml new file mode 100644 index 00000000..25bdd59d --- /dev/null +++ b/roles/minio_install/tasks/format_blockdevicefiles.yml @@ -0,0 +1,22 @@ +## +## Copyright (c) 2020-2022 Intel Corporation. +## +## Licensed under the Apache License, Version 2.0 (the "License"); +## you may not use this file except in compliance with the License. +## You may obtain a copy of the License at +## +## http://www.apache.org/licenses/LICENSE-2.0 +## +## Unless required by applicable law or agreed to in writing, software +## distributed under the License is distributed on an "AS IS" BASIS, +## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +## See the License for the specific language governing permissions and +## limitations under the License. +## +--- +- name: format file block device with xfs + filesystem: + fstype: xfs + dev: /tmp/diskimage{{ ansible_loop.index }} + force: yes + state: present diff --git a/roles/minio_install/tasks/main.yml b/roles/minio_install/tasks/main.yml new file mode 100644 index 00000000..6483d2de --- /dev/null +++ b/roles/minio_install/tasks/main.yml @@ -0,0 +1,70 @@ +## +## Copyright (c) 2020-2022 Intel Corporation. +## +## Licensed under the Apache License, Version 2.0 (the "License"); +## you may not use this file except in compliance with the License. +## You may obtain a copy of the License at +## +## http://www.apache.org/licenses/LICENSE-2.0 +## +## Unless required by applicable law or agreed to in writing, software +## distributed under the License is distributed on an "AS IS" BASIS, +## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +## See the License for the specific language governing permissions and +## limitations under the License. +## +--- +- name: collect the number of nodes + set_fact: + minio_nodes: "{{ groups['kube_node'] | length }}" + +- name: check the number of nodes for MinIO install + fail: + msg: | + Incorrect configuration pertaining the number of nodes for MinIO install. + Make sure the number of nodes you are deploying is more than or equal to {{ minio_tenant_servers }} + when: + - minio_nodes | int < minio_tenant_servers + +- name: install MinIO operator/console + import_tasks: minio_operator.yml + when: + - minio_nodes | int >= minio_tenant_servers + - inventory_hostname == groups['kube_control_plane'][0] + +- name: create MinIO local storage + import_tasks: file_blockdevice.yml + when: + - minio_tenant_enabled + - minio_deploy_test_mode + - minio_nodes | int >= minio_tenant_servers + - inventory_hostname in groups['kube_node'] + +- name: create MinIO nvme storage + import_tasks: nvme_blockdevice.yml + when: + - minio_tenant_enabled + - not minio_deploy_test_mode + - minio_nodes | int >= minio_tenant_servers + - inventory_hostname in groups['kube_node'] + +- name: pupulate MinIO local persisitent volumes + import_tasks: create_persistentvolumes.yml + when: + - minio_tenant_enabled + - minio_nodes | int >= minio_tenant_servers + - inventory_hostname == groups['kube_control_plane'][0] + +- name: pupulate MinIO storage class + import_tasks: create_storageclass.yml + when: + - minio_tenant_enabled + - minio_nodes | int >= minio_tenant_servers + - inventory_hostname == groups['kube_control_plane'][0] + +- name: install MinIO tenant + import_tasks: minio_tenant.yml + when: + - minio_tenant_enabled + - minio_nodes | int >= minio_tenant_servers + - inventory_hostname == groups['kube_control_plane'][0] diff --git a/roles/minio_install/tasks/minio_operator.yml b/roles/minio_install/tasks/minio_operator.yml new file mode 100644 index 00000000..0f676007 --- /dev/null +++ b/roles/minio_install/tasks/minio_operator.yml @@ -0,0 +1,78 @@ +## +## Copyright (c) 2020-2022 Intel Corporation. +## +## Licensed under the Apache License, Version 2.0 (the "License"); +## you may not use this file except in compliance with the License. +## You may obtain a copy of the License at +## +## http://www.apache.org/licenses/LICENSE-2.0 +## +## Unless required by applicable law or agreed to in writing, software +## distributed under the License is distributed on an "AS IS" BASIS, +## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +## See the License for the specific language governing permissions and +## limitations under the License. +## +--- +- name: check MinIO Operator Helm charts directory. + stat: + path: "{{ (project_root_dir, 'charts', 'operator') | path_join }}" + register: operator_dir + +- name: create MinIO Operator Helm charts directory if needed + file: + path: "{{ (project_root_dir, 'charts', 'operator') | path_join }}" + state: directory + mode: 0755 + when: + - operator_dir.stat.exists is defined and not operator_dir.stat.exists + +- name: copy MinIO Operator Helm charts to the controller node + copy: + src: "{{ (role_path, 'charts', 'operator') | path_join }}" + dest: "{{ (project_root_dir, 'charts') | path_join }}" + mode: 0755 + +- name: check MinIO Operator Helm charts temp directory. + stat: + path: "{{ (project_root_dir, 'charts', 'operator', 'temp') | path_join }}" + register: operator_temp_dir + +- name: create the temp folder for MinIO Operator custom values + file: + path: "{{ (project_root_dir, 'charts', 'operator', 'temp') | path_join }}" + state: directory + mode: 0755 + when: + - operator_temp_dir.stat.exists is defined and not operator_temp_dir.stat.exists + +- name: populate MinIO Operator Helm charts values template and push to controller node + template: + src: "minio_operator_custom_values.yml.j2" + dest: "{{ (project_root_dir, 'charts', 'operator', 'temp', 'minio-operator-custom-values.yml') | path_join }}" + force: yes + mode: preserve + +- name: create the namespace for MinIO Operator/Console + kubernetes.core.k8s: + name: "{{ minio_operator_namespace }}" + api_version: v1 + kind: Namespace + state: present + +- name: label minio on nodes + command: >- + kubectl label --overwrite nodes {{ item }} + {{ minio_storage_worker_key }}={{ minio_storage_worker_value }} + loop: "{{ groups['kube_node'] }}" + changed_when: true + +- name: install MinIO Operator Helm charts + command: >- + helm install + {{ minio_operator_release_name }} + {{ (project_root_dir, 'charts/operator') | path_join }} + --namespace {{ minio_operator_namespace }} + --set installCRDs=true + -f {{ (project_root_dir, 'charts', 'operator', 'temp', 'minio-operator-custom-values.yml') | path_join }} + changed_when: true diff --git a/roles/minio_install/tasks/minio_tenant.yml b/roles/minio_install/tasks/minio_tenant.yml new file mode 100644 index 00000000..dd806d32 --- /dev/null +++ b/roles/minio_install/tasks/minio_tenant.yml @@ -0,0 +1,71 @@ +## +## Copyright (c) 2020-2022 Intel Corporation. +## +## Licensed under the Apache License, Version 2.0 (the "License"); +## you may not use this file except in compliance with the License. +## You may obtain a copy of the License at +## +## http://www.apache.org/licenses/LICENSE-2.0 +## +## Unless required by applicable law or agreed to in writing, software +## distributed under the License is distributed on an "AS IS" BASIS, +## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +## See the License for the specific language governing permissions and +## limitations under the License. +## +--- +- name: check MinIO Tenant Helm charts directory. + stat: + path: "{{ (project_root_dir, 'charts', 'tenant') | path_join }}" + register: tenant_dir + +- name: create MinIO Tenant Helm charts directory if needed + file: + path: "{{ (project_root_dir, 'charts', 'tenant') | path_join }}" + state: directory + mode: 0755 + when: + - tenant_dir.stat.exists is defined and not tenant_dir.stat.exists + +- name: copy MinIO Tenant Helm charts to the controller node + copy: + src: "{{ (role_path, 'charts', 'tenant') | path_join }}" + dest: "{{ (project_root_dir, 'charts') | path_join }}" + mode: 0755 + +- name: check MinIO Tenant Helm charts temp directory. + stat: + path: "{{ (project_root_dir, 'charts', 'tenant', 'temp') | path_join }}" + register: tenant_temp_dir + +- name: create the temp folder for MinIO Tenant custom values + file: + path: "{{ (project_root_dir, 'charts', 'tenant', 'temp') | path_join }}" + state: directory + mode: 0755 + when: + - not tenant_temp_dir.stat.exists + +- name: populate MinIO Tenant Helm charts values template and push to controller node + template: + src: "minio_tenant_custom_values.yml.j2" + dest: "{{ (project_root_dir, 'charts', 'tenant', 'temp', 'minio-tenant-custom-values.yml') | path_join }}" + force: yes + mode: preserve + +- name: create the namespace for sample MinIO Tenants + kubernetes.core.k8s: + name: "{{ minio_tenant_namespace }}" + api_version: v1 + kind: Namespace + state: present + +- name: install MinIO Sample Tenant Helm chart + command: >- + helm install + {{ minio_tenant_release_name }} + {{ (project_root_dir, 'charts', 'tenant') | path_join }} + --namespace {{ minio_tenant_namespace }} + --set installCRDs=true + -f {{ (project_root_dir, 'charts', 'tenant', 'temp', 'minio-tenant-custom-values.yml') | path_join }} + changed_when: true diff --git a/roles/minio_install/tasks/mount_loopdevices.yml b/roles/minio_install/tasks/mount_loopdevices.yml new file mode 100644 index 00000000..94ea9d55 --- /dev/null +++ b/roles/minio_install/tasks/mount_loopdevices.yml @@ -0,0 +1,31 @@ +## +## Copyright (c) 2020-2022 Intel Corporation. +## +## Licensed under the Apache License, Version 2.0 (the "License"); +## you may not use this file except in compliance with the License. +## You may obtain a copy of the License at +## +## http://www.apache.org/licenses/LICENSE-2.0 +## +## Unless required by applicable law or agreed to in writing, software +## distributed under the License is distributed on an "AS IS" BASIS, +## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +## See the License for the specific language governing permissions and +## limitations under the License. +## +--- +- name: list loop devices + shell: >- + set -o pipefail && losetup -l |grep diskimage |awk -F " " '{ print $6,$1}' |sort |awk -F " " '{ print $2}' + args: + executable: /bin/bash + register: loopdevice_output + changed_when: true + +- name: mount file block devices +# noqa 303 - Ansible mount module doesn't support 'fstype:loop' + command: >- + mount -o loop {{ loopdevice_output.stdout_lines[ansible_loop.index0] }} {{ hostvars[inventory_hostname]['minio_pv'][ansible_loop.index0].mountPath }} + loop_control: + extended: yes + changed_when: true diff --git a/roles/minio_install/tasks/nvme_blockdevice.yml b/roles/minio_install/tasks/nvme_blockdevice.yml new file mode 100644 index 00000000..4b659c51 --- /dev/null +++ b/roles/minio_install/tasks/nvme_blockdevice.yml @@ -0,0 +1,26 @@ +## +## Copyright (c) 2020-2022 Intel Corporation. +## +## Licensed under the Apache License, Version 2.0 (the "License"); +## you may not use this file except in compliance with the License. +## You may obtain a copy of the License at +## +## http://www.apache.org/licenses/LICENSE-2.0 +## +## Unless required by applicable law or agreed to in writing, software +## distributed under the License is distributed on an "AS IS" BASIS, +## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +## See the License for the specific language governing permissions and +## limitations under the License. +## +--- +- name: set iteration number + set_fact: + num: 1 + req_num: "{{ hostvars[inventory_hostname]['minio_pv'] |length }}" + +- name: configuring nvme block device partition + include_tasks: create_nvme_partition.yml + loop: "{{ range(num, req_num|int + 1)|list }}" + loop_control: + extended: yes diff --git a/roles/collectd_install/vars/main.yml b/roles/minio_install/tasks/setup_loopdevices.yml similarity index 74% rename from roles/collectd_install/vars/main.yml rename to roles/minio_install/tasks/setup_loopdevices.yml index 1013a343..f44d6e62 100644 --- a/roles/collectd_install/vars/main.yml +++ b/roles/minio_install/tasks/setup_loopdevices.yml @@ -1,5 +1,5 @@ ## -## Copyright (c) 2020-2021 Intel Corporation. +## Copyright (c) 2020-2022 Intel Corporation. ## ## Licensed under the Apache License, Version 2.0 (the "License"); ## you may not use this file except in compliance with the License. @@ -14,9 +14,7 @@ ## limitations under the License. ## --- -collectd_configuration_files_dir: "{{ host_collectd_folder }}/collectd.conf.d" - -unsupported_power_skus: - - "06/6c" - - "$0000%@" - - "0000%@" +- name: setup loop devices for block devices files + command: >- + losetup --find --show /tmp/diskimage{{ ansible_loop.index }} + changed_when: true diff --git a/roles/minio_install/templates/minio_operator_custom_values.yml.j2 b/roles/minio_install/templates/minio_operator_custom_values.yml.j2 new file mode 100644 index 00000000..4893b2c8 --- /dev/null +++ b/roles/minio_install/templates/minio_operator_custom_values.yml.j2 @@ -0,0 +1,56 @@ +--- +operator: + ## Setup environment variables for the Operator + # env: + # - name: MINIO_OPERATOR_TLS_ENABLE + # value: "off" + # - name: CLUSTER_DOMAIN + # value: "cluster.domain" + # - name: WATCHED_NAMESPACE + # value: "" + image: + repository: {{ minio_operator_image | default("minio/operator") }} + tag: {{ minio_operator_version | default("v4.4.1")}} + pullPolicy: IfNotPresent + replicaCount: {{ minio_tenant_servers }} + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: storage + operator: In + values: + - minio + tolerations: [ ] + topologySpreadConstraints: [ ] + resources: + requests: + cpu: 200m + memory: 256Mi + ephemeral-storage: 500Mi + +console: + image: + repository: {{ minio_console_image | default("minio/console") }} + tag: {{ minio_console_version | default("v0.13.2")}} + pullPolicy: IfNotPresent + replicaCount: 1 + affinity: + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: storage + operator: In + values: + - minio + ingress: + enabled: false + ingressClassName: "" + labels: { } + annotations: { } + tls: [ ] + host: console.local + path: / diff --git a/roles/minio_install/templates/minio_tenant_custom_values.yml.j2 b/roles/minio_install/templates/minio_tenant_custom_values.yml.j2 new file mode 100644 index 00000000..15ed3eff --- /dev/null +++ b/roles/minio_install/templates/minio_tenant_custom_values.yml.j2 @@ -0,0 +1,71 @@ +--- +## MinIO Tenant Definition +tenants: + # Tenant name + - name: minio-tenant + ## Registry location and Tag to download MinIO Server image + image: + repository: quay.io/minio/minio + tag: RELEASE.2022-01-04T07-41-07Z + pullPolicy: IfNotPresent + ## Customize namespace for tenant deployment + namespace: minio-tenant + ## Customize any private registry image pull secret. + ## currently only one secret registry is supported + imagePullSecret: { } + ## If a scheduler is specified here, Tenant pods will be dispatched by specified scheduler. + ## If not specified, the Tenant pods will be dispatched by default scheduler. + scheduler: { } + ## Specification for MinIO Pool(s) in this Tenant. + pools: + ## Servers specifies the number of MinIO Tenant Pods / Servers in this pool. + ## For standalone mode, supply 1. For distributed mode, supply 4 or more. + ## Note that the operator does not support upgrading from standalone to distributed mode. + - servers: {{ minio_tenant_servers }} + ## volumesPerServer specifies the number of volumes attached per MinIO Tenant Pod / Server. + volumesPerServer: {{ minio_tenant_volumes_per_server }} + ## size specifies the capacity per volume + size: 1Gi + ## storageClass specifies the storage class name to be used for this pool + storageClassName: local-storage + ## Used to specify a toleration for a pod + tolerations: { } + ## nodeSelector parameters for MinIO Pods. It specifies a map of key-value pairs. For the pod to be + ## eligible to run on a node, the node must have each of the + ## indicated key-value pairs as labels. + ## Read more here: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/ + nodeSelector: { } + ## Affinity settings for MinIO pods. Read more about affinity + ## here: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#affinity-and-anti-affinity. + affinity: + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchExpressions: + - key: v1.min.io/tenant + operator: In + values: + - minio-tenant + topologyKey: kubernetes.io/hostname + ## Configure resource requests and limits for MinIO containers + resources: { } + ## Configure security context + securityContext: { } + ## Configure topology constraints + topologySpreadConstraints: [ ] + ## Mount path where PV will be mounted inside container(s). + mountPath: /mnt/data + ## Sub path inside Mount path where MinIO stores data. + subPath: /data + requestAutoCert: true + # pool secrets + secrets: + # create a kubernetes secret object with the accessKey and secretKey as defined here. + enabled: true + name: minio1-secret + accessKey: minio + secretKey: minio123 + # pool metrics to be read by Prometheus + metrics: + enabled: false + port: 9000 diff --git a/roles/minio_install/templates/minio_tenant_localpersistentvolumes.yml.j2 b/roles/minio_install/templates/minio_tenant_localpersistentvolumes.yml.j2 new file mode 100644 index 00000000..2d8c7ced --- /dev/null +++ b/roles/minio_install/templates/minio_tenant_localpersistentvolumes.yml.j2 @@ -0,0 +1,25 @@ +{%- for pv in hostvars[item]['minio_pv'] %} +--- +apiVersion: v1 +kind: PersistentVolume +metadata: + name: {{ pv.name }}-{{ item }} +spec: + capacity: + storage: {{ pv.capacity[:-1] }} + volumeMode: Filesystem + accessModes: + - {{ pv.accessMode }} + persistentVolumeReclaimPolicy: {{ pv.persistentVolumeReclaimPolicy }} + storageClassName: {{ pv.storageClassName }} + local: + path: {{ pv.mountPath }} + nodeAffinity: + required: + nodeSelectorTerms: + - matchExpressions: + - key: kubernetes.io/hostname + operator: In + values: + - {{ item }} +{% endfor %} diff --git a/roles/minio_install/templates/minio_tenant_storageclass.yml.j2 b/roles/minio_install/templates/minio_tenant_storageclass.yml.j2 new file mode 100644 index 00000000..42e292d2 --- /dev/null +++ b/roles/minio_install/templates/minio_tenant_storageclass.yml.j2 @@ -0,0 +1,7 @@ +--- +apiVersion: storage.k8s.io/v1 +kind: StorageClass +metadata: + name: local-storage +provisioner: kubernetes.io/no-provisioner +volumeBindingMode: WaitForFirstConsumer diff --git a/roles/net_attach_defs_create/tasks/main.yml b/roles/net_attach_defs_create/tasks/main.yml index ccb10390..2851bb85 100644 --- a/roles/net_attach_defs_create/tasks/main.yml +++ b/roles/net_attach_defs_create/tasks/main.yml @@ -1,5 +1,5 @@ ## -## Copyright (c) 2020-2021 Intel Corporation. +## Copyright (c) 2020-2022 Intel Corporation. ## ## Licensed under the Apache License, Version 2.0 (the "License"); ## you may not use this file except in compliance with the License. diff --git a/roles/nfd_install/charts/node-feature-discovery/Chart.yaml b/roles/nfd_install/charts/node-feature-discovery/Chart.yaml index f010878f..48c5b62e 100644 --- a/roles/nfd_install/charts/node-feature-discovery/Chart.yaml +++ b/roles/nfd_install/charts/node-feature-discovery/Chart.yaml @@ -1,5 +1,5 @@ ## -## Copyright (c) 2020-2021 Intel Corporation. +## Copyright (c) 2020-2022 Intel Corporation. ## ## Licensed under the Apache License, Version 2.0 (the "License"); ## you may not use this file except in compliance with the License. diff --git a/roles/nfd_install/charts/node-feature-discovery/templates/config.yml b/roles/nfd_install/charts/node-feature-discovery/templates/config.yml index 59ba85fa..031c51ba 100644 --- a/roles/nfd_install/charts/node-feature-discovery/templates/config.yml +++ b/roles/nfd_install/charts/node-feature-discovery/templates/config.yml @@ -9,4 +9,4 @@ metadata: data: nfd-worker.conf: | {{ toYaml .Values.config | indent 4 }} -{{- end }} \ No newline at end of file +{{- end }} diff --git a/roles/nfd_install/charts/node-feature-discovery/templates/master.yml b/roles/nfd_install/charts/node-feature-discovery/templates/master.yml index 02850419..732e6e04 100644 --- a/roles/nfd_install/charts/node-feature-discovery/templates/master.yml +++ b/roles/nfd_install/charts/node-feature-discovery/templates/master.yml @@ -44,9 +44,11 @@ spec: command: - "nfd-master" - --port={{ .Values.service.port }} -{{- if .Values.sgx_dp_nfd_labels }} - - "--resource-labels=sgx.intel.com/epc" - - "--extra-label-ns=sgx.intel.com" +{{- if .Values.nfd_extra_labels_ns | empty | not }} + - "--extra-label-ns={{- join "," .Values.nfd_extra_labels_ns }}" +{{- end }} +{{- if .Values.nfd_resource_labels | empty | not }} + - "--resource-labels={{- join "," .Values.nfd_resource_labels }}" {{- end }} {{- if .Values.tls.enabled }} args: diff --git a/roles/nfd_install/charts/node-feature-discovery/templates/rbac.yml b/roles/nfd_install/charts/node-feature-discovery/templates/rbac.yml index 896c2c12..cb3ec995 100644 --- a/roles/nfd_install/charts/node-feature-discovery/templates/rbac.yml +++ b/roles/nfd_install/charts/node-feature-discovery/templates/rbac.yml @@ -21,13 +21,16 @@ rules: - nodes # when using command line flag --resource-labels to create extended resources # "- nodes/status" is needed -{{- if .Values.sgx_dp_nfd_labels }} +{{- if or (.Values.sgx_dp_enabled) (.Values.gpu_dp.enabled) }} - nodes/status {{- end }} verbs: - get - patch - update +{{- if .Values.gpu_dp.enabled }} + - list +{{- end }} --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding diff --git a/roles/nfd_install/charts/node-feature-discovery/templates/worker.yml b/roles/nfd_install/charts/node-feature-discovery/templates/worker.yml index a4e865dc..d32c3ccd 100644 --- a/roles/nfd_install/charts/node-feature-discovery/templates/worker.yml +++ b/roles/nfd_install/charts/node-feature-discovery/templates/worker.yml @@ -23,6 +23,12 @@ spec: valueFrom: fieldRef: fieldPath: spec.nodeName +{{- if .Values.gpu_dp.enabled }} +{{- if .Values.gpu_dp.max_memory }} + - name: GPU_MEMORY_OVERRIDE + value: "{{ .Values.gpu_dp.max_memory }}" +{{- end }} +{{- end }} image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}" name: nfd-worker command: diff --git a/roles/nfd_install/charts/node-feature-discovery/values.yaml b/roles/nfd_install/charts/node-feature-discovery/values.yaml index a0e12655..ac5dadac 100644 --- a/roles/nfd_install/charts/node-feature-discovery/values.yaml +++ b/roles/nfd_install/charts/node-feature-discovery/values.yaml @@ -1,5 +1,5 @@ ## -## Copyright (c) 2020-2021 Intel Corporation. +## Copyright (c) 2020-2022 Intel Corporation. ## ## Licensed under the Apache License, Version 2.0 (the "License"); ## you may not use this file except in compliance with the License. @@ -16,7 +16,7 @@ --- image: repository: k8s.gcr.io/nfd/node-feature-discovery - tag: v0.9.0 + tag: v0.10.0 pullPolicy: IfNotPresent sleepInterval: "120s" @@ -50,8 +50,6 @@ service: #nameOverride: "node-feature-discovery" #fullnameOverride: "node-feature-discovery" -sgx_dp_nfd_labels: false - config: # sources: # cpu: diff --git a/roles/nfd_install/defaults/main.yml b/roles/nfd_install/defaults/main.yml index db79cf59..95a2b9a8 100644 --- a/roles/nfd_install/defaults/main.yml +++ b/roles/nfd_install/defaults/main.yml @@ -1,5 +1,5 @@ ## -## Copyright (c) 2020-2021 Intel Corporation. +## Copyright (c) 2020-2022 Intel Corporation. ## ## Licensed under the Apache License, Version 2.0 (the "License"); ## you may not use this file except in compliance with the License. @@ -15,20 +15,36 @@ ## --- nfd_git_url: "https://github.com/kubernetes-sigs/node-feature-discovery.git" -nfd_git_ref: "v0.9.0" -nfd_dir: "/usr/src/nfd" +nfd_git_ref: "v0.10.0" +nfd_dir: "{{ (project_root_dir, 'nfd') | path_join }}" nfd_external_image_name: "k8s.gcr.io/nfd/node-feature-discovery" -nfd_external_image_tag: "v0.9.0" +nfd_external_image_tag: "v0.10.0" nfd_build_image_locally: false nfd_namespace: "kube-system" nfd_sleep_interval: "60s" -nfd_sgx_dp_args: - sources: - cpu: - cpuid: - attributeWhitelist: - - "SGX" - - "SGXLC" + +nfd_tls_enabled: true +# automatically generate self-signed certificate +nfd_tls_generate: true +# base64-encoded nfd-master private TLS key, required when tls.enabled is true and tls.generate is false +nfd_tls_master_key: "" +# base64-encoded nfd-master private TLS certificate, required when tls.enabled is true and tls.generate is false +nfd_tls_master_cert: "" +# base64-encoded nfd-worker private TLS key, required when tls.enabled is true and tls.generate is false +nfd_tls_worker_key: "" +# base64-encoded nfd-worker private TLS certificate, required when tls.enabled is true and tls.generate is false +nfd_tls_worker_cert: "" +# base64-encoded additional CA certificate which will be used to validate peer certificates, required when tls.enabled is true and tls.generate is false +nfd_tls_ca_cert: "" + + +nfd_sa_create: true +nfd_sa_name: "" +nfd_rbac_enabled: true + +nfd_svc_port: 8080 +nfd_svc_type: ClusterIP + diff --git a/roles/nfd_install/tasks/main.yml b/roles/nfd_install/tasks/main.yml index 8fdb5bd6..9eaeec39 100644 --- a/roles/nfd_install/tasks/main.yml +++ b/roles/nfd_install/tasks/main.yml @@ -1,5 +1,5 @@ ## -## Copyright (c) 2020-2021 Intel Corporation. +## Copyright (c) 2020-2022 Intel Corporation. ## ## Licensed under the Apache License, Version 2.0 (the "License"); ## you may not use this file except in compliance with the License. @@ -14,7 +14,7 @@ ## limitations under the License. ## --- -- name: install dependencies +- name: install dependencies for NFD include_role: name: install_dependencies @@ -88,7 +88,7 @@ - name: create Helm charts directory if needed file: - path: /usr/src/charts + path: "{{ (project_root_dir, 'charts') | path_join }}" state: directory mode: 0755 when: @@ -96,8 +96,8 @@ - name: copy NFD Helm chart to the controller node copy: - src: "{{ role_path }}/charts/node-feature-discovery" - dest: "/usr/src/charts/" + src: "{{ (role_path, 'charts', 'node-feature-discovery') | path_join }}" + dest: "{{ (project_root_dir, 'charts') | path_join }}" mode: 0755 when: - inventory_hostname == groups['kube_control_plane'][0] @@ -118,18 +118,10 @@ - not nfd_build_image_locally - inventory_hostname == groups['kube_control_plane'][0] -- name: set values for NFD Helm chart values for SGX DP - set_fact: - sgx_dp_args: "{{ nfd_sgx_dp_args }}" - sgx_dp_labels: true - when: - - sgx_dp_enabled | default(false) - - inventory_hostname == groups['kube_control_plane'][0] - - name: populate NFD Helm chart values template and push to controller node template: src: "helm_values.yml.j2" - dest: "/usr/src/charts/nfd-values.yml" + dest: "{{ (project_root_dir, 'charts', 'nfd-values.yml') | path_join }}" force: yes mode: preserve when: @@ -153,6 +145,10 @@ - inventory_hostname == groups['kube_control_plane'][0] - name: install NFD helm chart - command: helm upgrade -i node-feature-discovery --namespace {{ nfd_namespace }} -f /usr/src/charts/nfd-values.yml /usr/src/charts/node-feature-discovery + command: >- + helm upgrade -i node-feature-discovery + --namespace {{ nfd_namespace }} + -f {{ (project_root_dir, 'charts', 'nfd-values.yml') | path_join }} + {{ (project_root_dir, 'charts', 'node-feature-discovery') | path_join }} when: - inventory_hostname == groups['kube_control_plane'][0] diff --git a/roles/nfd_install/templates/helm_values.yml.j2 b/roles/nfd_install/templates/helm_values.yml.j2 index 06ab0f6d..5a7ee501 100644 --- a/roles/nfd_install/templates/helm_values.yml.j2 +++ b/roles/nfd_install/templates/helm_values.yml.j2 @@ -1,11 +1,98 @@ --- image: - repository: {{ nfd_image | default("k8s.gcr.io/nfd/node-feature-discovery") }} - tag: {{ nfd_version | default("v0.9.0")}} - pullPolicy: IfNotPresent + repository: {{ nfd_image | default("k8s.gcr.io/nfd/node-feature-discovery") }} + tag: {{ nfd_version | default("v0.10.0")}} + pullPolicy: IfNotPresent -config: {{ sgx_dp_args | default(false)}} +{% if sgx_dp_enabled | default(false) or gpu_dp_enabled | default(false) %} +nfd_resource_labels: +{% if sgx_dp_enabled | default(false) %} + - "sgx.intel.com/epc" +{% endif %} +{% if gpu_dp_enabled | default(false) %} + - "gpu.intel.com/memory.max" + - "gpu.intel.com/millicores" +{% endif %} +{% else %} +nfd_resource_labels: [] +{% endif %} -sgx_dp_nfd_labels: {{ sgx_dp_labels | default(false)}} +{% if sgx_dp_enabled | default(false) or gpu_dp_enabled | default(false) %} +nfd_extra_labels_ns: +{% if sgx_dp_enabled | default(false) %} + - "sgx.intel.com" +{% endif %} +{% if gpu_dp_enabled | default(false) %} + - "gpu.intel.com" +{% endif %} +{% else %} +nfd_extra_labels_ns: [] +{% endif %} + +gpu_dp: + enabled: {{ gpu_dp_enabled | default(false) | bool | lower }} +{% if gpu_dp_max_memory is defined %} + max_memory: "{{ gpu_dp_max_memory | human_to_bytes }}" +{% else %} + max_memory: "0" +{% endif %} + +{% if sgx_dp_enabled | default(false) or qat_dp_enabled | default(false) %} +sgx_dp_enabled: {{ sgx_dp_enabled | default(false) | bool | lower }} + +config: + sources: + custom: +{% if sgx_dp_enabled | default(false) %} + - name: "intel.sgx" + labels: + intel.sgx: "true" + matchFeatures: + - feature: cpu.cpuid + matchExpressions: + SGX: {op: Exists} + SGXLC: {op: Exists} + - feature: cpu.sgx + matchExpressions: + enabled: {op: IsTrue} +{% if not (ansible_distribution == "Ubuntu" and ansible_distribution_version == "20.04") %} + - feature: kernel.config + matchExpressions: + X86_SGX: {op: Exists} +{% endif %} +{% endif %} +{% if qat_dp_enabled | default(false) %} + - name: "intel.qat" + labels: + intel.qat: "true" + matchFeatures: + - feature: pci.device + matchExpressions: + vendor: {op: In, value: ["8086"]} + device: {op: In, value: {{ qat_supported_pf_dev_ids | list + qat_supported_vf_dev_ids | list }}} +{% endif %} +{% endif %} sleepInterval: {{ nfd_sleep_interval | default("60s")}} + +tls: + enabled: {{ nfd_tls_enabled | default(false) | bool | lower }} + generate: {{ nfd_tls_generate | default(false) | bool | lower }} + masterKey: "{{ nfd_tls_master_key }}" + masterCert: "{{ nfd_tls_master_cert }}" + workerKey: "{{nfd_tls_worker_key}}" + workerCert: "{{ nfd_tls_worker_cert }}" + caCert: "{{ nfd_tls_ca_cert }}" + + +serviceAccount: + create: {{ nfd_sa_create | default(false) | bool | lower }} + name: "{{ nfd_sa_name }}" + +rbac: + enabled: {{ nfd_rbac_enabled | default(false) | bool | lower }} + +service: + port: {{ nfd_svc_port }} + type: "{{ nfd_svc_type }}" + diff --git a/roles/nfd_install/vars/main.yml b/roles/nfd_install/vars/main.yml index 9f60069c..170d32e5 100644 --- a/roles/nfd_install/vars/main.yml +++ b/roles/nfd_install/vars/main.yml @@ -1,5 +1,5 @@ ## -## Copyright (c) 2020-2021 Intel Corporation. +## Copyright (c) 2020-2022 Intel Corporation. ## ## Licensed under the Apache License, Version 2.0 (the "License"); ## you may not use this file except in compliance with the License. diff --git a/roles/openssl_engine_install/defaults/main.yml b/roles/openssl_engine_install/defaults/main.yml index d133629b..18f68e55 100644 --- a/roles/openssl_engine_install/defaults/main.yml +++ b/roles/openssl_engine_install/defaults/main.yml @@ -1,5 +1,5 @@ ## -## Copyright (c) 2020-2021 Intel Corporation. +## Copyright (c) 2020-2022 Intel Corporation. ## ## Licensed under the Apache License, Version 2.0 (the "License"); ## you may not use this file except in compliance with the License. @@ -14,13 +14,11 @@ ## limitations under the License. ## --- -openssl_engine_dir: "/usr/src/OpenSSL" +openssl_engine_dir: "{{ project_root_dir }}/openssl" openssl_engine_url: "https://github.com/intel/QAT_Engine.git" -openssl_engine_version: "v0.6.7" -nasm_ubuntu_url: "http://archive.ubuntu.com/ubuntu/pool/universe/n/nasm/nasm_2.15.04-1_amd64.deb" -nasm_centos_rhel_url: "https://www.nasm.us/pub/nasm/releasebuilds/2.15.02/linux/nasm-2.15.02-0.fc31.x86_64.rpm" -libarchive_url: "https://github.com/libarchive/libarchive/releases/download/3.5.1/libarchive-3.5.1.tar.xz" +openssl_engine_version: "v0.6.10" +libarchive_url: "https://github.com/libarchive/libarchive/releases/download/v3.5.1/libarchive-3.5.1.tar.xz" ipp_crypto_url: "https://github.com/intel/ipp-crypto.git" -ipp_crypto_version: "ippcp_2021.3" +ipp_crypto_version: "ippcp_2021.5" intel_ipsec_url: "https://github.com/intel/intel-ipsec-mb.git" -intel_ipsec_version: "v1.0" +intel_ipsec_version: "v1.1" diff --git a/roles/openssl_engine_install/tasks/main.yml b/roles/openssl_engine_install/tasks/main.yml index 6d4f8818..9160540a 100644 --- a/roles/openssl_engine_install/tasks/main.yml +++ b/roles/openssl_engine_install/tasks/main.yml @@ -1,5 +1,5 @@ ## -## Copyright (c) 2020-2021 Intel Corporation. +## Copyright (c) 2020-2022 Intel Corporation. ## ## Licensed under the Apache License, Version 2.0 (the "License"); ## you may not use this file except in compliance with the License. diff --git a/roles/openssl_engine_install/tasks/openssl_engine_config.yml b/roles/openssl_engine_install/tasks/openssl_engine_config.yml index fc2b35b9..105f8314 100644 --- a/roles/openssl_engine_install/tasks/openssl_engine_config.yml +++ b/roles/openssl_engine_install/tasks/openssl_engine_config.yml @@ -1,5 +1,5 @@ ## -## Copyright (c) 2020-2021 Intel Corporation. +## Copyright (c) 2020-2022 Intel Corporation. ## ## Licensed under the Apache License, Version 2.0 (the "License"); ## you may not use this file except in compliance with the License. @@ -18,99 +18,15 @@ include_role: name: install_dependencies -- name: remove pre-installed openssl version in CentOS <= 7.9 - yum: - name: openssl - state: absent - when: ansible_distribution == 'CentOS' and ansible_distribution_version <= '7.9' - -- name: install gcc and g++ required version for Ubuntu <= 18.04.4 - apt: - name: - - gcc-8 - - g++-8 - when: ansible_distribution == 'Ubuntu' and ansible_distribution_version <= '18.04.4' - -- name: set (gcc) in auto mode for <= Ubuntu 18.04.4 before OpenSSL Configuration - command: "update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-8 80 --slave /usr/bin/g++ g++ /usr/bin/g++-8 --slave /usr/bin/gcov gcov /usr/bin/gcov-8" - changed_when: true - when: ansible_distribution == 'Ubuntu' and ansible_distribution_version <= '18.04.4' - -# Need to install version 2.15 or later of nasm*, which is not provided by default for the Ubuntu 20.04 distribution -- name: create dir {{ openssl_engine_dir }}/nasm_ubuntu for nasm dependencies in Ubuntu - file: - path: "{{ openssl_engine_dir }}/nasm_ubuntu" - state: directory - mode: "u=rwx,g=rx,o=rx" - when: ansible_distribution == "Ubuntu" - -- name: download nasm required version for Ubuntu - get_url: - url: "{{ nasm_ubuntu_url }}" - dest: "{{ openssl_engine_dir }}/nasm_ubuntu" - register: ubuntu_version - until: ubuntu_version is not failed - retries: 5 - when: ansible_distribution == "Ubuntu" - -- name: install nasm required version for Ubuntu - command: 'dpkg -i nasm_2.15.04-1_amd64.deb' - args: - chdir: "{{ openssl_engine_dir }}/nasm_ubuntu" - become: yes - when: ansible_distribution == "Ubuntu" - changed_when: true - -# need to install version 2.15 or later of nasm*, which is not provided by default for the CentOS 8.2 distribution -- name: create dir {{ openssl_engine_dir }}/nasm_rhel_centos for nasm dependencies in CentOS / RHEL - file: - path: "{{ openssl_engine_dir }}/nasm_rhel_centos" - state: directory - mode: "u=rwx,g=rx,o=rx" - when: - - (ansible_distribution == 'CentOS' and ansible_distribution_version >= '8.2') or - (ansible_distribution == 'RedHat' and ansible_distribution_version >= '8.2') - -- name: download nasm required version for CentOS / RHEL - get_url: - url: "{{ nasm_centos_rhel_url }}" - dest: "{{ openssl_engine_dir }}/nasm_rhel_centos" - register: centos_rhel_version - until: centos_rhel_version is not failed - retries: 5 - when: - - (ansible_distribution == 'CentOS' and ansible_distribution_version >= '8.2') or - (ansible_distribution == 'RedHat' and ansible_distribution_version >= '8.2') - -# NOTE: dnf module refuse to install source package. -- name: install nasm required version for CentOS / RHEL - command: - cmd: "dnf install -y nasm-2.15.02-0.fc31.x86_64.rpm" - warn: false - args: - chdir: "{{ openssl_engine_dir }}/nasm_rhel_centos" - when: - - (ansible_distribution == 'CentOS' and ansible_distribution_version >= '8.2') or - (ansible_distribution == 'RedHat' and ansible_distribution_version >= '8.2') - changed_when: true - -- name: nasm version after installation must be >= 2.15 - command: "nasm --version" - register: nasm_version - changed_when: true - -- debug: - msg: "The nasm version after installation is: {{ nasm_version.stdout }}" - -# install libarchive required version in CentOS / RHEL >= 8.2 -- name: create dir {{ openssl_engine_dir }}/libarchive_install in CentOS / RHEL >= 8.2 +# install libarchive required version in CentOS / RHEL >= 8.3 +- name: create dir {{ openssl_engine_dir }}/libarchive_install in CentOS / RHEL >= 8.3 file: path: "{{ openssl_engine_dir }}/libarchive_install" state: directory mode: "u=rwx,g=rx,o=rx" - when: ansible_distribution in ["RedHat", "CentOS"] and ansible_distribution_version >= '8.2' + when: ansible_distribution in ["RedHat", "CentOS"] and ansible_distribution_version >= '8.3' -- name: download and unarchive libarchive required version for CentOS / RHEL >= 8.2 +- name: download and unarchive libarchive required version for CentOS / RHEL >= 8.3 unarchive: src: "{{ libarchive_url }}" dest: "{{ openssl_engine_dir }}/libarchive_install" @@ -119,84 +35,93 @@ register: libarchive_rhel_version until: libarchive_rhel_version is not failed retries: 5 - when: ansible_distribution in ["RedHat", "CentOS"] and ansible_distribution_version >= '8.2' + when: ansible_distribution in ["RedHat", "CentOS"] and ansible_distribution_version >= '8.3' -- name: configure libarchive with PATH in CentOS / RHEL >= 8.2 +- name: configure libarchive with PATH in CentOS / RHEL >= 8.3 command: './configure --prefix=/usr --disable-static' args: chdir: "{{ openssl_engine_dir }}/libarchive_install/libarchive-3.5.1" changed_when: true - when: ansible_distribution in ["RedHat", "CentOS"] and ansible_distribution_version >= '8.2' + when: ansible_distribution in ["RedHat", "CentOS"] and ansible_distribution_version >= '8.3' + +- name: detect number of cores on system + command: nproc + register: nproc_out + changed_when: false -- name: make install libarchive in CentOS / RHEL >= 8.2 +- name: make install libarchive in CentOS / RHEL >= 8.3 make: chdir: "/{{ openssl_engine_dir }}/libarchive_install/libarchive-3.5.1" target: install - when: ansible_distribution in ["RedHat", "CentOS"] and ansible_distribution_version >= '8.2' + environment: + "MAKEFLAGS": "-j{{ nproc_out.stdout | int }}" + when: ansible_distribution in ["RedHat", "CentOS"] and ansible_distribution_version >= '8.3' # Building Intel® Integrated Performance Primitives Cryptography -- name: create directory {{ openssl_engine_dir }}/IPP_Cryptography for IIPC dependencies +- name: create directory for Intel IPPC dependencies file: - path: "{{ openssl_engine_dir }}/IPP_Cryptography" + path: "{{ openssl_engine_dir }}/ipp_cryptography" state: directory mode: "u=rwx,g=rx,o=rx" -- name: clone IIP-Cryptography repository +- name: clone Intel IPP-Cryptography repository git: repo: "{{ ipp_crypto_url }}" version: "{{ ipp_crypto_version }}" - dest: "{{ openssl_engine_dir }}/IPP_Cryptography" + dest: "{{ openssl_engine_dir }}/ipp_cryptography" force: yes -- name: prepare the build by running cmake for IIP-Cryptography once successfully configured +- name: prepare the build by running cmake for Intel IPP-Cryptography once successfully configured command: 'cmake . -Bbuild -DCMAKE_INSTALL_PREFIX=/usr' args: - chdir: "{{ openssl_engine_dir }}/IPP_Cryptography/sources/ippcp/crypto_mb" + chdir: "{{ openssl_engine_dir }}/ipp_cryptography/sources/ippcp/crypto_mb" changed_when: true -- name: make perform IIP-Cryptography full build - command: 'make -j' +- name: build Intel IPP-Cryptography + command: 'make -j{{ nproc_out.stdout | int }}' args: - chdir: "{{ openssl_engine_dir }}/IPP_Cryptography/sources/ippcp/crypto_mb/build" + chdir: "{{ openssl_engine_dir }}/ipp_cryptography/sources/ippcp/crypto_mb/build" changed_when: true -- name: make install IIP-Cryptography +- name: make install Intel IPP-Cryptography make: - chdir: "{{ openssl_engine_dir }}/IPP_Cryptography/sources/ippcp/crypto_mb/build" + chdir: "{{ openssl_engine_dir }}/ipp_cryptography/sources/ippcp/crypto_mb/build" target: install + environment: + "MAKEFLAGS": "-j{{ nproc_out.stdout | int }}" become: yes # Building the Intel® Multi-Buffer Crypto for IPsec Library -- name: create directory {{ openssl_engine_dir }}/IMBC_IPsec for IMBC-IPsec dependencies +- name: create directory for Intel MBC-IPsec dependencies file: - path: "{{ openssl_engine_dir }}/IMBC_IPsec" + path: "{{ openssl_engine_dir }}/imbc_ipsec" state: directory mode: "u=rwx,g=rx,o=rx" -- name: clone IMBC-IPsec repository +- name: clone Intel MBC-IPsec repository git: repo: "{{ intel_ipsec_url }}" version: "{{ intel_ipsec_version }}" - dest: "{{ openssl_engine_dir }}/IMBC_IPsec" + dest: "{{ openssl_engine_dir }}/imbc_ipsec" force: yes -- name: build IMBC-IPsec Library - command: 'make -j SAFE_DATA=y SAFE_PARAM=y SAFE_LOOKUP=y' +- name: build Intel MBC-IPsec Library + command: 'make -j{{ nproc_out.stdout | int }} SAFE_DATA=y SAFE_PARAM=y SAFE_LOOKUP=y' args: - chdir: "{{ openssl_engine_dir }}/IMBC_IPsec" + chdir: "{{ openssl_engine_dir }}/imbc_ipsec" changed_when: true -- name: make install IMBC-IPsec Library - command: 'make install NOLDCONFIG=y' +- name: make install Intel MBC-IPsec Library + command: 'make install -j{{ nproc_out.stdout | int }} NOLDCONFIG=y' args: - chdir: "{{ openssl_engine_dir }}/IMBC_IPsec" + chdir: "{{ openssl_engine_dir }}/imbc_ipsec" become: yes changed_when: true # Building the Intel Quick Assist Technology Engine for OpenSSL -- name: create directory {{ openssl_engine_dir }}/Openssl_Engine for dependencies +- name: create directory for OpenSSL*Engine dependencies file: - path: "{{ openssl_engine_dir }}/Openssl_Engine" + path: "{{ openssl_engine_dir }}/openssl_engine" state: directory mode: "u=rwx,g=rx,o=rx" @@ -204,92 +129,45 @@ git: repo: "{{ openssl_engine_url }}" version: "{{ openssl_engine_version }}" - dest: "{{ openssl_engine_dir }}/Openssl_Engine" + dest: "{{ openssl_engine_dir }}/openssl_engine" force: yes - name: autogen configuration for OpenSSL*Engine command: ./autogen.sh args: - chdir: "{{ openssl_engine_dir }}/Openssl_Engine" + chdir: "{{ openssl_engine_dir }}/openssl_engine" changed_when: true - name: check all configuration is present for OpenSSL*Engine command: "./configure --enable-multibuff_offload --enable-ipsec_offload --enable-multibuff_ecx --enable-qat_sw" args: - chdir: "{{ openssl_engine_dir }}/Openssl_Engine" + chdir: "{{ openssl_engine_dir }}/openssl_engine" changed_when: false - name: Build OpenSSL*Engine Library once successfully configured - command: 'make -j' + command: 'make -j{{ nproc_out.stdout | int }}' args: - chdir: "{{ openssl_engine_dir }}/Openssl_Engine" + chdir: "{{ openssl_engine_dir }}/openssl_engine" changed_when: true - name: make install OpenSSL*Engine make: - chdir: "{{ openssl_engine_dir }}/Openssl_Engine" + chdir: "{{ openssl_engine_dir }}/openssl_engine" target: install - -- name: check OpenSSL new built after installation is completed - command: openssl version -b - register: openssl_new_built - changed_when: true - -- debug: - var: openssl_new_built.stdout_lines - -- name: verify OpenSSL dir after successful installation - command: "ls -al" - args: - chdir: "/usr/local/ssl" - register: verify_openssl - changed_when: true - -- debug: - var: verify_openssl.stdout_lines - -- name: check OpenSSL*Engine dir after successful installation - command: openssl version -e - register: openssl_ver_confirm - changed_when: true - when: ansible_distribution == "Ubuntu" - -- debug: - var: openssl_ver_confirm.stdout_lines - when: ansible_distribution == "Ubuntu" - -- name: verify that the engine is present in debug output in Ubuntu - command: 'ls -l /usr/lib/x86_64-linux-gnu/engines-1.1' - register: openssl_ls_ubuntu - changed_when: true - when: ansible_distribution == "Ubuntu" - -- debug: - var: openssl_ls_ubuntu.stdout_lines - when: ansible_distribution == "Ubuntu" - -- name: confirm OPENSSLDIR and ENGINEDIR dir after installation - shell: "set -o pipefail && openssl version -a | grep OPENSSLDIR: && openssl version -a | grep ENGINESDIR:" - args: - executable: /bin/bash - register: openssl_engine_dir - when: ansible_distribution in ["RedHat", "CentOS"] - -- debug: - var: openssl_engine_dir.stdout_lines - when: ansible_distribution in ["RedHat", "CentOS"] - -- name: verify that the engine is present in debug output in RHEL / CentOS - command: 'ls -l /usr/lib64/engines-1.1' - register: openssl_ls_rhel_centos - changed_when: true - when: ansible_distribution in ["RedHat", "CentOS"] - -- debug: - var: openssl_ls_rhel_centos.stdout_lines - when: ansible_distribution in ["RedHat", "CentOS"] + environment: + "MAKEFLAGS": "-j{{ nproc_out.stdout | int }}" - name: update the dynamic linker cache command: "ldconfig" when: ansible_distribution in ["RedHat", "CentOS"] changed_when: true + +- name: once OpenSSL*Engine in place ensure it loads successfully + command: openssl engine -v qatengine + register: confirm_openssl_engine + changed_when: true + +- name: OpenSSL*Engine command returns errors, playbook terminated + fail: + msg: "OpenSSL Engine failed to load... Cause of failure can be unsupported hardware or misconfiguration of Intel QAT OpenSSL*Engine" + when: "openssl_engine_version not in confirm_openssl_engine.stdout" diff --git a/roles/tas_install/vars/main.yml b/roles/openssl_engine_install/vars/main.yml similarity index 84% rename from roles/tas_install/vars/main.yml rename to roles/openssl_engine_install/vars/main.yml index c5deffd1..547e9cf5 100644 --- a/roles/tas_install/vars/main.yml +++ b/roles/openssl_engine_install/vars/main.yml @@ -1,5 +1,5 @@ ## -## Copyright (c) 2020-2021 Intel Corporation. +## Copyright (c) 2020-2022 Intel Corporation. ## ## Licensed under the Apache License, Version 2.0 (the "License"); ## you may not use this file except in compliance with the License. @@ -17,9 +17,7 @@ install_dependencies: Debian: - git - - make + - nasm RedHat: - git - - make - -helm_stable_url: "https://charts.helm.sh/stable" + - nasm diff --git a/roles/cmk_install/charts/cpu-manager-for-kubernetes/Chart.yaml b/roles/platform_aware_scheduling_install/charts/gpu-aware-scheduling/Chart.yaml similarity index 74% rename from roles/cmk_install/charts/cpu-manager-for-kubernetes/Chart.yaml rename to roles/platform_aware_scheduling_install/charts/gpu-aware-scheduling/Chart.yaml index bc0c0a5a..1937f457 100644 --- a/roles/cmk_install/charts/cpu-manager-for-kubernetes/Chart.yaml +++ b/roles/platform_aware_scheduling_install/charts/gpu-aware-scheduling/Chart.yaml @@ -1,5 +1,5 @@ ## -## Copyright (c) 2020-2021 Intel Corporation. +## Copyright (c) 2020-2022 Intel Corporation. ## ## Licensed under the Apache License, Version 2.0 (the "License"); ## you may not use this file except in compliance with the License. @@ -14,8 +14,9 @@ ## limitations under the License. ## apiVersion: v1 -name: cmk -version: 0.0.2 -description: CPU Manager for Kubernetes (CMK) +description: Intel® GPU Aware Scheduling +name: gpu-aware-scheduling +version: 0.1 sources: - - https://github.com/intel/CPU-Manager-for-Kubernetes + - https://github.com/intel/platform-aware-scheduling.git +appVersion: 0.1 diff --git a/roles/cmk_install/charts/cpu-manager-for-kubernetes/.helmignore b/roles/platform_aware_scheduling_install/charts/gpu-aware-scheduling/templates/.helmignore similarity index 100% rename from roles/cmk_install/charts/cpu-manager-for-kubernetes/.helmignore rename to roles/platform_aware_scheduling_install/charts/gpu-aware-scheduling/templates/.helmignore diff --git a/roles/platform_aware_scheduling_install/charts/gpu-aware-scheduling/templates/NOTES.txt b/roles/platform_aware_scheduling_install/charts/gpu-aware-scheduling/templates/NOTES.txt new file mode 100644 index 00000000..86348f28 --- /dev/null +++ b/roles/platform_aware_scheduling_install/charts/gpu-aware-scheduling/templates/NOTES.txt @@ -0,0 +1,31 @@ +The {{ .Chart.Name }}-{{ .Chart.Version }} was installed. + +You can now create a demo-app deployment. + +Demo-app example: + +apiVersion: apps/v1 +kind: Deployment +metadata: + name: bb-example +spec: + replicas: 1 + selector: + matchLabels: + app: bb-example + template: + metadata: + labels: + app: bb-example + spec: + containers: + - name: gpu-resource-request + image: busybox:1.33.1 + command: ['sh', '-c', 'echo The gpu resource request app is running! && sleep 6000'] + resources: + limits: + gpu.intel.com/i915: 1 + gpu.intel.com/millicores: 100 + gpu.intel.com/memory.max: 1G + +To learn more about GPU Aware Scheduling, visit https://github.com/intel/platform-aware-scheduling/tree/master/gpu-aware-scheduling. diff --git a/roles/qat_dp_install/charts/intel-qat-plugin/templates/_helpers.tpl b/roles/platform_aware_scheduling_install/charts/gpu-aware-scheduling/templates/_helpers.tpl similarity index 50% rename from roles/qat_dp_install/charts/intel-qat-plugin/templates/_helpers.tpl rename to roles/platform_aware_scheduling_install/charts/gpu-aware-scheduling/templates/_helpers.tpl index 85be03f6..23cd1455 100644 --- a/roles/qat_dp_install/charts/intel-qat-plugin/templates/_helpers.tpl +++ b/roles/platform_aware_scheduling_install/charts/gpu-aware-scheduling/templates/_helpers.tpl @@ -2,7 +2,7 @@ {{/* Expand the name of the chart. */}} -{{- define "qat-chart.name" -}} +{{- define "gpu-aware-scheduling.name" -}} {{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}} {{- end -}} @@ -11,7 +11,7 @@ Create a default fully qualified app name. We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). If release name contains chart name it will be used as a full name. */}} -{{- define "qat-chart.fullname" -}} +{{- define "gpu-aware-scheduling.fullname" -}} {{- if .Values.fullnameOverride -}} {{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}} {{- else -}} @@ -27,6 +27,33 @@ If release name contains chart name it will be used as a full name. {{/* Create chart name and version as used by the chart label. */}} -{{- define "qat-chart.chart" -}} +{{- define "gpu-aware-scheduling.chart" -}} {{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" -}} {{- end -}} + +{{/* +Common labels +*/}} +{{- define "gpu-aware-scheduling.labels" -}} +app.kubernetes.io/name: {{ include "gpu-aware-scheduling.name" . }} +helm.sh/chart: {{ include "gpu-aware-scheduling.chart" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end -}} + +{{/* +Service account name to use +*/}} +{{- define "gpu-aware-scheduling.serviceAccountName" -}} +{{ .Values.serviceAccount.name | default (include "gpu-aware-scheduling.fullname" .) }} +{{- end -}} + +{{/* +Service name to use +*/}} +{{- define "gpu-aware-scheduling.serviceName" -}} +{{ .Values.service.name | default (include "gpu-aware-scheduling.fullname" .) }} +{{- end -}} diff --git a/roles/platform_aware_scheduling_install/charts/gpu-aware-scheduling/templates/gas-deployment.yaml b/roles/platform_aware_scheduling_install/charts/gpu-aware-scheduling/templates/gas-deployment.yaml new file mode 100644 index 00000000..7957f369 --- /dev/null +++ b/roles/platform_aware_scheduling_install/charts/gpu-aware-scheduling/templates/gas-deployment.yaml @@ -0,0 +1,61 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "gpu-aware-scheduling.fullname" . }} + namespace: {{ .Values.namespace }} + labels: +{{ include "gpu-aware-scheduling.labels" . | indent 4 }} +spec: + replicas: {{ .Values.replicaCount }} + selector: + matchLabels: + app.kubernetes.io/name: {{ include "gpu-aware-scheduling.name" . }} + app.kubernetes.io/instance: {{ .Release.Name }} + template: + metadata: + annotations: + checksum/config: {{ randAlphaNum 5 | quote }} + labels: +{{ include "gpu-aware-scheduling.labels" . | indent 8 }} + spec: + {{- with .Values.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + serviceAccountName: {{ template "gpu-aware-scheduling.serviceAccountName" . }} + containers: + - name: gas-extender + command: + - /extender + - --v={{ .Values.verbosity }} + {{- if .Values.tls.enabled }} + - --cert={{ .Values.tls.gas_ssl }}/{{ .Values.tls.secretName }}.cert + - --cacert={{ .Values.tls.gas_ssl }}/ca.crt + - --key={{ .Values.tls.gas_ssl }}/{{ .Values.tls.secretName }}.key + {{- else }} + - --unsafe=true + {{- end }} + image: "{{ .Values.extenderImage.repository }}:{{ .Values.extenderImage.tag }}" + imagePullPolicy: {{ .Values.extenderImage.pullPolicy }} + securityContext: + runAsUser: 10001 + {{- if .Values.tls.enabled }} + volumeMounts: + - name: certs + mountPath: {{ .Values.tls.gas_ssl }} + volumes: + - name: certs + secret: + secretName: {{ .Values.tls.secretName }} + {{- end}} + tolerations: + - key: node-role.kubernetes.io/master + operator: Exists + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: node-role.kubernetes.io/master + operator: Exists + diff --git a/roles/platform_aware_scheduling_install/charts/gpu-aware-scheduling/templates/gas-rbac-accounts.yaml b/roles/platform_aware_scheduling_install/charts/gpu-aware-scheduling/templates/gas-rbac-accounts.yaml new file mode 100644 index 00000000..0187c49d --- /dev/null +++ b/roles/platform_aware_scheduling_install/charts/gpu-aware-scheduling/templates/gas-rbac-accounts.yaml @@ -0,0 +1,34 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ include "gpu-aware-scheduling.fullname" . }}-cluster-role +rules: +- apiGroups: [""] + resources: ["pods"] + verbs: ["get","list","watch","update","patch"] +- apiGroups: [""] + resources: ["nodes"] + verbs: ["get", "list", "watch", "patch"] +- apiGroups: [""] + resources: ["bindings","pods/binding"] + verbs: ["create"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ include "gpu-aware-scheduling.fullname" . }}-cluster-role-binding +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ include "gpu-aware-scheduling.fullname" . }}-cluster-role +subjects: +- kind: ServiceAccount + name: {{ template "gpu-aware-scheduling.serviceAccountName" . }} + namespace: {{ .Values.namespace }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "gpu-aware-scheduling.serviceAccountName" . }} + labels: +{{ include "gpu-aware-scheduling.labels" . | indent 4 }} diff --git a/roles/platform_aware_scheduling_install/charts/gpu-aware-scheduling/templates/gas-service.yaml b/roles/platform_aware_scheduling_install/charts/gpu-aware-scheduling/templates/gas-service.yaml new file mode 100644 index 00000000..969ec0af --- /dev/null +++ b/roles/platform_aware_scheduling_install/charts/gpu-aware-scheduling/templates/gas-service.yaml @@ -0,0 +1,14 @@ +apiVersion: v1 +kind: Service +metadata: + name: {{ template "gpu-aware-scheduling.serviceName" . }} + namespace: {{ .Values.namespace }} + labels: +{{ include "gpu-aware-scheduling.labels" . | indent 4 }} +spec: + selector: + app.kubernetes.io/name: {{ include "gpu-aware-scheduling.name" . }} + app.kubernetes.io/instance: {{ .Release.Name }} + type: {{ .Values.service.type }} + ports: + - port: {{ .Values.service.port }} diff --git a/roles/platform_aware_scheduling_install/charts/gpu-aware-scheduling/values.yaml b/roles/platform_aware_scheduling_install/charts/gpu-aware-scheduling/values.yaml new file mode 100644 index 00000000..bb623da7 --- /dev/null +++ b/roles/platform_aware_scheduling_install/charts/gpu-aware-scheduling/values.yaml @@ -0,0 +1,45 @@ +## +## Copyright (c) 2020-2022 Intel Corporation. +## +## Licensed under the Apache License, Version 2.0 (the "License"); +## you may not use this file except in compliance with the License. +## You may obtain a copy of the License at +## +## http://www.apache.org/licenses/LICENSE-2.0 +## +## Unless required by applicable law or agreed to in writing, software +## distributed under the License is distributed on an "AS IS" BASIS, +## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +## See the License for the specific language governing permissions and +## limitations under the License. +## + +# Default values for gpu-aware-schedulng. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +replicaCount: 1 + +extenderImage: + repository: "docker.io/intel/gpu-extender" + tag: "0.1" + pullPolicy: IfNotPresent + +nameOverride: "" +namespace: default +fullnameOverride: "" +clusterName: "cluster.local" + +serviceAccount: {} + +service: + type: ClusterIP + port: 80 + +tls: + enabled: false +# secretName: extender-secret +# key: +# certificate: +verbosity: 4 + diff --git a/roles/qat_dp_install/charts/intel-qat-plugin/.helmignore b/roles/platform_aware_scheduling_install/charts/telemetry-aware-scheduling/.helmignore similarity index 100% rename from roles/qat_dp_install/charts/intel-qat-plugin/.helmignore rename to roles/platform_aware_scheduling_install/charts/telemetry-aware-scheduling/.helmignore diff --git a/roles/tas_install/charts/telemetry-aware-scheduling/Chart.yaml b/roles/platform_aware_scheduling_install/charts/telemetry-aware-scheduling/Chart.yaml similarity index 83% rename from roles/tas_install/charts/telemetry-aware-scheduling/Chart.yaml rename to roles/platform_aware_scheduling_install/charts/telemetry-aware-scheduling/Chart.yaml index 6c462132..83dd52c9 100644 --- a/roles/tas_install/charts/telemetry-aware-scheduling/Chart.yaml +++ b/roles/platform_aware_scheduling_install/charts/telemetry-aware-scheduling/Chart.yaml @@ -1,5 +1,5 @@ ## -## Copyright (c) 2020-2021 Intel Corporation. +## Copyright (c) 2020-2022 Intel Corporation. ## ## Licensed under the Apache License, Version 2.0 (the "License"); ## you may not use this file except in compliance with the License. @@ -16,7 +16,7 @@ apiVersion: v1 description: Intel® Telemetry Aware Scheduling name: telemetry-aware-scheduling -version: 0.4.0 +version: 0.1 sources: - - https://github.com/intel/telemetry-aware-scheduling.git -appVersion: 0.4 + - https://github.com/intel/platform-aware-scheduling.git +appVersion: 0.1 diff --git a/roles/platform_aware_scheduling_install/charts/telemetry-aware-scheduling/crds/tas-policy-crd.yaml b/roles/platform_aware_scheduling_install/charts/telemetry-aware-scheduling/crds/tas-policy-crd.yaml new file mode 100644 index 00000000..5b9dd943 --- /dev/null +++ b/roles/platform_aware_scheduling_install/charts/telemetry-aware-scheduling/crds/tas-policy-crd.yaml @@ -0,0 +1,83 @@ +## +## Copyright (c) 2020-2022 Intel Corporation. +## +## Licensed under the Apache License, Version 2.0 (the "License"); +## you may not use this file except in compliance with the License. +## You may obtain a copy of the License at +## +## http://www.apache.org/licenses/LICENSE-2.0 +## +## Unless required by applicable law or agreed to in writing, software +## distributed under the License is distributed on an "AS IS" BASIS, +## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +## See the License for the specific language governing permissions and +## limitations under the License. +## +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: taspolicies.telemetry.intel.com +spec: + group: telemetry.intel.com + names: + kind: TASPolicy + listKind: TASPolicyList + plural: taspolicies + singular: taspolicy + scope: Namespaced + versions: + - name: v1alpha1 + served: true + storage: true + schema: + openAPIV3Schema: + type: object + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client' + type: string + metadata: + type: object + spec: + properties: + strategies: + additionalProperties: + properties: + policyName: + type: string + rules: + items: + properties: + metricname: + type: string + operator: + type: string + target: + format: int64 + type: integer + required: + - metricname + - operator + type: object + type: array + required: + - rules + type: object + type: object + required: + - strategies + type: object + status: + properties: + compliance: + type: string + message: + type: string + type: object + subresources: + status: {} diff --git a/roles/tas_install/charts/telemetry-aware-scheduling/templates/NOTES.txt b/roles/platform_aware_scheduling_install/charts/telemetry-aware-scheduling/templates/NOTES.txt similarity index 85% rename from roles/tas_install/charts/telemetry-aware-scheduling/templates/NOTES.txt rename to roles/platform_aware_scheduling_install/charts/telemetry-aware-scheduling/templates/NOTES.txt index 394c092b..b7fb20ec 100644 --- a/roles/tas_install/charts/telemetry-aware-scheduling/templates/NOTES.txt +++ b/roles/platform_aware_scheduling_install/charts/telemetry-aware-scheduling/templates/NOTES.txt @@ -26,6 +26,6 @@ spec: - metricname: node_metric operator: GreaterThan -To link a workload to a scheduling policy add a label of the form telemetry-policy=. +To link a workload to a scheduling policy add a label of the form telemetry-policy=. -To learn more about Telemetry Aware Scheduling, visit https://github.com/intel/telemetry-aware-scheduling. +To learn more about Telemetry Aware Scheduling, visit https://github.com/intel/platform-aware-scheduling/tree/master/telemetry-aware-scheduling. diff --git a/roles/tas_install/charts/telemetry-aware-scheduling/templates/_helpers.tpl b/roles/platform_aware_scheduling_install/charts/telemetry-aware-scheduling/templates/_helpers.tpl similarity index 84% rename from roles/tas_install/charts/telemetry-aware-scheduling/templates/_helpers.tpl rename to roles/platform_aware_scheduling_install/charts/telemetry-aware-scheduling/templates/_helpers.tpl index eb757d6c..1bd08bdc 100644 --- a/roles/tas_install/charts/telemetry-aware-scheduling/templates/_helpers.tpl +++ b/roles/platform_aware_scheduling_install/charts/telemetry-aware-scheduling/templates/_helpers.tpl @@ -45,12 +45,15 @@ app.kubernetes.io/managed-by: {{ .Release.Service }} {{- end -}} {{/* -Create the name of the service account to use +Service account name to use */}} {{- define "telemetry-aware-scheduling.serviceAccountName" -}} -{{- if .Values.serviceAccount.create -}} - {{ default (include "telemetry-aware-scheduling.fullname" .) .Values.serviceAccount.name }} -{{- else -}} - {{ default "default" .Values.serviceAccount.name }} +{{ .Values.serviceAccount.name | default (include "telemetry-aware-scheduling.fullname" .) }} {{- end -}} + +{{/* +Service name to use +*/}} +{{- define "telemetry-aware-scheduling.serviceName" -}} +{{ .Values.service.name | default (include "telemetry-aware-scheduling.fullname" .) }} {{- end -}} diff --git a/roles/tas_install/templates/tas-demo-policy.yml.j2 b/roles/platform_aware_scheduling_install/charts/telemetry-aware-scheduling/templates/tas-demo-policy.yaml similarity index 81% rename from roles/tas_install/templates/tas-demo-policy.yml.j2 rename to roles/platform_aware_scheduling_install/charts/telemetry-aware-scheduling/templates/tas-demo-policy.yaml index e7e0577a..6c4515a5 100644 --- a/roles/tas_install/templates/tas-demo-policy.yml.j2 +++ b/roles/platform_aware_scheduling_install/charts/telemetry-aware-scheduling/templates/tas-demo-policy.yaml @@ -1,8 +1,9 @@ +{{- if .Values.demoPolicy -}} apiVersion: telemetry.intel.com/v1alpha1 kind: TASPolicy metadata: name: demo-policy - namespace: {{ tas_namespace }} + namespace: {{ .Release.Namespace }} spec: strategies: deschedule: @@ -21,4 +22,6 @@ spec: scheduleonmetric: rules: - metricname: health_metric - operator: LessThan \ No newline at end of file + operator: LessThan +{{- end -}} + diff --git a/roles/tas_install/charts/telemetry-aware-scheduling/templates/tas-deployment.yml b/roles/platform_aware_scheduling_install/charts/telemetry-aware-scheduling/templates/tas-deployment.yaml similarity index 73% rename from roles/tas_install/charts/telemetry-aware-scheduling/templates/tas-deployment.yml rename to roles/platform_aware_scheduling_install/charts/telemetry-aware-scheduling/templates/tas-deployment.yaml index 06bc5f53..d8c2a43b 100644 --- a/roles/tas_install/charts/telemetry-aware-scheduling/templates/tas-deployment.yml +++ b/roles/platform_aware_scheduling_install/charts/telemetry-aware-scheduling/templates/tas-deployment.yaml @@ -13,6 +13,8 @@ spec: app.kubernetes.io/instance: {{ .Release.Name }} template: metadata: + annotations: + checksum/config: {{ randAlphaNum 5 | quote }} labels: {{ include "telemetry-aware-scheduling.labels" . | indent 8 }} spec: @@ -21,22 +23,16 @@ spec: {{- toYaml . | nindent 8 }} {{- end }} serviceAccountName: {{ template "telemetry-aware-scheduling.serviceAccountName" . }} - tolerations: - - key: cmk - operator: Equal - effect: NoSchedule - value: "true" containers: - name: tas-extender command: - /extender - --syncPeriod=2s - - --port=8111 - - --v=2 + - --v={{ .Values.verbosity }} {{- if .Values.tls.enabled }} - - --cert={{ .Values.tls.tas_ssl_mount_path }}/{{ .Values.tls.secretName }}.cert - - --cacert={{ .Values.tls.tas_ssl_mount_path }}/{{ .Values.tls.secretName }}.cert - - --key={{ .Values.tls.tas_ssl_mount_path }}/{{ .Values.tls.secretName }}.key + - --cert={{ .Values.tls.tas_ssl }}/{{ .Values.tls.secretName }}.cert + - --cacert={{ .Values.tls.tas_ssl }}/ca.crt + - --key={{ .Values.tls.tas_ssl }}/{{ .Values.tls.secretName }}.key {{- else }} - --unsafe=true {{- end }} @@ -47,9 +43,10 @@ spec: {{- if .Values.tls.enabled }} volumeMounts: - name: certs - mountPath: {{ .Values.tls.tas_ssl_mount_path }} + mountPath: {{ .Values.tls.tas_ssl }} volumes: - name: certs secret: secretName: {{ .Values.tls.secretName }} - {{- end}} + {{- end }} + diff --git a/roles/tas_install/charts/telemetry-aware-scheduling/templates/tas-rbac-accounts.yml b/roles/platform_aware_scheduling_install/charts/telemetry-aware-scheduling/templates/tas-rbac-accounts.yaml similarity index 63% rename from roles/tas_install/charts/telemetry-aware-scheduling/templates/tas-rbac-accounts.yml rename to roles/platform_aware_scheduling_install/charts/telemetry-aware-scheduling/templates/tas-rbac-accounts.yaml index 2e2156c6..b76e2dea 100644 --- a/roles/tas_install/charts/telemetry-aware-scheduling/templates/tas-rbac-accounts.yml +++ b/roles/platform_aware_scheduling_install/charts/telemetry-aware-scheduling/templates/tas-rbac-accounts.yaml @@ -17,29 +17,6 @@ rules: verbs: ["get", "list", "patch"] --- apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: {{ include "telemetry-aware-scheduling.fullname" . }}-configmapgetter - namespace: {{ .Values.namespace }} -rules: -- apiGroups: [""] - resources: ["configmaps"] - verbs: ["get"] ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: {{ include "telemetry-aware-scheduling.fullname" . }}-scheduler-config-map -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: {{ include "telemetry-aware-scheduling.fullname" . }}-configmapgetter -subjects: -- apiGroup: rbac.authorization.k8s.io - kind: User - name: system:kube-scheduler ---- -apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: name: {{ include "telemetry-aware-scheduling.fullname" . }}-handle-policies diff --git a/roles/tas_install/charts/telemetry-aware-scheduling/templates/tas-service.yml b/roles/platform_aware_scheduling_install/charts/telemetry-aware-scheduling/templates/tas-service.yaml similarity index 85% rename from roles/tas_install/charts/telemetry-aware-scheduling/templates/tas-service.yml rename to roles/platform_aware_scheduling_install/charts/telemetry-aware-scheduling/templates/tas-service.yaml index bda4bf47..4e6f7a5e 100644 --- a/roles/tas_install/charts/telemetry-aware-scheduling/templates/tas-service.yml +++ b/roles/platform_aware_scheduling_install/charts/telemetry-aware-scheduling/templates/tas-service.yaml @@ -1,7 +1,7 @@ apiVersion: v1 kind: Service metadata: - name: {{ include "telemetry-aware-scheduling.fullname" . }} + name: {{ template "telemetry-aware-scheduling.serviceName" . }} namespace: {{ .Values.namespace }} labels: {{ include "telemetry-aware-scheduling.labels" . | indent 4 }} diff --git a/roles/tas_install/charts/telemetry-aware-scheduling/values.yaml b/roles/platform_aware_scheduling_install/charts/telemetry-aware-scheduling/values.yaml similarity index 86% rename from roles/tas_install/charts/telemetry-aware-scheduling/values.yaml rename to roles/platform_aware_scheduling_install/charts/telemetry-aware-scheduling/values.yaml index d2daee80..2f2a9fbe 100644 --- a/roles/tas_install/charts/telemetry-aware-scheduling/values.yaml +++ b/roles/platform_aware_scheduling_install/charts/telemetry-aware-scheduling/values.yaml @@ -1,5 +1,5 @@ ## -## Copyright (c) 2020-2021 Intel Corporation. +## Copyright (c) 2020-2022 Intel Corporation. ## ## Licensed under the Apache License, Version 2.0 (the "License"); ## you may not use this file except in compliance with the License. @@ -13,6 +13,7 @@ ## See the License for the specific language governing permissions and ## limitations under the License. ## + # Default values for telemetry-aware-schedulng. # This is a YAML-formatted file. # Declare variables to be passed into your templates. @@ -20,8 +21,8 @@ replicaCount: 1 extenderImage: - repository: "localhost:30500/tasextender" - tag: "{{ tas_version }}" + repository: "docker.io/intel/telemetry-aware-scheduling" + tag: "0.1" pullPolicy: IfNotPresent nameOverride: "" @@ -29,8 +30,7 @@ namespace: default fullnameOverride: "" clusterName: "cluster.local" -serviceAccount: - create: true +serviceAccount: {} service: type: ClusterIP @@ -42,3 +42,4 @@ tls: # key: # certificate: +verbosity: 4 diff --git a/roles/platform_aware_scheduling_install/defaults/main.yml b/roles/platform_aware_scheduling_install/defaults/main.yml new file mode 100644 index 00000000..d3b10b78 --- /dev/null +++ b/roles/platform_aware_scheduling_install/defaults/main.yml @@ -0,0 +1,77 @@ +## +## Copyright (c) 2020-2022 Intel Corporation. +## +## Licensed under the Apache License, Version 2.0 (the "License"); +## you may not use this file except in compliance with the License. +## You may obtain a copy of the License at +## +## http://www.apache.org/licenses/LICENSE-2.0 +## +## Unless required by applicable law or agreed to in writing, software +## distributed under the License is distributed on an "AS IS" BASIS, +## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +## See the License for the specific language governing permissions and +## limitations under the License. +## +--- +install_dependencies: + Debian: + - git + - make + RedHat: + - git + - make + +# Platform Aware Scheduler +pas_git_url: "https://github.com/intel/platform-aware-scheduling.git" +pas_dir: "{{ (project_root_dir, 'platform-aware-scheduling') | path_join }}" +pas_namespace: kube-system + +# Descheduler +descheduler_git_url: https://github.com/kubernetes-sigs/descheduler.git +descheduler_git_version: "v0.22.1" +descheduler_dir: "{{ (project_root_dir, 'sigs.k8s.io/descheduler') | path_join }}" +sigs_k8s_io_dir: "{{ (project_root_dir, 'sigs.k8s.io') | path_join }}" + +# TAS deployment +tas_enabled: false +tas_build_image_locally: false +tas_git_version: "tas_v0.1" +tas_extender_image_tag_default: "0.1" +tas_version: "{{ ('tas' in tas_git_version) | ternary(tas_git_version[5:], tas_git_version[1:]) }}" +tas_extender_dir: "telemetry-aware-scheduling" +tas_extender_image: "{{ (tas_build_image_locally | bool) | ternary(registry_local_address + '/telemetry-aware-scheduling', 'docker.io/intel/telemetry-aware-scheduling') }}" +tas_extender_image_tag: "{{ (tas_build_image_locally | bool or 'tas' in tas_git_version) | ternary(tas_version, tas_extender_image_tag_default) }}" +tas_image_build_context: "{{ ('tas' in tas_git_version or not 'tas' in tas_git_version and tas_version is version('0.6', '>')) | ternary('../', './bin') }}" +tas_build_bin: true +tas_name: "tas" +tas_extender_secret_name: "tas-extender-secret" +tas_service_name: "telemetry-aware-scheduling" +tas_tls_enabled: true +tas_ssl: "/etc/ssl/tas" +tas_enable_demo_policy: false +tas_service_type: "ClusterIP" +tas_service_port: 9001 +tas_service_account_name: '' +tas_verbosity: 4 + +# GAS deployment +gas_enabled: false +gas_build_image_locally: false +gas_git_version: "gas_v0.1" +gas_extender_image_tag_default: "0.1" +gas_version: "{{ ('gas' in gas_git_version) | ternary(gas_git_version[5:], gas_git_version[1:]) }}" +gas_extender_dir: "gpu-aware-scheduling" +gas_extender_image: "{{ (gas_build_image_locally | bool) | ternary(registry_local_address + '/gpu-extender', 'docker.io/intel/gpu-extender') }}" +gas_extender_image_tag: "{{ (gas_build_image_locally | bool or 'gas' in gas_git_version) | ternary(gas_version, gas_extender_image_tag_default) }}" +gas_image_build_context: "../" +gas_build_bin: false +gas_name: "gas" +gas_extender_secret_name: "gas-extender-secret" +gas_service_name: "gpu-aware-scheduling" +gas_tls_enabled: true +gas_ssl: "/etc/ssl/gas" +gas_service_type: "ClusterIP" +gas_service_port: 9001 +gas_service_account_name: '' +gas_verbosity: 4 diff --git a/roles/platform_aware_scheduling_install/tasks/backups_restore.yml b/roles/platform_aware_scheduling_install/tasks/backups_restore.yml new file mode 100644 index 00000000..cd4638b2 --- /dev/null +++ b/roles/platform_aware_scheduling_install/tasks/backups_restore.yml @@ -0,0 +1,49 @@ +## +## Copyright (c) 2020-2022 Intel Corporation. +## +## Licensed under the Apache License, Version 2.0 (the "License"); +## you may not use this file except in compliance with the License. +## You may obtain a copy of the License at +## +## http://www.apache.org/licenses/LICENSE-2.0 +## +## Unless required by applicable law or agreed to in writing, software +## distributed under the License is distributed on an "AS IS" BASIS, +## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +## See the License for the specific language governing permissions and +## limitations under the License. +## +--- +- name: restore kube-scheduler config files backups + block: + - name: calculate current config checksum + stat: + path: "{{ item }}" + get_checksum: true + changed_when: false + register: config_current + + - name: calculate backup checksum + stat: + path: "{{ ('/etc/kubernetes', item | basename ) | path_join }}.bak" + get_checksum: true + changed_when: false + register: config_backup + + - name: restore config file from backup + copy: + src: "{{ ('/etc/kubernetes', item | basename ) | path_join }}.bak" + dest: "{{ item }}" + remote_src: true + mode: preserve + when: + - config_backup.stat.exists + - config_current.stat.checksum != config_backup.stat.checksum + + - name: remove backup files + file: + path: "{{ ('/etc/kubernetes', item | basename ) | path_join }}.bak" + state: absent + when: + - config_backup.stat.exists + diff --git a/roles/platform_aware_scheduling_install/tasks/build_image.yml b/roles/platform_aware_scheduling_install/tasks/build_image.yml new file mode 100644 index 00000000..e3317416 --- /dev/null +++ b/roles/platform_aware_scheduling_install/tasks/build_image.yml @@ -0,0 +1,72 @@ +## +## Copyright (c) 2020-2022 Intel Corporation. +## +## Licensed under the Apache License, Version 2.0 (the "License"); +## you may not use this file except in compliance with the License. +## You may obtain a copy of the License at +## +## http://www.apache.org/licenses/LICENSE-2.0 +## +## Unless required by applicable law or agreed to in writing, software +## distributed under the License is distributed on an "AS IS" BASIS, +## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +## See the License for the specific language governing permissions and +## limitations under the License. +## +--- +- name: clone PAS repository + git: + repo: "{{ pas_git_url }}" + version: "{{ item.version }}" + dest: "{{ pas_dir }}" + force: yes + retries: 5 + delay: 5 + register: result + until: not result.failed + +- name: "prepare to build {{ item.image }} extender image locally" + block: + - name: build extender binary externally + make: + target: build + chdir: "{{ (pas_dir, item.dir) | path_join }}" + when: + - item.bin_build + + - name: "find Dockerfile for {{ item.image }} extender image" + find: + paths: "{{ (pas_dir, item.dir, 'deploy/images') | path_join }}" + recurse: true + use_regex: true + patterns: "^[/a-zA-Z-_0-9]*Dockerfile.*$" + register: dockerfile + failed_when: dockerfile.skipped | default(false) or dockerfile.matched == 0 or dockerfile.files | length == 0 + +- name: "build {{ item.image }} extender image locally" + block: + - name: "build {{ item.image }} extender image" + command: "docker build -t {{ item.image }}:{{ item.tag }} -f {{ dockerfile.files[0].path }} {{ item.context }}" + args: + chdir: "{{ (pas_dir, item.dir) | path_join }}" + changed_when: false + + - name: "push {{ item.image }} extender image to local registry" + command: "docker push {{ item.image }}:{{ item.tag }}" + changed_when: true + when: + - container_runtime == "docker" + +- name: "build {{ item.image }} extender image locally" + block: + - name: "build and tag {{ item.image }} extender image" + command: "buildah bud -t {{ item.image }}:{{ item.tag }} -f {{ dockerfile.files[0].path }} {{ item.context }}" + args: + chdir: "{{ (pas_dir, item.dir) | path_join }}" + changed_when: true + + - name: "push {{ item.image }} extender image extender image to local registry" + command: buildah push {{ item.image }}:{{ item.tag }} + changed_when: true + when: + - '"docker" not in container_runtime' diff --git a/roles/platform_aware_scheduling_install/tasks/create-scheduler-config.yml b/roles/platform_aware_scheduling_install/tasks/create-scheduler-config.yml new file mode 100644 index 00000000..c7efbf22 --- /dev/null +++ b/roles/platform_aware_scheduling_install/tasks/create-scheduler-config.yml @@ -0,0 +1,194 @@ +## +## Copyright (c) 2020-2022 Intel Corporation. +## +## Licensed under the Apache License, Version 2.0 (the "License"); +## you may not use this file except in compliance with the License. +## You may obtain a copy of the License at +## +## http://www.apache.org/licenses/LICENSE-2.0 +## +## Unless required by applicable law or agreed to in writing, software +## distributed under the License is distributed on an "AS IS" BASIS, +## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +## See the License for the specific language governing permissions and +## limitations under the License. +## +--- +- name: backup kube-scheduler config files + copy: + src: "{{ item }}" + dest: "{{ ('/etc/kubernetes', item | basename) | path_join }}.bak" + remote_src: true + mode: preserve + loop: "{{ kube_scheduler_configs }}" + +- name: copy current kube-scheduler config files to tmp files + copy: + src: "{{ item }}" + dest: "{{ ('/tmp', item | basename) | path_join }}" + remote_src: true + mode: preserve + loop: "{{ kube_scheduler_configs }}" + +- name: Scheduler configuration + block: + - name: configure API version in kube-scheduler config file + lineinfile: + path: /tmp/kubescheduler-config.yaml + line: "apiVersion: kubescheduler.config.k8s.io/v1beta2" + regexp: "apiVersion: " + state: present + mode: 0600 + + - name: add extender conifguration in kube-scheduler config file + blockinfile: + path: /tmp/kubescheduler-config.yaml + mode: 0600 + block: | + extenders: + {% if tas_enabled %} + - urlPrefix: "http{% if tas_tls_enabled %}s{% endif %}://{{ tas_service_name }}.{{ pas_namespace }}.svc.{{ cluster_name }}:{{ tas_service_port }}" + prioritizeVerb: "scheduler/prioritize" + filterVerb: "scheduler/filter" + weight: 1 + enableHTTPS: {% if tas_tls_enabled %}true{% else %}false{% endif %}{{''}} + managedResources: + - name: "telemetry/scheduling" + ignoredByScheduler: true + ignorable: true + {% if tas_tls_enabled %} + tlsConfig: + insecure: false + certFile: "/host/certs/tas/tls-extender-client.pem" + keyFile: "/host/certs/tas/tls-extender-client-key.pem" + {% endif %} + {% endif %} + {% if gas_enabled %} + - urlPrefix: "http{% if gas_tls_enabled %}s{% endif %}://{{ gas_service_name }}.{{ pas_namespace }}.svc.{{ cluster_name }}:{{ gas_service_port }}" + prioritizeVerb: "scheduler/prioritize" + filterVerb: "scheduler/filter" + bindVerb: "scheduler/bind" + weight: 1 + enableHTTPS: {% if gas_tls_enabled %}true{% else %}false{% endif %}{{''}} + managedResources: + - name: "gpu.intel.com/i915" + ignoredByScheduler: true + ignorable: true + {% if gas_tls_enabled %} + tlsConfig: + insecure: false + certFile: "/host/certs/gas/tls-extender-client.pem" + keyFile: "/host/certs/gas/tls-extender-client-key.pem" + {% endif %} + {% endif %} + when: kube_version is version('v1.22', '>=') + +- name: configure kube-scheduler with ConfigMap + block: + - name: create extenders policy ConfigMap manifest + template: + src: "pas-extenders-configmap.yaml.j2" + dest: "{{ (project_root_dir, 'charts/pas-extenders-configmap.yaml') | path_join }}" + force: yes + mode: preserve + + - name: create ConfigMap instance + shell: >- + set -o pipefail && + kubectl apply -f ./pas-extenders-configmap.yaml + --namespace {{ pas_namespace }} + --dry-run=client -o yaml | kubectl apply -f - + args: + chdir: "{{ (project_root_dir, 'charts') | path_join }}" + executable: /bin/bash + changed_when: true + + - name: create ClusterRole for kube-scheduler to read ConfigMaps + template: + src: "pas-cluster-role-configmapgetter.yaml.j2" + dest: "{{ (project_root_dir, 'charts/pas-cluster-role-configmapgetter.yaml') | path_join }}" + force: yes + mode: preserve + + - name: create kube-scheduler ClusterRole + shell: >- + set -o pipefail && + kubectl apply -f ./pas-cluster-role-configmapgetter.yaml + --dry-run=client -o yaml | kubectl apply -f - + + args: + chdir: "{{ (project_root_dir, 'charts') | path_join }}" + executable: /bin/bash + changed_when: true + + - name: configure arguments to our kube-scheduler manifest - configmap + lineinfile: + path: /tmp/kube-scheduler.yaml + insertafter: " - kube-scheduler" + line: "{{ item.arg }}={{ item.value }}" + regexp: "{{ item.arg }}" + state: present + mode: 0600 + with_items: + - { arg: " - --policy-configmap", value: "pas-scheduler-extenders-policy" } + - { arg: " - --policy-configmap-namespace", value: "{{ pas_namespace }}" } + when: kube_version is version('v1.22', '<') + +- name: configure arguments from Kubernetes Scheduler file - dnsPolicy + lineinfile: + path: /tmp/kube-scheduler.yaml + insertafter: "spec:" + line: " dnsPolicy: ClusterFirstWithHostNet" + regexp: " dnsPolicy: " + state: present + mode: 0600 + +- name: add volume with certs + block: + - name: configure volumes + blockinfile: + path: /tmp/kube-scheduler.yaml + mode: 0600 + insertafter: " volumes:" + marker: "# {mark} - ANSIBLE MANAGED BLOCK" + marker_begin: "volumes" + marker_end: "volumes" + block: |2 + - hostPath: + path: {{ item.ssl }} + name: {{ item.name }}-extender-client + loop: "{{ extenders }}" + + - name: volumeMount with certs + blockinfile: + path: /tmp/kube-scheduler.yaml + mode: 0600 + insertafter: " volumeMounts:" + marker: "# {mark} - ANSIBLE MANAGED BLOCK" + marker_begin: "mounts" + marker_end: "mounts" + block: |4 + - mountPath: /host/certs/{{ item.name }} + name: {{ item.name }}-extender-client + readOnly: true + loop: "{{ extenders }}" + when: + - item.tls_enabled + +- name: copy new kube-scheduler config files from tmp files + copy: + src: "{{ ('/tmp', item | basename) | path_join }}" + dest: "{{ item }}" + remote_src: true + mode: preserve + loop: "{{ kube_scheduler_configs }}" + +- name: remove tmp config files + file: + state: absent + path: "{{ ('/tmp', item | basename) | path_join }}" + loop: "{{ kube_scheduler_configs }}" + +- name: wait for kube-scheduler + pause: + seconds: 10 diff --git a/roles/platform_aware_scheduling_install/tasks/gas.yml b/roles/platform_aware_scheduling_install/tasks/gas.yml new file mode 100644 index 00000000..e2e6b40f --- /dev/null +++ b/roles/platform_aware_scheduling_install/tasks/gas.yml @@ -0,0 +1,51 @@ +## +## Copyright (c) 2020-2022 Intel Corporation. +## +## Licensed under the Apache License, Version 2.0 (the "License"); +## you may not use this file except in compliance with the License. +## You may obtain a copy of the License at +## +## http://www.apache.org/licenses/LICENSE-2.0 +## +## Unless required by applicable law or agreed to in writing, software +## distributed under the License is distributed on an "AS IS" BASIS, +## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +## See the License for the specific language governing permissions and +## limitations under the License. +## +--- +- name: copy Helm chart resource definition to controller node + copy: + src: "{{ (role_path, 'charts/gpu-aware-scheduling') | path_join }}" + dest: "{{ (project_root_dir, 'charts') | path_join }}" + mode: preserve + +- name: populate gas Helm chart values template and push to controller node + template: + src: "{{ item.src }}" + dest: "{{ item.dest }}" + force: yes + mode: preserve + loop: + - { + src: "gas-values.yaml.j2", + dest: "{{ (project_root_dir, 'charts/gpu-aware-scheduling/values.yaml') | path_join }}" + } + - { + src: "gas-chart.yaml.j2", + dest: "{{ (project_root_dir, 'charts/gpu-aware-scheduling/Chart.yaml') | path_join }}" + } + +- name: install GAS helm chart + command: >- + helm upgrade -i {{ gas_name }} + --namespace {{ pas_namespace }} + ./ + args: + chdir: "{{ (project_root_dir, 'charts/gpu-aware-scheduling') | path_join }}" + retries: 5 + delay: 5 + register: result + until: result.rc == 0 + changed_when: true + diff --git a/roles/platform_aware_scheduling_install/tasks/main.yml b/roles/platform_aware_scheduling_install/tasks/main.yml new file mode 100644 index 00000000..4db7cf83 --- /dev/null +++ b/roles/platform_aware_scheduling_install/tasks/main.yml @@ -0,0 +1,80 @@ +## +## Copyright (c) 2020-2022 Intel Corporation. +## +## Licensed under the Apache License, Version 2.0 (the "License"); +## you may not use this file except in compliance with the License. +## You may obtain a copy of the License at +## +## http://www.apache.org/licenses/LICENSE-2.0 +## +## Unless required by applicable law or agreed to in writing, software +## distributed under the License is distributed on an "AS IS" BASIS, +## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +## See the License for the specific language governing permissions and +## limitations under the License. +## +--- +- name : install dependencies + include_role: + name: install_dependencies + when: + - inventory_hostname == groups['kube_control_plane'][0] + +- name: restore original kube-scheduler config files + include_tasks: backups_restore.yml + loop: "{{ kube_scheduler_configs }}" + when: inventory_hostname == groups['kube_control_plane'][0] + +- name: create Helm charts directory if needed + file: + path: "{{ (project_root_dir, 'charts') | path_join }}" + state: directory + mode: 0755 + when: + - inventory_hostname == groups['kube_control_plane'][0] + +- name: "create extenders ns if doesn't exist" + shell: "set -o pipefail && kubectl create ns {{ pas_namespace }} -o yaml --dry-run=client | kubectl apply -f -" + args: + executable: /bin/bash + changed_when: true + when: + - inventory_hostname == groups['kube_control_plane'][0] + +- name: build extenders images locally + include_tasks: build_image.yml + loop: "{{ extenders }}" + loop_control: + extended: yes + when: + - item.build + - inventory_hostname == groups['kube_control_plane'][0] + - tas_build_image_locally | default(false) or gas_build_image_locally | default(false) + +- name: create certificates + include_tasks: tls.yml + loop: "{{ extenders }}" + loop_control: + extended: yes + loop_var: extender + when: + - inventory_hostname == groups['kube_control_plane'][0] + - extender.tls_enabled + +- name: deploy TAS extender + include_tasks: tas.yml + when: + - inventory_hostname == groups['kube_control_plane'][0] + - tas_enabled | default(false) + +- name: deploy GAS extender + include_tasks: gas.yml + when: + - inventory_hostname == groups['kube_control_plane'][0] + - gas_enabled | default(false) + +- name: configure kube-scheduler + include_tasks: create-scheduler-config.yml + when: + - inventory_hostname == groups['kube_control_plane'][0] + diff --git a/roles/platform_aware_scheduling_install/tasks/tas.yml b/roles/platform_aware_scheduling_install/tasks/tas.yml new file mode 100644 index 00000000..0897b294 --- /dev/null +++ b/roles/platform_aware_scheduling_install/tasks/tas.yml @@ -0,0 +1,71 @@ +## +## Copyright (c) 2020-2022 Intel Corporation. +## +## Licensed under the Apache License, Version 2.0 (the "License"); +## you may not use this file except in compliance with the License. +## You may obtain a copy of the License at +## +## http://www.apache.org/licenses/LICENSE-2.0 +## +## Unless required by applicable law or agreed to in writing, software +## distributed under the License is distributed on an "AS IS" BASIS, +## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +## See the License for the specific language governing permissions and +## limitations under the License. +## +--- +- name: create Descheduler directory if needed + file: + path: "{{ sigs_k8s_io_dir }}" + state: directory + mode: 0755 + +- name: clone Descheduler for Kubernetes + git: + repo: "{{ descheduler_git_url }}" + dest: "{{ descheduler_dir }}" + force: yes + version: "{{ descheduler_git_version }}" + retries: 5 + delay: 5 + register: result + until: not result.failed + +- name: install descheduler + make: + chdir: "{{ descheduler_dir }}" + +- name: copy Helm chart resource definition to controller node + copy: + src: "{{ (role_path, 'charts/telemetry-aware-scheduling') | path_join }}" + dest: "{{ (project_root_dir, 'charts') | path_join }}" + mode: preserve + +- name: populate tas Helm chart values template and push to controller node + template: + src: "{{ item.src }}" + dest: "{{ item.dest }}" + force: yes + mode: preserve + loop: + - { + src: "tas-values.yaml.j2", + dest: "{{ (project_root_dir, 'charts/telemetry-aware-scheduling/values.yaml') | path_join }}" + } + - { + src: "tas-chart.yaml.j2", + dest: "{{ (project_root_dir, 'charts/telemetry-aware-scheduling/Chart.yaml') | path_join }}" + } + +- name: install TAS helm chart + command: >- + helm upgrade -i {{ tas_name }} + --namespace {{ pas_namespace }} + ./ + args: + chdir: "{{ (project_root_dir, 'charts/telemetry-aware-scheduling') | path_join }}" + retries: 5 + delay: 5 + register: result + until: result.rc == 0 + changed_when: true diff --git a/roles/platform_aware_scheduling_install/tasks/tls.yml b/roles/platform_aware_scheduling_install/tasks/tls.yml new file mode 100644 index 00000000..75703fc4 --- /dev/null +++ b/roles/platform_aware_scheduling_install/tasks/tls.yml @@ -0,0 +1,89 @@ +## +## Copyright (c) 2020-2022 Intel Corporation. +## +## Licensed under the Apache License, Version 2.0 (the "License"); +## you may not use this file except in compliance with the License. +## You may obtain a copy of the License at +## +## http://www.apache.org/licenses/LICENSE-2.0 +## +## Unless required by applicable law or agreed to in writing, software +## distributed under the License is distributed on an "AS IS" BASIS, +## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +## See the License for the specific language governing permissions and +## limitations under the License. +## +--- +- name: create tls folder + file: + path: "{{ extender.ssl }}" + state: directory + mode: u=rwX,g=,o= + owner: root + group: root + become: yes + +- name: populate cfssl tls templates + template: + src: "{{ item }}.j2" + dest: "{{ ('/etc/ssl', extender.name, item ) | path_join }}" + force: yes + mode: preserve + loop: + - tls-extender-ca.json + - tls-extender.json + - tls-extender-client.json + +- name: get GOPATH + command: /usr/local/go/bin/go env GOPATH + register: gopath + changed_when: false + +- name: generate CA root crt + shell: >- + set -o pipefail && {{ gopath.stdout }}/bin/cfssl gencert + -initca tls-extender-ca.json | {{ gopath.stdout }}/bin/cfssljson -bare ca - + args: + chdir: "{{ extender.ssl }}" + executable: /bin/bash + changed_when: true + +- name: generate certs + shell: >- + set -o pipefail && {{ gopath.stdout }}/bin/cfssl gencert + -ca=ca.pem -ca-key=ca-key.pem -profile={{ item.name }} {{ item.target }}.json | + {{ gopath.stdout }}/bin/cfssljson -bare {{ item.target }} + args: + chdir: "{{ extender.ssl }}" + executable: /bin/bash + loop: + - { name: server, target: "tls-extender" } + - { name: client, target: "tls-extender-client" } + changed_when: true + +- name: create secret {{ extender.secret_name }} + shell: >- + set -o pipefail && kubectl create secret generic {{ extender.secret_name }} + --namespace {{ pas_namespace }} + --from-file={{ extender.secret_name }}.key=tls-extender-key.pem + --from-file=ca.crt=ca.pem + --from-file={{ extender.secret_name }}.cert=tls-extender.pem + -o yaml --dry-run=client | kubectl apply -f - + args: + chdir: "{{ extender.ssl }}" + executable: /bin/bash + changed_when: true + +- name: remove unneeded files + file: + state: absent + path: "{{ (extender.ssl, item) | path_join }}" + loop: + - ca.csr + - tls-extender-key.pem + - tls-extender.pem + - tls-extender.csr + - tls-extender-client.csr + - tls-extender-ca.json + - tls-extender.json + - tls-extender-client.json diff --git a/roles/platform_aware_scheduling_install/templates/gas-chart.yaml.j2 b/roles/platform_aware_scheduling_install/templates/gas-chart.yaml.j2 new file mode 100644 index 00000000..3a923494 --- /dev/null +++ b/roles/platform_aware_scheduling_install/templates/gas-chart.yaml.j2 @@ -0,0 +1,7 @@ +apiVersion: v1 +description: Intel® GPU Aware Scheduling +name: gpu-aware-scheduling +version: "{{ gas_version }}" +sources: + - https://github.com/intel/platform-aware-scheduling.git +appVersion: "{{ gas_version }}" diff --git a/roles/platform_aware_scheduling_install/templates/gas-values.yaml.j2 b/roles/platform_aware_scheduling_install/templates/gas-values.yaml.j2 new file mode 100644 index 00000000..d6b46d5b --- /dev/null +++ b/roles/platform_aware_scheduling_install/templates/gas-values.yaml.j2 @@ -0,0 +1,30 @@ +# Default values for gpu-aware-scheduling. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +replicaCount: 1 + +extenderImage: + repository: {{ gas_extender_image }} + tag: {{ gas_extender_image_tag }} + pullPolicy: IfNotPresent + +nameOverride: "" +namespace: {{ pas_namespace | default("kube-system") }} +fullnameOverride: "" +clusterName: {{ cluster_name | default("cluster.local") }} + +serviceAccount: + name: {{ gas_service_account_name | default('') }} + +service: + type: {{ gas_service_type }} + port: {{ gas_service_port }} + name: {{ gas_service_name }} + +tls: + enabled: {{ gas_tls_enabled | default("false") }} + secretName: {{ gas_extender_secret_name | default("gas-extender-secret") }} + gas_ssl: {{ gas_ssl }} + +verbosity: {{ gas_verbosity }} diff --git a/roles/platform_aware_scheduling_install/templates/pas-cluster-role-configmapgetter.yaml.j2 b/roles/platform_aware_scheduling_install/templates/pas-cluster-role-configmapgetter.yaml.j2 new file mode 100644 index 00000000..16a548b7 --- /dev/null +++ b/roles/platform_aware_scheduling_install/templates/pas-cluster-role-configmapgetter.yaml.j2 @@ -0,0 +1,23 @@ +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: pas-configmapgetter +rules: +- apiGroups: [""] + resources: ["configmaps"] + verbs: ["get"] + +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: pas-configmapgetter +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: pas-configmapgetter +subjects: +- apiGroup: rbac.authorization.k8s.io + kind: User + name: system:kube-scheduler diff --git a/roles/platform_aware_scheduling_install/templates/pas-extenders-configmap.yaml.j2 b/roles/platform_aware_scheduling_install/templates/pas-extenders-configmap.yaml.j2 new file mode 100644 index 00000000..27fdea19 --- /dev/null +++ b/roles/platform_aware_scheduling_install/templates/pas-extenders-configmap.yaml.j2 @@ -0,0 +1,62 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: pas-scheduler-extenders-policy + namespace: "{{ pas_namespace }}" +data: + policy.cfg: | + { + "kind" : "Policy", + "apiVersion" : "v1", + "extenders" : [ + {% if tas_enabled | default(false) %} + { + "urlPrefix": "http{% if tas_tls_enabled %}s{% endif %}://{{ tas_service_name }}.{{ pas_namespace }}.svc.{{ cluster_name }}:{{ tas_service_port }}", + "apiVersion": "v1", + "prioritizeVerb": "scheduler/prioritize", + "filterVerb": "scheduler/filter", + "weight": 1, + "enableHttps": {% if gas_tls_enabled %}true{% else %}false{% endif %}, + "managedResources": [ + { + "name": "telemetry/scheduling", + "ignoredByScheduler": true + } + ], + "ignorable": true, + {% if tas_tls_enabled %} + "tlsConfig": { + "insecure": false, + "certFile": "/host/certs/tas/tls-extender-client.pem", + "keyFile" : "/host/certs/tas/tls-extender-client-key.pem" + } + {% endif %} + }{% if tas_enabled | default(false) and gas_enabled | default(false) %},{% endif %} + {% endif %} + {% if gas_enabled %} + { + "urlPrefix": "http{% if gas_tls_enabled %}s{% endif %}://{{ gas_service_name }}.{{ pas_namespace }}.svc.{{ cluster_name }}:{{ gas_service_port }}", + "apiVersion": "v1", + "filterVerb": "scheduler/filter", + "bindVerb": "scheduler/bind", + "weight": 1, + "enableHttps": {% if gas_tls_enabled %}true{% else %}false{% endif %}, + "managedResources": [ + { + "name": "gpu.intel.com/i915", + "ignoredByScheduler": false + } + ], + "ignorable": true, + {% if gas_tls_enabled %} + "tlsConfig": { + "insecure": false, + "certFile": "/host/certs/gas/tls-extender-client.pem", + "keyFile" : "/host/certs/gas/tls-extender-client-key.pem" + }, + {% endif %} + "nodeCacheCapable": true + } + {% endif %} + ] + } diff --git a/roles/platform_aware_scheduling_install/templates/tas-chart.yaml.j2 b/roles/platform_aware_scheduling_install/templates/tas-chart.yaml.j2 new file mode 100644 index 00000000..e43219db --- /dev/null +++ b/roles/platform_aware_scheduling_install/templates/tas-chart.yaml.j2 @@ -0,0 +1,7 @@ +apiVersion: v1 +description: Intel® Telemetry Aware Scheduling +name: telemetry-aware-scheduling +version: "{{ tas_version }}" +sources: + - https://github.com/intel/platform-aware-scheduling.git +appVersion: "{{ tas_version }}" diff --git a/roles/platform_aware_scheduling_install/templates/tas-values.yaml.j2 b/roles/platform_aware_scheduling_install/templates/tas-values.yaml.j2 new file mode 100644 index 00000000..18d76d01 --- /dev/null +++ b/roles/platform_aware_scheduling_install/templates/tas-values.yaml.j2 @@ -0,0 +1,32 @@ +# Default values for telemetry-aware-scheduling. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +replicaCount: 1 + +extenderImage: + repository: {{ tas_extender_image }} + tag: {{ tas_extender_image_tag }} + pullPolicy: IfNotPresent + +nameOverride: "" +namespace: {{ pas_namespace | default("kube-system") }} +fullnameOverride: "" +clusterName: {{ cluster_name | default("cluster.local") }} + +serviceAccount: + name: {{ tas_service_account_name | default('') }} + +service: + type: {{ tas_service_type }} + port: {{ tas_service_port }} + name: {{ tas_service_name }} + +tls: + enabled: {{ tas_tls_enabled | default("false") }} + secretName: {{ tas_extender_secret_name | default("tas-extender-secret") }} + tas_ssl: {{ tas_ssl }} + +demoPolicy: {{ tas_enable_demo_policy }} + +verbosity: {{ tas_verbosity }} diff --git a/roles/platform_aware_scheduling_install/templates/tls-extender-ca.json.j2 b/roles/platform_aware_scheduling_install/templates/tls-extender-ca.json.j2 new file mode 100644 index 00000000..3b8cd1f9 --- /dev/null +++ b/roles/platform_aware_scheduling_install/templates/tls-extender-ca.json.j2 @@ -0,0 +1,11 @@ +{ + "CN": "localhost", + "key": { + "algo": "rsa", + "size": 2048 + }, + "names": [{ + "O": "{{ extender.name }}", + "OU": "root" + }] +} diff --git a/roles/platform_aware_scheduling_install/templates/tls-extender-client.json.j2 b/roles/platform_aware_scheduling_install/templates/tls-extender-client.json.j2 new file mode 100644 index 00000000..b602e6fa --- /dev/null +++ b/roles/platform_aware_scheduling_install/templates/tls-extender-client.json.j2 @@ -0,0 +1,8 @@ +{ + "CN": "{{ extender.name }}-extender-client", + "hosts": [""], + "key": { + "algo": "rsa", + "size": 2048 + } +} diff --git a/roles/platform_aware_scheduling_install/templates/tls-extender.json.j2 b/roles/platform_aware_scheduling_install/templates/tls-extender.json.j2 new file mode 100644 index 00000000..a68708bf --- /dev/null +++ b/roles/platform_aware_scheduling_install/templates/tls-extender.json.j2 @@ -0,0 +1,17 @@ + +{ + "hosts": [ + "{{ extender.service_name }}.{{ pas_namespace }}.svc.{{ cluster_name | default("cluster.local") }}", + "{{ extender.service_name }}.{{ pas_namespace }}.svc", + "{{ extender.service_name }}.{{ pas_namespace }}", + "{{ extender.service_name }}" + ], + "CN": "{{ extender.service_name }}.{{ pas_namespace }}.svc", + "key": { + "algo": "rsa", + "size": 2048 + }, + "names": [{ + "O": "{{ extender.name }}-extender" + }] +} diff --git a/roles/platform_aware_scheduling_install/vars/main.yml b/roles/platform_aware_scheduling_install/vars/main.yml new file mode 100644 index 00000000..18e5098b --- /dev/null +++ b/roles/platform_aware_scheduling_install/vars/main.yml @@ -0,0 +1,52 @@ +## +## Copyright (c) 2020-2022 Intel Corporation. +## +## Licensed under the Apache License, Version 2.0 (the "License"); +## you may not use this file except in compliance with the License. +## You may obtain a copy of the License at +## +## http://www.apache.org/licenses/LICENSE-2.0 +## +## Unless required by applicable law or agreed to in writing, software +## distributed under the License is distributed on an "AS IS" BASIS, +## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +## See the License for the specific language governing permissions and +## limitations under the License. +## +--- +# variables in this file are not intended to be modified by the user directly +# please use defaults/main.yml instead +extenders: + - { + name: "{{ tas_name }}", + dir: "{{ tas_extender_dir }}", + image: "{{ tas_extender_image }}", + tag: "{{ tas_extender_image_tag }}", + context: "{{ tas_image_build_context }}", + build: "{{ tas_build_image_locally }}", + bin_build: "{{ tas_build_bin }}", + secret_name: "{{ tas_extender_secret_name }}", + service_name: "{{ tas_service_name }}", + tls_enabled: "{{ tas_tls_enabled }}", + ssl: "{{ tas_ssl }}", + version: "{{ tas_git_version }}" + } + - { + name: "{{ gas_name }}", + dir: "{{ gas_extender_dir }}", + image: "{{ gas_extender_image }}", + tag: "{{ gas_extender_image_tag }}", + context: "{{ gas_image_build_context }}", + build: "{{ gas_build_image_locally }}", + bin_build: "{{ gas_build_bin }}", + secret_name: "{{ gas_extender_secret_name }}", + service_name: "{{ gas_service_name }}", + tls_enabled: "{{ gas_tls_enabled }}", + ssl: "{{ gas_ssl }}", + version: "{{ gas_git_version }}" + } + +# kube-scheduler config files to backup +kube_scheduler_configs: + - "/etc/kubernetes/kubescheduler-config.yaml" + - "/etc/kubernetes/manifests/kube-scheduler.yaml" diff --git a/roles/qat_dp_install/charts/intel-qat-plugin/templates/NOTES.txt b/roles/qat_dp_install/charts/intel-qat-plugin/templates/NOTES.txt deleted file mode 100644 index 29863d3d..00000000 --- a/roles/qat_dp_install/charts/intel-qat-plugin/templates/NOTES.txt +++ /dev/null @@ -1,8 +0,0 @@ -Thank you for installing {{ .Chart.Name }}. - -Your release is named {{ .Release.Name }}. - -To learn more about the release, try: - - $ helm status {{ .Release.Name }} - $ helm get {{ .Release.Name }} diff --git a/roles/qat_dp_install/charts/intel-qat-plugin/templates/qat_plugin.yaml b/roles/qat_dp_install/charts/intel-qat-plugin/templates/qat_plugin.yaml deleted file mode 100644 index 45d29eb9..00000000 --- a/roles/qat_dp_install/charts/intel-qat-plugin/templates/qat_plugin.yaml +++ /dev/null @@ -1,65 +0,0 @@ -apiVersion: apps/v1 -kind: DaemonSet -metadata: - name: {{ .Release.Name }}-intel-qat-plugin - labels: - app: {{ .Release.Name }}-intel-qat-plugin -spec: - selector: - matchLabels: - app: {{ .Release.Name }}-intel-qat-plugin - template: - metadata: - labels: - app: {{ .Release.Name }}-intel-qat-plugin -{{ if .Values.apparmor.enabled }} - annotations: - container.apparmor.security.beta.kubernetes.io/{{ .Release.Name }}-intel-qat-plugin: {{ .Values.apparmor.profile }} -{{ end }} - spec: - tolerations: - - key: cmk - operator: Equal - effect: NoSchedule - value: "true" - containers: - - name: {{ .Release.Name }}-intel-qat-plugin - image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}" - securityContext: - readOnlyRootFilesystem: true - env: - - name: DPDK_DRIVER - valueFrom: - configMapKeyRef: - name: {{ .Release.Name }}-intel-qat-plugin-config - key: DPDK_DRIVER - - name: KERNEL_VF_DRIVERS - valueFrom: - configMapKeyRef: - name: {{ .Release.Name }}-intel-qat-plugin-config - key: KERNEL_VF_DRIVERS - - name: MAX_NUM_DEVICES - valueFrom: - configMapKeyRef: - name: {{ .Release.Name }}-intel-qat-plugin-config - key: MAX_NUM_DEVICES - imagePullPolicy: {{ .Values.image.pullPolicy }} - command: ["/usr/local/bin/intel_qat_device_plugin", "-dpdk-driver", "$(DPDK_DRIVER)", "-kernel-vf-drivers", "$(KERNEL_VF_DRIVERS)", "-max-num-devices", "$(MAX_NUM_DEVICES)"] - volumeMounts: - - name: devdir - mountPath: /dev/vfio - readOnly: true - - name: pcidir - mountPath: /sys/bus/pci - - name: kubeletsockets - mountPath: /var/lib/kubelet/device-plugins - volumes: - - name: devdir - hostPath: - path: /dev/vfio - - name: pcidir - hostPath: - path: /sys/bus/pci - - name: kubeletsockets - hostPath: - path: /var/lib/kubelet/device-plugins diff --git a/roles/qat_dp_install/charts/intel-qat-plugin/templates/qat_plugin_configmap.yaml b/roles/qat_dp_install/charts/intel-qat-plugin/templates/qat_plugin_configmap.yaml deleted file mode 100644 index 0dd2a171..00000000 --- a/roles/qat_dp_install/charts/intel-qat-plugin/templates/qat_plugin_configmap.yaml +++ /dev/null @@ -1,9 +0,0 @@ ---- -apiVersion: v1 -kind: ConfigMap -metadata: - name: {{ .Release.Name }}-intel-qat-plugin-config -data: - DPDK_DRIVER: "{{ .Values.drivers.dpdk }}" - KERNEL_VF_DRIVERS: "{{ .Values.drivers.kernel }}" - MAX_NUM_DEVICES: "{{ .Values.maxNumDevices }}" \ No newline at end of file diff --git a/roles/qat_dp_install/defaults/main.yml b/roles/qat_dp_install/defaults/main.yml index 0c512d7f..d7e82519 100644 --- a/roles/qat_dp_install/defaults/main.yml +++ b/roles/qat_dp_install/defaults/main.yml @@ -1,5 +1,5 @@ ## -## Copyright (c) 2020-2021 Intel Corporation. +## Copyright (c) 2020-2022 Intel Corporation. ## ## Licensed under the Apache License, Version 2.0 (the "License"); ## you may not use this file except in compliance with the License. @@ -15,13 +15,18 @@ ## --- intel_qat_dp_git_url: "https://github.com/intel/intel-device-plugins-for-kubernetes.git" -intel_qat_dp_git_ref: "v0.21.0" -intel_qat_dp_version: "0.21.0" -intel_qat_dp_dir: "/usr/src/intel-device-plugins-for-kubernetes-qat" +intel_qat_dp_git_ref: "v0.23.0" +intel_qat_dp_version: "0.23.0" +intel_qat_dp_dir: "{{ (project_root_dir, 'intel-qat-dp') | path_join }}" -qat_dp_build_image_locally: true qat_dp_namespace: "kube-system" qat_dp_dpdk_drivers: "vfio-pci" -qat_dp_kernel_drivers: "c6xxvf" +qat_dp_kernel_drivers: + - dh895xccvf + - c4xxxvf + - c3xxxvf + - 4xxxvf + - c6xxvf + - d15xxvf qat_dp_apparmor: "true" qat_dp_apparmor_profile: "unconfined" diff --git a/roles/qat_dp_install/files/noiommu.patch b/roles/qat_dp_install/files/noiommu.patch new file mode 100644 index 00000000..03a0da4a --- /dev/null +++ b/roles/qat_dp_install/files/noiommu.patch @@ -0,0 +1,78 @@ +diff --git a/cmd/qat_plugin/dpdkdrv/dpdkdrv.go b/cmd/qat_plugin/dpdkdrv/dpdkdrv.go +index 8a33e78..87da7e8 100644 +--- a/cmd/qat_plugin/dpdkdrv/dpdkdrv.go ++++ b/cmd/qat_plugin/dpdkdrv/dpdkdrv.go +@@ -169,6 +169,15 @@ func (dp *DevicePlugin) getDpdkDevice(vfBdf string) (string, error) { + + s := filepath.Base(group) + ++ // If the kernel has CONFIG_VFIO_NOIOMMU enabled and the node admin ++ // has explicitly set enable_unsafe_noiommu_mode VFIO parameter, ++ // VFIO taints the kernel and writes "vfio-noiommu" to the IOMMU ++ // group name. If these conditions are true, the /dev/vfio/ devices ++ // are prefixed with "noiommu-". ++ if isVfioNoIOMMU(vfioDirPath) { ++ s = fmt.Sprintf("noiommu-%s", s) ++ } ++ + return s, nil + + default: +@@ -176,6 +185,16 @@ func (dp *DevicePlugin) getDpdkDevice(vfBdf string) (string, error) { + } + } + ++func isVfioNoIOMMU(iommuGroupPath string) bool { ++ if fileData, err := os.ReadFile(filepath.Join(iommuGroupPath, "name")); err == nil { ++ if strings.TrimSpace(string(fileData)) == "vfio-noiommu" { ++ return true ++ } ++ } ++ ++ return false ++} ++ + func (dp *DevicePlugin) getDpdkDeviceSpecs(dpdkDeviceName string) []pluginapi.DeviceSpec { + switch dp.dpdkDriver { + case igbUio: +diff --git a/cmd/qat_plugin/dpdkdrv/dpdkdrv_test.go b/cmd/qat_plugin/dpdkdrv/dpdkdrv_test.go +index c55a4e9..a231beb 100644 +--- a/cmd/qat_plugin/dpdkdrv/dpdkdrv_test.go ++++ b/cmd/qat_plugin/dpdkdrv/dpdkdrv_test.go +@@ -348,7 +348,7 @@ func TestScan(t *testing.T) { + expectedErr: true, + }, + { +- name: "vfio-pci DPDKdriver with one kernel bound device (QAT device) where vfdevID is equal to qatDevId (37c9), running in a VM", ++ name: "vfio-pci DPDKdriver with one kernel bound device (QAT device) where vfdevID is equal to qatDevId (37c9), running in a VM with vIOMMU", + dpdkDriver: "vfio-pci", + kernelVfDrivers: []string{"c6xxvf"}, + dirs: []string{ +@@ -366,6 +366,27 @@ func TestScan(t *testing.T) { + maxDevNum: 1, + expectedDevNum: 1, + }, ++ { ++ name: "vfio-pci DPDKdriver in unsafe NOIOMMU mode with one kernel bound device (QAT device) where vfdevID is equal to qatDevId (37c9), running in a VM without IOMMU", ++ dpdkDriver: "vfio-pci", ++ kernelVfDrivers: []string{"c6xxvf"}, ++ dirs: []string{ ++ "sys/bus/pci/drivers/c6xxvf", ++ "sys/bus/pci/drivers/vfio-pci", ++ "sys/bus/pci/devices/0000:02:01.0", ++ "sys/kernel/iommu_groups/vfiotestfile", ++ }, ++ files: map[string][]byte{ ++ "sys/bus/pci/devices/0000:02:01.0/device": []byte("0x37c9"), ++ "sys/kernel/iommu_groups/vfiotestfile/name": []byte("vfio-noiommu"), ++ }, ++ symlinks: map[string]string{ ++ "sys/bus/pci/devices/0000:02:01.0/iommu_group": "sys/kernel/iommu_groups/vfiotestfile", ++ "sys/bus/pci/devices/0000:02:01.0/driver": "sys/bus/pci/drivers/c6xxvf", ++ }, ++ maxDevNum: 1, ++ expectedDevNum: 1, ++ }, + } + for _, tt := range tcases { + t.Run(tt.name, func(t *testing.T) { diff --git a/roles/qat_dp_install/tasks/main.yml b/roles/qat_dp_install/tasks/main.yml index 534ddeb8..6afd5a2a 100644 --- a/roles/qat_dp_install/tasks/main.yml +++ b/roles/qat_dp_install/tasks/main.yml @@ -1,5 +1,5 @@ ## -## Copyright (c) 2020-2021 Intel Corporation. +## Copyright (c) 2020-2022 Intel Corporation. ## ## Licensed under the Apache License, Version 2.0 (the "License"); ## you may not use this file except in compliance with the License. @@ -18,7 +18,7 @@ include_role: name: install_dependencies -- name: clone Intel Device Plugins repository +- name: clone Intel QAT Device Plugins repository git: repo: "{{ intel_qat_dp_git_url }}" version: "{{ intel_qat_dp_git_ref }}" @@ -28,143 +28,84 @@ - inventory_hostname == groups['kube_node'][0] - qat_dp_build_image_locally -- name: create Helm charts directory if needed - file: - path: /usr/src/charts - state: directory - mode: 0755 - when: - - inventory_hostname == groups['kube_control_plane'][0] - -- name: Build QAT Plugin before Intel QAT Device Plugin image - make: - target: qat_plugin - chdir: "{{ intel_qat_dp_dir }}" +# Once the patch is released in official intel_qat_dp version +# remove this task and disable local build in group_vars.j2 via +# qat_dp_build_image_locally: false +# Correction expected in intel_qat_dp version 0.24.0 +- name: apply qat_plugin patch + patch: + src: "{{ item }}" + basedir: "{{ intel_qat_dp_dir }}" + strip: 1 + state: present + with_items: + - "noiommu.patch" when: - inventory_hostname == groups['kube_node'][0] - qat_dp_build_image_locally - - container_runtime == "docker" + - on_vms is defined and on_vms -- name: update dynamic linker cache after Build QAT Plugin - command: "ldconfig" - changed_when: true +- name: prepare containers images + block: + - name: Build QAT Plugin before Intel QAT Device Plugin image + make: + target: qat_plugin + chdir: "{{ intel_qat_dp_dir }}" -- name: build Intel QAT Device Plugin image - make: - target: intel-qat-plugin - chdir: "{{ intel_qat_dp_dir }}" - when: - - inventory_hostname == groups['kube_node'][0] - - qat_dp_build_image_locally - - container_runtime == "docker" + - name: build Intel QAT Device Plugin image + make: + target: intel-qat-plugin + chdir: "{{ intel_qat_dp_dir }}" -- name: tag Intel QAT Device Plugin image - command: docker tag intel/intel-qat-plugin:{{ intel_qat_dp_version }} {{ registry_local_address }}/intel-qat-plugin:{{ intel_qat_dp_version }} - when: - - inventory_hostname == groups['kube_node'][0] - - qat_dp_build_image_locally - - container_runtime == "docker" + - name: tag Intel QAT Device Plugin image + command: docker tag intel/intel-qat-plugin:{{ intel_qat_dp_version }} {{ registry_local_address }}/intel-qat-plugin:{{ intel_qat_dp_version }} -- name: push Intel QAT Device Plugin image to local registry - command: docker push {{ registry_local_address }}/intel-qat-plugin:{{ intel_qat_dp_version }} + - name: push Intel QAT Device Plugin image to local registry + command: docker push {{ registry_local_address }}/intel-qat-plugin:{{ intel_qat_dp_version }} when: - inventory_hostname == groups['kube_node'][0] - qat_dp_build_image_locally - container_runtime == "docker" -- name: build and tag Intel QAT Device Plugin image - command: buildah bud -f build/docker/intel-qat-plugin.Dockerfile -t {{ registry_local_address }}/intel-qat-plugin:{{ intel_qat_dp_version }} - args: - chdir: "{{ intel_qat_dp_dir }}" - changed_when: true - with_items: - - { file: intel-qat-plugin.Dockerfile, name: intel-qat-plugin } - when: - - inventory_hostname == groups['kube_node'][0] - - qat_dp_build_image_locally - - '"docker" not in container_runtime' +- name: prepare containers images + block: + - name: build and tag Intel QAT Device Plugin image + command: buildah bud -f build/docker/intel-qat-plugin.Dockerfile -t {{ registry_local_address }}/intel-qat-plugin:{{ intel_qat_dp_version }} + args: + chdir: "{{ intel_qat_dp_dir }}" + changed_when: true + with_items: + - { file: intel-qat-plugin.Dockerfile, name: intel-qat-plugin } -- name: push Intel QAT Device Plugin image to local registry - command: buildah push {{ registry_local_address }}/intel-qat-plugin:{{ intel_qat_dp_version }} - changed_when: true + - name: push Intel QAT Device Plugin image to local registry + command: buildah push {{ registry_local_address }}/intel-qat-plugin:{{ intel_qat_dp_version }} + changed_when: true when: - inventory_hostname == groups['kube_node'][0] - qat_dp_build_image_locally - '"docker" not in container_runtime' -- name: copy Helm chart to the controller node - copy: - src: "{{ role_path }}/charts/intel-qat-plugin" - dest: "/usr/src/charts/" - mode: 0755 - when: - - inventory_hostname == groups['kube_control_plane'][0] - -- name: set values for Helm charts - set_fact: - qat_dp_image: "{{ registry_local_address }}/intel-qat-plugin" - qat_dp_kernel_drivers: "{{ hostvars[groups['kube_node'][0]].qat_devices[0].qat_pci_type | default('c6xx') }}vf" - when: - - inventory_hostname == groups['kube_control_plane'][0] - - qat_dp_build_image_locally - -- name: check if calculated VFs file exists - stat: - path: "/etc/bmra/bmra_sriov_qat_total_vfs" - register: total_vfs_folder - when: inventory_hostname == groups['kube_node'][0] - -- name: get calculated Vfs value form file - command: "cat /etc/bmra/bmra_sriov_qat_total_vfs" - register: calulated_vfs - when: - - inventory_hostname == groups['kube_node'][0] - - total_vfs_folder.stat.exists - changed_when: true - -- name: set calculated Vfs value from file - set_fact: - qat_calculated_vfs: "{{ calulated_vfs.stdout }}" - when: - - inventory_hostname == groups['kube_node'][0] - - total_vfs_folder.stat.exists - -- name: update calculated VFs before Helm chart values template push to controller node - lineinfile: - dest: "{{ role_path }}/templates/qat_values.yml.j2" - regexp: 'maxNumDevices:' - line: 'maxNumDevices: {{ qat_calculated_vfs }}' - mode: 0600 - delegate_to: localhost - become: false - when: - - inventory_hostname == groups['kube_node'][0] - - total_vfs_folder.stat.exists - changed_when: true - -- name: populate Intel QAT Plugin Helm chart values template and push to controller node - template: - src: "qat_values.yml.j2" - dest: "/usr/src/charts/intel-qat-plugin-values.yml" - force: yes - mode: preserve - when: - - inventory_hostname == groups['kube_control_plane'][0] - -- name: install Intel QAT Plugin Helm chart - command: helm upgrade -i intel-qat-plugin -f /usr/src/charts/intel-qat-plugin-values.yml --namespace {{ qat_dp_namespace }} /usr/src/charts/intel-qat-plugin - when: - - inventory_hostname == groups['kube_control_plane'][0] - -- name: revert template VFs values once Intel QAT Plugin Helm chart installation is finished - lineinfile: - dest: "{{ role_path }}/templates/qat_values.yml.j2" - regexp: 'maxNumDevices:' - line: 'maxNumDevices: 00' - mode: 0600 - delegate_to: localhost - become: false - when: - - inventory_hostname == groups['kube_node'][0] - - total_vfs_folder.stat.exists - changed_when: true +- name: prepare and deploy Intel QAT Device Plugin + block: + - name: set value for QAT DP image + set_fact: + qat_dp_image: "{{ registry_local_address }}/intel-qat-plugin" + when: + - inventory_hostname == groups['kube_control_plane'][0] + - qat_dp_build_image_locally + + - name: populate Intel QAT Device Plugin yaml file and push to controller node + template: + src: "intel-qat-plugin.yml.j2" + dest: "{{ project_root_dir }}/intel-qat-plugin.yml" + force: yes + mode: preserve + when: + - inventory_hostname == groups['kube_control_plane'][0] + + - name: deploy Intel QAT Device Plugin with the Intel Device Plugin Operator + k8s: + state: present + src: "{{ project_root_dir }}/intel-qat-plugin.yml" + when: + - inventory_hostname == groups['kube_control_plane'][0] diff --git a/roles/qat_dp_install/templates/intel-qat-plugin.yml.j2 b/roles/qat_dp_install/templates/intel-qat-plugin.yml.j2 new file mode 100644 index 00000000..8019026b --- /dev/null +++ b/roles/qat_dp_install/templates/intel-qat-plugin.yml.j2 @@ -0,0 +1,21 @@ +--- +apiVersion: deviceplugin.intel.com/v1 +kind: QatDevicePlugin +metadata: + name: intel-qat-device-plugin + # example apparmor annotation + # see more details here: + # - https://kubernetes.io/docs/tutorials/clusters/apparmor/#securing-a-pod + # - https://github.com/intel/intel-device-plugins-for-kubernetes/issues/381 +{% if ansible_distribution == "Ubuntu" and qat_dp_apparmor | default("true") %} + annotations: + container.apparmor.security.beta.kubernetes.io/intel-qat-plugin: {{ qat_dp_apparmor_profile | default("unconfined") }} +{% endif %} +spec: + image: {{ qat_dp_image | default("docker.io/intel/intel-qat-plugin") }}:{{ intel_qat_dp_version | default("0.23.0") }} + dpdkDriver: {{ qat_dp_dpdk_drivers }} + kernelVfDrivers: {{ qat_dp_kernel_drivers }} + maxNumDevices: {{ qat_dp_max_num_devices }} + logLevel: {{ qat_dp_verbosity | default(4) }} + nodeSelector: + feature.node.kubernetes.io/intel.qat: 'true' diff --git a/roles/qat_dp_install/templates/qat_values.yml.j2 b/roles/qat_dp_install/templates/qat_values.yml.j2 deleted file mode 100644 index c7702b56..00000000 --- a/roles/qat_dp_install/templates/qat_values.yml.j2 +++ /dev/null @@ -1,15 +0,0 @@ ---- -image: - repository: {{ qat_dp_image | default("docker.io/intel/intel-qat-plugin") }} - tag: {{ intel_qat_dp_version | default("0.21.0") }} - pullPolicy: IfNotPresent - -drivers: - dpdk: {{ qat_dp_dpdk_drivers }} - kernel: {{ qat_dp_kernel_drivers }} - -apparmor: - enabled: {{ qat_dp_apparmor | default("true") }} - profile: {{ qat_dp_apparmor_profile | default("unconfined") }} - -maxNumDevices: 00 diff --git a/roles/qat_dp_install/vars/main.yml b/roles/qat_dp_install/vars/main.yml index 9f60069c..170d32e5 100644 --- a/roles/qat_dp_install/vars/main.yml +++ b/roles/qat_dp_install/vars/main.yml @@ -1,5 +1,5 @@ ## -## Copyright (c) 2020-2021 Intel Corporation. +## Copyright (c) 2020-2022 Intel Corporation. ## ## Licensed under the Apache License, Version 2.0 (the "License"); ## you may not use this file except in compliance with the License. diff --git a/roles/redeploy_cleanup/defaults/main.yml b/roles/redeploy_cleanup/defaults/main.yml index 9f46bd68..d6aff17f 100644 --- a/roles/redeploy_cleanup/defaults/main.yml +++ b/roles/redeploy_cleanup/defaults/main.yml @@ -1,5 +1,5 @@ ## -## Copyright (c) 2020-2021 Intel Corporation. +## Copyright (c) 2020-2022 Intel Corporation. ## ## Licensed under the Apache License, Version 2.0 (the "License"); ## you may not use this file except in compliance with the License. @@ -13,6 +13,7 @@ ## See the License for the specific language governing permissions and ## limitations under the License. ## +--- keep_docker_images: false reboot_after_cleanup: true @@ -26,12 +27,13 @@ k8s_dirs_to_remove: - "$HOME/.kube/" intel_dirs_to_remove: - - "/etc/bmra" + - "/etc/ssl/tas" + - "/etc/ssl/gas" + - "/etc/cek" - "/opt/bin" - "/opt/cni" - "/opt/collect.d" - "/etc/ssl/etcd/" - - "/etc/ssl/cmk/" - "/lib/firmware/intel/i40e" - "/lib/firmware/updates/intel" - "/lib/firmware/i915" @@ -39,8 +41,8 @@ intel_dirs_to_remove: intel_services_to_stop: - "aesmd.service" - - "bmra_sriov_nic_init.service" - - "bmra_sriov_qat_init.service" + - "cek_sriov_nic_init.service" + - "cek_sriov_qat_init.service" - "ddp-ice.service" - "QAT.service" - "sst-cp-configure.service" diff --git a/roles/redeploy_cleanup/tasks/intel_cleanup.yml b/roles/redeploy_cleanup/tasks/intel_cleanup.yml index 5769fe70..e68dcc0d 100644 --- a/roles/redeploy_cleanup/tasks/intel_cleanup.yml +++ b/roles/redeploy_cleanup/tasks/intel_cleanup.yml @@ -1,5 +1,5 @@ ## -## Copyright (c) 2020-2021 Intel Corporation. +## Copyright (c) 2020-2022 Intel Corporation. ## ## Licensed under the Apache License, Version 2.0 (the "License"); ## you may not use this file except in compliance with the License. @@ -15,7 +15,7 @@ ## --- - debug: - msg: "Starting to remove Intel BMRA Container Experience Kit features ..." + msg: "Starting to remove Intel Container Experience Kit features ..." - name: remove OVS vswitchd block: @@ -42,6 +42,7 @@ register: ddp_i40e_service changed_when: false failed_when: false + - name: stop services service: name: "{{ item }}" @@ -56,7 +57,7 @@ block: - name: remove custom kernel build files make: - dest: "/usr/src/kernels/gpu_kernel/" + dest: "{{ project_root_dir }}/kernels/gpu_kernel/" target: "{{ item }}" with_items: - clean @@ -113,7 +114,7 @@ changed_when: true failed_when: false become: yes - when: ansible_distribution == "CentOS" + when: ansible_os_family == "RedHat" - include_role: name: bootstrap/update_grub @@ -141,17 +142,10 @@ changed_when: false failed_when: false -- name: cleanup /usr/src/* - # ansible find module too slow - # noqa 305 - shell is used intentionally here - shell: - cmd: | - find /usr/src/ \ - -not -wholename "/usr/src/" \ - -and -not -path "/usr/src/linux-headers-*" \ - -and -not -path "/usr/src/kernels*" \ - -and -not -path "/usr/src/debug*" \ - -delete +- name: cleanup "{{ project_root_dir }}" + file: + path: "{{ project_root_dir }}" + state: absent changed_when: false failed_when: false @@ -162,5 +156,8 @@ changed_when: false failed_when: false +- name: remove MinIO + include_tasks: remove_minio.yml + - debug: - msg: "Intel BMRA Container Experience Kit features have been removed ..." \ No newline at end of file + msg: "Intel Container Experience Kit features have been removed ..." diff --git a/roles/redeploy_cleanup/tasks/k8s_cleanup.yml b/roles/redeploy_cleanup/tasks/k8s_cleanup.yml index c8d3404e..6d7acfd2 100644 --- a/roles/redeploy_cleanup/tasks/k8s_cleanup.yml +++ b/roles/redeploy_cleanup/tasks/k8s_cleanup.yml @@ -1,5 +1,5 @@ ## -## Copyright (c) 2020-2021 Intel Corporation. +## Copyright (c) 2020-2022 Intel Corporation. ## ## Licensed under the Apache License, Version 2.0 (the "License"); ## you may not use this file except in compliance with the License. @@ -82,12 +82,31 @@ register: existing_containers changed_when: false failed_when: false + when: + - container_runtime == "docker" - name: remove running or stopped containers command: "docker rm -f {{ item }}" with_items: - "{{ existing_containers.stdout | default([]) }}" when: - existing_containers.stdout | length > 0 + - container_runtime == "docker" + changed_when: false + failed_when: false + - name: list running or stopped containers + command: crictl ps -a -q + register: crictl_existing_containers + changed_when: false + failed_when: false + when: + - container_runtime != "docker" + - name: remove running or stopped containers + command: "crictl rm -f {{ item }}" + with_items: + - "{{ crictl_existing_containers.stdout | default([]) }}" + when: + - crictl_existing_containers.stdout | length > 0 + - container_runtime != "docker" changed_when: false failed_when: false @@ -98,12 +117,31 @@ register: container_images changed_when: false failed_when: false + when: + - container_runtime == "docker" - name: remove container images command: "docker rmi -f {{ item }}" with_items: - "{{ container_images.stdout | default([]) }}" when: - container_images.stdout | length > 0 + - container_runtime == "docker" + changed_when: false + failed_when: false + - name: list existing containers images + command: crictl images -q + register: crictl_container_images + changed_when: false + failed_when: false + when: + - container_runtime != "docker" + - name: remove container images + command: "crictl rmi {{ item }}" + with_items: + - "{{ crictl_container_images.stdout | default([]) }}" + when: + - container_runtime != "docker" + - crictl_container_images.stdout | length > 0 changed_when: false failed_when: false when: @@ -141,6 +179,23 @@ state: restarted changed_when: false failed_when: false + when: container_runtime == "docker" + +- name: restart containerd service + service: + name: containerd + state: restarted + changed_when: false + failed_when: false + when: container_runtime == "containerd" + +- name: restart crio service + service: + name: crio + state: restarted + changed_when: false + failed_when: false + when: container_runtime == "crio" - debug: msg: "k8s cluster has been removed ..." diff --git a/roles/redeploy_cleanup/tasks/main.yml b/roles/redeploy_cleanup/tasks/main.yml index ad1d4533..1a6837b5 100644 --- a/roles/redeploy_cleanup/tasks/main.yml +++ b/roles/redeploy_cleanup/tasks/main.yml @@ -1,5 +1,5 @@ ## -## Copyright (c) 2020-2021 Intel Corporation. +## Copyright (c) 2020-2022 Intel Corporation. ## ## Licensed under the Apache License, Version 2.0 (the "License"); ## you may not use this file except in compliance with the License. @@ -17,7 +17,7 @@ - name: reset and remove Kubernetes cluster import_tasks: k8s_cleanup.yml -- name: remove Intel BMRA Container Experience Kit features artifacts +- name: remove Intel Container Experience Kit features artifacts import_tasks: intel_cleanup.yml - name: reboot diff --git a/roles/redeploy_cleanup/tasks/remove_files.yml b/roles/redeploy_cleanup/tasks/remove_files.yml index 6d3d75cd..0b0f3c2e 100644 --- a/roles/redeploy_cleanup/tasks/remove_files.yml +++ b/roles/redeploy_cleanup/tasks/remove_files.yml @@ -1,5 +1,5 @@ ## -## Copyright (c) 2020-2021 Intel Corporation. +## Copyright (c) 2020-2022 Intel Corporation. ## ## Licensed under the Apache License, Version 2.0 (the "License"); ## you may not use this file except in compliance with the License. @@ -20,4 +20,4 @@ state: absent with_items: "{{ files_to_delete }}" changed_when: false - failed_when: false \ No newline at end of file + failed_when: false diff --git a/roles/redeploy_cleanup/tasks/remove_minio.yml b/roles/redeploy_cleanup/tasks/remove_minio.yml new file mode 100644 index 00000000..e3f84027 --- /dev/null +++ b/roles/redeploy_cleanup/tasks/remove_minio.yml @@ -0,0 +1,85 @@ +## +## Copyright (c) 2020-2022 Intel Corporation. +## +## Licensed under the Apache License, Version 2.0 (the "License"); +## you may not use this file except in compliance with the License. +## You may obtain a copy of the License at +## +## http://www.apache.org/licenses/LICENSE-2.0 +## +## Unless required by applicable law or agreed to in writing, software +## distributed under the License is distributed on an "AS IS" BASIS, +## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +## See the License for the specific language governing permissions and +## limitations under the License. +## +--- +- name: uninstall MinIO + block: + - name: umount volumes + command: >- + umount --lazy "{{ item.mountPath }}" + loop: "{{ hostvars[inventory_hostname]['minio_pv'] }}" + when: + - hostvars[inventory_hostname]['minio_pv'] is defined and hostvars[inventory_hostname]['minio_pv']|length > 0 + changed_when: false + failed_when: false + + - name: remove paritions for nvme deploy + parted: + device: "{{ item.device }}" + number: 1 + state: absent + loop: "{{ hostvars[inventory_hostname]['minio_pv'] }}" + when: + - hostvars[inventory_hostname]['minio_pv'] is defined and hostvars[inventory_hostname]['minio_pv']|length > 0 + changed_when: false + failed_when: false + + - name: list loop devices + shell: >- + set -o pipefail && losetup -l |grep diskimage |awk -F " " '{ print $6,$1}' |sort |awk -F " " '{ print $2}' + args: + executable: /bin/bash + register: loopdevice_output + changed_when: false + failed_when: false + + - name: remove loop devices for test mode deploy (block device) + command: >- + losetup -d "{{ item }}" + loop: "{{ loopdevice_output.stdout_lines }}" + changed_when: false + failed_when: false + + - name: remove mount points + file: + path: "{{ item.mountPath }}" + state: absent + loop: "{{ hostvars[inventory_hostname]['minio_pv'] }}" + when: + - hostvars[inventory_hostname]['minio_pv'] is defined and hostvars[inventory_hostname]['minio_pv']|length > 0 + changed_when: false + failed_when: false + + - name: remove block device files (diskimages) + file: + path: "{{ item }}" + state: absent + loop: "{{ ['/root/diskimage1','/root/diskimage2','/root/diskimage3','/root/diskimage4'] }}" + changed_when: false + failed_when: false + when: + - inventory_hostname in groups['kube_node'] + +- name: remove mount points from /etc/fstab + shell: >- + set -o pipefail && cat /etc/fstab | sed "{{ item.mountPath | replace('/', '\\/') }}"/d | tee /etc/fstab + args: + executable: /bin/bash + loop: "{{ hostvars[inventory_hostname]['minio_pv'] }}" + when: + - hostvars[inventory_hostname]['minio_pv'] is defined and hostvars[inventory_hostname]['minio_pv']|length > 0 + - inventory_hostname in groups['kube_node'] + changed_when: false + failed_when: false diff --git a/roles/service_mesh_install/defaults/main.yml b/roles/service_mesh_install/defaults/main.yml new file mode 100644 index 00000000..579247e7 --- /dev/null +++ b/roles/service_mesh_install/defaults/main.yml @@ -0,0 +1,19 @@ +## +## Copyright (c) 2020-2022 Intel Corporation. +## +## Licensed under the Apache License, Version 2.0 (the "License"); +## you may not use this file except in compliance with the License. +## You may obtain a copy of the License at +## +## http://www.apache.org/licenses/LICENSE-2.0 +## +## Unless required by applicable law or agreed to in writing, software +## distributed under the License is distributed on an "AS IS" BASIS, +## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +## See the License for the specific language governing permissions and +## limitations under the License. +## +--- +service_mesh_download_url: "https://github.com/istio/istio/releases/download/{{ service_mesh.version }}/istio-{{ service_mesh.version }}-linux-amd64.tar.gz" +service_mesh_release_dir: "{{ project_root_dir }}/istio" +service_mesh_charts_dir: "{{ project_root_dir }}/charts/istio" diff --git a/roles/service_mesh_install/files/profiles/.gitkeep b/roles/service_mesh_install/files/profiles/.gitkeep new file mode 100644 index 00000000..e69de29b diff --git a/roles/service_mesh_install/tasks/cleanup.yml b/roles/service_mesh_install/tasks/cleanup.yml new file mode 100644 index 00000000..02772e29 --- /dev/null +++ b/roles/service_mesh_install/tasks/cleanup.yml @@ -0,0 +1,43 @@ +## +## Copyright (c) 2020-2022 Intel Corporation. +## +## Licensed under the Apache License, Version 2.0 (the "License"); +## you may not use this file except in compliance with the License. +## You may obtain a copy of the License at +## +## http://www.apache.org/licenses/LICENSE-2.0 +## +## Unless required by applicable law or agreed to in writing, software +## distributed under the License is distributed on an "AS IS" BASIS, +## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +## See the License for the specific language governing permissions and +## limitations under the License. +## +--- +- name: remove existing istio resources + command: + argv: + - "{{ service_mesh_release_dir }}/bin/istioctl" + - "x" + - "uninstall" + - "--purge" + - "-y" + failed_when: false + changed_when: true + +- name: remove existing tcpip-bypass-ebpf resources + command: "kubectl delete -f {{ service_mesh_charts_dir }}/tcpip-bypass-ebpf.yaml" + changed_when: true + failed_when: false + +- name: remove istio custom manifests directory + file: + path: "{{ service_mesh_charts_dir }}" + state: absent + failed_when: false + +- name: remove istio release directory + file: + path: "{{ service_mesh_release_dir }}" + state: absent + failed_when: false diff --git a/roles/service_mesh_install/tasks/install.yml b/roles/service_mesh_install/tasks/install.yml new file mode 100644 index 00000000..fc75f59c --- /dev/null +++ b/roles/service_mesh_install/tasks/install.yml @@ -0,0 +1,101 @@ +## +## Copyright (c) 2020-2022 Intel Corporation. +## +## Licensed under the Apache License, Version 2.0 (the "License"); +## you may not use this file except in compliance with the License. +## You may obtain a copy of the License at +## +## http://www.apache.org/licenses/LICENSE-2.0 +## +## Unless required by applicable law or agreed to in writing, software +## distributed under the License is distributed on an "AS IS" BASIS, +## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +## See the License for the specific language governing permissions and +## limitations under the License. +## +--- +- name: download and unarchive istio release + unarchive: + src: "{{ service_mesh_download_url }}" + dest: "{{ project_root_dir }}" + remote_src: yes + list_files: yes + mode: 0755 + extra_opts: + - "--transform" + - "s/^istio-{{ service_mesh.version }}/istio/" + register: istio_download + until: istio_download is not failed + retries: 5 + +- name: add istio binaries to the $PATH + copy: + content: | + export PATH={{ service_mesh_release_dir }}/bin:$PATH + dest: /etc/profile.d/istioctl.sh + mode: 0755 + +- name: create istio custom manifests directory + file: + path: "{{ service_mesh_charts_dir }}" + state: directory + mode: 0755 + +- name: copy custom istio profile manifest if exists + block: + - name: copy manifest + copy: + src: "profiles/{{ service_mesh.profile }}.yaml" + dest: "{{ service_mesh_charts_dir }}" + owner: root + mode: preserve + rescue: + - name: fallback to empty profile if no requested profile manifest exists + set_fact: + service_mesh: "{{ service_mesh | combine({'profile':'empty'}, recursive=True) }}" + when: + - service_mesh.profile not in ['default', 'demo', 'minimal', 'external', 'empty', 'preview'] + +- name: evaluate parameters to be used for istio service mesh installation + template: + src: "istioctl-options.yml.j2" + dest: "{{ role_path }}/tasks/istioctl-options.yml" + force: yes + mode: preserve + delegate_to: localhost + become: false + +- name: include temporary file contained options provided + include_vars: "{{ role_path }}/tasks/istioctl-options.yml" + +- name: print out arguments to be used for istio service mesh installation + debug: + msg: | + Following arguments to be used for istio service mesh installation: + {{ argv | join(' ') }} + delegate_to: localhost + become: false + +- name: check if istio system namespace exists + command: kubectl get namespace {{ service_mesh.istio_namespace }} + register: ns_exists + failed_when: false + changed_when: false + +- name: create istio system namespace for istio resources + command: kubectl create namespace "{{ service_mesh.istio_namespace }}" + when: ns_exists.stderr is defined and "NotFound" in ns_exists.stderr + +- name: install istio service mesh + # noqa 305 - shell is used intentionally here + shell: "{{ service_mesh_release_dir }}/bin/istioctl install {{ argv | join(' ') }}" + args: + executable: "/bin/bash" + changed_when: true + +- name: remove temporary options file + file: + path: "{{ role_path }}/tasks/istioctl-options.yml" + state: absent + delegate_to: localhost + become: false diff --git a/roles/istio_operator_install/tasks/main.yml b/roles/service_mesh_install/tasks/main.yml similarity index 58% rename from roles/istio_operator_install/tasks/main.yml rename to roles/service_mesh_install/tasks/main.yml index d899c9be..ca200def 100644 --- a/roles/istio_operator_install/tasks/main.yml +++ b/roles/service_mesh_install/tasks/main.yml @@ -1,5 +1,5 @@ ## -## Copyright (c) 2020-2021 Intel Corporation. +## Copyright (c) 2020-2022 Intel Corporation. ## ## Licensed under the Apache License, Version 2.0 (the "License"); ## you may not use this file except in compliance with the License. @@ -14,18 +14,24 @@ ## limitations under the License. ## --- -- name: remove existing istio operator - include: cleanup.yml +- name: combine defaults and user provided vars + set_fact: + service_mesh: "{{ service_mesh_defaults | combine(service_mesh | default({}), recursive=True) }}" when: - inventory_hostname == groups['kube_control_plane'][0] -- name: install istio operator - include: install.yml +- name: remove existing istio service mesh resources + include_tasks: cleanup.yml when: - inventory_hostname == groups['kube_control_plane'][0] -- name: deploy istio profile - include: deploy_istio_profile.yml +- name: deploy ebpf-tcpip-bypass + include_tasks: tcpip-bypass-ebpf.yml + when: + - inventory_hostname == groups['kube_control_plane'][0] + - service_mesh.tcpip_bypass_ebpf.enabled | default(false) | bool + +- name: install istio service mesh + include_tasks: install.yml when: - inventory_hostname == groups['kube_control_plane'][0] - - istio_profile | default('default') != 'none' diff --git a/roles/service_mesh_install/tasks/tcpip-bypass-ebpf.yml b/roles/service_mesh_install/tasks/tcpip-bypass-ebpf.yml new file mode 100644 index 00000000..03b87e19 --- /dev/null +++ b/roles/service_mesh_install/tasks/tcpip-bypass-ebpf.yml @@ -0,0 +1,41 @@ +## +## Copyright (c) 2020-2022 Intel Corporation. +## +## Licensed under the Apache License, Version 2.0 (the "License"); +## you may not use this file except in compliance with the License. +## You may obtain a copy of the License at +## +## http://www.apache.org/licenses/LICENSE-2.0 +## +## Unless required by applicable law or agreed to in writing, software +## distributed under the License is distributed on an "AS IS" BASIS, +## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +## See the License for the specific language governing permissions and +## limitations under the License. +## +--- +- name: Create istio profiles dir if does not exist + file: + path: "{{ service_mesh_charts_dir }}" + state: directory + owner: root + mode: 0755 + +- name: create tcpip-bypass-ebpf namespace if does not exist + shell: "set -o pipefail && kubectl create ns {{ service_mesh.tcpip_bypass_ebpf.namespace }} -o yaml --dry-run=client | kubectl apply -f -" + args: + executable: /bin/bash + changed_when: true + +- name: populate tcpip-bypass-ebpf manifest template with values + template: + src: "tcpip-bypass-ebpf.yaml.j2" + dest: "{{ service_mesh_charts_dir }}/tcpip-bypass-ebpf.yaml" + force: yes + mode: preserve + +- name: deploly tcpip-bypass-ebpf + shell: "set -o pipefail && kubectl apply -f {{ service_mesh_charts_dir }}/tcpip-bypass-ebpf.yaml -o yaml --dry-run=client | kubectl apply -f -" + args: + executable: /bin/bash + changed_when: true diff --git a/roles/service_mesh_install/templates/istioctl-options.yml.j2 b/roles/service_mesh_install/templates/istioctl-options.yml.j2 new file mode 100644 index 00000000..5ddacf14 --- /dev/null +++ b/roles/service_mesh_install/templates/istioctl-options.yml.j2 @@ -0,0 +1,60 @@ +argv: + - --skip-confirmation +{% if service_mesh.context is defined and service_mesh.context != '' %} + - --context + - {{ service_mesh.context }} +{% endif -%} +{% if service_mesh.filename is defined and service_mesh.filename != [] %} +{% for item in service_mesh.filename %} + - --filename + - {{ item }} +{% endfor -%} +{% endif -%} +{% if service_mesh.namespace is defined and service_mesh.namespace != '' %} + - --namespace + - {{ service_mesh.namespace }} +{% endif -%} +{% if service_mesh.istio_namespace is defined and service_mesh.istio_namespace != '' %} + - --istioNamespace + - {{ service_mesh.istio_namespace }} +{% endif -%} +{% if service_mesh.kubeconfig is defined and service_mesh.kubeconfig != '' %} + - --kubeconfig + - {{ service_mesh.kubeconfig }} +{% endif -%} +{% if service_mesh.vklog is defined and service_mesh.vklog != '' %} + - --vklog + - {{ service_mesh.vklog }} +{% endif -%} +{% if service_mesh.revision is defined and service_mesh.revision != '' %} + - --revision + - {{ service_mesh.revision }} +{% endif -%} +{% if service_mesh.manifest is defined and service_mesh.manifest != '' %} + - --manifests + - {{ service_mesh.manifest }} +{% endif -%} +{% if service_mesh.dry_run is defined and service_mesh.dry_run | bool %} + - --dry-run +{% endif -%} +{% if service_mesh.force is defined and service_mesh.force | bool %} + - --force +{% endif -%} +{% if service_mesh.readiness_timeout is defined and service_mesh.readiness_timeout != '' %} + - --readiness-timeout + - {{ service_mesh.readiness_timeout }} +{% endif -%} +{% if service_mesh.set is defined and service_mesh.set != [] %} +{% for item in service_mesh.set %} + - --set + - {{ item }} +{% endfor -%} +{% endif -%} +{% if service_mesh.verify is defined and service_mesh.verify | bool and service_mesh.profile != 'empty' %} + - --verify +{% endif -%} +{% if service_mesh.profile in ['default', 'demo', 'minimal', 'external', 'empty', 'preview'] %} + - --set profile={{ service_mesh.profile }} +{% else %} + - --filename={{ service_mesh_charts_dir }}/{{ service_mesh.profile }}.yaml +{% endif -%} diff --git a/roles/service_mesh_install/templates/tcpip-bypass-ebpf.yaml.j2 b/roles/service_mesh_install/templates/tcpip-bypass-ebpf.yaml.j2 new file mode 100644 index 00000000..c514ab8a --- /dev/null +++ b/roles/service_mesh_install/templates/tcpip-bypass-ebpf.yaml.j2 @@ -0,0 +1,37 @@ +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: {{ service_mesh.tcpip_bypass_ebpf.name }} + namespace: {{ service_mesh.tcpip_bypass_ebpf.namespace }} + labels: + k8s-app: {{ service_mesh.tcpip_bypass_ebpf.name }} +spec: + selector: + matchLabels: + name: {{ service_mesh.tcpip_bypass_ebpf.name }} + template: + metadata: + labels: + name: {{ service_mesh.tcpip_bypass_ebpf.name }} + spec: + tolerations: + # this toleration is to have the daemonset runnable on master nodes + # remove it if your masters can't run pods + - key: node-role.kubernetes.io/master + effect: NoSchedule + containers: + - name: {{ service_mesh.tcpip_bypass_ebpf.name }} + image: {{ service_mesh.tcpip_bypass_ebpf.image }} + imagePullPolicy: IfNotPresent + securityContext: + privileged: true + volumeMounts: + - name: sysfs + mountPath: /sys/fs + env: + - name: BPF_BYPASS_TCPIP + value: "enabled" + volumes: + - name: sysfs + hostPath: + path: /sys/fs diff --git a/roles/service_mesh_install/vars/main.yml b/roles/service_mesh_install/vars/main.yml new file mode 100644 index 00000000..f2eb01c0 --- /dev/null +++ b/roles/service_mesh_install/vars/main.yml @@ -0,0 +1,37 @@ +## +## Copyright (c) 2020-2022 Intel Corporation. +## +## Licensed under the Apache License, Version 2.0 (the "License"); +## you may not use this file except in compliance with the License. +## You may obtain a copy of the License at +## +## http://www.apache.org/licenses/LICENSE-2.0 +## +## Unless required by applicable law or agreed to in writing, software +## distributed under the License is distributed on an "AS IS" BASIS, +## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +## See the License for the specific language governing permissions and +## limitations under the License. +## +service_mesh_defaults: + enabled: false + version: 1.13.0 + context: '' + filename: [] + namespace: '' + istio_namespace: 'istio-system' + kubeconfig: '' + vklog: '' + revision: '' + manifests: '' + dry_run: false + profile: default + force: false + readiness_timeout: '' + set: [] + verify: true + tcpip_bypass_ebpf: + enabled: false + name: bypass-tcpip + namespace: kube-system + image: docker.io/intel/istio-tcpip-bypass@sha256:9f3944e8591d3eda51e547be039a9d5202f1403e024847273cfb98aafe31919f diff --git a/roles/sgx_configure_icx/defaults/main.yml b/roles/sgx_configure_icx/defaults/main.yml deleted file mode 100644 index 07fcf7a5..00000000 --- a/roles/sgx_configure_icx/defaults/main.yml +++ /dev/null @@ -1,80 +0,0 @@ -## -## Copyright (c) 2020-2021 Intel Corporation. -## -## Licensed under the Apache License, Version 2.0 (the "License"); -## you may not use this file except in compliance with the License. -## You may obtain a copy of the License at -## -## http://www.apache.org/licenses/LICENSE-2.0 -## -## Unless required by applicable law or agreed to in writing, software -## distributed under the License is distributed on an "AS IS" BASIS, -## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -## See the License for the specific language governing permissions and -## limitations under the License. -## ---- -# Intel SGX-DCAP drivers module for Ubuntu <= 18.04.4 -dcap_driver_series_ubuntu_18: "1.41" -dcap_driver_version_ubuntu_18: "sgx_linux_x64_driver_{{ dcap_driver_series_ubuntu_18 }}.bin" -dcap_driver_url_ubuntu_18: "https://download.01.org/intel-sgx/sgx-dcap/1.11/linux/distro/ubuntu18.04-server/{{ dcap_driver_version_ubuntu_18 }}" -dcap_driver_checksum_ubuntu_18: "sha256:010d46dbba4529609ff7532acef811fcb3d05aeb071872a33ea50f6578797a6f" -sgx_folder_check_ubuntu_18: "/usr/src/sgx-{{ dcap_driver_series_ubuntu_18 }}" - -sgx_sdk_version_ubuntu_18: "sgx_linux_x64_sdk_2.14.100.2.bin" -sgx_sdk_url_ubuntu_18: "https://download.01.org/intel-sgx/sgx-dcap/1.11/linux/distro/ubuntu18.04-server/{{ sgx_sdk_version_ubuntu_18 }}" -sgx_sdk_checksum_ubuntu_18: "sha256:3509a16e37e172369e1c4c4664047ad08bf3e608588a3a0df7367401e5f81e97" - -# Intel SGX-DCAP drivers module for Ubuntu 20.04 -dcap_driver_series_ubuntu_20: "1.41" -dcap_driver_version_ubuntu_20: "sgx_linux_x64_driver_{{ dcap_driver_series_ubuntu_20 }}.bin" -dcap_driver_url_ubuntu_20: "https://download.01.org/intel-sgx/sgx-dcap/1.11/linux/distro/ubuntu20.04-server/{{ dcap_driver_version_ubuntu_20 }}" -dcap_driver_checksum_ubuntu_20: "sha256:269a0062668c59212b1aa4c392b6de02fa575ec45435a935776aae2645482849" -sgx_folder_check_ubuntu_20: "/usr/src/sgx-{{ dcap_driver_series_ubuntu_20 }}" - -sgx_sdk_version_ubuntu_20: "sgx_linux_x64_sdk_2.14.100.2.bin" -sgx_sdk_url_ubuntu_20: "https://download.01.org/intel-sgx/sgx-dcap/1.11/linux/distro/ubuntu20.04-server/{{ sgx_sdk_version_ubuntu_20 }}" -sgx_sdk_checksum_ubuntu_20: "sha256:d0626ffb36c2e20c589d954fb968fded24ce51529b8b61a42febb312fd9debfc" - -# Intel SGX-SGX Key configuration for Ubuntu >= 18.04.4 -sgx_apt_source_list: "intel-sgx" -sgx_apt_repo_url: "https://download.01.org/intel-sgx/sgx_repo/ubuntu" -sgx_apt_repo_key: "{{ sgx_apt_repo_url }}/intel-sgx-deb.key" - -# Intel SGX-DCAP drivers module for CentOS 8.2 -dcap_driver_series_centos: "1.41" -dcap_driver_version_centos: "sgx_linux_x64_driver_{{ dcap_driver_series_centos }}.bin" -dcap_driver_url_centos: "https://download.01.org/intel-sgx/sgx-dcap/1.11/linux/distro/centos8.2-server/{{ dcap_driver_version_centos }}" -dcap_driver_checksum_centos: "sha256:32ed44af13aead9bbae123c9d309256d46f928aa7cc754594cded16ceeee1e0b" -sgx_folder_check_centos: "/usr/src/sgx-{{ dcap_driver_series_centos }}" - -sgx_sdk_version_centos: "sgx_linux_x64_sdk_2.14.100.2.bin" -sgx_sdk_url_centos: "https://download.01.org/intel-sgx/sgx-dcap/1.11/linux/distro/centos8.2-server/{{ sgx_sdk_version_centos }}" -sgx_sdk_checksum_centos: "sha256:4f317fe314efcf90d9bb951d7a754dffb70878d78b3beae2dd627bb4b3620dc8" - -# Intel SGX-DCAP drivers module for RHEL 8.2 -dcap_driver_series_rhel: "1.41" -dcap_driver_version_rhel: "sgx_linux_x64_driver_{{ dcap_driver_series_rhel }}.bin" -dcap_driver_url_rhel: "https://download.01.org/intel-sgx/sgx-dcap/1.11/linux/distro/rhel8.2-server/{{ dcap_driver_version_rhel }}" -dcap_driver_checksum_rhel: "sha256:55a9878ba778b1b2890e20a362009be951a99d6073e1dbf6ffef740dbd93be50" -sgx_folder_check_rhel: "/usr/src/sgx-{{ dcap_driver_series_centos }}" - -sgx_sdk_version_rhel: "sgx_linux_x64_sdk_2.14.100.2.bin" -sgx_sdk_url_rhel: "https://download.01.org/intel-sgx/sgx-dcap/1.11/linux/distro/rhel8.2-server/{{ sgx_sdk_version_rhel }}" -sgx_sdk_checksum_rhel: "sha256:11b1bc9e914ee3290b324c409a8c38cf7986651d70df69ebef834e0a732c7459" - -# Intel SGX RPM local repository for CentOS 8.2 -sgx_rpm_local_repo_version_centos: "sgx_rpm_local_repo.tgz" -sgx_rpm_local_repo_url_centos: "https://download.01.org/intel-sgx/sgx-dcap/1.11/linux/distro/centos8.2-server/{{ sgx_rpm_local_repo_version_centos }}" -sgx_rpm_local_repo_checksum_centos: "sha256:fae709696a21ed7776073942159edd890e820e6e98359b53325037cfe2407363" - -# Intel SGX RPM local repository for RHEL 8.2 -sgx_rpm_local_repo_version_rhel: "sgx_rpm_local_repo.tgz" -sgx_rpm_local_repo_url_rhel: "https://download.01.org/intel-sgx/sgx-dcap/1.11/linux/distro/rhel8.2-server/{{ sgx_rpm_local_repo_version_rhel }}" -sgx_rpm_local_repo_checksum_rhel: "sha256:2f5bbd4746231f9def7426b52e1d1f0c5972567badb50147ccda1534d81e6041" - -sgx_config_dir: "/usr/src" -sgx_rpm_directory: "/usr/src/sgx_rpm_local_repo" - -sgx_pkg_version: "2.14.100.2" -sgx_pkg_dcap_version: "1.11.100.2" diff --git a/roles/sgx_dp_install/charts/intel-sgx-aesmd/Chart.yaml b/roles/sgx_dp_install/charts/intel-sgx-aesmd/Chart.yaml index 4139e663..05e63eb4 100644 --- a/roles/sgx_dp_install/charts/intel-sgx-aesmd/Chart.yaml +++ b/roles/sgx_dp_install/charts/intel-sgx-aesmd/Chart.yaml @@ -1,5 +1,5 @@ ## -## Copyright (c) 2020-2021 Intel Corporation. +## Copyright (c) 2020-2022 Intel Corporation. ## ## Licensed under the Apache License, Version 2.0 (the "License"); ## you may not use this file except in compliance with the License. @@ -16,7 +16,7 @@ apiVersion: v1 description: Intel® Software Guard Extensions (SGX) device plugin for Kubernetes name: intel-sgx-plugin -version: 0.21.0 +version: 0.23.0 sources: - https://github.com/intel/intel-device-plugins-for-kubernetes.git -appVersion: 0.21.0 +appVersion: 0.23.0 diff --git a/roles/sgx_dp_install/defaults/main.yaml b/roles/sgx_dp_install/defaults/main.yaml index 9a4317d7..e29a21db 100644 --- a/roles/sgx_dp_install/defaults/main.yaml +++ b/roles/sgx_dp_install/defaults/main.yaml @@ -1,5 +1,5 @@ ## -## Copyright (c) 2020-2021 Intel Corporation. +## Copyright (c) 2020-2022 Intel Corporation. ## ## Licensed under the Apache License, Version 2.0 (the "License"); ## you may not use this file except in compliance with the License. @@ -15,10 +15,10 @@ ## --- intel_sgx_dp_git_url: "https://github.com/intel/intel-device-plugins-for-kubernetes.git" -intel_sgx_dp_git_ref: "v0.21.0" -intel_sgx_dp_version: "0.21.0" -intel_sgx_dp_dir: "/usr/src/intel-device-plugins-for-kubernetes-sgx" -intel_sgx_psp_rbac_dir: "/usr/src/psp-rbac-rules/" +intel_sgx_dp_git_ref: "v0.23.0" +intel_sgx_dp_version: "0.23.0" +intel_sgx_dp_dir: "{{ (project_root_dir, 'intel-sgx-dp') | path_join }}" +intel_sgx_psp_rbac_dir: "{{ project_root_dir}}/psp-rbac-rules/" sgx_dp_build_image_locally: true sgx_dp_provision_limit: 20 diff --git a/roles/sgx_dp_install/tasks/main.yaml b/roles/sgx_dp_install/tasks/main.yaml index c93f4ce2..8cd028a6 100644 --- a/roles/sgx_dp_install/tasks/main.yaml +++ b/roles/sgx_dp_install/tasks/main.yaml @@ -1,5 +1,5 @@ ## -## Copyright (c) 2020-2021 Intel Corporation. +## Copyright (c) 2020-2022 Intel Corporation. ## ## Licensed under the Apache License, Version 2.0 (the "License"); ## you may not use this file except in compliance with the License. @@ -17,11 +17,16 @@ - name: determine machine type include_role: name: check_machine_type + when: + - inventory_hostname == groups['kube_node'][0] - name: install dependencies include_role: name: install_dependencies - when: is_icx | default(false) | bool + when: + - inventory_hostname == groups['kube_node'][0] + - is_icx | default(false) | bool or + is_spr | default(false) | bool - name: clone Intel Device Plugins repository git: @@ -31,7 +36,8 @@ force: yes when: - inventory_hostname == groups['kube_node'][0] - - is_icx | default(false) | bool + - is_icx | default(false) | bool or + is_spr | default(false) | bool # docker is used as container runtime: - name: prepare containers images @@ -81,7 +87,8 @@ command: docker push {{ registry_local_address }}/intel-sgx-aesmd-demo:{{ intel_sgx_dp_version }} when: - inventory_hostname == groups['kube_node'][0] - - is_icx | default(false) | bool + - is_icx | default(false) | bool or + is_spr | default(false) | bool - container_runtime == "docker" # containerd/cri-o is used as container runtime: @@ -116,9 +123,11 @@ changed_when: true when: - inventory_hostname == groups['kube_node'][0] - - is_icx | default(false) | bool + - is_icx | default(false) | bool or + is_spr | default(false) | bool - '"docker" not in container_runtime' +# start deployment of SGX DP - name: prepare and deploy PSP and RBAC block: - name: make sure directory for PSP and RBAC rules exists @@ -153,7 +162,8 @@ - sgx-rbac-role-binding.yml when: - inventory_hostname == groups['kube_control_plane'][0] - - is_icx | default(false) | bool + - hostvars[groups['kube_node'][0]]['is_icx'] or + hostvars[groups['kube_node'][0]]['is_spr'] - psp_enabled | default(true) - name: prepare and deploy Intel SGX Device Plugin @@ -168,50 +178,54 @@ - name: populate Intel SGX Plugin yaml file and push to controller node template: src: "intel-sgx-plugin.yml.j2" - dest: "/usr/src/intel-sgx-plugin.yml" + dest: "{{ project_root_dir }}/intel-sgx-plugin.yml" force: yes mode: preserve - name: deploy Intel SGX Device Plugin with the Intel Device Plugin Operator k8s: state: present - src: "/usr/src/intel-sgx-plugin.yml" + src: "{{ project_root_dir }}/intel-sgx-plugin.yml" when: - inventory_hostname == groups['kube_control_plane'][0] - - is_icx | default(false) | bool + - hostvars[groups['kube_node'][0]]['is_icx'] or + hostvars[groups['kube_node'][0]]['is_spr'] - name: wait for Intel SGX Device Plugin pause: minutes: 1 - when: is_icx | default(false) | bool + when: + - is_icx | default(false) | bool or + is_spr | default(false) | bool - name: prepare and deploy Intel SGX aesmd block: - name: create Helm charts directory if needed file: - path: /usr/src/charts + path: "{{ project_root_dir }}/charts" state: directory mode: 0755 - name: populate Intel SGX aesmd Helm chart values template and push to controller node template: src: "aesmd-values.yml.j2" - dest: "/usr/src/charts/intel-sgx-aesmd-values.yml" + dest: "{{ project_root_dir }}/charts/intel-sgx-aesmd-values.yml" force: yes mode: preserve - name: copy Helm chart to the controller node copy: src: "{{ role_path }}/charts/intel-sgx-aesmd" - dest: "/usr/src/charts/" + dest: "{{ project_root_dir }}/charts/" mode: 0755 - name: install Intel SGX aesmd Helm chart command: >- helm upgrade -i intel-sgx-aesmd - -f /usr/src/charts/intel-sgx-aesmd-values.yml + -f {{ project_root_dir }}/charts/intel-sgx-aesmd-values.yml --namespace {{ sgx_aesmd_namespace }} - /usr/src/charts/intel-sgx-aesmd + {{ project_root_dir }}/charts/intel-sgx-aesmd when: - inventory_hostname == groups['kube_control_plane'][0] - - is_icx | default(false) | bool + - hostvars[groups['kube_node'][0]]['is_icx'] or + hostvars[groups['kube_node'][0]]['is_spr'] diff --git a/roles/sgx_dp_install/templates/intel-sgx-plugin.yml.j2 b/roles/sgx_dp_install/templates/intel-sgx-plugin.yml.j2 index 6d1d5209..d8ae5dd3 100644 --- a/roles/sgx_dp_install/templates/intel-sgx-plugin.yml.j2 +++ b/roles/sgx_dp_install/templates/intel-sgx-plugin.yml.j2 @@ -1,9 +1,12 @@ apiVersion: deviceplugin.intel.com/v1 kind: SgxDevicePlugin metadata: - name: intel-sgx-plugin + name: intel-sgx-device-plugin spec: - image: {{ sgx_dp_image | default("docker.io/intel/intel-sgx-plugin") }}:{{ sgx_dp_version | default("0.21.0") }} - initImage: {{ sgx_dp_init_image | default("docker.io/intel/intel-sgx-initcontainer") }}:{{ sgx_dp_version | default("0.21.0") }} + image: {{ sgx_dp_image | default("docker.io/intel/intel-sgx-plugin") }}:{{ sgx_dp_version | default("0.23.0") }} + initImage: {{ sgx_dp_init_image | default("docker.io/intel/intel-sgx-initcontainer") }}:{{ sgx_dp_version | default("0.23.0") }} enclaveLimit: {{ sgx_dp_enclave_limit }} provisionLimit: {{ sgx_dp_provision_limit }} + logLevel: {{ sgx_dp_verbosity | default(4) }} + nodeSelector: + feature.node.kubernetes.io/intel.sgx: 'true' diff --git a/roles/sgx_dp_install/vars/main.yml b/roles/sgx_dp_install/vars/main.yml index 9f60069c..170d32e5 100644 --- a/roles/sgx_dp_install/vars/main.yml +++ b/roles/sgx_dp_install/vars/main.yml @@ -1,5 +1,5 @@ ## -## Copyright (c) 2020-2021 Intel Corporation. +## Copyright (c) 2020-2022 Intel Corporation. ## ## Licensed under the Apache License, Version 2.0 (the "License"); ## you may not use this file except in compliance with the License. diff --git a/roles/sriov_cni_install/defaults/main.yml b/roles/sriov_cni_install/defaults/main.yml index 4f6c82e5..92cbf8ed 100644 --- a/roles/sriov_cni_install/defaults/main.yml +++ b/roles/sriov_cni_install/defaults/main.yml @@ -1,5 +1,5 @@ ## -## Copyright (c) 2020-2021 Intel Corporation. +## Copyright (c) 2020-2022 Intel Corporation. ## ## Licensed under the Apache License, Version 2.0 (the "License"); ## you may not use this file except in compliance with the License. @@ -14,6 +14,6 @@ ## limitations under the License. ## --- -sriov_cni_version: "v2.6.1" +sriov_cni_version: "v2.6.2" sriov_cni_url: "https://github.com/k8snetworkplumbingwg/sriov-cni.git" -sriov_cni_dir: "/usr/src/sriov-cni" +sriov_cni_dir: "{{ (project_root_dir, 'sriov-cni') | path_join }}" diff --git a/roles/sriov_cni_install/tasks/main.yml b/roles/sriov_cni_install/tasks/main.yml index 7072b71f..f21e2719 100644 --- a/roles/sriov_cni_install/tasks/main.yml +++ b/roles/sriov_cni_install/tasks/main.yml @@ -1,5 +1,5 @@ ## -## Copyright (c) 2020-2021 Intel Corporation. +## Copyright (c) 2020-2022 Intel Corporation. ## ## Licensed under the Apache License, Version 2.0 (the "License"); ## you may not use this file except in compliance with the License. @@ -25,9 +25,19 @@ version: "{{ sriov_cni_version }}" force: yes +- name: cleans up unused dependencies for golang before sriov-cni plugin + command: "go mod tidy" + args: + chdir: "{{ sriov_cni_dir }}" + changed_when: true + +- name: update GOFLAGS for golang env before sriov-cni plugin + command: "go env -w GOFLAGS=-mod=mod" + changed_when: true + - name: build sriov-cni plugin make: - chdir: /usr/src/sriov-cni + chdir: "{{ sriov_cni_dir }}" - name: create /opt/cni/bin file: @@ -38,7 +48,7 @@ - name: install sriov-cni binary to /opt/cni/bin directory copy: - src: /usr/src/sriov-cni/build/sriov + src: "{{ sriov_cni_dir }}/build/sriov" dest: /opt/cni/bin/sriov mode: 0755 remote_src: yes diff --git a/roles/sriov_cni_install/vars/main.yml b/roles/sriov_cni_install/vars/main.yml index 1819a19d..77dddbc0 100644 --- a/roles/sriov_cni_install/vars/main.yml +++ b/roles/sriov_cni_install/vars/main.yml @@ -1,5 +1,5 @@ ## -## Copyright (c) 2020-2021 Intel Corporation. +## Copyright (c) 2020-2022 Intel Corporation. ## ## Licensed under the Apache License, Version 2.0 (the "License"); ## you may not use this file except in compliance with the License. diff --git a/roles/sriov_dp_install/charts/sriov-net-dp/Chart.yaml b/roles/sriov_dp_install/charts/sriov-net-dp/Chart.yaml index 14718842..4076871c 100644 --- a/roles/sriov_dp_install/charts/sriov-net-dp/Chart.yaml +++ b/roles/sriov_dp_install/charts/sriov-net-dp/Chart.yaml @@ -1,5 +1,5 @@ ## -## Copyright (c) 2020-2021 Intel Corporation. +## Copyright (c) 2020-2022 Intel Corporation. ## ## Licensed under the Apache License, Version 2.0 (the "License"); ## you may not use this file except in compliance with the License. diff --git a/roles/sriov_dp_install/charts/sriov-net-dp/values.yaml b/roles/sriov_dp_install/charts/sriov-net-dp/values.yaml index 25e8117b..b4b8c36c 100644 --- a/roles/sriov_dp_install/charts/sriov-net-dp/values.yaml +++ b/roles/sriov_dp_install/charts/sriov-net-dp/values.yaml @@ -1,5 +1,5 @@ ## -## Copyright (c) 2020-2021 Intel Corporation. +## Copyright (c) 2020-2022 Intel Corporation. ## ## Licensed under the Apache License, Version 2.0 (the "License"); ## you may not use this file except in compliance with the License. @@ -17,8 +17,8 @@ namespace: kube-system image: - repository: docker.io/nfvpe/sriov-device-plugin - tag: v3.2 + repository: ghcr.io/k8snetworkplumbingwg/sriov-network-device-plugin + tag: v3.4.0 pullPolicy: IfNotPresent configPath: /etc/pcidp/config.json diff --git a/roles/sriov_dp_install/defaults/main.yml b/roles/sriov_dp_install/defaults/main.yml index 816129cb..e1dbf9e7 100644 --- a/roles/sriov_dp_install/defaults/main.yml +++ b/roles/sriov_dp_install/defaults/main.yml @@ -1,5 +1,5 @@ ## -## Copyright (c) 2020-2021 Intel Corporation. +## Copyright (c) 2020-2022 Intel Corporation. ## ## Licensed under the Apache License, Version 2.0 (the "License"); ## you may not use this file except in compliance with the License. @@ -15,10 +15,10 @@ ## --- sriov_net_dp_git_url: "https://github.com/k8snetworkplumbingwg/sriov-network-device-plugin.git" -sriov_net_dp_git_ref: "v3.3.2" -sriov_net_dp_dir: "/usr/src/sriov-network-device-plugin" -sriov_net_dp_build_image_locally: true -sriov_net_dp_autogenerate: true +sriov_net_dp_dir: "{{ (project_root_dir, 'sriov-network-device-plugin') | path_join }}" +sriov_net_dp_image: "ghcr.io/k8snetworkplumbingwg/sriov-network-device-plugin" +sriov_net_dp_tag: "v3.4.0" +sriov_net_dp_build_image_locally: false sriov_net_dp_namespace: kube-system sriovdp_config_data: | diff --git a/roles/sriov_dp_install/tasks/main.yml b/roles/sriov_dp_install/tasks/main.yml index 1bde0dd7..4a9feb43 100644 --- a/roles/sriov_dp_install/tasks/main.yml +++ b/roles/sriov_dp_install/tasks/main.yml @@ -1,5 +1,5 @@ ## -## Copyright (c) 2020-2021 Intel Corporation. +## Copyright (c) 2020-2022 Intel Corporation. ## ## Licensed under the Apache License, Version 2.0 (the "License"); ## you may not use this file except in compliance with the License. @@ -21,7 +21,7 @@ - name: clone SRIOV Network Device Plugin repository git: repo: "{{ sriov_net_dp_git_url }}" - version: "{{ sriov_net_dp_git_ref }}" + version: "{{ sriov_net_dp_tag }}" dest: "{{ sriov_net_dp_dir }}" force: yes when: @@ -60,14 +60,14 @@ until: sriov_dp_image_build is success - name: tag SRIOV Network Device Plugin image - command: docker tag nfvpe/sriov-device-plugin:latest {{ registry_local_address }}/sriov-device-plugin:{{ sriov_net_dp_git_ref }} + command: docker tag {{ sriov_net_dp_image }}:latest {{ registry_local_address }}/sriov-device-plugin:{{ sriov_net_dp_tag }} when: - inventory_hostname == groups['kube_node'][0] - sriov_net_dp_build_image_locally - container_runtime == "docker" - name: push SRIOV Network Device Plugin image to local registry - command: docker push {{ registry_local_address }}/sriov-device-plugin:{{ sriov_net_dp_git_ref }} + command: docker push {{ registry_local_address }}/sriov-device-plugin:{{ sriov_net_dp_tag }} when: - inventory_hostname == groups['kube_node'][0] - sriov_net_dp_build_image_locally @@ -75,7 +75,7 @@ - name: build and tag SRIOV Network Device Plugin image behind proxy shell: >- - buildah bud -t {{ registry_local_address }}/{{ item.name }}:{{ sriov_net_dp_git_ref }} + buildah bud -t {{ registry_local_address }}/{{ item.name }}:{{ sriov_net_dp_tag }} --build-arg=http_proxy=${http_proxy} --build-arg=https_proxy=${https_proxy} -f {{ item.file }} . @@ -95,7 +95,7 @@ - https_proxy is defined - name: build and tag SRIOV Network Device Plugin image - command: buildah bud -t {{ registry_local_address }}/{{ item.name }}:{{ sriov_net_dp_git_ref }} -f {{ item.file }} . + command: buildah bud -t {{ registry_local_address }}/{{ item.name }}:{{ sriov_net_dp_tag }} -f {{ item.file }} . args: chdir: "{{ sriov_net_dp_dir }}" changed_when: true @@ -112,7 +112,7 @@ - https_proxy is not defined - name: push SRIOV Network Device Plugin image to local registry - command: buildah push {{ registry_local_address }}/sriov-device-plugin:{{ sriov_net_dp_git_ref }} + command: buildah push {{ registry_local_address }}/sriov-device-plugin:{{ sriov_net_dp_tag }} changed_when: true when: - inventory_hostname == groups['kube_node'][0] @@ -121,7 +121,7 @@ - name: create Helm charts directory if needed file: - path: "/usr/src/charts" + path: "{{ (project_root_dir, 'charts') | path_join }}" state: directory mode: 0755 when: @@ -129,8 +129,8 @@ - name: copy SRIOV Network Device Plugin Helm chart to the controller node copy: - src: "{{ role_path }}/charts/sriov-net-dp" - dest: "/usr/src/charts/" + src: "{{ (role_path, 'charts', 'sriov-net-dp') | path_join }}" + dest: "{{ (project_root_dir, 'charts') | path_join }}" mode: 0755 when: - inventory_hostname == groups['kube_control_plane'][0] @@ -138,7 +138,6 @@ - name: set values for SRIOV Network Device Plugin Helm chart values for locally built and stored image set_fact: sriov_net_dp_image: "{{ registry_local_address }}/sriov-device-plugin" - sriov_net_dp_tag: "{{ sriov_net_dp_git_ref }}" when: - sriov_net_dp_build_image_locally - inventory_hostname == groups['kube_control_plane'][0] @@ -146,13 +145,16 @@ - name: populate SRIOV Network Device Plugin Helm chart values template and push to controller node template: src: "helm_values.yml.j2" - dest: "/usr/src/charts/sriov-net-dp-values.yml" + dest: "{{ (project_root_dir, 'charts', 'sriov-net-dp-values.yml') | path_join }}" force: yes mode: preserve when: - inventory_hostname == groups['kube_control_plane'][0] - name: install SRIOV Network Device Plugin helm chart - command: helm upgrade --install sriov-net-dp -f /usr/src/charts/sriov-net-dp-values.yml /usr/src/charts/sriov-net-dp + command: >- + helm upgrade -i sriov-net-dp + -f {{ (project_root_dir, 'charts', 'sriov-net-dp-values.yml') | path_join }} + {{ (project_root_dir, 'charts', 'sriov-net-dp') | path_join }} when: - inventory_hostname == groups['kube_control_plane'][0] diff --git a/roles/sriov_dp_install/templates/helm_values.yml.j2 b/roles/sriov_dp_install/templates/helm_values.yml.j2 index 01a641d1..3b5a24ca 100644 --- a/roles/sriov_dp_install/templates/helm_values.yml.j2 +++ b/roles/sriov_dp_install/templates/helm_values.yml.j2 @@ -2,8 +2,8 @@ namespace: {{ sriov_net_dp_namespace | default("kube-system") }} image: - repository: {{ sriov_net_dp_image | default("docker.io/nfvpe/sriov-device-plugin") }} - tag: {{ sriov_net_dp_tag | default("v3.2") }} + repository: {{ sriov_net_dp_image | default("ghcr.io/k8snetworkplumbingwg/sriov-network-device-plugin") }} + tag: {{ sriov_net_dp_tag | default("v3.4.0") }} pullPolicy: IfNotPresent configPath: {{ sriov_net_dp_config_path | default("/etc/pcidp/config.json") }} diff --git a/roles/sriov_dp_install/vars/main.yml b/roles/sriov_dp_install/vars/main.yml index 9f60069c..170d32e5 100644 --- a/roles/sriov_dp_install/vars/main.yml +++ b/roles/sriov_dp_install/vars/main.yml @@ -1,5 +1,5 @@ ## -## Copyright (c) 2020-2021 Intel Corporation. +## Copyright (c) 2020-2022 Intel Corporation. ## ## Licensed under the Apache License, Version 2.0 (the "License"); ## you may not use this file except in compliance with the License. diff --git a/roles/sriov_network_operator_install/charts/sriov-network-operator/Chart.yaml b/roles/sriov_network_operator_install/charts/sriov-network-operator/Chart.yaml index ccb91748..bf0c38fb 100644 --- a/roles/sriov_network_operator_install/charts/sriov-network-operator/Chart.yaml +++ b/roles/sriov_network_operator_install/charts/sriov-network-operator/Chart.yaml @@ -1,5 +1,5 @@ ## -## Copyright (c) 2020-2021 Intel Corporation. +## Copyright (c) 2020-2022 Intel Corporation. ## ## Licensed under the Apache License, Version 2.0 (the "License"); ## you may not use this file except in compliance with the License. diff --git a/roles/sriov_network_operator_install/charts/sriov-network-operator/README.md b/roles/sriov_network_operator_install/charts/sriov-network-operator/README.md index e195a9c1..8888b26a 100644 --- a/roles/sriov_network_operator_install/charts/sriov-network-operator/README.md +++ b/roles/sriov_network_operator_install/charts/sriov-network-operator/README.md @@ -1,5 +1,5 @@