From 77281b7f57c27b80edfee9108ba4e16f822d8311 Mon Sep 17 00:00:00 2001 From: Karl Cardenas Date: Wed, 24 Jan 2024 06:54:42 -0700 Subject: [PATCH 1/2] docs: added custom YAML override ability --- terraform/cluster-profiles/cp-modules/cp.tf | 2 +- .../cluster-profiles/cp-modules/inputs.tf | 16 +-- .../cp-profiles-with-module/cp.tf | 5 +- .../templates/kubernetes.yaml | 116 ++++++++++++++++++ 4 files changed, 130 insertions(+), 9 deletions(-) create mode 100644 terraform/cluster-profiles/cp-profiles-with-module/templates/kubernetes.yaml diff --git a/terraform/cluster-profiles/cp-modules/cp.tf b/terraform/cluster-profiles/cp-modules/cp.tf index 226c29e..b8a84cf 100644 --- a/terraform/cluster-profiles/cp-modules/cp.tf +++ b/terraform/cluster-profiles/cp-modules/cp.tf @@ -17,7 +17,7 @@ resource "spectrocloud_cluster_profile" "profile" { name = pack.value.name tag = pack.value.pack_data.version uid = pack.value.pack_data.id - values = pack.value.pack_data.values + values = lookup(var.custom_yaml_files, pack.value.name, null) != null ? var.custom_yaml_files[pack.value.name] : pack.value.pack_data.values } } diff --git a/terraform/cluster-profiles/cp-modules/inputs.tf b/terraform/cluster-profiles/cp-modules/inputs.tf index 812fb99..47650cb 100644 --- a/terraform/cluster-profiles/cp-modules/inputs.tf +++ b/terraform/cluster-profiles/cp-modules/inputs.tf @@ -51,12 +51,14 @@ variable "pack_order" { variable "packs" { type = map(string) description = "A list " - default = { - "csi-aws-ebs" = "1.22.0" - "cni-calico" = "3.26.1" - "kubernetes" = "1.27.5" - "ubuntu-aws" = "22.04" - } + default = {} +} + + +variable "custom_yaml_files" { + type = map(string) + description = "The file path to the custom YAML file that matches the pack name its values are overriding. Use the function 'file() to read the file. Example: 'kubernetes: 'file(path/to/file)'" + default = {} } variable "tags" { @@ -72,4 +74,4 @@ locals { version = var.packs[pack_name] pack_data = [for pack in data.spectrocloud_pack.generic : pack if pack.name == pack_name][0] }] -} \ No newline at end of file +} diff --git a/terraform/cluster-profiles/cp-profiles-with-module/cp.tf b/terraform/cluster-profiles/cp-profiles-with-module/cp.tf index 819623c..f600b68 100644 --- a/terraform/cluster-profiles/cp-profiles-with-module/cp.tf +++ b/terraform/cluster-profiles/cp-profiles-with-module/cp.tf @@ -16,6 +16,9 @@ module "primary-cp-1-0-0" { "kubernetes" = "1.27.5" "ubuntu-aws" = "22.04" } + custom_yaml_files = { + "kubernetes" = file("${path.module}/templates/kubernetes.yaml") + } } module "primary-cp-1-0-1" { @@ -33,4 +36,4 @@ module "primary-cp-1-0-1" { "kubernetes" = "1.27.5" "ubuntu-aws" = "22.04" } -} \ No newline at end of file +} diff --git a/terraform/cluster-profiles/cp-profiles-with-module/templates/kubernetes.yaml b/terraform/cluster-profiles/cp-profiles-with-module/templates/kubernetes.yaml new file mode 100644 index 0000000..86efddc --- /dev/null +++ b/terraform/cluster-profiles/cp-profiles-with-module/templates/kubernetes.yaml @@ -0,0 +1,116 @@ +pack: + k8sHardening: True + #CIDR Range for Pods in cluster + # Note : This must not overlap with any of the host or service network + podCIDR: "192.168.0.0/16" + #CIDR notation IP range from which to assign service cluster IPs + # Note : This must not overlap with any IP ranges assigned to nodes for pods. + serviceClusterIpRange: "10.96.0.0/12" + palette: + config: + dashboard: + identityProvider: palette + karl: 2.0.0 + # serviceDomain: "cluster.local" + + # KubeAdm customization for kubernetes hardening. Below config will be ignored if k8sHardening property above is disabled +kubeadmconfig: + apiServer: + extraArgs: + # Note : secure-port flag is used during kubeadm init. Do not change this flag on a running cluster + secure-port: "6443" + anonymous-auth: "true" + profiling: "false" + disable-admission-plugins: "AlwaysAdmit" + default-not-ready-toleration-seconds: "60" + default-unreachable-toleration-seconds: "60" + enable-admission-plugins: "AlwaysPullImages,NamespaceLifecycle,ServiceAccount,NodeRestriction,PodSecurity" + admission-control-config-file: "/etc/kubernetes/pod-security-standard.yaml" + audit-log-path: /var/log/apiserver/audit.log + audit-policy-file: /etc/kubernetes/audit-policy.yaml + audit-log-maxage: "30" + audit-log-maxbackup: "10" + audit-log-maxsize: "100" + authorization-mode: RBAC,Node + tls-cipher-suites: "TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256" + extraVolumes: + - name: audit-log + hostPath: /var/log/apiserver + mountPath: /var/log/apiserver + pathType: DirectoryOrCreate + - name: audit-policy + hostPath: /etc/kubernetes/audit-policy.yaml + mountPath: /etc/kubernetes/audit-policy.yaml + readOnly: true + pathType: File + - name: pod-security-standard + hostPath: /etc/kubernetes/pod-security-standard.yaml + mountPath: /etc/kubernetes/pod-security-standard.yaml + readOnly: true + pathType: File + controllerManager: + extraArgs: + profiling: "false" + terminated-pod-gc-threshold: "25" + use-service-account-credentials: "true" + feature-gates: "RotateKubeletServerCertificate=true" + scheduler: + extraArgs: + profiling: "false" + kubeletExtraArgs: + read-only-port: "0" + event-qps: "0" + feature-gates: "RotateKubeletServerCertificate=true" + protect-kernel-defaults: "true" + rotate-server-certificates: "true" + tls-cipher-suites: "TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256" + files: + - path: hardening/audit-policy.yaml + targetPath: /etc/kubernetes/audit-policy.yaml + targetOwner: "root:root" + targetPermissions: "0600" + - path: hardening/90-kubelet.conf + targetPath: /etc/sysctl.d/90-kubelet.conf + targetOwner: "root:root" + targetPermissions: "0600" + - targetPath: /etc/kubernetes/pod-security-standard.yaml + targetOwner: "root:root" + targetPermissions: "0600" + content: | + apiVersion: apiserver.config.k8s.io/v1 + kind: AdmissionConfiguration + plugins: + - name: PodSecurity + configuration: + apiVersion: pod-security.admission.config.k8s.io/v1 + kind: PodSecurityConfiguration + defaults: + enforce: "baseline" + enforce-version: "v1.26" + audit: "baseline" + audit-version: "v1.26" + warn: "restricted" + warn-version: "v1.26" + audit: "restricted" + audit-version: "v1.26" + exemptions: + # Array of authenticated usernames to exempt. + usernames: [] + # Array of runtime class names to exempt. + runtimeClasses: [] + # Array of namespaces to exempt. + namespaces: [kube-system] + + preKubeadmCommands: + # For enabling 'protect-kernel-defaults' flag to kubelet, kernel parameters changes are required + - 'echo "====> Applying kernel parameters for Kubelet"' + - "sysctl -p /etc/sysctl.d/90-kubelet.conf" + #postKubeadmCommands: + #- 'echo "List of post kubeadm commands to be executed"' + + # Client configuration to add OIDC based authentication flags in kubeconfig + #clientConfig: + #oidc-issuer-url: "{{ .spectro.pack.kubernetes.kubeadmconfig.apiServer.extraArgs.oidc-issuer-url }}" + #oidc-client-id: "{{ .spectro.pack.kubernetes.kubeadmconfig.apiServer.extraArgs.oidc-client-id }}" + #oidc-client-secret: 1gsranjjmdgahm10j8r6m47ejokm9kafvcbhi3d48jlc3rfpprhv + #oidc-extra-scope: profile,email From 03df14c5e2aaa9e3a8bad6fa5f412c53a720cf18 Mon Sep 17 00:00:00 2001 From: Karl Cardenas Date: Wed, 24 Jan 2024 06:55:14 -0700 Subject: [PATCH 2/2] docs: added custom YAML override ability --- .../cluster-profiles/cp-modules/README.md | 59 ++++++++++--------- 1 file changed, 30 insertions(+), 29 deletions(-) diff --git a/terraform/cluster-profiles/cp-modules/README.md b/terraform/cluster-profiles/cp-modules/README.md index 1a935bc..c2386ee 100644 --- a/terraform/cluster-profiles/cp-modules/README.md +++ b/terraform/cluster-profiles/cp-modules/README.md @@ -1,16 +1,16 @@ ## Requirements -| Name | Version | -|------|---------| -| [terraform](#requirement\_terraform) | >= 1.5 | -| [local](#requirement\_local) | >= 2.4.0 | -| [spectrocloud](#requirement\_spectrocloud) | >= 0.17.2 | +| Name | Version | +| --------------------------------------------------------------------------------- | --------- | +| [terraform](#requirement_terraform) | >= 1.5 | +| [local](#requirement_local) | >= 2.4.0 | +| [spectrocloud](#requirement_spectrocloud) | >= 0.17.2 | ## Providers -| Name | Version | -|------|---------| -| [spectrocloud](#provider\_spectrocloud) | 0.17.3 | +| Name | Version | +| --------------------------------------------------------------------------- | ------- | +| [spectrocloud](#provider_spectrocloud) | 0.17.3 | ## Modules @@ -18,30 +18,31 @@ No modules. ## Resources -| Name | Type | -|------|------| -| [spectrocloud_cluster_profile.profile](https://registry.terraform.io/providers/spectrocloud/spectrocloud/latest/docs/resources/cluster_profile) | resource | -| [spectrocloud_pack.generic](https://registry.terraform.io/providers/spectrocloud/spectrocloud/latest/docs/data-sources/pack) | data source | -| [spectrocloud_registry.public_registry](https://registry.terraform.io/providers/spectrocloud/spectrocloud/latest/docs/data-sources/registry) | data source | +| Name | Type | +| ----------------------------------------------------------------------------------------------------------------------------------------------- | ----------- | +| [spectrocloud_cluster_profile.profile](https://registry.terraform.io/providers/spectrocloud/spectrocloud/latest/docs/resources/cluster_profile) | resource | +| [spectrocloud_pack.generic](https://registry.terraform.io/providers/spectrocloud/spectrocloud/latest/docs/data-sources/pack) | data source | +| [spectrocloud_registry.public_registry](https://registry.terraform.io/providers/spectrocloud/spectrocloud/latest/docs/data-sources/registry) | data source | ## Inputs -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [cluster\_profile\_type](#input\_cluster\_profile\_type) | The type of cluster profile. Default value is 'cluster'. | `string` | `"cluster"` | no | -| [context](#input\_context) | The Palette scope to create the cluster profile in. | `string` | `"project"` | no | -| [description](#input\_description) | The description of the cluster profile. | `string` | `""` | no | -| [infrastructure\_provider](#input\_infrastructure\_provider) | The infrastructure provider the cluster profile is for. | `string` | n/a | yes | -| [name](#input\_name) | The name of the cluster profile. The version is appended. | `string` | n/a | yes | -| [pack\_order](#input\_pack\_order) | The Ordered list of pack names. The order must match with the expected layer of a cluster profile. | `list(string)` | n/a | yes | -| [packs](#input\_packs) | A list | `map(string)` | 
{
  "cni-calico": "3.26.1",
  "csi-aws-ebs": "1.22.0",
  "kubernetes": "1.27.5",
  "ubuntu-aws": "22.04"
}
| no | -| [profile\_version](#input\_profile\_version) | The version for the profile | `string` | `"1.0.0"` | no | -| [registry\_name](#input\_registry\_name) | n/a | `string` | `"Public Repo"` | no | -| [tags](#input\_tags) | The default tags to apply to Palette resources | `list(string)` | `[]` | no | +| Name | Description | Type | Default | Required | +| ------------------------------------------------------------------------------------------------------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | -------------- | --------------- | :------: | +| [cluster_profile_type](#input_cluster_profile_type) | The type of cluster profile. Default value is 'cluster'. | `string` | `"cluster"` | no | +| [context](#input_context) | The Palette scope to create the cluster profile in. | `string` | `"project"` | no | +| [custom_yaml_files](#input_custom_yaml_files) | The file path to the custom YAML file that matches the pack name its values are overriding. Use the function 'file() to read the file. Example: 'kubernetes: 'file(path/to/file)' | `map(string)` | `{}` | no | +| [description](#input_description) | The description of the cluster profile. | `string` | `""` | no | +| [infrastructure_provider](#input_infrastructure_provider) | The infrastructure provider the cluster profile is for. | `string` | n/a | yes | +| [name](#input_name) | The name of the cluster profile. The version is appended. | `string` | n/a | yes | +| [pack_order](#input_pack_order) | The Ordered list of pack names. The order must match with the expected layer of a cluster profile. The order goes from highest to lowest. For example. the first item has the highest priority order value assigned and so on. | `list(string)` | n/a | yes | +| [packs](#input_packs) | A list | `map(string)` | `{}` | no | +| [profile_version](#input_profile_version) | The version for the profile | `string` | `"1.0.0"` | no | +| [registry_name](#input_registry_name) | n/a | `string` | `"Public Repo"` | no | +| [tags](#input_tags) | The default tags to apply to Palette resources | `list(string)` | `[]` | no | ## Outputs -| Name | Description | -|------|-------------| -| [export](#output\_export) | n/a | -| [id](#output\_id) | n/a | +| Name | Description | +| ----------------------------------------------------- | ----------- | +| [export](#output_export) | n/a | +| [id](#output_id) | n/a |