From 17d35069b30ae6d7d92e4722c3fec8395d196680 Mon Sep 17 00:00:00 2001 From: Relk Li Date: Mon, 8 Jan 2024 15:02:28 +0800 Subject: [PATCH] FEATURE: support aws network policy controller and agent (#182) - feat: support aws network policy controller and agent - revert: replace docker to nerdctl --- docs/variables/aws/elastikube.md | 3 ++- docs/variables/aws/kube-worker.md | 8 ++++---- examples/kubernetes-cluster-aws-vpc/main.tf | 1 + examples/kubernetes-cluster-aws-vpc/variables.tf | 6 ++++++ modules/aws/elastikube/etcd.tf | 1 - modules/aws/elastikube/main.tf | 9 +++++---- modules/aws/elastikube/variables.tf | 6 ++++++ modules/aws/kube-etcd/ignition.tf | 3 +-- modules/aws/kube-master/ignition.tf | 3 ++- modules/aws/kube-master/variables.tf | 6 ++++++ modules/aws/kube-worker/ignition.tf | 2 +- 11 files changed, 34 insertions(+), 14 deletions(-) diff --git a/docs/variables/aws/elastikube.md b/docs/variables/aws/elastikube.md index 7280737..4d9fd17 100644 --- a/docs/variables/aws/elastikube.md +++ b/docs/variables/aws/elastikube.md @@ -64,6 +64,7 @@ This document gives an overview of variables used in the AWS platform of the ela | [enable\_eni\_prefix](#input\_enable\_eni\_prefix) | (Optional) assign prefix to AWS EC2 network interface | `bool` | `true` | no | | [enable\_iam\_auth](#input\_enable\_iam\_auth) | Enable AWS IAM authenticator or not. | `bool` | `false` | no | | [enable\_irsa](#input\_enable\_irsa) | (Optional) Enable AWS IAM role service account or not | `bool` | `false` | no | +| [enable\_network\_policy](#input\_enable\_network\_policy) | (Optional) [AWS VPC CNI] Enable AWS Network Policy Agent. | `bool` | `false` | no | | [endpoint\_public\_access](#input\_endpoint\_public\_access) | (Optional) kubernetes apiserver endpoint | `bool` | `false` | no | | [etcd\_instance\_config](#input\_etcd\_instance\_config) | (Optional) Desired etcd nodes configuration. | 
object({
    count              = number
    image_id           = string
    ec2_type           = string
    root_volume_size   = number
    data_volume_size   = number
    data_device_name   = string
    data_device_rename = string
    data_path          = string
  })
| 
{
  "count": 1,
  "data_device_name": "/dev/sdf",
  "data_device_rename": "/dev/nvme1n1",
  "data_path": "/var/lib/etcd",
  "data_volume_size": 100,
  "ec2_type": "t3.medium",
  "image_id": "ami-0b75e2f157200889f",
  "root_volume_size": 40
}
| no | | [etcd\_instance\_volume\_config](#input\_etcd\_instance\_volume\_config) | n/a | 
object({
    root = object({
      type       = string
      iops       = number
      throughput = number
    })
    data = object({
      type       = string
      iops       = number
      throughput = number
    })
  })
| 
{
  "data": {
    "iops": 0,
    "throughput": 0,
    "type": "gp2"
  },
  "root": {
    "iops": 0,
    "throughput": 0,
    "type": "gp2"
  }
}
| no | @@ -87,7 +88,7 @@ This document gives an overview of variables used in the AWS platform of the ela | [kubernetes\_version](#input\_kubernetes\_version) | Desired Kubernetes version. | `string` | `"v1.27.7"` | no | | [lb\_security\_group\_ids](#input\_lb\_security\_group\_ids) | (Optional) List of security group IDs for the cross-account elastic network interfaces
to use to allow communication to the kubernetes api server load balancer. | `list(string)` | `[]` | no | | [log\_level](#input\_log\_level) | Log level and verbosity of each components | 
object({
    aws_cloud_controller_manager = optional(string, "2")       # 2: Info, 3: Extended Info, 4: Debug, 5: Trace
    aws_vpc_cni                  = optional(string, "DEBUG")   # DEBUG, INFO, WARN, ERROR, FATAL
    containerd                   = optional(string, "info")    # trace, debug, info, warn, error, fatal, panic
    cilium_cni                   = optional(string, "DEBUG")   # DEBUG: enable debug logging, INFO: disable debug logging
    docker                       = optional(string, "info")    # debug, info, warn, error, fatal
    etcd                         = optional(string, "info")    # debug, info, warn, error, panic, fatal
    kube_apiserver               = optional(string, "2")       # 2: Info, 3: Extended Info, 4: Debug, 5: Trace
    kube_controller_manager      = optional(string, "2")       # 2: Info, 3: Extended Info, 4: Debug, 5: Trace
    kube_scheduler               = optional(string, "2")       # 2: Info, 3: Extended Info, 4: Debug, 5: Trace
    kube_proxy                   = optional(string, "2")       # 2: Info, 3: Extended Info, 4: Debug, 5: Trace
    kubelet                      = optional(string, "2")       # 2: Info, 3: Extended Info, 4: Debug, 5: Trace
    systemd_networkd             = optional(string, "warning") # emerg, alert, crit, err, warning, notice, info, debug
  })
| 
{
  "aws_cloud_controller_manager": "2",
  "aws_vpc_cni": "DEBUG",
  "cilium_cni": "DEBUG",
  "containerd": "info",
  "docker": "info",
  "etcd": "info",
  "kube_apiserver": "2",
  "kube_controller_manager": "2",
  "kube_proxy": "2",
  "kube_scheduler": "2",
  "kubelet": "2",
  "systemd_networkd": "warning"
}
| no | -| [master\_instance\_config](#input\_master\_instance\_config) | (Optional) Desired master nodes configuration. | 
object({
    count            = number
    image_id         = string
    ec2_type         = list(string)
    root_volume_iops = number
    root_volume_size = number
    root_volume_type = string

    default_cooldown          = number
    health_check_grace_period = number

    suspended_processes = list(string)

    instance_warmup        = number
    min_healthy_percentage = number

    on_demand_base_capacity                  = number
    on_demand_percentage_above_base_capacity = number
    spot_instance_pools                      = number
    spot_allocation_strategy                 = string
  })
| 
{
  "count": 1,
  "default_cooldown": 300,
  "ec2_type": [
    "t3.medium",
    "t2.medium"
  ],
  "health_check_grace_period": 300,
  "image_id": "ami-0b75e2f157200889f",
  "instance_warmup": 30,
  "min_healthy_percentage": 100,
  "on_demand_base_capacity": 0,
  "on_demand_percentage_above_base_capacity": 100,
  "root_volume_iops": 100,
  "root_volume_size": 256,
  "root_volume_type": "gp2",
  "spot_allocation_strategy": "lowest-price",
  "spot_instance_pools": 1,
  "suspended_processes": []
}
| no | +| [master\_instance\_config](#input\_master\_instance\_config) | (Optional) Desired master nodes configuration. | 
object({
    count            = number
    image_id         = string
    ec2_type         = list(string)
    root_volume_iops = number
    root_volume_size = number
    root_volume_type = string

    default_cooldown          = number
    health_check_grace_period = number

    suspended_processes = list(string)

    instance_refresh       = bool
    instance_warmup        = number
    min_healthy_percentage = number

    on_demand_base_capacity                  = number
    on_demand_percentage_above_base_capacity = number
    spot_instance_pools                      = number
    spot_allocation_strategy                 = string
  })
| 
{
  "count": 1,
  "default_cooldown": 300,
  "ec2_type": [
    "t3.medium",
    "t2.medium"
  ],
  "health_check_grace_period": 300,
  "image_id": "ami-0b75e2f157200889f",
  "instance_refresh": false,
  "instance_warmup": 30,
  "min_healthy_percentage": 100,
  "on_demand_base_capacity": 0,
  "on_demand_percentage_above_base_capacity": 100,
  "root_volume_iops": 100,
  "root_volume_size": 256,
  "root_volume_type": "gp2",
  "spot_allocation_strategy": "lowest-price",
  "spot_instance_pools": 1,
  "suspended_processes": []
}
| no | | [master\_instance\_spot\_max\_price](#input\_master\_instance\_spot\_max\_price) | Desired master nodes spot maximum price, default is the on-demand price. | `string` | `""` | no | | [max\_pods](#input\_max\_pods) | (Optional) the max pod number in the node when enable eni prefix | `string` | `"110"` | no | | [name](#input\_name) | (Required) Name of the cluster. | `string` | n/a | yes | diff --git a/docs/variables/aws/kube-worker.md b/docs/variables/aws/kube-worker.md index cee1ee5..d3b7155 100644 --- a/docs/variables/aws/kube-worker.md +++ b/docs/variables/aws/kube-worker.md @@ -12,8 +12,8 @@ This document gives an overview of variables used in the AWS platform of the kub | Name | Version | |------|---------| -| [aws](#provider\_aws) | n/a | -| [ignition](#provider\_ignition) | 2.1.2 | +| [aws](#provider\_aws) | 5.29.0 | +| [ignition](#provider\_ignition) | 1.2.1 | ## Modules @@ -22,7 +22,7 @@ This document gives an overview of variables used in the AWS platform of the kub | [ignition\_containerd](#module\_ignition\_containerd) | git::ssh://git@github.com/getamis/terraform-ignition-reinforcements//modules/containerd | v1.27.4.0 | | [ignition\_docker](#module\_ignition\_docker) | git::ssh://git@github.com/getamis/terraform-ignition-reinforcements//modules/docker | v1.27.4.0 | | [ignition\_ecr\_credentail\_provider](#module\_ignition\_ecr\_credentail\_provider) | git::ssh://git@github.com/getamis/terraform-ignition-reinforcements//modules/ecr-credential-provider | v1.27.4.0 | -| [ignition\_kubelet](#module\_ignition\_kubelet) | git::ssh://git@github.com/getamis/terraform-ignition-kubernetes//modules/kubelet | v1.27.7.0 | +| [ignition\_kubelet](#module\_ignition\_kubelet) | git::ssh://git@github.com/getamis/terraform-ignition-kubernetes//modules/kubelet | v1.27.7.1 | | [ignition\_locksmithd](#module\_ignition\_locksmithd) | git::ssh://git@github.com/getamis/terraform-ignition-reinforcements//modules/locksmithd | v1.27.4.0 | | [ignition\_sshd](#module\_ignition\_sshd) | git::ssh://git@github.com/getamis/terraform-ignition-reinforcements//modules/sshd | v1.27.4.0 | | [ignition\_systemd\_networkd](#module\_ignition\_systemd\_networkd) | git::ssh://git@github.com/getamis/terraform-ignition-reinforcements//modules/systemd-networkd | v1.27.4.0 | @@ -69,7 +69,7 @@ This document gives an overview of variables used in the AWS platform of the kub | [extra\_ignition\_file\_ids](#input\_extra\_ignition\_file\_ids) | Additional ignition file IDs. See https://www.terraform.io/docs/providers/ignition/d/file.html for more details. | `list(string)` | `[]` | no | | [extra\_ignition\_systemd\_unit\_ids](#input\_extra\_ignition\_systemd\_unit\_ids) | Additional ignition systemd unit IDs. See https://www.terraform.io/docs/providers/ignition/d/systemd_unit.html for more details. | `list(string)` | `[]` | no | | [extra\_tags](#input\_extra\_tags) | Extra AWS tags to be applied to created resources. | `map(string)` | `{}` | no | -| [instance\_config](#input\_instance\_config) | Desired worker nodes configuration. | 
object({
    count            = number
    max_count        = number
    name             = string
    image_id         = string
    ec2_type         = list(string)
    root_volume_iops = number
    root_volume_size = number
    root_volume_type = string

    default_cooldown          = number
    health_check_grace_period = number

    suspended_processes = list(string)

    instance_warmup        = number
    min_healthy_percentage = number

    on_demand_base_capacity                  = number
    on_demand_percentage_above_base_capacity = number
    spot_instance_pools                      = number
    spot_allocation_strategy                 = string
  })
| n/a | yes | +| [instance\_config](#input\_instance\_config) | Desired worker nodes configuration. | 
object({
    count            = number
    max_count        = number
    name             = string
    image_id         = string
    ec2_type         = list(string)
    root_volume_iops = number
    root_volume_size = number
    root_volume_type = string

    default_cooldown          = number
    health_check_grace_period = number

    suspended_processes = list(string)

    instance_refresh       = bool
    instance_warmup        = number
    min_healthy_percentage = number

    on_demand_base_capacity                  = number
    on_demand_percentage_above_base_capacity = number
    spot_instance_pools                      = number
    spot_allocation_strategy                 = string
  })
| n/a | yes | | [instance\_spot\_max\_price](#input\_instance\_spot\_max\_price) | Desired worker nodes spot maximum price, default is the on-demand price. | `string` | `""` | no | | [kubelet\_config](#input\_kubelet\_config) | The configuration of kubelet. The variables need to follow https://github.com/kubernetes/kubernetes/blob/master/staging/src/k8s.io/kubelet/config/v1beta1/types.go. Do not use underline. | `map` | `{}` | no | | [kubelet\_flags](#input\_kubelet\_flags) | The flags of kubelet. The variables need to follow https://kubernetes.io/docs/reference/command-line-tools-reference/kubelet/. Do not use underline. | `map(string)` | `{}` | no | diff --git a/examples/kubernetes-cluster-aws-vpc/main.tf b/examples/kubernetes-cluster-aws-vpc/main.tf index bfb91ea..9356ae5 100644 --- a/examples/kubernetes-cluster-aws-vpc/main.tf +++ b/examples/kubernetes-cluster-aws-vpc/main.tf @@ -100,6 +100,7 @@ module "master" { enable_eni_prefix = var.enable_eni_prefix enable_asg_life_cycle = var.enable_asg_life_cycle external_snat = var.external_snat + enable_network_policy = var.enable_network_policy debug_mode = var.debug_mode log_level = var.log_level diff --git a/examples/kubernetes-cluster-aws-vpc/variables.tf b/examples/kubernetes-cluster-aws-vpc/variables.tf index d4fc2e7..422b601 100644 --- a/examples/kubernetes-cluster-aws-vpc/variables.tf +++ b/examples/kubernetes-cluster-aws-vpc/variables.tf @@ -86,6 +86,12 @@ variable "external_snat" { default = true # AWS NAT Gateway is enabled by network module } +variable "enable_network_policy" { + description = "(Optional) [AWS VPC CNI] Enable AWS Network Policy Agent." + type = bool + default = true +} + variable "log_level" { description = "Log level and verbosity of each components" type = any diff --git a/modules/aws/elastikube/etcd.tf b/modules/aws/elastikube/etcd.tf index 8ff0e22..2fc95d7 100644 --- a/modules/aws/elastikube/etcd.tf +++ b/modules/aws/elastikube/etcd.tf @@ -7,7 +7,6 @@ module "etcd" { allowed_etcd_mgmt_cidr = var.allowed_etcd_mgmt_cidr instance_config = var.etcd_instance_config containers = var.override_containers - binaries = var.override_binaries instance_volume_config = var.etcd_instance_volume_config diff --git a/modules/aws/elastikube/main.tf b/modules/aws/elastikube/main.tf index e65a02a..ad2577a 100644 --- a/modules/aws/elastikube/main.tf +++ b/modules/aws/elastikube/main.tf @@ -59,10 +59,11 @@ module "master" { enable_irsa = var.enable_irsa oidc_config = var.irsa_oidc_config - enable_eni_prefix = var.enable_eni_prefix - annotate_pod_ip = var.annotate_pod_ip - external_snat = var.external_snat - max_pods = var.max_pods + enable_eni_prefix = var.enable_eni_prefix + annotate_pod_ip = var.annotate_pod_ip + external_snat = var.external_snat + enable_network_policy = var.enable_network_policy + max_pods = var.max_pods enable_asg_life_cycle = var.enable_asg_life_cycle diff --git a/modules/aws/elastikube/variables.tf b/modules/aws/elastikube/variables.tf index b4f83af..3b270be 100644 --- a/modules/aws/elastikube/variables.tf +++ b/modules/aws/elastikube/variables.tf @@ -272,6 +272,12 @@ variable "external_snat" { default = false } +variable "enable_network_policy" { + description = "(Optional) [AWS VPC CNI] Enable AWS Network Policy Agent." + type = bool + default = false +} + variable "max_pods" { description = "(Optional) the max pod number in the node when enable eni prefix" type = string diff --git a/modules/aws/kube-etcd/ignition.tf b/modules/aws/kube-etcd/ignition.tf index d08a538..5ce8c70 100644 --- a/modules/aws/kube-etcd/ignition.tf +++ b/modules/aws/kube-etcd/ignition.tf @@ -32,11 +32,10 @@ module "ignition_sshd" { } module "ignition_etcd" { - source = "git::ssh://git@github.com/getamis/terraform-ignition-etcd?ref=v1.27.4.1" + source = "git::ssh://git@github.com/getamis/terraform-ignition-etcd?ref=v1.27.4.2" name = var.name containers = var.containers - binaries = var.binaries discovery_service_srv = local.discovery_service client_port = local.client_port peer_port = local.peer_port diff --git a/modules/aws/kube-master/ignition.tf b/modules/aws/kube-master/ignition.tf index f04229d..e069305 100644 --- a/modules/aws/kube-master/ignition.tf +++ b/modules/aws/kube-master/ignition.tf @@ -12,7 +12,7 @@ resource "random_password" "encryption_secret" { } module "ignition_kubernetes" { - source = "git::ssh://git@github.com/getamis/terraform-ignition-kubernetes//?ref=v1.27.7.0" + source = "git::ssh://git@github.com/getamis/terraform-ignition-kubernetes//?ref=v1.27.7.1" binaries = var.binaries containers = var.containers @@ -58,6 +58,7 @@ module "ignition_kubernetes" { enable_eni_prefix = var.enable_eni_prefix annotate_pod_ip = var.annotate_pod_ip external_snat = var.external_snat + enable_network_policy = var.enable_network_policy max_pods = var.max_pods log_level = var.log_level diff --git a/modules/aws/kube-master/variables.tf b/modules/aws/kube-master/variables.tf index d610c50..416be29 100644 --- a/modules/aws/kube-master/variables.tf +++ b/modules/aws/kube-master/variables.tf @@ -269,6 +269,12 @@ variable "external_snat" { default = false } +variable "enable_network_policy" { + description = "(Optional) [AWS VPC CNI] Enable AWS Network Policy Agent." + type = bool + default = false +} + variable "max_pods" { description = "(Optional) the max pod number in the node when enable eni prefix" type = string diff --git a/modules/aws/kube-worker/ignition.tf b/modules/aws/kube-worker/ignition.tf index 730426e..5d4b1d8 100644 --- a/modules/aws/kube-worker/ignition.tf +++ b/modules/aws/kube-worker/ignition.tf @@ -52,7 +52,7 @@ data "aws_s3_object" "bootstrapping_kubeconfig" { } module "ignition_kubelet" { - source = "git::ssh://git@github.com/getamis/terraform-ignition-kubernetes//modules/kubelet?ref=v1.27.7.0" + source = "git::ssh://git@github.com/getamis/terraform-ignition-kubernetes//modules/kubelet?ref=v1.27.7.1" binaries = var.binaries containers = var.containers