Support TProxy for gateway-task submodule (#271)

CHANGELOG.md

@@ -15,6 +15,17 @@ BREAKING CHANGES
   - Adds the `CAP_NET_ADMIN` linux capability to the `mesh-init` container when `var.enable_transaparent_proxy` is set to `true`. This is needed to modify iptable rules within the ECS task.
   - `mesh-init` container is run as a `root` user.
   - Assign a UID of `5995` for the `consul-dataplane` container and `5996` for the `health-sync` container. This is done to selectively exclude the traffic flowing through these containers from the redirection rules.
+* Add support for configuring transparent proxy for gateway specific ECS EC2 tasks. Following are the changes made to the `gateway-task` submodule[[GH-271](https://github.com/hashicorp/terraform-aws-consul-ecs/pull/271)]
+  - Adds the following variables
+    - `enable_transparent_proxy` - Defaults to `true`. Fargate based tasks should explicitly pass `false` to avoid validation errors during terraform planning phase.
+    - `enable_consul_dns` - Defaults to `false`. Indicates whether Consul DNS should be configured for this task. Enabling this makes Consul dataplane start up a proxy DNS server that forwards requests to the Consul DNS server. `var.enable_transparent_proxy` should be `true` to enable this setting.
+    - `exclude_inbound_ports` - List of inbound ports to exclude from traffic redirection.
+    - `exclude_outbound_ports` - List of outbound ports to exclude from traffic redirection.
+    - `exclude_outbound_cidrs` - List of additional IP CIDRs to exclude from outbound traffic redirection.
+    - `exclude_outbound_uids` - List of additional process UIDs to exclude from traffic redirection.
+  - Adds the `CAP_NET_ADMIN` linux capability to the `mesh-init` container when `var.enable_transaparent_proxy` is set to `true`. This is needed to modify iptable rules within the ECS task.
+  - `mesh-init` container is run as a `root` user.
+  - Assign a UID of `5995` for the `consul-dataplane` container and `5996` for the `health-sync` container. This is done to selectively exclude the traffic flowing through these containers from the redirection rules.
 
 FEATURES
 * Add support for provisioning API gateways as ECS tasks [[GH-234](https://github.com/hashicorp/terraform-aws-consul-ecs/pull/234)]

modules/gateway-task/config.tf

@@ -2,7 +2,8 @@
 # SPDX-License-Identifier: MPL-2.0
 
 locals {
-  loginExtra = lookup(var.consul_ecs_config, "consulLogin", {})
+  loginExtra  = lookup(var.consul_ecs_config, "consulLogin", {})
+  tProxyExtra = lookup(var.consul_ecs_config, "transparentProxy", {})
 
   consulLogin = var.acls ? {
     enabled = var.acls
@@ -33,6 +34,17 @@ locals {
     var.grpc_config
   )
 
+  transparentProxy = {
+    enabled              = var.enable_transparent_proxy
+    excludeInboundPorts  = var.exclude_inbound_ports
+    excludeOutboundPorts = var.exclude_outbound_ports
+    excludeOutboundCIDRs = var.exclude_outbound_cidrs
+    excludeUIDs          = var.exclude_uids
+    consulDNS = {
+      enabled = var.enable_consul_dns
+    }
+  }
+
   config = {
     consulLogin = merge(local.consulLogin, local.loginExtra)
     gateway = {
@@ -63,6 +75,7 @@ locals {
       http = local.httpSettings
       grpc = local.grpcSettings
     }
+    transparentProxy = merge(local.tProxyExtra, local.transparentProxy)
   }
 
   encoded_config = jsonencode(local.config)

modules/gateway-task/main.tf

@@ -53,6 +53,61 @@ locals {
 
   https_ca_cert_arn = var.consul_https_ca_cert_arn != "" ? var.consul_https_ca_cert_arn : var.consul_ca_cert_arn
   grpc_ca_cert_arn  = var.consul_grpc_ca_cert_arn != "" ? var.consul_grpc_ca_cert_arn : var.consul_ca_cert_arn
+
+  mesh_init_container_definition = {
+    name             = "consul-ecs-mesh-init"
+    image            = var.consul_ecs_image
+    essential        = false
+    logConfiguration = var.log_configuration
+    command          = ["mesh-init"]
+    mountPoints = [
+      local.consul_data_mount_read_write,
+      {
+        sourceVolume  = local.consul_binary_volume_name
+        containerPath = "/bin/consul-inject"
+        readOnly      = true
+      }
+    ]
+    cpu         = 0
+    volumesFrom = []
+    environment = [
+      {
+        name  = "CONSUL_ECS_CONFIG_JSON",
+        value = local.encoded_config
+      }
+    ]
+    linuxParameters = {
+      initProcessEnabled = true
+      capabilities       = var.enable_transparent_proxy ? { add = ["NET_ADMIN"] } : {}
+    }
+    secrets = flatten(
+      concat(
+        var.tls ? [
+          concat(
+            local.https_ca_cert_arn != "" ? [
+              {
+                name      = "CONSUL_HTTPS_CACERT_PEM"
+                valueFrom = local.https_ca_cert_arn
+              },
+            ] : [],
+            local.grpc_ca_cert_arn != "" ? [
+              {
+                name      = "CONSUL_GRPC_CACERT_PEM"
+                valueFrom = local.grpc_ca_cert_arn
+              }
+            ] : [],
+            []
+          )
+        ] : [],
+        []
+      )
+    )
+  }
+
+  # Additional user attribute that needs to be added to run the mesh-init
+  # container with root access.
+  additional_user_attr                     = var.enable_transparent_proxy ? { user = "0" } : {}
+  finalized_mesh_init_container_definition = merge(local.mesh_init_container_definition, local.additional_user_attr)
 }
 
 resource "aws_ecs_task_definition" "this" {
@@ -152,58 +207,12 @@ resource "aws_ecs_task_definition" "this" {
     flatten(
       concat(
         [
-          {
-            name             = "consul-ecs-mesh-init"
-            image            = var.consul_ecs_image
-            essential        = false
-            logConfiguration = var.log_configuration
-            command          = ["mesh-init"]
-            mountPoints = [
-              local.consul_data_mount_read_write,
-              {
-                sourceVolume  = local.consul_binary_volume_name
-                containerPath = "/bin/consul-inject"
-                readOnly      = true
-              }
-            ]
-            cpu         = 0
-            volumesFrom = []
-            environment = [
-              {
-                name  = "CONSUL_ECS_CONFIG_JSON",
-                value = local.encoded_config
-              }
-            ]
-            linuxParameters = {
-              initProcessEnabled = true
-            }
-            secrets = flatten(
-              concat(
-                var.tls ? [
-                  concat(
-                    local.https_ca_cert_arn != "" ? [
-                      {
-                        name      = "CONSUL_HTTPS_CACERT_PEM"
-                        valueFrom = local.https_ca_cert_arn
-                      },
-                    ] : [],
-                    local.grpc_ca_cert_arn != "" ? [
-                      {
-                        name      = "CONSUL_GRPC_CACERT_PEM"
-                        valueFrom = local.grpc_ca_cert_arn
-                      }
-                    ] : [],
-                    []
-                  )
-                ] : [],
-                []
-              )
-            )
-          },
+          local.finalized_mesh_init_container_definition,
           {
             name             = "consul-dataplane"
             image            = var.consul_dataplane_image
             essential        = true
+            user             = "5995"
             logConfiguration = var.log_configuration
             entryPoint       = ["/consul/consul-ecs", "envoy-entrypoint"]
             command          = ["consul-dataplane", "-config-file", "/consul/consul-dataplane.json"] # consul-ecs-mesh-init dumps the dataplane's config into consul-dataplane.json

modules/gateway-task/validation.tf

@@ -10,5 +10,7 @@ locals {
   create_xor_modify_security_group = var.lb_create_security_group && var.lb_modify_security_group ? file("ERROR: Only one of lb_create_security_group or lb_modify_security_group may be true") : null
   require_sg_id_for_modify         = var.lb_modify_security_group && var.lb_modify_security_group_id == "" ? file("ERROR: lb_modify_security_group_id is required when lb_modify_security_group is true") : null
 
-  custom_lb_config_check = var.lb_enabled && length(var.custom_load_balancer_config) > 0 ? file("ERROR: custom_load_balancer_config must only be supplied when var.lb_enabled is false") : null
+  custom_lb_config_check                     = var.lb_enabled && length(var.custom_load_balancer_config) > 0 ? file("ERROR: custom_load_balancer_config must only be supplied when var.lb_enabled is false") : null
+  require_ec2_compability_for_tproxy_support = var.enable_transparent_proxy && (length(var.requires_compatibilities) != 1 || var.requires_compatibilities[0] != "EC2") ? file("ERROR: transparent proxy is supported only in ECS EC2 mode") : null
+  require_tproxy_enabled_for_consul_dns      = var.enable_consul_dns && !var.enable_transparent_proxy ? file("ERROR: var.enable_transparent_proxy must be set to true for Consul DNS to be enabled") : null
 }

modules/gateway-task/variables.tf

@@ -314,10 +314,10 @@ variable "consul_ecs_config" {
   EOT
 
   validation {
-    error_message = "Only 'consulLogin' field is allowed in consul_ecs_config."
+    error_message = "Only 'consulLogin' and 'transparentProxy' fields are allowed in consul_ecs_config."
     condition = alltrue([
       for key in keys(var.consul_ecs_config) :
-      contains(["consulLogin"], key)
+      contains(["consulLogin", "transparentProxy"], key)
     ])
   }
 
@@ -339,6 +339,23 @@ variable "consul_ecs_config" {
     ]))
   }
 
+  validation {
+    error_message = "Only the 'enabled','excludeInboundPorts','excludeOutboundPorts','excludeOutboundCIDRs', 'excludeUIDs' and 'consulDNS' fields are allowed in consul_ecs_config.transparentProxy."
+    condition = alltrue([
+      for key in keys(lookup(var.consul_ecs_config, "transparentProxy", {})) :
+      contains(["enabled", "excludeInboundPorts", "excludeOutboundPorts", "excludeUIDs", "excludeOutboundCIDRs", "consulDNS"], key)
+    ])
+  }
+
+  validation {
+    error_message = "Only the 'enabled' field is allowed in consul_ecs_config.transparentProxy.consulDNS."
+    condition = alltrue(flatten([
+      for tproxy in [lookup(var.consul_ecs_config, "transparentProxy", {})] : [
+        for key in keys(lookup(tproxy, "consulDNS", {})) :
+        contains(["enabled"], key)
+      ]
+    ]))
+  }
 }
 
 variable "http_config" {
@@ -383,3 +400,38 @@ variable "volumes" {
   default     = []
 }
 
+variable "enable_transparent_proxy" {
+  description = "Whether transparent proxy should be enabled for this task. Defaults to true"
+  type        = bool
+  default     = true
+}
+
+variable "enable_consul_dns" {
+  description = "Whether Consul DNS should be configured for this task. This rewrites the /etc/resolv.conf file to point to the Consul dataplane's DNS server as the primary nameserver. Defaults to false"
+  type        = bool
+  default     = false
+}
+
+variable "exclude_inbound_ports" {
+  description = "List of inbound ports to exclude from traffic redirection."
+  type        = list(number)
+  default     = []
+}
+
+variable "exclude_outbound_ports" {
+  description = "List of outbound ports to exclude from traffic redirection."
+  type        = list(number)
+  default     = []
+}
+
+variable "exclude_outbound_cidrs" {
+  description = "List of additional IP CIDRs to exclude from outbound traffic redirection."
+  type        = list(string)
+  default     = []
+}
+
+variable "exclude_uids" {
+  description = "List of additional UIDs to exclude from outbound traffic redirection."
+  type        = list(string)
+  default     = []
+}

test/acceptance/tests/validation/terraform/api-gateway-validate/main.tf

@@ -49,4 +49,5 @@ module "test_gateway" {
   lb_subnets                  = var.lb_subnets
   custom_load_balancer_config = var.custom_lb_config
   lb_create_security_group    = var.lb_enabled
+  enable_transparent_proxy    = false
 }

test/acceptance/tests/validation/terraform/mesh-gateway-validate/main.tf

@@ -71,6 +71,7 @@ module "test_gateway" {
   ecs_cluster_arn                    = "cluster"
   subnets                            = ["subnets"]
   security_groups                    = var.security_groups
+  enable_transparent_proxy           = false
   kind                               = var.kind
   gateway_count                      = var.gateway_count
   consul_server_hosts                = "localhost:8500"

test/acceptance/tests/validation/terraform/terminating-gateway-validate/main.tf

@@ -20,6 +20,7 @@ module "test_gateway" {
   ecs_cluster_arn          = "cluster"
   subnets                  = ["subnets"]
   kind                     = var.kind
+  enable_transparent_proxy = false
   gateway_count            = var.gateway_count
   consul_server_hosts      = "localhost:8500"
   tls                      = true

test/acceptance/tests/validation/terraform/tproxy-gateway-validate/main.tf

@@ -0,0 +1,37 @@
+# Copyright (c) HashiCorp, Inc.
+# SPDX-License-Identifier: MPL-2.0
+
+provider "aws" {
+  region = "us-west-2"
+}
+
+variable "requires_compatibilities" {
+  description = "Set of launch types required by the task."
+  type        = list(string)
+}
+
+variable "enable_transparent_proxy" {
+  description = "Whether to enable or disable transparent proxy for the task"
+  type        = bool
+  default     = true
+}
+
+variable "enable_consul_dns" {
+  description = "Whether to enable or disable Consul DNS for the task"
+  type        = bool
+  default     = true
+}
+
+module "test_gateway" {
+  source                   = "../../../../../../modules/gateway-task"
+  family                   = "family"
+  subnets                  = ["test-subnet"]
+  kind                     = "terminating-gateway"
+  ecs_cluster_arn          = "test-cluster-arn"
+  consul_server_hosts      = "consul.dc1.host"
+  lb_create_security_group = false
+  security_groups          = ["test-security-group"]
+  requires_compatibilities = var.requires_compatibilities
+  enable_transparent_proxy = var.enable_transparent_proxy
+  enable_consul_dns        = var.enable_consul_dns
+}

test/acceptance/tests/validation/validation_test.go

@@ -1154,3 +1154,68 @@ func TestValidation_TProxy(t *testing.T) {
 		})
 	}
 }
+
+func TestValidation_TProxy_Gateway(t *testing.T) {
+	t.Parallel()
+
+	cases := map[string]struct {
+		requiresCompatibilities []string
+		disableTProxy           bool
+		error                   bool
+		errorStr                string
+	}{
+		"only EC2": {
+			requiresCompatibilities: []string{"EC2"},
+		},
+		"only Fargate": {
+			requiresCompatibilities: []string{"FARGATE"},
+			error:                   true,
+			errorStr:                "transparent proxy is supported only in ECS EC2 mode.",
+		},
+		"both Fargate and EC2": {
+			requiresCompatibilities: []string{"FARGATE", "EC2"},
+			error:                   true,
+			errorStr:                "transparent proxy is supported only in ECS EC2 mode.",
+		},
+		"Consul DNS does not work without enabling tproxy": {
+			requiresCompatibilities: []string{"FARGATE", "EC2"},
+			disableTProxy:           true,
+			error:                   true,
+			errorStr:                "var.enable_transparent_proxy must be set to true for Consul DNS to be enabled.",
+		},
+	}
+
+	terraformOptions := &terraform.Options{
+		TerraformDir: "./terraform/tproxy-gateway-validate",
+		NoColor:      true,
+	}
+	terraform.Init(t, terraformOptions)
+
+	for name, c := range cases {
+		c := c
+
+		t.Run(name, func(t *testing.T) {
+			t.Parallel()
+
+			vars := map[string]interface{}{
+				"requires_compatibilities": c.requiresCompatibilities,
+			}
+			if c.disableTProxy {
+				vars["enable_transparent_proxy"] = false
+			}
+
+			out, err := terraform.PlanE(t, &terraform.Options{
+				TerraformDir: terraformOptions.TerraformDir,
+				NoColor:      true,
+				Vars:         vars,
+			})
+
+			if c.error {
+				require.Error(t, err)
+				require.Regexp(t, c.errorStr, out)
+			} else {
+				require.NoError(t, err)
+			}
+		})
+	}
+}