Add IPv6 support (#44)

In order to support IPv6 on AWS we had to do a major rework of the
networking configuration of the VPC and ECS clusters.

In case you like to enjoy some some trivia about why IPv6 support is so
weak in AWS, check these links:
* https://news.ycombinator.com/item?id=37608900
* https://github.com/DuckbillGroup/aws-ipv6-gaps
* aws/containers-roadmap#1340

More details follow:
* ECS tasks network mode has been switched to awsvpc, which is the
[official way to support
IPv6](https://aws.amazon.com/about-aws/whats-new/2020/11/amazon-ecs-supports-ipv6-in-awsvpc-networking-mode/).
* Separate the networking stack into private and public. The routing
table of the public network has direct IPv4 and IPv6 routes configured,
while the private network uses NAT to route to the internet.
* For NAT to work we also need to allocate elastic IPs, one for each
availability zone we would like to support.

In order to get the networking to work properly in ECS, we setup the
container host to make use of the public network to have direct access
to the internet, while the ECS container itself uses the private NATed
network. This is the desired configuration because we don't want to
directly expose the container to the internet, but rather map the
container port to the public internet accessible endpoint via load
balancer.

tf/environments/dev/main.tf

@@ -138,7 +138,7 @@ module "oonipg" {
   name                     = "ooni-tier0-postgres"
   aws_region               = var.aws_region
   vpc_id                   = module.network.vpc_id
-  subnet_ids               = module.network.vpc_subnet[*].id
+  subnet_ids               = module.network.vpc_subnet_public[*].id
   db_instance_class        = "db.t3.micro"
   db_storage_type          = "standard"
   db_allocated_storage     = "5"
@@ -251,7 +251,7 @@ module "ooni_backendproxy" {
   source = "../../modules/ooni_backendproxy"
 
   vpc_id     = module.network.vpc_id
-  subnet_ids = module.network.vpc_subnet[*].id
+  subnet_ids = module.network.vpc_subnet_public[*].id
 
   key_name      = module.adm_iam_roles.oonidevops_key_name
   instance_type = "t2.micro"
@@ -270,7 +270,7 @@ module "ooniapi_cluster" {
   name       = "ooniapi-ecs-cluster"
   key_name   = module.adm_iam_roles.oonidevops_key_name
   vpc_id     = module.network.vpc_id
-  subnet_ids = module.network.vpc_subnet[*].id
+  subnet_ids = module.network.vpc_subnet_public[*].id
 
   asg_min     = 2
   asg_max     = 6
@@ -290,11 +290,11 @@ module "oonith_cluster" {
   name       = "oonith-ecs-cluster"
   key_name   = module.adm_iam_roles.oonidevops_key_name
   vpc_id     = module.network.vpc_id
-  subnet_ids = module.network.vpc_subnet[*].id
+  subnet_ids = module.network.vpc_subnet_public[*].id
 
-  asg_min     = 2
-  asg_max     = 6
-  asg_desired = 2
+  asg_min     = 1
+  asg_max     = 4
+  asg_desired = 1
 
   instance_type = "t2.small"
 
@@ -329,8 +329,9 @@ module "ooniapi_ooniprobe" {
   # First run should be set on first run to bootstrap the task definition
   # first_run = true
 
-  vpc_id     = module.network.vpc_id
-  subnet_ids = module.network.vpc_subnet[*].id
+  vpc_id             = module.network.vpc_id
+  public_subnet_ids  = module.network.vpc_subnet_public[*].id
+  private_subnet_ids = module.network.vpc_subnet_private[*].id
 
   service_name             = "ooniprobe"
   default_docker_image_url = "ooni/api-ooniprobe:latest"
@@ -376,8 +377,9 @@ module "ooniapi_oonirun_deployer" {
 module "ooniapi_oonirun" {
   source = "../../modules/ooniapi_service"
 
-  vpc_id     = module.network.vpc_id
-  subnet_ids = module.network.vpc_subnet[*].id
+  vpc_id             = module.network.vpc_id
+  public_subnet_ids  = module.network.vpc_subnet_public[*].id
+  private_subnet_ids = module.network.vpc_subnet_private[*].id
 
   service_name             = "oonirun"
   default_docker_image_url = "ooni/api-oonirun:latest"
@@ -422,8 +424,9 @@ module "ooniapi_ooniauth_deployer" {
 module "ooniapi_ooniauth" {
   source = "../../modules/ooniapi_service"
 
-  vpc_id     = module.network.vpc_id
-  subnet_ids = module.network.vpc_subnet[*].id
+  vpc_id             = module.network.vpc_id
+  public_subnet_ids  = module.network.vpc_subnet_public[*].id
+  private_subnet_ids = module.network.vpc_subnet_private[*].id
 
   service_name             = "ooniauth"
   default_docker_image_url = "ooni/api-ooniauth:latest"
@@ -473,7 +476,7 @@ module "ooniapi_frontend" {
   source = "../../modules/ooniapi_frontend"
 
   vpc_id     = module.network.vpc_id
-  subnet_ids = module.network.vpc_subnet[*].id
+  subnet_ids = module.network.vpc_subnet_public[*].id
 
   oonibackend_proxy_target_group_arn = module.ooni_backendproxy.alb_target_group_id
   ooniapi_oonirun_target_group_arn   = module.ooniapi_oonirun.alb_target_group_id
@@ -513,8 +516,9 @@ module "oonith_oohelperd_deployer" {
 module "oonith_oohelperd" {
   source = "../../modules/oonith_service"
 
-  vpc_id     = module.network.vpc_id
-  subnet_ids = module.network.vpc_subnet[*].id
+  vpc_id             = module.network.vpc_id
+  public_subnet_ids  = module.network.vpc_subnet_public[*].id
+  private_subnet_ids = module.network.vpc_subnet_private[*].id
 
   service_name             = "oohelperd"
   default_docker_image_url = "ooni/oonith-oohelperd:latest"

tf/environments/prod/main.tf

@@ -143,7 +143,7 @@ module "oonipg" {
   name                     = "ooni-tier0-postgres"
   aws_region               = var.aws_region
   vpc_id                   = module.network.vpc_id
-  subnet_ids               = module.network.vpc_subnet[*].id
+  subnet_ids               = module.network.vpc_subnet_public[*].id
   db_instance_class        = "db.t3.micro"
   db_storage_type          = "standard"
   db_allocated_storage     = "5"
@@ -256,7 +256,7 @@ module "ooni_backendproxy" {
   source = "../../modules/ooni_backendproxy"
 
   vpc_id     = module.network.vpc_id
-  subnet_ids = module.network.vpc_subnet[*].id
+  subnet_ids = module.network.vpc_subnet_public[*].id
 
   key_name      = module.adm_iam_roles.oonidevops_key_name
   instance_type = "t2.micro"
@@ -275,11 +275,11 @@ module "ooniapi_cluster" {
   name       = "ooniapi-ecs-cluster"
   key_name   = module.adm_iam_roles.oonidevops_key_name
   vpc_id     = module.network.vpc_id
-  subnet_ids = module.network.vpc_subnet[*].id
+  subnet_ids = module.network.vpc_subnet_public[*].id
 
-  asg_min     = 2
-  asg_max     = 6
-  asg_desired = 2
+  asg_min     = 3
+  asg_max     = 8
+  asg_desired = 3
 
   instance_type = "t2.small"
 
@@ -295,7 +295,7 @@ module "oonith_cluster" {
   name       = "oonith-ecs-cluster"
   key_name   = module.adm_iam_roles.oonidevops_key_name
   vpc_id     = module.network.vpc_id
-  subnet_ids = module.network.vpc_subnet[*].id
+  subnet_ids = module.network.vpc_subnet_public[*].id
 
   asg_min     = 2
   asg_max     = 6
@@ -334,8 +334,9 @@ module "ooniapi_ooniprobe" {
   # First run should be set on first run to bootstrap the task definition
   #first_run = true
 
-  vpc_id     = module.network.vpc_id
-  subnet_ids = module.network.vpc_subnet[*].id
+  vpc_id             = module.network.vpc_id
+  private_subnet_ids = module.network.vpc_subnet_private[*].id
+  public_subnet_ids  = module.network.vpc_subnet_public[*].id
 
   service_name             = "ooniprobe"
   default_docker_image_url = "ooni/api-ooniprobe:latest"
@@ -344,6 +345,8 @@ module "ooniapi_ooniprobe" {
   key_name                 = module.adm_iam_roles.oonidevops_key_name
   ecs_cluster_id           = module.ooniapi_cluster.cluster_id
 
+  service_desired_count = 2
+
   task_secrets = {
     POSTGRESQL_URL              = aws_secretsmanager_secret_version.oonipg_url.arn
     JWT_ENCRYPTION_KEY          = aws_secretsmanager_secret_version.jwt_secret.arn
@@ -382,8 +385,9 @@ module "ooniapi_oonirun" {
   source = "../../modules/ooniapi_service"
   #first_run = true
 
-  vpc_id     = module.network.vpc_id
-  subnet_ids = module.network.vpc_subnet[*].id
+  vpc_id             = module.network.vpc_id
+  private_subnet_ids = module.network.vpc_subnet_private[*].id
+  public_subnet_ids  = module.network.vpc_subnet_public[*].id
 
   service_name             = "oonirun"
   default_docker_image_url = "ooni/api-oonirun:latest"
@@ -392,6 +396,8 @@ module "ooniapi_oonirun" {
   key_name                 = module.adm_iam_roles.oonidevops_key_name
   ecs_cluster_id           = module.ooniapi_cluster.cluster_id
 
+  service_desired_count = 2
+
   task_secrets = {
     POSTGRESQL_URL              = aws_secretsmanager_secret_version.oonipg_url.arn
     JWT_ENCRYPTION_KEY          = aws_secretsmanager_secret_version.jwt_secret.arn
@@ -429,8 +435,9 @@ module "ooniapi_ooniauth" {
   source = "../../modules/ooniapi_service"
   #first_run = true
 
-  vpc_id     = module.network.vpc_id
-  subnet_ids = module.network.vpc_subnet[*].id
+  vpc_id             = module.network.vpc_id
+  private_subnet_ids = module.network.vpc_subnet_private[*].id
+  public_subnet_ids  = module.network.vpc_subnet_public[*].id
 
   service_name             = "ooniauth"
   default_docker_image_url = "ooni/api-ooniauth:latest"
@@ -439,6 +446,8 @@ module "ooniapi_ooniauth" {
   key_name                 = module.adm_iam_roles.oonidevops_key_name
   ecs_cluster_id           = module.ooniapi_cluster.cluster_id
 
+  service_desired_count = 2
+
   task_secrets = {
     POSTGRESQL_URL              = aws_secretsmanager_secret_version.oonipg_url.arn
     JWT_ENCRYPTION_KEY          = aws_secretsmanager_secret_version.jwt_secret.arn
@@ -480,7 +489,7 @@ module "ooniapi_frontend" {
   source = "../../modules/ooniapi_frontend"
 
   vpc_id     = module.network.vpc_id
-  subnet_ids = module.network.vpc_subnet[*].id
+  subnet_ids = module.network.vpc_subnet_public[*].id
 
   oonibackend_proxy_target_group_arn = module.ooni_backendproxy.alb_target_group_id
   ooniapi_oonirun_target_group_arn   = module.ooniapi_oonirun.alb_target_group_id
@@ -521,8 +530,9 @@ module "oonith_oohelperd" {
   source = "../../modules/oonith_service"
   #first_run = true
 
-  vpc_id     = module.network.vpc_id
-  subnet_ids = module.network.vpc_subnet[*].id
+  vpc_id             = module.network.vpc_id
+  private_subnet_ids = module.network.vpc_subnet_private[*].id
+  public_subnet_ids  = module.network.vpc_subnet_public[*].id
 
   service_name             = "oohelperd"
   default_docker_image_url = "ooni/oonith-oohelperd:latest"
@@ -531,11 +541,14 @@ module "oonith_oohelperd" {
   key_name                 = module.adm_iam_roles.oonidevops_key_name
   ecs_cluster_id           = module.oonith_cluster.cluster_id
 
+  service_desired_count = 2
+
   task_secrets = {
     PROMETHEUS_METRICS_PASSWORD = aws_secretsmanager_secret_version.prometheus_metrics_password.arn
   }
 
   alternative_names = {
+    "4.th.ooni.org" = local.dns_root_zone_ooni_org,
     "5.th.ooni.org" = local.dns_root_zone_ooni_org,
     "6.th.ooni.org" = local.dns_root_zone_ooni_org,
   }

tf/modules/ecs_cluster/main.tf

@@ -72,6 +72,7 @@ resource "aws_security_group" "web" {
     cidr_blocks = [
       "0.0.0.0/0",
     ]
+    ipv6_cidr_blocks = ["::/0"]
   }
 
   tags = var.tags
@@ -116,10 +117,11 @@ resource "aws_security_group" "container_host" {
   }
 
   egress {
-    from_port   = 0
-    to_port     = 0
-    protocol    = "-1"
-    cidr_blocks = ["0.0.0.0/0"]
+    from_port        = 0
+    to_port          = 0
+    protocol         = "-1"
+    cidr_blocks      = ["0.0.0.0/0"]
+    ipv6_cidr_blocks = ["::/0"]
   }
 
   tags = var.tags
@@ -147,6 +149,7 @@ resource "aws_launch_template" "container_host" {
   network_interfaces {
     associate_public_ip_address = true
     delete_on_termination       = true
+    ipv6_address_count          = 1
     security_groups = [
       aws_security_group.container_host.id,
     ]

tf/modules/ecs_cluster/variables.tf

@@ -17,6 +17,7 @@ variable "vpc_id" {
 
 variable "subnet_ids" {
   description = "the ids of the subnet of the subnets to deploy the instance into"
+  type        = list(string)
 }
 
 variable "tags" {
@@ -30,13 +31,18 @@ variable "name" {
 }
 
 variable "asg_min" {
-  description = "Min numbers of servers in ASG"
+  description = <<EOT
+                Min numbers of servers in ASG. Keep in mind the ENI restrictions,
+                where small-large have a limit of 3 and micro has a limit of 2.
+                This means that if you plan to run 3 tasks in the ECS cluster with a task count of 2,
+                you need a minimum of 3  in the ASG
+                EOT
   default     = 1
 }
 
 variable "asg_max" {
   description = "Max numbers of servers in ASG"
-  default     = 4
+  default     = 6
 }
 
 variable "asg_desired" {