Merge branch 'main' of github.com:ooni/devops

* 'main' of github.com:ooni/devops:
  fix(scripts): add missing tags column while migrating (#90)
  feat: add oonifindings migration script (#89)
  feat: oonifindings container deployment (#57)
  fix: allow egress routing from private subnet (#88)
  fix: remove nat gateways from dev environment (#87)

scripts/migrate-oonifindings.py

@@ -0,0 +1,93 @@
+"""
+Dump OONI findings for clickhouse into postgres by performing appropriate
+transformations.
+
+To setup run:
+
+pip install psycopg2 clickhouse-driver
+
+Then:
+
+OONI_PG_PASSWORD=XXXX python migrate-oonifindings.py
+"""
+import os
+import json
+
+from clickhouse_driver import Client as Clickhouse
+import psycopg2
+
+
+def dump_oonifindings_clickhouse():
+    client = Clickhouse("localhost")
+
+    rows, cols = client.execute("SELECT * FROM incidents", with_column_types=True)
+    col_names = list(map(lambda x: x[0], cols))
+
+    findings = []
+    for row in rows:
+        d = dict(zip(col_names, row))
+
+        row = {
+            "finding_id": d["id"],
+            "update_time": d["update_time"],
+            "start_time": d["start_time"],
+            "end_time": d["end_time"],
+            "create_time": d["create_time"],
+            "creator_account_id": d["creator_account_id"],
+            "reported_by": d["reported_by"],
+            "title": d["title"],
+            "short_description": d["short_description"],
+            "text": d["text"],
+            "event_type": d["event_type"],
+            "published": d["published"],
+            "deleted": d["deleted"],
+            "email_address": d["email_address"],
+            "country_codes": json.dumps(d["CCs"]),
+            "tags": json.dumps(d["tags"]),
+            "asns": json.dumps(d["ASNs"]),
+            "domains": json.dumps(d["domains"]),
+            "links": json.dumps(d["links"]),
+            "test_names": json.dumps(d["test_names"]),
+        }
+        findings.append(row)
+
+    return findings
+
+
+def insert_findings_postgresql(data_to_insert):
+    db_params = {
+        'dbname': 'oonipg',
+        'user': 'oonipg',
+        'password': os.environ["OONI_PG_PASSWORD"],
+        'host': 'ooni-tier0-postgres'
+    }
+
+    conn = psycopg2.connect(**db_params)
+    cur = conn.cursor()
+
+    col_names = list(data_to_insert[0].keys())
+    col_values = ["%s"]*len(col_names)
+    insert_query = f'INSERT INTO oonifinding ({",".join(col_names)}) VALUES ({",".join(col_values)})'
+
+    insert_count = 0
+    try:
+        for row in data_to_insert:
+            values = [row[cn] for cn in col_names]
+            cur.execute(insert_query, values)
+            insert_count += 1
+        conn.commit()
+        print("Data inserted successfully")
+    except Exception as e:
+        conn.rollback()
+        print(f"Failed after {insert_count} rows at row:")
+        print(row)
+        print(f"An error occurred: {e}")
+        raise e
+    finally:
+        # Close the cursor and connection
+        cur.close()
+        conn.close()
+
+
+valid_links = dump_oonifindings_clickhouse()
+insert_findings_postgresql(valid_links)

tf/environments/dev/main.tf

@@ -107,7 +107,7 @@ module "ansible_inventory" {
 }
 
 module "network" {
-  source = "../../modules/network"
+  source = "../../modules/network_noipv6"
 
   az_count            = var.az_count
   vpc_main_cidr_block = "10.0.0.0/16"
@@ -273,11 +273,11 @@ module "ooniapi_cluster" {
   name       = "ooniapi-ecs-cluster"
   key_name   = module.adm_iam_roles.oonidevops_key_name
   vpc_id     = module.network.vpc_id
-  subnet_ids = module.network.vpc_subnet_public[*].id
+  subnet_ids = module.network.vpc_subnet_private[*].id
 
-  asg_min     = 2
+  asg_min     = 3
   asg_max     = 6
-  asg_desired = 2
+  asg_desired = 3
 
   instance_type = "t3.small"
 
@@ -293,7 +293,7 @@ module "oonith_cluster" {
   name       = "oonith-ecs-cluster"
   key_name   = module.adm_iam_roles.oonidevops_key_name
   vpc_id     = module.network.vpc_id
-  subnet_ids = module.network.vpc_subnet_public[*].id
+  subnet_ids = module.network.vpc_subnet_private[*].id
 
   asg_min     = 1
   asg_max     = 4
@@ -407,6 +407,55 @@ module "ooniapi_oonirun" {
   )
 }
 
+
+#### OONI Findings service
+
+module "ooniapi_oonifindings_deployer" {
+  source = "../../modules/ooniapi_service_deployer"
+
+  service_name            = "oonifindings"
+  repo                    = "ooni/backend"
+  branch_name             = "master"
+  buildspec_path          = "ooniapi/services/oonifindings/buildspec.yml"
+  codestar_connection_arn = aws_codestarconnections_connection.oonidevops.arn
+
+  codepipeline_bucket = aws_s3_bucket.ooniapi_codepipeline_bucket.bucket
+
+  ecs_service_name = module.ooniapi_oonifindings.ecs_service_name
+  ecs_cluster_name = module.ooniapi_cluster.cluster_name
+}
+
+module "ooniapi_oonifindings" {
+  source = "../../modules/ooniapi_service"
+
+  vpc_id             = module.network.vpc_id
+  public_subnet_ids  = module.network.vpc_subnet_public[*].id
+  private_subnet_ids = module.network.vpc_subnet_private[*].id
+
+  service_name             = "oonifindings"
+  default_docker_image_url = "ooni/api-oonifindings:latest"
+  stage                    = local.environment
+  dns_zone_ooni_io         = local.dns_zone_ooni_io
+  key_name                 = module.adm_iam_roles.oonidevops_key_name
+  ecs_cluster_id           = module.ooniapi_cluster.cluster_id
+
+  task_secrets = {
+    POSTGRESQL_URL              = aws_secretsmanager_secret_version.oonipg_url.arn
+    JWT_ENCRYPTION_KEY          = aws_secretsmanager_secret_version.jwt_secret.arn
+    PROMETHEUS_METRICS_PASSWORD = aws_secretsmanager_secret_version.prometheus_metrics_password.arn
+  }
+
+  ooniapi_service_security_groups = [
+    module.ooniapi_cluster.web_security_group_id
+  ]
+
+  tags = merge(
+    local.tags,
+    { Name = "ooni-tier0-oonifindings" }
+  )
+}
+
+
 #### OONI Auth service
 
 module "ooniapi_ooniauth_deployer" {
@@ -485,6 +534,7 @@ module "ooniapi_frontend" {
   ooniapi_oonirun_target_group_arn   = module.ooniapi_oonirun.alb_target_group_id
   ooniapi_ooniauth_target_group_arn  = module.ooniapi_ooniauth.alb_target_group_id
   ooniapi_ooniprobe_target_group_arn = module.ooniapi_ooniprobe.alb_target_group_id
+  ooniapi_oonifindings_target_group_arn = module.ooniapi_oonifindings.alb_target_group_id
 
   ooniapi_service_security_groups = [
     module.ooniapi_cluster.web_security_group_id

tf/modules/ecs_cluster/main.tf

@@ -157,7 +157,6 @@ resource "aws_launch_template" "container_host" {
   network_interfaces {
     associate_public_ip_address = true
     delete_on_termination       = true
-    ipv6_address_count          = 1
     security_groups = [
       aws_security_group.container_host.id,
     ]

tf/modules/network_noipv6/main.tf

@@ -0,0 +1,145 @@
+locals {
+  private_net_offset  = 100
+  cloudhsm_net_offset = 200
+}
+
+resource "aws_vpc" "main" {
+  cidr_block           = var.vpc_main_cidr_block
+  enable_dns_hostnames = true
+  enable_dns_support   = true
+
+  tags = var.tags
+}
+
+resource "aws_subnet" "public" {
+  count = var.az_count
+
+  cidr_block = cidrsubnet(aws_vpc.main.cidr_block, 8, count.index)
+
+  availability_zone       = element(var.aws_availability_zones_available.names, count.index)
+  vpc_id                  = aws_vpc.main.id
+  map_public_ip_on_launch = true
+
+  depends_on = [aws_internet_gateway.gw]
+
+  lifecycle {
+    create_before_destroy = true
+  }
+
+  tags = {
+    Name = "ooni-public-subnet-${count.index}"
+  }
+}
+
+resource "aws_subnet" "private" {
+  count = var.az_count
+
+  cidr_block = cidrsubnet(aws_vpc.main.cidr_block, 8, local.private_net_offset + count.index)
+
+  availability_zone       = element(var.aws_availability_zones_available.names, count.index)
+  vpc_id                  = aws_vpc.main.id
+  map_public_ip_on_launch = true
+
+  depends_on = [aws_internet_gateway.gw]
+
+  lifecycle {
+    create_before_destroy = true
+  }
+
+  tags = {
+    Name = "ooni-private-subnet-${count.index}"
+  }
+}
+
+resource "aws_internet_gateway" "gw" {
+  vpc_id = aws_vpc.main.id
+  tags = {
+    Name = "ooni-internet-gw"
+  }
+}
+
+resource "aws_route_table" "public" {
+  vpc_id = aws_vpc.main.id
+
+  route {
+    cidr_block = "0.0.0.0/0"
+    gateway_id = aws_internet_gateway.gw.id
+  }
+
+  tags = {
+    Name = "ooni-public-route-table"
+  }
+}
+
+resource "aws_route_table_association" "public" {
+  count          = var.az_count
+  subnet_id      = element(aws_subnet.public[*].id, count.index)
+  route_table_id = aws_route_table.public.id
+}
+
+resource "aws_route_table" "private" {
+  vpc_id = aws_vpc.main.id
+
+  route {
+    cidr_block = "0.0.0.0/0"
+    gateway_id = aws_internet_gateway.gw.id
+  }
+
+  tags = {
+    Name = "ooni-private-route-table"
+  }
+}
+
+resource "aws_route_table_association" "private" {
+  count          = var.az_count
+  subnet_id      = element(aws_subnet.private[*].id, count.index)
+  route_table_id = aws_route_table.private.id
+
+  lifecycle {
+    create_before_destroy = true
+  }
+}
+
+locals {
+  cloudhsm_network_count = (var.enable_codesign_network ? 1 : 0) * var.az_count
+}
+
+resource "aws_subnet" "cloudhsm" {
+  count      = local.cloudhsm_network_count
+  cidr_block = cidrsubnet(aws_vpc.main.cidr_block, 8, local.cloudhsm_net_offset + count.index)
+
+  availability_zone       = var.aws_availability_zones_available.names[count.index]
+  vpc_id                  = aws_vpc.main.id
+  map_public_ip_on_launch = false
+
+  depends_on = [aws_internet_gateway.gw]
+
+  lifecycle {
+    create_before_destroy = true
+  }
+
+  tags = {
+    Name = "ooni-cloudhsm-subnet-${count.index}"
+  }
+}
+
+resource "aws_route_table" "cloudhsm" {
+  count = local.cloudhsm_network_count
+
+  vpc_id = aws_vpc.main.id
+
+  route {
+    cidr_block = "0.0.0.0/0"
+    gateway_id = aws_internet_gateway.gw.id
+  }
+
+  tags = {
+    Name = "ooni-cloudhsm-route-table"
+  }
+}
+
+resource "aws_route_table_association" "cloudhsm" {
+  count          = local.cloudhsm_network_count
+  subnet_id      = element(aws_subnet.cloudhsm[*].id, count.index)
+  route_table_id = aws_route_table.cloudhsm[count.index].id
+}

tf/modules/network_noipv6/outputs.tf

@@ -0,0 +1,19 @@
+output "vpc_id" {
+  description = "The ID of the VPC"
+  value       = aws_vpc.main.id
+}
+
+output "vpc_subnet_public" {
+  description = "The value of the public subnet associated to the VPC"
+  value       = aws_subnet.public
+}
+
+output "vpc_subnet_private" {
+  description = "The value of the private subnet associated to the VPC"
+  value       = aws_subnet.private
+}
+
+output "vpc_subnet_cloudhsm" {
+  description = "The value of the cloudhsm subnet associated to the VPC"
+  value       = aws_subnet.cloudhsm
+}

tf/modules/network_noipv6/variables.tf

@@ -0,0 +1,26 @@
+variable "az_count" {
+  description = "Number of AZs to cover in a given AWS region"
+  type        = number
+  default     = "2"
+}
+
+variable "aws_availability_zones_available" {
+  description = "content of data.aws_availability_zones.available"
+}
+
+variable "vpc_main_cidr_block" {
+  description = "the start address of the main VPC cidr"
+  default     = "10.0.0.0/16"
+}
+
+variable "tags" {
+  description = "tags to apply to the resources"
+  default     = {}
+  type        = map(string)
+}
+
+variable "enable_codesign_network" {
+  description = "Enable codesign network"
+  default     = false
+  type        = bool
+}