From bc342ce116a005eba47a8e4aa8b8324d79a0bb48 Mon Sep 17 00:00:00 2001 From: Michael Barroco Date: Thu, 23 Nov 2023 00:41:40 +0100 Subject: [PATCH 01/28] 874: Automate creation of the infrastructure for the first ci environment --- .github/workflows/ci.yml | 24 ++ .gitignore | 5 +- deploy/operations/Dockerfile | 22 ++ deploy/operations/ci/aws-1/main.tf | 43 ++++ deploy/operations/ci/aws-1/output.tf | 15 ++ deploy/operations/ci/aws-1/terraform.tfvars | 27 ++ deploy/operations/ci/aws-1/test.sh | 25 ++ deploy/operations/ci/aws-1/variables.tf | 260 ++++++++++++++++++++ deploy/operations/docker-compose.yaml | 14 ++ 9 files changed, 434 insertions(+), 1 deletion(-) create mode 100644 deploy/operations/Dockerfile create mode 100644 deploy/operations/ci/aws-1/main.tf create mode 100644 deploy/operations/ci/aws-1/output.tf create mode 100644 deploy/operations/ci/aws-1/terraform.tfvars create mode 100755 deploy/operations/ci/aws-1/test.sh create mode 100644 deploy/operations/ci/aws-1/variables.tf create mode 100644 deploy/operations/docker-compose.yaml diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1e8f98bd8..b6b53f26e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -62,3 +62,27 @@ jobs: run: make probe-locally - name: Bring down local DSS instance run: make down-locally + + deploy-tests: + name: Deploy tests + runs-on: ubunutu-latest + concurrency: 1 + steps: + - name: Job information + run: | + echo "Job information" + echo "Trigger: ${{ github.event_name }}" + echo "Host: ${{ runner.os }}" + echo "Repository: ${{ github.repository }}" + echo "Branch: ${{ github.ref }}" + docker images + - name: Checkout + uses: actions/checkout@v2 + with: + submodules: true + - name: Test Scenario AWS-1 + env: + AWS_ACCESS_KEY_ID: ${{ secrets.CI_AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.CI_AWS_SECRET_ACCESS_KEY }} + COMPOSE_PROFILES: aws-1 + run: docker-compose up -d \ No newline at end of file diff --git a/.gitignore b/.gitignore index 9268591f8..a24e88084 100644 --- a/.gitignore +++ b/.gitignore @@ -128,4 +128,7 @@ test/e2e_test_result go # vscode files -.vscode \ No newline at end of file +.vscode + +# terraform +.terraform* \ No newline at end of file diff --git a/deploy/operations/Dockerfile b/deploy/operations/Dockerfile new file mode 100644 index 000000000..2ca5ff552 --- /dev/null +++ b/deploy/operations/Dockerfile @@ -0,0 +1,22 @@ +FROM ubuntu:22.04 + +RUN apt-get update \ +&& apt-get install -y unzip curl gnupg lsb-release + +# Terraform CLI +RUN curl -s https://apt.releases.hashicorp.com/gpg | gpg --dearmor -o /usr/share/keyrings/hashicorp-archive-keyring.gpg \ +&& echo "deb [signed-by=/usr/share/keyrings/hashicorp-archive-keyring.gpg] https://apt.releases.hashicorp.com $(lsb_release -cs) main" | tee /etc/apt/sources.list.d/hashicorp.list \ +&& apt-get update \ +&& apt-get install -y terraform + +# AWS CLI +WORKDIR /opt +RUN curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip" \ +&& unzip awscliv2.zip \ +&& rm awscliv2.zip \ +&& ./aws/install + +# Clean up apt +RUN apt-get clean && rm -rf /var/lib/apt/lists/* + +RUN terraform --version \ No newline at end of file diff --git a/deploy/operations/ci/aws-1/main.tf b/deploy/operations/ci/aws-1/main.tf new file mode 100644 index 000000000..7133ad3ba --- /dev/null +++ b/deploy/operations/ci/aws-1/main.tf @@ -0,0 +1,43 @@ +terraform { + backend "s3" { + bucket = "interuss-tf-backend-ci" + key = "aws-1" + region = "us-east-1" + } +} + +module "terraform-aws-kubernetes" { + # See variables.tf for variables description. + cluster_name = var.cluster_name + aws_region = var.aws_region + app_hostname = var.app_hostname + crdb_hostname_suffix = var.crdb_hostname_suffix + aws_instance_type = var.aws_instance_type + aws_route53_zone_id = var.aws_route53_zone_id + node_count = var.node_count + + source = "../../../infrastructure/dependencies/terraform-aws-kubernetes" +} + +module "terraform-commons-dss" { + # See variables.tf for variables description. + image = var.image + image_pull_secret = var.image_pull_secret + kubernetes_namespace = var.kubernetes_namespace + kubernetes_storage_class = var.aws_kubernetes_storage_class + app_hostname = var.app_hostname + crdb_hostname_suffix = var.crdb_hostname_suffix + should_init = var.should_init + authorization = var.authorization + crdb_locality = var.crdb_locality + crdb_internal_nodes = module.terraform-aws-kubernetes.crdb_nodes + ip_gateway = module.terraform-aws-kubernetes.ip_gateway + kubernetes_api_endpoint = module.terraform-aws-kubernetes.kubernetes_api_endpoint + kubernetes_cloud_provider_name = module.terraform-aws-kubernetes.kubernetes_cloud_provider_name + kubernetes_context_name = module.terraform-aws-kubernetes.kubernetes_context_name + kubernetes_get_credentials_cmd = module.terraform-aws-kubernetes.kubernetes_get_credentials_cmd + workload_subnet = module.terraform-aws-kubernetes.workload_subnet + gateway_cert_name = module.terraform-aws-kubernetes.app_hostname_cert_arn + + source = "../../../infrastructure/dependencies/terraform-commons-dss" +} diff --git a/deploy/operations/ci/aws-1/output.tf b/deploy/operations/ci/aws-1/output.tf new file mode 100644 index 000000000..4b0fceb15 --- /dev/null +++ b/deploy/operations/ci/aws-1/output.tf @@ -0,0 +1,15 @@ +output "crdb_addresses" { + value = module.terraform-aws-kubernetes.crdb_addresses +} + +output "gateway_address" { + value = module.terraform-aws-kubernetes.gateway_address +} + +output "generated_files_location" { + value = module.terraform-commons-dss.generated_files_location +} + +output "cluster_context" { + value = module.terraform-aws-kubernetes.kubernetes_context_name +} \ No newline at end of file diff --git a/deploy/operations/ci/aws-1/terraform.tfvars b/deploy/operations/ci/aws-1/terraform.tfvars new file mode 100644 index 000000000..9f5aa7205 --- /dev/null +++ b/deploy/operations/ci/aws-1/terraform.tfvars @@ -0,0 +1,27 @@ +# This file is an example, please adapt it to your configuration. +# See TFVARS.md for the full set of variables and related descriptions. + +# AWS account +aws_region = "eu-west-1" + +# DNS Management +aws_route53_zone_id = "" + +# Hostnames +app_hostname = "dss.aws-interuss-ci.uspace.dev" +crdb_hostname_suffix = "db.aws-interuss-ci.uspace.dev" + +# Kubernetes configuration +cluster_name = "dss-ci-aws-ew1" +node_count = 3 +aws_instance_type = "t3.medium" +aws_kubernetes_storage_class = "gp2" + +# DSS configuration +image = "latest" +authorization = { + public_key_pem_path = "/test-certs/auth2.pem" +} +should_init = true +crdb_locality = "interuss_dss-ci-aws-ew1" +crdb_external_nodes = [] \ No newline at end of file diff --git a/deploy/operations/ci/aws-1/test.sh b/deploy/operations/ci/aws-1/test.sh new file mode 100755 index 000000000..40b6fadbb --- /dev/null +++ b/deploy/operations/ci/aws-1/test.sh @@ -0,0 +1,25 @@ +#!/usr/bin/env bash + +set -eo pipefail + +# Find and change to repo root directory +OS=$(uname) +if [[ "$OS" == "Darwin" ]]; then + # OSX uses BSD readlink + BASEDIR="$(dirname "$0")" +else + BASEDIR=$(readlink -e "$(dirname "$0")") +fi +cd "${BASEDIR}" || exit 1 + + +clean () { + echo "Cleaning infrastructure" + terraform destroy -auto-approve +} + +clean +terraform init +terraform apply -auto-approve +clean + diff --git a/deploy/operations/ci/aws-1/variables.tf b/deploy/operations/ci/aws-1/variables.tf new file mode 100644 index 000000000..55c183ba5 --- /dev/null +++ b/deploy/operations/ci/aws-1/variables.tf @@ -0,0 +1,260 @@ + +# This file has been automatically generated by /deploy/infrastructure/utils/generate_terraform_variables.py. +# Please do not modify manually. + +variable "aws_region" { + type = string + description = <<-EOT + AWS region + List of available regions: https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/using-regions-availability-zones.html#concepts-regions + Currently, the terraform module uses the two first availability zones of the region. + + Example: `eu-west-1` + EOT +} + +variable "aws_instance_type" { + type = string + description = <<-EOT + AWS EC2 instance type used for the Kubernetes node pool. + + Example: `m6g.xlarge` for production and `t3.medium` for development + EOT +} + +variable "aws_route53_zone_id" { + type = string + description = <<-EOT + AWS Route 53 Zone ID + This module can automatically create DNS records in a Route 53 Zone. + Leave empty to disable record creation. + + Example: `Z0123456789ABCDEFGHIJ` + EOT +} + +variable "app_hostname" { + type = string + description = <<-EOT + Fully-qualified domain name of your HTTPS Gateway ingress endpoint. + + Example: `dss.example.com` + EOT +} + +variable "crdb_hostname_suffix" { + type = string + description = <<-EOT + The domain name suffix shared by all of your CockroachDB nodes. + For instance, if your CRDB nodes were addressable at 0.db.example.com, + 1.db.example.com and 2.db.example.com, then the value would be db.example.com. + + Example: db.example.com + EOT +} + +variable "cluster_name" { + type = string + description = <<-EOT + Name of the kubernetes cluster that will host this DSS instance (should generally describe the DSS instance being hosted) + + Example: `dss-che-1` + EOT +} + +variable "node_count" { + type = number + description = <<-EOT + Number of Kubernetes nodes which should correspond to the desired CockroachDB nodes. + **Always 3.** + + Example: `3` + EOT + + validation { + condition = var.node_count == 3 + error_message = "Node count should be 3. Only configuration supported at the moment" + } +} + +variable "aws_kubernetes_storage_class" { + type = string + description = <<-EOT + AWS Elastic Kubernetes Service Storage Class to use for CockroachDB and Prometheus persistent volumes. + See https://docs.aws.amazon.com/eks/latest/userguide/storage-classes.html for more details and + available options. + + Example: `gp2` + EOT +} + +variable "image" { + type = string + description = < kubectl create secret -n VAR_NAMESPACE docker-registry VAR_DOCKER_IMAGE_PULL_SECRET \ + --docker-server=DOCKER_REGISTRY_SERVER \ + --docker-username=DOCKER_USER \ + --docker-password=DOCKER_PASSWORD \ + --docker-email=DOCKER_EMAIL + + Replace `VAR_DOCKER_IMAGE_PULL_SECRET` with the secret name (for instance: `private-registry-credentials`). + For docker hub private repository, use `docker.io` as `DOCKER_REGISTRY_SERVER` and an + [access token](https://hub.docker.com/settings/security) as `DOCKER_PASSWORD`. + + Example: docker-registry + EOT + default = "" +} + +variable "authorization" { + type = object({ + public_key_pem_path = optional(string) + jwks = optional(object({ + endpoint = string + key_id = string + })) + }) + description = <_", + and the = character is not allowed. However, any unique (among all other participating + DSS instances) value is acceptable. + + Example: + EOT +} + +variable "crdb_external_nodes" { + type = list(string) + description = <<-EOT + Fully-qualified domain name of existing CRDB nodes outside of the cluster if you are joining an existing pool. + Example: ["0.db.dss.example.com", "1.db.dss.example.com", "2.db.dss.example.com"] + EOT + default = [] +} + +variable "kubernetes_namespace" { + type = string + description = <<-EOT + Namespace where to deploy Kubernetes resources. Only default is supported at the moment. + + Example: `default` + EOT + + default = "default" + + # TODO: Adapt current deployment scripts in /build/deploy to support default is supported for the moment. + validation { + condition = var.kubernetes_namespace == "default" + error_message = "Only default namespace is supported at the moment" + } +} + diff --git a/deploy/operations/docker-compose.yaml b/deploy/operations/docker-compose.yaml new file mode 100644 index 000000000..c9a645c21 --- /dev/null +++ b/deploy/operations/docker-compose.yaml @@ -0,0 +1,14 @@ +services: + ci-aws-1: + build: . + image: interuss-deploy + profiles: ["aws-1"] + command: operations/ci/aws-1/test.sh + working_dir: /opt/dss + environment: + - AWS_ACCESS_KEY_ID + - AWS_SECRET_ACCESS_KEY + volumes: + - type: bind + source: ../ + target: /opt/dss/ \ No newline at end of file From abc12015e28af84e4943ec1d377cb280f2ecd390 Mon Sep 17 00:00:00 2001 From: Michael Barroco Date: Thu, 30 Nov 2023 22:23:52 +0100 Subject: [PATCH 02/28] Fix login with session token --- .../terraform-aws-kubernetes/cluster.tf | 6 +- .../terraform-aws-kubernetes/ebs.tf | 9 --- .../terraform-aws-kubernetes/iam.tf | 55 +++++++++++-------- .../terraform-aws-kubernetes/main.tf | 14 +++-- .../terraform-aws-kubernetes/oidc.tf | 9 +++ deploy/operations/ci/aws-1/terraform.tfvars | 13 ++--- 6 files changed, 58 insertions(+), 48 deletions(-) create mode 100644 deploy/infrastructure/dependencies/terraform-aws-kubernetes/oidc.tf diff --git a/deploy/infrastructure/dependencies/terraform-aws-kubernetes/cluster.tf b/deploy/infrastructure/dependencies/terraform-aws-kubernetes/cluster.tf index dfca0743f..1c5153026 100644 --- a/deploy/infrastructure/dependencies/terraform-aws-kubernetes/cluster.tf +++ b/deploy/infrastructure/dependencies/terraform-aws-kubernetes/cluster.tf @@ -1,11 +1,11 @@ resource "aws_eks_cluster" "kubernetes_cluster" { name = var.cluster_name role_arn = aws_iam_role.dss-cluster.arn - + vpc_config { subnet_ids = aws_subnet.dss[*].id endpoint_public_access = true - public_access_cidrs = [ + public_access_cidrs = [ "0.0.0.0/0" ] } @@ -26,7 +26,7 @@ resource "aws_eks_node_group" "eks_node_group" { node_role_arn = aws_iam_role.dss-cluster-node-group.arn disk_size = 100 node_group_name_prefix = aws_eks_cluster.kubernetes_cluster.name - instance_types = [ + instance_types = [ var.aws_instance_type ] diff --git a/deploy/infrastructure/dependencies/terraform-aws-kubernetes/ebs.tf b/deploy/infrastructure/dependencies/terraform-aws-kubernetes/ebs.tf index eedf02822..dc7eefd8b 100644 --- a/deploy/infrastructure/dependencies/terraform-aws-kubernetes/ebs.tf +++ b/deploy/infrastructure/dependencies/terraform-aws-kubernetes/ebs.tf @@ -1,12 +1,3 @@ -data "tls_certificate" "cluster_oidc_provider" { - url = aws_eks_cluster.kubernetes_cluster.identity[0].oidc[0].issuer -} - -resource "aws_iam_openid_connect_provider" "cluster_provider" { - client_id_list = ["sts.amazonaws.com"] - thumbprint_list = data.tls_certificate.cluster_oidc_provider.certificates[*].sha1_fingerprint - url = data.tls_certificate.cluster_oidc_provider.url -} resource "aws_eks_addon" "aws-ebs-csi-driver" { addon_name = "aws-ebs-csi-driver" diff --git a/deploy/infrastructure/dependencies/terraform-aws-kubernetes/iam.tf b/deploy/infrastructure/dependencies/terraform-aws-kubernetes/iam.tf index 6eabc6ead..411ca4f6b 100644 --- a/deploy/infrastructure/dependencies/terraform-aws-kubernetes/iam.tf +++ b/deploy/infrastructure/dependencies/terraform-aws-kubernetes/iam.tf @@ -31,6 +31,8 @@ resource "aws_iam_role_policy_attachment" "dss-cluster-service" { role = aws_iam_role.dss-cluster.name } +# Roles + resource "aws_iam_role" "dss-cluster-node-group" { name = "${var.cluster_name}-cluster-node-group" @@ -48,6 +50,33 @@ resource "aws_iam_role" "dss-cluster-node-group" { }) } +// EBS + +resource "aws_iam_role" "AmazonEKS_EBS_CSI_DriverRole" { + name = "${var.cluster_name}-AmazonEKS_EBS_CSI_DriverRole" + + assume_role_policy = jsonencode({ + "Version" : "2012-10-17", + "Statement" : [ + { + "Effect" : "Allow", + "Principal" : { + "Federated" : format("arn:aws:iam::${local.aws_account_id}:%s", replace(local.aws_cluster_oidc_issuer, "https://", "oidc-provider/")), + }, + "Action" : "sts:AssumeRoleWithWebIdentity", + "Condition" : { + "StringEquals" : { + format("%s:aud", replace(local.aws_cluster_oidc_issuer, "https://", "")) : "sts.amazonaws.com", + format("%s:sub", replace(local.aws_cluster_oidc_issuer, "https://", "")) : "system:serviceaccount:kube-system:ebs-csi-controller-sa" + } + } + } + ] + }) +} + +// Policies + resource "aws_iam_policy" "AWSLoadBalancerControllerPolicy" { name = "${var.cluster_name}-AWSLoadBalancerControllerPolicy" # Source: https://docs.aws.amazon.com/eks/latest/userguide/aws-load-balancer-controller.html @@ -55,6 +84,8 @@ resource "aws_iam_policy" "AWSLoadBalancerControllerPolicy" { policy = file("${path.module}/AWSLoadBalancerControllerPolicy.json") } +// Attachments + resource "aws_iam_role_policy_attachment" "AWSLoadBalancerControllerPolicy" { policy_arn = aws_iam_policy.AWSLoadBalancerControllerPolicy.arn role = aws_iam_role.dss-cluster-node-group.name @@ -70,35 +101,11 @@ resource "aws_iam_role_policy_attachment" "AmazonEKS_CNI_Policy" { role = aws_iam_role.dss-cluster-node-group.name } -## Docker registry resource "aws_iam_role_policy_attachment" "AmazonEC2ContainerRegistryReadOnly" { policy_arn = "arn:aws:iam::aws:policy/AmazonEC2ContainerRegistryReadOnly" role = aws_iam_role.dss-cluster-node-group.name } -## EBS -resource "aws_iam_role" "AmazonEKS_EBS_CSI_DriverRole" { - name = "${var.cluster_name}-AmazonEKS_EBS_CSI_DriverRole" - - assume_role_policy = jsonencode({ - "Version" : "2012-10-17", - "Statement" : [ - { - "Effect" : "Allow", - "Principal" : { - "Federated" : format("arn:aws:iam::${local.aws_account_id}:%s", replace(local.aws_cluster_oidc_issuer, "https://", "oidc-provider/")), - }, - "Action" : "sts:AssumeRoleWithWebIdentity", - "Condition" : { - "StringEquals" : { - format("%s:aud", replace(local.aws_cluster_oidc_issuer, "https://", "")) : "sts.amazonaws.com", - format("%s:sub", replace(local.aws_cluster_oidc_issuer, "https://", "")) : "system:serviceaccount:kube-system:ebs-csi-controller-sa" - } - } - } - ] - }) -} resource "aws_iam_role_policy_attachment" "AmazonEKS_EBS_CSI_DriverRole" { policy_arn = "arn:aws:iam::aws:policy/service-role/AmazonEBSCSIDriverPolicy" diff --git a/deploy/infrastructure/dependencies/terraform-aws-kubernetes/main.tf b/deploy/infrastructure/dependencies/terraform-aws-kubernetes/main.tf index 9e0945db7..1a77c33dd 100644 --- a/deploy/infrastructure/dependencies/terraform-aws-kubernetes/main.tf +++ b/deploy/infrastructure/dependencies/terraform-aws-kubernetes/main.tf @@ -7,6 +7,10 @@ terraform { tls = { source = "hashicorp/tls" } + helm = { + source = "hashicorp/helm" + version = "2.12.0" + } } } @@ -20,14 +24,14 @@ provider "aws" { } } +data "aws_eks_cluster_auth" "access" { + name = aws_eks_cluster.kubernetes_cluster.name +} + provider "helm" { kubernetes { host = aws_eks_cluster.kubernetes_cluster.endpoint cluster_ca_certificate = base64decode(aws_eks_cluster.kubernetes_cluster.certificate_authority[0].data) - exec { - api_version = "client.authentication.k8s.io/v1beta1" - args = ["eks", "get-token", "--cluster-name", var.cluster_name] - command = "aws" - } + token = data.aws_eks_cluster_auth.access.token } } \ No newline at end of file diff --git a/deploy/infrastructure/dependencies/terraform-aws-kubernetes/oidc.tf b/deploy/infrastructure/dependencies/terraform-aws-kubernetes/oidc.tf new file mode 100644 index 000000000..0baaa6ed9 --- /dev/null +++ b/deploy/infrastructure/dependencies/terraform-aws-kubernetes/oidc.tf @@ -0,0 +1,9 @@ +data "tls_certificate" "cluster_oidc_provider" { + url = aws_eks_cluster.kubernetes_cluster.identity[0].oidc[0].issuer +} + +resource "aws_iam_openid_connect_provider" "cluster_provider" { + client_id_list = ["sts.amazonaws.com"] + thumbprint_list = data.tls_certificate.cluster_oidc_provider.certificates[*].sha1_fingerprint + url = data.tls_certificate.cluster_oidc_provider.url +} \ No newline at end of file diff --git a/deploy/operations/ci/aws-1/terraform.tfvars b/deploy/operations/ci/aws-1/terraform.tfvars index 9f5aa7205..ae36b31b6 100644 --- a/deploy/operations/ci/aws-1/terraform.tfvars +++ b/deploy/operations/ci/aws-1/terraform.tfvars @@ -1,18 +1,17 @@ -# This file is an example, please adapt it to your configuration. # See TFVARS.md for the full set of variables and related descriptions. # AWS account -aws_region = "eu-west-1" +aws_region = "us-east-1" # DNS Management -aws_route53_zone_id = "" +aws_route53_zone_id = "Z03377073HUSGB4L9FKEK" # Hostnames -app_hostname = "dss.aws-interuss-ci.uspace.dev" -crdb_hostname_suffix = "db.aws-interuss-ci.uspace.dev" +app_hostname = "dss.ci.aws-interuss.uspace.dev" +crdb_hostname_suffix = "db.ci.aws-interuss.uspace.dev" # Kubernetes configuration -cluster_name = "dss-ci-aws-ew1" +cluster_name = "dss-ci-aws-ue1" node_count = 3 aws_instance_type = "t3.medium" aws_kubernetes_storage_class = "gp2" @@ -23,5 +22,5 @@ authorization = { public_key_pem_path = "/test-certs/auth2.pem" } should_init = true -crdb_locality = "interuss_dss-ci-aws-ew1" +crdb_locality = "interuss_dss-ci-aws-ue1" crdb_external_nodes = [] \ No newline at end of file From 20fd00579bd1ae993a76ab80524161033e123494 Mon Sep 17 00:00:00 2001 From: Michael Barroco Date: Fri, 1 Dec 2023 01:03:12 +0100 Subject: [PATCH 03/28] Add aws_iam_path variable to scope resources created --- .../dependencies/terraform-aws-kubernetes/iam.tf | 4 ++++ .../dependencies/terraform-aws-kubernetes/main.tf | 4 ++-- .../terraform-aws-kubernetes/variables.tf | 11 +++++++++++ .../modules/terraform-aws-dss/TFVARS.md | 12 ++++++++++++ .../infrastructure/modules/terraform-aws-dss/main.tf | 1 + .../modules/terraform-aws-dss/variables.tf | 11 +++++++++++ deploy/infrastructure/utils/Dockerfile | 2 +- .../infrastructure/utils/definitions/aws_iam_path.tf | 10 ++++++++++ deploy/infrastructure/utils/variables.py | 6 ++++-- deploy/operations/ci/aws-1/main.tf | 1 + 10 files changed, 57 insertions(+), 5 deletions(-) create mode 100644 deploy/infrastructure/utils/definitions/aws_iam_path.tf diff --git a/deploy/infrastructure/dependencies/terraform-aws-kubernetes/iam.tf b/deploy/infrastructure/dependencies/terraform-aws-kubernetes/iam.tf index 411ca4f6b..2cb8ddff2 100644 --- a/deploy/infrastructure/dependencies/terraform-aws-kubernetes/iam.tf +++ b/deploy/infrastructure/dependencies/terraform-aws-kubernetes/iam.tf @@ -8,6 +8,7 @@ locals { resource "aws_iam_role" "dss-cluster" { name = "${var.cluster_name}-dss-cluster" + path = var.aws_iam_path assume_role_policy = < Date: Mon, 4 Dec 2023 14:29:21 -0500 Subject: [PATCH 04/28] Add support for permissions boundary --- .github/workflows/ci.yml | 24 ---------- .../terraform-aws-kubernetes/iam.tf | 45 ++++++++++--------- .../terraform-aws-kubernetes/variables.tf | 15 +++++-- .../modules/terraform-aws-dss/TFVARS.md | 15 +++++-- .../modules/terraform-aws-dss/main.tf | 17 +++---- .../terraform-aws-dss/terraform.tfvars | 18 ++++++++ .../modules/terraform-aws-dss/variables.tf | 15 +++++-- .../utils/definitions/aws_iam_path.tf | 2 +- .../aws_iam_permissions_boundary.tf | 8 ++++ deploy/infrastructure/utils/variables.py | 3 +- deploy/operations/ci/aws-1/main.tf | 3 +- 11 files changed, 101 insertions(+), 64 deletions(-) create mode 100644 deploy/infrastructure/modules/terraform-aws-dss/terraform.tfvars create mode 100644 deploy/infrastructure/utils/definitions/aws_iam_permissions_boundary.tf diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b6b53f26e..1e8f98bd8 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -62,27 +62,3 @@ jobs: run: make probe-locally - name: Bring down local DSS instance run: make down-locally - - deploy-tests: - name: Deploy tests - runs-on: ubunutu-latest - concurrency: 1 - steps: - - name: Job information - run: | - echo "Job information" - echo "Trigger: ${{ github.event_name }}" - echo "Host: ${{ runner.os }}" - echo "Repository: ${{ github.repository }}" - echo "Branch: ${{ github.ref }}" - docker images - - name: Checkout - uses: actions/checkout@v2 - with: - submodules: true - - name: Test Scenario AWS-1 - env: - AWS_ACCESS_KEY_ID: ${{ secrets.CI_AWS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.CI_AWS_SECRET_ACCESS_KEY }} - COMPOSE_PROFILES: aws-1 - run: docker-compose up -d \ No newline at end of file diff --git a/deploy/infrastructure/dependencies/terraform-aws-kubernetes/iam.tf b/deploy/infrastructure/dependencies/terraform-aws-kubernetes/iam.tf index 2cb8ddff2..006553837 100644 --- a/deploy/infrastructure/dependencies/terraform-aws-kubernetes/iam.tf +++ b/deploy/infrastructure/dependencies/terraform-aws-kubernetes/iam.tf @@ -7,23 +7,24 @@ locals { } resource "aws_iam_role" "dss-cluster" { + // EKS does not support a path in the role arn name = "${var.cluster_name}-dss-cluster" - path = var.aws_iam_path - assume_role_policy = < Date: Mon, 4 Dec 2023 14:29:52 -0500 Subject: [PATCH 05/28] Add AWS_SESSION_TOKEN to docker container --- deploy/operations/docker-compose.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/deploy/operations/docker-compose.yaml b/deploy/operations/docker-compose.yaml index c9a645c21..fdcfe382b 100644 --- a/deploy/operations/docker-compose.yaml +++ b/deploy/operations/docker-compose.yaml @@ -8,6 +8,7 @@ services: environment: - AWS_ACCESS_KEY_ID - AWS_SECRET_ACCESS_KEY + - AWS_SESSION_TOKEN volumes: - type: bind source: ../ From b19315a155fdaaec4bb48c8597f0e7c2c15c169e Mon Sep 17 00:00:00 2001 From: Michael Barroco Date: Mon, 4 Dec 2023 14:38:14 -0500 Subject: [PATCH 06/28] Only plan --- .github/workflows/dss-deploy.yml | 46 ++++++++++++++++++++++++++++++ deploy/operations/ci/aws-1/test.sh | 4 +-- 2 files changed, 48 insertions(+), 2 deletions(-) create mode 100644 .github/workflows/dss-deploy.yml diff --git a/.github/workflows/dss-deploy.yml b/.github/workflows/dss-deploy.yml new file mode 100644 index 000000000..1468fa4fd --- /dev/null +++ b/.github/workflows/dss-deploy.yml @@ -0,0 +1,46 @@ +on: + workflow_dispatch: {} +jobs: + deploy: + name: Deploy DSS to AWS + runs-on: ubuntu-latest + if: github.repository == 'interuss/dss' || github.repository == 'Orbitalize/dss' + concurrency: + group: dss-deploy-aws + cancel-in-progress: false + permissions: + id-token: write + contents: read + steps: + - name: Job information + run: | + echo "Job information" + echo "Trigger: ${{ github.event_name }}" + echo "Host: ${{ runner.os }}" + echo "Repository: ${{ github.repository }}" + echo "Branch: ${{ github.ref }}" + docker images + + - name: Checkout + uses: actions/checkout@v2 + with: + fetch-depth: 0 + + - name: Configure AWS Credentials + uses: aws-actions/configure-aws-credentials@v4 + with: + role-to-assume: arn:aws:iam::301042233698:role/InterUSSGithubCI + aws-region: us-east-1 + mask-aws-account-id: true + role-duration-seconds: 1800 + + - name: Caller id + run: | + aws sts get-caller-identity + + - name: Test Scenario AWS-1 + with: + working-directory: ./deploy/operations/ + env: + COMPOSE_PROFILES: aws-1 + run: docker-compose up -d \ No newline at end of file diff --git a/deploy/operations/ci/aws-1/test.sh b/deploy/operations/ci/aws-1/test.sh index 40b6fadbb..ece8e05f6 100755 --- a/deploy/operations/ci/aws-1/test.sh +++ b/deploy/operations/ci/aws-1/test.sh @@ -12,7 +12,6 @@ else fi cd "${BASEDIR}" || exit 1 - clean () { echo "Cleaning infrastructure" terraform destroy -auto-approve @@ -20,6 +19,7 @@ clean () { clean terraform init -terraform apply -auto-approve +terraform plan +#terraform apply -auto-approve clean From 1e4a279ca6ac8f6f118ef6b08d954a422f42dc85 Mon Sep 17 00:00:00 2001 From: Michael Barroco Date: Thu, 7 Dec 2023 16:33:09 -0500 Subject: [PATCH 07/28] Remove path from iam resources --- .../dependencies/terraform-aws-kubernetes/iam.tf | 4 +--- .../terraform-aws-kubernetes/variables.tf | 11 ----------- .../modules/terraform-aws-dss/TFVARS.md | 12 ------------ .../modules/terraform-aws-dss/variables.tf | 11 ----------- .../infrastructure/utils/definitions/aws_iam_path.tf | 10 ---------- deploy/infrastructure/utils/variables.py | 1 - deploy/operations/ci/aws-1/main.tf | 3 +-- deploy/operations/ci/aws-1/terraform.tfvars | 5 ++++- deploy/operations/ci/aws-1/variables.tf | 9 +++++++++ 9 files changed, 15 insertions(+), 51 deletions(-) delete mode 100644 deploy/infrastructure/utils/definitions/aws_iam_path.tf diff --git a/deploy/infrastructure/dependencies/terraform-aws-kubernetes/iam.tf b/deploy/infrastructure/dependencies/terraform-aws-kubernetes/iam.tf index 006553837..00131e28c 100644 --- a/deploy/infrastructure/dependencies/terraform-aws-kubernetes/iam.tf +++ b/deploy/infrastructure/dependencies/terraform-aws-kubernetes/iam.tf @@ -37,7 +37,6 @@ resource "aws_iam_role_policy_attachment" "dss-cluster-service" { resource "aws_iam_role" "dss-cluster-node-group" { name = "${var.cluster_name}-cluster-node-group" - path = var.aws_iam_path assume_role_policy = jsonencode({ Statement = [ @@ -59,7 +58,6 @@ resource "aws_iam_role" "dss-cluster-node-group" { resource "aws_iam_role" "AmazonEKS_EBS_CSI_DriverRole" { name = "${var.cluster_name}-AmazonEKS_EBS_CSI_DriverRole" - path = var.aws_iam_path assume_role_policy = jsonencode({ "Version" : "2012-10-17", @@ -87,7 +85,7 @@ resource "aws_iam_role" "AmazonEKS_EBS_CSI_DriverRole" { resource "aws_iam_policy" "AWSLoadBalancerControllerPolicy" { name = "${var.cluster_name}-AWSLoadBalancerControllerPolicy" - path = var.aws_iam_path + # Source: https://docs.aws.amazon.com/eks/latest/userguide/aws-load-balancer-controller.html # Template: https://raw.githubusercontent.com/kubernetes-sigs/aws-load-balancer-controller/v2.4.4/docs/install/iam_policy.json policy = file("${path.module}/AWSLoadBalancerControllerPolicy.json") diff --git a/deploy/infrastructure/dependencies/terraform-aws-kubernetes/variables.tf b/deploy/infrastructure/dependencies/terraform-aws-kubernetes/variables.tf index 098b9ad5f..d1bfb1375 100644 --- a/deploy/infrastructure/dependencies/terraform-aws-kubernetes/variables.tf +++ b/deploy/infrastructure/dependencies/terraform-aws-kubernetes/variables.tf @@ -33,17 +33,6 @@ variable "aws_route53_zone_id" { EOT } -variable "aws_iam_path" { - type = string - description = <<-EOT - AWS IAM Resources Path - IAM related resources will be created within the specified path - - Example: `ci/` - EOT - default = "/" -} - variable "aws_iam_permissions_boundary" { type = string description = <<-EOT diff --git a/deploy/infrastructure/modules/terraform-aws-dss/TFVARS.md b/deploy/infrastructure/modules/terraform-aws-dss/TFVARS.md index c94c867e2..e0bbc8e1f 100644 --- a/deploy/infrastructure/modules/terraform-aws-dss/TFVARS.md +++ b/deploy/infrastructure/modules/terraform-aws-dss/TFVARS.md @@ -40,18 +40,6 @@ Leave empty to disable record creation. Example: `Z0123456789ABCDEFGHIJ` -### aws_iam_path - -*Type: `string`* - -**Default: "/"** - -AWS IAM Resources Path -IAM related resources will be created within the specified path - -Example: `ci/` - - ### aws_iam_permissions_boundary *Type: `string`* diff --git a/deploy/infrastructure/modules/terraform-aws-dss/variables.tf b/deploy/infrastructure/modules/terraform-aws-dss/variables.tf index 329c67855..ee911afe5 100644 --- a/deploy/infrastructure/modules/terraform-aws-dss/variables.tf +++ b/deploy/infrastructure/modules/terraform-aws-dss/variables.tf @@ -33,17 +33,6 @@ variable "aws_route53_zone_id" { EOT } -variable "aws_iam_path" { - type = string - description = <<-EOT - AWS IAM Resources Path - IAM related resources will be created within the specified path - - Example: `ci/` - EOT - default = "/" -} - variable "aws_iam_permissions_boundary" { type = string description = <<-EOT diff --git a/deploy/infrastructure/utils/definitions/aws_iam_path.tf b/deploy/infrastructure/utils/definitions/aws_iam_path.tf deleted file mode 100644 index 18f95e4cd..000000000 --- a/deploy/infrastructure/utils/definitions/aws_iam_path.tf +++ /dev/null @@ -1,10 +0,0 @@ -variable "aws_iam_path" { - type = string - description = <<-EOT - AWS IAM Resources Path - IAM related resources will be created within the specified path - - Example: `ci/` - EOT - default = "/" -} \ No newline at end of file diff --git a/deploy/infrastructure/utils/variables.py b/deploy/infrastructure/utils/variables.py index e35605ae6..17ab38a3e 100755 --- a/deploy/infrastructure/utils/variables.py +++ b/deploy/infrastructure/utils/variables.py @@ -67,7 +67,6 @@ "aws_region", "aws_instance_type", "aws_route53_zone_id", - "aws_iam_path", "aws_iam_permissions_boundary" ] + COMMON_KUBERNETES_VARIABLES diff --git a/deploy/operations/ci/aws-1/main.tf b/deploy/operations/ci/aws-1/main.tf index 381c063f6..a9ee3093e 100644 --- a/deploy/operations/ci/aws-1/main.tf +++ b/deploy/operations/ci/aws-1/main.tf @@ -14,8 +14,7 @@ module "terraform-aws-kubernetes" { crdb_hostname_suffix = var.crdb_hostname_suffix aws_instance_type = var.aws_instance_type aws_route53_zone_id = var.aws_route53_zone_id - aws_iam_path = "/interuss/" - aws_iam_permissions_boundary = "arn:aws:iam::301042233698:policy/GithubCIPermissionBoundaries20231130225039606500000001" + aws_iam_permissions_boundary = var.aws_iam_permissions_boundary node_count = var.node_count source = "../../../infrastructure/dependencies/terraform-aws-kubernetes" diff --git a/deploy/operations/ci/aws-1/terraform.tfvars b/deploy/operations/ci/aws-1/terraform.tfvars index ae36b31b6..4f503cf21 100644 --- a/deploy/operations/ci/aws-1/terraform.tfvars +++ b/deploy/operations/ci/aws-1/terraform.tfvars @@ -23,4 +23,7 @@ authorization = { } should_init = true crdb_locality = "interuss_dss-ci-aws-ue1" -crdb_external_nodes = [] \ No newline at end of file +crdb_external_nodes = [] + +aws_iam_permissions_boundary = "arn:aws:iam::301042233698:policy/GithubCIPermissionBoundaries20231130225039606500000001" +aws_iam_administrator_role = "arn:aws:iam::301042233698:role/AWSReservedSSO_AdministratorAccess_9b637c80b830ea2c" \ No newline at end of file diff --git a/deploy/operations/ci/aws-1/variables.tf b/deploy/operations/ci/aws-1/variables.tf index 55c183ba5..ee911afe5 100644 --- a/deploy/operations/ci/aws-1/variables.tf +++ b/deploy/operations/ci/aws-1/variables.tf @@ -33,6 +33,15 @@ variable "aws_route53_zone_id" { EOT } +variable "aws_iam_permissions_boundary" { + type = string + description = <<-EOT + AWS IAM Policy to be used for permissions boundaries on created roles. + + Example: `GithubCIPermissionBoundaries` + EOT +} + variable "app_hostname" { type = string description = <<-EOT From d603eb14709ce888514144e3a2d85b3a87242b52 Mon Sep 17 00:00:00 2001 From: Michael Barroco Date: Thu, 7 Dec 2023 16:34:00 -0500 Subject: [PATCH 08/28] Expose node group arn as output --- .../dependencies/terraform-aws-kubernetes/output.tf | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/deploy/infrastructure/dependencies/terraform-aws-kubernetes/output.tf b/deploy/infrastructure/dependencies/terraform-aws-kubernetes/output.tf index 3f9594399..a6d238d77 100644 --- a/deploy/infrastructure/dependencies/terraform-aws-kubernetes/output.tf +++ b/deploy/infrastructure/dependencies/terraform-aws-kubernetes/output.tf @@ -52,4 +52,8 @@ output "gateway_address" { output "workload_subnet" { value = data.aws_subnet.main_subnet.id +} + +output "iam_role_node_group_arn" { + value = aws_iam_role.dss-cluster-node-group.arn } \ No newline at end of file From ea99e15a1d30fd36fc73df06f77106ee356ccb42 Mon Sep 17 00:00:00 2001 From: Michael Barroco Date: Thu, 7 Dec 2023 16:35:19 -0500 Subject: [PATCH 09/28] Grant administrator access to kubernetes --- deploy/operations/ci/aws-1/data.tf | 1 + .../ci/aws-1/kubernetes_admin_access.tf | 31 +++++++++++++++++++ deploy/operations/ci/aws-1/local_variables.tf | 11 +++++++ 3 files changed, 43 insertions(+) create mode 100644 deploy/operations/ci/aws-1/data.tf create mode 100644 deploy/operations/ci/aws-1/kubernetes_admin_access.tf create mode 100644 deploy/operations/ci/aws-1/local_variables.tf diff --git a/deploy/operations/ci/aws-1/data.tf b/deploy/operations/ci/aws-1/data.tf new file mode 100644 index 000000000..8fc4b38cc --- /dev/null +++ b/deploy/operations/ci/aws-1/data.tf @@ -0,0 +1 @@ +data "aws_caller_identity" "current" {} diff --git a/deploy/operations/ci/aws-1/kubernetes_admin_access.tf b/deploy/operations/ci/aws-1/kubernetes_admin_access.tf new file mode 100644 index 000000000..1bc4f761c --- /dev/null +++ b/deploy/operations/ci/aws-1/kubernetes_admin_access.tf @@ -0,0 +1,31 @@ +# This module is expected to be applied by the Github CI user. By default, only the user has permission to +# connect to the cluster. This file gathers resources to grant access to AWS administrators. + +resource "kubernetes_config_map_v1_data" "aws-auth" { + metadata { + name = "aws-auth" + namespace = "kube-system" + } + + force = true # EKS provisions this file by default. + + data = { + mapRoles = yamlencode([ + { + groups = [ + "system:bootstrappers", + "system:nodes" + ] + rolearn = module.terraform-aws-kubernetes.iam_role_node_group_arn + username = "system:node:{{EC2PrivateDNSName}}" + }, + { + groups = [ + "system:masters" + ] + rolearn = var.aws_iam_administrator_role + username = "aws-administrator" + } + ]) + } +} diff --git a/deploy/operations/ci/aws-1/local_variables.tf b/deploy/operations/ci/aws-1/local_variables.tf new file mode 100644 index 000000000..a4bf50497 --- /dev/null +++ b/deploy/operations/ci/aws-1/local_variables.tf @@ -0,0 +1,11 @@ +# This file contains variables only used by this module and which are not provided to child dependencies. + +variable "aws_iam_administrator_role" { + type = string + description = <<-EOT + AWS IAM administrator role + ARN of the role assumed by administrators when login into the AWS InterUSS account. + + Example: `arn:aws:iam::123456789012:role/AdminRole` + EOT +} From 2ecb54263391bd947b4c2592a24576e1c55f1da1 Mon Sep 17 00:00:00 2001 From: Michael Barroco Date: Thu, 7 Dec 2023 16:51:29 -0500 Subject: [PATCH 10/28] Add README with debugging instructions --- deploy/operations/ci/aws-1/README.md | 39 ++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) create mode 100644 deploy/operations/ci/aws-1/README.md diff --git a/deploy/operations/ci/aws-1/README.md b/deploy/operations/ci/aws-1/README.md new file mode 100644 index 000000000..9e7b04976 --- /dev/null +++ b/deploy/operations/ci/aws-1/README.md @@ -0,0 +1,39 @@ +# AWS-1 CI deployment + +This module deploys a kubernetes cluster to AWS and provisions the dss using helm. + +## Terraform state + +The terraform backend is configured to be shared using a S3 bucket. (see [`main.tf`](./main.tf)). + +## Debugging + +In case of issue, it is possible to connect to the cluster and retrieve the terraform state to manage it +locally. + +### Connection to the cluster + +To connect to the cluster, authenticate yourself to the AWS account. +Run the following command to load the kubernetes config: +``` +aws eks --region us-east-1 update-kubeconfig --name dss-ci-aws-ue1 +``` +Call the kubernetes cluster using `kubectl` + +#### Add other roles + +Access to the cluster is managed using the config map `aws-auth`. +Its definition is managed by [`kubernetes_admin_access.tf`](./kubernetes_admin_access.tf). +Currently only the user who bootstrapped the cluster and the ones assuming +the administrator role (see [`local_variables.tf`](./local_variables.tf)) have access. + +### Run terraform locally + +In case of failure, a user with administrator role can take over the deployment by cloning this +repository and retrieving the current deployment state by running the following command: + +``` +terraform init +``` + +At this point, the user can replay or clean the deployment as if it was the CI runner. \ No newline at end of file From 9211965eb3d4cad906ec806656ba10a7b90c09fb Mon Sep 17 00:00:00 2001 From: Michael Barroco Date: Thu, 7 Dec 2023 16:55:03 -0500 Subject: [PATCH 11/28] Trigger on PR --- .github/workflows/dss-deploy.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/dss-deploy.yml b/.github/workflows/dss-deploy.yml index 1468fa4fd..8a7c64c0e 100644 --- a/.github/workflows/dss-deploy.yml +++ b/.github/workflows/dss-deploy.yml @@ -1,5 +1,6 @@ on: workflow_dispatch: {} + pull_request: {} jobs: deploy: name: Deploy DSS to AWS From 606877c1d4fc86d07164dc98eef8bfceab5dda80 Mon Sep 17 00:00:00 2001 From: Michael Barroco Date: Thu, 7 Dec 2023 16:57:29 -0500 Subject: [PATCH 12/28] Update workflow --- .github/workflows/dss-deploy.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/dss-deploy.yml b/.github/workflows/dss-deploy.yml index 8a7c64c0e..f8865e711 100644 --- a/.github/workflows/dss-deploy.yml +++ b/.github/workflows/dss-deploy.yml @@ -44,4 +44,5 @@ jobs: working-directory: ./deploy/operations/ env: COMPOSE_PROFILES: aws-1 - run: docker-compose up -d \ No newline at end of file + run: | + docker-compose up -d \ No newline at end of file From 6917d6ea69372e8a2977d5d6a3f5a35cb87ac465 Mon Sep 17 00:00:00 2001 From: Michael Barroco Date: Thu, 7 Dec 2023 16:59:38 -0500 Subject: [PATCH 13/28] Update workflow --- .github/workflows/dss-deploy.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/dss-deploy.yml b/.github/workflows/dss-deploy.yml index f8865e711..2de6f3600 100644 --- a/.github/workflows/dss-deploy.yml +++ b/.github/workflows/dss-deploy.yml @@ -40,9 +40,10 @@ jobs: aws sts get-caller-identity - name: Test Scenario AWS-1 + shell: bash with: working-directory: ./deploy/operations/ env: COMPOSE_PROFILES: aws-1 run: | - docker-compose up -d \ No newline at end of file + docker-compose up -d From 18e6c4614314127b9ad519cef2c336ceea9ccd00 Mon Sep 17 00:00:00 2001 From: Michael Barroco Date: Thu, 7 Dec 2023 17:01:14 -0500 Subject: [PATCH 14/28] Update workflow --- .github/workflows/dss-deploy.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/dss-deploy.yml b/.github/workflows/dss-deploy.yml index 2de6f3600..58c196202 100644 --- a/.github/workflows/dss-deploy.yml +++ b/.github/workflows/dss-deploy.yml @@ -41,8 +41,7 @@ jobs: - name: Test Scenario AWS-1 shell: bash - with: - working-directory: ./deploy/operations/ + working-directory: ./deploy/operations/ env: COMPOSE_PROFILES: aws-1 run: | From de70af296e411189d91cfc53d5bd1455f8c378ee Mon Sep 17 00:00:00 2001 From: Michael Barroco Date: Thu, 7 Dec 2023 17:02:58 -0500 Subject: [PATCH 15/28] Update workflow --- .github/workflows/dss-deploy.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/dss-deploy.yml b/.github/workflows/dss-deploy.yml index 58c196202..b727a13a1 100644 --- a/.github/workflows/dss-deploy.yml +++ b/.github/workflows/dss-deploy.yml @@ -45,4 +45,4 @@ jobs: env: COMPOSE_PROFILES: aws-1 run: | - docker-compose up -d + docker-compose up From 770dbcaec90c78ee1ca7ac548fffd92f2498d26e Mon Sep 17 00:00:00 2001 From: Michael Barroco Date: Thu, 7 Dec 2023 17:05:32 -0500 Subject: [PATCH 16/28] Update workflow --- .github/workflows/dss-deploy.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/dss-deploy.yml b/.github/workflows/dss-deploy.yml index b727a13a1..4b2729dc5 100644 --- a/.github/workflows/dss-deploy.yml +++ b/.github/workflows/dss-deploy.yml @@ -35,11 +35,11 @@ jobs: mask-aws-account-id: true role-duration-seconds: 1800 - - name: Caller id + - name: Caller Id run: | aws sts get-caller-identity - - name: Test Scenario AWS-1 + - name: Test Deployment Scenario AWS-1 shell: bash working-directory: ./deploy/operations/ env: From 6b28d18c013b8cf0249d0c43be2c83dca5deb51c Mon Sep 17 00:00:00 2001 From: Michael Barroco Date: Thu, 7 Dec 2023 17:07:14 -0500 Subject: [PATCH 17/28] Update test script --- deploy/operations/ci/aws-1/test.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deploy/operations/ci/aws-1/test.sh b/deploy/operations/ci/aws-1/test.sh index ece8e05f6..28ec045a6 100755 --- a/deploy/operations/ci/aws-1/test.sh +++ b/deploy/operations/ci/aws-1/test.sh @@ -17,8 +17,8 @@ clean () { terraform destroy -auto-approve } -clean terraform init +clean terraform plan #terraform apply -auto-approve clean From 9d144f3bd5e88594bdd300ca036473b9ccd393b2 Mon Sep 17 00:00:00 2001 From: Michael Barroco Date: Thu, 7 Dec 2023 17:10:53 -0500 Subject: [PATCH 18/28] exit code from container --- .github/workflows/dss-deploy.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/dss-deploy.yml b/.github/workflows/dss-deploy.yml index 4b2729dc5..7c9b1b66c 100644 --- a/.github/workflows/dss-deploy.yml +++ b/.github/workflows/dss-deploy.yml @@ -45,4 +45,4 @@ jobs: env: COMPOSE_PROFILES: aws-1 run: | - docker-compose up + docker compose up --exit-code-from ci-aws-1 From 1f9ffb951a4242bf1f2f43195fec7037566b19be Mon Sep 17 00:00:00 2001 From: Michael Barroco Date: Thu, 7 Dec 2023 17:30:43 -0500 Subject: [PATCH 19/28] Add kubernetes provider configuration --- deploy/operations/ci/aws-1/providers.tf | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) create mode 100644 deploy/operations/ci/aws-1/providers.tf diff --git a/deploy/operations/ci/aws-1/providers.tf b/deploy/operations/ci/aws-1/providers.tf new file mode 100644 index 000000000..542349e86 --- /dev/null +++ b/deploy/operations/ci/aws-1/providers.tf @@ -0,0 +1,17 @@ +provider "aws" { + region = "us-east-1" +} + +data "aws_eks_cluster_auth" "kubernetes_cluster" { + name = var.cluster_name +} + +data "aws_eks_cluster" "kubernetes_cluster" { + name = var.cluster_name +} + +provider kubernetes { + host = data.aws_eks_cluster.kubernetes_cluster.endpoint + cluster_ca_certificate = base64decode(data.aws_eks_cluster.kubernetes_cluster.certificate_authority[0].data) + token = data.aws_eks_cluster_auth.kubernetes_cluster.token +} From 07bc714dbd8a1c029a27ff7a789f2dbdafa183d0 Mon Sep 17 00:00:00 2001 From: Michael Barroco Date: Fri, 8 Dec 2023 06:23:31 -0500 Subject: [PATCH 20/28] Grant explicitely GithubCI role to K8s --- deploy/operations/ci/aws-1/kubernetes_admin_access.tf | 9 ++++++++- deploy/operations/ci/aws-1/local_variables.tf | 10 ++++++++++ deploy/operations/ci/aws-1/providers.tf | 2 ++ deploy/operations/ci/aws-1/terraform.tfvars | 3 ++- deploy/operations/ci/aws-1/test.sh | 3 +-- 5 files changed, 23 insertions(+), 4 deletions(-) diff --git a/deploy/operations/ci/aws-1/kubernetes_admin_access.tf b/deploy/operations/ci/aws-1/kubernetes_admin_access.tf index 1bc4f761c..5e808c2f7 100644 --- a/deploy/operations/ci/aws-1/kubernetes_admin_access.tf +++ b/deploy/operations/ci/aws-1/kubernetes_admin_access.tf @@ -24,7 +24,14 @@ resource "kubernetes_config_map_v1_data" "aws-auth" { "system:masters" ] rolearn = var.aws_iam_administrator_role - username = "aws-administrator" + username = "interuss-aws-administrator" + }, + { + groups = [ + "system:masters" + ] + rolearn = var.aws_iam_ci_role + username = "interuss-ci" } ]) } diff --git a/deploy/operations/ci/aws-1/local_variables.tf b/deploy/operations/ci/aws-1/local_variables.tf index a4bf50497..628c3875e 100644 --- a/deploy/operations/ci/aws-1/local_variables.tf +++ b/deploy/operations/ci/aws-1/local_variables.tf @@ -9,3 +9,13 @@ variable "aws_iam_administrator_role" { Example: `arn:aws:iam::123456789012:role/AdminRole` EOT } + +variable "aws_iam_ci_role" { + type = string + description = <<-EOT + AWS IAM administrator role + ARN of the role assumed by administrators when login into the AWS InterUSS account. + + Example: `arn:aws:iam::123456789012:role/CiRole` + EOT +} diff --git a/deploy/operations/ci/aws-1/providers.tf b/deploy/operations/ci/aws-1/providers.tf index 542349e86..8daa12fe7 100644 --- a/deploy/operations/ci/aws-1/providers.tf +++ b/deploy/operations/ci/aws-1/providers.tf @@ -4,10 +4,12 @@ provider "aws" { data "aws_eks_cluster_auth" "kubernetes_cluster" { name = var.cluster_name + depends_on = [module.terraform-aws-kubernetes] } data "aws_eks_cluster" "kubernetes_cluster" { name = var.cluster_name + depends_on = [module.terraform-aws-kubernetes] } provider kubernetes { diff --git a/deploy/operations/ci/aws-1/terraform.tfvars b/deploy/operations/ci/aws-1/terraform.tfvars index 4f503cf21..90a5e8458 100644 --- a/deploy/operations/ci/aws-1/terraform.tfvars +++ b/deploy/operations/ci/aws-1/terraform.tfvars @@ -26,4 +26,5 @@ crdb_locality = "interuss_dss-ci-aws-ue1" crdb_external_nodes = [] aws_iam_permissions_boundary = "arn:aws:iam::301042233698:policy/GithubCIPermissionBoundaries20231130225039606500000001" -aws_iam_administrator_role = "arn:aws:iam::301042233698:role/AWSReservedSSO_AdministratorAccess_9b637c80b830ea2c" \ No newline at end of file +aws_iam_administrator_role = "arn:aws:iam::301042233698:role/AWSReservedSSO_AdministratorAccess_9b637c80b830ea2c" +aws_iam_ci_role = "arn:aws:iam::301042233698:role/InterUSSGithubCI" diff --git a/deploy/operations/ci/aws-1/test.sh b/deploy/operations/ci/aws-1/test.sh index 28ec045a6..cda5a9244 100755 --- a/deploy/operations/ci/aws-1/test.sh +++ b/deploy/operations/ci/aws-1/test.sh @@ -19,7 +19,6 @@ clean () { terraform init clean -terraform plan -#terraform apply -auto-approve +terraform apply -auto-approve clean From 1a983baec2911e65b9e757a482cd3a03d051f8ea Mon Sep 17 00:00:00 2001 From: Michael Barroco Date: Fri, 8 Dec 2023 06:41:54 -0500 Subject: [PATCH 21/28] Keep only generated files output --- deploy/operations/ci/aws-1/output.tf | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/deploy/operations/ci/aws-1/output.tf b/deploy/operations/ci/aws-1/output.tf index 4b0fceb15..ade9609d1 100644 --- a/deploy/operations/ci/aws-1/output.tf +++ b/deploy/operations/ci/aws-1/output.tf @@ -1,15 +1,4 @@ -output "crdb_addresses" { - value = module.terraform-aws-kubernetes.crdb_addresses -} - -output "gateway_address" { - value = module.terraform-aws-kubernetes.gateway_address -} - output "generated_files_location" { value = module.terraform-commons-dss.generated_files_location } -output "cluster_context" { - value = module.terraform-aws-kubernetes.kubernetes_context_name -} \ No newline at end of file From 47fd8fa10decc14c4068dfddce25fc910706cdfa Mon Sep 17 00:00:00 2001 From: Michael Barroco Date: Fri, 8 Dec 2023 07:16:27 -0500 Subject: [PATCH 22/28] Add title --- .github/workflows/dss-deploy.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/dss-deploy.yml b/.github/workflows/dss-deploy.yml index 7c9b1b66c..7b20dc265 100644 --- a/.github/workflows/dss-deploy.yml +++ b/.github/workflows/dss-deploy.yml @@ -1,3 +1,4 @@ +name: Deploy DSS on: workflow_dispatch: {} pull_request: {} From b3f11cb50907197c8537240ea46462b76265caa2 Mon Sep 17 00:00:00 2001 From: Michael Barroco Date: Fri, 8 Dec 2023 09:04:10 -0500 Subject: [PATCH 23/28] Remove unnecessary version --- .../dependencies/terraform-aws-kubernetes/main.tf | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/deploy/infrastructure/dependencies/terraform-aws-kubernetes/main.tf b/deploy/infrastructure/dependencies/terraform-aws-kubernetes/main.tf index 516b182e7..fc14c133f 100644 --- a/deploy/infrastructure/dependencies/terraform-aws-kubernetes/main.tf +++ b/deploy/infrastructure/dependencies/terraform-aws-kubernetes/main.tf @@ -9,7 +9,6 @@ terraform { } helm = { source = "hashicorp/helm" - version = "2.12.0" } } } @@ -34,4 +33,4 @@ provider "helm" { cluster_ca_certificate = base64decode(aws_eks_cluster.kubernetes_cluster.certificate_authority[0].data) token = data.aws_eks_cluster_auth.kubernetes_cluster.token } -} \ No newline at end of file +} From 29ff2d55c9959c7038fa59d14d4f54530e6c7ac4 Mon Sep 17 00:00:00 2001 From: Michael Barroco Date: Fri, 8 Dec 2023 09:14:41 -0500 Subject: [PATCH 24/28] Clean up --- deploy/infrastructure/utils/Dockerfile | 2 +- deploy/infrastructure/utils/README.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/deploy/infrastructure/utils/Dockerfile b/deploy/infrastructure/utils/Dockerfile index d6f1dfbaa..117653ff1 100644 --- a/deploy/infrastructure/utils/Dockerfile +++ b/deploy/infrastructure/utils/Dockerfile @@ -1,4 +1,4 @@ -FROM python:3.11.5 +FROM python:3.10 RUN pip install python-hcl2 diff --git a/deploy/infrastructure/utils/README.md b/deploy/infrastructure/utils/README.md index 015b7f50d..e9c336372 100644 --- a/deploy/infrastructure/utils/README.md +++ b/deploy/infrastructure/utils/README.md @@ -4,4 +4,4 @@ This directory contains the following tools to simplify the management of the te 1. `generate_terraform_variables.sh`: Terraform variables can't be shared between modules without repeating their definition at every level of encapsulation. To prevent repeating ourselves and to maintain a consistent level of quality for every module and dependencies, this script takes variables - in the `definitions` directory and creates a `variables.tf` file in each modules with the appropriate content. \ No newline at end of file + in the `definitions` directory and creates a `variables.tf` file in each modules with the appropriate content. From 6adc9521fd4baadc15b2b0f5b87a6b5b8671153c Mon Sep 17 00:00:00 2001 From: Michael Barroco Date: Fri, 8 Dec 2023 09:22:49 -0500 Subject: [PATCH 25/28] Manage operations/ci/aws-1/variables.tf with utility --- .../modules/terraform-aws-dss/terraform.tfvars | 18 ------------------ deploy/infrastructure/utils/variables.py | 3 +++ 2 files changed, 3 insertions(+), 18 deletions(-) delete mode 100644 deploy/infrastructure/modules/terraform-aws-dss/terraform.tfvars diff --git a/deploy/infrastructure/modules/terraform-aws-dss/terraform.tfvars b/deploy/infrastructure/modules/terraform-aws-dss/terraform.tfvars deleted file mode 100644 index 40bd9b004..000000000 --- a/deploy/infrastructure/modules/terraform-aws-dss/terraform.tfvars +++ /dev/null @@ -1,18 +0,0 @@ -aws_region = "eu-west-1" -cluster_name = "dss-1" -aws_route53_zone_id = "Z01554482LNPDVY7FW95I" -app_hostname = "dss.aws-interuss.uspace.dev" -aws_instance_type = "t3.medium" -crdb_hostname_suffix = "db.aws-interuss.uspace.dev" -node_count = 3 - -aws_kubernetes_storage_class = "gp2" - -# DSS configuration -image = "latest" -authorization = { - public_key_pem_path = "/test-certs/auth2.pem" -} -should_init = true -crdb_locality = "interuss_dss-aws-ew1" -crdb_external_nodes = [] diff --git a/deploy/infrastructure/utils/variables.py b/deploy/infrastructure/utils/variables.py index 17ab38a3e..633686634 100755 --- a/deploy/infrastructure/utils/variables.py +++ b/deploy/infrastructure/utils/variables.py @@ -85,6 +85,9 @@ "../dependencies/terraform-aws-kubernetes": AWS_KUBERNETES_VARIABLES, "../dependencies/terraform-google-kubernetes": GOOGLE_KUBERNETES_VARIABLES, "../dependencies/terraform-commons-dss": COMMONS_DSS_VARIABLES, + "../../operations/ci/aws-1": list( + dict.fromkeys(AWS_MODULE_VARIABLES) + ) } From 1a2f5701f898894a5cf0c68b8e9eba4024b76baf Mon Sep 17 00:00:00 2001 From: Michael Barroco Date: Fri, 8 Dec 2023 09:26:02 -0500 Subject: [PATCH 26/28] new line end of file --- deploy/operations/docker-compose.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deploy/operations/docker-compose.yaml b/deploy/operations/docker-compose.yaml index fdcfe382b..4618089db 100644 --- a/deploy/operations/docker-compose.yaml +++ b/deploy/operations/docker-compose.yaml @@ -12,4 +12,4 @@ services: volumes: - type: bind source: ../ - target: /opt/dss/ \ No newline at end of file + target: /opt/dss/ From 3b53cdd75b5449511f16db73ebd9099b9fd50e42 Mon Sep 17 00:00:00 2001 From: Michael Barroco Date: Fri, 8 Dec 2023 09:43:15 -0500 Subject: [PATCH 27/28] Update documentation --- .../dependencies/terraform-aws-kubernetes/variables.tf | 5 +++-- deploy/infrastructure/modules/terraform-aws-dss/TFVARS.md | 4 ++-- .../infrastructure/modules/terraform-aws-dss/variables.tf | 5 +++-- .../utils/definitions/aws_iam_permissions_boundary.tf | 6 +++--- deploy/operations/ci/aws-1/variables.tf | 5 +++-- 5 files changed, 14 insertions(+), 11 deletions(-) diff --git a/deploy/infrastructure/dependencies/terraform-aws-kubernetes/variables.tf b/deploy/infrastructure/dependencies/terraform-aws-kubernetes/variables.tf index d1bfb1375..1b73a3e34 100644 --- a/deploy/infrastructure/dependencies/terraform-aws-kubernetes/variables.tf +++ b/deploy/infrastructure/dependencies/terraform-aws-kubernetes/variables.tf @@ -36,12 +36,13 @@ variable "aws_route53_zone_id" { variable "aws_iam_permissions_boundary" { type = string description = <<-EOT - AWS IAM Policy to be used for permissions boundaries on created roles. + AWS IAM Policy ARN to be used for permissions boundaries on created roles. - Example: `GithubCIPermissionBoundaries` + Example: `arn:aws:iam::123456789012:policy/GithubCIPermissionBoundaries` EOT } + variable "app_hostname" { type = string description = <<-EOT diff --git a/deploy/infrastructure/modules/terraform-aws-dss/TFVARS.md b/deploy/infrastructure/modules/terraform-aws-dss/TFVARS.md index e0bbc8e1f..99b04c8ed 100644 --- a/deploy/infrastructure/modules/terraform-aws-dss/TFVARS.md +++ b/deploy/infrastructure/modules/terraform-aws-dss/TFVARS.md @@ -44,9 +44,9 @@ Example: `Z0123456789ABCDEFGHIJ` *Type: `string`* -AWS IAM Policy to be used for permissions boundaries on created roles. +AWS IAM Policy ARN to be used for permissions boundaries on created roles. -Example: `GithubCIPermissionBoundaries` +Example: `arn:aws:iam::123456789012:policy/GithubCIPermissionBoundaries` ### app_hostname diff --git a/deploy/infrastructure/modules/terraform-aws-dss/variables.tf b/deploy/infrastructure/modules/terraform-aws-dss/variables.tf index ee911afe5..3d276e351 100644 --- a/deploy/infrastructure/modules/terraform-aws-dss/variables.tf +++ b/deploy/infrastructure/modules/terraform-aws-dss/variables.tf @@ -36,12 +36,13 @@ variable "aws_route53_zone_id" { variable "aws_iam_permissions_boundary" { type = string description = <<-EOT - AWS IAM Policy to be used for permissions boundaries on created roles. + AWS IAM Policy ARN to be used for permissions boundaries on created roles. - Example: `GithubCIPermissionBoundaries` + Example: `arn:aws:iam::123456789012:policy/GithubCIPermissionBoundaries` EOT } + variable "app_hostname" { type = string description = <<-EOT diff --git a/deploy/infrastructure/utils/definitions/aws_iam_permissions_boundary.tf b/deploy/infrastructure/utils/definitions/aws_iam_permissions_boundary.tf index 16dc72db7..1279be202 100644 --- a/deploy/infrastructure/utils/definitions/aws_iam_permissions_boundary.tf +++ b/deploy/infrastructure/utils/definitions/aws_iam_permissions_boundary.tf @@ -1,8 +1,8 @@ variable "aws_iam_permissions_boundary" { type = string description = <<-EOT - AWS IAM Policy to be used for permissions boundaries on created roles. + AWS IAM Policy ARN to be used for permissions boundaries on created roles. - Example: `GithubCIPermissionBoundaries` + Example: `arn:aws:iam::123456789012:policy/GithubCIPermissionBoundaries` EOT -} \ No newline at end of file +} diff --git a/deploy/operations/ci/aws-1/variables.tf b/deploy/operations/ci/aws-1/variables.tf index ee911afe5..3d276e351 100644 --- a/deploy/operations/ci/aws-1/variables.tf +++ b/deploy/operations/ci/aws-1/variables.tf @@ -36,12 +36,13 @@ variable "aws_route53_zone_id" { variable "aws_iam_permissions_boundary" { type = string description = <<-EOT - AWS IAM Policy to be used for permissions boundaries on created roles. + AWS IAM Policy ARN to be used for permissions boundaries on created roles. - Example: `GithubCIPermissionBoundaries` + Example: `arn:aws:iam::123456789012:policy/GithubCIPermissionBoundaries` EOT } + variable "app_hostname" { type = string description = <<-EOT From 174cda76af5dfdb2afb57972bf770b0e53994d9c Mon Sep 17 00:00:00 2001 From: Michael Barroco Date: Fri, 8 Dec 2023 09:59:33 -0500 Subject: [PATCH 28/28] Improve README --- deploy/operations/ci/aws-1/README.md | 4 ++-- deploy/operations/ci/aws-1/test.sh | 2 ++ 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/deploy/operations/ci/aws-1/README.md b/deploy/operations/ci/aws-1/README.md index 9e7b04976..bdb0ebe21 100644 --- a/deploy/operations/ci/aws-1/README.md +++ b/deploy/operations/ci/aws-1/README.md @@ -1,6 +1,6 @@ # AWS-1 CI deployment -This module deploys a kubernetes cluster to AWS and provisions the dss using helm. +This module deploys a Kubernetes cluster to AWS. ## Terraform state @@ -36,4 +36,4 @@ repository and retrieving the current deployment state by running the following terraform init ``` -At this point, the user can replay or clean the deployment as if it was the CI runner. \ No newline at end of file +At this point, the user can replay or clean the deployment as if it was the CI runner. diff --git a/deploy/operations/ci/aws-1/test.sh b/deploy/operations/ci/aws-1/test.sh index cda5a9244..7caeafa93 100755 --- a/deploy/operations/ci/aws-1/test.sh +++ b/deploy/operations/ci/aws-1/test.sh @@ -20,5 +20,7 @@ clean () { terraform init clean terraform apply -auto-approve +# TODO: Deploy the DSS +# TODO: Test the deployment of the DSS clean