From d4e22fc99e26be5241f0a7b70555fc442a7c8671 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arturo=20Filast=C3=B2?= Date: Wed, 14 Feb 2024 17:22:27 +0100 Subject: [PATCH] Add support for persistent data volume (#2) * add basic readme file * also fix deploy checks when a branch lands on master, since it fails because the comment script doesn't have in it the reference to the open PR number --- .github/workflows/check_deploy.yml | 15 ------ Readme.md | 52 +++++++++++++++++++ .../production/ansible/inventory.ini | 1 + tf/environments/production/main.tf | 42 +++++++++++---- .../production/templates/clickhouse-setup.sh | 4 +- 5 files changed, 87 insertions(+), 27 deletions(-) create mode 100644 Readme.md diff --git a/.github/workflows/check_deploy.yml b/.github/workflows/check_deploy.yml index c4c6a2dd..0038d8de 100644 --- a/.github/workflows/check_deploy.yml +++ b/.github/workflows/check_deploy.yml @@ -3,9 +3,6 @@ # * https://docs.github.com/en/actions/using-workflows/events-that-trigger-workflows#pull_request # * https://docs.github.com/en/webhooks/webhook-events-and-payloads?actionType=synchronize#pull_request on: - push: - branches: - - main pull_request: types: - opened @@ -193,18 +190,6 @@ jobs: - #### Apply 📖\`${{ steps.apply.outcome }}\` - * **${terraformApplyPlanLine}** - * **${terraformApplyApplyLine}** - -
Show Apply - - \`\`\`\n - ${terraformApplyOutput} - \`\`\` - -
- | | | |-------------------|------------------------------------| | Pusher | @${{ github.actor }} | diff --git a/Readme.md b/Readme.md new file mode 100644 index 00000000..ef690164 --- /dev/null +++ b/Readme.md @@ -0,0 +1,52 @@ +# OONI Devops + +This repository contains the code necessary for managing the OONI +infrastructure as code and all the necessary tooling for day to day operations +of it. + +## Setup + +* Install [terraform](https://developer.hashicorp.com/terraform/install) +* Install [ansible](https://docs.ansible.com/ansible/latest/installation_guide/intro_installation.html) + +## Using + +For most up to date information, always look at the github workflow. + +You should have setup the following environment variables: +``` +AWS_ACCESS_KEY_ID=XXXX +AWS_SECRET_ACCESS_KEY=YYYY +TF_VAR_aws_access_key_id=XXX +TF_VAR_aws_secret_access_key=YYYY +TF_VAR_datadog_api_key=ZZZZ +``` + +### Deploying IaC + +``` +cd tf/environments/production/ +terraform plan +``` + +Check the plan looks good, then apply: + +``` +terraform apply +``` + +This will update the ansible inventory file. + +### Deploying Configuration + +You can now run: +``` +ansible-playbook -i inventory.ini --check --diff playbook.yml +``` + +And the apply it with: + +``` +ansible-playbook -i inventory.ini playbook.yml +``` + diff --git a/tf/environments/production/ansible/inventory.ini b/tf/environments/production/ansible/inventory.ini index d81954c4..bbc18cea 100755 --- a/tf/environments/production/ansible/inventory.ini +++ b/tf/environments/production/ansible/inventory.ini @@ -5,3 +5,4 @@ clickhouse.tier1.prod.ooni.nu [clickhouse_servers] clickhouse.tier1.prod.ooni.nu + diff --git a/tf/environments/production/main.tf b/tf/environments/production/main.tf index 0fc073ea..827dd2a2 100644 --- a/tf/environments/production/main.tf +++ b/tf/environments/production/main.tf @@ -11,10 +11,6 @@ terraform { } } -# You cannot create a new backend by simply defining this and then -# immediately proceeding to "terraform apply". The S3 backend must -# be bootstrapped according to the simple yet essential procedure in -# https://github.com/cloudposse/terraform-aws-tfstate-backend#usage # You cannot create a new backend by simply defining this and then # immediately proceeding to "terraform apply". The S3 backend must # be bootstrapped according to the simple yet essential procedure in @@ -166,16 +162,42 @@ resource "aws_instance" "clickhouse_server_prod_tier1" { ) } -resource "aws_ebs_volume" "clickhouse_data_volume" { - availability_zone = aws_instance.clickhouse_server_prod_tier1.availability_zone - size = 1024 # 1 TB - type = "gp3" # SSD-based volume type, provides up to 16,000 IOPS and 1,000 MiB/s throughput - tags = local.tags +# We care to ensure this data volume is not destroyed across re-applies. To do +# that you can either run first an apply with this commented out and then +# specify the data volume below. You can also just create a data volume with the +# appropriate tag manually and then edit the section below to indicate the name. +# If you do that, you will then have to manually also run: +# $ terraform state rm aws_ebs_volume.clickhouse_data_volume +#resource "aws_ebs_volume" "clickhouse_data_volume" { +# availability_zone = aws_instance.clickhouse_server_prod_tier1.availability_zone +# size = 1024 # 1 TB +# type = "gp3" # SSD-based volume type, provides up to 16,000 IOPS and 1,000 MiB/s throughput +# tags = merge(local.tags, { +# Name = "ooni-tier1-prod-clickhouse-vol1" +# }) +# +# lifecycle { +# prevent_destroy = true +# } +#} + +data "aws_ebs_volume" "clickhouse_data_volume" { + most_recent = true + + filter { + name = "tag:Name" + values = ["ooni-tier1-prod-clickhouse-vol1"] + } + + filter { + name = "availability-zone" + values = [aws_instance.clickhouse_server_prod_tier1.availability_zone] + } } resource "aws_volume_attachment" "clickhouse_data_volume_attachment" { device_name = local.clickhouse_device_name - volume_id = aws_ebs_volume.clickhouse_data_volume.id + volume_id = data.aws_ebs_volume.clickhouse_data_volume.id instance_id = aws_instance.clickhouse_server_prod_tier1.id force_detach = true } diff --git a/tf/environments/production/templates/clickhouse-setup.sh b/tf/environments/production/templates/clickhouse-setup.sh index 50a307cc..4ebc4f5e 100644 --- a/tf/environments/production/templates/clickhouse-setup.sh +++ b/tf/environments/production/templates/clickhouse-setup.sh @@ -4,9 +4,9 @@ sudo hostnamectl set-hostname --static ${hostname} # Install datadog agent DD_API_KEY=${datadog_api_key} DD_SITE="datadoghq.eu" bash -c "$(curl -L https://s3.amazonaws.com/dd-agent/scripts/install_script_agent7.sh)" -sudo mkfs.ext4 -q -F ${device_name} +# This only needs to be run the first time to initialize the volume +# sudo mkfs.ext4 -q -F ${device_name} sudo mkdir -p /var/lib/clickhouse sudo mount ${device_name} /var/lib/clickhouse echo "${device_name} /var/lib/clickhouse ext4 defaults,nofail 0 2" | sudo tee -a /etc/fstab sudo chown -R clickhouse:clickhouse /var/lib/clickhouse -