From 2e5e77add9c9bd09d3a25afc3b2d9b51e41aced1 Mon Sep 17 00:00:00 2001 From: Pedro Guimaraes Date: Fri, 15 Nov 2024 12:38:15 +0100 Subject: [PATCH 1/7] Add terraform module --- .github/workflows/ci.yaml | 50 ++++++++++++++++ .gitignore | 47 +++++++++++++++ terraform/README.md | 90 ++++++++++++++++++++++++++++ terraform/main.tf | 41 +++++++++++++ terraform/outputs.tf | 26 ++++++++ terraform/tests/preamble.tf | 18 ++++++ terraform/tests/providers.tf | 16 +++++ terraform/tests/simple_deployment.tf | 89 +++++++++++++++++++++++++++ terraform/tests/variables.tf | 22 +++++++ terraform/variables.tf | 67 +++++++++++++++++++++ terraform/versions.tf | 12 ++++ 11 files changed, 478 insertions(+) create mode 100644 terraform/README.md create mode 100644 terraform/main.tf create mode 100644 terraform/outputs.tf create mode 100644 terraform/tests/preamble.tf create mode 100644 terraform/tests/providers.tf create mode 100644 terraform/tests/simple_deployment.tf create mode 100644 terraform/tests/variables.tf create mode 100644 terraform/variables.tf create mode 100644 terraform/versions.tf diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 22d48146..f231b487 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -62,6 +62,56 @@ jobs: - name: Run tests run: tox run -e unit + terraform-test: + name: Terraform - Lint and Simple Deployment + runs-on: ubuntu-latest + timeout-minutes: 120 + steps: + - name: Checkout repo + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: lint charm module + run: | + pushd ./terraform + terraform init + terraform fmt + # -test-directory is not really working, it is ignoring some errors + terraform validate + pushd ./tests + terraform init + terraform fmt + # -test-directory is not really working, it is ignoring some errors + terraform validate + popd + popd + - name: run checks - prepare + run: | + sudo snap install juju --channel=3.6/beta --classic + sudo snap install juju-wait --channel=latest/stable --classic + sudo snap install jq + - name: LXD setup + run: | + sudo snap refresh lxd --channel=latest/stable + sudo adduser "$USER" 'lxd' + # `newgrp` does not work in GitHub Actions; use `sg` instead + sg 'lxd' -c "lxd waitready" + sg 'lxd' -c "lxd init --auto" + sg 'lxd' -c "lxc network set lxdbr0 ipv6.address none" + sudo iptables -F FORWARD + sudo iptables -P FORWARD ACCEPT + - name: Juju setup + run: | + sg 'lxd' -c "juju bootstrap 'localhost' --config model-logs-size=10G" + juju model-defaults logging-config='=INFO; unit=DEBUG' + juju add-model test + - name: Terraform deploy + run: | + pushd ./terraform/tests/ + TF_VAR_model_name="test" terraform apply -target null_resource.simple_deployment_juju_wait_deployment -auto-approve + popd + build: strategy: matrix: diff --git a/.gitignore b/.gitignore index 51ccad86..ffa1ed05 100644 --- a/.gitignore +++ b/.gitignore @@ -16,3 +16,50 @@ cloudinit-userdata.yaml # Moving to Poetry, we do not need this file to be pushed any longer /requirements.txt /requirements-last-build.txt + +######################################################## +# +# Terraform .gitignore +# +######################################################## + + +# Local .terraform directories +**/.terraform/* +*.terraform.lock.hcl + +# .tfstate files +*.tfstate +*.tfstate.* + +# Crash log files +crash.log +crash.*.log + +# Generated files +*.key +credentials* + +# Exclude all .tfvars files, which are likely to contain sensitive data, such as +# password, private keys, and other secrets. These should not be part of version +# control as they are data points which are potentially sensitive and subject +# to change depending on the environment. +*.tfvars +*.tfvars.json + +# Ignore override files as they are usually used to override resources locally and so +# are not checked in +override.tf +override.tf.json +*_override.tf +*_override.tf.json + +# Include override files you do wish to add to version control using negated pattern +# !example_override.tf + +# Include tfplan files to ignore the plan output of command: terraform plan -out=tfplan +# example: *tfplan* + +# Ignore CLI configuration files +.terraformrc +terraform.rc \ No newline at end of file diff --git a/terraform/README.md b/terraform/README.md new file mode 100644 index 00000000..16c2be3a --- /dev/null +++ b/terraform/README.md @@ -0,0 +1,90 @@ +# Terraform module for opensearch-dashboards-operator + +This is a Terraform module facilitating the deployment of the OpenSearch charm with [Terraform juju provider](https://github.com/juju/terraform-provider-juju/). For more information, refer to the provider [documentation](https://registry.terraform.io/providers/juju/juju/latest/docs). + +## Requirements +This module requires a `juju` model to be available. Refer to the [usage section](#usage) below for more details. + +## API + +### Inputs +The module offers the following configurable inputs: + +| Name | Type | Description | Required | +| - | - | - | - | +| `app_name`| string | Application name | False | +| `channel`| string | Channel that the charm is deployed from | False | +| `base`| string | The series to be used for this charm | False | +| `config`| map(string) | Map of the charm configuration options | False | +| `model_name`| string | Name of the model that the charm is deployed on | True | +| `resources`| map(string) | Map of the charm resources | False | +| `revision`| number | Revision number of the charm name | False | +| `units`| number | Number of units to be deployed | False | +| `constraints`| string | Machine constraints for the charm | False | + + +### Outputs +Upon applied, the module exports the following outputs: + +| Name | Description | +| - | - | +| `app_name`| Application name | +| `provides`| Map of `provides` endpoints | +| `requires`| Map of `requires` endpoints | + +## Usage + +This module is intended to be used as part of a higher-level module. When defining one, users should ensure that Terraform is aware of the `juju_model` dependency of the charm module. There are two options to do so when creating a high-level module: + +### Define a `juju_model` resource +Define a `juju_model` resource and pass to the `model_name` input a reference to the `juju_model` resource's name. For example: + +``` +resource "juju_model" "opensearch" { + name = opensearch +} + +module "opensearch-dashboards" { + source = "" + model_name = juju_model.opensearch.name +} +``` + +### Define a `data` source +Define a `data` source and pass to the `model_name` input a reference to the `data.juju_model` resource's name. This will enable Terraform to look for a `juju_model` resource with a name attribute equal to the one provided, and apply only if this is present. Otherwise, it will fail before applying anything. + +``` +data "juju_model" "opensearch" { + name = var.model_name +} + +module "opensearch-dashboards" { + source = "" + model_name = data.juju_model.opensearch.name +} +``` + +### Deploy OpenSearch Module and Relate + +Follow the description in: https://github.com/canonical/opensearch-operator/, in the `terraform` folder. Deploy an opensearch +cluster and relate both of them together: + +``` +module "opensearch" { + ... +} + +resource "juju_integration" "opensearch-and-dashboard" { + model = juju_model.opensearch.name + + application { + name = module.opensearch.app_name + endpoint = module.opensearch.opensearch_client_endpoint + } + + application { + name = module.opensearch-dashboards.app_name + endpoint = module.opensearch-dashboards.opensearch_client_endpoint + } +} +``` \ No newline at end of file diff --git a/terraform/main.tf b/terraform/main.tf new file mode 100644 index 00000000..b84dc6da --- /dev/null +++ b/terraform/main.tf @@ -0,0 +1,41 @@ +# Copyright 2024 Canonical Ltd. +# See LICENSE file for licensing details. + +resource "juju_application" "opensearch-dashboards" { + + charm { + name = "opensearch-dashboards" + channel = var.channel + revision = var.revision + base = var.base + } + config = var.config + model = var.model + name = var.app_name + units = var.units + constraints = var.constraints + + + # TODO: uncomment once final fixes have been added for: + # Error: juju/terraform-provider-juju#443, juju/terraform-provider-juju#182 + # placement = join(",", var.machines) + + endpoint_bindings = [ + for k, v in var.endpoint_bindings : { + endpoint = k, space = v + } + ] + + storage_directives = var.storage + + lifecycle { + precondition { + condition = length(var.machines) == 0 || (length(var.machines) > 0 && length(var.machines) == var.units) + error_message = "Machine count does not match unit count" + } + precondition { + condition = length(var.storage) == 0 || (length(var.storage) > 0 && (lookup(var.storage, "count", 0) <= 1) && (lookup(var.storage, "count", 0) > 0)) + error_message = "Only one storage is supported" + } + } +} diff --git a/terraform/outputs.tf b/terraform/outputs.tf new file mode 100644 index 00000000..218b5635 --- /dev/null +++ b/terraform/outputs.tf @@ -0,0 +1,26 @@ +# Copyright 2024 Canonical Ltd. +# See LICENSE file for licensing details. + +output "app_name" { + description = "Name of the deployed application." + value = juju_application.opensearch-dashboards.name +} + +# Required integration endpoints + +output "certificates_endpoint" { + description = "Name of the endpoint used to integrate with the TLS certificates provider." + value = "certificates" +} + +output "opensearch_client_endpoint" { + description = "Name of the endpoint opensearch-client endpoint." + value = "opensearch-client" +} + +# Provided integration endpoints + +output "cos_agent_endpoint" { + description = "Name of the endpoint used to provide COS agent integration." + value = "cos-agent-endpoint" +} diff --git a/terraform/tests/preamble.tf b/terraform/tests/preamble.tf new file mode 100644 index 00000000..efd68202 --- /dev/null +++ b/terraform/tests/preamble.tf @@ -0,0 +1,18 @@ +resource "null_resource" "preamble" { + provisioner "local-exec" { + command = <<-EOT + sudo snap install juju-wait --classic || true + sudo sysctl -w vm.max_map_count=262144 vm.swappiness=0 net.ipv4.tcp_retries2=5 + EOT + } + +} + +resource "juju_application" "self-signed-certificates" { + charm { + name = "self-signed-certificates" + channel = "latest/stable" + } + model = var.model_name + depends_on = [null_resource.preamble] +} \ No newline at end of file diff --git a/terraform/tests/providers.tf b/terraform/tests/providers.tf new file mode 100644 index 00000000..d3b63473 --- /dev/null +++ b/terraform/tests/providers.tf @@ -0,0 +1,16 @@ +terraform { + required_providers { + juju = { + source = "juju/juju" + version = "~> 0.14.0" + } + http = { + source = "hashicorp/http" + version = "~> 3.4.5" + } + external = { + source = "hashicorp/external" + version = "~> 2.3.4" + } + } +} \ No newline at end of file diff --git a/terraform/tests/simple_deployment.tf b/terraform/tests/simple_deployment.tf new file mode 100644 index 00000000..30333454 --- /dev/null +++ b/terraform/tests/simple_deployment.tf @@ -0,0 +1,89 @@ +module "opensearch-dashboards" { + source = "../" + app_name = var.app_name + model = var.model_name + units = var.simple_opensearch_dashboards_units + + channel = "2/edge" + + depends_on = [juju_application.self-signed-certificates] +} + +module "opensearch" { + source = "git::https://github.com/canonical/opensearch-operator//terraform?ref=DPE-5866-terraform" + app_name = "opensearch" + model = var.model_name + units = var.simple_opensearch_units + config = { + profile = "testing" + } + + channel = "2/edge" + + depends_on = [juju_application.self-signed-certificates] +} + +resource "juju_integration" "dashboards_opensearch-integration" { + model = var.model_name + + application { + name = module.opensearch-dashboards.app_name + endpoint = module.opensearch-dashboards.opensearch_client_endpoint + } + application { + name = module.opensearch.app_name + endpoint = module.opensearch.opensearch_client_endpoint + } + depends_on = [ + module.opensearch-dashboards, + module.opensearch + ] + +} + +resource "juju_integration" "simple_deployment_tls-operator_opensearch-dashboards-integration" { + model = var.model_name + + application { + name = juju_application.self-signed-certificates.name + } + application { + name = module.opensearch.app_name + } + depends_on = [ + juju_application.self-signed-certificates, + module.opensearch + ] + +} + + +resource "juju_integration" "simple_deployment_tls-operator_opensearch-integration" { + model = var.model_name + + application { + name = juju_application.self-signed-certificates.name + } + application { + name = module.opensearch.app_name + } + depends_on = [ + juju_application.self-signed-certificates, + module.opensearch + ] + +} + +resource "null_resource" "simple_deployment_juju_wait_deployment" { + provisioner "local-exec" { + command = <<-EOT + juju-wait -v --model ${var.model_name} + EOT + } + + depends_on = [ + juju_integration.simple_deployment_tls-operator_opensearch-integration, + juju_integration.dashboards_opensearch-integration, + imple_deployment_tls-operator_opensearch-dashboards-integration, + ] +} diff --git a/terraform/tests/variables.tf b/terraform/tests/variables.tf new file mode 100644 index 00000000..efbc3749 --- /dev/null +++ b/terraform/tests/variables.tf @@ -0,0 +1,22 @@ +variable "model_name" { + description = "Model name" + type = string +} + +variable "app_name" { + description = "OpenSearch app name" + type = string + default = "opensearch-dashboards" +} + +variable "simple_opensearch_units" { + description = "Node count" + type = number + default = 1 +} + +variable "simple_opensearch_dashboards_units" { + description = "OpenSearch dashboards node count" + type = number + default = 1 +} \ No newline at end of file diff --git a/terraform/variables.tf b/terraform/variables.tf new file mode 100644 index 00000000..b147a061 --- /dev/null +++ b/terraform/variables.tf @@ -0,0 +1,67 @@ +# Copyright 2024 Canonical Ltd. +# See LICENSE file for licensing details. + +variable "app_name" { + description = "Application name" + type = string + default = "opensearch-dashboards" +} + +variable "channel" { + description = "Charm channel" + type = string + default = null +} + +variable "base" { + description = "Charm base (old name: series)" + type = string + default = "ubuntu@22.04" +} + +variable "config" { + description = "Map of charm configuration options" + type = map(string) + default = {} +} + +variable "model" { + description = "Model name" + type = string +} + +variable "revision" { + description = "Charm revision" + type = number + default = null +} + +variable "units" { + description = "Charm units" + type = number + default = 1 +} + +variable "constraints" { + description = "String listing constraints for this application" + type = string + default = "arch=amd64" +} + +variable "machines" { + description = "List of machines for placement" + type = list(string) + default = [] +} + +variable "storage" { + description = "Map of storage used by the application" + type = map(string) + default = {} +} + +variable "endpoint_bindings" { + description = "Map of endpoint bindings" + type = map(string) + default = {} +} \ No newline at end of file diff --git a/terraform/versions.tf b/terraform/versions.tf new file mode 100644 index 00000000..3d6e5a5f --- /dev/null +++ b/terraform/versions.tf @@ -0,0 +1,12 @@ +# Copyright 2024 Canonical Ltd. +# See LICENSE file for licensing details. + +terraform { + required_version = ">= 1.6" + required_providers { + juju = { + source = "juju/juju" + version = "~> 0.14.0" + } + } +} \ No newline at end of file From 50251e4c61771a503c5d16f59dbf00f78b22994e Mon Sep 17 00:00:00 2001 From: Pedro Guimaraes Date: Fri, 15 Nov 2024 12:39:51 +0100 Subject: [PATCH 2/7] Extend size to 3x units per deployment --- terraform/tests/variables.tf | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/terraform/tests/variables.tf b/terraform/tests/variables.tf index efbc3749..c652af46 100644 --- a/terraform/tests/variables.tf +++ b/terraform/tests/variables.tf @@ -12,11 +12,11 @@ variable "app_name" { variable "simple_opensearch_units" { description = "Node count" type = number - default = 1 + default = 3 } variable "simple_opensearch_dashboards_units" { description = "OpenSearch dashboards node count" type = number - default = 1 -} \ No newline at end of file + default = 3 +} From 6790019fbda062384f9bff89f0b21a65e239fad1 Mon Sep 17 00:00:00 2001 From: Pedro Guimaraes Date: Fri, 15 Nov 2024 13:45:46 +0100 Subject: [PATCH 3/7] Fix the typo in the simple_deployment.tf --- terraform/tests/simple_deployment.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/terraform/tests/simple_deployment.tf b/terraform/tests/simple_deployment.tf index 30333454..04dad109 100644 --- a/terraform/tests/simple_deployment.tf +++ b/terraform/tests/simple_deployment.tf @@ -84,6 +84,6 @@ resource "null_resource" "simple_deployment_juju_wait_deployment" { depends_on = [ juju_integration.simple_deployment_tls-operator_opensearch-integration, juju_integration.dashboards_opensearch-integration, - imple_deployment_tls-operator_opensearch-dashboards-integration, + juju_integration.simple_deployment_tls-operator_opensearch-dashboards-integration, ] } From bb882bd91223986c310802c093377a193795e613 Mon Sep 17 00:00:00 2001 From: Pedro Guimaraes Date: Fri, 15 Nov 2024 14:58:48 +0100 Subject: [PATCH 4/7] Fix simple deployment and variables to 1x clusters --- terraform/tests/simple_deployment.tf | 2 +- terraform/tests/variables.tf | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/terraform/tests/simple_deployment.tf b/terraform/tests/simple_deployment.tf index 04dad109..9694eb8e 100644 --- a/terraform/tests/simple_deployment.tf +++ b/terraform/tests/simple_deployment.tf @@ -65,7 +65,7 @@ resource "juju_integration" "simple_deployment_tls-operator_opensearch-integrati name = juju_application.self-signed-certificates.name } application { - name = module.opensearch.app_name + name = module.opensearch-dashboards.app_name } depends_on = [ juju_application.self-signed-certificates, diff --git a/terraform/tests/variables.tf b/terraform/tests/variables.tf index c652af46..c4515fb4 100644 --- a/terraform/tests/variables.tf +++ b/terraform/tests/variables.tf @@ -12,11 +12,11 @@ variable "app_name" { variable "simple_opensearch_units" { description = "Node count" type = number - default = 3 + default = 2 } variable "simple_opensearch_dashboards_units" { description = "OpenSearch dashboards node count" type = number - default = 3 + default = 1 } From b1e76f92d09a0134c21b3faf6b41567c0e0c4d1a Mon Sep 17 00:00:00 2001 From: Pedro Guimaraes Date: Mon, 25 Nov 2024 12:31:44 +0100 Subject: [PATCH 5/7] Update following opensearch charm TF --- .github/workflows/ci.yaml | 2 -- .gitignore | 2 +- terraform/README.md | 2 +- terraform/main.tf | 6 +----- terraform/tests/simple_deployment.tf | 2 +- 5 files changed, 4 insertions(+), 10 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index f231b487..c6c49009 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -77,12 +77,10 @@ jobs: pushd ./terraform terraform init terraform fmt - # -test-directory is not really working, it is ignoring some errors terraform validate pushd ./tests terraform init terraform fmt - # -test-directory is not really working, it is ignoring some errors terraform validate popd popd diff --git a/.gitignore b/.gitignore index ffa1ed05..41e33295 100644 --- a/.gitignore +++ b/.gitignore @@ -62,4 +62,4 @@ override.tf.json # Ignore CLI configuration files .terraformrc -terraform.rc \ No newline at end of file +terraform.rc diff --git a/terraform/README.md b/terraform/README.md index 16c2be3a..9a616243 100644 --- a/terraform/README.md +++ b/terraform/README.md @@ -87,4 +87,4 @@ resource "juju_integration" "opensearch-and-dashboard" { endpoint = module.opensearch-dashboards.opensearch_client_endpoint } } -``` \ No newline at end of file +``` diff --git a/terraform/main.tf b/terraform/main.tf index b84dc6da..925f92dc 100644 --- a/terraform/main.tf +++ b/terraform/main.tf @@ -30,12 +30,8 @@ resource "juju_application" "opensearch-dashboards" { lifecycle { precondition { - condition = length(var.machines) == 0 || (length(var.machines) > 0 && length(var.machines) == var.units) + condition = length(var.machines) == 0 || length(var.machines) == var.units error_message = "Machine count does not match unit count" } - precondition { - condition = length(var.storage) == 0 || (length(var.storage) > 0 && (lookup(var.storage, "count", 0) <= 1) && (lookup(var.storage, "count", 0) > 0)) - error_message = "Only one storage is supported" - } } } diff --git a/terraform/tests/simple_deployment.tf b/terraform/tests/simple_deployment.tf index 9694eb8e..0c0a56e0 100644 --- a/terraform/tests/simple_deployment.tf +++ b/terraform/tests/simple_deployment.tf @@ -10,7 +10,7 @@ module "opensearch-dashboards" { } module "opensearch" { - source = "git::https://github.com/canonical/opensearch-operator//terraform?ref=DPE-5866-terraform" + source = "git::https://github.com/canonical/opensearch-operator//terraform?ref=2/edge" app_name = "opensearch" model = var.model_name units = var.simple_opensearch_units From 1f39a227930d67512e260101b737c9110704e533 Mon Sep 17 00:00:00 2001 From: phvalguima Date: Mon, 25 Nov 2024 13:52:48 +0100 Subject: [PATCH 6/7] Update main.tf --- terraform/main.tf | 2 -- 1 file changed, 2 deletions(-) diff --git a/terraform/main.tf b/terraform/main.tf index 925f92dc..42c33939 100644 --- a/terraform/main.tf +++ b/terraform/main.tf @@ -26,8 +26,6 @@ resource "juju_application" "opensearch-dashboards" { } ] - storage_directives = var.storage - lifecycle { precondition { condition = length(var.machines) == 0 || length(var.machines) == var.units From 4b59abe0c1b67eb9182c313f2320a727a9fbbfda Mon Sep 17 00:00:00 2001 From: phvalguima Date: Mon, 25 Nov 2024 13:53:24 +0100 Subject: [PATCH 7/7] Update variables.tf --- terraform/variables.tf | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/terraform/variables.tf b/terraform/variables.tf index b147a061..b470d227 100644 --- a/terraform/variables.tf +++ b/terraform/variables.tf @@ -54,14 +54,8 @@ variable "machines" { default = [] } -variable "storage" { - description = "Map of storage used by the application" - type = map(string) - default = {} -} - variable "endpoint_bindings" { description = "Map of endpoint bindings" type = map(string) default = {} -} \ No newline at end of file +}