chore: refactoring terraform script

use GTK for Terraform to replace native GKE and ASM provisioning and configuration. temporary disable output variables.
GoogleCloudPlatform · Jan 10, 2024 · 85e38e8 · 85e38e8
1 parent b782bbc
commit 85e38e8
Show file tree

Hide file tree

Showing 7 changed files with 106 additions and 137 deletions.
diff --git a/provisioning/terraform/asm.tf b/provisioning/terraform/asm.tf
@@ -13,36 +13,13 @@
 # limitations under the License.
 
 
-resource "null_resource" "install_asm" {
-  count = var.enable_asm ? 1 : 0
+module "asm" {
+  source = "terraform-google-modules/kubernetes-engine/google//modules/asm"
 
-  triggers = {
-    project_id       = var.gcp_project_id
-    cluster_name     = google_container_cluster.sandbox.name
-    cluster_location = google_container_cluster.sandbox.location
-  }
-
-  provisioner "local-exec" {
-    interpreter = ["bash", "-exc"]
-    command     = <<-EOT
-      ./scripts/install_asm.sh --project ${self.triggers.project_id} \
-        --channel ${var.asm_channel} \
-        --cluster_name ${self.triggers.cluster_name} \
-        --cluster_location ${self.triggers.cluster_location}
-EOT
-  }
-
-  provisioner "local-exec" {
-    when    = destroy
-    command = <<-EOT
-      gcloud container fleet memberships unregister ${self.triggers.cluster_name} \
-        --gke-cluster '${self.triggers.cluster_location}/${self.triggers.cluster_name}' \
-        --project=${self.triggers.project_id}
-EOT
-  }
-
-  depends_on = [
-    resource.google_container_cluster.sandbox,
-    module.gcloud,
-  ]
+  project_id                = var.project_id
+  cluster_name              = module.gke.name
+  cluster_location          = module.gke.location
+  enable_mesh_feature       = true
+  enable_fleet_registration = true
+  count                     = var.enable_asm == true ? 1 : 0
 }
diff --git a/provisioning/terraform/cloudops.tf b/provisioning/terraform/cloudops.tf
@@ -18,7 +18,7 @@ module "monitoring" {
   gcp_project_number   = data.google_project.info.number
   enable_asm           = var.enable_asm
   frontend_external_ip = data.kubernetes_service.frontend_external_service.status[0].load_balancer[0].ingress[0].ip
-  gke_cluster_name     = var.gke_cluster_name
+  gke_cluster_name     = var.cluster_name
   # re-use prefix to customize resources within the same project
   name_suffix = length(var.state_prefix) > 0 ? "-${var.state_prefix}" : ""
 

diff --git a/provisioning/terraform/gke.tf b/provisioning/terraform/gke.tf
@@ -13,69 +13,65 @@
 # limitations under the License.
 
 locals {
-  location_label  = length(split("-", var.gke_cluster_location)) == 2 ? "--region" : (length(split("-", var.gke_cluster_location)) == 3 ? "--zone" : "--location")
+  zones           = length(split("-", var.cluster_location)) == 3 ? [var.cluster_location] : []
+  region          = length(split("-", var.cluster_location)) == 2 ? var.cluster_location : null
   resource_labels = var.enable_asm ? { "mesh_id" = "proj-${data.google_project.info.number}" } : {}
 }
 
-resource "google_container_cluster" "sandbox" {
-  name     = var.gke_cluster_name
-  location = var.gke_cluster_location
+# look at https://registry.terraform.io/modules/terraform-google-modules/kubernetes-engine/google/latest
+module "gke" {
+  source = "terraform-google-modules/kubernetes-engine/google"
 
-  release_channel {
-    channel = "STABLE"
-  }
+  project_id                 = var.project_id
+  name                       = var.cluster_name
+  description                = "Provisioned for Cloud Ops Sandbox version ${file("../version.txt")}"
+  region                     = local.region
+  regional                   = (local.region != null)
+  zones                      = local.zones
+  cluster_resource_labels    = local.resource_labels
+  network                    = var.cluster_network
+  subnetwork                 = var.cluster_subnetwork
+  ip_range_pods              = ""
+  ip_range_services          = ""
+  http_load_balancing        = true
+  network_policy             = false
+  horizontal_pod_autoscaling = true
+  filestore_csi_driver       = false
+  create_service_account     = false
+  deletion_protection        = false
 
-  gateway_api_config {
-    channel = "CHANNEL_STANDARD"
-  }
+  gateway_api_channel = "CHANNEL_STANDARD"
+  release_channel     = "STABLE"
+  identity_namespace  = "enabled"
 
-  resource_labels = local.resource_labels
+  node_pools = [
+    {
+      name               = "default-node-pool"
+      initial_node_count = var.node_pool_config.initial_node_count
+      machine_type       = var.node_pool_config.machine_type
+      min_count          = var.node_pool_config.min_count
+      max_count          = var.node_pool_config.max_count
 
-  description = "Provisioned for Cloud Ops Sandbox version ${file("../version.txt")}"
+    },
+  ]
 
-  # Enables Workload Identity
-  workload_identity_config {
-    workload_pool = "${data.google_project.info.project_id}.svc.id.goog"
+  node_pools_oauth_scopes = {
+    all = ["https://www.googleapis.com/auth/cloud-platform"]
   }
 
-  # Configures default node pool
-  node_pool {
-    initial_node_count = var.gke_node_pool.initial_node_count
-
-    node_config {
-      machine_type = var.gke_node_pool.machine_type
-      labels       = var.gke_node_pool.labels
-      oauth_scopes = ["https://www.googleapis.com/auth/cloud-platform"]
-
-      # Enables Workload Identity
-      workload_metadata_config {
-        mode = "GKE_METADATA"
-      }
-    }
+  node_pools_labels = {
+    all = {}
 
-    dynamic "autoscaling" {
-      for_each = var.gke_node_pool.autoscaling != null ? [var.gke_node_pool.autoscaling] : []
-      content {
-        min_node_count = autoscaling.value.min_node_count
-        max_node_count = autoscaling.value.max_node_count
-      }
-    }
+    default-node-pool = var.node_pool_config.labels
   }
 
-  depends_on = [
-    module.enable_google_apis
-  ]
-}
+  node_pools_tags = {
+    all = []
 
-module "gcloud" {
-  source  = "terraform-google-modules/gcloud/google"
-  version = "~> 3.1.0"
-
-  platform              = "linux"
-  additional_components = ["kubectl", "beta"]
+    default-node-pool = [
+      "default-node-pool",
+    ]
+  }
 
-  create_cmd_entrypoint = "gcloud"
-  # Module does not support explicit dependency
-  # Use 'local.cluster_name' to enforce implicit dependency because 'depends_on' is not available for this module
-  create_cmd_body = "container clusters get-credentials ${resource.google_container_cluster.sandbox.name} ${local.location_label}=${resource.google_container_cluster.sandbox.location} --project=${var.gcp_project_id}"
+  depends_on = [module.google_apis]
 }
diff --git a/provisioning/terraform/output.tf b/provisioning/terraform/output.tf
@@ -12,6 +12,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-output "frontend_external_ip" {
-  value = length(data.kubernetes_service.frontend_external_service.status) > 0 ? data.kubernetes_service.frontend_external_service.status[0].load_balancer[0].ingress[0].ip : null
-}
+# output "frontend_external_ip" {
+#   value = length(data.kubernetes_service.frontend_external_service.status) > 0 ? data.kubernetes_service.frontend_external_service.status[0].load_balancer[0].ingress[0].ip : null
+# }
diff --git a/provisioning/terraform/project.tf b/provisioning/terraform/project.tf
@@ -21,22 +21,21 @@ locals {
   ]
   mesh_apis = [
     "mesh.googleapis.com",
-    #    "meshtelemetry.googleapis.com",
+    "gkehub.googleapis.com",
     "cloudresourcemanager.googleapis.com",
   ]
+  google_apis = concat(local.base_apis, var.enable_asm ? local.mesh_apis : [])
 }
 
 # Enable Google Cloud APIs
-module "enable_google_apis" {
-  source  = "terraform-google-modules/project-factory/google//modules/project_services"
-  version = "~> 14.1.0"
+module "google_apis" {
+  source = "terraform-google-modules/project-factory/google//modules/project_services"
 
-  project_id                  = var.gcp_project_id
+  project_id                  = var.project_id
   disable_services_on_destroy = false
-
-  activate_apis = concat(local.base_apis, var.enable_asm ? local.mesh_apis : [])
+  activate_apis               = local.google_apis
 }
 
 data "google_project" "info" {
-  project_id = var.gcp_project_id
+  project_id = var.project_id
 }
diff --git a/provisioning/terraform/providers.tf b/provisioning/terraform/providers.tf
@@ -25,48 +25,45 @@ terraform {
   required_providers {
     google = {
       source  = "hashicorp/google"
-      version = "~> 4.54.0"
+      version = ">= 5.0"
     }
     google-beta = {
       source  = "hashicorp/google-beta"
-      version = "4.54.0"
+      version = ">= 5.0"
     }
     null = {
       source  = "hashicorp/null"
-      version = "~>3.2.1"
+      version = "~> 3.2.1"
     }
     kubernetes = {
       source  = "hashicorp/kubernetes"
-      version = "~>2.18.1"
+      version = "~> 2.18.1"
     }
   }
-  backend "gcs" {}
+  #  backend "gcs" {}
 }
 
 # tflint-ignore: terraform_unused_declarations
-data "terraform_remote_state" "state" {
-  backend = "gcs"
-  config = {
-    bucket = var.state_bucket_name
-    prefix = var.state_prefix
-  }
-}
+# data "terraform_remote_state" "state" {
+#   backend = "gcs"
+#   config = {
+#     bucket = var.state_bucket_name
+#     prefix = var.state_prefix
+#   }
+# }
 
 provider "google" {
-  project = var.gcp_project_id
+  project = var.project_id
 }
 
-# Retrieve an access token as the Terraform runner
-data "google_client_config" "default" {}
-
 provider "google-beta" {
-  project = var.gcp_project_id
+  project = var.project_id
 }
 
+data "google_client_config" "default" {}
+
 provider "kubernetes" {
-  host  = "https://${resource.google_container_cluster.sandbox.endpoint}"
-  token = data.google_client_config.default.access_token
-  cluster_ca_certificate = base64decode(
-    resource.google_container_cluster.sandbox.master_auth[0].cluster_ca_certificate,
-  )
+  host                   = "https://${module.gke.endpoint}"
+  token                  = data.google_client_config.default.access_token
+  cluster_ca_certificate = base64decode(module.gke.ca_certificate)
 }
diff --git a/provisioning/terraform/variables.tf b/provisioning/terraform/variables.tf
@@ -15,7 +15,7 @@
  */
 
 # Required input variables
-variable "gcp_project_id" {
+variable "project_id" {
   type        = string
   description = "The GCP project ID to apply this config to"
 }
@@ -25,17 +25,6 @@ variable "state_bucket_name" {
   description = "The GCS bucket URL where Terraform stores the state"
 }
 
-# Optional input variables
-variable "asm_channel" {
-  type        = string
-  description = "Defines one of the following managed ASM channels/revisions: 'rapid', 'regular' or stable'"
-  default     = "stable"
-  validation {
-    condition     = can(regex("^(rapid|regular|stable)$", var.asm_channel))
-    error_message = "ASM channel/revision can be only 'rapid', 'regular' or stable'"
-  }
-}
-
 variable "enable_asm" {
   type        = bool
   description = "If true, installs Anthos Service Mesh (managed version of Istio) on the GKE cluster"
@@ -48,37 +37,48 @@ variable "filepath_manifest" {
   default     = "../kustomize/online-boutique/"
 }
 
-variable "gke_cluster_name" {
+variable "cluster_name" {
   type        = string
   description = "Name given to the new GKE cluster"
   default     = "cloud-ops-sandbox"
 }
 
-variable "gke_cluster_location" {
+variable "cluster_location" {
   type        = string
   description = "Region or zone of the new GKE cluster"
   default     = "us-central1"
 }
 
+variable "cluster_network" {
+  type        = string
+  description = " The VPC network to host the cluster in"
+  default     = "default"
+}
+
+variable "cluster_subnetwork" {
+  type        = string
+  description = " The subnetwork to host the cluster in"
+  default     = "default"
+}
+
+
 # Default values for node pool support connecting the cluster to ASM
 # https://cloud.google.com/service-mesh/docs/unified-install/anthos-service-mesh-prerequisites#cluster_requirements
-variable "gke_node_pool" {
+variable "node_pool_config" {
   type = object({
     initial_node_count = number
     labels             = map(string)
     machine_type       = string
-
-    autoscaling = object({
-      max_node_count = number
-      min_node_count = number
-    })
+    max_node_count     = number
+    min_node_count     = number
   })
   description = "Initial settings and autoscale configuration of the GKE cluster's default node pool"
   default = {
-    initial_node_count = 4
+    initial_node_count = 3
     labels             = {}
     machine_type       = "e2-standard-4"
-    autoscaling        = null
+    min_count          = 3
+    max_count          = 3
   }
 }