diff --git a/Makefile b/Makefile index 8dbec2b..ae8d641 100644 --- a/Makefile +++ b/Makefile @@ -16,52 +16,47 @@ # Export the required environment variables before using the make command. SHELL := /usr/bin/env bash ROOT := ${CURDIR} +provider := ${} .PHONY: help help: @echo 'Usage:' - @echo ' make init Initialize and configure Backend.' - @echo ' make plan Plan all GCP resources.' - @echo ' make apply Create or update GCP resources.' - @echo ' make destroy Destroy all GCP resources.' - @echo ' make validate Check that installed resources work as expected.' - @echo ' make lint Check syntax of all scripts.' - @echo ' make get pods Get running pods IP's and Namespaces run this command after apply' + @echo ' make init "provider=" Initialize and configure Terraform Backend.' + @echo ' make plan "provider=" Plan all Terraform resources.' + @echo ' make apply "provider=" Create or update Terraform resources.' + @echo ' make destroy "provider=" Destroy all Terraform resources.' + @echo ' make lint Check syntax of all scripts.' + @echo ' make getpods Get running pods IPs and Namespaces run this command after apply' @echo # Before you run this command please export the required variables. # Initialize the environment variables .PHONY: init init: - bash $(ROOT)/scripts/init.sh + bash $(ROOT)/$(provider)/scripts/init.sh -# Plan the google cloud resources +# Plan the Terraform resources .PHONY: plan plan: - bash $(ROOT)/scripts/plan.sh + bash $(ROOT)/$(provider)/scripts/plan.sh -# Apply the google cloud resources +# Apply the Terraform resources .PHONY: apply apply: - bash $(ROOT)/scripts/apply.sh + bash $(ROOT)/$(provider)/scripts/apply.sh # Destroy the terraform resources .PHONY: destroy destroy: - bash $(ROOT)/scripts/destroy.sh + bash $(ROOT)/$(provider)/scripts/destroy.sh -# Validate terraform resources -.PHONY: validate -validate: - bash ${ROOT}/scripts/validate.sh - -# Get pods details of the running container cluster. -.PHONY: get pods -get pods: +# Get pods details of the running cluster. +.PHONY: getpods +getpods: python ${ROOT}/test/kube-pods.py .PHONY: lint -lint: Check_shell check_terraform check_shebangs check_trailing_whitespace +lint: check_shell check_terraform check_shebangs check_trailing_whitespace # Shell check .PHONY: check_shell diff --git a/README.md b/README.md index 2b95d6e..157b5b7 100644 --- a/README.md +++ b/README.md @@ -30,8 +30,27 @@ A [Terraform Module](https://www.terraform.io/docs/language/modules/develop/inde Instead of figuring out the details of how to run a piece of infrastructure from scratch, you can reuse existing code that has been proven in production. And instead of maintaining all that infrastructure code yourself, you can leverage the work of the Module to pick up infrastructure improvements through a version number bump. +## Prerequisites +At a minimum 61 GiB of memory, 8 vCPUs virtual machines are needed to run k8ssandra. Minimum recommendation for volumes is 1.5 - 2 TB, but that's all set up through the persistent volume requests. -## GCP Prerequisites +## Resource Naming Conventions +* Naming Conventions: All the resources will be created with the prefix of `environment`-`project_name`. + - eg: environment="development" and Project_name="k8ssandra" + resource_name "development-k8ssandra-gke-cluster" + +* Naming Limitation: Every cloud provider have limitations on the resource names, they will only allow resource names up to some characters long. + - eg: If we pass `environment`=**production** the `project_name`=**K8ssandra-terraform-project-resources-for-multiple-cloud-providers** + your resource will create as resource_name =**production-K8ssandra-terraform-project-resources-for-multiple-cloud-providers-gke-cluster** + + * In the above example the resource name exceeds more than 63 characters long. It is an invalid resource name, these will error out when you run `Terraform plan` or `Terraform validate` commands. These limitations are hard limitations which can not be changed by your cloud provider. + make sure you followed naming standards while creating your resources. **It is a good practice maintain limits on length of resource names**. + + * refer the following documentation + * [azure](https://docs.microsoft.com/en-us/azure/azure-resource-manager/management/resource-name-rules). + * [gcp](https://stepan.wtf/cloud-naming-convention/) + + +### GCP Prerequisites | NAME | Version | |-------------------|------------| @@ -92,12 +111,70 @@ k8ssandra-terraform/ | README.md +## How to use Makefile +* Terraform scripts are in different folders choose your cloud provider to create resources in. + list of available providers "aws, gcp, azure " + * `make help` To list out the available options to use. + * `make init "provider="` Initialize and configure Backend. + * `make plan "provider="` Plan all Terraform resources. + * `make apply "provider="` Create or update Terraform resources. + * `make destroy "provider="` Destroy all Terraform resources. + * `make lint` Check syntax of all scripts. + * `make getpods` Get running pods IPs and Namespaces run this command after apply + ## Create GKE resources * Testing this project Locally [gcp](./gcp#test-this-project-locally) +* Set up environment on your machine before running the make commands. use the following links to setup your machine. + * [Tools](./gcp#Tools) + * [GCP-authentication](./gcp#GCP-authentication) + * [Configure-gcloud-settings](./gcp/#Configure-gcloud-settings) + * How to create GKE cluster resources by using the make command -[ WORK IN PROGRESS ] +Before using the make commands export the following terraform environment variables(TFVARS) for terraform to create the resources. + +```console + +export TF_VAR_environment= +ex:- export TF_VAR_environment=dev + +export TF_VAR_name= +ex:- export TF_VAR_name=k8ssandra + +export TF_VAR_project_id= +ex:- export TF_VAR_project_id=k8ssandra-testing + +export TF_VAR_region= +ex:- export TF_VAR_region=us-central-1 + +``` + +```console +#To list out the available options to use. +make help +``` +### important: Before running the following command, we need to Export the environment variables as show above. + +```console +# Initialize and configure Backend. +make init "provider=gcp" +``` +```console +# Plan all GCP resources. +make plan "provider=gcp" +``` +### This command will create a Kubernetes cluster and deploy k8ssandra on the cluster. +```console +# Create or update GCP resources +# This command takes some time to execute. +make apply "provider=gcp" +``` +```console +# Destroy all GCP resources + +make destroy "provider=gcp" +``` ## Create EKS resources @@ -112,8 +189,13 @@ k8ssandra-terraform/ ## Troubleshooting * **The create script fails with a `Permission denied` when running Terraform** - The credentials that Terraform is using do not provide the necessary permissions to create resources in the selected projects. Ensure that the account listed in `gcloud config list` has necessary permissions to create resources. If it does, regenerate the application default credentials using `gcloud auth application-default login`. + * **Terraform timeouts** - Sometimes resources may take longer than usual to create and Terraform will timeout. The solution is to just run `make create` again. Terraform should pick up where it left off. +* **Terraform statelock** - Sometime if two are more people working on the same Terraform statefile a lock will be placed on your remote Terraform statefile, to unlock the state run the following command `terraform force-unlock `. + +* **Terraform Incomplete resource deletion** - If you created some resources manually on the cloud console and attach those resources to the resources created by the Terraform, `terraform destroy` or `make destroy` commands will fail. To resolve those errors you will have to login into the cloud console, delete those resource manually and run `make destroy` or `terraform destory`. + ## Relevant Material * [Private GKE Clusters](https://cloud.google.com/kubernetes-engine/docs/how-to/creating-a-cluster) diff --git a/gcp/README.md b/gcp/README.md index 285b9c3..5c783f7 100644 --- a/gcp/README.md +++ b/gcp/README.md @@ -1,12 +1,12 @@ # K8ssandra GCP Terraform Example ## What is Google Kubernetes Engine (GKE)? -[Google Kubernetes Engine](https://cloud.google.com/kubernetes-engine) or "GKE" is a Google-managed Kubernetes environment. GKE is a fully managed experience; it handles the management/upgrading of the Kubernetes cluster master as well as autoscaling of "nodes" through "node pool" templates. +[Google Kubernetes Engine](https://cloud.google.com/kubernetes-engine) or "GKE" is a Google-managed Kubernetes environment. GKE is a fully managed experience; it handles the management/upgrading of the Kubernetes cluster master as well as Autoscaling of "nodes" through "node pool" templates. Through GKE, your Kubernetes deployments will have first-class support for GCP IAM identities, built-in configuration of high-availability and secured clusters, as well as native access to GCP's networking features such as load balancers. ## What is GKE Node Pool ? -GKE Node Pools are a group of nodes who share the same configuration, defined as a NodeConfig. Node pools also control the autoscaling of their nodes, and autoscaling configuration is done inline, alongside the node config definition. A GKE Cluster can have multiple node pools defined. Initially by default we have only defined a `1` node pool. +GKE Node Pools are a group of nodes who share the same configuration, defined as a NodeConfig. Node pools also control the Autoscaling of their nodes, and Autoscaling configuration is done inline, alongside the node config definition. A GKE Cluster can have multiple node pools defined. Initially by default we have only defined a `1` node pool. ## VPC Network You must explicitly specify the network and subnetwork of your GKE cluster using the network and subnetwork fields; We will not use the default network with an automatically generated subnetwork. @@ -16,16 +16,13 @@ You must explicitly specify the network and subnetwork of your GKE cluster using * GKE cluster * Cluster node pool * Service account -* Iam member -* Custom Iam member +* Iam members * Google compute network(VPC) -* Public subnet -* Private subnet +* Subnet * Router * Compute router NAT * Google storage bucket * Google storage bucket IAM member -* Google compute address ## Project directory Structure @@ -85,7 +82,7 @@ The steps to create kubernetes cluster in this document require the following to ### Cloud project -You will need a google cloud project created, If you do not have a google cloud account plese sigup [google](https://cloud.google.com). +You will need a google cloud project created, If you do not have a google cloud account please signup [google](https://cloud.google.com). ### Required GCP APIs @@ -104,6 +101,9 @@ Execute the following commands on the linux machine in order to setup gcloud cli ```console gcloud init ``` +### GCP Quotas + +If you created your Google cloud account newly, Google Compute Engine enforces quotas on resource usage for a variety of reasons. For example, quotas protect the community of Google Cloud users by preventing unforeseen spikes in usage, Google keep some soft limitations on the resources, you can always make a request to increase your quota limit. If you are planning to deploy k8ssandra cluster on GKE, you will need to make a request to increase your **Compute Engine API (backend services)** quota to `50` for the future use. ### Tools @@ -121,12 +121,8 @@ Terraform is used to automate the manipulation of cloud infrastructure. Its [Ter The Google Cloud SDK is used to interact with your GCP resources. [Google cloud SDK Installation instructions](https://cloud.google.com/sdk/downloads) for multiple platforms are available online. -#### Install kubectl CLI - -The kubectl CLI is used to interteract with both Kubernetes Engine and Kubernetes in general. [kubectl CLI Installation instructions](https://cloud.google.com/kubernetes-engine/docs/quickstart) for multiple platforms are available online. - -### GCP authenticaion +### GCP-authentication Ensure you have authenticated your gcloud client by running the following command: @@ -134,13 +130,20 @@ Ensure you have authenticated your gcloud client by running the following comman gcloud auth login ``` -if you are already using another profile on your machine, use the following command to update the credentials: +If you are already using another profile on your machine, use the following command to update the credentials: ```console gcloud auth application-default login ``` +#### Install Kubectl +The kubectl CLI is used to interact with both Kubernetes Engine and Kubernetes in general. Use the following command to install kubectl using gcloud: + +```console +# install Kubectl +gcloud components install kubectl. +``` -### Configure gcloud settings +### Configure-gcloud-settings Run `gcloud config list` and make sure that `compute/zone`, `compute/region` and `core/project` are populated with values that work for you. You can choose a [region and zone near you](https://cloud.google.com/compute/docs/regions-zones/). You can set their values with the following commands: @@ -184,7 +187,7 @@ ex:- export TF_VAR_region=us-central-1 ``` -Importent: Initialize the terraform modules delete the backend file for local testing. +Important: Initialize the terraform modules delete the backend file for local testing. ```console cd env/ diff --git a/gcp/modules/gke/main.tf b/gcp/modules/gke/main.tf index 7403a17..bc6a97d 100644 --- a/gcp/modules/gke/main.tf +++ b/gcp/modules/gke/main.tf @@ -16,7 +16,7 @@ resource "google_container_cluster" "container_cluster" { name = var.name project = var.project_id - description = format("%s-GKE Cluster", var.name) + description = format("%s-gke-cluster", var.name) location = var.region remove_default_node_pool = true initial_node_count = var.initial_node_count diff --git a/gcp/modules/iam/main.tf b/gcp/modules/iam/main.tf index 076f998..702bcd1 100644 --- a/gcp/modules/iam/main.tf +++ b/gcp/modules/iam/main.tf @@ -14,7 +14,7 @@ # Create Google Service Account resource "google_service_account" "service_account" { - account_id = format("%s-service-account", var.name) + account_id = format("%s-sa", var.name) display_name = "GKE Security Service Account" project = var.project_id } diff --git a/gcp/modules/vpc/outputs.tf b/gcp/modules/vpc/outputs.tf index 893bcd1..a157ac0 100644 --- a/gcp/modules/vpc/outputs.tf +++ b/gcp/modules/vpc/outputs.tf @@ -21,15 +21,3 @@ output "network_selflink" { output "subnetwork_selflink" { value = google_compute_subnetwork.compute_subnetwork.self_link } - -/* -# Output variable for the NAT address selflink. -output "compute_address" { - value = google_compute_address.compute_address.self_link -} - -# Output variable for the NAT ip address. -output "compute_address_id" { - value = google_compute_address.compute_address.id -} -*/ \ No newline at end of file diff --git a/gcp/scripts/README.md b/gcp/scripts/README.md new file mode 100644 index 0000000..560c90b --- /dev/null +++ b/gcp/scripts/README.md @@ -0,0 +1,12 @@ +# Scripts to create and destroy the resources + +## What's in this folder + +* [apply.sh](./apply.sh): By using this script we can apply changes to the Terraform resources in GCP. +* [common.sh](./common.sh): By using this script we can validate the required packages and variables on your system. This script will be called in `apply.sh`, `init.sh`, `validate.sh`, `destroy.sh`, `plan.sh`. +* [delete_bucket.py](./delete_bucket.py): By using this script we can delete the GCP bucket created for terraform state files. +* [destroy.sh](./destroy.sh): By using this script we can destroy all the resource Created Terraform. +* [enable.sh](./enable.sh): By using this script we can enable the google api's +* [init.sh](./init.sh): By using this script we can initialize the modules, Terraform workspace, environment and create terraform statefile bucket. +* [make_bucket.py](./make_bucket.py): This script is used by [init.sh](./init.sh) to create terraform Statefiles bucket in GCP. +* [plan.sh](./plan.sh): By using this script we plan the resources by running `terraform plan` command. diff --git a/scripts/apply.sh b/gcp/scripts/apply.sh similarity index 95% rename from scripts/apply.sh rename to gcp/scripts/apply.sh index 90430d4..2a53cbe 100644 --- a/scripts/apply.sh +++ b/gcp/scripts/apply.sh @@ -26,7 +26,7 @@ ROOT="$( cd "$( dirname "${BASH_SOURCE[0]}" )/.." && pwd )" source "${ROOT}/scripts/common.sh" # Make apply : this command will apply the infrastructure changes -(cd "${ROOT}/gcp/env"; terraform apply -input=false -auto-approve) +(cd "${ROOT}/env"; terraform apply -input=false -auto-approve) # Get cluster outputs from the gke cluster. GET_OUTPUTS="$(terraform output endpoint master_version)" diff --git a/scripts/common.sh b/gcp/scripts/common.sh similarity index 98% rename from scripts/common.sh rename to gcp/scripts/common.sh index fa717cb..b13f056 100644 --- a/scripts/common.sh +++ b/gcp/scripts/common.sh @@ -64,7 +64,7 @@ command -v terraform >/dev/null 2>&1 || { \ # Make sure python3 is installed. If not, refer to: # https://www.python.org/downloads/ -command -v terraform >/dev/null 2>&1 || { \ +command -v python3 >/dev/null 2>&1 || { \ echo >&2 "I require python3 but it's not installed. Aborting." echo >&2 "https://www.python.org/downloads/" exit 1 diff --git a/scripts/delete_bucket.py b/gcp/scripts/delete_bucket.py similarity index 100% rename from scripts/delete_bucket.py rename to gcp/scripts/delete_bucket.py diff --git a/scripts/destroy.sh b/gcp/scripts/destroy.sh similarity index 98% rename from scripts/destroy.sh rename to gcp/scripts/destroy.sh index 3c2b486..d45c466 100644 --- a/scripts/destroy.sh +++ b/gcp/scripts/destroy.sh @@ -29,7 +29,7 @@ source "${ROOT}/scripts/common.sh" export bucket_name="${TF_VAR_name}-${TF_VAR_project_id}-statefiles" # Make destroy : this command will destroy the GKE cluster- infrastructure -cd "${ROOT}"/gcp/env +cd "${ROOT}"/env # Select the environment workspace where you want destroy all your resources terraform workspace select $"TF_VAR_environment" diff --git a/scripts/enable.sh b/gcp/scripts/enable.sh similarity index 100% rename from scripts/enable.sh rename to gcp/scripts/enable.sh diff --git a/scripts/init.sh b/gcp/scripts/init.sh similarity index 92% rename from scripts/init.sh rename to gcp/scripts/init.sh index 1448425..8bb6bf3 100644 --- a/scripts/init.sh +++ b/gcp/scripts/init.sh @@ -22,20 +22,19 @@ set -o pipefail # Locate the root directory ROOT="$( cd "$( dirname "${BASH_SOURCE[0]}" )/.." && pwd )" -# Make will use bash instead of sh +# Run common.sh script for validation +source "${ROOT}/scripts/common.sh" + # Set environment variables # Create a gcs bucket through cli to store terraform statefiles # Create google storage bucket for the terraform backend. export bucket_name="${TF_VAR_name}-${TF_VAR_project_id}-statefiles" # Create Google cloud storage bucket to store the state files. -source "${ROOT}/scripts/make_bucket.py" - -# Run common.sh script for validation -source "${ROOT}/scripts/common.sh" +source "${ROOT}/gcp/scripts/make_bucket.py" # Terraform initialize and run cd gcp/env -cd "${ROOT}/gcp/env" +cd "${ROOT}/env" # Terraform initinalize the backend bucket terraform init -input=false -backend-config="bucket=${bucket_name}" -backend-config="prefix=terraform/${TF_VAR_environment}/" @@ -43,5 +42,8 @@ terraform init -input=false -backend-config="bucket=${bucket_name}" -backend-con # Create workspace based on the environment, by doing this you don't overlap wih the resources in different environments. terraform workspace new "$TF_VAR_environment" || terraform workspace select "$TF_VAR_environment" +#make validate : this command will validate the terraform code +terraform validate + # Functions Script to enable google api's source "${ROOT}/scripts/enable.sh" diff --git a/scripts/k8ssandra.yaml b/gcp/scripts/k8ssandra.yaml similarity index 100% rename from scripts/k8ssandra.yaml rename to gcp/scripts/k8ssandra.yaml diff --git a/scripts/make_bucket.py b/gcp/scripts/make_bucket.py similarity index 100% rename from scripts/make_bucket.py rename to gcp/scripts/make_bucket.py diff --git a/scripts/plan.sh b/gcp/scripts/plan.sh similarity index 98% rename from scripts/plan.sh rename to gcp/scripts/plan.sh index cf192f6..fe05fde 100644 --- a/scripts/plan.sh +++ b/gcp/scripts/plan.sh @@ -26,7 +26,7 @@ ROOT="$( cd "$( dirname "${BASH_SOURCE[0]}" )/.." && pwd )" source "${ROOT}/scripts/common.sh" #make plan : this command will validate the terraform code -cd "${ROOT}"/gcp/env +cd "${ROOT}"/env # Terraform validate before the plan terraform validate diff --git a/scripts/README.md b/scripts/README.md deleted file mode 100644 index 084d69b..0000000 --- a/scripts/README.md +++ /dev/null @@ -1,2 +0,0 @@ -# Scripts to create and destroy the resources -[ WORK IN PROGRESS ] diff --git a/scripts/validate.sh b/scripts/validate.sh deleted file mode 100644 index 06a16a2..0000000 --- a/scripts/validate.sh +++ /dev/null @@ -1,30 +0,0 @@ -#!/usr/bin/env bash - -# Copyright 2021 Datastax LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Bash safeties: exit on error, no unset variables, pipelines can't hide errors -set -o errexit -set -o nounset -set -o pipefail - -# Locate the root directory -ROOT="$( cd "$( dirname "${BASH_SOURCE[0]}" )/.." && pwd )" - -# Run common.sh script for variable declaration and validation -source "${ROOT}/scripts/common.sh" - -#make validate : this command will validate the terraform code -cd "${ROOT}"/gcp/env -terraform validate diff --git a/test/README.md b/test/README.md index abbe390..0e79a99 100644 --- a/test/README.md +++ b/test/README.md @@ -2,4 +2,4 @@ This folder the contains [lint.sh](./lint.sh) script. To check the linting on all type project files on this repository. -[ WORK IN PROGRESS ] + diff --git a/test/reaper-nodeport.yaml b/test/reaper-nodeport.yaml new file mode 100644 index 0000000..0b61650 --- /dev/null +++ b/test/reaper-nodeport.yaml @@ -0,0 +1,14 @@ +apiVersion: v1 +kind: Service +metadata: + name: reaper-nodeport +spec: + ports: + - name: app + port: 8080 + protocol: TCP + targetPort: 8080 + selector: + app.kubernetes.io/managed-by: reaper-operator + reaper.cassandra-reaper.io/reaper: test-reaper + type: NodePort