diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml new file mode 100644 index 0000000..92f8842 --- /dev/null +++ b/.github/workflows/lint.yml @@ -0,0 +1,27 @@ +name: Lint + +on: + push: + branches: [main] + pull_request: + +jobs: + lint: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - uses: opentofu/setup-opentofu@v1 + with: + tofu_version: v1.7.2 # renovate: datasource=github-releases depName=opentofu/opentofu + tofu_wrapper: false + + - run: | + tofu validate + tofu fmt + + - name: Check uncommitted changes + run: git diff --exit-code + + - if: failure() + run: echo "::error::Check failed, please run 'tofu validate; tofu fmt' and commit the changes." diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml new file mode 100644 index 0000000..6099e89 --- /dev/null +++ b/.github/workflows/test.yml @@ -0,0 +1,64 @@ +name: Test + +on: + push: + branches: [main] + pull_request: + +jobs: + example: + runs-on: ubuntu-latest + + concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + + permissions: + id-token: write # Required by hetznercloud/tps-action + + defaults: + run: + working-directory: example + + steps: + - uses: actions/checkout@v4 + + - uses: opentofu/setup-opentofu@v1 + with: + tofu_version: v1.7.2 # renovate: datasource=github-releases depName=opentofu/opentofu + tofu_wrapper: false + + - uses: yokawasa/action-setup-kube-tools@v0.11.1 + with: + setup-tools: | + kubectl + skaffold + kubectl: v1.29.6 # renovate: datasource=github-releases depName=kubernetes/kubernetes + skaffold: v2.12.0 # renovate: datasource=github-releases depName=GoogleContainerTools/skaffold + + - uses: actions/setup-go@v5 + with: + go-version-file: go.mod + + - uses: hetznercloud/tps-action@main + + - name: Swap module source + run: sed -i -e 's|source = ".*"|source = ".."|' main.tf + + - name: Setup environment + run: make up + + - name: Verify environment + run: | + source files/env.sh + + kubectl wait --for=condition=Ready --all node + kubectl wait --for=condition=Ready --all --all-namespaces deployment + kubectl wait --for=condition=Ready --all --all-namespaces pod + + skaffold run ../test-app + kubectl wait --for=condition=Ready --all pod -l app=test-app + + - name: Cleanup + if: always() + run: make down diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000..962f6ce --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,30 @@ +--- +# See https://pre-commit.com for more information +# See https://pre-commit.com/hooks.html for more hooks +repos: + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: v4.6.0 + hooks: + - id: check-added-large-files + - id: check-case-conflict + - id: check-executables-have-shebangs + - id: check-shebang-scripts-are-executable + - id: check-symlinks + - id: destroyed-symlinks + + - id: check-json + - id: check-yaml + - id: check-toml + + - id: check-merge-conflict + - id: end-of-file-fixer + - id: mixed-line-ending + args: [--fix=lf] + - id: trailing-whitespace + + - repo: https://github.com/pre-commit/mirrors-prettier + rev: v3.1.0 + hooks: + - id: prettier + files: \.(md|ya?ml)$ + exclude: ^CHANGELOG.md$ diff --git a/README.md b/README.md index 4529002..3da1fc9 100644 --- a/README.md +++ b/README.md @@ -4,3 +4,39 @@ This repository contains a terraform module used to setup a Kubernetes developme > [!WARNING] > This project is not an official Hetzner Cloud Integration and is intended to be used internally. There is no backwards-compatibility promise. + +## Usage + +To setup a development environment, make sure you installed the following tools: + +- [tofu](https://opentofu.org/) +- [k3sup](https://github.com/alexellis/k3sup) + +1. Configure a `HCLOUD_TOKEN` in your shell session. + +> [!WARNING] +> The development environment runs on Hetzner Cloud servers which will induce costs. + +2. Deploy the development cluster: + +```sh +make -C dev up +``` + +3. Load the generated configuration to access the development cluster: + +```sh +source files/env.sh +``` + +4. Check that the development cluster is healthy: + +```sh +kubectl get nodes -o wide +``` + +⚠️ Do not forget to clean up the development cluster once are finished: + +```sh +make -C dev down +``` diff --git a/example/.gitignore b/example/.gitignore new file mode 100644 index 0000000..fbef7b5 --- /dev/null +++ b/example/.gitignore @@ -0,0 +1,9 @@ +.terraform* +terraform.tfstate +terraform.tfstate.backup +*.auto.tfvars + +files/* +!files/.gitkeep + +.env diff --git a/example/Makefile b/example/Makefile new file mode 100644 index 0000000..63f6ea6 --- /dev/null +++ b/example/Makefile @@ -0,0 +1,27 @@ +SHELL = bash +.ONESHELL: + +ENV ?= dev +K3S_CHANNEL ?= stable + +env.auto.tfvars: + @echo 'name = "$(ENV)"' > "$@" + @echo 'hcloud_token = "$(HCLOUD_TOKEN)"' >> "$@" + @echo 'k3s_channel = "$(K3S_CHANNEL)"' >> "$@" + +.terraform: + tofu init + +up: .terraform env.auto.tfvars + tofu apply -auto-approve + $(MAKE) port-forward + +down: .terraform env.auto.tfvars + tofu destroy -auto-approve + +port-forward: + source files/env.sh + bash files/registry-port-forward.sh + +clean: + rm -Rf files/* .terraform* terraform.tfstate* env.auto.tfvars diff --git a/example/main.tf b/example/main.tf new file mode 100644 index 0000000..fa31a4c --- /dev/null +++ b/example/main.tf @@ -0,0 +1,8 @@ +module "dev" { + source = "github.com/hetznercloud/terraform-k8s-dev?ref=0.1.0" + + name = "k8s-dev-${replace(var.name, "/[^a-zA-Z0-9-_]/", "-")}" + hcloud_token = var.hcloud_token + + k3s_channel = var.k3s_channel +} diff --git a/example/variables.tf b/example/variables.tf new file mode 100644 index 0000000..46b3405 --- /dev/null +++ b/example/variables.tf @@ -0,0 +1,10 @@ +variable "name" { + type = string +} +variable "hcloud_token" { + type = string + sensitive = true +} +variable "k3s_channel" { + type = string +} diff --git a/main-infra.tf b/main-infra.tf new file mode 100644 index 0000000..3910198 --- /dev/null +++ b/main-infra.tf @@ -0,0 +1,102 @@ +# Setup the infrastructure + +provider "hcloud" { + token = var.hcloud_token +} + +locals { + labels = { + env = var.name + } +} + +# SSH Key + +resource "tls_private_key" "ssh" { + algorithm = "ED25519" +} + +resource "local_sensitive_file" "ssh" { + content = tls_private_key.ssh.private_key_openssh + filename = abspath("${path.root}/files/id_ed25519") +} + +resource "hcloud_ssh_key" "default" { + name = var.name + public_key = tls_private_key.ssh.public_key_openssh + labels = local.labels +} + +# Network + +resource "hcloud_network" "cluster" { + name = var.name + ip_range = "10.0.0.0/8" + labels = local.labels +} + +resource "hcloud_network_subnet" "cluster" { + network_id = hcloud_network.cluster.id + network_zone = "eu-central" + type = "cloud" + ip_range = "10.0.0.0/24" +} + +# Control Plane Node + +resource "hcloud_server" "control" { + name = "${var.name}-control" + server_type = var.hcloud_server_type + location = var.hcloud_location + image = var.hcloud_image + ssh_keys = [hcloud_ssh_key.default.id] + labels = local.labels + + connection { + host = self.ipv4_address + private_key = tls_private_key.ssh.private_key_openssh + } + + provisioner "remote-exec" { + inline = ["cloud-init status --wait || test $? -eq 2"] + } +} + +resource "hcloud_server_network" "control" { + server_id = hcloud_server.control.id + subnet_id = hcloud_network_subnet.cluster.id +} + +# Worker / Agent Nodes + +variable "worker_count" { + type = number + default = 3 +} + +resource "hcloud_server" "worker" { + count = var.worker_count + + name = "${var.name}-worker-${count.index}" + server_type = var.hcloud_server_type + location = var.hcloud_location + image = var.hcloud_image + ssh_keys = [hcloud_ssh_key.default.id] + labels = local.labels + + connection { + host = self.ipv4_address + private_key = tls_private_key.ssh.private_key_openssh + } + + provisioner "remote-exec" { + inline = ["cloud-init status --wait || test $? -eq 2"] + } +} + +resource "hcloud_server_network" "worker" { + count = var.worker_count + + server_id = hcloud_server.worker[count.index].id + subnet_id = hcloud_network_subnet.cluster.id +} diff --git a/main-setup.tf b/main-setup.tf new file mode 100644 index 0000000..faf64aa --- /dev/null +++ b/main-setup.tf @@ -0,0 +1,216 @@ +# Setup the k3s cluster + +locals { + # The CIDR range for the Pods, must be included in the range of the + # network (10.0.0.0/8) but must not overlap with the Subnet (10.0.0.0/24) + cluster_cidr = "10.244.0.0/16" + + registry_service_ip = "10.43.0.2" + registry_port = 30666 + + kubeconfig_path = abspath("${path.root}/files/kubeconfig.yaml") + env_path = abspath("${path.root}/files/env.sh") +} + +resource "null_resource" "k3sup_control" { + triggers = { + id = hcloud_server.control.id + ip = hcloud_server_network.control.ip + } + + connection { + host = hcloud_server.control.ipv4_address + private_key = tls_private_key.ssh.private_key_openssh + } + + provisioner "remote-exec" { + inline = ["mkdir -p /etc/rancher/k3s"] + } + provisioner "file" { + content = yamlencode({ + "mirrors" : { + "localhost:${local.registry_port}" : { + "endpoint" : ["http://${local.registry_service_ip}:5000"] + } + } + }) + destination = "/etc/rancher/k3s/registries.yaml" + } + + provisioner "local-exec" { + command = <<-EOT + k3sup install --print-config=false \ + --ssh-key='${local_sensitive_file.ssh.filename}' \ + --ip='${hcloud_server.control.ipv4_address}' \ + --k3s-channel='${var.k3s_channel}' \ + --k3s-extra-args="\ + --kubelet-arg=cloud-provider=external \ + --cluster-cidr='${local.cluster_cidr}' \ + --disable-cloud-controller \ + --disable-network-policy \ + --disable=local-storage \ + --disable=servicelb \ + --disable=traefik \ + --flannel-backend=none \ + --node-external-ip='${hcloud_server.control.ipv4_address}' \ + --node-ip='${hcloud_server_network.control.ip}'" \ + --local-path='${local.kubeconfig_path}' + EOT + } +} + +resource "null_resource" "k3sup_worker" { + count = var.worker_count + + triggers = { + id = hcloud_server.worker[count.index].id + ip = hcloud_server_network.worker[count.index].ip + + # Wait the control-plane to be initialized, and re-join the new cluster if the + # control-plane server changed. + control_id = null_resource.k3sup_control.id + } + + connection { + host = hcloud_server.worker[count.index].ipv4_address + private_key = tls_private_key.ssh.private_key_openssh + } + + provisioner "remote-exec" { + inline = ["mkdir -p /etc/rancher/k3s"] + } + provisioner "file" { + content = yamlencode({ + "mirrors" : { + "localhost:${local.registry_port}" : { + "endpoint" : ["http://${local.registry_service_ip}:5000"] + } + } + }) + destination = "/etc/rancher/k3s/registries.yaml" + } + + provisioner "local-exec" { + command = <<-EOT + k3sup join \ + --ssh-key='${local_sensitive_file.ssh.filename}' \ + --ip='${hcloud_server.worker[count.index].ipv4_address}' \ + --server-ip='${hcloud_server.control.ipv4_address}' \ + --k3s-channel='${var.k3s_channel}' \ + --k3s-extra-args="\ + --kubelet-arg='cloud-provider=external' \ + --node-external-ip='${hcloud_server.worker[count.index].ipv4_address}' \ + --node-ip='${hcloud_server_network.worker[count.index].ip}'" + EOT + } +} + +# Configure kubernetes + +data "local_sensitive_file" "kubeconfig" { + depends_on = [null_resource.k3sup_control] + filename = local.kubeconfig_path +} + +provider "kubernetes" { + config_path = data.local_sensitive_file.kubeconfig.filename +} + +resource "kubernetes_secret_v1" "hcloud_token" { + metadata { + name = "hcloud" + namespace = "kube-system" + } + + data = { + token = var.hcloud_token + network = hcloud_network.cluster.id + } +} + +provider "helm" { + kubernetes { + config_path = data.local_sensitive_file.kubeconfig.filename + } +} + +resource "helm_release" "cilium" { + name = "cilium" + chart = "cilium" + repository = "https://helm.cilium.io" + namespace = "kube-system" + version = "1.13.1" + wait = true + + set { + name = "operator.replicas" + value = "1" + } + set { + name = "ipam.mode" + value = "kubernetes" + } + set { + name = "tunnel" + value = "disabled" + } + set { + name = "ipv4NativeRoutingCIDR" + value = local.cluster_cidr + } +} + +resource "helm_release" "hcloud_cloud_controller_manager" { + name = "hcloud-cloud-controller-manager" + chart = "hcloud-cloud-controller-manager" + repository = "https://charts.hetzner.cloud" + namespace = "kube-system" + version = "1.19.0" + wait = true + + set { + name = "networking.enabled" + value = "true" + } +} + +resource "helm_release" "docker_registry" { + name = "docker-registry" + chart = "docker-registry" + repository = "https://helm.twun.io" + namespace = "kube-system" + version = "2.2.3" + wait = true + + set { + name = "service.clusterIP" + value = local.registry_service_ip + } + set { + name = "tolerations[0].key" + value = "node.cloudprovider.kubernetes.io/uninitialized" + } + set { + name = "tolerations[0].operator" + value = "Exists" + } +} + +# Export files + +resource "local_file" "registry_port_forward" { + source = "${path.module}/registry-port-forward.sh" + filename = "${path.root}/files/registry-port-forward.sh" + file_permission = "0755" +} + +resource "local_file" "env" { + content = <<-EOT + #!/usr/bin/env bash + + export KUBECONFIG=${data.local_sensitive_file.kubeconfig.filename} + export SKAFFOLD_DEFAULT_REPO=localhost:${local.registry_port} + EOT + filename = local.env_path + file_permission = "0644" +} diff --git a/providers.tf b/providers.tf new file mode 100644 index 0000000..fc3089e --- /dev/null +++ b/providers.tf @@ -0,0 +1,29 @@ +terraform { + required_providers { + local = { + source = "hashicorp/local" + version = "2.5.1" + } + null = { + source = "hashicorp/null" + version = "3.2.2" + } + tls = { + source = "hashicorp/tls" + version = "4.0.5" + } + helm = { + source = "hashicorp/helm" + version = "2.14.0" + } + kubernetes = { + source = "hashicorp/kubernetes" + version = "2.31.0" + } + + hcloud = { + source = "hetznercloud/hcloud" + version = "1.47.0" + } + } +} diff --git a/registry-port-forward.sh b/registry-port-forward.sh new file mode 100755 index 0000000..863adeb --- /dev/null +++ b/registry-port-forward.sh @@ -0,0 +1,20 @@ +#!/usr/bin/env bash + +set -ue -o pipefail + +run() { + unit="k8s-registry-port-forward.service" + description="Port Forward for Container Registry of k8s dev environment" + + systemctl --user stop "$unit" 2> /dev/null || true + + systemd-run --user \ + --unit="$unit" \ + --description="$description" \ + --same-dir \ + --setenv="KUBECONFIG=$KUBECONFIG" \ + --collect \ + kubectl port-forward -n kube-system svc/docker-registry 30666:5000 +} + +run diff --git a/test-app/go.mod b/test-app/go.mod new file mode 100644 index 0000000..51a8f82 --- /dev/null +++ b/test-app/go.mod @@ -0,0 +1,3 @@ +module github.com/hetznercloud/terraform-k8s-env/test-app + +go 1.22.4 diff --git a/test-app/main.go b/test-app/main.go new file mode 100644 index 0000000..14026c1 --- /dev/null +++ b/test-app/main.go @@ -0,0 +1,17 @@ +package main + +import ( + "fmt" + "os" + "os/signal" + "syscall" +) + +func main() { + fmt.Println("Hello Kubernetes!") + + sigs := make(chan os.Signal, 1) + signal.Notify(sigs, syscall.SIGINT, syscall.SIGTERM) + <-sigs + fmt.Println("exiting") +} diff --git a/test-app/skaffold.yaml b/test-app/skaffold.yaml new file mode 100644 index 0000000..286f9a1 --- /dev/null +++ b/test-app/skaffold.yaml @@ -0,0 +1,16 @@ +apiVersion: skaffold/v4beta11 +kind: Config +metadata: + name: test-app + +build: + artifacts: + - image: test-app + ko: {} + +manifests: + rawYaml: + - test-app.yaml + +deploy: + kubectl: {} diff --git a/test-app/test-app.yaml b/test-app/test-app.yaml new file mode 100644 index 0000000..96d2d5c --- /dev/null +++ b/test-app/test-app.yaml @@ -0,0 +1,20 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: test-app + labels: + app: test-app +spec: + replicas: 1 + selector: + matchLabels: + app: test-app + template: + metadata: + name: test-app + labels: + app: test-app + spec: + containers: + - name: test-app + image: test-app diff --git a/variables.tf b/variables.tf new file mode 100644 index 0000000..d558345 --- /dev/null +++ b/variables.tf @@ -0,0 +1,35 @@ +# Environement +variable "name" { + description = "Name of the environement" + type = string + default = "dev" +} + +# Hetzner Cloud +variable "hcloud_token" { + description = "Hetzner Cloud API token" + type = string + sensitive = true +} +variable "hcloud_server_type" { + description = "Hetzner Cloud Server Type used for the environement" + type = string + default = "cpx21" +} +variable "hcloud_location" { + description = "Hetzner Cloud Location used for the environement" + type = string + default = "fsn1" +} +variable "hcloud_image" { + description = "Hetzner Cloud Image used for the environement" + type = string + default = "ubuntu-24.04" +} + +# K3S +variable "k3s_channel" { + description = "k3S channel used for the environement" + type = string + default = "stable" +}