From a58392912eca03a4ff2e44b80bf32a4bdb4f5bc4 Mon Sep 17 00:00:00 2001 From: Jonas L Date: Mon, 24 Jun 2024 10:09:40 +0200 Subject: [PATCH] test: implement new development environment (#628) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replace the multiple bash scripts used to setup the `csi-driver` test environment with Terraform and Make. --------- Co-authored-by: Julian Tölle --- .github/workflows/test.yml | 83 ++++---- .gitignore | 3 - README.md | 126 ++++-------- dev/.gitignore | 9 + {hack => dev}/Dockerfile | 0 dev/Makefile | 27 +++ dev/files/.gitkeep | 0 dev/hcloud-k8s-env/main-infra.tf | 102 +++++++++ dev/hcloud-k8s-env/main-setup.tf | 216 ++++++++++++++++++++ dev/hcloud-k8s-env/providers.tf | 29 +++ dev/hcloud-k8s-env/registry-port-forward.sh | 20 ++ dev/hcloud-k8s-env/variables.tf | 35 ++++ {hack => dev}/kustomization.yaml | 0 dev/main.tf | 8 + dev/variables.tf | 10 + hack/dev-down.sh | 37 ---- hack/dev-up.sh | 176 ---------------- hack/k3s-registries.yaml | 3 - hack/registry-port-forward.sh | 16 -- kustomization.yaml | 2 +- skaffold.yaml | 7 +- test/e2e/kubernetes/.gitignore | 2 +- test/e2e/kubernetes/Makefile | 31 +++ test/e2e/kubernetes/run-e2e-tests.sh | 30 --- 24 files changed, 583 insertions(+), 389 deletions(-) create mode 100644 dev/.gitignore rename {hack => dev}/Dockerfile (100%) create mode 100644 dev/Makefile create mode 100644 dev/files/.gitkeep create mode 100644 dev/hcloud-k8s-env/main-infra.tf create mode 100644 dev/hcloud-k8s-env/main-setup.tf create mode 100644 dev/hcloud-k8s-env/providers.tf create mode 100755 dev/hcloud-k8s-env/registry-port-forward.sh create mode 100644 dev/hcloud-k8s-env/variables.tf rename {hack => dev}/kustomization.yaml (100%) create mode 100644 dev/main.tf create mode 100644 dev/variables.tf delete mode 100755 hack/dev-down.sh delete mode 100755 hack/dev-up.sh delete mode 100644 hack/k3s-registries.yaml delete mode 100755 hack/registry-port-forward.sh create mode 100644 test/e2e/kubernetes/Makefile delete mode 100755 test/e2e/kubernetes/run-e2e-tests.sh diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 185a00e0..c072396f 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -23,8 +23,8 @@ jobs: with: token: ${{ secrets.CODECOV_TOKEN }} - test-kubernetes: - name: e2e k3s ${{ matrix.k3s }} + kubernetes: + name: kubernetes ${{ matrix.k3s }} runs-on: ubuntu-latest permissions: @@ -33,21 +33,22 @@ jobs: concurrency: ${{ github.workflow }}-${{ github.ref }}-${{ matrix.k3s }} strategy: - # The e2e tests are flaky and often one of the jobs fails. The default setting - # causes all other currently running jobs to abort and all need to be restarted. - fail-fast: false + fail-fast: false # Continue tests matrix if a flaky run occur. matrix: include: - # All k3s after January 2024 break our e2e tests, we hardcode - # the versions for now until we can fix the source of this. - - k3s: v1.26.12+k3s1 - k8s-test: v1.26.12 - - k3s: v1.27.9+k3s1 - k8s-test: v1.27.9 - - k3s: v1.28.5+k3s1 - k8s-test: v1.28.5 - - k3s: v1.29.0+k3s1 - k8s-test: v1.29.0 + - k3s: v1.26 + k8s-test: v1.26.15 + - k3s: v1.27 + k8s-test: v1.27.15 + - k3s: v1.28 + k8s-test: v1.28.11 + - k3s: v1.29 + k8s-test: v1.29.6 + + env: + K3S_CHANNEL: ${{ matrix.k3s }} + K8S_TEST_VERSION: ${{ matrix.k8s-test }} + ENV: gha-${{ github.run_id }}-${{ github.run_attempt }}-${{ matrix.k3s }} steps: - uses: actions/checkout@v4 @@ -56,9 +57,12 @@ jobs: with: go-version-file: go.mod - - uses: hetznercloud/tps-action@main + - uses: opentofu/setup-opentofu@v1 + with: + tofu_version: v1.7.2 # renovate: datasource=github-releases depName=opentofu/opentofu + tofu_wrapper: false - - uses: hetznercloud/setup-hcloud@v1 + - uses: docker/setup-buildx-action@v3 - uses: yokawasa/action-setup-kube-tools@v0.11.1 with: @@ -66,26 +70,35 @@ jobs: helm kubectl skaffold - helm: v3.11.2 - kubectl: v1.29.0 - skaffold: v2.3.0 + helm: v3.15.2 # renovate: datasource=github-releases depName=helm/helm + kubectl: v1.29.6 # renovate: datasource=github-releases depName=kubernetes/kubernetes + skaffold: v2.12.0 # renovate: datasource=github-releases depName=GoogleContainerTools/skaffold - - name: Run tests - env: - K3S_VERSION: ${{ matrix.k3s }} - K8S_TEST_VERSION: ${{ matrix.k8s-test }} - SCOPE: gha-${{ github.run_id }}-${{ github.run_attempt }}-${{ matrix.k3s }} - run: | - curl -sLS https://get.k3sup.dev | sh + - name: Setup k3sup + run: curl -sLS https://get.k3sup.dev | sh + + - name: Setup k8s test binaries + run: make -C test/e2e/kubernetes bin - trap "hack/dev-down.sh" EXIT - source <(hack/dev-up.sh) + - uses: hetznercloud/tps-action@main - skaffold build --tag="e2e-${GITHUB_RUN_ID}-${GITHUB_RUN_NUMBER}" - tag=$(skaffold build --tag="e2e-${GITHUB_RUN_ID}-${GITHUB_RUN_NUMBER}" --quiet --output="{{ (index .Builds 0).Tag }}") - skaffold deploy --images=docker.io/hetznercloud/hcloud-csi-driver=$tag + - name: Setup environment + run: make -C dev up - test/e2e/kubernetes/run-e2e-tests.sh + - name: Run skaffold + run: | + source dev/files/env.sh + skaffold run + + - name: Run parallel tests + run: | + source dev/files/env.sh + make -C test/e2e/kubernetes parallel + + - name: Run serial tests + run: | + source dev/files/env.sh + make -C test/e2e/kubernetes serial deploy-manifests: runs-on: ubuntu-latest @@ -95,7 +108,7 @@ jobs: with: setup-tools: | helm - helm: v3.11.2 + helm: v3.11.2 # renovate: datasource=github-releases depName=helm/helm - uses: actions/checkout@v4 @@ -117,7 +130,7 @@ jobs: with: setup-tools: | helm - helm: v3.11.2 + helm: v3.11.2 # renovate: datasource=github-releases depName=helm/helm - uses: actions/checkout@v4 diff --git a/.gitignore b/.gitignore index 6fa7c4f8..a904188b 100644 --- a/.gitignore +++ b/.gitignore @@ -1,9 +1,6 @@ # rootfs for docker plugin deploy/docker-swarm/pkg/plugin -# Tmp files for dev env -hack/.* - # Build output dist/ hcloud-csi-*.tgz diff --git a/README.md b/README.md index 1c3d41bc..80b47591 100644 --- a/README.md +++ b/README.md @@ -17,117 +17,77 @@ relevant to that Container Orchestrator behind the link: - [Docker Swarm](./docs/docker-swarm/README.md)️ _⚠️ Not officially supported_ - [HashiCorp Nomad](./docs/nomad/README.md)️ _⚠️ Not officially supported_ -## Tests +## Development -### Integration Tests +### Setup a development environment -**Requirements: Docker** +To setup a development environment, make sure you installed the following tools: -The core operations like publishing and resizing can be tested locally with Docker. +- [tofu](https://opentofu.org/) +- [k3sup](https://github.com/alexellis/k3sup) +- [docker](https://www.docker.com/) +- [skaffold](https://skaffold.dev/) -```bash -go test $(go list ./... | grep integration) -v -``` +1. Configure a `HCLOUD_TOKEN` in your shell session. -### E2E Tests +> [!WARNING] +> The development environment runs on Hetzner Cloud servers which will induce costs. -> ⚠️ Kubernetes E2E Tests were recently refactored and the docs are now outdated. -> See the [GitHub Actions workflow](.github/workflows/test_e2e.yml) for an -> up-to-date script to run the e2e tests. +2. Deploy the development cluster: -The Hetzner Cloud CSI Driver was tested against the official k8s e2e -tests for a specific version. You can run the tests with the following -commands. Keep in mind, that these tests run on real cloud servers and -will create volumes that will be billed. +```sh +make -C dev up +``` -**Test Server Setup**: +3. Load the generated configuration to access the development cluster: -1x CPX21 (Ubuntu 18.04) +```sh +source dev/files/env.sh +``` -**Requirements: Docker and Go 1.17** +4. Check that the development cluster is healthy: -1. Configure your environment correctly - ```bash - export HCLOUD_TOKEN= - export K8S_VERSION=1.21.0 # The specific (latest) version is needed here - export USE_SSH_KEYS=key1,key2 # Name or IDs of your SSH Keys within the Hetzner Cloud, the servers will be accessible with that keys - ``` -2. Run the tests - ```bash - go test $(go list ./... | grep e2e) -v -timeout 60m - ``` +```sh +kubectl get nodes -o wide +``` -The tests will now run, this will take a while (~30 min). +5. Start developing the CSI driver in the development cluster: -**If the tests fail, make sure to clean up the project with the Hetzner Cloud Console or the hcloud cli.** +```sh +skaffold dev +``` -### Local test setup +On code change, skaffold will rebuild the image, redeploy it and print all logs from csi components. -> ⚠️ Local Kubernetes Dev Setup was recently refactored and the docs are now -> outdated. Check out the scripts [dev-up.sh](hack/dev-up.sh) & -> [dev-down.sh](hack/dev-down.sh) for an automatic dev setup. +⚠️ Do not forget to clean up the development cluster once are finished: -This repository provides [skaffold](https://skaffold.dev/) to easily deploy / debug this driver on demand +```sh +make -C dev down +``` -#### Requirements -1. Install [hcloud-cli](https://github.com/hetznercloud/cli) -2. Install [k3sup](https://github.com/alexellis/k3sup) -3. Install [cilium](https://github.com/cilium/cilium-cli) -4. Install [docker](https://www.docker.com/) +### Run the docker e2e tests -You will also need to set a `HCLOUD_TOKEN` in your shell session +To run the integrations tests, make sure you installed the following tools: -#### Manual Installation guide +- [docker](https://www.docker.com/) -1. Create an SSH key +1. Run the following command to run the integrations tests: -Assuming you already have created an ssh key via `ssh-keygen` -``` -hcloud ssh-key create --name ssh-key-csi-test --public-key-from-file ~/.ssh/id_rsa.pub +```sh +go test -v ./test/integration ``` -2. Create a server -``` -hcloud server create --name csi-test-server --image ubuntu-20.04 --ssh-key ssh-key-csi-test --type cx22 -``` +### Run the kubernetes e2e tests -3. Setup k3s on this server -``` -k3sup install --ip $(hcloud server ip csi-test-server) --local-path=/tmp/kubeconfig --cluster --k3s-channel=v1.23 --k3s-extra-args='--no-flannel --no-deploy=servicelb --no-deploy=traefik --disable-cloud-controller --disable-network-policy --kubelet-arg=cloud-provider=external' -``` -- The kubeconfig will be created under `/tmp/kubeconfig` -- Kubernetes version can be configured via `--k3s-channel` +The Hetzner Cloud CSI driver is tested against the official kubernetes e2e tests. -4. Switch your kubeconfig to the test cluster -``` -export KUBECONFIG=/tmp/kubeconfig -``` - -5. Install cilium + test your cluster -``` -cilium install -``` +Before running the integrations tests, make sure you followed the [Setup a development environment](#setup-a-development-environment) steps. -6. Add your secret to the cluster -``` -kubectl -n kube-system create secret generic hcloud --from-literal="token=$HCLOUD_TOKEN" -``` +1. Run the kubernetes e2e tests using the following command: -7. Install hcloud-cloud-controller-manager + test your cluster +```sh +make -C test/e2e/kubernetes test ``` -kubectl apply -f https://github.com/hetznercloud/hcloud-cloud-controller-manager/releases/latest/download/ccm.yaml -kubectl config set-context default -kubectl get node -o wide -``` - -8. Deploy your CSI driver -``` -SKAFFOLD_DEFAULT_REPO=naokiii skaffold dev -``` -- `docker login` required -- Skaffold is using your own dockerhub repo to push the CSI image. - -On code change, skaffold will repack the image & deploy it to your test cluster again. Also, it is printing all logs from csi components. ## License diff --git a/dev/.gitignore b/dev/.gitignore new file mode 100644 index 00000000..fbef7b5d --- /dev/null +++ b/dev/.gitignore @@ -0,0 +1,9 @@ +.terraform* +terraform.tfstate +terraform.tfstate.backup +*.auto.tfvars + +files/* +!files/.gitkeep + +.env diff --git a/hack/Dockerfile b/dev/Dockerfile similarity index 100% rename from hack/Dockerfile rename to dev/Dockerfile diff --git a/dev/Makefile b/dev/Makefile new file mode 100644 index 00000000..63f6ea67 --- /dev/null +++ b/dev/Makefile @@ -0,0 +1,27 @@ +SHELL = bash +.ONESHELL: + +ENV ?= dev +K3S_CHANNEL ?= stable + +env.auto.tfvars: + @echo 'name = "$(ENV)"' > "$@" + @echo 'hcloud_token = "$(HCLOUD_TOKEN)"' >> "$@" + @echo 'k3s_channel = "$(K3S_CHANNEL)"' >> "$@" + +.terraform: + tofu init + +up: .terraform env.auto.tfvars + tofu apply -auto-approve + $(MAKE) port-forward + +down: .terraform env.auto.tfvars + tofu destroy -auto-approve + +port-forward: + source files/env.sh + bash files/registry-port-forward.sh + +clean: + rm -Rf files/* .terraform* terraform.tfstate* env.auto.tfvars diff --git a/dev/files/.gitkeep b/dev/files/.gitkeep new file mode 100644 index 00000000..e69de29b diff --git a/dev/hcloud-k8s-env/main-infra.tf b/dev/hcloud-k8s-env/main-infra.tf new file mode 100644 index 00000000..39101980 --- /dev/null +++ b/dev/hcloud-k8s-env/main-infra.tf @@ -0,0 +1,102 @@ +# Setup the infrastructure + +provider "hcloud" { + token = var.hcloud_token +} + +locals { + labels = { + env = var.name + } +} + +# SSH Key + +resource "tls_private_key" "ssh" { + algorithm = "ED25519" +} + +resource "local_sensitive_file" "ssh" { + content = tls_private_key.ssh.private_key_openssh + filename = abspath("${path.root}/files/id_ed25519") +} + +resource "hcloud_ssh_key" "default" { + name = var.name + public_key = tls_private_key.ssh.public_key_openssh + labels = local.labels +} + +# Network + +resource "hcloud_network" "cluster" { + name = var.name + ip_range = "10.0.0.0/8" + labels = local.labels +} + +resource "hcloud_network_subnet" "cluster" { + network_id = hcloud_network.cluster.id + network_zone = "eu-central" + type = "cloud" + ip_range = "10.0.0.0/24" +} + +# Control Plane Node + +resource "hcloud_server" "control" { + name = "${var.name}-control" + server_type = var.hcloud_server_type + location = var.hcloud_location + image = var.hcloud_image + ssh_keys = [hcloud_ssh_key.default.id] + labels = local.labels + + connection { + host = self.ipv4_address + private_key = tls_private_key.ssh.private_key_openssh + } + + provisioner "remote-exec" { + inline = ["cloud-init status --wait || test $? -eq 2"] + } +} + +resource "hcloud_server_network" "control" { + server_id = hcloud_server.control.id + subnet_id = hcloud_network_subnet.cluster.id +} + +# Worker / Agent Nodes + +variable "worker_count" { + type = number + default = 3 +} + +resource "hcloud_server" "worker" { + count = var.worker_count + + name = "${var.name}-worker-${count.index}" + server_type = var.hcloud_server_type + location = var.hcloud_location + image = var.hcloud_image + ssh_keys = [hcloud_ssh_key.default.id] + labels = local.labels + + connection { + host = self.ipv4_address + private_key = tls_private_key.ssh.private_key_openssh + } + + provisioner "remote-exec" { + inline = ["cloud-init status --wait || test $? -eq 2"] + } +} + +resource "hcloud_server_network" "worker" { + count = var.worker_count + + server_id = hcloud_server.worker[count.index].id + subnet_id = hcloud_network_subnet.cluster.id +} diff --git a/dev/hcloud-k8s-env/main-setup.tf b/dev/hcloud-k8s-env/main-setup.tf new file mode 100644 index 00000000..952f8aee --- /dev/null +++ b/dev/hcloud-k8s-env/main-setup.tf @@ -0,0 +1,216 @@ +# Setup the k3s cluster + +locals { + # The CIDR range for the Pods, must be included in the range of the + # network (10.0.0.0/8) but must not overlap with the Subnet (10.0.0.0/24) + cluster_cidr = "10.244.0.0/16" + + registry_service_ip = "10.43.0.2" + registry_port = 30666 + + kubeconfig_path = abspath("${path.root}/files/kubeconfig.yaml") + env_path = abspath("${path.root}/files/env.sh") +} + +resource "null_resource" "k3sup_control" { + triggers = { + id = hcloud_server.control.id + ip = hcloud_server_network.control.ip + } + + connection { + host = hcloud_server.control.ipv4_address + private_key = tls_private_key.ssh.private_key_openssh + } + + provisioner "remote-exec" { + inline = ["mkdir -p /etc/rancher/k3s"] + } + provisioner "file" { + content = yamlencode({ + "mirrors" : { + "localhost:${local.registry_port}" : { + "endpoint" : ["http://${local.registry_service_ip}:5000"] + } + } + }) + destination = "/etc/rancher/k3s/registries.yaml" + } + + provisioner "local-exec" { + command = <<-EOT + k3sup install --print-config=false \ + --ssh-key='${local_sensitive_file.ssh.filename}' \ + --ip='${hcloud_server.control.ipv4_address}' \ + --k3s-channel='${var.k3s_channel}' \ + --k3s-extra-args="\ + --kubelet-arg=cloud-provider=external \ + --cluster-cidr='${local.cluster_cidr}' \ + --disable-cloud-controller \ + --disable-network-policy \ + --disable=local-storage \ + --disable=servicelb \ + --disable=traefik \ + --flannel-backend=none \ + --node-external-ip='${hcloud_server.control.ipv4_address}' \ + --node-ip='${hcloud_server_network.control.ip}'" \ + --local-path='${local.kubeconfig_path}' + EOT + } +} + +resource "null_resource" "k3sup_worker" { + count = var.worker_count + + triggers = { + id = hcloud_server.worker[count.index].id + ip = hcloud_server_network.worker[count.index].ip + + # Wait the control-plane to be initialized, and re-join the new cluster if the + # control-plane server changed. + control_id = null_resource.k3sup_control.id + } + + connection { + host = hcloud_server.worker[count.index].ipv4_address + private_key = tls_private_key.ssh.private_key_openssh + } + + provisioner "remote-exec" { + inline = ["mkdir -p /etc/rancher/k3s"] + } + provisioner "file" { + content = yamlencode({ + "mirrors" : { + "localhost:${local.registry_port}" : { + "endpoint" : ["http://${local.registry_service_ip}:5000"] + } + } + }) + destination = "/etc/rancher/k3s/registries.yaml" + } + + provisioner "local-exec" { + command = <<-EOT + k3sup join \ + --ssh-key='${local_sensitive_file.ssh.filename}' \ + --ip='${hcloud_server.worker[count.index].ipv4_address}' \ + --server-ip='${hcloud_server.control.ipv4_address}' \ + --k3s-channel='${var.k3s_channel}' \ + --k3s-extra-args="\ + --kubelet-arg='cloud-provider=external' \ + --node-external-ip='${hcloud_server.worker[count.index].ipv4_address}' \ + --node-ip='${hcloud_server_network.worker[count.index].ip}'" + EOT + } +} + +# Configure kubernetes + +data "local_sensitive_file" "kubeconfig" { + depends_on = [null_resource.k3sup_control] + filename = local.kubeconfig_path +} + +provider "kubernetes" { + config_path = data.local_sensitive_file.kubeconfig.filename +} + +resource "kubernetes_secret_v1" "hcloud_token" { + metadata { + name = "hcloud" + namespace = "kube-system" + } + + data = { + token = var.hcloud_token + network = hcloud_network.cluster.id + } +} + +provider "helm" { + kubernetes { + config_path = data.local_sensitive_file.kubeconfig.filename + } +} + +resource "helm_release" "cilium" { + name = "cilium" + chart = "cilium" + repository = "https://helm.cilium.io" + namespace = "kube-system" + version = "1.13.1" # renovate: datasource=github-releases depName=cilium/cilium extractVersion=v(?.+) + wait = true + + set { + name = "operator.replicas" + value = "1" + } + set { + name = "ipam.mode" + value = "kubernetes" + } + set { + name = "tunnel" + value = "disabled" + } + set { + name = "ipv4NativeRoutingCIDR" + value = local.cluster_cidr + } +} + +resource "helm_release" "hcloud_cloud_controller_manager" { + name = "hcloud-cloud-controller-manager" + chart = "hcloud-cloud-controller-manager" + repository = "https://charts.hetzner.cloud" + namespace = "kube-system" + version = "1.19.0" # renovate: datasource=github-releases depName=hetznercloud/hcloud-cloud-controller-manager extractVersion=v(?.+) + wait = true + + set { + name = "networking.enabled" + value = "true" + } +} + +resource "helm_release" "docker_registry" { + name = "docker-registry" + chart = "docker-registry" + repository = "https://helm.twun.io" + namespace = "kube-system" + version = "2.2.2" # renovate: datasource=github-releases depName=distribution/distribution extractVersion=v(?.+) + wait = true + + set { + name = "service.clusterIP" + value = local.registry_service_ip + } + set { + name = "tolerations[0].key" + value = "node.cloudprovider.kubernetes.io/uninitialized" + } + set { + name = "tolerations[0].operator" + value = "Exists" + } +} + +# Export files + +resource "local_file" "registry_port_forward" { + source = "${path.module}/registry-port-forward.sh" + filename = "${path.root}/files/registry-port-forward.sh" + file_permission = "0755" +} + +resource "local_file" "env" { + content = <<-EOT + #!/usr/bin/env bash + + export KUBECONFIG=${data.local_sensitive_file.kubeconfig.filename} + export SKAFFOLD_DEFAULT_REPO=localhost:${local.registry_port} + EOT + filename = local.env_path + file_permission = "0644" +} diff --git a/dev/hcloud-k8s-env/providers.tf b/dev/hcloud-k8s-env/providers.tf new file mode 100644 index 00000000..1dcc2da6 --- /dev/null +++ b/dev/hcloud-k8s-env/providers.tf @@ -0,0 +1,29 @@ +terraform { + required_providers { + local = { + source = "hashicorp/local" + version = "2.5.1" + } + null = { + source = "hashicorp/null" + version = "3.2.2" + } + tls = { + source = "hashicorp/tls" + version = "4.0.5" + } + helm = { + source = "hashicorp/helm" + version = "2.14.0" + } + kubernetes = { + source = "hashicorp/kubernetes" + version = "2.31.0" + } + + hcloud = { + source = "hetznercloud/hcloud" + version = "1.45.0" + } + } +} diff --git a/dev/hcloud-k8s-env/registry-port-forward.sh b/dev/hcloud-k8s-env/registry-port-forward.sh new file mode 100755 index 00000000..863adeb4 --- /dev/null +++ b/dev/hcloud-k8s-env/registry-port-forward.sh @@ -0,0 +1,20 @@ +#!/usr/bin/env bash + +set -ue -o pipefail + +run() { + unit="k8s-registry-port-forward.service" + description="Port Forward for Container Registry of k8s dev environment" + + systemctl --user stop "$unit" 2> /dev/null || true + + systemd-run --user \ + --unit="$unit" \ + --description="$description" \ + --same-dir \ + --setenv="KUBECONFIG=$KUBECONFIG" \ + --collect \ + kubectl port-forward -n kube-system svc/docker-registry 30666:5000 +} + +run diff --git a/dev/hcloud-k8s-env/variables.tf b/dev/hcloud-k8s-env/variables.tf new file mode 100644 index 00000000..d558345b --- /dev/null +++ b/dev/hcloud-k8s-env/variables.tf @@ -0,0 +1,35 @@ +# Environement +variable "name" { + description = "Name of the environement" + type = string + default = "dev" +} + +# Hetzner Cloud +variable "hcloud_token" { + description = "Hetzner Cloud API token" + type = string + sensitive = true +} +variable "hcloud_server_type" { + description = "Hetzner Cloud Server Type used for the environement" + type = string + default = "cpx21" +} +variable "hcloud_location" { + description = "Hetzner Cloud Location used for the environement" + type = string + default = "fsn1" +} +variable "hcloud_image" { + description = "Hetzner Cloud Image used for the environement" + type = string + default = "ubuntu-24.04" +} + +# K3S +variable "k3s_channel" { + description = "k3S channel used for the environement" + type = string + default = "stable" +} diff --git a/hack/kustomization.yaml b/dev/kustomization.yaml similarity index 100% rename from hack/kustomization.yaml rename to dev/kustomization.yaml diff --git a/dev/main.tf b/dev/main.tf new file mode 100644 index 00000000..51475870 --- /dev/null +++ b/dev/main.tf @@ -0,0 +1,8 @@ +module "dev" { + source = "./hcloud-k8s-env" + + name = "csi-driver-${replace(var.name, "/[^a-zA-Z0-9-_]/", "-")}" + hcloud_token = var.hcloud_token + + k3s_channel = var.k3s_channel +} diff --git a/dev/variables.tf b/dev/variables.tf new file mode 100644 index 00000000..46b34057 --- /dev/null +++ b/dev/variables.tf @@ -0,0 +1,10 @@ +variable "name" { + type = string +} +variable "hcloud_token" { + type = string + sensitive = true +} +variable "k3s_channel" { + type = string +} diff --git a/hack/dev-down.sh b/hack/dev-down.sh deleted file mode 100755 index bf3097fb..00000000 --- a/hack/dev-down.sh +++ /dev/null @@ -1,37 +0,0 @@ -#!/usr/bin/env bash -set -ue -o pipefail -SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )" - -scope="${SCOPE:-dev}" -scope=${scope//[^a-zA-Z0-9_]/-} -scope_name=csi-${scope} -label="managedby=hack" - -if [[ "${ALL:-}" == "" ]]; then - label="$label,scope=$scope_name" - rm -f $SCRIPT_DIR/.ssh-$scope $SCRIPT_DIR/.kubeconfig-$scope -else - rm -f $SCRIPT_DIR/.ssh* $SCRIPT_DIR/.kubeconfig* -fi - -for instance in $(hcloud server list -o noheader -o columns=id -l $label); do - ( - hcloud server delete $instance - ) & -done - - -for key in $(hcloud ssh-key list -o noheader -o columns=name -l $label); do - ( - hcloud ssh-key delete $key - ) & -done - - -for key in $(hcloud network list -o noheader -o columns=name -l $label); do - ( - hcloud network delete $key - ) & -done - -wait \ No newline at end of file diff --git a/hack/dev-up.sh b/hack/dev-up.sh deleted file mode 100755 index 943258ed..00000000 --- a/hack/dev-up.sh +++ /dev/null @@ -1,176 +0,0 @@ -#!/usr/bin/env bash -set -ueo pipefail -SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )" - -if [[ -n "${DEBUG:-}" ]]; then set -x; fi - -# Redirect all stdout to stderr. -{ - if ! hcloud version >/dev/null; then echo "ERROR: 'hcloud' CLI not found, please install it and make it available on your \$PATH"; exit 1; fi - if ! k3sup version >/dev/null; then echo "ERROR: 'k3sup' not found, please install it and make it available on your \$PATH"; exit 1; fi - if ! helm version >/dev/null; then echo "ERROR: 'helm' not found, please install it and make it available on your \$PATH"; exit 1; fi - if [[ "${HCLOUD_TOKEN:-}" == "" ]]; then echo "ERROR: please set \$HCLOUD_TOKEN"; exit 1; fi - - # We run a lot of subshells below for speed. If any encounter an error, we shut down the whole process group, pronto. - function error() { - echo "Onoes, something went wrong! :( The output above might have some clues." - kill 0 - } - - trap error ERR - - image_name=${IMAGE_NAME:-ubuntu-20.04} - instance_count=${INSTANCES:-3} - instance_type=${INSTANCE_TYPE:-cpx11} - location=${LOCATION:-fsn1} - network_zone=${NETWORK_ZONE:-eu-central} - ssh_keys=${SSH_KEYS:-} - # All k3s after January 2024 break our e2e tests, we hardcode - # the versions for now until we can fix the source of this. - # channel=${K3S_CHANNEL:-stable} - k3s_version=${K3S_VERSION:-v1.28.5+k3s1} - network_cidr=${NETWORK_CIDR:-10.0.0.0/8} - subnet_cidr=${SUBNET_CIDR:-10.0.0.0/24} - cluster_cidr=${CLUSTER_CIDR:-10.244.0.0/16} - scope="${SCOPE:-dev}" - scope=${scope//[^a-zA-Z0-9_]/-} - scope_name=csi-${scope} - label="managedby=hack,scope=$scope_name" - ssh_private_key="$SCRIPT_DIR/.ssh-$scope" - k3s_opts=${K3S_OPTS:-"--kubelet-arg cloud-provider=external"} - k3s_server_opts=${K3S_SERVER_OPTS:-"--disable-cloud-controller --disable=traefik --disable=servicelb --disable=local-storage --flannel-backend=none --cluster-cidr ${cluster_cidr}"} - - echo -n "$HCLOUD_TOKEN" > "$SCRIPT_DIR/.token-$scope" - - export KUBECONFIG="$SCRIPT_DIR/.kubeconfig-$scope" - - ssh_command="ssh -i $ssh_private_key -o StrictHostKeyChecking=off -o BatchMode=yes -o ConnectTimeout=5" - - # Generate SSH keys and upload publkey to Hetzner Cloud. - ( trap error ERR - [[ ! -f $ssh_private_key ]] && ssh-keygen -t ed25519 -f $ssh_private_key -C '' -N '' - [[ ! -f $ssh_private_key.pub ]] && ssh-keygen -y -f $ssh_private_key > $ssh_private_key.pub - if ! hcloud ssh-key describe $scope_name >/dev/null 2>&1; then - hcloud ssh-key create --label $label --name $scope_name --public-key-from-file $ssh_private_key.pub - fi - ) & - - # Create Network - ( trap error ERR - if ! hcloud network describe $scope_name >/dev/null 2>&1; then - hcloud network create --label $label --ip-range $network_cidr --name $scope_name - hcloud network add-subnet --network-zone $network_zone --type cloud --ip-range $subnet_cidr $scope_name - fi - ) & - - - for num in $(seq $instance_count); do - # Create server and initialize Kubernetes on it with k3sup. - ( trap error ERR - - server_name="$scope_name-$num" - - # Maybe cluster is already up and node is already there. - if kubectl get node $server_name >/dev/null 2>&1; then - exit 0 - fi - - ip=$(hcloud server ip $server_name 2>/dev/null || true) - - if [[ -z "${ip:-}" ]]; then - # Wait for SSH key - until hcloud ssh-key describe $scope_name >/dev/null 2>&1; do sleep 1; done - until hcloud network describe $scope_name 2>&1 | grep $subnet_cidr >/dev/null; do sleep 1; done - - createcmd="hcloud server create --image $image_name --label $label --location $location --name $server_name --ssh-key=$scope_name --type $instance_type --network $scope_name" - for key in $ssh_keys; do - createcmd+=" --ssh-key $key" - done - $createcmd - ip=$(hcloud server ip $server_name) - fi - - # Wait for SSH. - until [ "$($ssh_command root@$ip echo ok 2>/dev/null)" = "ok" ]; do - sleep 1 - done - - $ssh_command root@$ip 'mkdir -p /etc/rancher/k3s && cat > /etc/rancher/k3s/registries.yaml' < $SCRIPT_DIR/k3s-registries.yaml - - private_ip=$(hcloud server describe $server_name -o format="{{ (index .PrivateNet 0).IP }}") - k3s_node_ip_opts="--node-external-ip ${ip} --node-ip ${private_ip}" - - if [[ "$num" == "1" ]]; then - # First node is control plane. - k3sup install --print-config=false --ip $ip --k3s-version "${k3s_version}" --k3s-extra-args "${k3s_server_opts} ${k3s_opts} ${k3s_node_ip_opts}" --local-path $KUBECONFIG --ssh-key $ssh_private_key - else - # All subsequent nodes are initialized as workers. - - # Can't go any further until control plane has bootstrapped a bit though. - until $ssh_command root@$(hcloud server ip $scope_name-1 || true) stat /etc/rancher/node/password >/dev/null 2>&1; do - sleep 1 - done - - k3sup join --server-ip $(hcloud server ip $scope_name-1) --ip $ip --k3s-version "${k3s_version}" --k3s-extra-args "${k3s_opts} ${k3s_node_ip_opts}" --ssh-key $ssh_private_key - fi - ) & - - # Wait for this node to show up in the cluster. - ( trap error ERR; set +x - until kubectl wait --for=condition=Ready node/$scope_name-$num >/dev/null 2>&1; do sleep 1; done - echo $scope_name-$num is up and in cluster - ) & - done - - ( trap error ERR - # Control plane init tasks. - # This is running in parallel with the server init, above. - - # Wait for control plane to look alive. - until kubectl get nodes >/dev/null 2>&1; do sleep 1; done; - - # Deploy private registry. - ( trap error ERR - if ! helm status -n kube-system registry >/dev/null 2>&1; then - helm install registry docker-registry \ - --repo=https://helm.twun.io \ - -n kube-system \ - --version 2.2.2 \ - --set service.clusterIP=10.43.0.2 \ - --set 'tolerations[0].key=node.cloudprovider.kubernetes.io/uninitialized' \ - --set 'tolerations[0].operator=Exists' - fi - ) & - - # Install Cilium. - ( trap error ERR - if ! helm status -n kube-system cilium >/dev/null 2>&1; then - helm install cilium cilium --repo https://helm.cilium.io/ -n kube-system --version 1.13.1 \ - --set tunnel=disabled \ - --set ipv4NativeRoutingCIDR=$cluster_cidr \ - --set ipam.mode=kubernetes - fi) & - - # Create HCLOUD_TOKEN Secret for hcloud-cloud-controller-manager. - ( trap error ERR - if ! kubectl -n kube-system get secret hcloud >/dev/null 2>&1; then - kubectl -n kube-system create secret generic hcloud --from-literal="token=$HCLOUD_TOKEN" --from-literal="network=$scope_name" - fi) & - wait - - # Install hcloud-cloud-controller-manager - ( trap error ERR - if ! helm status -n kube-system hccm >/dev/null 2>&1; then - helm install hccm hcloud-cloud-controller-manager --repo https://charts.hetzner.cloud/ -n kube-system --version 1.14.2 --set networking.enabled=true - fi) & - wait - ) & - wait - echo "Success - cluster fully initialized and ready, why not see for yourself?" - echo '$ kubectl get nodes' - kubectl get nodes -} >&2 - -echo "export KUBECONFIG=$KUBECONFIG" -$SCRIPT_DIR/registry-port-forward.sh -echo "export SKAFFOLD_DEFAULT_REPO=localhost:30666" diff --git a/hack/k3s-registries.yaml b/hack/k3s-registries.yaml deleted file mode 100644 index 8c808b12..00000000 --- a/hack/k3s-registries.yaml +++ /dev/null @@ -1,3 +0,0 @@ -mirrors: - localhost:30666: - endpoint: ["http://10.43.0.2:5000"] diff --git a/hack/registry-port-forward.sh b/hack/registry-port-forward.sh deleted file mode 100755 index 082079d2..00000000 --- a/hack/registry-port-forward.sh +++ /dev/null @@ -1,16 +0,0 @@ -#!/usr/bin/env bash -set -ue -o pipefail -SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )" - -{ -until kubectl -n kube-system --timeout=30s rollout status deployment/registry-docker-registry >/dev/null 2>&1; do sleep 1; done -old_pid=$(cat $SCRIPT_DIR/.reg-pf 2>/dev/null || true) -if [[ -n "$old_pid" ]]; then - echo "killing old port-forward with PID $old_pid" - kill $old_pid || true -fi - -nohup kubectl port-forward -n kube-system svc/registry-docker-registry 30666:5000 >$SCRIPT_DIR/.reg-pf.out 2>$SCRIPT_DIR/.reg-pf.err & -} >&2 - -echo $! > $SCRIPT_DIR/.reg-pf diff --git a/kustomization.yaml b/kustomization.yaml index cf5e63c0..73de684e 100644 --- a/kustomization.yaml +++ b/kustomization.yaml @@ -1,4 +1,4 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization resources: -- deploy/kubernetes/ + - deploy/kubernetes diff --git a/skaffold.yaml b/skaffold.yaml index 549caddd..b9831bae 100644 --- a/skaffold.yaml +++ b/skaffold.yaml @@ -1,4 +1,4 @@ -apiVersion: skaffold/v4beta3 +apiVersion: skaffold/v4beta11 kind: Config metadata: name: csi-driver @@ -6,13 +6,12 @@ build: artifacts: - image: docker.io/hetznercloud/hcloud-csi-driver docker: - dockerfile: hack/Dockerfile - cacheFrom: - - docker.io/hetznercloud/hcloud-csi-driver:buildcache + dockerfile: dev/Dockerfile local: useBuildkit: true insecureRegistries: - localhost:30666 + manifests: helm: releases: diff --git a/test/e2e/kubernetes/.gitignore b/test/e2e/kubernetes/.gitignore index a459cf81..ba077a40 100644 --- a/test/e2e/kubernetes/.gitignore +++ b/test/e2e/kubernetes/.gitignore @@ -1 +1 @@ -test-binaries/ \ No newline at end of file +bin diff --git a/test/e2e/kubernetes/Makefile b/test/e2e/kubernetes/Makefile new file mode 100644 index 00000000..58479eb3 --- /dev/null +++ b/test/e2e/kubernetes/Makefile @@ -0,0 +1,31 @@ +SHELL = bash + +K8S_TEST_VERSION ?= v1.29.0 + +GINKGO = bin/ginkgo +GINKGO_ARGS = -v --flake-attempts=2 + +E2E = bin/e2e.test +E2E_ARGS = -storage.testdriver=$(CURDIR)/testdriver.yaml + +bin: + mkdir -p bin + curl -sSL "https://dl.k8s.io/$(K8S_TEST_VERSION)/kubernetes-test-linux-amd64.tar.gz" | \ + tar --strip-components=3 -C bin -zxf - \ + kubernetes/test/bin/e2e.test \ + kubernetes/test/bin/ginkgo + +parallel: bin + $(GINKGO) $(GINKGO_ARGS) \ + -nodes=12 \ + -focus='External.Storage' \ + -skip='\[Feature:|\[Disruptive\]|\[Serial\]' \ + $(E2E) -- $(E2E_ARGS) + +serial: bin + $(GINKGO) $(GINKGO_ARGS) \ + -focus='External.Storage.*(\[Feature:|\[Serial\])' \ + -skip='\[Feature:SELinuxMountReadWriteOncePod\]' \ + $(E2E) -- $(E2E_ARGS) + +test: parallel serial diff --git a/test/e2e/kubernetes/run-e2e-tests.sh b/test/e2e/kubernetes/run-e2e-tests.sh deleted file mode 100755 index 8db8edad..00000000 --- a/test/e2e/kubernetes/run-e2e-tests.sh +++ /dev/null @@ -1,30 +0,0 @@ -#!/usr/bin/env bash -set -uex -o pipefail -SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )" - -k8s_test_version="${K8S_TEST_VERSION:-v1.29.0}" - -mkdir -p "${SCRIPT_DIR}/test-binaries" -# TODO: Read linux-amd64 from env -curl --location "https://dl.k8s.io/${k8s_test_version}/kubernetes-test-linux-amd64.tar.gz" | \ - tar --strip-components=3 -C "${SCRIPT_DIR}/test-binaries" -zxf - kubernetes/test/bin/e2e.test kubernetes/test/bin/ginkgo - -ginkgo="${SCRIPT_DIR}/test-binaries/ginkgo" -ginkgo_flags="-v --flakeAttempts=2" - -e2e="${SCRIPT_DIR}/test-binaries/e2e.test" -e2e_flags="-storage.testdriver=${SCRIPT_DIR}/testdriver.yaml" - -echo "Executing parallel tests" -${ginkgo} ${ginkgo_flags} \ - -nodes=6 \ - -focus='External.Storage' \ - -skip='\[Feature:|\[Disruptive\]|\[Serial\]' \ - "${e2e}" -- ${e2e_flags} - -echo "Executing serial tests" -${ginkgo} ${ginkgo_flags} \ - -focus='External.Storage.*(\[Feature:|\[Serial\])' \ - -skip='\[Feature:SELinuxMountReadWriteOncePod\]' \ - "${e2e}" -- ${e2e_flags} -