From eac3aaa2542ee4e98366f21a0c9abc99e9ddae01 Mon Sep 17 00:00:00 2001 From: lukasmetzner Date: Thu, 28 Nov 2024 12:00:12 +0100 Subject: [PATCH 01/26] chore(nomad): integrated nomad development environment --- go.mod | 9 ++++ go.sum | 19 +++++++++ nomad/README.md | 49 +++++++++++++++++++++ nomad/dev/.gitignore | 6 +++ nomad/dev/.terraform.lock.hcl | 76 +++++++++++++++++++++++++++++++++ nomad/dev/Makefile | 30 +++++++++++++ nomad/dev/main.tf | 5 +++ nomad/dev/variables.tf | 3 ++ nomad/hcloud-csi-controller.hcl | 60 ++++++++++++++++++++++++++ nomad/hcloud-csi-node.hcl | 41 ++++++++++++++++++ nomad/skaffold.yaml | 17 ++++++++ nomad/start_nomad.sh | 23 ++++++++++ nomad/stop_nomad.sh | 9 ++++ 13 files changed, 347 insertions(+) create mode 100644 nomad/README.md create mode 100644 nomad/dev/.gitignore create mode 100644 nomad/dev/.terraform.lock.hcl create mode 100644 nomad/dev/Makefile create mode 100644 nomad/dev/main.tf create mode 100644 nomad/dev/variables.tf create mode 100644 nomad/hcloud-csi-controller.hcl create mode 100644 nomad/hcloud-csi-node.hcl create mode 100644 nomad/skaffold.yaml create mode 100755 nomad/start_nomad.sh create mode 100755 nomad/stop_nomad.sh diff --git a/go.mod b/go.mod index 855edab8..5eb7d928 100644 --- a/go.mod +++ b/go.mod @@ -31,8 +31,17 @@ require ( github.com/google/go-cmp v0.6.0 // indirect github.com/google/pprof v0.0.0-20240525223248-4bfdf5a9a2af // indirect github.com/google/uuid v1.6.0 // indirect + github.com/gorilla/websocket v1.5.0 // indirect github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.1.0 // indirect + github.com/hashicorp/cronexpr v1.1.2 // indirect + github.com/hashicorp/errwrap v1.1.0 // indirect + github.com/hashicorp/go-cleanhttp v0.5.2 // indirect + github.com/hashicorp/go-multierror v1.1.1 // indirect + github.com/hashicorp/go-rootcerts v1.0.2 // indirect + github.com/hashicorp/nomad/api v0.0.0-20241125123754-1f29a95c2413 // indirect github.com/klauspost/compress v1.17.11 // indirect + github.com/mitchellh/go-homedir v1.1.0 // indirect + github.com/mitchellh/mapstructure v1.5.0 // indirect github.com/moby/sys/mountinfo v0.7.2 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/onsi/ginkgo/v2 v2.19.0 // indirect diff --git a/go.sum b/go.sum index 8af8615c..0617bbf0 100644 --- a/go.sum +++ b/go.sum @@ -29,10 +29,25 @@ github.com/google/pprof v0.0.0-20240525223248-4bfdf5a9a2af h1:kmjWCqn2qkEml422C2 github.com/google/pprof v0.0.0-20240525223248-4bfdf5a9a2af/go.mod h1:K1liHPHnj73Fdn/EKuT8nrFqBihUSKXoLYU0BuatOYo= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/gorilla/websocket v1.5.0 h1:PPwGk2jz7EePpoHN/+ClbZu8SPxiqlu12wZP/3sWmnc= +github.com/gorilla/websocket v1.5.0/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/grpc-ecosystem/go-grpc-middleware/providers/prometheus v1.0.1 h1:qnpSQwGEnkcRpTqNOIR6bJbR0gAorgP9CSALpRcKoAA= github.com/grpc-ecosystem/go-grpc-middleware/providers/prometheus v1.0.1/go.mod h1:lXGCsh6c22WGtjr+qGHj1otzZpV/1kwTMAqkwZsnWRU= github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.1.0 h1:pRhl55Yx1eC7BZ1N+BBWwnKaMyD8uC+34TLdndZMAKk= github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.1.0/go.mod h1:XKMd7iuf/RGPSMJ/U4HP0zS2Z9Fh8Ps9a+6X26m/tmI= +github.com/hashicorp/cronexpr v1.1.2 h1:wG/ZYIKT+RT3QkOdgYc+xsKWVRgnxJ1OJtjjy84fJ9A= +github.com/hashicorp/cronexpr v1.1.2/go.mod h1:P4wA0KBl9C5q2hABiMO7cp6jcIg96CDh1Efb3g1PWA4= +github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= +github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY2I= +github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= +github.com/hashicorp/go-cleanhttp v0.5.2 h1:035FKYIWjmULyFRBKPs8TBQoi0x6d9G4xc9neXJWAZQ= +github.com/hashicorp/go-cleanhttp v0.5.2/go.mod h1:kO/YDlP8L1346E6Sodw+PrpBSV4/SoxCXGY6BqNFT48= +github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo= +github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM= +github.com/hashicorp/go-rootcerts v1.0.2 h1:jzhAVGtqPKbwpyCPELlgNWhE1znq+qwJtW5Oi2viEzc= +github.com/hashicorp/go-rootcerts v1.0.2/go.mod h1:pqUvnprVnM5bf7AOirdbb01K4ccR319Vf4pU3K5EGc8= +github.com/hashicorp/nomad/api v0.0.0-20241125123754-1f29a95c2413 h1:k0Z3HkPTwMY51/P6gRe20+oCQq6mszc7xJnpM1A+T4w= +github.com/hashicorp/nomad/api v0.0.0-20241125123754-1f29a95c2413/go.mod h1:svtxn6QnrQ69P23VvIWMR34tg3vmwLz4UdUzm1dSCgE= github.com/hetznercloud/hcloud-go/v2 v2.14.0 h1:WQW72DuOGqT486F0eNp92lDH5cwDTmyn9Mhin93m1To= github.com/hetznercloud/hcloud-go/v2 v2.14.0/go.mod h1:h8sHav+27Xa+48cVMAvAUMELov5h298Ilg2vflyTHgg= github.com/hetznercloud/hcloud-go/v2 v2.15.0 h1:6mpMJ/RuX1woZj+MCJdyKNEX9129KDkEIDeeyfr4GD4= @@ -53,6 +68,10 @@ github.com/kubernetes-csi/csi-test/v5 v5.3.1 h1:Wiukp1In+kif+BFo6q2ExjgB+MbrAz4j github.com/kubernetes-csi/csi-test/v5 v5.3.1/go.mod h1:7hA2cSYJ6T8CraEZPA6zqkLZwemjBD54XAnPsPC3VpA= github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= +github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= +github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= +github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= +github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/moby/buildkit v0.16.0 h1:wOVBj1o5YNVad/txPQNXUXdelm7Hs/i0PUFjzbK0VKE= github.com/moby/buildkit v0.16.0/go.mod h1:Xqx/5GlrqE1yIRORk0NSCVDFpQAU1WjlT6KHYZdisIQ= github.com/moby/buildkit v0.17.0 h1:ZA/4AxwBbve1f3ZaNNJQiCBtTV62R6YweWNwq4A+sTc= diff --git a/nomad/README.md b/nomad/README.md new file mode 100644 index 00000000..09c400fd --- /dev/null +++ b/nomad/README.md @@ -0,0 +1,49 @@ +# Nomad Development + +There is no official support from our side for Nomad. Nonetheless, we would still like to offer a satisfying developer experience and have automated e2e tests to avoid breaking Nomad accidentally. + +## Nomad Development Environment +As a prerequisite for developing Nomad, a setup of the [`nomad-dev-env`](https://github.com/hetznercloud/nomad-dev-env) is necessary, which is located in `nomad/dev`. + +1. Setup the `HCLOUD_TOKEN` environment variable +2. Deploy the development cluster: +``` bash +make -C nomad/dev up +``` +3. Load the generated configuration to access the development cluster: +``` bash +source nomad/dev/files/env.sh +``` +4. Check that the cluster is healthy: +``` bash +nomad node status +``` + +## Skaffold + +Skaffold commands should be executed from the `csi-driver` root directory and use the `-f` flag to point to the Nomad specific `skaffold.yaml`. + +``` bash +skaffold -f nomad/skaffold.yaml build +``` + +Skaffold does not offer any native support for Nomad. For this reason we use the Nomad post build hooks to deploy/redeploy the csi plugin. To delete the csi plugin a manual execution of `stop_nomad.sh` is necessary. + +``` bash +bash ./nomad/stop_nomad.sh +``` + +## E2E Tests + +The nomad e2e tests are located in `test/e2e/nomad` and need a working development environment. +1. Deploy the csi-driver + +``` bash +skaffold -f nomad/skaffold.yaml build +``` + +2. Run the e2e tests + +``` bash +go test -v -tags e2e ./test/e2e/nomad/... +``` \ No newline at end of file diff --git a/nomad/dev/.gitignore b/nomad/dev/.gitignore new file mode 100644 index 00000000..3aa31043 --- /dev/null +++ b/nomad/dev/.gitignore @@ -0,0 +1,6 @@ +.terraform/ +terraform.tfstate* +.terraform.tfstate* +*.auto.tfvars + +files/ diff --git a/nomad/dev/.terraform.lock.hcl b/nomad/dev/.terraform.lock.hcl new file mode 100644 index 00000000..90045e04 --- /dev/null +++ b/nomad/dev/.terraform.lock.hcl @@ -0,0 +1,76 @@ +# This file is maintained automatically by "tofu init". +# Manual edits may be lost in future updates. + +provider "registry.opentofu.org/hashicorp/external" { + version = "2.3.4" + constraints = "2.3.4" + hashes = [ + "h1:saRbzhRhYh4urj+ARe8vIB0mlRspy6E/fPSyvwqjUW8=", + "zh:0e5eb3513d6ad5cc3196799a6e413c6a9c0b642ba6d8f84fc11efa48f58358a4", + "zh:1658beae42b4614d4009a3191710c86571ccf1dc526c4bac520a87ab701dd2e9", + "zh:28d937c13e90c170fc1e4b726a9bcf113aade53b95b3eccd335cd9eaba8acff5", + "zh:2ac19917bb83233f24391d4cbaf301bb6ec14013d3b7f93bdf64823280622daa", + "zh:3263d1808fc5252d586a9aa98d89086c912f53e1a3dc43bc5306364b358f04fa", + "zh:463469836637342495e22a936ef9ab3c8ab2fb47eb0fae09c346d63f3331af59", + "zh:53398a27492cd11f61b2f24c2601c12f50c39da32b90fd25aed7011a1e25a225", + "zh:5a44cfbcef52fd3c970144a69a934cab320bd3bb57939ae4682fc516783e2996", + "zh:65edb579d9d0dac42e77728d81da6e85ea30d3fe8f2cb6e5db82602ee8afa17e", + "zh:f2edd3027b7ae0d31a690fd5dcdcd22b467b4f1e045f84f2bc88289353ef9a5b", + ] +} + +provider "registry.opentofu.org/hashicorp/local" { + version = "2.5.2" + hashes = [ + "h1:6lS+5A/4WFAqY3/RHWFRBSiFVLPRjvLaUgxPQvjXLHU=", + "zh:25b95b76ceaa62b5c95f6de2fa6e6242edbf51e7fc6c057b7f7101aa4081f64f", + "zh:3c974fdf6b42ca6f93309cf50951f345bfc5726ec6013b8832bcd3be0eb3429e", + "zh:5de843bf6d903f5cca97ce1061e2e06b6441985c68d013eabd738a9e4b828278", + "zh:86beead37c7b4f149a54d2ae633c99ff92159c748acea93ff0f3603d6b4c9f4f", + "zh:8e52e81d3dc50c3f79305d257da7fde7af634fed65e6ab5b8e214166784a720e", + "zh:9882f444c087c69559873b2d72eec406a40ede21acb5ac334d6563bf3a2387df", + "zh:a4484193d110da4a06c7bffc44cc6b61d3b5e881cd51df2a83fdda1a36ea25d2", + "zh:a53342426d173e29d8ee3106cb68abecdf4be301a3f6589e4e8d42015befa7da", + "zh:d25ef2aef6a9004363fc6db80305d30673fc1f7dd0b980d41d863b12dacd382a", + "zh:fa2d522fb323e2121f65b79709fd596514b293d816a1d969af8f72d108888e4c", + ] +} + +provider "registry.opentofu.org/hashicorp/tls" { + version = "4.0.6" + hashes = [ + "h1:EJoUGDo7L52Iu22cA1KCndJ9B1Rrfd75wyZzsScEnc0=", + "zh:4b53b372767e5068d9bbfc89199201c1ae4283dde2f0c301974f8abb4215791f", + "zh:5b4c308bd074c6d0bd560220e6ee10a9859ca9a1f29a59367b0477a740ff265e", + "zh:674dd6bc85597677e160ee601d88b21c5a974759a658769812d2904bd94bc042", + "zh:6ccc1c448349b56677ba66112aec7e0a58eb827f66209ca5f4077b81cce240fb", + "zh:8aa6e13a5d722b74230937ea21e8b4994e53340d95b5691cf6cf3518b9f38e6e", + "zh:8b27e55e4c7fa887774860113b95c8f7f68804b002fa47f0eb8e3a485997287e", + "zh:a430b5a3e8753d8f61784de49e538ac4abed19fb665fccd8a10b55402fe9f076", + "zh:b07c978c335ae9fc12f9c221629610775e4ae36691ed4e7ba258d275dd58a243", + "zh:bbec8cb1efc84ee3026c793956a4a4cd0ece20b89d2d4f7d954c68e7f6d596d0", + "zh:e684e247424188dc3b500a543b1a8046d1c0ec08c2a90aedca0c4f6bb56bedbd", + ] +} + +provider "registry.opentofu.org/hetznercloud/hcloud" { + version = "1.49.1" + constraints = "~> 1.45" + hashes = [ + "h1:FKGRNHVbcfQJd8EWrb8Ze5QHkaGr8zI+ZKxBMjvOwPk=", + "zh:3d5f9773da4f8203cf625d04a5a0e4ff7e202684c010a801a945756140c61cde", + "zh:446305d492017cda91e5c15122ec16ff15bfe3ef4d3fd6bcea0cdf7742ab1b86", + "zh:44d4f9156ed8b4f0444bd4dc456825940be49048828565964a192286d28c9f20", + "zh:492ad893d2f89bb17c9beb877c8ceb4a16caf39db1a79030fefeada6c7aa217f", + "zh:68dc552c19ad9d209ec6018445df6e06fb77a637513a53cc66ddce1b024082be", + "zh:7492495ffda6f6c49ab38b539bd2eb965b1150a63fb6b191a27dec07d17601cb", + "zh:850fe92005981ea00db86c3e49ba5b49732fdf1f7bd5530a68f6e272847059fc", + "zh:8cb67f744c233acfb1d68a6c27686315439d944edf733b95f113b4aa63d86713", + "zh:8e13dac46e8c2497772ed1baee701b1d1c26bcc95a63b5c4566c83468f504868", + "zh:c44249c6a8ba931e208a334792686b5355ab2da465cadea03c1ea8e73c02db12", + "zh:d103125a28a85c89aea0cb0c534fe3f504416c4d4fc75c37364b9ec5f66dd77d", + "zh:ed8f64e826aa9bfca95b72892271678cb78411b40d7b404a52404141e05a4ab1", + "zh:f40efad816de00b279bd1e2cbf62c76b0e5b2da150a0764f259984b318e30945", + "zh:f5e912d0873bf4ecc43feba4ceccdf158048080c76d557e47f34749139fdd452", + ] +} diff --git a/nomad/dev/Makefile b/nomad/dev/Makefile new file mode 100644 index 00000000..77d6f999 --- /dev/null +++ b/nomad/dev/Makefile @@ -0,0 +1,30 @@ +SHELL = bash + +ENV ?= dev + +env.auto.tfvars: + @echo 'hcloud_token = "$(HCLOUD_TOKEN)"' >> "$@" + +.terraform: + tofu init + +validate: .terraform + tofu validate + +up: .terraform env.auto.tfvars + tofu apply -auto-approve + $(MAKE) port-forward + +down: .terraform env.auto.tfvars + if test -f files/registry-port-forward.sh; then files/registry-port-forward.sh down; fi + tofu destroy -auto-approve + +port-forward: + files/registry-port-forward.sh up + +clean: + rm -Rf files/ .terraform/ terraform.tfstate* env.auto.tfvars + +deps: clean + sed -i -e 's|source = ".*"|source = "./.."|' main.tf + tofu init -upgrade \ No newline at end of file diff --git a/nomad/dev/main.tf b/nomad/dev/main.tf new file mode 100644 index 00000000..5b14d200 --- /dev/null +++ b/nomad/dev/main.tf @@ -0,0 +1,5 @@ +module "dev" { + source = "github.com/hetznercloud/nomad-dev-env?ref=nomad-dev-env" + + hcloud_token = var.hcloud_token +} \ No newline at end of file diff --git a/nomad/dev/variables.tf b/nomad/dev/variables.tf new file mode 100644 index 00000000..58b29de0 --- /dev/null +++ b/nomad/dev/variables.tf @@ -0,0 +1,3 @@ +variable "hcloud_token" { + sensitive = true +} \ No newline at end of file diff --git a/nomad/hcloud-csi-controller.hcl b/nomad/hcloud-csi-controller.hcl new file mode 100644 index 00000000..d67c4101 --- /dev/null +++ b/nomad/hcloud-csi-controller.hcl @@ -0,0 +1,60 @@ +job "hcloud-csi-controller" { + datacenters = ["dc1"] + namespace = "default" + type = "service" + + group "controller" { + + ### NOTE + # We define (at least) 2 allocations to increase the availability in case of a node failure with + # a controller allocation running on that node. On a "Single Node Cluster", the group stanzas + # might need modification or should be removed. + count = 2 + + constraint { + distinct_hosts = true + } + + update { + max_parallel = 1 + canary = 1 + min_healthy_time = "10s" + healthy_deadline = "1m" + auto_revert = true + auto_promote = true + } + + task "plugin" { + driver = "docker" + + config { + image = "$SKAFFOLD_IMAGE" + command = "bin/hcloud-csi-driver-controller" + } + + env { + CSI_ENDPOINT = "unix://csi/csi.sock" + ENABLE_METRICS = true + } + + template { + data = < /dev/null +then + nomad job stop -purge hcloud-csi-controller +fi + +controller="$(mktemp)" +envsubst < "./nomad/hcloud-csi-controller.hcl" > $controller +sed -i 's/localhost:30666/docker-registry.service.consul:5000/' $controller +nomad job run $controller + +if nomad job inspect hcloud-csi-node > /dev/null +then + nomad job stop -purge hcloud-csi-node +fi + +node="$(mktemp)" +envsubst < "./nomad/hcloud-csi-node.hcl" > $node +sed -i 's/localhost:30666/docker-registry.service.consul:5000/' $node +nomad job run $node diff --git a/nomad/stop_nomad.sh b/nomad/stop_nomad.sh new file mode 100755 index 00000000..736d087d --- /dev/null +++ b/nomad/stop_nomad.sh @@ -0,0 +1,9 @@ +#!/bin/bash + +set -euo pipefail + +nomad job stop -purge hcloud-csi-controller +echo "Deleted hcloud-csi-controller" + +nomad job stop -purge hcloud-csi-node +echo "Deleted hcloud-csi-node" \ No newline at end of file From 22dc2dd747974d4f1cfddbabd449a3aa18222013 Mon Sep 17 00:00:00 2001 From: lukasmetzner Date: Thu, 28 Nov 2024 12:02:01 +0100 Subject: [PATCH 02/26] test(nomad): implemented nomad e2e tests --- test/e2e/nomad/e2e_test.go | 93 ++++++++++++++++++++++++++++ test/e2e/nomad/utils.go | 120 +++++++++++++++++++++++++++++++++++++ 2 files changed, 213 insertions(+) create mode 100644 test/e2e/nomad/e2e_test.go create mode 100644 test/e2e/nomad/utils.go diff --git a/test/e2e/nomad/e2e_test.go b/test/e2e/nomad/e2e_test.go new file mode 100644 index 00000000..935ed721 --- /dev/null +++ b/test/e2e/nomad/e2e_test.go @@ -0,0 +1,93 @@ +//go:build e2e + +package e2e + +import ( + "context" + "fmt" + "os" + "strconv" + "testing" + + nomad "github.com/hashicorp/nomad/api" + "github.com/stretchr/testify/assert" +) + +var cluster *Cluster + +func TestMain(m *testing.M) { + var err error + cluster, err = NewCluster() + if err != nil { + fmt.Printf("%v\n", err) + os.Exit(1) + } + + exitCode := m.Run() + + if err := cluster.Cleanup(); err != nil { + fmt.Printf("%v\n", err) + os.Exit(1) + } + + os.Exit(exitCode) +} + +func TestVolumeLifecycle(t *testing.T) { + volReq := &nomad.CSIVolume{ + ID: "db-vol", + Name: "db-vol", + Namespace: "default", + PluginID: "csi.hetzner.cloud", + RequestedCapacityMin: 10737418240, + RequestedCapabilities: []*nomad.CSIVolumeCapability{ + { + AccessMode: "single-node-writer", + AttachmentMode: "file-system", + }, + }, + MountOptions: &nomad.CSIMountOptions{ + FSType: "ext4", + MountFlags: []string{ + "discard", + "defaults", + }, + }, + } + + var volID int64 + t.Run("volume creation", func(t *testing.T) { + vol, _, err := cluster.CreateVolume(volReq, &nomad.WriteOptions{}) + if err != nil { + t.Error(err) + } + assert.Len(t, vol, 1) + + volID, err := strconv.ParseInt(vol[0].ExternalID, 10, 64) + if err != nil { + t.Error(err) + } + + hcloudVolume, _, err := cluster.hcloudClient.Volume.GetByID(context.Background(), volID) + if err != nil { + t.Error(err) + } + + assert.NotNilf(t, hcloudVolume, "could not find volume with ID %d on hcloud", volID) + }) + + t.Run("volume deletion", func(t *testing.T) { + err := cluster.DeleteVolume(volReq.ID, &nomad.WriteOptions{}) + if err != nil { + t.Error(err) + } + + hcloudVolume, _, err := cluster.hcloudClient.Volume.GetByID(context.Background(), volID) + if err != nil { + t.Error(err) + } + + assert.Nil(t, hcloudVolume, "hcloud volume was deleted in nomad, but still exists") + }) + +} diff --git a/test/e2e/nomad/utils.go b/test/e2e/nomad/utils.go new file mode 100644 index 00000000..2d5b5af4 --- /dev/null +++ b/test/e2e/nomad/utils.go @@ -0,0 +1,120 @@ +package e2e + +import ( + "context" + "fmt" + "os" + "sync" + "time" + + nomad "github.com/hashicorp/nomad/api" + "github.com/hetznercloud/hcloud-go/v2/hcloud" +) + +type Cluster struct { + hcloudClient *hcloud.Client + nomadClient *nomad.Client + + volumesCreated map[string]struct{} + lock sync.Mutex +} + +func NewCluster() (*Cluster, error) { + token := os.Getenv("HCLOUD_TOKEN") + if token == "" { + return nil, fmt.Errorf("HCLOUD_TOKEN env variable is not set") + } + + hcloudOpts := []hcloud.ClientOption{ + hcloud.WithToken(token), + hcloud.WithApplication("nomad-csi-e2e", "v2.11.0"), // x-releaser-pleaser-version + hcloud.WithPollOpts(hcloud.PollOpts{ + BackoffFunc: hcloud.ExponentialBackoffWithOpts(hcloud.ExponentialBackoffOpts{ + Base: time.Second, + Multiplier: 2, + Cap: 10 * time.Second, + }), + }), + } + + hcloudClient := hcloud.NewClient(hcloudOpts...) + if hcloudClient == nil { + return nil, fmt.Errorf("hcloud client could not be initialized") + } + + nomadAddr := os.Getenv("NOMAD_ADDR") + if nomadAddr == "" { + return nil, fmt.Errorf("NOMAD_ADDR env variable is not set") + } + nomadCACert := os.Getenv("NOMAD_CACERT") + if nomadCACert == "" { + return nil, fmt.Errorf("NOMAD_CACERT env variable is not set") + } + nomadClientCert := os.Getenv("NOMAD_CLIENT_CERT") + if nomadClientCert == "" { + return nil, fmt.Errorf("NOMAD_CLIENT_CERT env variable is not set") + } + nomadClientKey := os.Getenv("NOMAD_CLIENT_KEY") + if nomadClientKey == "" { + return nil, fmt.Errorf("NOMAD_CLIENT_KEY env variable is not set") + } + + nomadConfig := nomad.DefaultConfig() + + nomadClient, err := nomad.NewClient(nomadConfig) + if err != nil { + return nil, err + } + + return &Cluster{ + hcloudClient: hcloudClient, + nomadClient: nomadClient, + volumesCreated: make(map[string]struct{}), + lock: sync.Mutex{}, + }, nil +} + +func (cluster *Cluster) Cleanup() []error { + var cleanupErrors []error + + for volName := range cluster.volumesCreated { + vol, _, err := cluster.hcloudClient.Volume.GetByName(context.Background(), volName) + if err != nil { + cleanupErrors = append(cleanupErrors, err) + } + _, err = cluster.hcloudClient.Volume.Delete(context.Background(), vol) + if err != nil { + cleanupErrors = append(cleanupErrors, err) + } + } + + return cleanupErrors +} + +func (cluster *Cluster) CreateVolume(volReq *nomad.CSIVolume, w *nomad.WriteOptions) ([]*nomad.CSIVolume, *nomad.WriteMeta, error) { + vol, meta, err := cluster.nomadClient.CSIVolumes().Create(volReq, w) + if err != nil { + return nil, nil, err + } + + cluster.lock.Lock() + defer cluster.lock.Unlock() + + cluster.volumesCreated[volReq.ID] = struct{}{} + + return vol, meta, err +} + +func (cluster *Cluster) DeleteVolume(externalVolID string, w *nomad.WriteOptions) error { + err := cluster.nomadClient.CSIVolumes().Delete(externalVolID, w) + if err != nil { + return err + } + + cluster.lock.Lock() + defer cluster.lock.Unlock() + + delete(cluster.volumesCreated, externalVolID) + + return nil +} From 56d13f25bd03fdd5a186acd21a95ad14a873cb3e Mon Sep 17 00:00:00 2001 From: lukasmetzner Date: Thu, 28 Nov 2024 12:11:26 +0100 Subject: [PATCH 03/26] refactor(nomad): linting fix --- test/e2e/nomad/utils.go | 1 + 1 file changed, 1 insertion(+) diff --git a/test/e2e/nomad/utils.go b/test/e2e/nomad/utils.go index 2d5b5af4..90bbe117 100644 --- a/test/e2e/nomad/utils.go +++ b/test/e2e/nomad/utils.go @@ -8,6 +8,7 @@ import ( "time" nomad "github.com/hashicorp/nomad/api" + "github.com/hetznercloud/hcloud-go/v2/hcloud" ) From c504381f398cad67f6ecea3f6b2bbb62adcfa959 Mon Sep 17 00:00:00 2001 From: lukasmetzner Date: Mon, 2 Dec 2024 11:21:47 +0100 Subject: [PATCH 04/26] ci: consul setup --- .github/workflows/test.yml | 74 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 74 insertions(+) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 83dc6512..67e161c2 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -115,3 +115,77 @@ jobs: if: always() continue-on-error: true run: make -C dev down + + nomad: + name: nomad ${{ matrix.nomad }} + runs-on: ubuntu-latest + + permissions: + id-token: write + + concurrency: + group: ${{ github.workflow }}-${{ github.ref }}-${{ matrix.nomad }} + cancel-in-progress: true + + strategy: + fail-fast: false + matrix: + include: + - nomad: 1.8.7 + consul: 1.18.4 + + env: + TF_VAR_nomad_version: ${{ matrix.nomad }} + TF_VAR_consul_version: ${{ matrix.consul }} + + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-go@v5 + with: + go-version: "1.23" + + - uses: opentofu/setup-opentofu@v1 + with: + tofu_version: v1.8.6 # renovate: datasource=github-releases depName=opentofu/opentofu + tofu_wrapper: false + + - uses: docker/setup-buildx-action@v3 + + - uses: yokawasa/action-setup-kube-tools@v0.11.2 + with: + setup-tools: | + skaffold + skaffold: v2.13.2 # renovate: datasource=github-releases depName=GoogleContainerTools/skaffold + + - uses: hashicorp/setup-nomad@main + id: setup + with: + version: ${{ matrix.nomad }} + + # used for generating the certificates + - name: setup consul binary + run: | + curl -o consul.zip https://releases.hashicorp.com/consul/${{ matrix.consul }}/consul_${{ matrix.consul }}_linux_amd64.zip + unzip consul.zip + mv consul /usr/local/bin/ + + - uses: hetznercloud/tps-action@main + + - name: Setup environment + run: make -C nomad/dev up + + - name: Run skaffold + run: | + source nomad/dev/files/env.sh + skaffold -f nomad/skaffold.yaml build + + - name: Run tests + run: | + source nomad/dev/files/env.sh + go test -v -tags e2e test/e2e/nomad/... + + - name: Cleanup + if: always() + continue-on-error: true + run: make -C nomad/dev down \ No newline at end of file From 63c3fdf73b345e5f484fe6f5ab5705870da754dd Mon Sep 17 00:00:00 2001 From: lukasmetzner Date: Mon, 2 Dec 2024 11:31:42 +0100 Subject: [PATCH 05/26] ci: updated consul and nomad version --- .github/workflows/test.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 67e161c2..3a5ae22b 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -131,8 +131,8 @@ jobs: fail-fast: false matrix: include: - - nomad: 1.8.7 - consul: 1.18.4 + - nomad: 1.9.3 + consul: 1.20.1 env: TF_VAR_nomad_version: ${{ matrix.nomad }} From a397ecee70f4e04d5ac5c0b9bef5cd8efbdafede Mon Sep 17 00:00:00 2001 From: lukasmetzner Date: Mon, 2 Dec 2024 11:40:16 +0100 Subject: [PATCH 06/26] ci: fixed go test command --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 3a5ae22b..ceddf3ae 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -183,7 +183,7 @@ jobs: - name: Run tests run: | source nomad/dev/files/env.sh - go test -v -tags e2e test/e2e/nomad/... + go test -v -tags e2e ./test/e2e/nomad/... - name: Cleanup if: always() From 9543a9a16e327ecf88f436e58d11081ab0e66666 Mon Sep 17 00:00:00 2001 From: lukasmetzner Date: Wed, 4 Dec 2024 12:58:19 +0100 Subject: [PATCH 07/26] test(nomad): extended e2e test suite Added volume resize request and validation. Added plugin information request and validation. --- test/e2e/nomad/e2e_test.go | 63 ++++++++++++++++++++++++++++++++++---- test/e2e/nomad/utils.go | 1 + 2 files changed, 58 insertions(+), 6 deletions(-) diff --git a/test/e2e/nomad/e2e_test.go b/test/e2e/nomad/e2e_test.go index 935ed721..140241b5 100644 --- a/test/e2e/nomad/e2e_test.go +++ b/test/e2e/nomad/e2e_test.go @@ -10,9 +10,16 @@ import ( "testing" nomad "github.com/hashicorp/nomad/api" + "github.com/hetznercloud/csi-driver/internal/driver" "github.com/stretchr/testify/assert" ) +const ( + initialCapacity = 10737418240 // 10GiB + resizedCapacity = 11811160064 // 11GiB + resizedCapacityGB = 11 +) + var cluster *Cluster func TestMain(m *testing.M) { @@ -33,13 +40,31 @@ func TestMain(m *testing.M) { os.Exit(exitCode) } +func TestGetPluginInfo(t *testing.T) { + plugin, _, err := cluster.nomadClient.CSIPlugins().Info(driver.PluginName, &nomad.QueryOptions{}) + if err != nil { + t.Error(err) + } + + assert.NotNil(t, plugin, "Expected plugin from Nomad to be not nil") + + assert.Equalf( + t, + plugin.Version, + driver.PluginVersion, + "Expected plugin version %s, but got %s", + driver.PluginVersion, + plugin.Version, + ) +} + func TestVolumeLifecycle(t *testing.T) { volReq := &nomad.CSIVolume{ ID: "db-vol", Name: "db-vol", Namespace: "default", PluginID: "csi.hetzner.cloud", - RequestedCapacityMin: 10737418240, + RequestedCapacityMin: initialCapacity, RequestedCapabilities: []*nomad.CSIVolumeCapability{ { AccessMode: "single-node-writer", @@ -55,7 +80,7 @@ func TestVolumeLifecycle(t *testing.T) { }, } - var volID int64 + var hcloudVolID int64 t.Run("volume creation", func(t *testing.T) { vol, _, err := cluster.CreateVolume(volReq, &nomad.WriteOptions{}) if err != nil { @@ -63,26 +88,51 @@ func TestVolumeLifecycle(t *testing.T) { } assert.Len(t, vol, 1) - volID, err := strconv.ParseInt(vol[0].ExternalID, 10, 64) + hcloudVolID, err = strconv.ParseInt(vol[0].ExternalID, 10, 64) if err != nil { t.Error(err) } - hcloudVolume, _, err := cluster.hcloudClient.Volume.GetByID(context.Background(), volID) + hcloudVolume, _, err := cluster.hcloudClient.Volume.GetByID(context.Background(), hcloudVolID) if err != nil { t.Error(err) } - assert.NotNilf(t, hcloudVolume, "could not find volume with ID %d on hcloud", volID) + assert.NotNilf(t, hcloudVolume, "could not find volume with ID %d on hcloud", hcloudVolID) }) + t.Run("volume resize", func(t *testing.T) { + volReq.RequestedCapacityMin = resizedCapacity + + _, _, err := cluster.nomadClient.CSIVolumes().Create(volReq, &nomad.WriteOptions{}) + if err != nil { + t.Error(err) + } + + hcloudVolume, _, err := cluster.hcloudClient.Volume.GetByID(context.Background(), hcloudVolID) + if err != nil { + t.Error(err) + } + + if assert.NotNilf(t, hcloudVolume, "could not find volume with ID %d on hcloud", hcloudVolID) { + assert.Equalf( + t, + hcloudVolume.Size, + resizedCapacityGB, + "Expected vol size %d, but got %d", + resizedCapacityGB, + hcloudVolume.Size, + ) + } + }) + t.Run("volume deletion", func(t *testing.T) { err := cluster.DeleteVolume(volReq.ID, &nomad.WriteOptions{}) if err != nil { t.Error(err) } - hcloudVolume, _, err := cluster.hcloudClient.Volume.GetByID(context.Background(), volID) + hcloudVolume, _, err := cluster.hcloudClient.Volume.GetByID(context.Background(), hcloudVolID) if err != nil { t.Error(err) } @@ -91,3 +141,4 @@ func TestVolumeLifecycle(t *testing.T) { }) } + diff --git a/test/e2e/nomad/utils.go b/test/e2e/nomad/utils.go index 90bbe117..6ce8f03b 100644 --- a/test/e2e/nomad/utils.go +++ b/test/e2e/nomad/utils.go @@ -119,3 +119,4 @@ func (cluster *Cluster) DeleteVolume(externalVolID string, w *nomad.WriteOptions return nil } + From c358c6b7f59b35174c392c2835f5cc5c7f0c5a88 Mon Sep 17 00:00:00 2001 From: lukasmetzner Date: Wed, 4 Dec 2024 13:02:43 +0100 Subject: [PATCH 08/26] style: fix linting --- test/e2e/nomad/utils.go | 1 - 1 file changed, 1 deletion(-) diff --git a/test/e2e/nomad/utils.go b/test/e2e/nomad/utils.go index 6ce8f03b..90bbe117 100644 --- a/test/e2e/nomad/utils.go +++ b/test/e2e/nomad/utils.go @@ -119,4 +119,3 @@ func (cluster *Cluster) DeleteVolume(externalVolID string, w *nomad.WriteOptions return nil } - From c8b3fb33be4b5c84dee9ccdd3d286f4fdce5afe3 Mon Sep 17 00:00:00 2001 From: lukasmetzner Date: Wed, 4 Dec 2024 15:23:00 +0100 Subject: [PATCH 09/26] ci: renovate config for nomad and consul --- .github/workflows/test.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index ceddf3ae..4a1a38ee 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -131,8 +131,8 @@ jobs: fail-fast: false matrix: include: - - nomad: 1.9.3 - consul: 1.20.1 + - nomad: 1.9.3 # renovate: datasource=github-releases depName=hashicorp/nomad + consul: 1.20.1 # renovate: datasource=github-releases depName=hashicorp/consul env: TF_VAR_nomad_version: ${{ matrix.nomad }} From ab35d487b67abf0cd5f0e7ccac579f4646d0be28 Mon Sep 17 00:00:00 2001 From: lukasmetzner Date: Wed, 11 Dec 2024 10:00:28 +0100 Subject: [PATCH 10/26] deps: switched to first nomad-dev-env release --- nomad/dev/main.tf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nomad/dev/main.tf b/nomad/dev/main.tf index 5b14d200..95cb75ba 100644 --- a/nomad/dev/main.tf +++ b/nomad/dev/main.tf @@ -1,5 +1,5 @@ module "dev" { - source = "github.com/hetznercloud/nomad-dev-env?ref=nomad-dev-env" + source = "github.com/hetznercloud/nomad-dev-env?ref=v0.1.0" # renovate: datasource=github-releases depName=hetznercloud/nomad-dev-env hcloud_token = var.hcloud_token -} \ No newline at end of file +} From d9a43f87f7194b1b35250d609d649a092933d6ce Mon Sep 17 00:00:00 2001 From: lukasmetzner Date: Wed, 11 Dec 2024 10:09:50 +0100 Subject: [PATCH 11/26] chore: removed copy unused tab --- nomad/dev/Makefile | 4 ---- 1 file changed, 4 deletions(-) diff --git a/nomad/dev/Makefile b/nomad/dev/Makefile index 77d6f999..fb88e42d 100644 --- a/nomad/dev/Makefile +++ b/nomad/dev/Makefile @@ -24,7 +24,3 @@ port-forward: clean: rm -Rf files/ .terraform/ terraform.tfstate* env.auto.tfvars - -deps: clean - sed -i -e 's|source = ".*"|source = "./.."|' main.tf - tofu init -upgrade \ No newline at end of file From e124b4095f3ccc08278b5f9c9677967916e17e70 Mon Sep 17 00:00:00 2001 From: lukasmetzner Date: Fri, 13 Dec 2024 10:17:06 +0100 Subject: [PATCH 12/26] test(nomad): write test data and verify on another node In this test a job is created which writes a test file of 1M to a Hetzner Cloud volume. Afterwards, this job is deleted and another job on another node is started, that verifies that the file is present and has a size greater than zero. --- test/e2e/nomad/e2e_test.go | 207 ++++++++++++++++++++++++++----------- test/e2e/nomad/utils.go | 110 ++++++++++++++++++++ 2 files changed, 258 insertions(+), 59 deletions(-) diff --git a/test/e2e/nomad/e2e_test.go b/test/e2e/nomad/e2e_test.go index 140241b5..9f0abeab 100644 --- a/test/e2e/nomad/e2e_test.go +++ b/test/e2e/nomad/e2e_test.go @@ -15,9 +15,8 @@ import ( ) const ( - initialCapacity = 10737418240 // 10GiB - resizedCapacity = 11811160064 // 11GiB - resizedCapacityGB = 11 + ResizedCapacity = 11811160064 // 11GiB + ResizedCapacityGB = 11 ) var cluster *Cluster @@ -41,44 +40,25 @@ func TestMain(m *testing.M) { } func TestGetPluginInfo(t *testing.T) { - plugin, _, err := cluster.nomadClient.CSIPlugins().Info(driver.PluginName, &nomad.QueryOptions{}) - if err != nil { - t.Error(err) - } - - assert.NotNil(t, plugin, "Expected plugin from Nomad to be not nil") - - assert.Equalf( - t, - plugin.Version, - driver.PluginVersion, - "Expected plugin version %s, but got %s", - driver.PluginVersion, - plugin.Version, - ) + plugin, _, err := cluster.nomadClient.CSIPlugins().Info(driver.PluginName, &nomad.QueryOptions{}) + if err != nil { + t.Error(err) + } + + assert.NotNil(t, plugin, "Expected plugin from Nomad to be not nil") + + assert.Equalf( + t, + plugin.Version, + driver.PluginVersion, + "Expected plugin version %s, but got %s", + driver.PluginVersion, + plugin.Version, + ) } func TestVolumeLifecycle(t *testing.T) { - volReq := &nomad.CSIVolume{ - ID: "db-vol", - Name: "db-vol", - Namespace: "default", - PluginID: "csi.hetzner.cloud", - RequestedCapacityMin: initialCapacity, - RequestedCapabilities: []*nomad.CSIVolumeCapability{ - { - AccessMode: "single-node-writer", - AttachmentMode: "file-system", - }, - }, - MountOptions: &nomad.CSIMountOptions{ - FSType: "ext4", - MountFlags: []string{ - "discard", - "defaults", - }, - }, - } + volReq := createVolumeSpec("db-vol") var hcloudVolID int64 t.Run("volume creation", func(t *testing.T) { @@ -101,30 +81,30 @@ func TestVolumeLifecycle(t *testing.T) { assert.NotNilf(t, hcloudVolume, "could not find volume with ID %d on hcloud", hcloudVolID) }) - t.Run("volume resize", func(t *testing.T) { - volReq.RequestedCapacityMin = resizedCapacity + t.Run("volume resize", func(t *testing.T) { + volReq.RequestedCapacityMin = ResizedCapacity - _, _, err := cluster.nomadClient.CSIVolumes().Create(volReq, &nomad.WriteOptions{}) - if err != nil { - t.Error(err) - } + _, _, err := cluster.nomadClient.CSIVolumes().Create(volReq, &nomad.WriteOptions{}) + if err != nil { + t.Error(err) + } - hcloudVolume, _, err := cluster.hcloudClient.Volume.GetByID(context.Background(), hcloudVolID) - if err != nil { - t.Error(err) - } + hcloudVolume, _, err := cluster.hcloudClient.Volume.GetByID(context.Background(), hcloudVolID) + if err != nil { + t.Error(err) + } if assert.NotNilf(t, hcloudVolume, "could not find volume with ID %d on hcloud", hcloudVolID) { - assert.Equalf( - t, - hcloudVolume.Size, - resizedCapacityGB, - "Expected vol size %d, but got %d", - resizedCapacityGB, - hcloudVolume.Size, - ) - } - }) + assert.Equalf( + t, + hcloudVolume.Size, + ResizedCapacityGB, + "Expected vol size %d, but got %d", + ResizedCapacityGB, + hcloudVolume.Size, + ) + } + }) t.Run("volume deletion", func(t *testing.T) { err := cluster.DeleteVolume(volReq.ID, &nomad.WriteOptions{}) @@ -139,6 +119,115 @@ func TestVolumeLifecycle(t *testing.T) { assert.Nil(t, hcloudVolume, "hcloud volume was deleted in nomad, but still exists") }) - } +func TestVolumeWrite(t *testing.T) { + volID := "test-vol" + jobID := "test-writer" + volReq := createVolumeSpec(volID) + job := createBusyboxWithVolumeJobSpec(jobID, volID, "/test") + + t.Run("create volume", func(t *testing.T) { + vol, _, err := cluster.CreateVolume(volReq, &nomad.WriteOptions{}) + if err != nil { + t.Error(err) + } + assert.Len(t, vol, 1) + }) + + // Used to ensure that the job for verifying the data is scheduled on another node + var previousNodeID string + t.Run("write to volume", func(t *testing.T) { + _, _, err := cluster.nomadClient.Jobs().Register(job, &nomad.WriteOptions{}) + if err != nil { + t.Error(err) + } + defer func() { + _, _, err = cluster.nomadClient.Jobs().Deregister(*job.ID, true, &nomad.WriteOptions{}) + if err != nil { + t.Error(err) + } + }() + + allocStub, err := cluster.WaitForRunningJob(*job.ID) + if err != nil { + t.Error(err) + return + } + + previousNodeID = allocStub.NodeID + + alloc, _, err := cluster.nomadClient.Allocations().Info(allocStub.ID, &nomad.QueryOptions{}) + if err != nil { + t.Error(err) + return + } + + exitCode, err := cluster.ExecInAlloc(alloc, jobID, []string{ + "dd", + "if=/dev/random", + "of=/test/data", + "bs=1M", + "count=1", + }) + if err != nil { + t.Error(err) + } + assert.Equalf(t, 0, exitCode, "could not write test data - exit code: %d", exitCode) + }) + + t.Run("verify volume data", func(t *testing.T) { + // try to schedule job on another node + constraint := &nomad.Affinity{ + LTarget: "${node.unique.id}", + RTarget: previousNodeID, + Operand: "!=", + } + job.Affinities = append(job.Affinities, constraint) + + _, _, err := cluster.nomadClient.Jobs().Register(job, &nomad.WriteOptions{}) + if err != nil { + t.Error(err) + } + defer func() { + _, _, err = cluster.nomadClient.Jobs().Deregister(*job.ID, true, &nomad.WriteOptions{}) + if err != nil { + t.Error(err) + } + }() + + allocStub, err := cluster.WaitForRunningJob(*job.ID) + if err != nil { + t.Error(err) + return + } + + alloc, _, err := cluster.nomadClient.Allocations().Info(allocStub.ID, &nomad.QueryOptions{}) + if err != nil { + t.Error(err) + return + } + + // verify that file exists and has a size greater than zero + exitCode, err := cluster.ExecInAlloc(alloc, jobID, []string{ + "test", + "-s", + "/test/data", + }) + if err != nil { + t.Error(err) + } + assert.Equalf(t, 0, exitCode, "could not verify test data - exit code: %d", exitCode) + }) + + t.Run("delete volume", func(t *testing.T) { + // with retries, as volume can still be in use for a couple of seconds after job got deleted, + // which results in a internal server error + for i := range 10 { + if err := cluster.DeleteVolume(volReq.ID, &nomad.WriteOptions{}); err == nil { + break + } + backoffSleep(i) + } + }) +} diff --git a/test/e2e/nomad/utils.go b/test/e2e/nomad/utils.go index 90bbe117..864c04ca 100644 --- a/test/e2e/nomad/utils.go +++ b/test/e2e/nomad/utils.go @@ -3,6 +3,7 @@ package e2e import ( "context" "fmt" + "math" "os" "sync" "time" @@ -12,6 +13,8 @@ import ( "github.com/hetznercloud/hcloud-go/v2/hcloud" ) +const InitialVolumeCapacity = 10737418240 // 10GiB + type Cluster struct { hcloudClient *hcloud.Client nomadClient *nomad.Client @@ -82,6 +85,11 @@ func (cluster *Cluster) Cleanup() []error { vol, _, err := cluster.hcloudClient.Volume.GetByName(context.Background(), volName) if err != nil { cleanupErrors = append(cleanupErrors, err) + continue + } + if vol == nil { + cleanupErrors = append(cleanupErrors, fmt.Errorf("volume %s not found on hcloud", volName)) + continue } _, err = cluster.hcloudClient.Volume.Delete(context.Background(), vol) if err != nil { @@ -119,3 +127,105 @@ func (cluster *Cluster) DeleteVolume(externalVolID string, w *nomad.WriteOptions return nil } + +func (cluster *Cluster) ExecInAlloc(alloc *nomad.Allocation, task string, command []string) (int, error) { + exitCode, err := cluster.nomadClient.Allocations().Exec( + context.Background(), + alloc, + task, + true, + command, + os.Stdin, + os.Stdout, + os.Stderr, + make(<-chan nomad.TerminalSize), + &nomad.QueryOptions{}, + ) + if err != nil { + return exitCode, err + } + return exitCode, nil +} + +func (cluster *Cluster) WaitForRunningJob(jobID string) (*nomad.AllocationListStub, error) { + for retry := range 10 { + allocs, _, err := cluster.nomadClient.Jobs().Allocations( + jobID, + false, + &nomad.QueryOptions{}, + ) + if err != nil { + return nil, err + } + + for _, alloc := range allocs { + if alloc.ClientStatus == "running" { + return alloc, nil + } + } + + backoffSleep(retry) + } + return nil, fmt.Errorf("no running allocation for job %s", jobID) +} + +func createVolumeSpec(id string) *nomad.CSIVolume { + return &nomad.CSIVolume{ + ID: id, + Name: id, + Namespace: "default", + PluginID: "csi.hetzner.cloud", + RequestedCapacityMin: InitialVolumeCapacity, + RequestedCapabilities: []*nomad.CSIVolumeCapability{ + { + AccessMode: "single-node-writer", + AttachmentMode: "file-system", + }, + }, + MountOptions: &nomad.CSIMountOptions{ + FSType: "ext4", + MountFlags: []string{ + "discard", + "defaults", + }, + }, + } + +} + +func createBusyboxWithVolumeJobSpec(id string, volumeID string, mountPath string) *nomad.Job { + job := nomad.NewServiceJob(id, id, "global", 50) + taskGroup := nomad.NewTaskGroup(id, 1) + + taskGroup.Volumes = map[string]*nomad.VolumeRequest{ + volumeID: { + Name: volumeID, + ReadOnly: false, + Type: "csi", + Source: volumeID, + AttachmentMode: "file-system", + AccessMode: "single-node-writer", + PerAlloc: false, + }, + } + + task := nomad.NewTask(id, "docker") + task = task.SetConfig("image", "busybox:stable") + task = task.SetConfig("command", "sleep") + task = task.SetConfig("args", []string{"3600"}) + + task.VolumeMounts = append(task.VolumeMounts, &nomad.VolumeMount{ + Volume: &volumeID, + Destination: &mountPath, + }) + + taskGroup = taskGroup.AddTask(task) + job = job.AddTaskGroup(taskGroup) + return job +} + +func backoffSleep(retry int) { + delay := math.Pow(2, float64(retry)) + delay = math.Min(delay, 16) + time.Sleep(time.Second * time.Duration(delay)) +} From 8fe1a6d7fcbbc97d2501dfbb80ac4cff377b6632 Mon Sep 17 00:00:00 2001 From: lukasmetzner Date: Fri, 13 Dec 2024 10:20:54 +0100 Subject: [PATCH 13/26] style: file formatting and linting --- nomad/README.md | 23 +++++++++++++++-------- nomad/hcloud-csi-node.hcl | 2 +- nomad/skaffold.yaml | 2 +- nomad/start_nomad.sh | 18 ++++++++---------- nomad/stop_nomad.sh | 2 +- test/e2e/nomad/e2e_test.go | 6 +++--- test/e2e/nomad/utils.go | 5 ++--- 7 files changed, 31 insertions(+), 27 deletions(-) diff --git a/nomad/README.md b/nomad/README.md index 09c400fd..6295be29 100644 --- a/nomad/README.md +++ b/nomad/README.md @@ -3,19 +3,25 @@ There is no official support from our side for Nomad. Nonetheless, we would still like to offer a satisfying developer experience and have automated e2e tests to avoid breaking Nomad accidentally. ## Nomad Development Environment + As a prerequisite for developing Nomad, a setup of the [`nomad-dev-env`](https://github.com/hetznercloud/nomad-dev-env) is necessary, which is located in `nomad/dev`. 1. Setup the `HCLOUD_TOKEN` environment variable 2. Deploy the development cluster: -``` bash + +```bash make -C nomad/dev up ``` + 3. Load the generated configuration to access the development cluster: -``` bash + +```bash source nomad/dev/files/env.sh ``` + 4. Check that the cluster is healthy: -``` bash + +```bash nomad node status ``` @@ -23,27 +29,28 @@ nomad node status Skaffold commands should be executed from the `csi-driver` root directory and use the `-f` flag to point to the Nomad specific `skaffold.yaml`. -``` bash +```bash skaffold -f nomad/skaffold.yaml build ``` Skaffold does not offer any native support for Nomad. For this reason we use the Nomad post build hooks to deploy/redeploy the csi plugin. To delete the csi plugin a manual execution of `stop_nomad.sh` is necessary. -``` bash +```bash bash ./nomad/stop_nomad.sh ``` ## E2E Tests The nomad e2e tests are located in `test/e2e/nomad` and need a working development environment. + 1. Deploy the csi-driver -``` bash +```bash skaffold -f nomad/skaffold.yaml build ``` 2. Run the e2e tests -``` bash +```bash go test -v -tags e2e ./test/e2e/nomad/... -``` \ No newline at end of file +``` diff --git a/nomad/hcloud-csi-node.hcl b/nomad/hcloud-csi-node.hcl index 0d5310d8..2ff2c62b 100644 --- a/nomad/hcloud-csi-node.hcl +++ b/nomad/hcloud-csi-node.hcl @@ -38,4 +38,4 @@ EOH } } } -} \ No newline at end of file +} diff --git a/nomad/skaffold.yaml b/nomad/skaffold.yaml index 94a7b6a3..12cb3046 100644 --- a/nomad/skaffold.yaml +++ b/nomad/skaffold.yaml @@ -14,4 +14,4 @@ build: local: useBuildkit: true insecureRegistries: - - localhost:30666 \ No newline at end of file + - localhost:30666 diff --git a/nomad/start_nomad.sh b/nomad/start_nomad.sh index 7691d9ee..19f874ce 100755 --- a/nomad/start_nomad.sh +++ b/nomad/start_nomad.sh @@ -2,22 +2,20 @@ set -euo pipefail -if nomad job inspect hcloud-csi-controller > /dev/null -then +if nomad job inspect hcloud-csi-controller > /dev/null; then nomad job stop -purge hcloud-csi-controller fi controller="$(mktemp)" -envsubst < "./nomad/hcloud-csi-controller.hcl" > $controller -sed -i 's/localhost:30666/docker-registry.service.consul:5000/' $controller -nomad job run $controller +envsubst < "./nomad/hcloud-csi-controller.hcl" > "$controller" +sed -i 's/localhost:30666/docker-registry.service.consul:5000/' "$controller" +nomad job run "$controller" -if nomad job inspect hcloud-csi-node > /dev/null -then +if nomad job inspect hcloud-csi-node > /dev/null; then nomad job stop -purge hcloud-csi-node fi node="$(mktemp)" -envsubst < "./nomad/hcloud-csi-node.hcl" > $node -sed -i 's/localhost:30666/docker-registry.service.consul:5000/' $node -nomad job run $node +envsubst < "./nomad/hcloud-csi-node.hcl" > "$node" +sed -i 's/localhost:30666/docker-registry.service.consul:5000/' "$node" +nomad job run "$node" diff --git a/nomad/stop_nomad.sh b/nomad/stop_nomad.sh index 736d087d..dc1d5541 100755 --- a/nomad/stop_nomad.sh +++ b/nomad/stop_nomad.sh @@ -6,4 +6,4 @@ nomad job stop -purge hcloud-csi-controller echo "Deleted hcloud-csi-controller" nomad job stop -purge hcloud-csi-node -echo "Deleted hcloud-csi-node" \ No newline at end of file +echo "Deleted hcloud-csi-node" diff --git a/test/e2e/nomad/e2e_test.go b/test/e2e/nomad/e2e_test.go index 9f0abeab..918801f3 100644 --- a/test/e2e/nomad/e2e_test.go +++ b/test/e2e/nomad/e2e_test.go @@ -58,7 +58,7 @@ func TestGetPluginInfo(t *testing.T) { } func TestVolumeLifecycle(t *testing.T) { - volReq := createVolumeSpec("db-vol") + volReq := CreateVolumeSpec("db-vol") var hcloudVolID int64 t.Run("volume creation", func(t *testing.T) { @@ -124,8 +124,8 @@ func TestVolumeLifecycle(t *testing.T) { func TestVolumeWrite(t *testing.T) { volID := "test-vol" jobID := "test-writer" - volReq := createVolumeSpec(volID) - job := createBusyboxWithVolumeJobSpec(jobID, volID, "/test") + volReq := CreateVolumeSpec(volID) + job := CreateBusyboxWithVolumeJobSpec(jobID, volID, "/test") t.Run("create volume", func(t *testing.T) { vol, _, err := cluster.CreateVolume(volReq, &nomad.WriteOptions{}) diff --git a/test/e2e/nomad/utils.go b/test/e2e/nomad/utils.go index 864c04ca..f2347860 100644 --- a/test/e2e/nomad/utils.go +++ b/test/e2e/nomad/utils.go @@ -169,7 +169,7 @@ func (cluster *Cluster) WaitForRunningJob(jobID string) (*nomad.AllocationListSt return nil, fmt.Errorf("no running allocation for job %s", jobID) } -func createVolumeSpec(id string) *nomad.CSIVolume { +func CreateVolumeSpec(id string) *nomad.CSIVolume { return &nomad.CSIVolume{ ID: id, Name: id, @@ -190,10 +190,9 @@ func createVolumeSpec(id string) *nomad.CSIVolume { }, }, } - } -func createBusyboxWithVolumeJobSpec(id string, volumeID string, mountPath string) *nomad.Job { +func CreateBusyboxWithVolumeJobSpec(id string, volumeID string, mountPath string) *nomad.Job { job := nomad.NewServiceJob(id, id, "global", 50) taskGroup := nomad.NewTaskGroup(id, 1) From 5cfd0bae96a54975de8648529ac8b5622bca521a Mon Sep 17 00:00:00 2001 From: lukasmetzner Date: Fri, 13 Dec 2024 10:29:25 +0100 Subject: [PATCH 14/26] style: missing newline at eof --- nomad/dev/variables.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nomad/dev/variables.tf b/nomad/dev/variables.tf index 58b29de0..4c0c0b86 100644 --- a/nomad/dev/variables.tf +++ b/nomad/dev/variables.tf @@ -1,3 +1,3 @@ variable "hcloud_token" { sensitive = true -} \ No newline at end of file +} From 5586260673e33ce2885f7f4592a8aa7ca96a4456 Mon Sep 17 00:00:00 2001 From: lukasmetzner Date: Fri, 13 Dec 2024 11:08:10 +0100 Subject: [PATCH 15/26] docs: updated docs for nomad --- docs/nomad/README.md | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/docs/nomad/README.md b/docs/nomad/README.md index f7201f7e..b5c9c2d1 100644 --- a/docs/nomad/README.md +++ b/docs/nomad/README.md @@ -2,7 +2,7 @@ ## Preconditions -- Nomad >= 1.4.x cluster installed following the [Nomad Reference Architecture for production deployments](https://developer.hashicorp.com/nomad/tutorials/enterprise/production-reference-architecture-vm-with-consul). The setup was tested on Nomad Community, version 1.5.x. +- Nomad >= 1.4.x cluster installed following the [Nomad Reference Architecture for production deployments](https://developer.hashicorp.com/nomad/tutorials/enterprise/production-reference-architecture-vm-with-consul). The setup was tested on Nomad Community, version 1.9.3. - The cluster nodes need to have the `docker` driver installed & configured with [`allow_privileged = true`](https://developer.hashicorp.com/nomad/docs/drivers/docker#allow_privileged). - The HCL resources are meant to be executed on a machine having nomad installed (with access to the Nomad API). @@ -33,12 +33,7 @@ job "hcloud-csi-controller" { type = "service" group "controller" { - - ### NOTE - # We define (at least) 2 allocations to increase the availability in case of a node failure with - # a controller allocation running on that node. On a "Single Node Cluster", the group stanzas - # might need modification or should be removed. - count = 2 + count = 1 constraint { distinct_hosts = true From 12fc9cf2d200c4d8bda77de601878f000797a9df Mon Sep 17 00:00:00 2001 From: Lukas Metzner Date: Mon, 16 Dec 2024 14:25:15 +0100 Subject: [PATCH 16/26] Update nomad/start_nomad.sh MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Julian Tölle --- nomad/start_nomad.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nomad/start_nomad.sh b/nomad/start_nomad.sh index 19f874ce..106021e3 100755 --- a/nomad/start_nomad.sh +++ b/nomad/start_nomad.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash set -euo pipefail From 64e5694c49791333957fff83b050b721498c21a4 Mon Sep 17 00:00:00 2001 From: Lukas Metzner Date: Mon, 16 Dec 2024 14:25:21 +0100 Subject: [PATCH 17/26] Update nomad/stop_nomad.sh MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Julian Tölle --- nomad/stop_nomad.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nomad/stop_nomad.sh b/nomad/stop_nomad.sh index dc1d5541..644cbf7a 100755 --- a/nomad/stop_nomad.sh +++ b/nomad/stop_nomad.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash set -euo pipefail From 327241b581245345de270236b25dcafd0a1ee402 Mon Sep 17 00:00:00 2001 From: lukasmetzner Date: Mon, 16 Dec 2024 15:07:10 +0100 Subject: [PATCH 18/26] ci: controller deployment count Accidentally removed the wrong commit from the history. This reintroduces the changes and adds a controller count of 2 back to the official guide. --- docs/nomad/README.md | 7 ++++++- nomad/hcloud-csi-controller.hcl | 7 +------ 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/docs/nomad/README.md b/docs/nomad/README.md index b5c9c2d1..68264610 100644 --- a/docs/nomad/README.md +++ b/docs/nomad/README.md @@ -33,7 +33,12 @@ job "hcloud-csi-controller" { type = "service" group "controller" { - count = 1 + + ### NOTE + # We define (at least) 2 allocations to increase the availability in case of a node failure with + # a controller allocation running on that node. On a "Single Node Cluster", the group stanzas + # might need modification or should be removed. + count = 2 constraint { distinct_hosts = true diff --git a/nomad/hcloud-csi-controller.hcl b/nomad/hcloud-csi-controller.hcl index d67c4101..685b0e39 100644 --- a/nomad/hcloud-csi-controller.hcl +++ b/nomad/hcloud-csi-controller.hcl @@ -4,12 +4,7 @@ job "hcloud-csi-controller" { type = "service" group "controller" { - - ### NOTE - # We define (at least) 2 allocations to increase the availability in case of a node failure with - # a controller allocation running on that node. On a "Single Node Cluster", the group stanzas - # might need modification or should be removed. - count = 2 + count = 1 constraint { distinct_hosts = true From 6943e9a4e74aa3b21ed559024439c595ba3926a3 Mon Sep 17 00:00:00 2001 From: lukasmetzner Date: Mon, 16 Dec 2024 15:11:40 +0100 Subject: [PATCH 19/26] build(nomad): removed unnecessary -c --- nomad/skaffold.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nomad/skaffold.yaml b/nomad/skaffold.yaml index 12cb3046..05d5f340 100644 --- a/nomad/skaffold.yaml +++ b/nomad/skaffold.yaml @@ -9,7 +9,7 @@ build: dockerfile: dev/Dockerfile hooks: after: - - command: ["sh", "-c", "./nomad/start_nomad.sh"] + - command: ["sh", "./nomad/start_nomad.sh"] os: [darwin, linux] local: useBuildkit: true From 94871a003ff3552daeb21a50e23fd0222ddfc0e0 Mon Sep 17 00:00:00 2001 From: Lukas Metzner Date: Mon, 16 Dec 2024 15:14:06 +0100 Subject: [PATCH 20/26] Update nomad/README.md Co-authored-by: Jonas L. --- nomad/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nomad/README.md b/nomad/README.md index 6295be29..eb83110a 100644 --- a/nomad/README.md +++ b/nomad/README.md @@ -1,6 +1,6 @@ # Nomad Development -There is no official support from our side for Nomad. Nonetheless, we would still like to offer a satisfying developer experience and have automated e2e tests to avoid breaking Nomad accidentally. +Hetzner Cloud does not provide official support for running the CSI driver in Nomad. Nonetheless, we would still like to offer a satisfying developer experience and have automated e2e tests to avoid breaking Nomad accidentally. ## Nomad Development Environment From 50ce492ff185c3274bc983ccc42f76d0787c9bbd Mon Sep 17 00:00:00 2001 From: Lukas Metzner Date: Mon, 16 Dec 2024 15:14:13 +0100 Subject: [PATCH 21/26] Update nomad/README.md Co-authored-by: Jonas L. --- nomad/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nomad/README.md b/nomad/README.md index eb83110a..801eb3be 100644 --- a/nomad/README.md +++ b/nomad/README.md @@ -1,4 +1,4 @@ -# Nomad Development +# Nomad Hetzner Cloud does not provide official support for running the CSI driver in Nomad. Nonetheless, we would still like to offer a satisfying developer experience and have automated e2e tests to avoid breaking Nomad accidentally. From 931b9266f5884f0ba94635951a5d86e142380178 Mon Sep 17 00:00:00 2001 From: lukasmetzner Date: Mon, 16 Dec 2024 15:15:33 +0100 Subject: [PATCH 22/26] style(nomad): fix indent --- test/e2e/nomad/e2e_test.go | 46 +++++++++++++++++++------------------- 1 file changed, 23 insertions(+), 23 deletions(-) diff --git a/test/e2e/nomad/e2e_test.go b/test/e2e/nomad/e2e_test.go index 918801f3..7d19fa84 100644 --- a/test/e2e/nomad/e2e_test.go +++ b/test/e2e/nomad/e2e_test.go @@ -135,8 +135,8 @@ func TestVolumeWrite(t *testing.T) { assert.Len(t, vol, 1) }) - // Used to ensure that the job for verifying the data is scheduled on another node - var previousNodeID string + // Used to ensure that the job for verifying the data is scheduled on another node + var previousNodeID string t.Run("write to volume", func(t *testing.T) { _, _, err := cluster.nomadClient.Jobs().Register(job, &nomad.WriteOptions{}) if err != nil { @@ -155,7 +155,7 @@ func TestVolumeWrite(t *testing.T) { return } - previousNodeID = allocStub.NodeID + previousNodeID = allocStub.NodeID alloc, _, err := cluster.nomadClient.Allocations().Info(allocStub.ID, &nomad.QueryOptions{}) if err != nil { @@ -164,12 +164,12 @@ func TestVolumeWrite(t *testing.T) { } exitCode, err := cluster.ExecInAlloc(alloc, jobID, []string{ - "dd", - "if=/dev/random", - "of=/test/data", - "bs=1M", - "count=1", - }) + "dd", + "if=/dev/random", + "of=/test/data", + "bs=1M", + "count=1", + }) if err != nil { t.Error(err) } @@ -177,13 +177,13 @@ func TestVolumeWrite(t *testing.T) { }) t.Run("verify volume data", func(t *testing.T) { - // try to schedule job on another node - constraint := &nomad.Affinity{ - LTarget: "${node.unique.id}", - RTarget: previousNodeID, - Operand: "!=", - } - job.Affinities = append(job.Affinities, constraint) + // try to schedule job on another node + constraint := &nomad.Affinity{ + LTarget: "${node.unique.id}", + RTarget: previousNodeID, + Operand: "!=", + } + job.Affinities = append(job.Affinities, constraint) _, _, err := cluster.nomadClient.Jobs().Register(job, &nomad.WriteOptions{}) if err != nil { @@ -208,12 +208,12 @@ func TestVolumeWrite(t *testing.T) { return } - // verify that file exists and has a size greater than zero + // verify that file exists and has a size greater than zero exitCode, err := cluster.ExecInAlloc(alloc, jobID, []string{ - "test", - "-s", - "/test/data", - }) + "test", + "-s", + "/test/data", + }) if err != nil { t.Error(err) } @@ -221,8 +221,8 @@ func TestVolumeWrite(t *testing.T) { }) t.Run("delete volume", func(t *testing.T) { - // with retries, as volume can still be in use for a couple of seconds after job got deleted, - // which results in a internal server error + // with retries, as volume can still be in use for a couple of seconds after job got deleted, + // which results in a internal server error for i := range 10 { if err := cluster.DeleteVolume(volReq.ID, &nomad.WriteOptions{}); err == nil { break From 56c2ff49435504bf9ab3bc718318a931968cec9f Mon Sep 17 00:00:00 2001 From: lukasmetzner Date: Mon, 16 Dec 2024 15:18:44 +0100 Subject: [PATCH 23/26] style(nomad): fix newline --- nomad/hcloud-csi-controller.hcl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nomad/hcloud-csi-controller.hcl b/nomad/hcloud-csi-controller.hcl index 685b0e39..b45659be 100644 --- a/nomad/hcloud-csi-controller.hcl +++ b/nomad/hcloud-csi-controller.hcl @@ -52,4 +52,4 @@ EOH } } } -} \ No newline at end of file +} From 2e2220a0ec16fc061a9e803bff1e97acd49309cc Mon Sep 17 00:00:00 2001 From: lukasmetzner Date: Mon, 16 Dec 2024 16:13:35 +0100 Subject: [PATCH 24/26] style: fix newline --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 4a1a38ee..a6fbc738 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -188,4 +188,4 @@ jobs: - name: Cleanup if: always() continue-on-error: true - run: make -C nomad/dev down \ No newline at end of file + run: make -C nomad/dev down From 95c6ffeec689388f820f86f66030845ab5ab15e2 Mon Sep 17 00:00:00 2001 From: lukasmetzner Date: Mon, 16 Dec 2024 16:14:58 +0100 Subject: [PATCH 25/26] refactor: removed unnecassary version for hcloud client setup --- test/e2e/nomad/utils.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/e2e/nomad/utils.go b/test/e2e/nomad/utils.go index f2347860..81794381 100644 --- a/test/e2e/nomad/utils.go +++ b/test/e2e/nomad/utils.go @@ -31,7 +31,7 @@ func NewCluster() (*Cluster, error) { hcloudOpts := []hcloud.ClientOption{ hcloud.WithToken(token), - hcloud.WithApplication("nomad-csi-e2e", "v2.11.0"), // x-releaser-pleaser-version + hcloud.WithApplication("nomad-csi-e2e", ""), hcloud.WithPollOpts(hcloud.PollOpts{ BackoffFunc: hcloud.ExponentialBackoffWithOpts(hcloud.ExponentialBackoffOpts{ Base: time.Second, From 53f20dcdb568ca93366fded3718a15a279e84eb2 Mon Sep 17 00:00:00 2001 From: lukasmetzner Date: Mon, 16 Dec 2024 16:29:35 +0100 Subject: [PATCH 26/26] build: switched to bash for nomads skaffold.yaml --- nomad/skaffold.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nomad/skaffold.yaml b/nomad/skaffold.yaml index 05d5f340..d659f074 100644 --- a/nomad/skaffold.yaml +++ b/nomad/skaffold.yaml @@ -9,7 +9,7 @@ build: dockerfile: dev/Dockerfile hooks: after: - - command: ["sh", "./nomad/start_nomad.sh"] + - command: ["bash", "./nomad/start_nomad.sh"] os: [darwin, linux] local: useBuildkit: true