feat(nomad): improved nomad support

.github/workflows/test.yml

@@ -115,3 +115,77 @@ jobs:
         if: always()
         continue-on-error: true
         run: make -C dev down
+
+  nomad:
+    name: nomad ${{ matrix.nomad }}
+    runs-on: ubuntu-latest
+
+    permissions:
+      id-token: write
+
+    concurrency:
+      group: ${{ github.workflow }}-${{ github.ref }}-${{ matrix.nomad }}
+      cancel-in-progress: true
+
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - nomad: 1.9.3 # renovate: datasource=github-releases depName=hashicorp/nomad
+            consul: 1.20.1 # renovate: datasource=github-releases depName=hashicorp/consul
+
+    env:
+      TF_VAR_nomad_version: ${{ matrix.nomad }}
+      TF_VAR_consul_version: ${{ matrix.consul }}
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: actions/setup-go@v5
+        with:
+          go-version: "1.23"
+
+      - uses: opentofu/setup-opentofu@v1
+        with:
+          tofu_version: v1.8.6 # renovate: datasource=github-releases depName=opentofu/opentofu
+          tofu_wrapper: false
+
+      - uses: docker/setup-buildx-action@v3
+
+      - uses: yokawasa/[email protected]
+        with:
+          setup-tools: |
+            skaffold
+          skaffold: v2.13.2 # renovate: datasource=github-releases depName=GoogleContainerTools/skaffold
+
+      - uses: hashicorp/setup-nomad@main
+        id: setup
+        with:
+          version: ${{ matrix.nomad }}
+
+      # used for generating the certificates
+      - name: setup consul binary
+        run: |
+          curl -o consul.zip https://releases.hashicorp.com/consul/${{ matrix.consul }}/consul_${{ matrix.consul }}_linux_amd64.zip
+          unzip consul.zip
+          mv consul /usr/local/bin/
+
+      - uses: hetznercloud/tps-action@main
+
+      - name: Setup environment
+        run: make -C nomad/dev up
+
+      - name: Run skaffold
+        run: |
+          source nomad/dev/files/env.sh
+          skaffold -f nomad/skaffold.yaml build
+
+      - name: Run tests
+        run: |
+          source nomad/dev/files/env.sh
+          go test -v -tags e2e ./test/e2e/nomad/...
+
+      - name: Cleanup
+        if: always()
+        continue-on-error: true
+        run: make -C nomad/dev down

docs/nomad/README.md

@@ -2,7 +2,7 @@
 
 ## Preconditions
 
-- Nomad >= 1.4.x cluster installed following the [Nomad Reference Architecture for production deployments](https://developer.hashicorp.com/nomad/tutorials/enterprise/production-reference-architecture-vm-with-consul). The setup was tested on Nomad Community, version 1.5.x.
+- Nomad >= 1.4.x cluster installed following the [Nomad Reference Architecture for production deployments](https://developer.hashicorp.com/nomad/tutorials/enterprise/production-reference-architecture-vm-with-consul). The setup was tested on Nomad Community, version 1.9.3.
 - The cluster nodes need to have the `docker` driver installed & configured with [`allow_privileged = true`](https://developer.hashicorp.com/nomad/docs/drivers/docker#allow_privileged).
 - The HCL resources are meant to be executed on a machine having nomad installed (with access to the Nomad API).
 

go.mod

@@ -31,8 +31,17 @@ require (
 	github.com/google/go-cmp v0.6.0 // indirect
 	github.com/google/pprof v0.0.0-20240525223248-4bfdf5a9a2af // indirect
 	github.com/google/uuid v1.6.0 // indirect
+	github.com/gorilla/websocket v1.5.0 // indirect
 	github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.1.0 // indirect
+	github.com/hashicorp/cronexpr v1.1.2 // indirect
+	github.com/hashicorp/errwrap v1.1.0 // indirect
+	github.com/hashicorp/go-cleanhttp v0.5.2 // indirect
+	github.com/hashicorp/go-multierror v1.1.1 // indirect
+	github.com/hashicorp/go-rootcerts v1.0.2 // indirect
+	github.com/hashicorp/nomad/api v0.0.0-20241125123754-1f29a95c2413 // indirect
 	github.com/klauspost/compress v1.17.11 // indirect
+	github.com/mitchellh/go-homedir v1.1.0 // indirect
+	github.com/mitchellh/mapstructure v1.5.0 // indirect
 	github.com/moby/sys/mountinfo v0.7.2 // indirect
 	github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
 	github.com/onsi/ginkgo/v2 v2.19.0 // indirect

go.sum

@@ -29,10 +29,25 @@ github.com/google/pprof v0.0.0-20240525223248-4bfdf5a9a2af h1:kmjWCqn2qkEml422C2
 github.com/google/pprof v0.0.0-20240525223248-4bfdf5a9a2af/go.mod h1:K1liHPHnj73Fdn/EKuT8nrFqBihUSKXoLYU0BuatOYo=
 github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
 github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/gorilla/websocket v1.5.0 h1:PPwGk2jz7EePpoHN/+ClbZu8SPxiqlu12wZP/3sWmnc=
+github.com/gorilla/websocket v1.5.0/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
 github.com/grpc-ecosystem/go-grpc-middleware/providers/prometheus v1.0.1 h1:qnpSQwGEnkcRpTqNOIR6bJbR0gAorgP9CSALpRcKoAA=
 github.com/grpc-ecosystem/go-grpc-middleware/providers/prometheus v1.0.1/go.mod h1:lXGCsh6c22WGtjr+qGHj1otzZpV/1kwTMAqkwZsnWRU=
 github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.1.0 h1:pRhl55Yx1eC7BZ1N+BBWwnKaMyD8uC+34TLdndZMAKk=
 github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.1.0/go.mod h1:XKMd7iuf/RGPSMJ/U4HP0zS2Z9Fh8Ps9a+6X26m/tmI=
+github.com/hashicorp/cronexpr v1.1.2 h1:wG/ZYIKT+RT3QkOdgYc+xsKWVRgnxJ1OJtjjy84fJ9A=
+github.com/hashicorp/cronexpr v1.1.2/go.mod h1:P4wA0KBl9C5q2hABiMO7cp6jcIg96CDh1Efb3g1PWA4=
+github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
+github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY2I=
+github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
+github.com/hashicorp/go-cleanhttp v0.5.2 h1:035FKYIWjmULyFRBKPs8TBQoi0x6d9G4xc9neXJWAZQ=
+github.com/hashicorp/go-cleanhttp v0.5.2/go.mod h1:kO/YDlP8L1346E6Sodw+PrpBSV4/SoxCXGY6BqNFT48=
+github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo=
+github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM=
+github.com/hashicorp/go-rootcerts v1.0.2 h1:jzhAVGtqPKbwpyCPELlgNWhE1znq+qwJtW5Oi2viEzc=
+github.com/hashicorp/go-rootcerts v1.0.2/go.mod h1:pqUvnprVnM5bf7AOirdbb01K4ccR319Vf4pU3K5EGc8=
+github.com/hashicorp/nomad/api v0.0.0-20241125123754-1f29a95c2413 h1:k0Z3HkPTwMY51/P6gRe20+oCQq6mszc7xJnpM1A+T4w=
+github.com/hashicorp/nomad/api v0.0.0-20241125123754-1f29a95c2413/go.mod h1:svtxn6QnrQ69P23VvIWMR34tg3vmwLz4UdUzm1dSCgE=
 github.com/hetznercloud/hcloud-go/v2 v2.14.0 h1:WQW72DuOGqT486F0eNp92lDH5cwDTmyn9Mhin93m1To=
 github.com/hetznercloud/hcloud-go/v2 v2.14.0/go.mod h1:h8sHav+27Xa+48cVMAvAUMELov5h298Ilg2vflyTHgg=
 github.com/hetznercloud/hcloud-go/v2 v2.15.0 h1:6mpMJ/RuX1woZj+MCJdyKNEX9129KDkEIDeeyfr4GD4=
@@ -53,6 +68,10 @@ github.com/kubernetes-csi/csi-test/v5 v5.3.1 h1:Wiukp1In+kif+BFo6q2ExjgB+MbrAz4j
 github.com/kubernetes-csi/csi-test/v5 v5.3.1/go.mod h1:7hA2cSYJ6T8CraEZPA6zqkLZwemjBD54XAnPsPC3VpA=
 github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc=
 github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw=
+github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y=
+github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
+github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY=
+github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo=
 github.com/moby/buildkit v0.16.0 h1:wOVBj1o5YNVad/txPQNXUXdelm7Hs/i0PUFjzbK0VKE=
 github.com/moby/buildkit v0.16.0/go.mod h1:Xqx/5GlrqE1yIRORk0NSCVDFpQAU1WjlT6KHYZdisIQ=
 github.com/moby/buildkit v0.17.0 h1:ZA/4AxwBbve1f3ZaNNJQiCBtTV62R6YweWNwq4A+sTc=

nomad/README.md

@@ -0,0 +1,56 @@
+# Nomad
+
+Hetzner Cloud does not provide official support for running the CSI driver in Nomad. Nonetheless, we would still like to offer a satisfying developer experience and have automated e2e tests to avoid breaking Nomad accidentally.
+
+## Nomad Development Environment
+
+As a prerequisite for developing Nomad, a setup of the [`nomad-dev-env`](https://github.com/hetznercloud/nomad-dev-env) is necessary, which is located in `nomad/dev`.
+
+1. Setup the `HCLOUD_TOKEN` environment variable
+2. Deploy the development cluster:
+
+```bash
+make -C nomad/dev up
+```
+
+3. Load the generated configuration to access the development cluster:
+
+```bash
+source nomad/dev/files/env.sh
+```
+
+4. Check that the cluster is healthy:
+
+```bash
+nomad node status
+```
+
+## Skaffold
+
+Skaffold commands should be executed from the `csi-driver` root directory and use the `-f` flag to point to the Nomad specific `skaffold.yaml`.
+
+```bash
+skaffold -f nomad/skaffold.yaml build
+```
+
+Skaffold does not offer any native support for Nomad. For this reason we use the Nomad post build hooks to deploy/redeploy the csi plugin. To delete the csi plugin a manual execution of `stop_nomad.sh` is necessary.
+
+```bash
+bash ./nomad/stop_nomad.sh
+```
+
+## E2E Tests
+
+The nomad e2e tests are located in `test/e2e/nomad` and need a working development environment.
+
+1. Deploy the csi-driver
+
+```bash
+skaffold -f nomad/skaffold.yaml build
+```
+
+2. Run the e2e tests
+
+```bash
+go test -v -tags e2e ./test/e2e/nomad/...
+```

nomad/dev/.gitignore

@@ -0,0 +1,6 @@
+.terraform/
+terraform.tfstate*
+.terraform.tfstate*
+*.auto.tfvars
+
+files/

nomad/dev/Makefile

@@ -0,0 +1,26 @@
+SHELL = bash
+
+ENV ?= dev
+
+env.auto.tfvars:
+	@echo 'hcloud_token = "$(HCLOUD_TOKEN)"' >> "$@"
+
+.terraform:
+	tofu init
+
+validate: .terraform
+	tofu validate
+
+up: .terraform env.auto.tfvars
+	tofu apply -auto-approve
+	$(MAKE) port-forward
+
+down: .terraform env.auto.tfvars
+	if test -f files/registry-port-forward.sh; then files/registry-port-forward.sh down; fi
+	tofu destroy -auto-approve
+
+port-forward:
+	files/registry-port-forward.sh up
+
+clean:
+	rm -Rf files/ .terraform/ terraform.tfstate* env.auto.tfvars

nomad/dev/main.tf

@@ -0,0 +1,5 @@
+module "dev" {
+  source = "github.com/hetznercloud/nomad-dev-env?ref=v0.1.0" # renovate: datasource=github-releases depName=hetznercloud/nomad-dev-env
+
+  hcloud_token = var.hcloud_token
+}

nomad/dev/variables.tf

@@ -0,0 +1,3 @@
+variable "hcloud_token" {
+  sensitive = true
+}

nomad/hcloud-csi-controller.hcl

@@ -0,0 +1,55 @@
+job "hcloud-csi-controller" {
+  datacenters = ["dc1"]
+  namespace   = "default"
+  type        = "service"
+
+  group "controller" {
+    count = 1
+
+    constraint {
+      distinct_hosts = true
+    }
+
+    update {
+      max_parallel     = 1
+      canary           = 1
+      min_healthy_time = "10s"
+      healthy_deadline = "1m"
+      auto_revert      = true
+      auto_promote     = true
+    }
+
+    task "plugin" {
+      driver = "docker"
+
+      config {
+        image   = "$SKAFFOLD_IMAGE"
+        command = "bin/hcloud-csi-driver-controller"
+      }
+
+      env {
+        CSI_ENDPOINT   = "unix://csi/csi.sock"
+        ENABLE_METRICS = true
+      }
+
+      template {
+        data        = <<EOH
+HCLOUD_TOKEN="{{ with nomadVar "secrets/hcloud" }}{{ .hcloud_token }}{{ end }}"
+EOH
+        destination = "${NOMAD_SECRETS_DIR}/hcloud-token.env"
+        env         = true
+      }
+
+      csi_plugin {
+        id        = "csi.hetzner.cloud"
+        type      = "controller"
+        mount_dir = "/csi"
+      }
+
+      resources {
+        cpu    = 100
+        memory = 64
+      }
+    }
+  }
+}