Kuttl tests

.github/workflows/kuttl_tests.yaml

@@ -0,0 +1,90 @@
+name: kuttl tests
+
+on:
+  push:
+    branches:
+      - main
+    paths-ignore:
+      - 'docs/**'
+      - 'CHANGELOG/**'
+  pull_request:
+    branches: [ main ]
+    paths-ignore:
+      - 'docs/**'
+      - 'CHANGELOG/**'
+jobs:
+  build_image:
+    name: Build image
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check out code into the Go module directory
+        uses: actions/checkout@v2
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v1
+      - name: Cache Docker layers
+        uses: actions/cache@v2
+        with:
+          path: /tmp/.buildx-cache
+          key: ${{ runner.os }}-buildx-${{ github.sha }}
+          restore-keys: |
+            ${{ runner.os }}-buildx-  uses: docker/setup-buildx-action@v1
+      - name: Set git parsed values
+        id: vars
+        run: |
+          echo ::set-output name=sha_short::$(git rev-parse --short=8 ${{ github.sha }})
+      - name: Build Docker image
+        id: docker_build
+        uses: docker/build-push-action@v2
+        with:
+          file: Dockerfile
+          context: .
+          push: false
+          tags: k8ssandra/k8ssandra-operator:${{ steps.vars.outputs.sha_short }}, k8ssandra/k8ssandra-operator:latest
+          platforms: linux/amd64
+          cache-from: type=local,src=/tmp/.buildx-cache
+          cache-to: type=local,dest=/tmp/.buildx-cache
+          outputs: type=docker,dest=/tmp/k8ssandra-k8ssandra-operator.tar
+      - name: Upload k8ssandra-operator image
+        uses: actions/upload-artifact@v2
+        with:
+          name: k8ssandra-operator
+          path: /tmp/k8ssandra-k8ssandra-operator.tar
+  run_kuttl_test:
+    runs-on: ubuntu-latest
+    needs: [build_image]
+    name: Run kuttl tests
+    strategy:
+      matrix:
+        k8s_version: ["1.21"]
+    steps:
+    - name: Check out code into the Go module directory
+      uses: actions/checkout@v2
+    - name: Cache Docker layers
+      uses: actions/cache@v2
+      with:
+        path: /tmp/.buildx-cache
+        key: ${{ runner.os }}-buildx-${{ github.sha }}
+        restore-keys: |
+          ${{ runner.os }}-buildx-  uses: docker/setup-buildx-action@v1
+    - name: Set git parsed values
+      id: vars
+      run: |
+        echo ::set-output name=sha_short::$(git rev-parse --short=8 ${{ github.sha }})
+    - name: Kind kube-proxy issue workaround
+      run: sudo sysctl net/netfilter/nf_conntrack_max=524288
+    - name: Download k8ssandra-operator image
+      uses: actions/download-artifact@v2
+      with:
+        name: k8ssandra-operator
+        path: /tmp
+    - name: Load Docker images
+      run: |
+        docker load --input /tmp/k8ssandra-k8ssandra-operator.tar
+    - name: install kustomize
+      run: |
+        make kustomize
+    - name: Run kuttl-test
+      run: |
+        # We are running tests against k8s 1.20 - 1.22 currently. 
+        # Additional versions must be added in kind config files under ./test/config/kind
+        make KUTTL_KIND_CFG="./test/kuttl/config/kind/w3k${{ matrix.k8s_version }}.yaml" kuttl-test

.gitignore

@@ -26,4 +26,6 @@ build
 *~
 clientconfig
 .vscode/
-.devcontainer/
+.devcontainer/
+kind-log*/
+kubeconfig

Makefile

@@ -315,3 +315,16 @@ catalog-build: opm ## Build a catalog image.
 .PHONY: catalog-push
 catalog-push: ## Push a catalog image.
 	$(MAKE) docker-push IMG=$(CATALOG_IMG)
+
+# E2E tests from kuttl
+kuttl-test: install-kuttl docker-build
+	./bin/kubectl-kuttl test --kind-config=${KUTTL_KIND_CFG}
+
+ # Install kuttl for e2e tests.
+install-kuttl: 
+	mkdir -p ./bin ; \
+	cd ./bin ; \
+	OS="$$(uname | tr '[:upper:]' '[:lower:]')" ; \
+  	ARCH="$$(uname -m | sed -e 's/\(arm\)\(64\)\?.*/\1\2/' -e 's/aarch64$$/arm64/')" ; \
+	curl -LO https://github.com/kudobuilder/kuttl/releases/download/v0.11.1/kuttl_0.11.1_$${OS}_$${ARCH}.tar.gz ; \
+	tar -zxvf kuttl_0.11.1_$${OS}_$${ARCH}.tar.gz ; 

kuttl-test.yaml

@@ -0,0 +1,13 @@
+apiVersion: kuttl.dev/v1beta1
+kind: TestSuite
+testDirs:
+- ./test/kuttl
+
+startKIND: true
+kindNodeCache: true
+# we are using a k8s 1.21 as default. Other versions can be run using the --kind-config flag.
+kindConfig: test/kuttl/config/kind/w3k1.21.yaml 
+timeout: 1400
+namespace: k8ssandra-operator
+kindContainers:
+- k8ssandra/k8ssandra-operator:latest

test/kuttl/config/kind/w1k1.20.yaml

@@ -0,0 +1,9 @@
+kind: Cluster
+apiVersion: kind.x-k8s.io/v1alpha4
+networking:
+  apiServerPort: 45451
+nodes:
+- role: control-plane
+  image: kindest/node:v1.20.7
+- role: worker
+  image: kindest/node:v1.20.7

test/kuttl/config/kind/w1k1.21.yaml

@@ -0,0 +1,9 @@
+kind: Cluster
+apiVersion: kind.x-k8s.io/v1alpha4
+networking:
+  apiServerPort: 45451
+nodes:
+- role: control-plane
+  image: kindest/node:v1.21.2
+- role: worker
+  image: kindest/node:v1.21.2

test/kuttl/config/kind/w1k1.22.yaml

@@ -0,0 +1,9 @@
+kind: Cluster
+apiVersion: kind.x-k8s.io/v1alpha4
+networking:
+  apiServerPort: 45451
+nodes:
+- role: control-plane
+  image: kindest/node:v1.22.2
+- role: worker
+  image: kindest/node:v1.22.2

test/kuttl/config/kind/w3k1.21.yaml

@@ -0,0 +1,12 @@
+kind: Cluster
+apiVersion: kind.x-k8s.io/v1alpha4
+networking:
+  apiServerPort: 45451
+nodes:
+- role: control-plane
+- role: worker
+  image: kindest/node:v1.21.2
+- role: worker
+  image: kindest/node:v1.21.2
+- role: worker
+  image: kindest/node:v1.21.2

test/kuttl/test-cassDC-deploys/00-assert.yaml

@@ -0,0 +1,45 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: cert-manager-cainjector
+  namespace: "cert-manager"
+  labels:
+    app: cainjector
+    app.kubernetes.io/name: cainjector
+status:
+  conditions:
+  - status: "True"
+    type: Available
+  - status: "True"
+    type: Progressing
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: cert-manager
+  namespace: "cert-manager"
+  labels:
+    app: cert-manager
+    app.kubernetes.io/name: cert-manager
+    app.kubernetes.io/instance: cert-manager
+status:
+  conditions:
+  - status: "True"
+    type: Available
+  - status: "True"
+    type: Progressing
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: cert-manager-webhook
+  namespace: "cert-manager"
+  labels:
+    app: webhook
+    app.kubernetes.io/name: webhook
+status:
+  conditions:
+  - status: "True"
+    type: Available
+  - status: "True"
+    type: Progressing

test/kuttl/test-cassDC-deploys/00-cert-manager.yaml

@@ -0,0 +1,5 @@
+apiVersion: kuttl.dev/v1beta1
+kind: TestStep
+commands: 
+- command: kubectl apply -f https://github.com/jetstack/cert-manager/releases/download/v1.6.0/cert-manager.yaml
+  ignoreFailure: false 

test/kuttl/test-cassDC-deploys/01-assert.yaml

@@ -0,0 +1,22 @@
+# apiVersion: apiextensions.k8s.io/v1
+# kind: CustomResourceDefinition
+#   name: k8ssandraclusters.k8ssandra.io
+# spec:
+#   group: k8ssandra.io
+#   names:
+#     kind: K8ssandraCluster
+#     listKind: K8ssandraClusterList
+#     plural: k8ssandraclusters
+#     singular: k8ssandracluster
+# ---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: k8ssandra-operator
+  namespace: k8ssandra-operator
+status:
+  conditions:
+  - status: "True"
+    type: Available
+  - status: "True"
+    type: Progressing

test/kuttl/test-cassDC-deploys/01-k8ssandra-operator.yaml

@@ -0,0 +1,6 @@
+apiVersion: kuttl.dev/v1beta1
+kind: TestStep
+commands: 
+- script: |
+    ../../../bin/kustomize build ../../../config/deployments/control-plane | kubectl apply -n $NAMESPACE --server-side --force-conflicts -f -
+  ignoreFailure: false 

test/kuttl/test-cassDC-deploys/02-assert.yaml

@@ -0,0 +1,7 @@
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+  name: demo-dc1-default-sts
+  namespace: k8ssandra-operator
+status:
+  readyReplicas: 3

test/kuttl/test-cassDC-deploys/02-k8ssandra-cluster.yaml

@@ -0,0 +1,5 @@
+apiVersion: kuttl.dev/v1beta1
+kind: TestStep
+commands: 
+- command:  kubectl apply -n $NAMESPACE --server-side --force-conflicts -f ../../testdata/samples/k8ssandra-multi-kind.yaml
+  ignoreFailure: false