From 7e7185d1215ce7f8ca3ef8126dd90482a4204850 Mon Sep 17 00:00:00 2001 From: Miles-Garnsey Date: Thu, 2 Dec 2021 10:18:53 +1100 Subject: [PATCH 1/2] * Add kind config yamls. * Add kuttl test for management API heap size. * Add kuttl makefile targets. --- .gitignore | 4 +- Makefile | 13 ++++++ kuttl-test.yaml | 13 ++++++ test/kuttl/config/kind/w1k1.20.yaml | 9 ++++ test/kuttl/config/kind/w1k1.21.yaml | 9 ++++ test/kuttl/config/kind/w1k1.22.yaml | 9 ++++ test/kuttl/config/kind/w3k1.21.yaml | 12 +++++ test/kuttl/test-cassDC-deploys/00-assert.yaml | 45 +++++++++++++++++++ .../test-cassDC-deploys/00-cert-manager.yaml | 5 +++ test/kuttl/test-cassDC-deploys/01-assert.yaml | 22 +++++++++ .../01-k8ssandra-operator.yaml | 6 +++ test/kuttl/test-cassDC-deploys/02-assert.yaml | 7 +++ .../02-k8ssandra-cluster.yaml | 5 +++ 13 files changed, 158 insertions(+), 1 deletion(-) create mode 100644 kuttl-test.yaml create mode 100644 test/kuttl/config/kind/w1k1.20.yaml create mode 100644 test/kuttl/config/kind/w1k1.21.yaml create mode 100644 test/kuttl/config/kind/w1k1.22.yaml create mode 100644 test/kuttl/config/kind/w3k1.21.yaml create mode 100644 test/kuttl/test-cassDC-deploys/00-assert.yaml create mode 100644 test/kuttl/test-cassDC-deploys/00-cert-manager.yaml create mode 100644 test/kuttl/test-cassDC-deploys/01-assert.yaml create mode 100644 test/kuttl/test-cassDC-deploys/01-k8ssandra-operator.yaml create mode 100644 test/kuttl/test-cassDC-deploys/02-assert.yaml create mode 100644 test/kuttl/test-cassDC-deploys/02-k8ssandra-cluster.yaml diff --git a/.gitignore b/.gitignore index a5251286d..347c6280d 100644 --- a/.gitignore +++ b/.gitignore @@ -26,4 +26,6 @@ build *~ clientconfig .vscode/ -.devcontainer/ \ No newline at end of file +.devcontainer/ +kind-log*/ +kubeconfig \ No newline at end of file diff --git a/Makefile b/Makefile index 81d815741..1ea61fae7 100644 --- a/Makefile +++ b/Makefile @@ -315,3 +315,16 @@ catalog-build: opm ## Build a catalog image. .PHONY: catalog-push catalog-push: ## Push a catalog image. $(MAKE) docker-push IMG=$(CATALOG_IMG) + +# E2E tests from kuttl +kuttl-test: install-kuttl docker-build + ./bin/kubectl-kuttl test --kind-config=${KUTTL_KIND_CFG} + + # Install kuttl for e2e tests. +install-kuttl: + mkdir -p ./bin ; \ + cd ./bin ; \ + OS="$$(uname | tr '[:upper:]' '[:lower:]')" ; \ + ARCH="$$(uname -m | sed -e 's/\(arm\)\(64\)\?.*/\1\2/' -e 's/aarch64$$/arm64/')" ; \ + curl -LO https://github.com/kudobuilder/kuttl/releases/download/v0.11.1/kuttl_0.11.1_$${OS}_$${ARCH}.tar.gz ; \ + tar -zxvf kuttl_0.11.1_$${OS}_$${ARCH}.tar.gz ; diff --git a/kuttl-test.yaml b/kuttl-test.yaml new file mode 100644 index 000000000..f27def0e1 --- /dev/null +++ b/kuttl-test.yaml @@ -0,0 +1,13 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestSuite +testDirs: +- ./test/kuttl + +startKIND: true +kindNodeCache: true +# we are using a k8s 1.21 as default. Other versions can be run using the --kind-config flag. +kindConfig: test/kuttl/config/kind/w3k1.21.yaml +timeout: 1400 +namespace: k8ssandra-operator +kindContainers: +- k8ssandra/k8ssandra-operator:latest \ No newline at end of file diff --git a/test/kuttl/config/kind/w1k1.20.yaml b/test/kuttl/config/kind/w1k1.20.yaml new file mode 100644 index 000000000..034f66c12 --- /dev/null +++ b/test/kuttl/config/kind/w1k1.20.yaml @@ -0,0 +1,9 @@ +kind: Cluster +apiVersion: kind.x-k8s.io/v1alpha4 +networking: + apiServerPort: 45451 +nodes: +- role: control-plane + image: kindest/node:v1.20.7 +- role: worker + image: kindest/node:v1.20.7 \ No newline at end of file diff --git a/test/kuttl/config/kind/w1k1.21.yaml b/test/kuttl/config/kind/w1k1.21.yaml new file mode 100644 index 000000000..27a38cb30 --- /dev/null +++ b/test/kuttl/config/kind/w1k1.21.yaml @@ -0,0 +1,9 @@ +kind: Cluster +apiVersion: kind.x-k8s.io/v1alpha4 +networking: + apiServerPort: 45451 +nodes: +- role: control-plane + image: kindest/node:v1.21.2 +- role: worker + image: kindest/node:v1.21.2 \ No newline at end of file diff --git a/test/kuttl/config/kind/w1k1.22.yaml b/test/kuttl/config/kind/w1k1.22.yaml new file mode 100644 index 000000000..c611a2b96 --- /dev/null +++ b/test/kuttl/config/kind/w1k1.22.yaml @@ -0,0 +1,9 @@ +kind: Cluster +apiVersion: kind.x-k8s.io/v1alpha4 +networking: + apiServerPort: 45451 +nodes: +- role: control-plane + image: kindest/node:v1.22.2 +- role: worker + image: kindest/node:v1.22.2 \ No newline at end of file diff --git a/test/kuttl/config/kind/w3k1.21.yaml b/test/kuttl/config/kind/w3k1.21.yaml new file mode 100644 index 000000000..0d81a9ac5 --- /dev/null +++ b/test/kuttl/config/kind/w3k1.21.yaml @@ -0,0 +1,12 @@ +kind: Cluster +apiVersion: kind.x-k8s.io/v1alpha4 +networking: + apiServerPort: 45451 +nodes: +- role: control-plane +- role: worker + image: kindest/node:v1.21.2 +- role: worker + image: kindest/node:v1.21.2 +- role: worker + image: kindest/node:v1.21.2 diff --git a/test/kuttl/test-cassDC-deploys/00-assert.yaml b/test/kuttl/test-cassDC-deploys/00-assert.yaml new file mode 100644 index 000000000..e14b8f28b --- /dev/null +++ b/test/kuttl/test-cassDC-deploys/00-assert.yaml @@ -0,0 +1,45 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: cert-manager-cainjector + namespace: "cert-manager" + labels: + app: cainjector + app.kubernetes.io/name: cainjector +status: + conditions: + - status: "True" + type: Available + - status: "True" + type: Progressing +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: cert-manager + namespace: "cert-manager" + labels: + app: cert-manager + app.kubernetes.io/name: cert-manager + app.kubernetes.io/instance: cert-manager +status: + conditions: + - status: "True" + type: Available + - status: "True" + type: Progressing +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: cert-manager-webhook + namespace: "cert-manager" + labels: + app: webhook + app.kubernetes.io/name: webhook +status: + conditions: + - status: "True" + type: Available + - status: "True" + type: Progressing \ No newline at end of file diff --git a/test/kuttl/test-cassDC-deploys/00-cert-manager.yaml b/test/kuttl/test-cassDC-deploys/00-cert-manager.yaml new file mode 100644 index 000000000..d8ebb5cd7 --- /dev/null +++ b/test/kuttl/test-cassDC-deploys/00-cert-manager.yaml @@ -0,0 +1,5 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +commands: +- command: kubectl apply -f https://github.com/jetstack/cert-manager/releases/download/v1.6.0/cert-manager.yaml + ignoreFailure: false diff --git a/test/kuttl/test-cassDC-deploys/01-assert.yaml b/test/kuttl/test-cassDC-deploys/01-assert.yaml new file mode 100644 index 000000000..4abd7e69d --- /dev/null +++ b/test/kuttl/test-cassDC-deploys/01-assert.yaml @@ -0,0 +1,22 @@ +# apiVersion: apiextensions.k8s.io/v1 +# kind: CustomResourceDefinition +# name: k8ssandraclusters.k8ssandra.io +# spec: +# group: k8ssandra.io +# names: +# kind: K8ssandraCluster +# listKind: K8ssandraClusterList +# plural: k8ssandraclusters +# singular: k8ssandracluster +# --- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: k8ssandra-operator + namespace: k8ssandra-operator +status: + conditions: + - status: "True" + type: Available + - status: "True" + type: Progressing \ No newline at end of file diff --git a/test/kuttl/test-cassDC-deploys/01-k8ssandra-operator.yaml b/test/kuttl/test-cassDC-deploys/01-k8ssandra-operator.yaml new file mode 100644 index 000000000..fcb135a1b --- /dev/null +++ b/test/kuttl/test-cassDC-deploys/01-k8ssandra-operator.yaml @@ -0,0 +1,6 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +commands: +- script: | + ../../../bin/kustomize build ../../../config/deployments/control-plane | kubectl apply -n $NAMESPACE --server-side --force-conflicts -f - + ignoreFailure: false diff --git a/test/kuttl/test-cassDC-deploys/02-assert.yaml b/test/kuttl/test-cassDC-deploys/02-assert.yaml new file mode 100644 index 000000000..52a6a6748 --- /dev/null +++ b/test/kuttl/test-cassDC-deploys/02-assert.yaml @@ -0,0 +1,7 @@ +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: demo-dc1-default-sts + namespace: k8ssandra-operator +status: + readyReplicas: 3 \ No newline at end of file diff --git a/test/kuttl/test-cassDC-deploys/02-k8ssandra-cluster.yaml b/test/kuttl/test-cassDC-deploys/02-k8ssandra-cluster.yaml new file mode 100644 index 000000000..7257961af --- /dev/null +++ b/test/kuttl/test-cassDC-deploys/02-k8ssandra-cluster.yaml @@ -0,0 +1,5 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +commands: +- command: kubectl apply -n $NAMESPACE --server-side --force-conflicts -f ../../testdata/samples/k8ssandra-multi-kind.yaml + ignoreFailure: false \ No newline at end of file From a03580fb76609e48beb0a5426660913ea4d7ec27 Mon Sep 17 00:00:00 2001 From: Miles-Garnsey Date: Thu, 2 Dec 2021 18:16:08 +1100 Subject: [PATCH 2/2] Add GHA workflow for kuttl tests. --- .github/workflows/kuttl_tests.yaml | 90 ++++++++++++++++++++++++++++++ 1 file changed, 90 insertions(+) create mode 100644 .github/workflows/kuttl_tests.yaml diff --git a/.github/workflows/kuttl_tests.yaml b/.github/workflows/kuttl_tests.yaml new file mode 100644 index 000000000..392125486 --- /dev/null +++ b/.github/workflows/kuttl_tests.yaml @@ -0,0 +1,90 @@ +name: kuttl tests + +on: + push: + branches: + - main + paths-ignore: + - 'docs/**' + - 'CHANGELOG/**' + pull_request: + branches: [ main ] + paths-ignore: + - 'docs/**' + - 'CHANGELOG/**' +jobs: + build_image: + name: Build image + runs-on: ubuntu-latest + steps: + - name: Check out code into the Go module directory + uses: actions/checkout@v2 + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v1 + - name: Cache Docker layers + uses: actions/cache@v2 + with: + path: /tmp/.buildx-cache + key: ${{ runner.os }}-buildx-${{ github.sha }} + restore-keys: | + ${{ runner.os }}-buildx- uses: docker/setup-buildx-action@v1 + - name: Set git parsed values + id: vars + run: | + echo ::set-output name=sha_short::$(git rev-parse --short=8 ${{ github.sha }}) + - name: Build Docker image + id: docker_build + uses: docker/build-push-action@v2 + with: + file: Dockerfile + context: . + push: false + tags: k8ssandra/k8ssandra-operator:${{ steps.vars.outputs.sha_short }}, k8ssandra/k8ssandra-operator:latest + platforms: linux/amd64 + cache-from: type=local,src=/tmp/.buildx-cache + cache-to: type=local,dest=/tmp/.buildx-cache + outputs: type=docker,dest=/tmp/k8ssandra-k8ssandra-operator.tar + - name: Upload k8ssandra-operator image + uses: actions/upload-artifact@v2 + with: + name: k8ssandra-operator + path: /tmp/k8ssandra-k8ssandra-operator.tar + run_kuttl_test: + runs-on: ubuntu-latest + needs: [build_image] + name: Run kuttl tests + strategy: + matrix: + k8s_version: ["1.21"] + steps: + - name: Check out code into the Go module directory + uses: actions/checkout@v2 + - name: Cache Docker layers + uses: actions/cache@v2 + with: + path: /tmp/.buildx-cache + key: ${{ runner.os }}-buildx-${{ github.sha }} + restore-keys: | + ${{ runner.os }}-buildx- uses: docker/setup-buildx-action@v1 + - name: Set git parsed values + id: vars + run: | + echo ::set-output name=sha_short::$(git rev-parse --short=8 ${{ github.sha }}) + - name: Kind kube-proxy issue workaround + run: sudo sysctl net/netfilter/nf_conntrack_max=524288 + - name: Download k8ssandra-operator image + uses: actions/download-artifact@v2 + with: + name: k8ssandra-operator + path: /tmp + - name: Load Docker images + run: | + docker load --input /tmp/k8ssandra-k8ssandra-operator.tar + - name: install kustomize + run: | + make kustomize + - name: Run kuttl-test + run: | + # We are running tests against k8s 1.20 - 1.22 currently. + # Additional versions must be added in kind config files under ./test/config/kind + make KUTTL_KIND_CFG="./test/kuttl/config/kind/w3k${{ matrix.k8s_version }}.yaml" kuttl-test \ No newline at end of file