Merge pull request #540 from cyberark/e2e-ci-improvements

Run golang e2e tests in CI

CONTRIBUTING.md

@@ -100,7 +100,7 @@ You can also view Conjur Enterprise / Conjur Open Source pod logs by running `ku
 
 #### Run integration tests against local development environment
 
-Assuming you have a running local cluster per the previous section and a valid kubeconfig, you can run the Golang-based integration tests by simply running `./bin/test_e2e` from the project root folder. This will build a test container with the kubeconfig mounted, and execute the tests against the local cluster.
+Assuming you have a running local cluster per the previous section and a valid kubeconfig, you can run the Golang-based integration tests by simply running `go test -timeout 0 -v ./e2e --tags=e2e` from the project root folder. This will execute the tests against the local cluster.
 
 NOTE: This assumes that the cluster is running with the default values in `bootstrap.env` for local DEV clusters.
 

Dockerfile.e2e

@@ -1,19 +1,59 @@
-FROM golang:1.20-alpine
-MAINTAINER CyberArk Software Ltd.
-LABEL id="secrets-provider-for-k8s-e2e-test-runner"
+FROM google/cloud-sdk:437.0.0-slim
 
-WORKDIR /secrets-provider-for-k8s
+RUN mkdir -p /src
+WORKDIR /src
+
+# Install Docker client
+RUN apt-get update -y && \
+    apt-get install -y apt-transport-https ca-certificates curl gnupg2 software-properties-common wget google-cloud-sdk-gke-gcloud-auth-plugin && \
+    curl -fsSL https://download.docker.com/linux/$(. /etc/os-release; echo "$ID")/gpg | apt-key add - && \
+    add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/$(. /etc/os-release; echo "$ID") $(lsb_release -cs) stable" && \
+    apt-get update && \
+    apt-get install -y docker-ce && \
+    rm -rf /var/lib/apt/lists/*
+
+# Install kubectl CLI
+ARG KUBECTL_CLI_URL
+RUN wget -O /usr/local/bin/kubectl ${KUBECTL_CLI_URL:-https://storage.googleapis.com/kubernetes-release/release/v1.7.6/bin/linux/amd64/kubectl} && \
+    chmod +x /usr/local/bin/kubectl
+
+# Install OpenShift oc CLI
+ARG OPENSHIFT_CLI_URL
+RUN mkdir -p ocbin && \
+    wget -O oc.tar.gz ${OPENSHIFT_CLI_URL:-https://mirror.openshift.com/pub/openshift-v4/x86_64/clients/ocp/stable/openshift-client-linux.tar.gz} && \
+    tar xvf oc.tar.gz -C ocbin && \
+    cp "$(find ./ocbin -name 'oc' -type f | tail -1)"  /usr/local/bin/oc  && \
+    rm -rf ocbin oc.tar.gz
+
+# Install Helm
+RUN curl -fsSL -o get_helm.sh https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 && \
+    chmod 700 get_helm.sh && \
+    ./get_helm.sh
 
-RUN apk add -u curl \
-               gcc \
-               git \
-               mercurial \
-               musl-dev
+# Adds ability to perform mathematical operations with floats for testing
+RUN apt-get update -y && \
+    apt-get install -y bc
+
+# Add Golang-based E2E tests
+# Install necessary packages to download and install Golang
+RUN apt-get update && \
+    apt-get install -y --no-install-recommends \
+    ca-certificates \
+    curl \
+    git
+
+# Download and install Golang
+ENV GOLANG_VERSION 1.20
+RUN curl -L https://golang.org/dl/go${GOLANG_VERSION}.linux-amd64.tar.gz | tar -C /usr/local -xz
+ENV PATH=$PATH:/usr/local/go/bin
+
+RUN mkdir -p /secrets-provider-for-k8s
+WORKDIR /secrets-provider-for-k8s
 
-COPY ./go.mod ./go.sum /secrets-provider-for-k8s/
+COPY go.mod go.sum /secrets-provider-for-k8s/
 
 RUN go mod download
 
 COPY . .
 
-ENTRYPOINT [ "go", "test", "-v", "-timeout", "3m", "-tags", "e2e", "./e2e/..." ]
+WORKDIR /src

bin/start

@@ -56,6 +56,7 @@ while true ; do
     --oldest ) SUMMON_ENV=oldest ; shift ;;
     --current ) SUMMON_ENV=current ; shift ;;
     --next ) SUMMON_ENV=next ; shift ;;
+    --template=* ) TEMPLATE_OVERRIDE="${1#*=}" ; export TEMPLATE_OVERRIDE; shift ;;
     -h | --help ) print_help ; shift ;;
      * ) if [ -z "$1" ]; then break; else echo "$1 is not a valid option"; exit 1; fi;;
   esac
@@ -70,10 +71,16 @@ export RUN_IN_DOCKER
 export CONJUR_DEPLOYMENT
 export DEV
 export SUMMON_ENV
+export TEMPLATE_OVERRIDE
+
+echo $TEMPLATE_OVERRIDE
 
 # summon environment variable
 export CONJUR_MAJOR_VERSION=5
 
+# make sure we are in the project root
+cd $(git rev-parse --show-toplevel)
+
 source bootstrap.env
 
 pushd deploy

bootstrap.env

@@ -19,6 +19,10 @@ export APP_NAMESPACE_NAME=app-$UNIQUE_TEST_ID
 # export AUTHENTICATOR_ID=authn-dev-env
 # export APP_NAMESPACE_NAME=local-secrets-provider
 # export CONJUR_NAMESPACE_NAME=local-conjur
+# export RUN_IN_DOCKER=false
+# export CONJUR_DEPLOYMENT=dap
+# export SUMMON_ENV=gke
+# export RELOAD_ENV=false
 # export STOP_RUNNING_ENV=true
 # export CONJUR_ACCOUNT=cucumber
 # export CONJUR_LOG_LEVEL=debug

deploy/redeploy.sh

@@ -0,0 +1,60 @@
+#!/bin/bash
+set -euxo pipefail
+
+pushd $(dirname "${BASH_SOURCE[0]}")
+
+# If DEV env variable isn't set, source the bootstrap file
+if [[ -z "${DEV:-}" ]]; then
+  source "../bootstrap.env"
+fi
+
+source "../deploy/utils.sh"
+
+set_config_directory_path
+
+../deploy/teardown_resources.sh
+
+if [ "${DEV}" = "false" ]; then
+  announce "Creating image pull secret."
+  if [[ "${PLATFORM}" == "kubernetes" ]]; then
+   $cli_with_timeout delete --ignore-not-found secret $IMAGE_PULL_SECRET
+
+   $cli_with_timeout create secret docker-registry dockerpullsecret \
+    --docker-server="${PULL_DOCKER_REGISTRY_URL}" \
+    --docker-username=_ \
+    --docker-password=_ \
+    --docker-email=_
+  elif [[ "$PLATFORM" == "openshift" ]]; then
+    $cli_with_timeout delete --ignore-not-found secrets dockerpullsecret
+
+    $cli_with_timeout create secret docker-registry $IMAGE_PULL_SECRET \
+      --docker-server="${PULL_DOCKER_REGISTRY_PATH}" \
+      --docker-username=_ \
+      --docker-password=$($cli_with_timeout whoami -t) \
+      --docker-email=_
+
+    $cli_with_timeout secrets link serviceaccount/default dockerpullsecret --for=pull
+  fi
+fi
+
+echo "Create secret k8s-secret"
+$cli_with_timeout create -f "$CONFIG_DIR/k8s-secret.yml"
+
+wait_for_it 600  "$CONFIG_DIR/secrets-access-role.sh.yml | $cli_without_timeout apply -f -"
+
+wait_for_it 600  "$CONFIG_DIR/secrets-access-role-binding.sh.yml | $cli_without_timeout apply -f -"
+
+selector="role=follower"
+cert_location="/opt/conjur/etc/ssl/conjur.pem"
+if [ "$CONJUR_DEPLOYMENT" = "oss" ]; then
+    selector="app=conjur-cli"
+    cert_location="/root/conjur-server.pem"
+fi
+conjur_pod_name="$(get_pod_name "$CONJUR_NAMESPACE_NAME" "$selector")"
+ssl_cert=$($cli_with_timeout "exec ${conjur_pod_name} --namespace $CONJUR_NAMESPACE_NAME -- cat $cert_location")
+
+export CONJUR_SSL_CERTIFICATE=$ssl_cert
+
+deploy_env
+
+popd

deploy/test/test_cases/run_tests.sh

@@ -22,6 +22,20 @@ set +a
 
 times=1
 
+announce "Preparing to run E2E tests"
+
+# Uncomment for Golang-based tests
+./test_case_setup.sh
+create_secret_access_role
+create_secret_access_role_binding
+deploy_env
+pushd /secrets-provider-for-k8s
+go test -v -tags e2e -timeout 0 ./e2e/...
+popd
+
+../../teardown_resources.sh
+
+# Uncomment for Bash tests
 for c in {1..$times}
 do
   for filename in ./$TEST_NAME_PREFIX*.sh; do

deploy/test/test_in_docker.sh

@@ -24,11 +24,13 @@ main() {
 }
 
 buildTestRunnerImage() {
+  pushd ..
   docker build --tag $TEST_RUNNER_IMAGE:$CONJUR_NAMESPACE_NAME \
-    --file test/Dockerfile \
+    --file Dockerfile.e2e \
     --build-arg OPENSHIFT_CLI_URL=$OPENSHIFT_CLI_URL \
     --build-arg KUBECTL_CLI_URL=$KUBECTL_CLI_URL \
     .
+  popd
 }
 
 deployConjur() {

deploy/utils.sh

@@ -128,8 +128,8 @@ runDockerCommand() {
       -v /var/run/docker.sock:/var/run/docker.sock \
       -v ~/.config:/root/.config \
       -v "$PWD/../helm":/helm \
-      -v "$PWD":/src \
-      -w /src \
+      -v "$PWD/..":/src \
+      -w /src/deploy \
       $TEST_RUNNER_IMAGE:$CONJUR_NAMESPACE_NAME \
       bash -c "
         ./platform_login.sh
@@ -168,10 +168,9 @@ runDockerCommand() {
       -e SUMMON_ENV \
       -e IMAGE_PULL_SECRET \
       -v /var/run/docker.sock:/var/run/docker.sock \
-      -v ~/.config:/root/.config \
       -v "$PWD/../helm":/helm \
-      -v "$PWD":/src \
-      -w /src \
+      -v "$PWD/..":/src \
+      -w /src/deploy \
       $TEST_RUNNER_IMAGE:$CONJUR_NAMESPACE_NAME \
       bash -c "
         ./platform_login.sh

e2e/consts.go

@@ -4,11 +4,11 @@
 package e2e
 
 const (
-	// Namespaces and container names
-	SecretsProviderNamespace = "local-secrets-provider"
-	ConjurNamespace          = "local-conjur"
-	TestAppContainer         = "test-app"
-	CLIContainer             = "conjur-cli"
+	// Container names
+	TestAppContainer             = "test-app"
+	CLIContainer                 = "conjur-cli"
+	SecretsProviderLabelSelector = "app=test-env"
+	CLILabelSelector             = "app=conjur-cli"
 
 	// Available templates:
 	K8sTemplate         = "secrets-provider-init-container"