From 9314b7731ed35ca1a4aa89f8105d15ebdb873a5b Mon Sep 17 00:00:00 2001 From: Sushanta Das Date: Wed, 18 Sep 2024 15:34:53 +0530 Subject: [PATCH] tests for inidividual tasks/stepactions --- hack/common.sh | 266 ++++++++++++++++++++++++++++++ hack/run-test.sh | 52 ++++++ hack/test-tasks.sh | 74 +++++++++ task/git-clone/0.1/tests/run.yaml | 30 ++++ 4 files changed, 422 insertions(+) create mode 100644 hack/common.sh create mode 100755 hack/run-test.sh create mode 100755 hack/test-tasks.sh create mode 100644 task/git-clone/0.1/tests/run.yaml diff --git a/hack/common.sh b/hack/common.sh new file mode 100644 index 0000000000..855322dbe5 --- /dev/null +++ b/hack/common.sh @@ -0,0 +1,266 @@ + +#!/bin/bash + +# Define a custom kubectl path if you like +KUBECTL_CMD=${KUBECTL_CMD:-kubectl} + +# Checks whether the given function exists. +function function_exists() { + [[ "$(type -t $1)" == "function" ]] +} + +function detect_changed_e2e_test() { + # detect for changes in hack dir + # TODO: change PULL_BASE_SHA and PULL_PULL_SHA with values applicable for PAC/GA + git --no-pager diff --name-only "${PULL_BASE_SHA}".."${PULL_PULL_SHA}"|grep "^test/[^/]*" +} + +function detect_new_changed_resources() { + # TODO: change PULL_BASE_SHA and PULL_PULL_SHA with values applicable for PAC/GA + # detect for changes in tests dir of the task + git --no-pager diff --name-only "${PULL_BASE_SHA}".."${PULL_PULL_SHA}"|grep 'task/[^\/]*/[^\/]*/tests/[^/]*'|xargs -I {} dirname {}|sed 's/\(tests\).*/\1/g' + # detect for changes in the task manifest + git --no-pager diff --name-only "${PULL_BASE_SHA}".."${PULL_PULL_SHA}"|grep 'task/[^\/]*/[^\/]*/*[^/]*.yaml'|xargs -I {} dirname {}|awk '{print $1"/tests"}' + # detect for changes in tests dir of the stepaction + git --no-pager diff --name-only "${PULL_BASE_SHA}".."${PULL_PULL_SHA}"|grep 'stepactions/[^\/]*/[^\/]*/tests/[^/]*'|xargs -I {} dirname {}|sed 's/\(tests\).*/\1/g' + # detect for changes in the stepaction manifest + git --no-pager diff --name-only "${PULL_BASE_SHA}".."${PULL_PULL_SHA}"|grep 'stepactions/[^\/]*/[^\/]*/*[^/]*.yaml'|xargs -I {} dirname {}|awk '{print $1"/tests"}' +} + +# Signal (as return code and in the logs) that all E2E tests passed. +function success() { + set_test_return_code 0 + echo "**************************************" + echo "*** E2E TESTS PASSED ***" + echo "**************************************" + exit 0 +} + +function show_failure() { + local testname=$1 tns=$2 + + echo "FAILED: ${testname} task has failed to comeback properly" ; + ${KUBECTL_CMD} api-resources + echo "Namespace: ${tns}" + echo "--- TaskRun Dump" + ${KUBECTL_CMD} get --ignore-not-found=true -n ${tns} taskrun -o yaml + echo "--- Task Dump" + ${KUBECTL_CMD} get --ignore-not-found=true -n ${tns} task -o yaml + echo "--- PipelineRun Dump" + ${KUBECTL_CMD} get --ignore-not-found=true -n ${tns} pipelinerun -o yaml + echo "--- Pipeline Dump" + ${KUBECTL_CMD} get --ignore-not-found=true -n ${tns} pipeline -o yaml + echo "--- StepAction Dump" + ${KUBECTL_CMD} get --ignore-not-found=true -n ${tns} stepaction -o yaml + echo "--- Container Logs" + for pod in $(${KUBECTL_CMD} get pod -o name -n ${tns}); do + echo "----POD_NAME: ${pod}---" + ${KUBECTL_CMD} logs --all-containers -n ${tns} ${pod} || true + done + exit 1 + +} + +function test_yaml_can_install() { + # Validate that all the StepAction/Task CRDs in this repo are valid by creating them in a NS. + ns="test-ns" + all_tasks="$*" + ${KUBECTL_CMD} create ns "${ns}" || true + local runtest + for runtest in ${all_tasks}; do + # remove task/ or stepaction/ from beginning + local runtestdir=${runtest#*/} + # remove /0.1/tests from end + local testname=${runtestdir%%/*} + runtest=${runtest//tests} + + # in case a stepaction/task is being removed then it's directory + # doesn't exists, so skip the test for YAML + [ ! -d "${runtest%%/*}/${testname}" ] && continue + + runtest="${runtest}${testname}.yaml" + skipit= + for ignore in ${TEST_YAML_IGNORES};do + [[ ${ignore} == "${testname}" ]] && skipit=True + done + + # don't test the stepactions/tasks which are deprecated + cat ${runtest} | grep 'tekton.dev/deprecated: \"true\"' && skipit=True + + # In case if PLATFORM env variable is specified, do the tests only for matching stepactions/tasks + [[ -n ${PLATFORM} ]] && [[ $(grep "tekton.dev/platforms" ${runtest} 2>/dev/null) != *"${PLATFORM}"* ]] && skipit=True + + [[ -n ${skipit} ]] && continue + echo "Checking ${testname}" + ${KUBECTL_CMD} -n ${ns} apply -f <(sed "s/namespace:.*/namespace: ${ns}/" "${runtest}") + done +} + +function test_resource_creation() { + local runtest + declare -A resource_to_wait_for + + for runtest in $@;do + # remove task/ from beginning + local runtestdir=${runtest#*/} + # remove //tests from end + local testname=${runtestdir%%/*} + local version=$(basename $(basename $(dirname $runtest))) + # check version is in given format + [[ ${version} =~ ^[0-9]+\.[0-9]+$ ]] || { echo "ERROR: version of the task is not set properly"; exit 1;} + # replace . with - in version as not supported in namespace name + version="$( echo $version | tr '.' '-' )" + + #local tns="${testname}-${version}" + local tns=$2 + local skipit= + + for ignore in ${TEST_TASKRUN_IGNORES};do + [[ ${ignore} == ${testname} ]] && skipit=True + done + + # remove /tests from end + local resourcedir=${runtest%/*} + + # check whether tests folder exists or not inside task dir + # if not then run the tests for next task (if any) + [[ ! -d $runtest ]] && skipit=True + + ls ${resourcedir}/*.yaml 2>/dev/null >/dev/null || skipit=True + + cat ${resourcedir}/*.yaml | grep 'tekton.dev/deprecated: \"true\"' && skipit=True + + # In case if PLATFORM env variable is specified, do the tests only for matching tasks + [[ -n ${PLATFORM} ]] && [[ $(grep "tekton.dev/platforms" ${resourcedir}/*.yaml 2>/dev/null) != *"${PLATFORM}"* ]] && skipit=True + + [[ -n ${skipit} ]] && continue + + # In case of rerun it's fine to ignore this error + #${KUBECTL_CMD} create namespace ${tns} >/dev/null 2>/dev/null || : + + # Install the task itself first. We can only have one YAML file + yaml=$(printf ${resourcedir}/*.yaml) + started=$(date '+%Hh%M:%S') + echo "${started} STARTING: ${testname}/${version}" + + # dry-run this YAML to validate and also get formatting side-effects. + ${KUBECTL_CMD} -n ${tns} create -f ${yaml} --dry-run=client -o yaml >${TMPF} + [[ -f ${resourcedir}/tests/pre-apply-task-hook.sh ]] && source ${resourcedir}/tests/pre-apply-task-hook.sh + function_exists pre-apply-task-hook && pre-apply-task-hook + + [[ -d ${resourcedir}/tests/fixtures ]] && { + # Create a configmap to make every file under fixture + # available to the sidecar. + ${KUBECTL_CMD} -n ${tns} create configmap fixtures --from-file=${resourcedir}/tests/fixtures + # The task may already have a volumes section and in that case, we + # need to append fixtures volume. + if [[ -n $(grep "^[[:space:]]\{2,\}volumes:$" ${TMPF}) ]]; then + sed -i "s/^[[:space:]]\{2,\}volumes:$/ volumes:\\n - name: fixtures\\n configMap:\\n name: fixtures/g" ${TMPF} + else + cat <>${TMPF} + volumes: + - name: fixtures + configMap: + name: fixtures +EOF + fi + cat <>${TMPF} + sidecars: + - image: gcr.io/tekton-releases/dogfooding/go-rest-api-test:latest + name: go-rest-api + volumeMounts: + - name: fixtures + mountPath: /fixtures + env: + - name: CONFIG + value: | +$(cat ${resourcedir}/tests/fixtures/*.yaml|sed 's/^/ /') +EOF + } + + # Make sure we have deleted the content, this is in case of rerun + # and namespace hasn't been cleaned up or there is some Cluster* + # stuff, which really should not be allowed. + ${KUBECTL_CMD} -n ${tns} delete -f ${TMPF} >/dev/null 2>/dev/null || true + ${KUBECTL_CMD} -n ${tns} create -f ${TMPF} + + # Install resource and run + for yaml in ${runtest}/*.yaml;do + cp ${yaml} ${TMPF} + #[[ -f ${resourcedir}/tests/pre-apply-taskrun-hook.sh ]] && source ${resourcedir}/tests/pre-apply-taskrun-hook.sh + #function_exists pre-apply-taskrun-hook && pre-apply-taskrun-hook + + # Make sure we have deleted the content, this is in case of rerun + # and namespace hasn't been cleaned up or there is some Cluster* + # stuff, which really should not be allowed. + ${KUBECTL_CMD} -n ${tns} delete -f ${TMPF} >/dev/null 2>/dev/null || true + ${KUBECTL_CMD} -n ${tns} create -f ${TMPF} + done + + resource_to_wait_for["$testname/${version}"]="${tns}|$started" + + done + + # I would refactor this to a function but bash limitation is too great, really need a rewrite the sooner + # the uglness to pass a hashmap to a function https://stackoverflow.com/a/17557904/145125 + local cnt=0 + local all_status='' + local reason='' + local maxloop=60 # 10 minutes max + + set +x + while true;do + # If we have timed out then show failures of what's remaining in + # resource_to_wait_for we assume only first one fails this + [[ ${cnt} == "${maxloop}" ]] && { + for testname in "${!resource_to_wait_for[@]}";do + target_ns=${resource_to_wait_for[$testname]} + show_failure "${testname}" "${target_ns}" + done + } + [[ -z ${resource_to_wait_for[*]} ]] && { + break + } + + for testname in "${!resource_to_wait_for[@]}";do + target_ns=${resource_to_wait_for[$testname]%|*} + started=${resource_to_wait_for[$testname]#*|} + # sometimes we don't get all_status and reason in one go so + # wait until we get the reason and all_status for 5 iterations + for tektontype in pipelinerun taskrun;do + for _ in {1..10}; do + all_status=$(${KUBECTL_CMD} get -n ${target_ns} ${tektontype} --output=jsonpath='{.items[*].status.conditions[*].status}') + reason=$(${KUBECTL_CMD} get -n ${target_ns} ${tektontype} --output=jsonpath='{.items[*].status.conditions[*].reason}') + [[ ! -z ${all_status} ]] && [[ ! -z ${reason} ]] && break + sleep 1 + done + # No need to check taskrun if pipelinerun has been set + [[ ! -z ${all_status} ]] && [[ ! -z ${reason} ]] && break + done + + if [[ -z ${all_status} || -z ${reason} ]];then + echo "Could not find a created taskrun or pipelinerun in ${target_ns}" + fi + + breakit=True + for status in ${all_status};do + [[ ${status} == *ERROR || ${reason} == *Fail* || ${reason} == Couldnt* ]] && show_failure ${testname} ${target_ns} + + if [[ ${status} != True ]];then + breakit= + fi + done + + if [[ ${breakit} == True ]];then + unset resource_to_wait_for[$testname] + [[ -z ${CATALOG_TEST_SKIP_CLEANUP} ]] && ${KUBECTL_CMD} delete ns ${target_ns} >/dev/null + echo "${started}::$(date '+%Hh%M:%S') SUCCESS: ${testname} testrun has successfully executed" ; + fi + + done + + sleep 10 + cnt=$((cnt+1)) + done + set -x +} \ No newline at end of file diff --git a/hack/run-test.sh b/hack/run-test.sh new file mode 100755 index 0000000000..9a0da0de33 --- /dev/null +++ b/hack/run-test.sh @@ -0,0 +1,52 @@ +#!/bin/bash + +set -ex + +cd $(git rev-parse --show-toplevel) +source hack/common.sh + +if [[ -z ${@} || ${1} == "-h" ]];then + cat </dev/null 2>/dev/null && ${KUBECTL_CMD} delete ns ${RESOURCE}-${NAME}-${VERSION//./-} +# TODO: Need to link appstudio-pipeline SA with the TEST_NS namespace if not already + +if [[ ! -d ${resourcedir}/tests ]];then + echo "No 'tests' directory is located in ${resourcedir}" + exit 1 +fi + + + +test_resource_creation ${RESOURCE}/${NAME}/${VERSION}/tests $TEST_NS \ No newline at end of file diff --git a/hack/test-tasks.sh b/hack/test-tasks.sh new file mode 100755 index 0000000000..36d6c92ca4 --- /dev/null +++ b/hack/test-tasks.sh @@ -0,0 +1,74 @@ +#!/bin/bash + +source $(dirname $0)/common.sh + +TMPF=$(mktemp /tmp/.mm.XXXXXX) +clean() { rm -f ${TMPF}; } +trap clean EXIT + +# Configure the number of parallel tests running at the same time, start from 0 +MAX_NUMBERS_OF_PARALLEL_TASKS=7 # => 8 + +# You can ignore some yaml tests by providing the TEST_YAML_IGNORES variable +# with the test name separated by a space, for example: +# +# TEST_YAML_IGNORES="kaniko s2i" +# +# will ignore the kaniko and s2i tests, +# +TEST_YAML_IGNORES=${TEST_YAML_IGNORES:-""} + +# Allow ignoring some yaml tests, space separated, should be the basename of the +# test for example "s2i" +TEST_TASKRUN_IGNORES=${TEST_TASKRUN_IGNORES:-""} + +# Define this variable if you want to run all tests and not just the modified one. +TEST_RUN_ALL_TESTS=${TEST_RUN_ALL_TESTS:-""} + +#DEFINE A TEST NS +TEST_NS=${TEST_NS:-"konflux-ci"} + +set -ex +set -o pipefail + +all_tests=$(echo task/*/*/tests) +all_stepactions=$(echo stepactions/*/*/tests) + +[[ -z ${TEST_RUN_ALL_TESTS} ]] && [[ ! -z $(detect_changed_e2e_test) ]] && TEST_RUN_ALL_TESTS=1 + +if [[ -z ${TEST_RUN_ALL_TESTS} ]];then + all_tests=$(detect_new_changed_resources|sort -u || true) + [[ -z ${all_tests} ]] && { + echo "No tests has been detected in this PR. exiting." + success + } +fi + +#test_yaml_can_install "${all_stepactions}" +test_yaml_can_install "${all_tests}" + +function test_resources { + local cnt=0 + local resource_to_tests="" + + for runtest in $@;do + resource_to_tests="${resource_to_tests} ${runtest}" + if [[ ${cnt} == "${MAX_NUMBERS_OF_PARALLEL_TASKS}" ]];then + test_resource_creation "${resource_to_tests}" + cnt=0 + resource_to_tests="" + continue + fi + cnt=$((cnt+1)) + done + + # in case if there are some remaining resources + if [[ -n ${resource_to_tests} ]];then + test_resource_creation "${resource_to_tests}" + fi +} + +#test_resources "${all_stepactions}" +test_resources "${all_tests}" + +success \ No newline at end of file diff --git a/task/git-clone/0.1/tests/run.yaml b/task/git-clone/0.1/tests/run.yaml new file mode 100644 index 0000000000..817009d9b2 --- /dev/null +++ b/task/git-clone/0.1/tests/run.yaml @@ -0,0 +1,30 @@ +--- +apiVersion: tekton.dev/v1beta1 +kind: TaskRun +metadata: + name: git-clone-run-noargs +spec: + workspaces: + - name: output + emptyDir: {} + taskRef: + name: git-clone + params: + - name: url + value: https://github.com/kelseyhightower/nocode +--- +apiVersion: tekton.dev/v1beta1 +kind: TaskRun +metadata: + name: git-clone-run-tag +spec: + workspaces: + - name: output + emptyDir: {} + taskRef: + name: git-clone + params: + - name: url + value: https://github.com/kelseyhightower/nocode + - name: revision + value: 1.0.0 \ No newline at end of file