From dd4f0a968c4fc8ced45d79c0a29ab2cb3d09647a Mon Sep 17 00:00:00 2001 From: Petr Michalec Date: Mon, 4 Jun 2018 11:11:53 +0200 Subject: [PATCH] Add kitchen tests Change-Id: I81b03691a9ecab9052ea2ae6cbae30c0873709e3 --- .gitignore | 4 ++ .kitchen.yml | 55 ++++++++++++++++++ .travis.yml | 46 +++++++++++++++ Makefile | 6 ++ tests/run_tests.sh | 142 ++++++++++++++++++++++++++++++++++++++++++--- 5 files changed, 245 insertions(+), 8 deletions(-) create mode 100644 .kitchen.yml create mode 100644 .travis.yml diff --git a/.gitignore b/.gitignore index aa8e42a..43daaac 100644 --- a/.gitignore +++ b/.gitignore @@ -3,3 +3,7 @@ tests/build/ *.swp *.pyc .ropeproject +.kitchen/ +.kitchen.local.yml +.bundle +.vendor diff --git a/.kitchen.yml b/.kitchen.yml new file mode 100644 index 0000000..658dcc4 --- /dev/null +++ b/.kitchen.yml @@ -0,0 +1,55 @@ +--- +driver: + name: docker + hostname: freeipa.ci.local + #socket: tcp://127.0.0.1:2376 + use_sudo: false + + + +provisioner: + name: salt_solo + salt_install: bootstrap + salt_bootstrap_url: https://bootstrap.saltstack.com + salt_version: latest + require_chef: false + formula: freeipa + log_level: info + state_top: + base: + "*": + - freeipa + pillars: + top.sls: + base: + "*": + - freeipa + grains: + noservices: True + + +platforms: + - name: <%=ENV['PLATFORM'] || 'saltstack-centos-7-salt-stable' %> + driver_config: + image: <%=ENV['PLATFORM'] || 'epcim/salt:saltstack-centos-7-salt-stable'%> + platform: <%=ENV['PLATFORM_NAME'] || 'rhel'%> + + +verifier: + name: inspec + sudo: true + + +suites: + + - name: client + provisioner: + pillars-from-files: + freeipa.sls: tests/pillar/client.sls + + - name: server + provisioner: + pillars-from-files: + freeipa.sls: tests/pillar/server.sls + +# vim: ft=yaml sw=2 ts=2 sts=2 tw=125 diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 0000000..42ad5c9 --- /dev/null +++ b/.travis.yml @@ -0,0 +1,46 @@ +sudo: required +services: + - docker + +addons: + apt: + packages: + - apt-transport-https + +install: + - pip install PyYAML + - pip install virtualenv + - | + test -e Gemfile || cat < Gemfile + source 'https://rubygems.org' + gem 'rake' + gem 'test-kitchen' + gem 'kitchen-docker' + gem 'kitchen-inspec' + gem 'inspec' + gem 'kitchen-salt' #, :git => 'https://github.com/salt-formulas/kitchen-salt.git' + - bundle install + +env: + - PLATFORM=epcim/salt:saltstack-centos-7-salt-stable PLATFORM_NAME:centos + - PLATFORM=epcim/salt:saltstack-ubuntu-xenial-salt-stable PLATFORM_NAME:ubuntu + - PLATFORM=epcim/salt:saltstack-ubuntu-bionic-salt-stable + +before_script: + - set -o pipefail + - make test | tail + +script: + - test ! -e .kitchen.yml || bundle exec kitchen converge ${SUITE} || true + - test ! -e .kitchen.yml || bundle exec kitchen verify ${SUITE} -t tests/integration + +notifications: + webhooks: + urls: + - https://webhooks.gitter.im/e/6123573504759330786b + on_success: change # options: [always|never|change] default: always + on_failure: never # options: [always|never|change] default: always + on_start: never # options: [always|never|change] default: always + on_cancel: never # options: [always|never|change] default: always + on_error: never # options: [always|never|change] default: always + email: false diff --git a/Makefile b/Makefile index d166862..f9b4f47 100644 --- a/Makefile +++ b/Makefile @@ -34,6 +34,7 @@ all: @echo "make release-major - Generate new major release" @echo "make release-minor - Generate new minor release" @echo "make changelog - Show changes since last release" + @echo "make test-model-validate - Run salt jsonschema validation" install: # Formula @@ -41,6 +42,7 @@ install: cp -a $(FORMULANAME) $(DESTDIR)/$(SALTENVDIR)/ [ ! -d _modules ] || cp -a _modules $(DESTDIR)/$(SALTENVDIR)/ [ ! -d _states ] || cp -a _states $(DESTDIR)/$(SALTENVDIR)/ || true + [ ! -d _engines ] || cp -a _engines $(DESTDIR)/$(SALTENVDIR)/ || true [ ! -d _grains ] || cp -a _grains $(DESTDIR)/$(SALTENVDIR)/ || true # Metadata [ -d $(DESTDIR)/$(RECLASSDIR)/service/$(FORMULANAME) ] || mkdir -p $(DESTDIR)/$(RECLASSDIR)/service/$(FORMULANAME) @@ -52,6 +54,10 @@ lint: test: [ ! -d tests ] || (cd tests; ./run_tests.sh) +test-model-validate: + # TODO make it actually fail + [ ! -d $(FORMULANAME)/schemas/ ] || (cd tests; ./run_tests.sh model-validate) + release-major: check-changes @echo "Current version is $(VERSION), new version is $(NEW_MAJOR_VERSION)" @[ $(VERSION_MAJOR) != $(NEW_MAJOR_VERSION) ] || (echo "Major version $(NEW_MAJOR_VERSION) already released, nothing to do. Do you want release-minor?" && exit 1) diff --git a/tests/run_tests.sh b/tests/run_tests.sh index 316692b..a348912 100755 --- a/tests/run_tests.sh +++ b/tests/run_tests.sh @@ -1,24 +1,34 @@ #!/usr/bin/env bash +### +# Script requirments: +#apt-get install -y python-yaml virtualenv git + set -e [ -n "$DEBUG" ] && set -x CURDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" METADATA=${CURDIR}/../metadata.yml FORMULA_NAME=$(cat $METADATA | python -c "import sys,yaml; print yaml.load(sys.stdin)['name']") +FORMULA_META_DIR=${CURDIR}/../${FORMULA_NAME}/meta ## Overrideable parameters PILLARDIR=${PILLARDIR:-${CURDIR}/pillar} BUILDDIR=${BUILDDIR:-${CURDIR}/build} VENV_DIR=${VENV_DIR:-${BUILDDIR}/virtualenv} +MOCK_BIN_DIR=${MOCK_BIN_DIR:-${CURDIR}/mock_bin} DEPSDIR=${BUILDDIR}/deps +SCHEMARDIR=${SCHEMARDIR:-"${CURDIR}/../${FORMULA_NAME}/schemas/"} SALT_FILE_DIR=${SALT_FILE_DIR:-${BUILDDIR}/file_root} SALT_PILLAR_DIR=${SALT_PILLAR_DIR:-${BUILDDIR}/pillar_root} SALT_CONFIG_DIR=${SALT_CONFIG_DIR:-${BUILDDIR}/salt} SALT_CACHE_DIR=${SALT_CACHE_DIR:-${SALT_CONFIG_DIR}/cache} +SALT_CACHE_EXTMODS_DIR=${SALT_CACHE_EXTMODS_DIR:-${SALT_CONFIG_DIR}/cache_master_extmods} + +SALT_OPTS="${SALT_OPTS} --retcode-passthrough --local -c ${SALT_CONFIG_DIR} --log-file=/dev/null" -SALT_OPTS="${SALT_OPTS} --retcode-passthrough --local -c ${SALT_CONFIG_DIR}" +IGNORE_MODELVALIDATE_MASK=${IGNORE_MODELVALIDATE_MASK:-"novalidate"} if [ "x${SALT_VERSION}" != "x" ]; then PIP_SALT_VERSION="==${SALT_VERSION}" @@ -26,24 +36,38 @@ fi ## Functions log_info() { - echo "[INFO] $*" + echo -e "[INFO] $*" } log_err() { - echo "[ERROR] $*" >&2 + echo -e "[ERROR] $*" >&2 } setup_virtualenv() { log_info "Setting up Python virtualenv" + dependency_check virtualenv virtualenv $VENV_DIR source ${VENV_DIR}/bin/activate python -m pip install salt${PIP_SALT_VERSION} + if [[ -f ${CURDIR}/test-requirements.txt ]]; then + python -m pip install -r ${CURDIR}/test-requirements.txt + fi +} + +setup_mock_bin() { + # If some state requires a binary, a lightweight replacement for + # such binary can be put into MOCK_BIN_DIR for test purposes + if [ -d "${MOCK_BIN_DIR}" ]; then + PATH="${MOCK_BIN_DIR}:$PATH" + export PATH + fi } setup_pillar() { [ ! -d ${SALT_PILLAR_DIR} ] && mkdir -p ${SALT_PILLAR_DIR} echo "base:" > ${SALT_PILLAR_DIR}/top.sls for pillar in ${PILLARDIR}/*; do + grep ${FORMULA_NAME}: ${pillar} &>/dev/null || continue state_name=$(basename ${pillar%.sls}) echo -e " ${state_name}:\n - ${state_name}" >> ${SALT_PILLAR_DIR}/top.sls done @@ -53,9 +77,11 @@ setup_salt() { [ ! -d ${SALT_FILE_DIR} ] && mkdir -p ${SALT_FILE_DIR} [ ! -d ${SALT_CONFIG_DIR} ] && mkdir -p ${SALT_CONFIG_DIR} [ ! -d ${SALT_CACHE_DIR} ] && mkdir -p ${SALT_CACHE_DIR} + [ ! -d ${SALT_CACHE_EXTMODS_DIR} ] && mkdir -p ${SALT_CACHE_EXTMODS_DIR} echo "base:" > ${SALT_FILE_DIR}/top.sls for pillar in ${PILLARDIR}/*.sls; do + grep ${FORMULA_NAME}: ${pillar} &>/dev/null || continue state_name=$(basename ${pillar%.sls}) echo -e " ${state_name}:\n - ${FORMULA_NAME}" >> ${SALT_FILE_DIR}/top.sls done @@ -63,13 +89,14 @@ setup_salt() { cat << EOF > ${SALT_CONFIG_DIR}/minion file_client: local cachedir: ${SALT_CACHE_DIR} +extension_modules: ${SALT_CACHE_EXTMODS_DIR} verify_env: False +minion_id_caching: False file_roots: base: - ${SALT_FILE_DIR} - ${CURDIR}/.. - - /usr/share/salt-formulas/env pillar_roots: base: @@ -79,13 +106,14 @@ EOF } fetch_dependency() { + # example: fetch_dependency "linux:https://github.com/salt-formulas/salt-formula-linux" dep_name="$(echo $1|cut -d : -f 1)" dep_source="$(echo $1|cut -d : -f 2-)" dep_root="${DEPSDIR}/$(basename $dep_source .git)" dep_metadata="${dep_root}/metadata.yml" - [ -d /usr/share/salt-formulas/env/${dep_name} ] && log_info "Dependency $dep_name already present in system-wide salt env" && return 0 - [ -d $dep_root ] && log_info "Dependency $dep_name already fetched" && return 0 + dependency_check git + [ -d $dep_root ] && { log_info "Dependency $dep_name already fetched"; return 0; } log_info "Fetching dependency $dep_name" [ ! -d ${DEPSDIR} ] && mkdir -p ${DEPSDIR} @@ -95,6 +123,19 @@ fetch_dependency() { METADATA="${dep_metadata}" install_dependencies } +link_modules(){ + # Link modules *.py files to temporary salt-root + local SALT_ROOT=${1:-$SALT_FILE_DIR} + local SALT_ENV=${2:-$DEPSDIR} + + mkdir -p "${SALT_ROOT}/_modules/" + # from git, development versions + find ${SALT_ENV} -maxdepth 3 -mindepth 3 -path '*_modules*' -iname "*.py" -type f -print0 | while read -d $'\0' file; do + ln -fs $(readlink -e ${file}) "$SALT_ROOT"/_modules/$(basename ${file}) ; + done + salt_run saltutil.sync_all +} + install_dependencies() { grep -E "^dependencies:" ${METADATA} >/dev/null || return 0 (python - | while read dep; do fetch_dependency "$dep"; done) << EOF @@ -115,19 +156,92 @@ salt_run() { } prepare() { - [ -d ${BUILDDIR} ] && mkdir -p ${BUILDDIR} + if [[ -f ${BUILDDIR}/.prepare_done ]]; then + log_info "${BUILDDIR}/.prepare_done exist, not rebuilding BUILDDIR" + return + fi + [[ -d ${BUILDDIR} ]] && mkdir -p ${BUILDDIR} - which salt-call || setup_virtualenv + [[ ! -f "${VENV_DIR}/bin/activate" ]] && setup_virtualenv + setup_mock_bin setup_pillar setup_salt install_dependencies + link_modules + touch ${BUILDDIR}/.prepare_done +} + +lint_releasenotes() { + [[ ! -f "${VENV_DIR}/bin/activate" ]] && setup_virtualenv + source ${VENV_DIR}/bin/activate + reno lint ${CURDIR}/../ +} + +lint() { +# lint_releasenotes + log_err "TODO: lint_releasenotes" } run() { for pillar in ${PILLARDIR}/*.sls; do + grep ${FORMULA_NAME}: ${pillar} &>/dev/null || continue state_name=$(basename ${pillar%.sls}) + salt_run grains.set 'noservices' False force=True + + echo "Checking state ${FORMULA_NAME}.${state_name} ..." salt_run --id=${state_name} state.show_sls ${FORMULA_NAME} || (log_err "Execution of ${FORMULA_NAME}.${state_name} failed"; exit 1) + + # Check that all files in 'meta' folder can be rendered using any valid pillar + for meta in `find ${FORMULA_META_DIR} -type f`; do + meta_name=$(basename ${meta}) + echo "Checking meta ${meta_name} ..." + salt_run --out=quiet --id=${state_name} cp.get_template ${meta} ${SALT_CACHE_DIR}/${meta_name} \ + || { log_err "Failed to render meta ${meta} using pillar ${FORMULA_NAME}.${state_name}"; exit 1; } + cat ${SALT_CACHE_DIR}/${meta_name} + done + done +} + +real_run() { + for pillar in ${PILLARDIR}/*.sls; do + state_name=$(basename ${pillar%.sls}) + salt_run --id=${state_name} state.sls ${FORMULA_NAME} || { log_err "Execution of ${FORMULA_NAME}.${state_name} failed"; exit 1; } + done +} + +run_model_validate(){ + # Run modelschema.model_validate validation. + # TEST iterateble, run for `each formula ROLE against each ROLE_PILLARNAME` + # Pillars should be named in conviend ROLE_XXX.sls or ROLE.sls + # Example: + # client.sls client_auth.sls server.sls server_auth.sls + if [ -d ${SCHEMARDIR} ]; then + # model validator require py modules + fetch_dependency "salt:https://github.com/salt-formulas/salt-formula-salt" + link_modules + salt_run saltutil.clear_cache; salt_run saltutil.refresh_pillar; salt_run saltutil.sync_all; + for role in ${SCHEMARDIR}/*.yaml; do + role_name=$(basename "${role%*.yaml}") + for pillar in $(ls pillar/${role_name}*.sls | grep -v ${IGNORE_MODELVALIDATE_MASK} ); do + pillar_name=$(basename "${pillar%*.sls}") + local _message="FORMULA:${FORMULA_NAME} ROLE:${role_name} against PILLAR:${pillar_name}" + log_info "model_validate ${_message}" + # Rendered Example: + # python $(which salt-call) --local -c /test1/maas/tests/build/salt --id=maas_cluster modelschema.model_validate maas cluster + salt_run -m ${DEPSDIR}/salt-formula-salt --id=${pillar_name} modelschema.model_validate ${FORMULA_NAME} ${role_name} || { log_err "Execution of model_validate ${_message} failed"; exit 1 ; } + done done + else + log_info "${SCHEMARDIR} not found!"; + fi +} + +dependency_check() { + local DEPENDENCY_COMMANDS=$* + + for DEPENDENCY_COMMAND in $DEPENDENCY_COMMANDS; do + which $DEPENDENCY_COMMAND > /dev/null || ( log_err "Command \"$DEPENDENCY_COMMAND\" can not be found in default path."; exit 1; ) + done } _atexit() { @@ -152,11 +266,23 @@ case $1 in prepare) prepare ;; + lint) + lint + ;; run) run ;; + real-run) + real_run + ;; + model-validate) + prepare + run_model_validate + ;; *) prepare +# lint run + run_model_validate ;; esac