.murdock

#!/bin/sh

# uncomment and change this to limit builds, e.g.,
#export BOARDS="samr21-xpro native"
# and / or
#export APPS="examples/hello-world tests/unittests"

# this configures boards that are available via pifleet
case "${CI_MURDOCK_PROJECT}" in
    riot)
        : ${TEST_BOARDS_AVAILABLE:="esp32-wroom-32 samr21-xpro"}
        ;;
    riot-staging)
        : ${TEST_BOARDS_AVAILABLE:=""}
        ;;
    *)
        : ${TEST_BOARDS_AVAILABLE:=""}
        ;;
esac

#: ${EMULATED_BOARDS_AVAILABLE:="microbit"}

# temporarily disabling llvm builds until https://github.com/RIOT-OS/RIOT/pull/15595
# is in
#: ${TEST_BOARDS_LLVM_COMPILE:="iotlab-m3 native nrf52dk mulle nucleo-f401re samr21-xpro slstk3402a"}
: ${TEST_BOARDS_LLVM_COMPILE:=""}

: ${TEST_KCONFIG_BOARDS_AVAILABLE:="
adafruit-itsybitsy-m4
arduino-due
arduino-leonardo
arduino-mkrzero
arduino-mega2560
arduino-nano
atxmega-a1-xplained
atxmega-a3bu-xplained
avr-rss2
avsextrem
bastwan
bluepill
b-l475e-iot01a
cc1352-launchpad
cc2650-launchpad
derfmega128
dwm1001
esp32-ci
esp32-heltec-lora32-v2
esp32-olimex-evb
esp32c3-ci
esp32c3-devkit
esp32s3-ci
esp32s3-devkit
esp32s2-ci
esp32s2-devkit
esp8266-ci
esp8266-esp-12x
hamilton
hifive1
mbed_lpc1768
mega-xplained
microbit
mulle
native
nrf52840dk
nrf9160dk
nucleo-f072rb
nucleo-f103rb
nucleo-f207zg
nucleo-f334r8
nucleo-f429zi
nucleo-f767zi
nucleo-g071rb
nucleo-g474re
nucleo-l011k4
nucleo-l073rz
nucleo-l152re
nucleo-l433rc
nucleo-l552ze-q
pba-d-01-kw2x
p-nucleo-wb55
qn9080dk
remote-revb
same54-xpro
samr21-xpro
seeedstudio-gd32
slstk3400a
sltb001a
slwstk6220a
stm32f723e-disco
stm32mp157c-dk2
waspmote-pro
waveshare-nrf52840-eval-kit
weact-f401ce
z1
"}

: ${TEST_KCONFIG_ENFORCE_APP_GROUPS:="
tests/cb_mux*
tests/congure_*
tests/driver_b*
tests/driver_f*
tests/driver_g*
tests/driver_h*
tests/driver_i*
tests/driver_j*
tests/driver_l*
tests/driver_o*
tests/driver_p*
tests/driver_q*
tests/driver_r*
tests/driver_s*
tests/driver_t*
tests/driver_u*
tests/driver_v*
tests/external_board_dirs
tests/periph_*
tests/pkg_elk
tests/pkg_uzlib
tests/prng_*
tests/trace
tests/xtimer_*
tests/ztimer_*
examples/hello-world
tests/ieee802154_hal
"}

# This list prevents boards from being tested by the TEST_KCONFIG_TEST_ALLOWLIST
# This should only be used for boards that require a lot of work, such as,
# requiring modelling of a clock tree.
# As a rule of thumb, only add boards here if there are not differences in
# modules or packages.
# Eventually this list will be removed...
#
: ${TEST_KCONFIG_BOARD_BLOCKLIST:="
"}

# This list will force all boards that are not in the TEST_KCONFIG_BOARD_BLOCKLIST
# to be tested on the apps. The purpose is to prevent new boards from skipping
# the kconfig implementations.
: ${TEST_KCONFIG_TEST_ALLOWLIST:="
examples/hello-world
tests/saul
tests/mtd_mapper
"}

: ${TEST_WITH_CONFIG_SUPPORTED:="examples/suit_update tests/driver_at86rf2xx_aes"}

export RIOT_CI_BUILD=1
export CC_NOCOLOR=1
export STATIC_TESTS=0
export CFLAGS_DBG=""
export DLCACHE_DIR=${DLCACHE_DIR:-~/.dlcache}
export ENABLE_TEST_CACHE=${ENABLE_TEST_CACHE:-1}
export MURDOCK_REDIS_HOST=${MURDOCK_REDIS_HOST:-127.0.0.1}

NIGHTLY=${NIGHTLY:-0}
FULL_BUILD=${FULL_BUILD:-${NIGHTLY}}

# This is a work around for a bug in CCACHE which interacts very badly with
# some features of RIOT and of murdock. The result is that ccache is
# ineffective (i.e. objects are never reused, resulting in extreme cache miss
# rate) and murdock becomes slow.
#
# - CCACHE thinks that -gz by itself enables debugging, which is not true.
#   see https://github.com/ccache/ccache/issues/464
#   - When debug info is included, CCACHE hashes the file paths, as these
#     influence the debug information (the name of compile units and/or their
#     "comp_dir" attribute)
# - Riot does not set -fdebug-prefix-map. This is not that easy as it may not
#   be supported in every toolchain (some are quite old).
# - Murdock builds PRs in different directories each time.
#
# It is only the combination of these three factors which causes this bug.
export OPTIONAL_CFLAGS_BLACKLIST="-gz"

DWQ_ENV="-E BOARDS -E APPS -E NIGHTLY -E RUN_TESTS -E ENABLE_TEST_CACHE
         -E TEST_HASH -E CI_PULL_LABELS -ECI_BASE_BRANCH -ECI_BASE_COMMIT
         -EPKG_USE_MIRROR -EAPPS_CHANGED -EBOARDS_CHANGED -ESTATIC_TESTS
         -E CI_MURDOCK_PROJECT"

if [ ${NIGHTLY} -eq 1 ]; then
    export PKG_USE_MIRROR=0
fi

CFCR_ARGS="--upstreambranch ${CI_BASE_COMMIT}"

get_supported_kconfig_board_app() {
    local board=$1
    local appdir=$2
    if is_in_list "${board}" "${TEST_KCONFIG_BOARD_BLOCKLIST}"; then
        return 1
    fi

    # On nightlies run all possible kconfig tests on all boards
    # Normally we don't want to do this as it adds quite a bit to build time
    # and the subset of boards we are testing for are pretty good at catching
    # any bugs...
    # This will be over one day, I promise.
    if [ ${NIGHTLY} -eq 1 ]; then
        if is_in_list "${appdir}" "${TEST_KCONFIG_ENFORCE_APP_GROUPS}"; then
            return 0
        fi
        if [ -f "${appdir}/app.config.test" ]; then
            return 0
        fi
    fi
    if is_in_list "${appdir}" "${TEST_KCONFIG_TEST_ALLOWLIST}"; then
        return 0
    fi
    if is_in_list "${board}" "${TEST_KCONFIG_BOARDS_AVAILABLE}"; then
        if is_in_list "${appdir}" "${TEST_KCONFIG_ENFORCE_APP_GROUPS}"; then
            return 0
        fi
        if [ -f "${appdir}/app.config.test" ]; then
            return 0
        fi
    fi
    return 1
}

kconfig_module_packages_diff() {
    local board=$1
    local appdir=$2

    make clean --no-print-directory -C ${appdir} BOARD=${board}
    mmp=$(make --no-print-directory info-modules info-packages -C ${appdir} BOARD=${board})

    make clean --no-print-directory -C ${appdir} BOARD=${board}
    kmp=$(TEST_KCONFIG=1 make --no-print-directory info-modules info-packages -C ${appdir} BOARD=${board})


    bash -c "diff <(echo \"${mmp}\") <(echo \"${kmp}\")"
}

check_label() {
    local label="${1}"
    [ -z "${CI_PULL_LABELS}" ] && return 1
    echo "${CI_PULL_LABELS}" | grep -q "${label}"
    return $?
}

# if RUN_TESTS is unset (e.g., not passed from the outside),
# set to 1 if NIGHTLY=1 or if the label "CI: run tests" is set,
# otherwise set 0.
if [ -z "$RUN_TESTS" ]; then
    if [ "$NIGHTLY" = "1" ] || check_label "CI: run tests" ; then
        RUN_TESTS=1
    else
        RUN_TESTS=0
    fi
fi

[ "$ENABLE_TEST_CACHE" = "1" ] && {
    check_label "CI: disable test cache" && export ENABLE_TEST_CACHE=0
}

error() {
    echo "$@"
    exit 1
}

# true if "$2" starts with "$1", false otherwise
startswith() {
  case "${2}" in
    ${1}*) true ;;
    *) false ;;
  esac
}

# if MURDOCK_HOOK is set, this function will execute it and pass on all it's
# parameters. should the hook script exit with negative exit code, hook() makes
# this script exit with error, too.
# hook() will be called from different locations of this script.
# currently, the only caller is "run_test", which calls "hook run_test_pre".
# More hooks will be added as needed.
hook() {
    if [ -n "${MURDOCK_HOOK}" ]; then
        echo "- executing hook $1"
        "${MURDOCK_HOOK}" "$@" || {
            error "$0: hook \"${MURDOCK_HOOK} $@\" failed!"
        }
        echo "- hook $1 finished"
    fi
}

# true if word "$1" is in list of words "$2", false otherwise
# uses grep -w, thus only alphanum and "_" count as word bounderies
# (word "def" matches "abc-def")
is_in_list() {
    [ $# -ne 2 ] && return 1

    local needle="$1"
    local haystack="$2"

    echo "$haystack" | grep -q -w "$needle"
}

# grep that doesn't return error on empty input
_grep() {
    grep "$@"
    true
}

_greplist() {
    if [ $# -eq 0 ]; then
        echo cat
    else
        echo -n "_grep -E ($1"
        shift
        for i in $*; do
            echo -n "|$i"
        done
        echo ")"
    fi
}

# get list of all app directories
get_apps() {
    make -f makefiles/app_dirs.inc.mk info-applications \
        | $(_greplist $APPS) | sort
}

# take app dir as parameter, print all boards that are supported
# Only print for boards in $BOARDS.
get_supported_boards() {
    local appdir=$1
    local only_changed=0

    if [ -n "$APPS_CHANGED" ]; then
        if is_in_list "$appdir" "${APPS_CHANGED}"; then
            # this app has changed -> build for all boards
            true
        else
            # this is not a changed app -> build for changed boards
            only_changed=1
        fi
    else
        if [ -n "$BOARDS_CHANGED" ]; then
            echo d
            # no changed apps, some changed boards -> build for changed boards
            only_changed=1
        else
            # no changed apps list, no changed boards list -> build all boards
            true
        fi
    fi

    if [ $only_changed -eq 1 ]; then
        BOARDS_=${BOARDS}
        export BOARDS="$BOARDS_CHANGED"
    fi

    local boards="$(make --no-print-directory -C$appdir info-boards-supported 2>/dev/null || echo broken)"

    export BOARDS="${BOARDS_}"

    if [ "$boards" = broken ]; then
        echo "makefile_broken"
        return
    fi

    for board in $boards
    do
        echo $board
    done | $(_greplist $BOARDS)
}

get_supported_toolchains() {
    local appdir=$1
    local board=$2
    local toolchains="gnu"

    if is_in_list "${board}" "${TEST_BOARDS_LLVM_COMPILE}"; then
        toolchains="$(make -s --no-print-directory -C${appdir} BOARD=${board} \
                      info-toolchains-supported 2> /dev/null | grep -o -e "llvm" -e "gnu")"
    fi
    echo "${toolchains}"
}

# given an app dir as parameter, print "$appdir $board:$toolchain" for each
# supported board and toolchain. Only print for boards in $BOARDS.
# if extra args are given, they will be prepended to each output line.
get_app_board_toolchain_pairs() {
    local appdir=$1
    local boards="$(get_supported_boards $appdir)"

    # collect extra arguments into prefix variable
    shift
    local prefix="$*"

    if [ "$boards" = makefile_broken ]; then
        echo "$0 error \"error: ${DWQ_WORKER}: get_supported_boards failed in $appdir\""
        return
    fi

    for board in ${boards}
    do
        for toolchain in $(get_supported_toolchains $appdir $board)
        do
            echo $prefix $appdir $board:$toolchain
        done
    done | $(_greplist $BOARDS)
}

# use dwqc to create full "appdir board toolchain" compile job list
get_compile_jobs() {
    check_label "CI: skip compile test" && return
    update_changed_modules || return

    get_apps | \
        maybe_filter_changed_apps | \
        dwqc ${DWQ_ENV} --queue default-first -s \
        ${DWQ_JOBID:+--subjob} \
        "$0 get_app_board_toolchain_pairs \${1} $0 compile"
}

print_worker() {
    [ -n "$DWQ_WORKER" ] && \
        echo "-- running on worker ${DWQ_WORKER} thread ${DWQ_WORKER_THREAD}, build number $DWQ_WORKER_BUILDNUM."
}

test_hash_calc() {
    local bindir=$1

    # Why two times cut?
    # "test-input-hash.sha1" contains a list of lines containing
    # "<hash> <filename>" on each line.
    # We need to filter out the filename, as it contains the full path name,
    # which differs depending on the build machine.
    #
    # After piping through sha1sum, we get "<hash> -". " -" has to go so we save the
    # hassle of escaping the resulting hash.

    cat ${bindir}/test-input-hash.sha1 | cut -f1 -d' ' | sha1sum | cut -f1 -d' '
}

test_cache_get() {
    test "${ENABLE_TEST_CACHE}" = "1" || return 1
    test -n "$(redis-cli -h ${MURDOCK_REDIS_HOST} get $1)" > /dev/null
}

test_cache_put() {
    redis-cli -h ${MURDOCK_REDIS_HOST} set "$1" ok
}

# compile one app for one board with one toolchain. delete intermediates.
compile() {
    local appdir=$1
    local board=$(echo $2 | cut -f 1 -d':')
    local toolchain=$(echo $2 | cut -f 2 -d':')

    [ "$board" = "makefile_broken" ] && {
        echo "$0: There seems to be a problem in \"$appdir\" while getting supported boards!"
        echo "$0: testing \"make -C$appdir info-boards-supported\"..."
        make -C$appdir info-boards-supported && echo "$0: success. no idea what's wrong." || echo "$0: failed!"
        exit 1
    }

    # set build directory. CI ensures only one build at a time in $(pwd).
    export BINDIR="$(pwd)/build"
    export PKGDIRBASE="${BINDIR}/pkg"

    # Pre-build cleanup
    rm -rf ${BINDIR}

    print_worker

    # sanity checks
    [ $# -ne 2 ] && error "$0: compile: invalid parameters (expected \$appdir \$board:\$toolchain)"
    [ ! -d "$appdir" ] && error "$0: compile: error: application directory \"$appdir\" doesn't exist"

    # We compile a first time with Kconfig based dependency
    # resolution for regression purposes. $TEST_KCONFIG contains a
    # list of board-application tuples that are currently modeled to
    # run with Kconfig

    should_check_kconfig_hash=0

    if get_supported_kconfig_board_app "${board}" "${appdir}"; then
        should_check_kconfig_hash=1
        # As we have some issues with occasional unrelated hash mismatches
        # we will stop the binary checks and rely only in a module/package
        # check to ensure kconfig is matching make.
        # Only nightlies will check the hash...
        # Once we figure out the problem we can check the hashes again or
        # better yet just finish!
        if [ ${NIGHTLY} -eq 1 ]; then
            BOARD=${board} make -C${appdir} clean
            CCACHE_BASEDIR="$(pwd)" BOARD=${board} TOOLCHAIN=${toolchain} RIOT_CI_BUILD=1 TEST_KCONFIG=1 \
                            make -C${appdir} all test-input-hash -j${JOBS:-4}
            RES=$?
            if [ $RES -eq 0 ]; then
                kconfig_hashes="$(cat ${BINDIR}/test-input-hash.sha1)"
            else
                kconfig_hashes="kconfig-build-failed"
                echo "An error occurred while compiling using Kconfig";
            fi
        fi
    fi

    # compile without Kconfig
    BOARD=${board} make -C${appdir} clean
    CCACHE_BASEDIR="$(pwd)" BOARD=$board TOOLCHAIN=$toolchain RIOT_CI_BUILD=1 \
        make -C${appdir} all test-input-hash -j${JOBS:-4}
    RES=$?

    if [ ${NIGHTLY} -eq 1 ]; then
        no_kconfig_hashes="$(cat ${BINDIR}/test-input-hash.sha1)"
    fi
    # test hash is used to cache test results, not for comparing binaries
    # generated with and without KConfig
    test_hash=$(test_hash_calc "$BINDIR")


    if [ ${should_check_kconfig_hash} != 0 ]; then
        if [ ${NIGHTLY} -eq 1 ]; then
            if [ "${kconfig_hashes}" != "${no_kconfig_hashes}" ]; then
                echo "Hashes of binaries with and without Kconfig mismatch for ${appdir} with ${board}";
                echo "Please check that all used modules are modelled in Kconfig and enabled";
                echo "Input without KConfig:"
                echo "${no_kconfig_hashes}"
                echo "Input with KConfig:"
                echo "${kconfig_hashes}"
                kconfig_module_packages_diff ${board} ${appdir}
                RES=1
            fi
        else
            kconfig_module_packages_diff ${board} ${appdir}
            RES=$(( $RES | $? ))
        fi
    fi

    # run tests
    if [ $RES -eq 0 ]; then
        if is_in_list "$board" "$EMULATED_BOARDS_AVAILABLE"; then
            EMULATED=1
        else
            EMULATED=0
        fi

        if [ $RUN_TESTS -eq 1 -o "$board" = "native" -o "$EMULATED" = "1" ]; then
            if [ -f "${BINDIR}/.test" ]; then
                if [ "$board" = "native" -o "${EMULATED}" = "1" ]; then
                    EMULATE=${EMULATED} BOARD=$board make -C${appdir} test
                    RES=$?
                elif is_in_list "$board" "$TEST_BOARDS_AVAILABLE"; then
                    echo "-- test_hash=$test_hash"
                    if test_cache_get $test_hash; then
                        echo "-- skipping test due to positive cache hit"
                    else
                        BOARD=$board TOOLCHAIN=$toolchain TEST_HASH=$test_hash \
                            make -C${appdir} test-murdock
                        RES=$?
                    fi
                fi
            fi
        fi
    fi

    # log some build stats
    if [ -d ${BINDIR} ]
    then
        echo "{\"build/\": $(du -s ${BINDIR} | cut -f1)}"
        # cleanup
        rm -rf ${BINDIR}
    fi

    return $RES
}

test_job() {
    local appdir=$1
    local board=$(echo $2 | cut -f 1 -d':')
    local toolchain=$(echo $2 | cut -f 2 -d':')

    # interpret any extra arguments as file names.
    # They will be sent along with the job to the test worker
    # and stored in the application's binary folder.
    shift 2
    local files=""
    for filename in "$@"; do
        # check if the file is within $(BINDIR)
        if startswith "${BINDIR}" "${filename}"; then
          # get relative (to $(BINDIR) path
          local relpath="$(realpath --relative-to ${BINDIR} ${filename})"
        else
          error "$0: error: extra test files not within \${BINDIR}!"
        fi

        # set remote file path.
        # currently, the test workers build in the default build path.
        local remote_bindir="${appdir}/bin/${board}"
        files="${files} --file ${filename}:${remote_bindir}/${relpath}"
    done

    dwqc \
        ${DWQ_ENV} \
        ${DWQ_JOBID:+--subjob} \
        --queue ${TEST_QUEUE:-$board} \
        --maxfail 1 \
        $files \
        "./.murdock run_test $appdir $board:$toolchain"
}

run_test() {
    local appdir=$1
    local board=$(echo $2 | cut -f 1 -d':')
    local toolchain=$(echo $2 | cut -f 2 -d':')
    print_worker
    echo "-- executing tests for $appdir on $board (compiled with $toolchain toolchain):"
    hook run_test_pre

    # do flashing and building of termdeps simultaneously
    BOARD=$board TOOLCHAIN=${toolchain} make -C$appdir flash-only termdeps -j2

    # now run the actual test
    if is_in_list "${appdir}" "${TEST_WITH_CONFIG_SUPPORTED}"; then
        BOARD=${board} TOOLCHAIN=${toolchain} make -C${appdir} test-with-config
    else
        BOARD=$board TOOLCHAIN=${toolchain} make -C$appdir test
    fi
    RES=$?

    if [ $RES -eq 0 -a -n "$TEST_HASH" ]; then
        echo -n "-- saving test result to cache: "
        test_cache_put $TEST_HASH
    fi

    return $RES
}

basename_list () {
    for path in $*; do
        basename $path
    done
}

# calls out to can_fast_ci_run.py.
#
# returns 1 if nothing should be built.
# or, returns 0, potentially meddling with BOARDS, APPS, BOARDS_CHANGED, APPS_CHANGED.
update_changed_modules() {
    # early out if there's no base commit info
    if [ -z "${CI_BASE_COMMIT}" ]; then
        return 0
    fi

    # early out if a full build is requested
    if [ $FULL_BUILD -eq 1 ]; then
        return 0
    fi

    # if these are set, just build what's requested.
    if [ -n "$BOARDS" -o -n "$APPS" ]; then
        return 0
    fi

    # do full build if requested by label
    check_label "CI: full build" && return 0

    # call out to can_fast_ci_run.py
    # that script will output e.g.,
    #    APPS_CHANGED="foo/bar foo/barfoo"
    #    BOARDS_CHANGED="foo/bar foo/barfoo"
    # just eval that output, so we set those variables directly in this shell.
    eval $(dist/tools/ci/can_fast_ci_run.py \
        ${CFCR_ARGS} \
        --changed-boards --changed-apps \
        2>/dev/null || echo CFCR_ERROR=1)

    # if this errors, it means we cannot skip any builds.
    if [ -n "$CFCR_ERROR" ]; then
        return 0
    fi

    # if can_fast_ci_run exits 0 but doesn't output any changed apps or boards,
    # it means we can skip all compile jobs.
    if [ -z "$APPS_CHANGED" -a -z "$BOARDS_CHANGED" ]; then
        return 1
    fi

    # can_fast_ci_run.py outputs "board/fooboard", but the rest of this script
    # expects a board name without leading "board/".
    if [ -n "$BOARDS_CHANGED" ]; then
        export BOARDS_CHANGED="$(basename_list $BOARDS_CHANGED)"
    fi

    # if no board has changed, filter by just setting APPS.
    # same for apps -> BOARDS.
    if [ -z "$APPS_CHANGED" -a -n "$BOARDS_CHANGED" ]; then
        export BOARDS="$(echo $BOARDS_CHANGED | $(_greplist ${BOARDS}))"
        unset BOARDS_CHANGED
    elif [ -n "$APPS_CHANGED" -a -z "$BOARDS_CHANGED" ]; then
        export APPS="$(echo $APPS_CHANGED | $(_greplist ${APPS}))"
        unset APPS_CHANGED
    fi

    export APPS_CHANGED
    export BOARDS_CHANGED
}

maybe_filter_changed_apps() {
    # if no boards have changes, only a subset of the apps need to be built.
    # else, all apps need to be passed, as they'll need to be built for
    # the changed boards.
    if [ -n "$BOARDS_CHANGED" ]; then
        cat
    else
        $(_greplist $APPS_CHANGED)
    fi
}

# execute static tests
static_tests() {
    print_worker
    build_filter_status

    [ "$STATIC_TESTS" = "1" ] && \
        ./dist/tools/ci/static_tests.sh
    true
}

build_filter_status() {
    echo "--- can_fast_ci_run:"
    if [ $FULL_BUILD -eq 1 ]; then
        echo "--- doing full build."
        return
    fi

    dist/tools/ci/can_fast_ci_run.py ${CFCR_ARGS} --explain --json --changed-boards --changed-apps
    if [ -n "$MURDOCK_TEST_CHANGE_FILTER" ]; then
        echo MURDOCK_TEST_CHANGE_FILTER=$MURDOCK_TEST_CHANGE_FILTER
    fi

    echo ""
    update_changed_modules

    if [ -n "$CFCR_ERROR" ]; then
        echo "-- can_fast_ci_run.py exited non-zero"
    fi
}

get_non_compile_jobs() {
    echo "$0 static_tests"
}

get_jobs() {
    get_non_compile_jobs
    get_compile_jobs
}

$*