diff --git a/.gitignore b/.gitignore index 3c3629e..9daa824 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,2 @@ +.DS_Store node_modules diff --git a/balena.sh b/balena.sh new file mode 100755 index 0000000..79c2cf0 --- /dev/null +++ b/balena.sh @@ -0,0 +1,186 @@ +#!/usr/bin/env bash + +# shellcheck disable=SC2154,SC2034,SC1090 +set -ea + +[[ $VERBOSE =~ on|On|Yes|yes|true|True ]] && set -x + +if [[ -n "${BALENA_DEVICE_UUID}" ]]; then + # prepend the device UUID if running on balenaOS + TLD="${BALENA_DEVICE_UUID}.${DNS_TLD}" +else + TLD="${DNS_TLD}" +fi + +BALENA_API_URL=${BALENA_API_URL:-https://api.balena-cloud.com} +certs=${CERTS:-/certs} +conf=${CONF:-/balena/${TLD}.env} +test_fleet=${TEST_FLEET:-test-fleet} +device_type=${DEVICE_TYPE:-qemux86-64} +os_version=${OS_VERSION:-$(balena os versions "${device_type}" | grep \.dev | head -n 1)} +guest_disk_size=${GUEST_DISK_SIZE:-16} +guest_image=${GUEST_IMAGE:-/balena/balena.img} +attempts=${ATTEMPTS:-3} + +function set_update_lock { + if [[ -n $BALENA_SUPERVISOR_ADDRESS ]] && [[ -n $BALENA_SUPERVISOR_API_KEY ]]; then + while [[ $(curl --silent --retry "${attempts}" --fail \ + "${BALENA_SUPERVISOR_ADDRESS}/v1/device?apikey=${BALENA_SUPERVISOR_API_KEY}" \ + -H "Content-Type: application/json" | jq -r '.update_pending') == 'true' ]]; do + + curl --silent --retry "${attempts}" --fail \ + "${BALENA_SUPERVISOR_ADDRESS}/v1/device?apikey=${BALENA_SUPERVISOR_API_KEY}" \ + -H "Content-Type: application/json" | jq -r + + sleep "$(( (RANDOM % 3) + 3 ))s" + done + sleep "$(( (RANDOM % 5) + 5 ))s" + + # https://www.balena.io/docs/learn/deploy/release-strategy/update-locking/ + lockfile /tmp/balena/updates.lock + fi +} + +function remove_update_lock() { + rm -f /tmp/balena/updates.lock +} + +function cleanup() { + rm -f /tmp/balena.zip + remove_update_lock + + # crash loop backoff + sleep 10s +} + +trap 'cleanup' EXIT + +function update_ca_certificates() { + # only set CA bundle if using private certificate chain + if [[ -e "${certs}/ca-bundle.pem" ]]; then + if [[ "$(readlink -f "${certs}/${TLD}-chain.pem")" =~ \/private\/ ]]; then + mkdir -p /usr/local/share/ca-certificates + cat < "${certs}/ca-bundle.pem" > /usr/local/share/ca-certificates/balenaRootCA.crt + # shellcheck disable=SC2034 + CURL_CA_BUNDLE=${CURL_CA_BUNDLE:-${certs}/ca-bundle.pem} + NODE_EXTRA_CA_CERTS=${NODE_EXTRA_CA_CERTS:-${CURL_CA_BUNDLE}} + # (TBC) refactor to use NODE_EXTRA_CA_CERTS instead of ROOT_CA + # https://github.com/balena-io/e2e/blob/master/conf.js#L12-L14 + # https://github.com/balena-io/e2e/blob/master/Dockerfile#L82-L83 + # ... or + # https://thomas-leister.de/en/how-to-import-ca-root-certificate/ + # https://github.com/puppeteer/puppeteer/issues/2377 + ROOT_CA=${ROOT_CA:-$(cat < "${NODE_EXTRA_CA_CERTS}" | openssl base64 -A)} + else + rm -f /usr/local/share/ca-certificates/balenaRootCA.crt + unset NODE_EXTRA_CA_CERTS CURL_CA_BUNDLE ROOT_CA + fi + update-ca-certificates + fi +} + +function wait_for_api() { + while ! curl --silent --fail "https://api.${DNS_TLD}/ping"; do + sleep "$(( (RANDOM % 5) + 5 ))s" + done +} + +function open_balena_login() { + balena login --credentials \ + --email "${SUPERUSER_EMAIL}" \ + --password "${SUPERUSER_PASSWORD}" +} + +function create_fleet() { + if ! balena fleet "${test_fleet}"; then + # wait for API to load DT contracts + # (TBC) 'balena devices supported' always returns empty list + while ! balena fleet create "${test_fleet}" --type "${device_type}"; do + sleep "$(( (RANDOM % 5) + 5 ))s" + done + fi +} + +function download_os_image() { + if ! [[ -e $guest_image ]]; then + wget -qO /tmp/balena.zip \ + "${BALENA_API_URL}/download?deviceType=${device_type}&version=${os_version:1}&fileType=.zip&developmentMode=true" + + unzip -oq /tmp/balena.zip -d /tmp + + cat < "$(find /tmp -type f -name "*.img" | head -n 1)" > "${guest_image}" + fi +} + +function configure_virtual_device() { + while ! [[ -e $guest_image ]]; do sleep "$(( (RANDOM % 5) + 5 ))s"; done + + if ! [[ -e /balena/config.json ]]; then + balena_device_uuid="$(openssl rand -hex 16)" + + balena device register "${test_fleet}" --uuid "${balena_device_uuid}" + + balena config generate \ + --version "${os_version:1}" \ + --device "${balena_device_uuid}" \ + --network ethernet \ + --appUpdatePollInterval 10 \ + --output /balena/config.json + fi + + balena os configure "${guest_image}" \ + --fleet "${test_fleet}" \ + --config /balena/config.json + +} + +function resize_disk_image() { + if ! [[ -e /balena/standard0.qcow2 ]]; then + qemu-img convert -f raw -O qcow2 \ + "${guest_image}" \ + "/balena/standard0.qcow2" + + qemu-img resize "/balena/standard0.qcow2" "${guest_disk_size}G" + fi +} + +function convert_raw_image() { + if ! [[ -e /balena/standard0-snapshot.qcow2 ]]; then + qemu-img create \ + -f qcow2 -b "/balena/standard0.qcow2" \ + -F qcow2 "/balena/standard0-snapshot.qcow2" \ + "$(( guest_disk_size / 2 ))G" + fi +} + +function enable_nested_virtualisation() { + if modprobe kvm_intel; then + echo 1 > /sys/kernel/mm/ksm/run + else + sed -i '/accel: kvm/d' guests.yml + fi +} + +if [[ $TLD =~ ^.*\.local\.? ]]; then + echo 'mDNS configurations not supported' + sleep infinity +fi + +[[ -f $conf ]] && source "${conf}" + +BALENARC_BALENA_URL="${DNS_TLD}" + +enable_nested_virtualisation +update_ca_certificates +wait_for_api +balena whoami || open_balena_login +create_fleet +download_os_image +configure_virtual_device + +set_update_lock +resize_disk_image +convert_raw_image +remove_update_lock + +exec /root/cli.js "$@" diff --git a/cli/Dockerfile b/cli/Dockerfile index 92484ea..2051c15 100644 --- a/cli/Dockerfile +++ b/cli/Dockerfile @@ -9,23 +9,55 @@ RUN apk add --update --no-cache \ WORKDIR /root +RUN apk add --no-cache \ + bash \ + curl \ + docker-cli \ + jq \ + minicom \ + netcat-openbsd \ + openssh-client \ + qemu-img \ + qemu-system-x86_64 \ + unzip \ + wget \ + && apk add --no-cache --repository http://dl-cdn.alpinelinux.org/alpine/v3.11/main \ + procmail + + +# --- build FROM base AS build -# install npm in a separate build stage to save space in the runtime image -RUN apk add --update --no-cache npm +RUN apk add --no-cache -t .build-deps \ + build-base \ + git \ + linux-headers \ + python3 + +COPY run-tests.sh package*.json *.js ./ + +RUN npm i && apk del --purge .build-deps -COPY package.json *.js ./ -RUN npm i +# --- runtime FROM base AS run +WORKDIR /data/ + COPY --from=build /root/ /root/ +COPY *.yml ./ + +COPY docker-hc balena.sh /usr/sbin/ + +RUN ln -sf /root/node_modules/balena-cli/bin/balena /usr/bin/balena + # create qemu-bridge-helper ACL file # https://wiki.qemu.org/Features/HelperNetworking RUN mkdir -p /etc/qemu \ && echo "allow all" > /etc/qemu/bridge.conf \ - && chmod 0640 /etc/qemu/bridge.conf + && chmod 0640 /etc/qemu/bridge.conf \ + && addgroup root qemu \ + && addgroup root kvm -WORKDIR /data/ CMD /root/cli.js diff --git a/cli/package.json b/cli/package.json index 3776d00..5e681a3 100644 --- a/cli/package.json +++ b/cli/package.json @@ -11,6 +11,7 @@ "author": "", "license": "ISC", "dependencies": { + "balena-cli": "^13.2.1", "mz": "^2.7.0", "yaml": "^1.10.2", "yargs": "^17.5.0" diff --git a/docker-hc b/docker-hc new file mode 100755 index 0000000..18ee18e --- /dev/null +++ b/docker-hc @@ -0,0 +1,45 @@ +#!/usr/bin/env bash + +# shellcheck disable=SC2154,SC2034,SC1090 +set -ea + +[[ $VERBOSE =~ on|On|Yes|yes|true|True ]] && set -x + +conf=${CONF:-/balena/${BALENA_DEVICE_UUID}.${DNS_TLD}.env} +test_fleet=${TEST_FLEET:-test-fleet} + +function open_balena_login() { + balena login --credentials \ + --email "${SUPERUSER_EMAIL}" \ + --password "${SUPERUSER_PASSWORD}" +} + +function check_device_status() { + if [[ -e /balena/config.json ]]; then + balena_device_uuid="$(cat < /balena/config.json | jq -r .uuid)" + + if [[ -n $balena_device_uuid ]]; then + is_online="$(balena devices --json --fleet "${test_fleet}" \ + | jq -r --arg uuid "${balena_device_uuid}" '.[] | select(.uuid==$uuid).is_online == true')" + + if [[ $is_online =~ true ]]; then + exit 0 + else + exit 1 + fi + fi + fi +} + +[[ -f $conf ]] && source "${conf}" + +BALENARC_BALENA_URL="${DNS_TLD}" + +balena whoami || open_balena_login + +if [[ $TLD =~ ^.*\.local\.? ]]; then + # mDNS configurations not supported + exit 0 +else + check_device_status +fi diff --git a/guests.yml b/guests.yml new file mode 100644 index 0000000..32c3b82 --- /dev/null +++ b/guests.yml @@ -0,0 +1,49 @@ +# ~ https://github.com/balena-io/autohat/blob/master/resources/qemu.robot +templates: + standard: + machine: + - type: q35 + # (TBC) not supported on AWS/EC2 unless using metal instance classes|types + # only supported on AMIs build on AWS Nitro System + accel: kvm + smp: cores=2 + m: 512M + drive: + - file: /balena/standard0.qcow2 + format: qcow2 + if: none + index: 0 + media: disk + id: disk + device: + - ahci: + id: ahci + - ide-hd: + drive: disk + bus: ahci.0 + - virtio-net-pci: + netdev: n1 + netdev: + #- "user,id=n1,dns=127.0.0.1,guestfwd=tcp:10.0.2.100:80-cmd:netcat haproxy 80,tcp:10.0.2.100:443-cmd:netcat haproxy 443" + - user: + id: n1 + dns: 127.0.0.1 + guestfwd: + # (TBC) escape spaces in command + #- tcp:10.0.2.100:80-cmd:nc haproxy 80 + - tcp:10.0.2.100:443-cmd:nc haproxy 443 + # (minicom) https://github.com/balena-io/autohat#troubleshooting + chardev: + - socket: + id: serial0 + path: /tmp/console.sock + server: on + wait: off + serial: chardev:serial0 + monitor: none + nographic: + +guests: + - template: standard + arch: x86_64 + count: 1 diff --git a/run-tests.sh b/run-tests.sh new file mode 100755 index 0000000..908d9c1 --- /dev/null +++ b/run-tests.sh @@ -0,0 +1,107 @@ +#!/usr/bin/env bash + +# shellcheck disable=SC2154,SC2034,SC1090 +set -ea + +[[ $VERBOSE =~ on|On|Yes|yes|true|True ]] && set -x + +if [[ -n "${BALENA_DEVICE_UUID}" ]]; then + # prepend the device UUID if running on balenaOS + TLD="${BALENA_DEVICE_UUID}.${DNS_TLD}" +else + TLD="${DNS_TLD}" +fi + +conf=${CONF:-/balena/${TLD}.env} +test_fleet=${TEST_FLEET:-test-fleet} +device_type=${DEVICE_TYPE:-qemux86-64} + +function cleanup() { + rm -f Dockerfile + sleep "$(( (RANDOM % 5) + 5 ))s" +} + +trap 'cleanup' EXIT + +function wait_for_device() { + while ! [[ -e /balena/config.json ]]; do sleep "$(( (RANDOM % 5) + 5 ))s"; done + while ! pgrep qemu-system-x86_64; do sleep "$(( (RANDOM % 5) + 5 ))s"; done + while ! /usr/sbin/docker-hc; do sleep "$(( (RANDOM % 5) + 5 ))s"; done +} + +function registry_auth() { + if [[ -n $REGISTRY_USER ]] && [[ -n $REGISTRY_PASS ]]; then + docker login -u "${REGISTRY_USER}" -p "${REGISTRY_PASS}" + + echo "{\"https://index.docker.io/v1/\":{\"username\":\"${REGISTRY_USER}\",\"password\":\"${REGISTRY_PASS}\"}}" \ + | jq -r > ~/.balena/secrets.json + fi +} + +function deploy_release() { + printf 'FROM balenalib/%s-alpine\n\nCMD [ "balena-idle" ]\n' "${device_type}" > Dockerfile + + while ! balena deploy \ + --ca "${DOCKER_CERT_PATH}/ca.pem" \ + --cert "${DOCKER_CERT_PATH}/cert.pem" \ + --key "${DOCKER_CERT_PATH}/key.pem" \ + "${test_fleet}" --logs; do + + sleep "$(( (RANDOM % 5) + 5 ))s" + done +} + +function get_last_release() { + balena releases "${test_fleet}" | head -n 2 | tail -n 1 \ + | grep -E '^.*\s+success\s+.*\s+true$' \ + | awk '{print $2}' +} + +function check_running_release() { + balena_device_uuid="$(cat < /balena/config.json | jq -r .uuid)" + + if [[ -n $balena_device_uuid ]] && [[ -n $1 ]]; then + while ! [[ $(balena device "${balena_device_uuid}" | grep -E ^COMMIT | awk '{print $2}') =~ ${should_be_running_release_id} ]]; do + running_release_id="$(balena device "${balena_device_uuid}" | grep -E ^COMMIT | awk '{print $2}')" + printf 'please wait, device %s should be running %s, but is still running %s...\n' \ + "${balena_device_uuid}" \ + "${1}" \ + "${running_release_id}" + + sleep "$(( (RANDOM % 5) + 5 ))s" + done + + balena device "${balena_device_uuid}" + fi +} + +function supervisor_poll_target_state() { + balena_device_uuid="$(cat < /balena/config.json | jq -r .uuid)" + + if [[ -n $balena_device_uuid ]]; then + while ! curl -X POST --silent --fail \ + --header "Content-Type:application/json" \ + --header "Authorization: Bearer $(cat ~/.balena/token)" \ + --data "{\"uuid\": \"${balena_device_uuid}\", \"data\": {\"force\": true}}" \ + "https://api.${DNS_TLD}/supervisor/v1/update"; do + + sleep "$(( (RANDOM % 5) + 5 ))s" + done + fi +} + +if [[ $TLD =~ ^.*\.local\.? ]]; then + echo 'mDNS configurations not supported' + exit 1 +fi + +[[ -f $conf ]] && source "${conf}" + +BALENARC_BALENA_URL="${DNS_TLD}" + +wait_for_device +registry_auth +deploy_release +should_be_running_release_id="$(get_last_release)" +supervisor_poll_target_state +check_running_release "${should_be_running_release_id}"