From ccac0fbdb08693ce84c0f17a1a7ac0ee683167c2 Mon Sep 17 00:00:00 2001 From: Randy Fay Date: Tue, 3 Sep 2024 18:05:33 -0600 Subject: [PATCH] test: apt key expiration management, fixes #5829 (#6516) --- .ci-scripts/linux_arm64_setup.sh | 2 +- .github/workflows/linux-setup.sh | 2 +- containers/ddev-dbserver/test/functions.sh | 2 +- .../ddev-dbserver/test/image_general.bats | 21 +++--- .../ddev-dbserver/test/test_dbserver.sh | 1 + containers/ddev-ssh-agent/Dockerfile | 2 +- containers/ddev-ssh-agent/test/functions.sh | 2 +- .../ddev-ssh-agent/test/image_general.bats | 15 +--- containers/ddev-ssh-agent/test/test.sh | 5 +- .../tests/ddev-webserver/general.bats | 17 ++--- .../tests/ddev-webserver/test.sh | 6 +- .../testscripts/check_key_expirations.sh | 68 +++++++++++++++++++ 12 files changed, 98 insertions(+), 45 deletions(-) create mode 100755 containers/testscripts/check_key_expirations.sh diff --git a/.ci-scripts/linux_arm64_setup.sh b/.ci-scripts/linux_arm64_setup.sh index 9748521c2b8..1a40c1f06e5 100755 --- a/.ci-scripts/linux_arm64_setup.sh +++ b/.ci-scripts/linux_arm64_setup.sh @@ -26,7 +26,7 @@ echo "capath=/etc/ssl/certs/" >>~/.curlrc curl -sSL https://golang.org/dl/go${GO_VERSION}.linux-arm64.tar.gz -o /tmp/go.tgz && sudo rm -rf /usr/local/go && sudo tar -zxf /tmp/go.tgz -C /usr/local -git clone --branch v1.2.1 https://github.com/bats-core/bats-core.git /tmp/bats-core && pushd /tmp/bats-core >/dev/null && sudo ./install.sh /usr/local +git clone --branch v1.11.0 https://github.com/bats-core/bats-core.git /tmp/bats-core && pushd /tmp/bats-core >/dev/null && sudo ./install.sh /usr/local # Install mkcert sudo curl --fail -JL -s -o /usr/local/bin/mkcert "https://dl.filippo.io/mkcert/latest?for=linux/arm64" && sudo chmod +x /usr/local/bin/mkcert diff --git a/.github/workflows/linux-setup.sh b/.github/workflows/linux-setup.sh index 4cf60f6ab8d..c5261990b82 100755 --- a/.github/workflows/linux-setup.sh +++ b/.github/workflows/linux-setup.sh @@ -35,7 +35,7 @@ done mkcert -install -git clone --branch v1.2.1 https://github.com/bats-core/bats-core.git /tmp/bats-core && pushd /tmp/bats-core >/dev/null && sudo ./install.sh /usr/local >/dev/null +git clone --branch v1.11.0 https://github.com/bats-core/bats-core.git /tmp/bats-core && pushd /tmp/bats-core >/dev/null && sudo ./install.sh /usr/local >/dev/null primary_ip=$(ip route get 1 | awk '{gsub("^.*src ",""); print $1; exit}') diff --git a/containers/ddev-dbserver/test/functions.sh b/containers/ddev-dbserver/test/functions.sh index fa478c3a799..61bb9ed316f 100644 --- a/containers/ddev-dbserver/test/functions.sh +++ b/containers/ddev-dbserver/test/functions.sh @@ -2,7 +2,7 @@ function basic_setup { export CONTAINER_NAME="testserver" - export HOSTPORT=33000 + export HOSTPORT=31000 export MYTMPDIR="${HOME}/tmp/testserver-sh_${RANDOM}_$$" export outdir="${HOME}/tmp/mariadb_testserver/output_${RANDOM}_$$" export VOLUME="dbserver_test-${RANDOM}_$$" diff --git a/containers/ddev-dbserver/test/image_general.bats b/containers/ddev-dbserver/test/image_general.bats index 8479f1fa233..0a0e8bed29e 100644 --- a/containers/ddev-dbserver/test/image_general.bats +++ b/containers/ddev-dbserver/test/image_general.bats @@ -12,22 +12,17 @@ function setup { containercheck } -@test "verify apt keys are not expiring" { - DDEV_MAX_DAYS_BEFORE_CERT_EXPIRATION=${DDEV_MAX_DAYS_BEFORE_CERT_EXPIRATION:-90} +@test "verify apt keys are not expiring within ${DDEV_MAX_DAYS_BEFORE_CERT_EXPIRATION:-90} days" { if [ "${DDEV_IGNORE_EXPIRING_KEYS:-}" = "true" ]; then skip "Skipping because DDEV_IGNORE_EXPIRING_KEYS is set" fi - echo "# DDEV_MAX_DAYS_BEFORE_CERT_EXPIRATION='${DDEV_MAX_DAYS_BEFORE_CERT_EXPIRATION}' DDEV_IGNORE_EXPIRING_KEYS='${DDEV_IGNORE_EXPIRING_KEYS}'" >&3 - docker exec -e "max=${DDEV_MAX_DAYS_BEFORE_CERT_EXPIRATION}" ${CONTAINER_NAME} bash -c ' - dates=$(apt-key list 2>/dev/null | awk "/\[expires/ { gsub(/[\[\]]/, \"\"); print \$6;}") - for item in ${dates}; do - today=$(date -I) - let diff=($(date +%s -d ${item})-$(date +%s -d ${today}))/86400 - if [ ${diff} -le ${max} ]; then - exit 1 - fi - done - ' + if [ "${DB_TYPE:-}" = "mysql" ] && [[ ${DB_VERSION} =~ ^5.[56]$ ]]; then + skip "Skipping mysql:${DB_VERSION} as its keys are long since expired" + fi + + docker cp ${TEST_SCRIPT_DIR}/check_key_expirations.sh ${CONTAINER_NAME}:/tmp + docker exec -u root -e "DDEV_MAX_DAYS_BEFORE_CERT_EXPIRATION=$DDEV_MAX_DAYS_BEFORE_CERT_EXPIRATION" ${CONTAINER_NAME} /tmp/check_key_expirations.sh >&3 + } @test "verify xtrabackup version equal to mysql-server version" { diff --git a/containers/ddev-dbserver/test/test_dbserver.sh b/containers/ddev-dbserver/test/test_dbserver.sh index 5ccc7a1f8bb..b8d587bd6f8 100755 --- a/containers/ddev-dbserver/test/test_dbserver.sh +++ b/containers/ddev-dbserver/test/test_dbserver.sh @@ -2,6 +2,7 @@ # Find the directory of this script DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null && pwd)" +export TEST_SCRIPT_DIR=${DIR}/../../testscripts set -o errexit set -o pipefail diff --git a/containers/ddev-ssh-agent/Dockerfile b/containers/ddev-ssh-agent/Dockerfile index b57a2227d42..026824dd721 100644 --- a/containers/ddev-ssh-agent/Dockerfile +++ b/containers/ddev-ssh-agent/Dockerfile @@ -1,6 +1,6 @@ FROM debian:bullseye-slim -RUN apt-get update && apt-get install -y bash expect file openssh-client socat psmisc && apt-get autoclean +RUN apt-get update && apt-get install -y bash expect file gpg openssh-client socat psmisc && apt-get autoclean # Copy container files COPY files / diff --git a/containers/ddev-ssh-agent/test/functions.sh b/containers/ddev-ssh-agent/test/functions.sh index 20d885c449d..59928112751 100644 --- a/containers/ddev-ssh-agent/test/functions.sh +++ b/containers/ddev-ssh-agent/test/functions.sh @@ -2,7 +2,7 @@ function basic_setup { export CONTAINER_NAME="testserver" - export HOSTPORT=33000 + export HOSTPORT=31000 export MYTMPDIR="${HOME}/tmp/testserver-sh_${RANDOM}_$$" export outdir="${HOME}/tmp/mariadb_testserver/output_${RANDOM}_$$" export VOLUME="dbserver_test-${RANDOM}_$$" diff --git a/containers/ddev-ssh-agent/test/image_general.bats b/containers/ddev-ssh-agent/test/image_general.bats index c71f52b3971..03cd181a631 100644 --- a/containers/ddev-ssh-agent/test/image_general.bats +++ b/containers/ddev-ssh-agent/test/image_general.bats @@ -12,20 +12,11 @@ function setup { containercheck } -@test "verify apt keys are not expiring" { - DDEV_MAX_DAYS_BEFORE_CERT_EXPIRATION=${DDEV_MAX_DAYS_BEFORE_CERT_EXPIRATION:-90} +@test "verify apt keys are not expiring within ${DDEV_MAX_DAYS_BEFORE_CERT_EXPIRATION:-90} days" { if [ "${DDEV_IGNORE_EXPIRING_KEYS:-}" = "true" ]; then skip "Skipping because DDEV_IGNORE_EXPIRING_KEYS is set" fi - docker exec -e "max=$DDEV_MAX_DAYS_BEFORE_CERT_EXPIRATION" ${CONTAINER_NAME} bash -c ' - dates=$(apt-key list 2>/dev/null | awk "/\[expires/ { gsub(/[\[\]]/, \"\"); print \$6;}") - for item in ${dates}; do - today=$(date -I) - let diff=($(date +%s -d ${item})-$(date +%s -d ${today}))/86400 - if [ ${diff} -le ${max} ]; then - exit 1 - fi - done - ' + docker cp ${TEST_SCRIPT_DIR}/check_key_expirations.sh ${CONTAINER_NAME}:/tmp + docker exec -u root -e "DDEV_MAX_DAYS_BEFORE_CERT_EXPIRATION=$DDEV_MAX_DAYS_BEFORE_CERT_EXPIRATION" ${CONTAINER_NAME} /tmp/check_key_expirations.sh >&3 } diff --git a/containers/ddev-ssh-agent/test/test.sh b/containers/ddev-ssh-agent/test/test.sh index 1b817c15fcb..bec721297af 100755 --- a/containers/ddev-ssh-agent/test/test.sh +++ b/containers/ddev-ssh-agent/test/test.sh @@ -1,7 +1,8 @@ #!/bin/bash # Find the directory of this script -DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null && pwd)" +export DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null && pwd)" +export TEST_SCRIPT_DIR=${DIR}/../../testscripts set -o errexit set -o pipefail @@ -17,5 +18,5 @@ export CURRENT_ARCH=$(../get_arch.sh) # /usr/local/bin is added for git-bash, where it may not be in the $PATH. export PATH="/usr/local/bin:$PATH" -bats test || (echo "bats tests failed for IMAGE=${IMAGE}" && exit 2) +bats --show-output-of-passing-tests test || (echo "bats tests failed for IMAGE=${IMAGE}" && exit 2) printf "Test successful for IMAGE=${IMAGE}\n\n" diff --git a/containers/ddev-webserver/tests/ddev-webserver/general.bats b/containers/ddev-webserver/tests/ddev-webserver/general.bats index 9eb81b39441..93f92882caa 100644 --- a/containers/ddev-webserver/tests/ddev-webserver/general.bats +++ b/containers/ddev-webserver/tests/ddev-webserver/general.bats @@ -57,19 +57,12 @@ docker run --rm $DOCKER_IMAGE bash -c 'php --version | grep -v "with Xdebug"' } -@test "verify apt keys are not expiring" { - DDEV_MAX_DAYS_BEFORE_CERT_EXPIRATION=${DDEV_MAX_DAYS_BEFORE_CERT_EXPIRATION:-90} +@test "verify apt keys are not expiring within ${DDEV_MAX_DAYS_BEFORE_CERT_EXPIRATION:-90} days" { if [ "${DDEV_IGNORE_EXPIRING_KEYS:-}" = "true" ]; then skip "Skipping because DDEV_IGNORE_EXPIRING_KEYS is set" fi - docker exec -e "max=$DDEV_MAX_DAYS_BEFORE_CERT_EXPIRATION" ${CONTAINER_NAME} bash -c ' - dates=$(apt-key list 2>/dev/null | awk "/\[expires/ { gsub(/[\[\]]/, \"\"); print \$6;}") - for item in ${dates}; do - today=$(date -I) - let diff=($(date +%s -d ${item})-$(date +%s -d ${today}))/86400 - if [ ${diff} -le ${max} ]; then - exit 1 - fi - done - ' + docker cp ${TEST_SCRIPT_DIR}/check_key_expirations.sh ${CONTAINER_NAME}:/tmp + docker exec -u root -e "DDEV_MAX_DAYS_BEFORE_CERT_EXPIRATION=$DDEV_MAX_DAYS_BEFORE_CERT_EXPIRATION" ${CONTAINER_NAME} /tmp/check_key_expirations.sh >&3 + } + diff --git a/containers/ddev-webserver/tests/ddev-webserver/test.sh b/containers/ddev-webserver/tests/ddev-webserver/test.sh index e1636e1b5d7..3b0a8d5b769 100755 --- a/containers/ddev-webserver/tests/ddev-webserver/test.sh +++ b/containers/ddev-webserver/tests/ddev-webserver/test.sh @@ -12,6 +12,10 @@ if [[ "${DOCKER_REPO}" == "*prod*" ]]; then IS_HARDENED=true fi +# Find the directory of this script +DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null && pwd)" +export TEST_SCRIPT_DIR=${DIR}/../../../testscripts + export HOST_HTTP_PORT="8080" export HOST_HTTPS_PORT="8443" export CONTAINER_HTTP_PORT="80" @@ -71,7 +75,7 @@ if ! containerwait; then echo "=============== Failed containerwait after docker run with DDEV_WEBSERVER_TYPE=${WEBSERVER_TYPE} DDEV_PHP_VERSION=$PHP_VERSION ===================" exit 100 fi -bats tests/ddev-webserver/general.bats +bats --show-output-of-passing-tests tests/ddev-webserver/general.bats cleanup diff --git a/containers/testscripts/check_key_expirations.sh b/containers/testscripts/check_key_expirations.sh new file mode 100755 index 00000000000..70488bdab14 --- /dev/null +++ b/containers/testscripts/check_key_expirations.sh @@ -0,0 +1,68 @@ +#!/bin/bash + +# Test all keyrings to make sure they are not going to expire +# within DDEV_MAX_DAYS_BEFORE_CERT_EXPIRATION days + +set -eu -o pipefail + +# Directories containing keyrings +directories=("/etc/apt/trusted.gpg.d/" "/usr/share/keyrings/") +# Today's date in Unix time +today=$(date +%s) +# Days ahead to check for expiration +days_ahead=${DDEV_MAX_DAYS_BEFORE_CERT_EXPIRATION:-90} +printf "Checking key expirations for ${days_ahead} days ahead\n" +# Seconds ahead (DDEV_MAX_DAYS_BEFORE_CERT_EXPIRATION days) +seconds_ahead=$((days_ahead * 24 * 3600)) + +# Process each directory +for dir in "${directories[@]}"; do + if [ ! -d ${dir} ]; then + echo "Skipping non-existent ${dir}" + continue + fi + echo "Checking directory: $dir" + cd "$dir" + shopt -s nullglob + for keyring in *.{gpg,asc}; do + # Skip specific keyrings + if [[ "$keyring" =~ -removed-keys\.gpg$ ]]; then + continue + fi + + # Prepare keyring path for GPG command + keyring_path="$dir$keyring" + # Determine if temporary keyring is needed (for .asc files) + temp_keyring="/tmp/temp-${keyring%.asc}.gpg" + if [[ "$keyring" == *.asc ]]; then + gpg --dearmor < "$keyring_path" > "$temp_keyring" 2>/dev/null + keyring_to_check="$temp_keyring" + else + keyring_to_check="$keyring_path" + fi + + echo "Checking keyring: $keyring" + gpg --no-default-keyring --keyring "$keyring_to_check" --list-keys --with-colons --fixed-list-mode | \ + awk -F: -v today="$today" -v seconds_ahead="$seconds_ahead" ' + /^pub/ {keyid = $5; expiry_date = $7} + /^uid/ && keyid { + # Calculate the time until expiration only if the key has an expiration date + if (expiry_date == 0 || expiry_date == "") { + print "Key ID:", keyid, "Name:", $10, "has no expiration date" + } else if (expiry_date > 0) { + time_to_expire = expiry_date - today; + print "Key ID:", keyid, "Name:", $10, "Expires in:", int(time_to_expire / 86400), "days" + # Check if the key expires within DDEV_MAX_DAYS_BEFORE_CERT_EXPIRATION days + if (expiry_date != "" && expiry_date > 0 && time_to_expire <= seconds_ahead) { + print "Key ID:", keyid, "Name:", $10, "Expires in:", int(time_to_expire / 86400), "days"; + exit 1; + } + keyid = ""; expiry_date = ""; + } else { + print "Error: Key MESSUP negative:", keyid, "Expiry date:", expiry_date, "Name:", $10, "Expires in:", int(time_to_expire / 86400), "days" + exit 2 + } + } + ' + done +done