From c4df1f7b8bf546f447175ab7a8f7583cb15d9809 Mon Sep 17 00:00:00 2001 From: Yuri Astrakhan Date: Sat, 30 Sep 2023 15:41:42 -0400 Subject: [PATCH] Improve SSL mode testing --- .github/workflows/ci.yml | 37 ++++++++++++++++++++++------ docker-compose.yml | 28 +++++++++++++++++++++ justfile | 29 +++++++++++++++++++--- tests/fixtures/initdb-dc-ssl-cert.sh | 18 ++++++++++++++ tests/test.sh | 11 ++++++--- 5 files changed, 108 insertions(+), 15 deletions(-) create mode 100755 tests/fixtures/initdb-dc-ssl-cert.sh diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d67a1d464..3befbb81b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -337,8 +337,8 @@ jobs: path: tests/output/* retention-days: 5 - test-legacy: - name: Test Legacy DB + test-service: + name: Test with PG ${{ matrix.img_ver }} sslmode=${{ matrix.sslmode }} runs-on: ubuntu-latest needs: [ build ] strategy: @@ -346,16 +346,19 @@ jobs: matrix: include: # These must match the versions of postgres used in the docker-compose.yml - - image: postgis/postgis:11-3.0-alpine + - img_ver: 11-3.0-alpine args: postgres sslmode: disable - - image: postgis/postgis:14-3.3-alpine + - img_ver: 14-3.3-alpine args: postgres sslmode: disable # alpine images don't support SSL, so for this we use the debian images - - image: postgis/postgis:15-3.3 + - img_ver: 15-3.3 args: postgres -c ssl=on -c ssl_cert_file=/etc/ssl/certs/ssl-cert-snakeoil.pem -c ssl_key_file=/etc/ssl/private/ssl-cert-snakeoil.key sslmode: require + - img_ver: 15-3.3 + args: postgres -c ssl=on -c ssl_cert_file=/etc/ssl/certs/ssl-cert-snakeoil.pem -c ssl_key_file=/etc/ssl/private/ssl-cert-snakeoil.key + sslmode: verify-ca env: # PG_* variables are used by psql PGDATABASE: test @@ -364,7 +367,7 @@ jobs: PGPASSWORD: postgres services: postgres: - image: ${{ matrix.image }} + image: postgis/postgis:${{ matrix.img_ver }} ports: # will assign a random free host port - 5432/tcp @@ -382,7 +385,7 @@ jobs: --health-timeout 5s --health-retries 5 --entrypoint sh - ${{ matrix.image }} + postgis/postgis:${{ matrix.img_ver }} -c "exec docker-entrypoint.sh ${{ matrix.args }}" steps: - name: Checkout sources @@ -393,6 +396,13 @@ jobs: run: tests/fixtures/initdb.sh env: PGPORT: ${{ job.services.postgres.ports[5432] }} + - name: Get DB SSL cert (sslmode=verify-*) + if: matrix.sslmode == 'verify-ca' || matrix.sslmode == 'verify-full' + run: | + set -x + mkdir -p target/certs + docker cp ${{ job.services.postgres.id }}:/etc/ssl/certs/ssl-cert-snakeoil.pem target/certs/server.crt + docker cp ${{ job.services.postgres.id }}:/etc/ssl/private/ssl-cert-snakeoil.key target/certs/server.key - name: Download build artifact build-x86_64-unknown-linux-gnu uses: actions/download-artifact@v3 with: @@ -400,6 +410,11 @@ jobs: path: target_releases/ - name: Integration Tests run: | + if [[ "${{ matrix.sslmode }}" == "verify-ca" ]]; then + export PGSSLROOTCERT=target/certs/server.crt + elif [[ "${{ matrix.sslmode }}" == "verify-full" ]]; then + export PGSSLROOTCERT=target/certs/server.crt + fi export MARTIN_BUILD=- export MARTIN_BIN=target_releases/martin export MBTILES_BUILD=- @@ -417,6 +432,11 @@ jobs: - name: Tests Debian package run: | sudo dpkg -i target_releases/debian-x86_64.deb + if [[ "${{ matrix.sslmode }}" == "verify-ca" ]]; then + export PGSSLROOTCERT=target/certs/server.crt + elif [[ "${{ matrix.sslmode }}" == "verify-full" ]]; then + export PGSSLROOTCERT=target/certs/server.crt + fi export MARTIN_BUILD=- export MARTIN_BIN=/usr/bin/martin export MBTILES_BUILD=- @@ -427,6 +447,7 @@ jobs: env: DATABASE_URL: postgres://${{ env.PGUSER }}:${{ env.PGUSER }}@${{ env.PGHOST }}:${{ job.services.postgres.ports[5432] }}/${{ env.PGDATABASE }}?sslmode=${{ matrix.sslmode }} - name: Unit Tests + if: matrix.sslmode != 'verify-ca' && matrix.sslmode != 'verify-full' run: | echo "Running unit tests, connecting to DATABASE_URL=$DATABASE_URL" echo "Same but as base64 to prevent GitHub obfuscation (this is not a secret):" @@ -447,7 +468,7 @@ jobs: package: name: Package ${{ matrix.target }} runs-on: ${{ matrix.os }} - needs: [ docker, test, test-legacy ] + needs: [ docker, test, test-service ] strategy: fail-fast: true matrix: diff --git a/docker-compose.yml b/docker-compose.yml index bda054125..04754d296 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -75,6 +75,34 @@ services: - ./tests/fixtures/initdb-dc-ssl.sh:/docker-entrypoint-initdb.d/10_martin.sh - ./tests/fixtures/initdb-dc.sh:/docker-entrypoint-initdb.d/20_martin.sh + db-ssl-cert: + # This should match the version of postgres used in the CI workflow + image: postgis/postgis:15-3.3 + command: + - "postgres" + - "-c" + - "ssl=on" + - "-c" + - "ssl_cert_file=/etc/ssl/certs/ssl-cert-snakeoil.pem" + - "-c" + - "ssl_key_file=/etc/ssl/private/ssl-cert-snakeoil.key" + restart: unless-stopped + ports: + - "${PGPORT:-5411}:5432" + environment: + # POSTGRES_* variables are used by the postgis/postgres image + # PG_* variables are used by psql + - POSTGRES_DB=db + - POSTGRES_USER=postgres + - POSTGRES_PASSWORD=postgres + - PGDATABASE=db + - PGUSER=postgres + - PGPASSWORD=postgres + volumes: + - ./tests/fixtures:/fixtures + - ./tests/fixtures/initdb-dc-ssl-cert.sh:/docker-entrypoint-initdb.d/10_martin.sh + - ./tests/fixtures/initdb-dc.sh:/docker-entrypoint-initdb.d/20_martin.sh + db-legacy: # This should match the version of postgres used in the CI workflow image: postgis/postgis:11-3.0-alpine diff --git a/justfile b/justfile index 08799320e..b5484a2e5 100644 --- a/justfile +++ b/justfile @@ -43,18 +43,25 @@ clean-test: rm -rf tests/output # Start a test database -start: (docker-up "db") +start: (docker-up "db") docker-is-ready # Start an ssl-enabled test database -start-ssl: (docker-up "db-ssl") +start-ssl: (docker-up "db-ssl") docker-is-ready + +# Start an ssl-enabled test database that requires a client certificate +start-ssl-cert: (docker-up "db-ssl-cert") docker-is-ready # Start a legacy test database -start-legacy: (docker-up "db-legacy") +start-legacy: (docker-up "db-legacy") docker-is-ready # Start a specific test database, e.g. db or db-legacy [private] docker-up name: docker-compose up -d {{ name }} + +# Wait for the test database to be ready +[private] +docker-is-ready: docker-compose run -T --rm db-is-ready alias _down := stop @@ -87,6 +94,22 @@ test: start test-unit test-int test-ssl: start-ssl test-unit clean-test tests/test.sh +# Run all tests using an SSL connection with client cert to a test database. Expected output won't match. +test-ssl-cert: start-ssl-cert + #!/usr/bin/env bash + set -euxo pipefail + # copy client cert to the tests folder from the docker container + KEY_DIR=target/certs + mkdir -p $KEY_DIR + docker cp martin-db-ssl-cert-1:/etc/ssl/certs/ssl-cert-snakeoil.pem $KEY_DIR/ssl-cert-snakeoil.pem + docker cp martin-db-ssl-cert-1:/etc/ssl/private/ssl-cert-snakeoil.key $KEY_DIR/ssl-cert-snakeoil.key + # export DATABASE_URL="$DATABASE_URL?sslmode=verify-full&sslrootcert=$KEY_DIR/ssl-cert-snakeoil.pem&sslcert=$KEY_DIR/ssl-cert-snakeoil.pem&sslkey=$KEY_DIR/ssl-cert-snakeoil.key" + export PGSSLROOTCERT="$KEY_DIR/ssl-cert-snakeoil.pem" + export PGSSLCERT="$KEY_DIR/ssl-cert-snakeoil.pem" + export PGSSLKEY="$KEY_DIR/ssl-cert-snakeoil.key" + {{just_executable()}} test-unit clean-test + tests/test.sh + # Run all tests using the oldest supported version of the database test-legacy: start-legacy test-unit test-int diff --git a/tests/fixtures/initdb-dc-ssl-cert.sh b/tests/fixtures/initdb-dc-ssl-cert.sh new file mode 100755 index 000000000..84ba461b9 --- /dev/null +++ b/tests/fixtures/initdb-dc-ssl-cert.sh @@ -0,0 +1,18 @@ +#!/usr/bin/env sh +set -e + +mv /var/lib/postgresql/data/pg_hba.conf /var/lib/postgresql/data/pg_hba.conf.bak +cat > /var/lib/postgresql/data/pg_hba.conf <&1 | tee test_log_1.txt & +$MARTIN_BIN "${ARG[@]}" 2>&1 | tee "${TMP_DIR}/test_log_1.txt" & PROCESS_ID=`jobs -p` { set +x; } 2> /dev/null @@ -227,7 +230,7 @@ test_pbf mb_mvt_2_3_1 world_cities/2/3/1 test_pbf points_empty_srid_0_0_0 points_empty_srid/0/0/0 kill_process $PROCESS_ID -validate_log test_log_1.txt +validate_log "${TMP_DIR}/test_log_1.txt" echo "------------------------------------------------------------------------------------------------------------------------" @@ -237,7 +240,7 @@ mkdir -p "$TEST_OUT_DIR" ARG=(--config tests/config.yaml --max-feature-count 1000 --save-config "$(dirname "$0")/output/given_config.yaml" -W 1) set -x -$MARTIN_BIN "${ARG[@]}" 2>&1 | tee test_log_2.txt & +$MARTIN_BIN "${ARG[@]}" 2>&1 | tee "${TMP_DIR}/test_log_2.txt" & PROCESS_ID=`jobs -p` { set +x; } 2> /dev/null trap "kill -9 $PROCESS_ID 2> /dev/null || true" EXIT @@ -266,7 +269,7 @@ test_jsn spr_cmp_2x sprite/src1,mysrc@2x.json test_png spr_cmp_2x sprite/src1,mysrc@2x.png kill_process $PROCESS_ID -validate_log test_log_2.txt +validate_log "${TMP_DIR}/test_log_2.txt" remove_line "$(dirname "$0")/output/given_config.yaml" " connection_string: " remove_line "$(dirname "$0")/output/generated_config.yaml" " connection_string: "