From df334254c61a18e26ccf3ca28d9fc7a972d6f285 Mon Sep 17 00:00:00 2001 From: Jeremy Tubongbanua <79019866+JeremyTubongbanua@users.noreply.github.com> Date: Thu, 14 Sep 2023 09:52:53 -0400 Subject: [PATCH] ci: alternate_port_test, sshrvd healthcheck, prod_tests (#468) * refactor: prod test secrets * ci: prod tests ensure unique device name * ci: try [] for test health check * refactor: image /atsign -> ${HOMEDIR} * ci: health check params * docs: setup-sshrvd-entrypoint.sh * refactor: sshnp/npd/rvd.log * feat: healthcheck for sshrvd * refactor: sshnpd.log for healthcheck sshnpd * ci: healthcheck * ci: healthcheck condition * ci: add condition service healthy to alternate_port_test and prod_tests * ci: fix healthchecks for sshrvd * ci: fix service health/started * ci: maybe no quotes * ci: 5 min * ci: reduce retries sshrvd healthcheck * ci: refactor sshrvd atsign env var * refactor: names of action & job * ci: &> sshrvd redirection log * ci: * ci: * ci: sad * ci: use `d` token for prod tests unique device name * ci: prod tests now use device name env var * chore: format prod_tests --- .github/workflows/end2end_tests.yaml | 9 ++-- .github/workflows/prod_tests.yaml | 44 ++++++++++++------- .../_init_/setup-sshrvd-entrypoint.sh | 2 +- .../entrypoints/sshnp_entrypoint.sh | 16 +++---- .../entrypoints/sshnp_installer_entrypoint.sh | 16 +++---- .../entrypoints/sshnpd_entrypoint.sh | 2 +- .../sshnpd_installer_entrypoint.sh | 2 +- .../entrypoints/sshrvd_entrypoint.sh | 4 +- tests/end2end_tests/image/Dockerfile | 10 ++--- .../tests/service-container-sshnpd.yaml | 4 +- .../tests/service-container-sshrvd.yaml | 6 +++ 11 files changed, 67 insertions(+), 48 deletions(-) diff --git a/.github/workflows/end2end_tests.yaml b/.github/workflows/end2end_tests.yaml index 64567f439..b61e54b3d 100644 --- a/.github/workflows/end2end_tests.yaml +++ b/.github/workflows/end2end_tests.yaml @@ -563,12 +563,15 @@ jobs: cat service-container-sshnp.yaml >> docker-compose.yaml echo ' image: atsigncompany/sshnp-e2e-runtime:local' >> docker-compose.yaml echo ' depends_on:' >> docker-compose.yaml - echo ' - image-runtime-local' >> docker-compose.yaml - echo ' - container-sshnpd' >> docker-compose.yaml + echo ' image-runtime-local:' >> docker-compose.yaml + echo ' condition: service_started' >> docker-compose.yaml + echo ' container-sshnpd:' >> docker-compose.yaml + echo ' condition: service_healthy' >> docker-compose.yaml cat service-container-sshnpd.yaml >> docker-compose.yaml echo ' image: atsigncompany/sshnp-e2e-runtime:local' >> docker-compose.yaml echo ' depends_on:' >> docker-compose.yaml - echo ' - image-runtime-local' >> docker-compose.yaml + echo ' image-runtime-local:' >> docker-compose.yaml + echo ' condition: service_started' >> docker-compose.yaml - name: docker-compose.yaml if: always() diff --git a/.github/workflows/prod_tests.yaml b/.github/workflows/prod_tests.yaml index f97fd2a77..468c30c75 100644 --- a/.github/workflows/prod_tests.yaml +++ b/.github/workflows/prod_tests.yaml @@ -10,19 +10,16 @@ on: env: SSHNP_ATSIGN: "@8incanteater" - SSHNP_ATSIGN_KEYS: ${{ secrets.ATKEYS_8INCANTEATER }} SSHNPD_ATSIGN: "@8052simple" - SSHNPD_ATSIGN_KEYS: ${{ secrets.ATKEYS_8052SIMPLE }} SSHRVD_ATSIGN: "@8485wealthy51" - SSHRVD_ATSIGN_KEYS: ${{ secrets.ATKEYS_8485WEALTHY51 }} SSHRVD_AM_ATSIGN: "@rv_am" SSHRVD_AP_ATSIGN: "@rv_ap" SSHRVD_EU_ATSIGN: "@rv_eu" DOCKER_COMPOSE_BUILD_CMD: "docker compose build" - DOCKER_COMPOSE_UP_CMD: "docker compose up --abort-on-container-exit --timeout 900" + DOCKER_COMPOSE_UP_CMD: "docker compose up --abort-on-container-exit" jobs: - e2e_test: + prod_rvd_test: runs-on: ubuntu-latest strategy: matrix: @@ -51,24 +48,33 @@ jobs: SSHNPD_ATKEYS="$(tr '[:lower:]' '[:upper:]' <<< '${{ env.SSHNPD_ATSIGN }}')" echo "SSHNPD_ATKEYS=ATKEYS_${SSHNPD_ATKEYS:1}" >> $GITHUB_ENV + SSHRVD_ATKEYS="$(tr '[:lower:]' '[:upper:]' <<< '${{ env.SSHRVD_ATSIGN }}')" + echo "SSHRVD_ATKEYS=ATKEYS_${SSHRVD_ATKEYS:1}" >> $GITHUB_ENV + - name: Setup NP/NPD keys working-directory: tests/end2end_tests/contexts run: | echo "${{ secrets[env.SSHNP_ATKEYS] }}" > sshnp/.atsign/keys/${{ env.SSHNP_ATSIGN }}_key.atKeys echo "${{ secrets[env.SSHNPD_ATKEYS] }}" > sshnpd/.atsign/keys/${{ env.SSHNPD_ATSIGN }}_key.atKeys + - name: Setup Devicename + # First two guarantee a unique # per workflow call + # Last two guarantee a unique # per job per strategy in matrix + run: | + echo "DEVICENAME=${{ github.run_id }}${{ github.run_attempt }}p${{ strategy.job-index }}" >> $GITHUB_ENV + - name: Set up NP/NPD entrypoints working-directory: tests/end2end_tests/contexts/_init_ run: | ./setup-sshnp-entrypoint.sh \ - ${{ github.run_id }}${{ github.run_attempt }}${{ strategy.job-index }} \ + ${{ env.DEVICENAME }} \ ${{ env.SSHNP_ATSIGN }} \ ${{ env.SSHNPD_ATSIGN }} \ ${{ matrix.rvd }} \ sshnp_entrypoint.sh ./setup-sshnpd-entrypoint.sh \ - ${{ github.run_id }}${{ github.run_attempt }}${{ strategy.job-index }} \ + ${{ env.DEVICENAME }} \ ${{ env.SSHNP_ATSIGN }} \ ${{ env.SSHNPD_ATSIGN }} \ sshnpd_entrypoint.sh @@ -78,7 +84,7 @@ jobs: working-directory: tests/end2end_tests run: | # setup keys - echo "${{ env.SSHRVD_ATSIGN_KEYS }}" > contexts/sshrvd/.atsign/keys/${{ env.SSHRVD_ATSIGN }}_key.atKeys + echo "${{ secrets[env.SSHRVD_ATKEYS] }}" > contexts/sshrvd/.atsign/keys/${{ env.SSHRVD_ATSIGN }}_key.atKeys # set up sshrvd entrypoint cd contexts/_init_ @@ -104,17 +110,22 @@ jobs: cat service-container-sshnp.yaml >> docker-compose.yaml echo " image: atsigncompany/sshnp-e2e-runtime:latest" >> docker-compose.yaml echo " depends_on:" >> docker-compose.yaml - echo " - image-runtime-release" >> docker-compose.yaml - echo " - container-sshnpd" >> docker-compose.yaml - if [ "${{ matrix.rvd }}" == "@8485wealthy51" ]; then - echo " - container-sshrvd" >> docker-compose.yaml + echo " image-runtime-release:" >> docker-compose.yaml + echo " condition: service_started" >> docker-compose.yaml + echo " container-sshnpd:" >> docker-compose.yaml + echo " condition: service_healthy" >> docker-compose.yaml + if [ "${{ matrix.rvd }}" == "${{ env.SSHRVD_ATSIGN }}" ]; then + echo " container-sshrvd:" >> docker-compose.yaml + echo " condition: service_healthy" >> docker-compose.yaml fi cat service-container-sshnpd.yaml >> docker-compose.yaml echo " image: atsigncompany/sshnp-e2e-runtime:latest" >> docker-compose.yaml echo " depends_on:" >> docker-compose.yaml - echo " - image-runtime-release" >> docker-compose.yaml - if [ "${{ matrix.rvd }}" == "@8485wealthy51" ]; then - echo " - container-sshrvd" >> docker-compose.yaml + echo " image-runtime-release:" >> docker-compose.yaml + echo " condition: service_started" >> docker-compose.yaml + if [ "${{ matrix.rvd }}" == "${{ env.SSHRVD_ATSIGN }}" ]; then + echo " container-sshrvd:" >> docker-compose.yaml + echo " condition: service_healthy" >> docker-compose.yaml fi - name: Add RVD service to docker-compose.yaml @@ -124,7 +135,8 @@ jobs: cat service-container-sshrvd.yaml >> docker-compose.yaml echo " image: atsigncompany/sshnp-e2e-runtime:latest" >> docker-compose.yaml echo " depends_on:" >> docker-compose.yaml - echo " - image-runtime-release" >> docker-compose.yaml + echo " image-runtime-release:" >> docker-compose.yaml + echo " condition: service_started" >> docker-compose.yaml - name: docker-compose.yaml working-directory: tests/end2end_tests/tests diff --git a/tests/end2end_tests/contexts/_init_/setup-sshrvd-entrypoint.sh b/tests/end2end_tests/contexts/_init_/setup-sshrvd-entrypoint.sh index bc782c786..743c7cb9e 100755 --- a/tests/end2end_tests/contexts/_init_/setup-sshrvd-entrypoint.sh +++ b/tests/end2end_tests/contexts/_init_/setup-sshrvd-entrypoint.sh @@ -5,7 +5,7 @@ # example usage: ./setup-sshrvd-entrypoint.sh @alice sshrvd=$1 # e.g. @alice -template_name=$2 +template_name=$2 # e.g. "sshrvd_entrypoint.sh" cp ../../entrypoints/"$template_name" ../sshrvd/entrypoint.sh # copy template to the mounted folder diff --git a/tests/end2end_tests/entrypoints/sshnp_entrypoint.sh b/tests/end2end_tests/entrypoints/sshnp_entrypoint.sh index 2d88226d0..4ca29872e 100644 --- a/tests/end2end_tests/entrypoints/sshnp_entrypoint.sh +++ b/tests/end2end_tests/entrypoints/sshnp_entrypoint.sh @@ -1,21 +1,21 @@ #!/bin/bash echo "SSHNP START ENTRY" -SSHNP_COMMAND="$HOME/.local/bin/sshnp -f @sshnpatsign -t @sshnpdatsign -d deviceName -h @sshrvdatsign -s id_ed25519.pub -v > logs.txt" +SSHNP_COMMAND="$HOME/.local/bin/sshnp -f @sshnpatsign -t @sshnpdatsign -d deviceName -h @sshrvdatsign -s id_ed25519.pub -v > sshnp.log" echo "Running: $SSHNP_COMMAND" eval "$SSHNP_COMMAND" -cat logs.txt -tail -n 5 logs.txt | grep "ssh -p" > sshcommand.txt +cat sshnp.log +tail -n 5 sshnp.log | grep "ssh -p" > sshcommand.txt if [ ! -s sshcommand.txt ]; then # try again echo "Running: $SSHNP_COMMAND" eval "$SSHNP_COMMAND" - cat logs.txt - tail -n 5 logs.txt | grep "ssh -p" > sshcommand.txt + cat sshnp.log + tail -n 5 sshnp.log | grep "ssh -p" > sshcommand.txt if [ ! -s sshcommand.txt ]; then - echo "could not find 'ssh -p' command in logs.txt" - echo "last 5 lines of logs.txt:" - tail -n 5 logs.txt || echo + echo "could not find 'ssh -p' command in sshnp.log" + echo "last 5 lines of sshnp.log:" + tail -n 5 sshnp.log || echo exit 1 fi fi diff --git a/tests/end2end_tests/entrypoints/sshnp_installer_entrypoint.sh b/tests/end2end_tests/entrypoints/sshnp_installer_entrypoint.sh index 0f35ee512..97ea189b8 100644 --- a/tests/end2end_tests/entrypoints/sshnp_installer_entrypoint.sh +++ b/tests/end2end_tests/entrypoints/sshnp_installer_entrypoint.sh @@ -1,22 +1,22 @@ #!/bin/bash sleep WAITING_TIME # time for sshnpd to share device name -SSHNP_COMMAND="$HOME/.local/bin/sshnp -f @sshnpatsign -t @sshnpdatsign -d deviceName -h @sshrvdatsign -s id_ed25519.pub -v > logs.txt" +SSHNP_COMMAND="$HOME/.local/bin/sshnp -f @sshnpatsign -t @sshnpdatsign -d deviceName -h @sshrvdatsign -s id_ed25519.pub -v > sshnp.log" echo "Running: $SSHNP_COMMAND" eval "$SSHNP_COMMAND" -cat logs.txt -tail -n 5 logs.txt | grep "ssh -p" > sshcommand.txt +cat sshnp.log +tail -n 5 sshnp.log | grep "ssh -p" > sshcommand.txt if [ ! -s sshcommand.txt ]; then # try again echo "Running: $SSHNP_COMMAND" eval "$SSHNP_COMMAND" - cat logs.txt - tail -n 5 logs.txt | grep "ssh -p" > sshcommand.txt + cat sshnp.log + tail -n 5 sshnp.log | grep "ssh -p" > sshcommand.txt if [ ! -s sshcommand.txt ]; then - echo "could not find 'ssh -p' command in logs.txt" - echo "last 5 lines of logs.txt:" - tail -n 5 logs.txt || echo + echo "could not find 'ssh -p' command in sshnp.log" + echo "last 5 lines of sshnp.log:" + tail -n 5 sshnp.log || echo exit 1 fi fi diff --git a/tests/end2end_tests/entrypoints/sshnpd_entrypoint.sh b/tests/end2end_tests/entrypoints/sshnpd_entrypoint.sh index 834bca8b8..16341ec0d 100644 --- a/tests/end2end_tests/entrypoints/sshnpd_entrypoint.sh +++ b/tests/end2end_tests/entrypoints/sshnpd_entrypoint.sh @@ -1,5 +1,5 @@ #!/bin/bash echo "SSHNPD START ENTRY" -SSHNPD_COMMAND="$HOME/.local/bin/sshnpd -a @sshnpdatsign -m @sshnpatsign -d deviceName -s -u -v 2>&1 | tee all.txt" +SSHNPD_COMMAND="$HOME/.local/bin/sshnpd -a @sshnpdatsign -m @sshnpatsign -d deviceName -s -u -v 2>&1 | tee -a sshnpd.log" echo "Running: $SSHNPD_COMMAND" eval "$SSHNPD_COMMAND" diff --git a/tests/end2end_tests/entrypoints/sshnpd_installer_entrypoint.sh b/tests/end2end_tests/entrypoints/sshnpd_installer_entrypoint.sh index 237344874..63bce9eb1 100644 --- a/tests/end2end_tests/entrypoints/sshnpd_installer_entrypoint.sh +++ b/tests/end2end_tests/entrypoints/sshnpd_installer_entrypoint.sh @@ -1,4 +1,4 @@ #!/bin/bash -SSHNPD_COMMAND="$HOME/.local/bin/sshnpd@sshnpatsign 2>&1 | tee all.txt" +SSHNPD_COMMAND="$HOME/.local/bin/sshnpd@sshnpatsign 2>&1 | tee -a sshnpd.log" echo "Running: $SSHNPD_COMMAND" eval "$SSHNPD_COMMAND" \ No newline at end of file diff --git a/tests/end2end_tests/entrypoints/sshrvd_entrypoint.sh b/tests/end2end_tests/entrypoints/sshrvd_entrypoint.sh index bc4cc0371..c41a11bc8 100644 --- a/tests/end2end_tests/entrypoints/sshrvd_entrypoint.sh +++ b/tests/end2end_tests/entrypoints/sshrvd_entrypoint.sh @@ -1,4 +1,2 @@ #!/bin/bash -"$HOME"/.local/bin/sshrvd -a @sshrvdatsign -i "$(hostname -i)" -v -s -sleep 60 # sleep 60 because other containers depend on it. And if it's not being used (let's say you're using @rv_am), then it will just sleep until sshnp exits -exit 0 +"$HOME"/.local/bin/sshrvd -a @sshrvdatsign -i "$(hostname -i)" -v -s 2>&1 | tee -a sshrvd.log diff --git a/tests/end2end_tests/image/Dockerfile b/tests/end2end_tests/image/Dockerfile index 50956536e..ee0822dbb 100644 --- a/tests/end2end_tests/image/Dockerfile +++ b/tests/end2end_tests/image/Dockerfile @@ -57,7 +57,7 @@ WORKDIR ${HOMEDIR} USER ${USER} -ENTRYPOINT cp -r /mount/. /atsign && sudo service ssh start && sh ${HOMEDIR}/entrypoint.sh +ENTRYPOINT cp -r /mount/. ${HOMEDIR} && sudo service ssh start && sh ${HOMEDIR}/entrypoint.sh # END BRANCH # LOCAL @@ -90,7 +90,7 @@ WORKDIR ${HOMEDIR} USER ${USER} -ENTRYPOINT cp -r /mount/. /atsign && sudo service ssh start && sh ${HOMEDIR}/entrypoint.sh +ENTRYPOINT cp -r /mount/. ${HOMEDIR} && sudo service ssh start && sh ${HOMEDIR}/entrypoint.sh # END LOCAL # RELEASE @@ -139,7 +139,7 @@ WORKDIR ${HOMEDIR} USER ${USER} -ENTRYPOINT cp -r /mount/. /atsign && sudo service ssh start && sh ${HOMEDIR}/entrypoint.sh +ENTRYPOINT cp -r /mount/. ${HOMEDIR} && sudo service ssh start && sh ${HOMEDIR}/entrypoint.sh # END RELEASE # SSHNP INSTALLER @@ -170,7 +170,7 @@ FROM build-sshnp-installer AS runtime-sshnp-installer USER ${USER} WORKDIR ${HOMEDIR} -ENTRYPOINT cp -r /mount/. /atsign && sudo service ssh start && sh ${HOMEDIR}/entrypoint.sh +ENTRYPOINT cp -r /mount/. ${HOMEDIR} && sudo service ssh start && sh ${HOMEDIR}/entrypoint.sh # END SSHNP INSTALLER # SSHNPD INSTALLER @@ -204,7 +204,7 @@ FROM build-sshnpd-installer AS runtime-sshnpd-installer USER ${USER} WORKDIR ${HOMEDIR} -ENTRYPOINT cp -r /mount/. /atsign && sudo service ssh start && sh ${HOMEDIR}/entrypoint.sh +ENTRYPOINT cp -r /mount/. ${HOMEDIR} && sudo service ssh start && sh ${HOMEDIR}/entrypoint.sh # END SSHNPD INSTALLER # MANUAL diff --git a/tests/end2end_tests/tests/service-container-sshnpd.yaml b/tests/end2end_tests/tests/service-container-sshnpd.yaml index bf019c46a..17f319fd9 100644 --- a/tests/end2end_tests/tests/service-container-sshnpd.yaml +++ b/tests/end2end_tests/tests/service-container-sshnpd.yaml @@ -5,11 +5,11 @@ networks: - sshnpd healthcheck: - test: grep -Eq "monitor started for @" /atsign/all.txt + test: ["CMD", "grep", "-Eq", "monitor started for @", "/atsign/sshnpd.log"] start_period: 10s # Wait 10 seconds before checking interval: 5s # Check every 5 seconds timeout: 1s # If a check takes longer than a second, consider it a failed check - retries: 180 # Retry the check 180 times (180 * 5s = 15 mins) + retries: 36 # Retry the check n times # auto added: # - image # - depends_on: (sshrvd + runtime service) diff --git a/tests/end2end_tests/tests/service-container-sshrvd.yaml b/tests/end2end_tests/tests/service-container-sshrvd.yaml index a3cc54c42..ee3da03cf 100644 --- a/tests/end2end_tests/tests/service-container-sshrvd.yaml +++ b/tests/end2end_tests/tests/service-container-sshrvd.yaml @@ -3,6 +3,12 @@ volumes: - ../contexts/sshrvd:/mount network_mode: host + healthcheck: + test: ["CMD", "grep", "-Eq", "monitor started for @", "/atsign/sshrvd.log"] + start_period: 10s # Wait 10 seconds before checking + interval: 5s # Check every 5 seconds + timeout: 1s # If a check takes longer than a second, consider it a failed check + retries: 36 # Retry the check n times (180 * 5s = 15 mins) # auto added: # - image # - depends_on: (runtime service)