Concurrent essential and extended test workflows

stellar · Nov 10, 2023 · e01e72d · e01e72d
1 parent 8e595ea
commit e01e72d
Show file tree

Hide file tree

Showing 13 changed files with 439 additions and 208 deletions.
diff --git a/.github/workflows/on_pull_request.yml b/.github/workflows/on_pull_request.yml
@@ -0,0 +1,33 @@
+name: On Pull Request Workflow
+
+on:
+  pull_request:
+
+jobs:
+  gradle_build:
+    uses: ./.github/workflows/sub_gradle_build.yml
+
+  jacoco_report:
+    needs: [ gradle_build ]
+    uses: ./.github/workflows/sub_jacoco_report.yml
+    with:
+      forceRebuild: true
+
+  essential_tests:
+    needs: [ gradle_build ]
+    uses: ./.github/workflows/sub_essential_tests.yml
+
+  extended_tests:
+    needs: [ gradle_build ]
+    uses: ./.github/workflows/sub_extended_tests.yml
+
+  codeql_analysis:
+    uses: ./.github/workflows/sub_codeql_analysis.yml
+
+  complete:
+    if: always()
+    needs: [ gradle_build, essential_tests, extended_tests, jacoco_report, codeql_analysis ]
+    runs-on: ubuntu-22.04
+    steps:
+      - if: contains(needs.*.result, 'failure') || contains(needs.*.result, 'cancelled')
+        run: exit 1
diff --git a/.github/workflows/on_pull_request_comments.yml b/.github/workflows/on_pull_request_comments.yml
@@ -0,0 +1,24 @@
+name: Pull Request Comments
+
+on:
+  issue_comment:
+    types: [ created, edited ]
+
+jobs:
+  run-extended-tests:
+    if: github.event.issue.pull_request && contains(github.event.comment.body, '/run-extended-tests')
+    uses: ./.github/workflows/sub_extended_tests.yml
+
+  run-essential-tests:
+    if: github.event.issue.pull_request && contains(github.event.comment.body, '/run-essential-tests')
+    uses: ./.github/workflows/sub_essential_tests.yml
+
+  run-codeql-analysis:
+    if: github.event.issue.pull_request && contains(github.event.comment.body, '/run-codeql-analysis')
+    uses: ./.github/workflows/sub_codeql_analysis.yml
+
+  run-jacoco-report:
+    if: github.event.issue.pull_request && contains(github.event.comment.body, '/run-jacoco-report')
+    uses: ./.github/workflows/sub_jacoco_report.yml
+    with:
+      forceRebuild: true
diff --git a/.github/workflows/wk_push_to_develop.yml → .github/workflows/on_push_to_develop.yml b/.github/workflows/wk_push_to_develop.yml → .github/workflows/on_push_to_develop.yml
@@ -7,12 +7,27 @@ name: Push/Merge to `develop` Branch
 
 on:
   push:
-    # when commits are pushed or merged onto `develop` branch
+    # when commits are pushed or pull requests merged onto `develop` branch
     branches: [ develop ]
 
 jobs:
-  gradle_test_and_build:
-    uses: ./.github/workflows/sub_gradle_test_and_build.yml
+  gradle_build:
+    uses: ./.github/workflows/sub_gradle_build.yml
+
+  jacoco_report:
+    needs: [ gradle_build ]
+    uses: ./.github/workflows/sub_jacoco_report.yml
+
+  essential_tests:
+    needs: [ gradle_build ]
+    uses: ./.github/workflows/sub_essential_tests.yml
+
+  extended_tests:
+    needs: [ gradle_build ]
+    uses: ./.github/workflows/sub_extended_tests.yml
+
+  codeql_analysis:
+    uses: ./.github/workflows/sub_codeql_analysis.yml
 
   build_and_push_docker_image:
     # stellar/anchor-platform:edge-${{ steps.get_date.outputs.date }}-${{ steps.get_sha.outputs.SHA }}
@@ -45,24 +60,23 @@ jobs:
           tags: stellar/anchor-platform:edge,stellar/anchor-platform:edge-${{ steps.get_date.outputs.DATE }}-${{ steps.get_sha.outputs.SHA }}
           file: Dockerfile
 
-  # TODO: enable purge-image when we got the dockerhub token
-  #  purge-image:
-  #    name: Purge Docker edge Images (tags=edge-*)
-  #    runs-on: ubuntu-22.04
-  #    steps:
-  #      - name: Delete image
-  #        uses: bots-house/[email protected]
-  #        with:
-  #          owner: stellarproducteng
-  #          name: ${{ secrets.DOCKERHUB_USERNAME }}
-  #          token: ${{ secrets.DOCKERHUB_TOKEN }}
-  #          tag: edge-*
+    # TODO: enable purge-image when we got the dockerhub token
+    #  purge-image:
+    #    name: Purge Docker edge Images (tags=edge-*)
+    #    runs-on: ubuntu-22.04
+    #    steps:
+    #      - name: Delete image
+    #        uses: bots-house/[email protected]
+    #        with:
+    #          owner: stellarproducteng
+    #          name: ${{ secrets.DOCKERHUB_USERNAME }}
+    #          token: ${{ secrets.DOCKERHUB_TOKEN }}
+    #          tag: edge-*
 
   complete:
     if: always()
-    # TODO: add sep_validation_suite when Jenkins develop build is complete
-    needs: [ gradle_test_and_build, build_and_push_docker_image ]
+    needs: [ essential_tests, extended_tests, build_and_push_docker_image ]
     runs-on: ubuntu-latest
     steps:
       - if: contains(needs.*.result, 'failure') || contains(needs.*.result, 'cancelled')
-        run: exit 1
+        run: exit 1
diff --git a/...rkflows/wf_release_created_or_updated.yml → ...rkflows/on_release_created_or_updated.yml b/...rkflows/wf_release_created_or_updated.yml → ...rkflows/on_release_created_or_updated.yml
@@ -6,6 +6,20 @@ on:
     types: [ created, edited ]
 
 jobs:
+  gradle_build:
+    uses: ./.github/workflows/sub_gradle_build.yml
+
+  essential_tests:
+    needs: [ gradle_build ]
+    uses: ./.github/workflows/sub_essential_tests.yml
+
+  extended_tests:
+    needs: [ gradle_build ]
+    uses: ./.github/workflows/sub_extended_tests.yml
+
+  codeql_analysis:
+    uses: ./.github/workflows/sub_codeql_analysis.yml
+
   build_and_push_docker_image:
     name: Push to DockerHub (tag=stellar/anchor-platform:${{ github.event.release.tag_name }})
     runs-on: ubuntu-22.04
@@ -41,9 +55,7 @@ jobs:
 
   complete:
     if: always()
-    # TODO: Enable end_to_end tests after it is fixed.
-    #    needs: [ end_to_end_tests, build_and_push_docker_image ]
-    needs: [ build_and_push_docker_image ]
+    needs: [ essential_tests, extended_tests, build_and_push_docker_image ]
     runs-on: ubuntu-latest
     steps:
       - if: contains(needs.*.result, 'failure') || contains(needs.*.result, 'cancelled')

diff --git a/.github/workflows/sub_codeql_analysis.yml b/.github/workflows/sub_codeql_analysis.yml
@@ -0,0 +1,41 @@
+name: CodeQL Analysis Workflow
+
+on:
+  # allows this workflow to be called from another workflow
+  workflow_dispatch:
+  workflow_call:
+
+jobs:
+  analyze:
+    name: CodeQL Analysis
+    runs-on: ubuntu-latest-4-cores
+    permissions:
+      actions: read
+      contents: read
+      security-events: write
+
+    strategy:
+      fail-fast: false
+      matrix:
+        language: [ 'java-kotlin' ]
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v3
+
+      - name: Set up Gradle properties
+        run: |
+          echo "kotlin.daemon.jvmargs=-Xmx2g" >> gradle.properties
+
+      - name: Initialize CodeQL
+        uses: github/codeql-action/init@v2
+        with:
+          languages: ${{ matrix.language }}
+
+      - name: Autobuild
+        uses: github/codeql-action/autobuild@v2
+
+      - name: Perform CodeQL Analysis
+        uses: github/codeql-action/analyze@v2
+        with:
+          category: "/language:${{matrix.language}}"
diff --git a/.github/workflows/sub_essential_tests.yml b/.github/workflows/sub_essential_tests.yml
@@ -0,0 +1,110 @@
+name: Run Essential Tests Workflow
+
+on:
+  # allows this workflow to be called from another workflow
+  workflow_dispatch:
+  workflow_call:
+
+jobs:
+  essential_tests:
+    name: Run Essential Tests (Integration Tests, Faster End-2-End Tests, Unit Tests, and Stellar Validation Tools)
+    runs-on: ubuntu-latest-16-cores
+    #    runs-on: ubuntu-latest
+    steps:
+      #############################################
+      # Setup JDK 11
+      # Download, and Extract java-stellar-anchor-sdk.tar
+      # Setup hostnames (/etc/hosts)
+      #############################################
+      - name: Set up JDK 11
+        uses: actions/setup-java@v3
+        with:
+          java-version: '11'
+          distribution: 'adopt'
+
+      - name: Download java-stellar-anchor-sdk.tar
+        uses: actions/download-artifact@v3
+        with:
+          name: java-stellar-anchor-sdk-tar
+          path: /home/runner/
+
+      - name: Extract java-stellar-anchor-sdk.tar
+        run: |
+          cd /home/runner
+          tar -xf /home/runner/java-stellar-anchor-sdk.tar
+          cd /home/runner/java-stellar-anchor-sdk
+
+      - name: Set up hostnames (/etc/hosts)
+        run: |
+          sudo echo "127.0.0.1 db" | sudo tee -a /etc/hosts
+          sudo echo "127.0.0.1 zookeeper" | sudo tee -a /etc/hosts
+          sudo echo "127.0.0.1 kafka" | sudo tee -a /etc/hosts
+          sudo echo "127.0.0.1 sep24-reference-ui" | sudo tee -a /etc/hosts
+          sudo echo "127.0.0.1 reference-server" | sudo tee -a /etc/hosts
+          sudo echo "127.0.0.1 wallet-server" | sudo tee -a /etc/hosts
+          sudo echo "127.0.0.1 platform" | sudo tee -a /etc/hosts
+          sudo echo "127.0.0.1 custody-server" | sudo tee -a /etc/hosts
+          sudo echo "127.0.0.1 host.docker.internal" | sudo tee -a /etc/hosts 
+
+      #############################################
+
+
+      - name: Pull Stellar Validation Tests Docker Image
+        run: docker pull stellar/anchor-tests:v0.6.9 &
+
+      - name: Run Zookeeper, Kafka, Postgres, and Sep24 UI with docker-compose
+        env:
+          TEST_PROFILE_NAME: default
+        run: docker-compose -f /home/runner/java-stellar-anchor-sdk/service-runner/src/main/resources/docker-compose-test.yaml up -d --build
+
+      - name: Run sep server, platform server, observer, and reference servers for integration tests
+        env:
+          RUN_DOCKER: false
+          RUN_ALL_SERVERS: false
+          RUN_SEP_SERVER: true
+          RUN_PLATFORM_SERVER: true
+          RUN_OBSERVER: true
+          RUN_KOTLIN_REFERENCE_SERVER: true
+          RUN_EVENT_PROCESSING_SERVER: true
+          RUN_WALLET_SERVER: true
+          SEP1_TOML_VALUE: /home/runner/java-stellar-anchor-sdk/service-runner/src/main/resources/config/stellar.host.docker.internal.toml
+          SEP10_HOME_DOMAIN: host.docker.internal:8080
+        run: |
+          cp /home/runner/java-stellar-anchor-sdk/service-runner/build/libs/anchor-platform-runner-*.jar /home/runner/anchor-platform-runner.jar
+          java -jar /home/runner/anchor-platform-runner.jar -t &
+          echo "PID=$!" >> $GITHUB_ENV
+
+      - name: Wait for the sep server to start and get ready
+        uses: mydea/action-wait-for-api@v1
+        with:
+          url: "http://localhost:8080/.well-known/stellar.toml"
+          expected-status: "200"
+          timeout: "300"
+          interval: "1"
+
+      #
+      # Run Essential Tests
+      #
+      - name: Run Essential Tests
+        env:
+          RUN_DOCKER: false
+          ANCHOR_DOMAIN: http://host.docker.internal:8080
+        run: |
+          cd /home/runner/java-stellar-anchor-sdk
+          ./gradlew test -x extended-tests:test jacocoTestReport --stacktrace -x spotlessApply -x spotlessKotlinApply -x javadoc -x javadocJar -x sourcesJar -x distTar -x distZip -x shadowJar -x shadowDistZip -x shadowDistTar -x bootDistTar -x bootDistZip
+
+      - name: Run Stellar validation tool
+        run: |
+          docker run --network host -v /home/runner/java-stellar-anchor-sdk/platform/src/test/resources://config stellar/anchor-tests:v0.6.9 --home-domain http://host.docker.internal:8080 --seps 1 6 10 12 24 31 38 --sep-config //config/stellar-anchor-tests-sep-config.json --verbose
+
+      - name: Upload Essential Tests Report
+        if: always()
+        uses: actions/upload-artifact@v3
+        with:
+          name: essential-tests-report
+          path: |
+            /home/runner/java-stellar-anchor-sdk/api-schema/build/reports/
+            /home/runner/java-stellar-anchor-sdk/core/build/reports/
+            /home/runner/java-stellar-anchor-sdk/platform/build/reports/
+            /home/runner/java-stellar-anchor-sdk/essential-tests/build/reports/
+        
diff --git a/.github/workflows/sub_extended_tests.yml b/.github/workflows/sub_extended_tests.yml
@@ -0,0 +1,66 @@
+name: Run Extended Tests Workflow
+
+on:
+  # allows this workflow to be called from another workflow
+  workflow_dispatch:
+  workflow_call:
+
+jobs:
+  extended_tests:
+    name: Run Extended Tests
+    runs-on: ubuntu-latest-8-cores
+    steps:
+      #############################################
+      # Setup JDK 11
+      # Download, and Extract java-stellar-anchor-sdk.tar
+      # Setup hostnames (/etc/hosts)
+      #############################################
+      - name: Set up JDK 11
+        uses: actions/setup-java@v3
+        with:
+          java-version: '11'
+          distribution: 'adopt'
+
+      - name: Download java-stellar-anchor-sdk.tar
+        uses: actions/download-artifact@v3
+        with:
+          name: java-stellar-anchor-sdk-tar
+          path: /home/runner/
+
+      - name: Extract java-stellar-anchor-sdk.tar
+        run: |
+          cd /home/runner
+          tar -xf /home/runner/java-stellar-anchor-sdk.tar
+          cd /home/runner/java-stellar-anchor-sdk
+
+      - name: Set up hostnames (/etc/hosts)
+        run: |
+          sudo echo "127.0.0.1 db" | sudo tee -a /etc/hosts
+          sudo echo "127.0.0.1 zookeeper" | sudo tee -a /etc/hosts
+          sudo echo "127.0.0.1 kafka" | sudo tee -a /etc/hosts
+          sudo echo "127.0.0.1 sep24-reference-ui" | sudo tee -a /etc/hosts
+          sudo echo "127.0.0.1 reference-server" | sudo tee -a /etc/hosts
+          sudo echo "127.0.0.1 wallet-server" | sudo tee -a /etc/hosts
+          sudo echo "127.0.0.1 platform" | sudo tee -a /etc/hosts
+          sudo echo "127.0.0.1 custody-server" | sudo tee -a /etc/hosts
+          sudo echo "127.0.0.1 host.docker.internal" | sudo tee -a /etc/hosts 
+
+      #############################################
+
+      - name: Run Extended Tests
+        env:
+          ANCHOR_DOMAIN: http://host.docker.internal:8080
+          SEP1_TOML_VALUE: /home/runner/java-stellar-anchor-sdk/service-runner/src/main/resources/config/stellar.host.docker.internal.toml
+          SEP10_HOME_DOMAIN: host.docker.internal:8080
+        run: |
+          cd /home/runner/java-stellar-anchor-sdk
+          ./gradlew extended-tests:test --stacktrace -x spotlessApply -x spotlessKotlinApply -x javadoc -x javadocJar -x sourcesJar -x distTar -x distZip -x shadowJar -x shadowDistZip -x shadowDistTar -x bootDistTar -x bootDistZip
+
+      - name: Upload Extended Tests Report
+        if: always()
+        uses: actions/upload-artifact@v3
+        with:
+          name: extended-tests-report
+          path: |
+            /home/runner/java-stellar-anchor-sdk/extended-tests/build/reports/
+