[improve][broker] Reduce the pressure from the transaction buffer in rolling restarts #1076
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # | |
| # Licensed to the Apache Software Foundation (ASF) under one | |
| # or more contributor license agreements. See the NOTICE file | |
| # distributed with this work for additional information | |
| # regarding copyright ownership. The ASF licenses this file | |
| # to you under the Apache License, Version 2.0 (the | |
| # "License"); you may not use this file except in compliance | |
| # with the License. You may obtain a copy of the License at | |
| # | |
| # http://www.apache.org/licenses/LICENSE-2.0 | |
| # | |
| # Unless required by applicable law or agreed to in writing, | |
| # software distributed under the License is distributed on an | |
| # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY | |
| # KIND, either express or implied. See the License for the | |
| # specific language governing permissions and limitations | |
| # under the License. | |
| # | |
| name: Pulsar CI | |
| on: | |
| pull_request: | |
| branches: | |
| - master | |
| - branch-* | |
| - pulsar-* | |
| schedule: | |
| # scheduled job with JDK 17 | |
| - cron: '0 12 * * *' | |
| # scheduled job with JDK 21 | |
| # if cron expression is changed, make sure to update the expression in jdk_major_version step in preconditions job | |
| - cron: '0 6 * * *' | |
| workflow_dispatch: | |
| inputs: | |
| collect_coverage: | |
| description: 'Collect test coverage and upload to Codecov' | |
| required: true | |
| type: boolean | |
| default: true | |
| jdk_major_version: | |
| description: 'JDK major version to use for the build' | |
| required: true | |
| type: choice | |
| options: | |
| - '17' | |
| - '21' | |
| default: '17' | |
| trace_test_resource_cleanup: | |
| description: 'Collect thread & heap information before exiting a test JVM. When set to "on", thread dump and heap histogram will be collected. When set to "full", a heap dump will also be collected.' | |
| required: true | |
| type: choice | |
| options: | |
| - 'off' | |
| - 'on' | |
| - 'full' | |
| default: 'off' | |
| thread_leak_detector_wait_millis: | |
| description: 'Duration in ms to wait for threads to exit in thread leak detection between test classes. It is necessary to wait for threads to complete before they are determined to be leaked threads.' | |
| required: true | |
| type: number | |
| default: 10000 | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.ref }}-${{ github.event_name }}${{ github.event_name == 'workflow_dispatch' && github.event.inputs.jdk_major_version || '' }} | |
| cancel-in-progress: true | |
| env: | |
| MAVEN_OPTS: -Xss1500k -Xmx1500m -Daether.connector.http.reuseConnections=false -Daether.connector.requestTimeout=60000 -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.class=standard -Dmaven.wagon.http.retryHandler.count=3 -Dmaven.wagon.http.retryHandler.requestSentEnabled=true -Dmaven.wagon.http.serviceUnavailableRetryStrategy.class=standard -Dmaven.wagon.rto=60000 | |
| # defines the retention period for the intermediate build artifacts needed for rerunning a failed build job | |
| # it's possible to rerun individual failed jobs when the build artifacts are available | |
| # if the artifacts have already been expired, the complete workflow can be rerun by closing and reopening the PR or by rebasing the PR | |
| ARTIFACT_RETENTION_DAYS: 3 | |
| JDK_DISTRIBUTION: corretto | |
| jobs: | |
| preconditions: | |
| name: Preconditions | |
| runs-on: ubuntu-22.04 | |
| outputs: | |
| docs_only: ${{ steps.check_changes.outputs.docs_only }} | |
| changed_tests: ${{ steps.changes.outputs.tests_files }} | |
| need_owasp: ${{ steps.changes.outputs.need_owasp }} | |
| collect_coverage: ${{ steps.check_coverage.outputs.collect_coverage }} | |
| jdk_major_version: ${{ steps.jdk_major_version.outputs.jdk_major_version }} | |
| java_non_tests: ${{ steps.changes.outputs.java_non_tests }} | |
| steps: | |
| - name: Cancel scheduled jobs in forks by default | |
| if: ${{ github.repository != 'apache/pulsar' && github.event_name == 'schedule' }} | |
| uses: actions/github-script@v6 | |
| with: | |
| script: | | |
| await github.rest.actions.cancelWorkflowRun({owner: context.repo.owner, repo: context.repo.repo, run_id: context.runId}); | |
| process.exit(1); | |
| - name: Select JDK major version | |
| id: jdk_major_version | |
| run: | | |
| # use JDK 21 for the scheduled build with cron expression '0 6 * * *' | |
| if [[ "${{ github.event_name == 'schedule' && github.event.schedule == '0 6 * * *' && 'true' || 'false' }}" == "true" ]]; then | |
| echo "jdk_major_version=21" >> $GITHUB_OUTPUT | |
| exit 0 | |
| fi | |
| # use JDK 17 for build unless overridden with workflow_dispatch input | |
| echo "jdk_major_version=${{ github.event_name == 'workflow_dispatch' && github.event.inputs.jdk_major_version || '17'}}" >> $GITHUB_OUTPUT | |
| - name: checkout | |
| if: ${{ github.event_name == 'pull_request' }} | |
| uses: actions/checkout@v4 | |
| - name: Detect changed files | |
| if: ${{ github.event_name == 'pull_request' }} | |
| id: changes | |
| uses: apache/pulsar-test-infra/paths-filter@master | |
| with: | |
| filters: .github/changes-filter.yaml | |
| list-files: csv | |
| - name: Check changed files | |
| if: ${{ github.event_name == 'pull_request' }} | |
| id: check_changes | |
| run: | | |
| if [[ "${GITHUB_EVENT_NAME}" != "schedule" && "${GITHUB_EVENT_NAME}" != "workflow_dispatch" ]]; then | |
| echo "docs_only=${{ fromJSON(steps.changes.outputs.all_count) == fromJSON(steps.changes.outputs.docs_count) && fromJSON(steps.changes.outputs.docs_count) > 0 }}" >> $GITHUB_OUTPUT | |
| else | |
| echo docs_only=false >> $GITHUB_OUTPUT | |
| fi | |
| - name: Check if coverage should be collected | |
| id: check_coverage | |
| run: | | |
| echo "collect_coverage=${{ | |
| (steps.check_changes.outputs.docs_only != 'true' && github.event_name != 'workflow_dispatch' | |
| && (github.base_ref == 'master' || github.ref_name == 'master')) | |
| || (github.event_name == 'workflow_dispatch' && github.event.inputs.collect_coverage == 'true') | |
| }}" >> $GITHUB_OUTPUT | |
| - name: Check if the PR has been approved for testing | |
| if: ${{ steps.check_changes.outputs.docs_only != 'true' && github.repository == 'apache/pulsar' && github.event_name == 'pull_request' }} | |
| env: | |
| GITHUB_RUN_ATTEMPT: ${{ github.run_attempt }} | |
| GITHUB_TOKEN: ${{ github.token }} | |
| run: | | |
| build/pulsar_ci_tool.sh check_ready_to_test | |
| build-and-license-check: | |
| needs: preconditions | |
| name: Build and License check | |
| env: | |
| JOB_NAME: Build and License check | |
| GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} | |
| CI_JDK_MAJOR_VERSION: ${{ needs.preconditions.outputs.jdk_major_version }} | |
| runs-on: ubuntu-22.04 | |
| timeout-minutes: 60 | |
| if: ${{ needs.preconditions.outputs.docs_only != 'true' }} | |
| steps: | |
| - name: checkout | |
| uses: actions/checkout@v4 | |
| - name: Tune Runner VM | |
| uses: ./.github/actions/tune-runner-vm | |
| - name: Setup ssh access to build runner VM | |
| # ssh access is enabled for builds in own forks | |
| if: ${{ github.repository != 'apache/pulsar' && github.event_name == 'pull_request' }} | |
| uses: ./.github/actions/ssh-access | |
| continue-on-error: true | |
| with: | |
| limit-access-to-actor: true | |
| - name: Cache local Maven repository | |
| uses: actions/cache@v4 | |
| timeout-minutes: 5 | |
| with: | |
| path: | | |
| ~/.m2/repository/*/*/* | |
| !~/.m2/repository/org/apache/pulsar | |
| key: ${{ runner.os }}-m2-dependencies-core-modules-${{ hashFiles('**/pom.xml') }} | |
| restore-keys: | | |
| ${{ runner.os }}-m2-dependencies-core-modules- | |
| - name: Set up JDK ${{ env.CI_JDK_MAJOR_VERSION }} | |
| uses: actions/setup-java@v4 | |
| with: | |
| distribution: ${{ env.JDK_DISTRIBUTION }} | |
| java-version: ${{ env.CI_JDK_MAJOR_VERSION }} | |
| - name: Check source code license headers | |
| run: mvn -B -T 8 -ntp initialize apache-rat:check license:check | |
| - name: Check source code style | |
| run: mvn -B -T 8 -ntp initialize checkstyle:check | |
| - name: Build core-modules | |
| run: | | |
| mvn -B -T 1C -ntp -Pcore-modules,-main clean install -DskipTests -Dlicense.skip=true -Drat.skip=true -Dcheckstyle.skip=true | |
| - name: Check binary licenses | |
| run: src/check-binary-license.sh ./distribution/server/target/apache-pulsar-*-bin.tar.gz | |
| - name: Install gh-actions-artifact-client.js | |
| uses: apache/pulsar-test-infra/gh-actions-artifact-client/dist@master | |
| - name: Save maven build results to Github artifact cache so that the results can be reused | |
| run: | | |
| cd $HOME | |
| $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh store_tar_to_github_actions_artifacts pulsar-maven-repository-binaries \ | |
| tar --exclude '.m2/repository/org/apache/pulsar/pulsar-*-distribution' \ | |
| -I zstd -cf - .m2/repository/org/apache/pulsar | |
| cd $GITHUB_WORKSPACE | |
| $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh store_tar_to_github_actions_artifacts pulsar-server-distribution \ | |
| tar -I zstd -cf - distribution/server/target/apache-pulsar-*-bin.tar.gz | |
| - name: Wait for ssh connection when build fails | |
| # ssh access is enabled for builds in own forks | |
| uses: ./.github/actions/ssh-access | |
| if: ${{ failure() && github.repository != 'apache/pulsar' && github.event_name == 'pull_request' }} | |
| continue-on-error: true | |
| with: | |
| action: wait | |
| unit-tests: | |
| name: CI - Unit - ${{ matrix.name }} | |
| env: | |
| JOB_NAME: CI - Unit - ${{ matrix.name }} | |
| COLLECT_COVERAGE: "${{ needs.preconditions.outputs.collect_coverage }}" | |
| GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} | |
| CI_JDK_MAJOR_VERSION: ${{ needs.preconditions.outputs.jdk_major_version }} | |
| TRACE_TEST_RESOURCE_CLEANUP: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.trace_test_resource_cleanup || 'off' }} | |
| TRACE_TEST_RESOURCE_CLEANUP_DIR: ${{ github.workspace }}/target/trace-test-resource-cleanup | |
| THREAD_LEAK_DETECTOR_WAIT_MILLIS: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.thread_leak_detector_wait_millis || 10000 }} | |
| THREAD_LEAK_DETECTOR_DIR: ${{ github.workspace }}/target/thread-leak-dumps | |
| runs-on: ubuntu-22.04 | |
| timeout-minutes: ${{ matrix.timeout || 60 }} | |
| needs: ['preconditions', 'build-and-license-check'] | |
| if: ${{ needs.preconditions.outputs.docs_only != 'true' }} | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| include: | |
| - name: Other | |
| group: OTHER | |
| timeout: 75 | |
| - name: Brokers - Broker Group 1 | |
| group: BROKER_GROUP_1 | |
| - name: Brokers - Broker Group 2 | |
| group: BROKER_GROUP_2 | |
| - name: Brokers - Broker Group 3 | |
| group: BROKER_GROUP_3 | |
| - name: Brokers - Broker Group 4 | |
| group: BROKER_GROUP_4 | |
| - name: Brokers - Client Api | |
| group: BROKER_CLIENT_API | |
| - name: Brokers - Client Impl | |
| group: BROKER_CLIENT_IMPL | |
| - name: Proxy | |
| group: PROXY | |
| - name: Pulsar IO | |
| group: PULSAR_IO | |
| timeout: 75 | |
| - name: Pulsar IO - Elastic Search | |
| group: PULSAR_IO_ELASTIC | |
| - name: Pulsar IO - Kafka Connect Adaptor | |
| group: PULSAR_IO_KAFKA_CONNECT | |
| - name: Pulsar Client | |
| group: CLIENT | |
| - name: Pulsar Metadata | |
| group: METADATA | |
| steps: | |
| - name: checkout | |
| uses: actions/checkout@v4 | |
| - name: Tune Runner VM | |
| uses: ./.github/actions/tune-runner-vm | |
| - name: Clean Disk when tracing test resource cleanup | |
| if: ${{ env.TRACE_TEST_RESOURCE_CLEANUP != 'off' }} | |
| uses: ./.github/actions/clean-disk | |
| - name: Setup ssh access to build runner VM | |
| # ssh access is enabled for builds in own forks | |
| if: ${{ github.repository != 'apache/pulsar' && github.event_name == 'pull_request' }} | |
| uses: ./.github/actions/ssh-access | |
| continue-on-error: true | |
| with: | |
| limit-access-to-actor: true | |
| - name: Cache Maven dependencies | |
| uses: actions/cache@v4 | |
| timeout-minutes: 5 | |
| with: | |
| path: | | |
| ~/.m2/repository/*/*/* | |
| !~/.m2/repository/org/apache/pulsar | |
| key: ${{ runner.os }}-m2-dependencies-core-modules-${{ hashFiles('**/pom.xml') }} | |
| restore-keys: | | |
| ${{ runner.os }}-m2-dependencies-core-modules- | |
| - name: Set up JDK ${{ matrix.jdk || env.CI_JDK_MAJOR_VERSION }} | |
| uses: actions/setup-java@v4 | |
| with: | |
| distribution: ${{ env.JDK_DISTRIBUTION }} | |
| java-version: ${{ matrix.jdk || env.CI_JDK_MAJOR_VERSION }} | |
| - name: Install gh-actions-artifact-client.js | |
| uses: apache/pulsar-test-infra/gh-actions-artifact-client/dist@master | |
| - name: Restore maven build results from Github artifact cache | |
| run: | | |
| cd $HOME | |
| $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh restore_tar_from_github_actions_artifacts pulsar-maven-repository-binaries | |
| $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh snapshot_pulsar_maven_artifacts | |
| - name: Run setup commands | |
| run: | | |
| ${{ matrix.setup }} | |
| - name: Run unit test group '${{ matrix.group }}' | |
| run: | | |
| CHANGED_TESTS="${{ needs.preconditions.outputs.tests_files }}" ./build/run_unit_group.sh ${{ matrix.group }} | |
| - name: Upload coverage to build artifacts | |
| if: ${{ needs.preconditions.outputs.collect_coverage == 'true' }} | |
| run: $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh upload_unittest_coverage_files ${{ matrix.group }} | |
| - name: print JVM thread dumps when cancelled | |
| if: cancelled() | |
| run: $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh print_thread_dumps | |
| - name: Aggregates all test reports to ./test-reports and ./surefire-reports directories | |
| if: ${{ always() }} | |
| uses: ./.github/actions/copy-test-reports | |
| - name: Publish Test Report | |
| uses: apache/pulsar-test-infra/action-junit-report@master | |
| if: ${{ always() }} | |
| with: | |
| report_paths: 'test-reports/TEST-*.xml' | |
| annotate_only: 'true' | |
| - name: Report detected thread leaks | |
| if: ${{ always() }} | |
| run: | | |
| if [ -d "$THREAD_LEAK_DETECTOR_DIR" ]; then | |
| cd "$THREAD_LEAK_DETECTOR_DIR" | |
| cat threadleak*.txt | awk '/^Summary:/ {print "::warning::" $0 "\n"; next} {print}' | |
| fi | |
| - name: Upload Surefire reports | |
| uses: actions/upload-artifact@v4 | |
| if: ${{ !success() || env.TRACE_TEST_RESOURCE_CLEANUP != 'off' }} | |
| with: | |
| name: Unit-${{ matrix.group }}-surefire-reports | |
| path: surefire-reports | |
| retention-days: 7 | |
| - name: Upload possible heap dump, core dump or crash files | |
| uses: actions/upload-artifact@v4 | |
| if: ${{ always() }} | |
| with: | |
| name: Unit-${{ matrix.group }}-dumps | |
| path: | | |
| /tmp/*.hprof | |
| **/hs_err_*.log | |
| **/core.* | |
| ${{ env.TRACE_TEST_RESOURCE_CLEANUP_DIR }}/* | |
| ${{ env.THREAD_LEAK_DETECTOR_DIR }}/* | |
| retention-days: 7 | |
| if-no-files-found: ignore | |
| - name: Wait for ssh connection when build fails | |
| # ssh access is enabled for builds in own forks | |
| uses: ./.github/actions/ssh-access | |
| if: ${{ failure() && github.repository != 'apache/pulsar' && github.event_name == 'pull_request' }} | |
| continue-on-error: true | |
| with: | |
| action: wait | |
| unit-tests-upload-coverage: | |
| name: CI - Unit - Upload Coverage | |
| runs-on: ubuntu-22.04 | |
| timeout-minutes: 30 | |
| needs: ['preconditions', 'unit-tests'] | |
| env: | |
| CI_JDK_MAJOR_VERSION: ${{ needs.preconditions.outputs.jdk_major_version }} | |
| if: ${{ needs.preconditions.outputs.collect_coverage == 'true' }} | |
| steps: | |
| - name: checkout | |
| uses: actions/checkout@v4 | |
| - name: Tune Runner VM | |
| uses: ./.github/actions/tune-runner-vm | |
| - name: Setup ssh access to build runner VM | |
| # ssh access is enabled for builds in own forks | |
| if: ${{ github.repository != 'apache/pulsar' && github.event_name == 'pull_request' }} | |
| uses: ./.github/actions/ssh-access | |
| continue-on-error: true | |
| with: | |
| limit-access-to-actor: true | |
| - name: Cache Maven dependencies | |
| uses: actions/cache@v4 | |
| timeout-minutes: 5 | |
| with: | |
| path: | | |
| ~/.m2/repository/*/*/* | |
| !~/.m2/repository/org/apache/pulsar | |
| key: ${{ runner.os }}-m2-dependencies-core-modules-${{ hashFiles('**/pom.xml') }} | |
| restore-keys: | | |
| ${{ runner.os }}-m2-dependencies-core-modules- | |
| - name: Set up JDK ${{ matrix.jdk || env.CI_JDK_MAJOR_VERSION }} | |
| uses: actions/setup-java@v4 | |
| with: | |
| distribution: ${{ env.JDK_DISTRIBUTION }} | |
| java-version: ${{ matrix.jdk || env.CI_JDK_MAJOR_VERSION }} | |
| - name: Install gh-actions-artifact-client.js | |
| uses: apache/pulsar-test-infra/gh-actions-artifact-client/dist@master | |
| - name: Restore maven build results from Github artifact cache | |
| run: | | |
| cd $HOME | |
| $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh restore_tar_from_github_actions_artifacts pulsar-maven-repository-binaries | |
| $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh snapshot_pulsar_maven_artifacts | |
| - name: Restore coverage files from build artifacts and create Jacoco reports | |
| run: | | |
| $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh restore_unittest_coverage_files | |
| $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh create_test_coverage_report | |
| cd $GITHUB_WORKSPACE/target | |
| zip -qr jacoco_test_coverage_report_unittests.zip jacoco_test_coverage_report || true | |
| - name: Upload Jacoco report files to build artifacts | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: Jacoco-coverage-report-unittests | |
| path: target/jacoco_test_coverage_report_unittests.zip | |
| retention-days: 3 | |
| - name: Upload to Codecov | |
| uses: ./.github/actions/upload-coverage | |
| with: | |
| flags: unittests | |
| - name: Delete coverage files from build artifacts | |
| run: | | |
| $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh delete_unittest_coverage_files | |
| - name: Wait for ssh connection when build fails | |
| # ssh access is enabled for builds in own forks | |
| uses: ./.github/actions/ssh-access | |
| if: ${{ failure() && github.repository != 'apache/pulsar' && github.event_name == 'pull_request' }} | |
| continue-on-error: true | |
| with: | |
| action: wait | |
| pulsar-java-test-image: | |
| name: Build Pulsar java-test-image docker image | |
| runs-on: ubuntu-22.04 | |
| timeout-minutes: 60 | |
| needs: ['preconditions', 'build-and-license-check'] | |
| if: ${{ needs.preconditions.outputs.docs_only != 'true'}} | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| platform: | |
| - linux/amd64 | |
| - linux/arm64 | |
| env: | |
| GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} | |
| CI_JDK_MAJOR_VERSION: ${{ needs.preconditions.outputs.jdk_major_version }} | |
| IMAGE_JDK_MAJOR_VERSION: ${{ needs.preconditions.outputs.jdk_major_version }} | |
| steps: | |
| - name: checkout | |
| uses: actions/checkout@v4 | |
| - name: Tune Runner VM | |
| uses: ./.github/actions/tune-runner-vm | |
| - name: Setup ssh access to build runner VM | |
| # ssh access is enabled for builds in own forks | |
| if: ${{ github.repository != 'apache/pulsar' && github.event_name == 'pull_request' }} | |
| uses: ./.github/actions/ssh-access | |
| continue-on-error: true | |
| with: | |
| limit-access-to-actor: true | |
| - name: Cache Maven dependencies | |
| uses: actions/cache@v4 | |
| timeout-minutes: 5 | |
| with: | |
| path: | | |
| ~/.m2/repository/*/*/* | |
| !~/.m2/repository/org/apache/pulsar | |
| key: ${{ runner.os }}-m2-dependencies-core-modules-${{ hashFiles('**/pom.xml') }} | |
| restore-keys: | | |
| ${{ runner.os }}-m2-dependencies-core-modules- | |
| - name: Set up JDK ${{ env.CI_JDK_MAJOR_VERSION }} | |
| uses: actions/setup-java@v4 | |
| with: | |
| distribution: ${{ env.JDK_DISTRIBUTION }} | |
| java-version: ${{ env.CI_JDK_MAJOR_VERSION }} | |
| - name: Install gh-actions-artifact-client.js | |
| uses: apache/pulsar-test-infra/gh-actions-artifact-client/dist@master | |
| - name: restore maven build results from Github artifact cache | |
| run: | | |
| cd $HOME | |
| $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh restore_tar_from_github_actions_artifacts pulsar-maven-repository-binaries | |
| - name: Set up QEMU | |
| uses: docker/setup-qemu-action@v3 | |
| with: | |
| platforms: arm64 | |
| - name: Build java-test-image docker image - ${{ matrix.platform }} | |
| run: | | |
| # build docker image | |
| DOCKER_CLI_EXPERIMENTAL=enabled mvn -B -am -pl docker/pulsar,tests/docker-images/java-test-image install -Pcore-modules,-main,integrationTests,docker \ | |
| -Ddocker.platforms=${{ matrix.platform }} \ | |
| -Dmaven.test.skip=true -DskipSourceReleaseAssembly=true \ | |
| -Dspotbugs.skip=true -Dlicense.skip=true -Dcheckstyle.skip=true -Drat.skip=true | |
| - name: save docker image apachepulsar/java-test-image:latest to Github artifact cache | |
| if: ${{ matrix.platform == 'linux/amd64' }} | |
| run: | | |
| $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh docker_save_image_to_github_actions_artifacts apachepulsar/java-test-image:latest pulsar-java-test-image | |
| - name: Wait for ssh connection when build fails | |
| # ssh access is enabled for builds in own forks | |
| uses: ./.github/actions/ssh-access | |
| if: ${{ failure() && github.repository != 'apache/pulsar' && github.event_name == 'pull_request' }} | |
| continue-on-error: true | |
| with: | |
| action: wait | |
| integration-tests: | |
| name: CI - Integration - ${{ matrix.name }} | |
| runs-on: ubuntu-22.04 | |
| timeout-minutes: ${{ matrix.timeout || 60 }} | |
| needs: ['preconditions', 'pulsar-java-test-image'] | |
| if: ${{ needs.preconditions.outputs.docs_only != 'true' }} | |
| env: | |
| JOB_NAME: CI - Integration - ${{ matrix.name }} | |
| PULSAR_TEST_IMAGE_NAME: apachepulsar/java-test-image:latest | |
| GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} | |
| CI_JDK_MAJOR_VERSION: ${{ needs.preconditions.outputs.jdk_major_version }} | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| include: | |
| - name: Backwards Compatibility | |
| group: BACKWARDS_COMPAT | |
| no_coverage: true | |
| - name: Cli | |
| group: CLI | |
| - name: Messaging | |
| group: MESSAGING | |
| - name: LoadBalance | |
| group: LOADBALANCE | |
| no_coverage: true | |
| - name: Shade on Java 8 | |
| group: SHADE_RUN | |
| runtime_jdk: 8 | |
| setup: ./build/run_integration_group.sh SHADE_BUILD | |
| no_coverage: true | |
| - name: Shade on Java 11 | |
| group: SHADE_RUN | |
| runtime_jdk: 11 | |
| setup: ./build/run_integration_group.sh SHADE_BUILD | |
| no_coverage: true | |
| - name: Shade on Java 17 | |
| group: SHADE_RUN | |
| runtime_jdk: 17 | |
| setup: ./build/run_integration_group.sh SHADE_BUILD | |
| no_coverage: true | |
| - name: Shade on Java 21 | |
| group: SHADE_RUN | |
| runtime_jdk: 21 | |
| setup: ./build/run_integration_group.sh SHADE_BUILD | |
| no_coverage: true | |
| - name: Standalone | |
| group: STANDALONE | |
| - name: Transaction | |
| group: TRANSACTION | |
| - name: Metrics | |
| group: METRICS | |
| steps: | |
| - name: checkout | |
| uses: actions/checkout@v4 | |
| - name: Tune Runner VM | |
| uses: ./.github/actions/tune-runner-vm | |
| - name: Setup ssh access to build runner VM | |
| # ssh access is enabled for builds in own forks | |
| if: ${{ github.repository != 'apache/pulsar' && github.event_name == 'pull_request' }} | |
| uses: ./.github/actions/ssh-access | |
| continue-on-error: true | |
| with: | |
| limit-access-to-actor: true | |
| - name: Cache Maven dependencies | |
| uses: actions/cache@v4 | |
| timeout-minutes: 5 | |
| with: | |
| path: | | |
| ~/.m2/repository/*/*/* | |
| !~/.m2/repository/org/apache/pulsar | |
| key: ${{ runner.os }}-m2-dependencies-core-modules-${{ hashFiles('**/pom.xml') }} | |
| restore-keys: | | |
| ${{ runner.os }}-m2-dependencies-core-modules- | |
| - name: Set up JDK ${{ env.CI_JDK_MAJOR_VERSION }} | |
| uses: actions/setup-java@v4 | |
| with: | |
| distribution: ${{ env.JDK_DISTRIBUTION }} | |
| java-version: ${{ env.CI_JDK_MAJOR_VERSION }} | |
| - name: Install gh-actions-artifact-client.js | |
| uses: apache/pulsar-test-infra/gh-actions-artifact-client/dist@master | |
| - name: Restore maven build results from Github artifact cache | |
| run: | | |
| cd $HOME | |
| $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh restore_tar_from_github_actions_artifacts pulsar-maven-repository-binaries | |
| $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh snapshot_pulsar_maven_artifacts | |
| - name: Load docker image apachepulsar/java-test-image:latest from Github artifact cache | |
| run: | | |
| $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh docker_load_image_from_github_actions_artifacts pulsar-java-test-image | |
| - name: Run setup commands | |
| if: ${{ matrix.setup }} | |
| run: | | |
| ${{ matrix.setup }} | |
| - name: Set up runtime JDK ${{ matrix.runtime_jdk }} | |
| uses: actions/setup-java@v4 | |
| if: ${{ matrix.runtime_jdk }} | |
| with: | |
| distribution: ${{ env.JDK_DISTRIBUTION }} | |
| java-version: ${{ matrix.runtime_jdk }} | |
| - name: Run integration test group '${{ matrix.group }}' | |
| run: | | |
| if [[ "${{ matrix.no_coverage }}" != "true" && "${{ needs.preconditions.outputs.collect_coverage }}" == "true" ]]; then | |
| coverage_args="--coverage" | |
| fi | |
| ./build/run_integration_group.sh ${{ matrix.group }} $coverage_args | |
| - name: Upload coverage to build artifacts | |
| if: ${{ !matrix.no_coverage && needs.preconditions.outputs.collect_coverage == 'true' }} | |
| run: $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh upload_inttest_coverage_files ${{ matrix.group }} | |
| - name: print JVM thread dumps when cancelled | |
| if: cancelled() | |
| run: $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh print_thread_dumps | |
| - name: Aggregates all test reports to ./test-reports and ./surefire-reports directories | |
| if: ${{ always() }} | |
| uses: ./.github/actions/copy-test-reports | |
| - name: Publish Test Report | |
| uses: apache/pulsar-test-infra/action-junit-report@master | |
| if: ${{ always() }} | |
| with: | |
| report_paths: 'test-reports/TEST-*.xml' | |
| annotate_only: 'true' | |
| - name: Upload Surefire reports | |
| uses: actions/upload-artifact@v4 | |
| if: ${{ !success() }} | |
| with: | |
| name: Integration-${{ matrix.group }}-surefire-reports | |
| path: surefire-reports | |
| retention-days: 7 | |
| - name: Upload container logs | |
| uses: actions/upload-artifact@v4 | |
| if: ${{ !success() }} | |
| continue-on-error: true | |
| with: | |
| name: Integration-${{ matrix.group }}-container-logs | |
| path: tests/integration/target/container-logs | |
| retention-days: 7 | |
| - name: Wait for ssh connection when build fails | |
| # ssh access is enabled for builds in own forks | |
| uses: ./.github/actions/ssh-access | |
| if: ${{ failure() && github.repository != 'apache/pulsar' && github.event_name == 'pull_request' }} | |
| continue-on-error: true | |
| with: | |
| action: wait | |
| integration-tests-upload-coverage: | |
| name: CI - Integration - Upload Coverage | |
| runs-on: ubuntu-22.04 | |
| timeout-minutes: 30 | |
| needs: ['preconditions', 'integration-tests'] | |
| if: ${{ needs.preconditions.outputs.collect_coverage == 'true' }} | |
| env: | |
| PULSAR_TEST_IMAGE_NAME: apachepulsar/java-test-image:latest | |
| CI_JDK_MAJOR_VERSION: ${{ needs.preconditions.outputs.jdk_major_version }} | |
| steps: | |
| - name: checkout | |
| uses: actions/checkout@v4 | |
| - name: Tune Runner VM | |
| uses: ./.github/actions/tune-runner-vm | |
| - name: Setup ssh access to build runner VM | |
| # ssh access is enabled for builds in own forks | |
| if: ${{ github.repository != 'apache/pulsar' && github.event_name == 'pull_request' }} | |
| uses: ./.github/actions/ssh-access | |
| continue-on-error: true | |
| with: | |
| limit-access-to-actor: true | |
| - name: Cache Maven dependencies | |
| uses: actions/cache@v4 | |
| timeout-minutes: 5 | |
| with: | |
| path: | | |
| ~/.m2/repository/*/*/* | |
| !~/.m2/repository/org/apache/pulsar | |
| key: ${{ runner.os }}-m2-dependencies-core-modules-${{ hashFiles('**/pom.xml') }} | |
| restore-keys: | | |
| ${{ runner.os }}-m2-dependencies-core-modules- | |
| - name: Set up JDK ${{ env.CI_JDK_MAJOR_VERSION }} | |
| uses: actions/setup-java@v4 | |
| with: | |
| distribution: ${{ env.JDK_DISTRIBUTION }} | |
| java-version: ${{ env.CI_JDK_MAJOR_VERSION }} | |
| - name: Install gh-actions-artifact-client.js | |
| uses: apache/pulsar-test-infra/gh-actions-artifact-client/dist@master | |
| - name: Restore maven build results from Github artifact cache | |
| run: | | |
| cd $HOME | |
| $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh restore_tar_from_github_actions_artifacts pulsar-maven-repository-binaries | |
| $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh snapshot_pulsar_maven_artifacts | |
| - name: Load docker image apachepulsar/java-test-image:latest from Github artifact cache | |
| run: | | |
| $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh docker_load_image_from_github_actions_artifacts pulsar-java-test-image | |
| - name: Restore coverage files from build artifacts and create Jacoco reports | |
| run: | | |
| $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh restore_inttest_coverage_files | |
| $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh create_inttest_coverage_report | |
| $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh create_test_coverage_report | |
| cd $GITHUB_WORKSPACE/target | |
| zip -qr jacoco_test_coverage_report_inttests.zip jacoco_test_coverage_report jacoco_inttest_coverage_report || true | |
| - name: Upload Jacoco report files to build artifacts | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: Jacoco-coverage-report-inttests | |
| path: target/jacoco_test_coverage_report_inttests.zip | |
| retention-days: 3 | |
| - name: Upload to Codecov | |
| uses: ./.github/actions/upload-coverage | |
| with: | |
| flags: inttests | |
| - name: Delete coverage files from build artifacts | |
| run: | | |
| $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh delete_inttest_coverage_files | |
| - name: Wait for ssh connection when build fails | |
| # ssh access is enabled for builds in own forks | |
| uses: ./.github/actions/ssh-access | |
| if: ${{ failure() && github.repository != 'apache/pulsar' && github.event_name == 'pull_request' }} | |
| continue-on-error: true | |
| with: | |
| action: wait | |
| delete-integration-test-docker-image-artifact: | |
| name: "Delete integration test docker image artifact" | |
| runs-on: ubuntu-22.04 | |
| timeout-minutes: 10 | |
| needs: [ | |
| 'preconditions', | |
| 'integration-tests', | |
| 'integration-tests-upload-coverage' | |
| ] | |
| if: ${{ needs.preconditions.outputs.docs_only != 'true' }} | |
| steps: | |
| - name: checkout | |
| uses: actions/checkout@v4 | |
| - name: Tune Runner VM | |
| uses: ./.github/actions/tune-runner-vm | |
| - name: Install gh-actions-artifact-client.js | |
| uses: apache/pulsar-test-infra/gh-actions-artifact-client/dist@master | |
| - name: Delete docker image from GitHub Actions Artifacts | |
| run: | | |
| gh-actions-artifact-client.js delete pulsar-java-test-image.zst | |
| pulsar-test-latest-version-image: | |
| name: Build Pulsar docker image | |
| runs-on: ubuntu-22.04 | |
| timeout-minutes: 60 | |
| needs: ['preconditions', 'build-and-license-check'] | |
| if: ${{ needs.preconditions.outputs.docs_only != 'true' }} | |
| env: | |
| GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} | |
| CI_JDK_MAJOR_VERSION: ${{ needs.preconditions.outputs.jdk_major_version }} | |
| IMAGE_JDK_MAJOR_VERSION: ${{ needs.preconditions.outputs.jdk_major_version }} | |
| steps: | |
| - name: checkout | |
| uses: actions/checkout@v4 | |
| - name: Tune Runner VM | |
| uses: ./.github/actions/tune-runner-vm | |
| - name: Setup ssh access to build runner VM | |
| # ssh access is enabled for builds in own forks | |
| if: ${{ github.repository != 'apache/pulsar' && github.event_name == 'pull_request' }} | |
| uses: ./.github/actions/ssh-access | |
| continue-on-error: true | |
| with: | |
| limit-access-to-actor: true | |
| - name: Clean Disk | |
| uses: ./.github/actions/clean-disk | |
| with: | |
| mode: full | |
| - name: Cache local Maven repository | |
| uses: actions/cache@v4 | |
| timeout-minutes: 5 | |
| with: | |
| path: | | |
| ~/.m2/repository/*/*/* | |
| !~/.m2/repository/org/apache/pulsar | |
| key: ${{ runner.os }}-m2-dependencies-all-${{ hashFiles('**/pom.xml') }} | |
| restore-keys: | | |
| ${{ runner.os }}-m2-dependencies-core-modules-${{ hashFiles('**/pom.xml') }} | |
| ${{ runner.os }}-m2-dependencies-core-modules- | |
| - name: Set up JDK ${{ env.CI_JDK_MAJOR_VERSION }} | |
| uses: actions/setup-java@v4 | |
| with: | |
| distribution: ${{ env.JDK_DISTRIBUTION }} | |
| java-version: ${{ env.CI_JDK_MAJOR_VERSION }} | |
| - name: Install gh-actions-artifact-client.js | |
| uses: apache/pulsar-test-infra/gh-actions-artifact-client/dist@master | |
| - name: restore maven build results from Github artifact cache | |
| run: | | |
| cd $HOME | |
| $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh restore_tar_from_github_actions_artifacts pulsar-maven-repository-binaries | |
| - name: Build latest-version-image docker image | |
| run: | | |
| # build docker image | |
| # include building of Connectors, Offloaders and server distros | |
| DOCKER_CLI_EXPERIMENTAL=enabled mvn -B -am -pl distribution/io,distribution/offloaders,distribution/server,distribution/shell,tests/docker-images/latest-version-image install \ | |
| -Pmain,docker -Dmaven.test.skip=true \ | |
| -Dspotbugs.skip=true -Dlicense.skip=true -Dcheckstyle.skip=true -Drat.skip=true | |
| # check full build artifacts licenses | |
| - name: Check binary licenses | |
| run: src/check-binary-license.sh ./distribution/server/target/apache-pulsar-*-bin.tar.gz && src/check-binary-license.sh ./distribution/shell/target/apache-pulsar-shell-*-bin.tar.gz | |
| - name: Run Trivy container scan | |
| uses: aquasecurity/trivy-action@master | |
| if: ${{ github.repository == 'apache/pulsar' && github.event_name != 'pull_request' }} | |
| with: | |
| image-ref: "apachepulsar/pulsar:latest" | |
| scanners: vuln | |
| severity: CRITICAL,HIGH,MEDIUM,LOW | |
| limit-severities-for-sarif: true | |
| format: 'sarif' | |
| output: 'trivy-results.sarif' | |
| - name: Upload Trivy scan results to GitHub Security tab | |
| uses: github/codeql-action/upload-sarif@v3 | |
| if: ${{ github.repository == 'apache/pulsar' && github.event_name != 'pull_request' }} | |
| with: | |
| sarif_file: 'trivy-results.sarif' | |
| - name: Clean up disk space | |
| run: | | |
| # release disk space since saving docker image consumes local disk space | |
| # | |
| echo "::group::Available diskspace before cleaning" | |
| time df -BM / /mnt | |
| echo "::endgroup::" | |
| echo "::group::Clean build directory" | |
| # docker build changes some files to root ownership, fix this before deleting files | |
| sudo chown -R $USER:$GROUP . | |
| # clean build directories | |
| time git clean -fdx | |
| echo "::endgroup::" | |
| echo "::group::Available diskspace after cleaning build directory" | |
| time df -BM / /mnt | |
| echo "::endgroup::" | |
| echo "::group::Delete maven repository" | |
| # delete maven repository | |
| time rm -rf ~/.m2/repository | |
| echo "::endgroup::" | |
| echo "::group::Available diskspace after cleaning maven repository" | |
| time df -BM / /mnt | |
| echo "::endgroup::" | |
| - name: save docker image apachepulsar/pulsar-test-latest-version:latest to Github artifact cache | |
| run: | | |
| $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh docker_save_image_to_github_actions_artifacts apachepulsar/pulsar-test-latest-version:latest pulsar-test-latest-version-image | |
| - name: Wait for ssh connection when build fails | |
| # ssh access is enabled for builds in own forks | |
| uses: ./.github/actions/ssh-access | |
| if: ${{ failure() && github.repository != 'apache/pulsar' && github.event_name == 'pull_request' }} | |
| continue-on-error: true | |
| with: | |
| action: wait | |
| system-tests: | |
| name: CI - System - ${{ matrix.name }} | |
| runs-on: ubuntu-22.04 | |
| timeout-minutes: 60 | |
| needs: ['preconditions', 'pulsar-test-latest-version-image'] | |
| if: ${{ needs.preconditions.outputs.docs_only != 'true' }} | |
| env: | |
| JOB_NAME: CI - System - ${{ matrix.name }} | |
| PULSAR_TEST_IMAGE_NAME: apachepulsar/pulsar-test-latest-version:latest | |
| GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} | |
| CI_JDK_MAJOR_VERSION: ${{ needs.preconditions.outputs.jdk_major_version }} | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| include: | |
| - name: Tiered FileSystem | |
| group: TIERED_FILESYSTEM | |
| - name: Tiered JCloud | |
| group: TIERED_JCLOUD | |
| - name: Function | |
| group: FUNCTION | |
| - name: Schema | |
| group: SCHEMA | |
| - name: Pulsar Connectors - Thread | |
| group: PULSAR_CONNECTORS_THREAD | |
| - name: Pulsar Connectors - Process | |
| group: PULSAR_CONNECTORS_PROCESS | |
| - name: Pulsar IO | |
| group: PULSAR_IO | |
| clean_disk: true | |
| steps: | |
| - name: checkout | |
| uses: actions/checkout@v4 | |
| - name: Tune Runner VM | |
| uses: ./.github/actions/tune-runner-vm | |
| - name: Clean Disk when needed | |
| if: ${{ matrix.clean_disk }} | |
| uses: ./.github/actions/clean-disk | |
| - name: Setup ssh access to build runner VM | |
| # ssh access is enabled for builds in own forks | |
| if: ${{ github.repository != 'apache/pulsar' && github.event_name == 'pull_request' }} | |
| uses: ./.github/actions/ssh-access | |
| continue-on-error: true | |
| with: | |
| limit-access-to-actor: true | |
| - name: Cache local Maven repository | |
| uses: actions/cache@v4 | |
| timeout-minutes: 5 | |
| with: | |
| path: | | |
| ~/.m2/repository/*/*/* | |
| !~/.m2/repository/org/apache/pulsar | |
| key: ${{ runner.os }}-m2-dependencies-all-${{ hashFiles('**/pom.xml') }} | |
| restore-keys: | | |
| ${{ runner.os }}-m2-dependencies-core-modules-${{ hashFiles('**/pom.xml') }} | |
| ${{ runner.os }}-m2-dependencies-core-modules- | |
| - name: Set up JDK ${{ env.CI_JDK_MAJOR_VERSION }} | |
| uses: actions/setup-java@v4 | |
| with: | |
| distribution: ${{ env.JDK_DISTRIBUTION }} | |
| java-version: ${{ env.CI_JDK_MAJOR_VERSION }} | |
| - name: Install gh-actions-artifact-client.js | |
| uses: apache/pulsar-test-infra/gh-actions-artifact-client/dist@master | |
| - name: Restore maven build results from Github artifact cache | |
| run: | | |
| cd $HOME | |
| $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh restore_tar_from_github_actions_artifacts pulsar-maven-repository-binaries | |
| $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh snapshot_pulsar_maven_artifacts | |
| - name: Load docker image apachepulsar/pulsar-test-latest-version:latest from Github artifact cache | |
| run: | | |
| $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh docker_load_image_from_github_actions_artifacts pulsar-test-latest-version-image | |
| - name: Run setup commands | |
| if: ${{ matrix.setup }} | |
| run: | | |
| ${{ matrix.setup }} | |
| - name: Run system test group '${{ matrix.group }}' | |
| run: | | |
| if [[ "${{ matrix.no_coverage }}" != "true" && "${{ needs.preconditions.outputs.collect_coverage }}" == "true" ]]; then | |
| coverage_args="--coverage" | |
| fi | |
| ./build/run_integration_group.sh ${{ matrix.group }} $coverage_args | |
| - name: Upload coverage to build artifacts | |
| if: ${{ !matrix.no_coverage && needs.preconditions.outputs.collect_coverage == 'true' }} | |
| run: $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh upload_systest_coverage_files ${{ matrix.group }} | |
| - name: print JVM thread dumps when cancelled | |
| if: cancelled() | |
| run: $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh print_thread_dumps | |
| - name: Aggregates all test reports to ./test-reports and ./surefire-reports directories | |
| if: ${{ always() }} | |
| uses: ./.github/actions/copy-test-reports | |
| - name: Publish Test Report | |
| uses: apache/pulsar-test-infra/action-junit-report@master | |
| if: ${{ always() }} | |
| with: | |
| report_paths: 'test-reports/TEST-*.xml' | |
| annotate_only: 'true' | |
| - name: Upload container logs | |
| uses: actions/upload-artifact@v4 | |
| if: ${{ !success() }} | |
| continue-on-error: true | |
| with: | |
| name: System-${{ matrix.group }}-container-logs | |
| path: tests/integration/target/container-logs | |
| retention-days: 7 | |
| - name: Upload Surefire reports | |
| uses: actions/upload-artifact@v4 | |
| if: ${{ !success() }} | |
| with: | |
| name: System-${{ matrix.name }}-surefire-reports | |
| path: surefire-reports | |
| retention-days: 7 | |
| - name: Wait for ssh connection when build fails | |
| # ssh access is enabled for builds in own forks | |
| uses: ./.github/actions/ssh-access | |
| if: ${{ failure() && github.repository != 'apache/pulsar' && github.event_name == 'pull_request' }} | |
| continue-on-error: true | |
| with: | |
| action: wait | |
| system-tests-upload-coverage: | |
| name: CI - System - Upload Coverage | |
| runs-on: ubuntu-22.04 | |
| timeout-minutes: 30 | |
| needs: ['preconditions', 'system-tests'] | |
| if: ${{ needs.preconditions.outputs.collect_coverage == 'true' }} | |
| env: | |
| PULSAR_TEST_IMAGE_NAME: apachepulsar/pulsar-test-latest-version:latest | |
| CI_JDK_MAJOR_VERSION: ${{ needs.preconditions.outputs.jdk_major_version }} | |
| steps: | |
| - name: checkout | |
| uses: actions/checkout@v4 | |
| - name: Tune Runner VM | |
| uses: ./.github/actions/tune-runner-vm | |
| - name: Setup ssh access to build runner VM | |
| # ssh access is enabled for builds in own forks | |
| if: ${{ github.repository != 'apache/pulsar' && github.event_name == 'pull_request' }} | |
| uses: ./.github/actions/ssh-access | |
| continue-on-error: true | |
| with: | |
| limit-access-to-actor: true | |
| - name: Cache local Maven repository | |
| uses: actions/cache@v4 | |
| timeout-minutes: 5 | |
| with: | |
| path: | | |
| ~/.m2/repository/*/*/* | |
| !~/.m2/repository/org/apache/pulsar | |
| key: ${{ runner.os }}-m2-dependencies-all-${{ hashFiles('**/pom.xml') }} | |
| restore-keys: | | |
| ${{ runner.os }}-m2-dependencies-core-modules-${{ hashFiles('**/pom.xml') }} | |
| ${{ runner.os }}-m2-dependencies-core-modules- | |
| - name: Set up JDK ${{ env.CI_JDK_MAJOR_VERSION }} | |
| uses: actions/setup-java@v4 | |
| with: | |
| distribution: ${{ env.JDK_DISTRIBUTION }} | |
| java-version: ${{ env.CI_JDK_MAJOR_VERSION }} | |
| - name: Install gh-actions-artifact-client.js | |
| uses: apache/pulsar-test-infra/gh-actions-artifact-client/dist@master | |
| - name: Restore maven build results from Github artifact cache | |
| run: | | |
| cd $HOME | |
| $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh restore_tar_from_github_actions_artifacts pulsar-maven-repository-binaries | |
| - name: Load docker image apachepulsar/pulsar-test-latest-version:latest from Github artifact cache | |
| run: | | |
| $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh docker_load_image_from_github_actions_artifacts pulsar-test-latest-version-image | |
| - name: Restore coverage files from build artifacts and create Jacoco reports | |
| run: | | |
| $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh restore_systest_coverage_files | |
| $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh create_inttest_coverage_report | |
| $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh create_test_coverage_report | |
| cd $GITHUB_WORKSPACE/target | |
| zip -qr jacoco_test_coverage_report_systests.zip jacoco_test_coverage_report jacoco_inttest_coverage_report || true | |
| - name: Upload Jacoco report files to build artifacts | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: Jacoco-coverage-report-systests | |
| path: target/jacoco_test_coverage_report_systests.zip | |
| retention-days: 3 | |
| - name: Upload to Codecov | |
| uses: ./.github/actions/upload-coverage | |
| with: | |
| flags: systests | |
| - name: Delete coverage files from build artifacts | |
| run: | | |
| $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh delete_systest_coverage_files | |
| - name: Wait for ssh connection when build fails | |
| # ssh access is enabled for builds in own forks | |
| uses: ./.github/actions/ssh-access | |
| if: ${{ failure() && github.repository != 'apache/pulsar' && github.event_name == 'pull_request' }} | |
| continue-on-error: true | |
| with: | |
| action: wait | |
| flaky-system-tests: | |
| name: CI Flaky - System - ${{ matrix.name }} | |
| runs-on: ubuntu-22.04 | |
| timeout-minutes: 60 | |
| needs: [ 'preconditions', 'pulsar-test-latest-version-image' ] | |
| if: ${{ needs.preconditions.outputs.docs_only != 'true' }} | |
| env: | |
| JOB_NAME: CI Flaky - System - ${{ matrix.name }} | |
| PULSAR_TEST_IMAGE_NAME: apachepulsar/pulsar-test-latest-version:latest | |
| GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} | |
| CI_JDK_MAJOR_VERSION: ${{ needs.preconditions.outputs.jdk_major_version }} | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| include: | |
| - name: Plugin | |
| group: PLUGIN | |
| - name: Pulsar IO - Oracle | |
| group: PULSAR_IO_ORA | |
| clean_disk: true | |
| steps: | |
| - name: checkout | |
| uses: actions/checkout@v4 | |
| - name: Tune Runner VM | |
| uses: ./.github/actions/tune-runner-vm | |
| - name: Clean Disk when needed | |
| if: ${{ matrix.clean_disk }} | |
| uses: ./.github/actions/clean-disk | |
| - name: Setup ssh access to build runner VM | |
| # ssh access is enabled for builds in own forks | |
| if: ${{ github.repository != 'apache/pulsar' && github.event_name == 'pull_request' }} | |
| uses: ./.github/actions/ssh-access | |
| continue-on-error: true | |
| with: | |
| limit-access-to-actor: true | |
| - name: Cache local Maven repository | |
| uses: actions/cache@v4 | |
| timeout-minutes: 5 | |
| with: | |
| path: | | |
| ~/.m2/repository/*/*/* | |
| !~/.m2/repository/org/apache/pulsar | |
| key: ${{ runner.os }}-m2-dependencies-all-${{ hashFiles('**/pom.xml') }} | |
| restore-keys: | | |
| ${{ runner.os }}-m2-dependencies-core-modules-${{ hashFiles('**/pom.xml') }} | |
| ${{ runner.os }}-m2-dependencies-core-modules- | |
| - name: Set up JDK ${{ env.CI_JDK_MAJOR_VERSION }} | |
| uses: actions/setup-java@v4 | |
| with: | |
| distribution: ${{ env.JDK_DISTRIBUTION }} | |
| java-version: ${{ env.CI_JDK_MAJOR_VERSION }} | |
| - name: Install gh-actions-artifact-client.js | |
| uses: apache/pulsar-test-infra/gh-actions-artifact-client/dist@master | |
| - name: Restore maven build results from Github artifact cache | |
| run: | | |
| cd $HOME | |
| $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh restore_tar_from_github_actions_artifacts pulsar-maven-repository-binaries | |
| - name: Load docker image apachepulsar/pulsar-test-latest-version:latest from Github artifact cache | |
| run: | | |
| $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh docker_load_image_from_github_actions_artifacts pulsar-test-latest-version-image | |
| - name: Run setup commands | |
| if: ${{ matrix.setup }} | |
| run: | | |
| ${{ matrix.setup }} | |
| - name: Run system test group '${{ matrix.group }}' | |
| run: | | |
| ./build/run_integration_group.sh ${{ matrix.group }} | |
| - name: print JVM thread dumps when cancelled | |
| if: cancelled() | |
| run: $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh print_thread_dumps | |
| - name: Aggregates all test reports to ./test-reports and ./surefire-reports directories | |
| if: ${{ always() }} | |
| uses: ./.github/actions/copy-test-reports | |
| - name: Publish Test Report | |
| uses: apache/pulsar-test-infra/action-junit-report@master | |
| if: ${{ always() }} | |
| with: | |
| report_paths: 'test-reports/TEST-*.xml' | |
| annotate_only: 'true' | |
| - name: Upload container logs | |
| uses: actions/upload-artifact@v4 | |
| if: ${{ !success() }} | |
| continue-on-error: true | |
| with: | |
| name: System-${{ matrix.group }}-container-logs | |
| path: tests/integration/target/container-logs | |
| retention-days: 7 | |
| - name: Upload Surefire reports | |
| uses: actions/upload-artifact@v4 | |
| if: ${{ !success() }} | |
| with: | |
| name: System-${{ matrix.name }}-surefire-reports | |
| path: surefire-reports | |
| retention-days: 7 | |
| - name: Wait for ssh connection when build fails | |
| # ssh access is enabled for builds in own forks | |
| uses: ./.github/actions/ssh-access | |
| if: ${{ failure() && github.repository != 'apache/pulsar' && github.event_name == 'pull_request' }} | |
| continue-on-error: true | |
| with: | |
| action: wait | |
| delete-system-test-docker-image-artifact: | |
| name: "Delete system test docker image artifact" | |
| runs-on: ubuntu-22.04 | |
| timeout-minutes: 10 | |
| needs: [ | |
| 'preconditions', | |
| 'system-tests', | |
| 'system-tests-upload-coverage', | |
| 'flaky-system-tests' | |
| ] | |
| if: ${{ needs.preconditions.outputs.docs_only != 'true' }} | |
| steps: | |
| - name: checkout | |
| uses: actions/checkout@v4 | |
| - name: Tune Runner VM | |
| uses: ./.github/actions/tune-runner-vm | |
| - name: Install gh-actions-artifact-client.js | |
| uses: apache/pulsar-test-infra/gh-actions-artifact-client/dist@master | |
| - name: Delete docker image from GitHub Actions Artifacts | |
| run: | | |
| gh-actions-artifact-client.js delete pulsar-test-latest-version-image.zst | |
| macos-build: | |
| name: Build Pulsar on MacOS | |
| runs-on: macos-latest | |
| timeout-minutes: 120 | |
| needs: ['preconditions', 'integration-tests'] | |
| if: ${{ needs.preconditions.outputs.docs_only != 'true' }} | |
| env: | |
| GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} | |
| CI_JDK_MAJOR_VERSION: ${{ needs.preconditions.outputs.jdk_major_version }} | |
| steps: | |
| - name: checkout | |
| uses: actions/checkout@v4 | |
| - name: Tune Runner VM | |
| uses: ./.github/actions/tune-runner-vm | |
| - name: Cache Maven dependencies | |
| uses: actions/cache@v4 | |
| timeout-minutes: 5 | |
| with: | |
| path: | | |
| ~/.m2/repository/*/*/* | |
| !~/.m2/repository/org/apache/pulsar | |
| key: ${{ runner.os }}-m2-dependencies-all-${{ hashFiles('**/pom.xml') }} | |
| restore-keys: | | |
| ${{ runner.os }}-m2-dependencies-all- | |
| - name: Set up JDK ${{ env.CI_JDK_MAJOR_VERSION }} | |
| uses: actions/setup-java@v4 | |
| with: | |
| distribution: ${{ env.JDK_DISTRIBUTION }} | |
| java-version: ${{ env.CI_JDK_MAJOR_VERSION }} | |
| - name: build package | |
| run: mvn -B clean package -DskipTests -T 1C -ntp | |
| codeql: | |
| name: Run CodeQL Analysis | |
| runs-on: ubuntu-22.04 | |
| timeout-minutes: 60 | |
| needs: ['preconditions', 'unit-tests'] | |
| if: ${{ (needs.preconditions.outputs.java_non_tests == 'true' || github.event_name != 'pull_request') && ((github.event_name == 'pull_request' && github.base_ref == 'master') || (github.event_name != 'pull_request' && github.ref_name == 'master')) }} | |
| permissions: | |
| actions: read | |
| contents: read | |
| security-events: write | |
| env: | |
| GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} | |
| CI_JDK_MAJOR_VERSION: ${{ needs.preconditions.outputs.jdk_major_version }} | |
| CODEQL_LANGUAGE: java-kotlin | |
| steps: | |
| - name: checkout | |
| uses: actions/checkout@v4 | |
| - name: Tune Runner VM | |
| uses: ./.github/actions/tune-runner-vm | |
| - name: Clean Disk when needed | |
| if: ${{ matrix.clean_disk }} | |
| uses: ./.github/actions/clean-disk | |
| - name: Setup ssh access to build runner VM | |
| # ssh access is enabled for builds in own forks | |
| if: ${{ github.repository != 'apache/pulsar' && github.event_name == 'pull_request' }} | |
| uses: ./.github/actions/ssh-access | |
| continue-on-error: true | |
| with: | |
| limit-access-to-actor: true | |
| - name: Cache local Maven repository | |
| uses: actions/cache@v4 | |
| timeout-minutes: 5 | |
| with: | |
| path: | | |
| ~/.m2/repository/*/*/* | |
| !~/.m2/repository/org/apache/pulsar | |
| key: ${{ runner.os }}-m2-dependencies-all-${{ hashFiles('**/pom.xml') }} | |
| restore-keys: | | |
| ${{ runner.os }}-m2-dependencies-core-modules-${{ hashFiles('**/pom.xml') }} | |
| ${{ runner.os }}-m2-dependencies-core-modules- | |
| - name: Set up JDK ${{ env.CI_JDK_MAJOR_VERSION }} | |
| uses: actions/setup-java@v4 | |
| with: | |
| distribution: ${{ env.JDK_DISTRIBUTION }} | |
| java-version: ${{ env.CI_JDK_MAJOR_VERSION }} | |
| - name: Initialize CodeQL | |
| uses: github/codeql-action/init@v3 | |
| with: | |
| languages: ${{ env.CODEQL_LANGUAGE }} | |
| - name: Build Java code | |
| run: | | |
| mvn -B -ntp -Pcore-modules,-main install -DskipTests -Dlicense.skip=true -Drat.skip=true -Dcheckstyle.skip=true | |
| - name: Perform CodeQL Analysis | |
| uses: github/codeql-action/analyze@v3 | |
| with: | |
| category: "/language:${{ env.CODEQL_LANGUAGE }}" | |
| owasp-dep-check: | |
| name: OWASP dependency check | |
| runs-on: ubuntu-22.04 | |
| timeout-minutes: 120 | |
| needs: [ 'preconditions', 'integration-tests' ] | |
| if: ${{ needs.preconditions.outputs.need_owasp == 'true' }} | |
| env: | |
| GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} | |
| CI_JDK_MAJOR_VERSION: ${{ needs.preconditions.outputs.jdk_major_version }} | |
| NIST_NVD_API_KEY: ${{ secrets.NIST_NVD_API_KEY }} | |
| steps: | |
| - name: checkout | |
| uses: actions/checkout@v4 | |
| - name: Tune Runner VM | |
| uses: ./.github/actions/tune-runner-vm | |
| - name: Setup ssh access to build runner VM | |
| # ssh access is enabled for builds in own forks | |
| if: ${{ github.repository != 'apache/pulsar' && github.event_name == 'pull_request' }} | |
| uses: ./.github/actions/ssh-access | |
| continue-on-error: true | |
| with: | |
| limit-access-to-actor: true | |
| - name: Restore Maven repository cache | |
| uses: actions/cache/restore@v4 | |
| timeout-minutes: 5 | |
| with: | |
| path: | | |
| ~/.m2/repository/*/*/* | |
| !~/.m2/repository/org/apache/pulsar | |
| key: ${{ runner.os }}-m2-dependencies-core-modules-${{ hashFiles('**/pom.xml') }} | |
| restore-keys: | | |
| ${{ runner.os }}-m2-dependencies-core-modules- | |
| - name: Set up JDK ${{ matrix.jdk || env.CI_JDK_MAJOR_VERSION }} | |
| uses: actions/setup-java@v4 | |
| with: | |
| distribution: ${{ env.JDK_DISTRIBUTION }} | |
| java-version: ${{ matrix.jdk || env.CI_JDK_MAJOR_VERSION }} | |
| - name: Clean Disk | |
| uses: ./.github/actions/clean-disk | |
| - name: Install gh-actions-artifact-client.js | |
| uses: apache/pulsar-test-infra/gh-actions-artifact-client/dist@master | |
| - name: Restore maven build results from Github artifact cache | |
| run: | | |
| cd $HOME | |
| $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh restore_tar_from_github_actions_artifacts pulsar-maven-repository-binaries | |
| - name: OWASP cache key weeknum | |
| id: get-weeknum | |
| run: | | |
| echo "weeknum=$(date -u +"%Y-%U")" >> $GITHUB_OUTPUT | |
| shell: bash | |
| - name: Restore OWASP Dependency Check data | |
| id: restore-owasp-dependency-check-data | |
| uses: actions/cache/restore@v4 | |
| timeout-minutes: 5 | |
| with: | |
| path: ~/.m2/repository/org/owasp/dependency-check-data | |
| key: owasp-dependency-check-data-${{ steps.get-weeknum.outputs.weeknum }} | |
| enableCrossOsArchive: true | |
| restore-keys: | | |
| owasp-dependency-check-data- | |
| # Projects dependent on flume, hdfs, and hbase currently excluded from the scan. | |
| - name: trigger dependency check | |
| run: | | |
| mvn -B -ntp verify -PskipDocker,skip-all,owasp-dependency-check -Dcheckstyle.skip=true -DskipTests \ | |
| -pl '!distribution/server,!distribution/io,!distribution/offloaders,!tiered-storage/file-system,!pulsar-io/flume,!pulsar-io/hbase,!pulsar-io/hdfs2,!pulsar-io/hdfs3,!pulsar-io/docs,!pulsar-io/jdbc/openmldb' | |
| - name: Upload report | |
| uses: actions/upload-artifact@v4 | |
| if: ${{ cancelled() || failure() }} | |
| continue-on-error: true | |
| with: | |
| name: dependency report | |
| path: target/dependency-check-report.html | |
| - name: Wait for ssh connection when build fails | |
| # ssh access is enabled for builds in own forks | |
| uses: ./.github/actions/ssh-access | |
| if: ${{ failure() && github.repository != 'apache/pulsar' && github.event_name == 'pull_request' }} | |
| continue-on-error: true | |
| with: | |
| action: wait | |
| # This job is required for pulls to be merged. This job is referenced by name in .asf.yaml file in the | |
| # protected_branches section for master branch required_status_checks. | |
| # It depends on all other jobs in this workflow. | |
| # This job also cleans up the binaries at the end of the workflow. | |
| pulsar-ci-checks-completed: | |
| name: "Pulsar CI checks completed" | |
| # run always, but skip for other repositories than apache/pulsar when a scheduled workflow is cancelled | |
| # this is to allow the workflow scheduled jobs to show as cancelled instead of failed since scheduled | |
| # jobs are not enabled for other than apache/pulsar repository. | |
| if: ${{ always() && !(cancelled() && github.repository != 'apache/pulsar' && github.event_name == 'schedule') }} | |
| runs-on: ubuntu-22.04 | |
| timeout-minutes: 10 | |
| needs: [ | |
| 'preconditions', | |
| 'unit-tests', | |
| 'integration-tests', | |
| 'system-tests', | |
| 'flaky-system-tests', | |
| 'macos-build', | |
| 'unit-tests-upload-coverage', | |
| 'integration-tests-upload-coverage', | |
| 'system-tests-upload-coverage', | |
| 'owasp-dep-check', | |
| 'codeql' | |
| ] | |
| steps: | |
| - name: Check that all required jobs were completed successfully | |
| if: ${{ needs.preconditions.result != 'success' || needs.preconditions.outputs.docs_only != 'true' }} | |
| run: | | |
| if [[ ! ( \ | |
| "${{ needs.preconditions.result }}" == "success" \ | |
| && "${{ needs.unit-tests.result }}" == "success" \ | |
| && "${{ needs.integration-tests.result }}" == "success" \ | |
| && "${{ needs.system-tests.result }}" == "success" \ | |
| && "${{ needs.macos-build.result }}" == "success" \ | |
| && ( "${{ needs.codeql.result }}" == "success" || "${{ needs.codeql.result }}" == "skipped" ) \ | |
| ) ]]; then | |
| echo "Required jobs haven't been completed successfully." | |
| exit 1 | |
| fi | |
| - name: checkout | |
| if: ${{ needs.preconditions.outputs.docs_only != 'true' }} | |
| uses: actions/checkout@v4 | |
| - name: Tune Runner VM | |
| if: ${{ needs.preconditions.outputs.docs_only != 'true' }} | |
| uses: ./.github/actions/tune-runner-vm | |
| - name: Install gh-actions-artifact-client.js | |
| if: ${{ needs.preconditions.outputs.docs_only != 'true' }} | |
| uses: apache/pulsar-test-infra/gh-actions-artifact-client/dist@master | |
| - name: Delete maven repository binaries from GitHub Actions Artifacts | |
| if: ${{ needs.preconditions.outputs.docs_only != 'true' && !contains(needs.*.result, 'failure') && !contains(needs.*.result, 'cancelled') }} | |
| run: | | |
| gh-actions-artifact-client.js delete pulsar-maven-repository-binaries.tar.zst || true | |
| gh-actions-artifact-client.js delete pulsar-server-distribution.tar.zst || true |