From 934e43558bd4f6ed546e076d27990b5d032426f5 Mon Sep 17 00:00:00 2001 From: Justin Florentine <justin+github@florentine.us> Date: Tue, 12 Mar 2024 18:24:22 -0400 Subject: [PATCH 1/7] Gha updates (#6705) * make artifacts more snapshot friendly * break out new workflows for snapshots, and a develop releease * removes checking for approval, runs on pr update * adds concurrency so updated refs cancel prior runs if still running * explicitly disable caching on gradle setup tasks --------- Signed-off-by: Justin Florentine <justin+github@florentine.us> --- .github/workflows/acceptance-tests.yml | 47 ++------- .github/workflows/artifacts.yml | 46 ++++++--- .github/workflows/codeql.yml | 16 ++- .github/workflows/develop.yml | 64 ++++++++++++ .github/workflows/docker.yml | 7 ++ .github/workflows/integration-tests.yml | 52 ++-------- .github/workflows/nightly.yml | 125 ------------------------ .github/workflows/pre-review.yml | 20 +++- .github/workflows/reference-tests.yml | 57 ++--------- .github/workflows/release.yml | 3 +- .github/workflows/sonarcloud.yml | 2 + 11 files changed, 154 insertions(+), 285 deletions(-) create mode 100644 .github/workflows/develop.yml delete mode 100644 .github/workflows/nightly.yml diff --git a/.github/workflows/acceptance-tests.yml b/.github/workflows/acceptance-tests.yml index 39af80ba6e6..bd89270942e 100644 --- a/.github/workflows/acceptance-tests.yml +++ b/.github/workflows/acceptance-tests.yml @@ -5,58 +5,22 @@ on: branches: - main - release-* - pull_request_review: - types: [submitted] + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true env: GRADLE_OPTS: "-Xmx6g -Dorg.gradle.daemon=false" total-runners: 16 jobs: - shouldRun: - name: checks to ensure we should run - # necessary because there is no single PR approved event, need to check all comments/approvals/denials - runs-on: ubuntu-22.04 - outputs: - shouldRun: ${{steps.shouldRun.outputs.result}} - steps: - - name: required check - id: shouldRun - uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea - env: - # fun fact, this changes based on incoming event, it will be different when we run this on pushes to main - RELEVANT_SHA: ${{ github.event.pull_request.head.sha || github.sha }} - with: - script: | - const { RELEVANT_SHA } = process.env; - const { data: { statuses } } = await github.rest.repos.getCombinedStatusForRef({ - owner: context.repo.owner, - repo: context.repo.repo, - ref: RELEVANT_SHA, - }); - const acceptanceTested = statuses && statuses.filter(({ context }) => context === 'accepttests-passed'); - const alreadyRun = acceptanceTested && acceptanceTested.find(({ state }) => state === 'success') > 0; - const { data: reviews } = await github.rest.pulls.listReviews({ - owner: context.repo.owner, - repo: context.repo.repo, - pull_number: context.issue.number, - }); - const approvingReviews = reviews && reviews.filter(review => review.state === 'APPROVED'); - const shouldRun = !alreadyRun && github.actor != 'dependabot[bot]' && (approvingReviews.length > 0); - - console.log("tests should be run = %j", shouldRun); - console.log("alreadyRun = %j", alreadyRun); - console.log("approvingReviews = %j", approvingReviews.length); - - return shouldRun; acceptanceTestEthereum: runs-on: ubuntu-22.04 name: "Acceptance Runner" - needs: shouldRun permissions: statuses: write checks: write - if: ${{ needs.shouldRun.outputs.shouldRun == 'true'}} strategy: fail-fast: true matrix: @@ -81,6 +45,8 @@ jobs: if_no_artifact_found: true - name: setup gradle uses: gradle/actions/setup-gradle@9e899d11ad247ec76be7a60bc1cf9d3abbb9e7f1 + with: + cache-disabled: true - name: Split tests id: split-tests uses: r7kamura/split-tests-by-timings@9322bd292d9423e2bc5a65bec548901801341e3f @@ -111,6 +77,7 @@ jobs: report_paths: 'acceptance-tests/tests/build/test-results/**/TEST-*.xml' annotate_only: true accepttests-passed: + name: "accepttests-passed" runs-on: ubuntu-22.04 needs: [ acceptanceTestEthereum ] permissions: diff --git a/.github/workflows/artifacts.yml b/.github/workflows/artifacts.yml index 6923a43a94e..6a737252bcc 100644 --- a/.github/workflows/artifacts.yml +++ b/.github/workflows/artifacts.yml @@ -1,11 +1,12 @@ -name: artifacts +name: release artifacts on: - workflow_dispatch: release: types: - prereleased +env: + GRADLE_OPTS: "-Dorg.gradle.parallel=true -Dorg.gradle.caching=true" jobs: artifacts: @@ -22,9 +23,11 @@ jobs: java-version: '17' - name: setup gradle uses: gradle/actions/setup-gradle@9e899d11ad247ec76be7a60bc1cf9d3abbb9e7f1 - - name: assemble distributions + with: + cache-disabled: true + - name: assemble release run: - ./gradlew -Prelease.releaseVersion=${{github.ref_name}} -Pversion=${{github.ref_name}} assemble -Dorg.gradle.parallel=true -Dorg.gradle.caching=true + ./gradlew -Prelease.releaseVersion=${{github.ref_name}} -Pversion=${{github.ref_name}} assemble - name: hashes id: hashes run: | @@ -43,16 +46,6 @@ jobs: path: 'build/distributions/besu*.zip' name: besu-${{ github.ref_name }}.zip compression-level: 0 - - name: Upload Release assets - uses: softprops/action-gh-release@de2c0eb89ae2a093876385947365aca7b0e5f844 - with: - append_body: true - files: | - build/distributions/besu*.tar.gz - build/distributions/besu*.zip - body: | - ${{steps.hashes.outputs.tarSha}} - ${{steps.hashes.outputs.zipSha}} testWindows: runs-on: windows-2022 needs: artifacts @@ -67,13 +60,34 @@ jobs: - name: Download zip uses: actions/download-artifact@eaceaf801fd36c7dee90939fad912460b18a1ffe with: - name: besu-${{ github.ref_name }}.zip + pattern: besu-*.zip + merge-multiple: true - name: test Besu run: | + dir unzip besu-*.zip -d besu-tmp cd besu-tmp mv besu-* ../besu cd .. besu\bin\besu.bat --help besu\bin\besu.bat --version - + publish: + runs-on: ubuntu-22.04 + needs: testWindows + steps: + - name: Download archives + uses: actions/download-artifact@eaceaf801fd36c7dee90939fad912460b18a1ffe + with: + pattern: besu-* + merge-multiple: true + path: 'build/distributions' + - name: Upload Release assets + uses: softprops/action-gh-release@de2c0eb89ae2a093876385947365aca7b0e5f844 + with: + append_body: true + files: | + build/distributions/besu*.tar.gz + build/distributions/besu*.zip + body: | + ${{steps.hashes.outputs.tarSha}} + ${{steps.hashes.outputs.zipSha}} \ No newline at end of file diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index f0340e168c7..0a50ec57c2d 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -13,15 +13,11 @@ name: "CodeQL" on: workflow_dispatch: - push: - branches: [ main ] - pull_request: - branches: [ main ] - paths-ignore: - - '**/*.json' - - '**/*.md' - - '**/*.properties' - - '**/*.txt' + schedule: + # * is a special character in YAML so you have to quote this string + # expression evaluates to midnight every night + - cron: '0 0 * * *' + jobs: analyze: name: Analyze @@ -50,6 +46,8 @@ jobs: - name: setup gradle uses: gradle/actions/setup-gradle@9e899d11ad247ec76be7a60bc1cf9d3abbb9e7f1 + with: + cache-disabled: true - name: compileJava noscan run: | JAVA_OPTS="-Xmx2048M" ./gradlew --no-scan compileJava diff --git a/.github/workflows/develop.yml b/.github/workflows/develop.yml new file mode 100644 index 00000000000..fbafc9e4f23 --- /dev/null +++ b/.github/workflows/develop.yml @@ -0,0 +1,64 @@ + +name: develop pre-release + +on: + push: + branches: + - main +env: + GRADLE_OPTS: "-Dorg.gradle.parallel=true -Dorg.gradle.caching=true" + +jobs: + artifacts: + runs-on: ubuntu-22.04 + permissions: + contents: write + steps: + - name: checkout + uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 + - name: Set up JDK 17 + uses: actions/setup-java@387ac29b308b003ca37ba93a6cab5eb57c8f5f93 + with: + distribution: 'temurin' + java-version: '17' + - name: setup gradle + uses: gradle/actions/setup-gradle@9e899d11ad247ec76be7a60bc1cf9d3abbb9e7f1 + with: + cache-disabled: true + - name: assemble release + run: + ./gradlew assemble + - name: upload tarball + uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 + with: + path: 'build/distributions/besu*.tar.gz' + name: besu-develop.tar.gz + compression-level: 0 + - name: hashes + id: hashes + run: | + cd build/distributions + echo "zipSha=$(shasum -a 256 besu*.zip)" >> $GITHUB_OUTPUT + echo "tarSha=$(shasum -a 256 besu*.tar.gz)" >> $GITHUB_OUTPUT + - name: upload zipfile + uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 + with: + path: 'build/distributions/besu*.zip' + name: besu-develop.zip + compression-level: 0 + - name: Upload Release assets + uses: softprops/action-gh-release@de2c0eb89ae2a093876385947365aca7b0e5f844 + with: + prerelease: true + name: develop + tag_name: develop + fail_on_unmatched_files: true + append_body: false + files: | + build/distributions/besu*.tar.gz + build/distributions/besu*.zip + body: | + This is an automated, bleeding edge build from the tip of ${{ github.ref_name }}. No promises. YOLO. + + ${{steps.hashes.outputs.tarSha}} + ${{steps.hashes.outputs.zipSha}} diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 8889b0dc7ad..3155667b9e6 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -19,6 +19,8 @@ jobs: java-version: 17 - name: setup gradle uses: gradle/actions/setup-gradle@9e899d11ad247ec76be7a60bc1cf9d3abbb9e7f1 + with: + cache-disabled: true - name: hadoLint_openj9-jdk_17 run: docker run --rm -i hadolint/hadolint < docker/openj9-jdk-17/Dockerfile - name: hadoLint_openjdk_17 @@ -66,6 +68,8 @@ jobs: java-version: 17 - name: setup gradle uses: gradle/actions/setup-gradle@9e899d11ad247ec76be7a60bc1cf9d3abbb9e7f1 + with: + cache-disabled: true - name: install goss run: | mkdir -p docker/reports @@ -81,6 +85,7 @@ jobs: env: architecture: ${{ steps.prep.outputs.ARCH }} with: + cache-disabled: true arguments: testDocker -PdockerOrgName=${{ env.registry }}/${{ secrets.DOCKER_ORG }} -Pversion=${{github.ref_name}} -Prelease.releaseVersion=${{ github.ref_name }} - name: publish env: @@ -102,6 +107,8 @@ jobs: java-version: 17 - name: setup gradle uses: gradle/actions/setup-gradle@9e899d11ad247ec76be7a60bc1cf9d3abbb9e7f1 + with: + cache-disabled: true - name: login to ${{ env.registry }} uses: docker/login-action@343f7c4344506bcbf9b4de18042ae17996df046d with: diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml index 80a2eca1e27..944422cbdf8 100644 --- a/.github/workflows/integration-tests.yml +++ b/.github/workflows/integration-tests.yml @@ -5,54 +5,18 @@ on: branches: - main - release-* - pull_request_review: - types: - - submitted + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true env: - GRADLE_OPTS: "-Xmx6g -Dorg.gradle.daemon=false" + GRADLE_OPTS: "-Xmx6g -Dorg.gradle.daemon=false -Dorg.gradle.parallel=true -Dorg.gradle.caching=true" jobs: - shouldRun: - name: checks to ensure we should run - runs-on: ubuntu-22.04 - outputs: - shouldRun: ${{steps.shouldRun.outputs.result}} - steps: - - name: required check - id: shouldRun - uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea - env: - # fun fact, this changes based on incoming event, it will be different when we run this on pushes to main - RELEVANT_SHA: ${{ github.event.pull_request.head.sha || github.sha }} - with: - script: | - const { RELEVANT_SHA } = process.env; - const { data: { statuses } } = await github.rest.repos.getCombinedStatusForRef({ - owner: context.repo.owner, - repo: context.repo.repo, - ref: RELEVANT_SHA, - }); - - const intTested = statuses && statuses.filter(({ context }) => context === 'integration-tests'); - const alreadyRun = intTested && intTested.find(({ state }) => state === 'success') > 0; - const { data: reviews } = await github.rest.pulls.listReviews({ - owner: context.repo.owner, - repo: context.repo.repo, - pull_number: context.issue.number, - }); - const approvingReviews = reviews && reviews.filter(review => review.state === 'APPROVED'); - const shouldRun = !alreadyRun && github.actor != 'dependabot[bot]' && (approvingReviews.length > 0); - - console.log("tests should be run = %j", shouldRun); - console.log("alreadyRun = %j", alreadyRun); - console.log("approvingReviews = %j", approvingReviews.length); - - return shouldRun; integration-tests: + name: "integration-passed" runs-on: ubuntu-22.04 - needs: shouldRun - if: ${{ needs.shouldRun.outputs.shouldRun == 'true' }} permissions: statuses: write checks: write @@ -68,8 +32,10 @@ jobs: java-version: 17 - name: setup gradle uses: gradle/actions/setup-gradle@9e899d11ad247ec76be7a60bc1cf9d3abbb9e7f1 + with: + cache-disabled: true - name: run integration tests - run: ./gradlew integrationTest compileJmh -Dorg.gradle.parallel=true -Dorg.gradle.caching=true + run: ./gradlew integrationTest compileJmh - name: Publish Test Report uses: mikepenz/action-junit-report@5f47764eec0e1c1f19f40c8e60a5ba47e47015c5 if: (success() || failure()) diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml deleted file mode 100644 index 3defa71ab5a..00000000000 --- a/.github/workflows/nightly.yml +++ /dev/null @@ -1,125 +0,0 @@ -name: nightly - -on: - workflow_dispatch: - schedule: - # * is a special character in YAML so you have to quote this string - # expression evaluates to midnight every night - - cron: '0 0 * * *' - -env: - nightly-tag: develop - registry: docker.io - -jobs: - hadolint: - runs-on: ubuntu-22.04 - steps: - - name: Checkout Repo - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 - - name: Set up Java - uses: actions/setup-java@387ac29b308b003ca37ba93a6cab5eb57c8f5f93 - with: - distribution: temurin - java-version: 17 - - name: setup gradle - uses: gradle/actions/setup-gradle@9e899d11ad247ec76be7a60bc1cf9d3abbb9e7f1 - - name: hadoLint_openj9-jdk_17 - run: docker run --rm -i hadolint/hadolint < docker/openj9-jdk-17/Dockerfile - - name: hadoLint_openjdk_17 - run: docker run --rm -i hadolint/hadolint < docker/openjdk-17/Dockerfile - - name: hadoLint_openjdk_17_debug - run: docker run --rm -i hadolint/hadolint < docker/openjdk-17-debug/Dockerfile - - name: hadoLint_openjdk_latest - run: docker run --rm -i hadolint/hadolint < docker/openjdk-latest/Dockerfile - - name: hadoLint_graalvm - run: docker run --rm -i hadolint/hadolint < docker/graalvm/Dockerfile - buildDocker: - needs: hadolint - permissions: - contents: read - packages: write - strategy: - fail-fast: false - matrix: - platform: - - ubuntu-22.04 - - [self-hosted, ARM64] - runs-on: ${{ matrix.platform }} - steps: - - name: Prepare - id: prep - run: | - platform=${{ matrix.platform }} - if [ "$platform" = 'ubuntu-22.04' ]; then - echo "PLATFORM_PAIR=linux-amd64" >> $GITHUB_OUTPUT - echo "ARCH=amd64" >> $GITHUB_OUTPUT - else - echo "PLATFORM_PAIR=linux-arm64" >> $GITHUB_OUTPUT - echo "ARCH=arm64" >> $GITHUB_OUTPUT - fi - - name: Checkout Repo - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 - - name: short sha - id: shortSha - run: echo "sha=$(git rev-parse --short HEAD)" >> $GITHUB_OUTPUT - - name: Set up Java - uses: actions/setup-java@387ac29b308b003ca37ba93a6cab5eb57c8f5f93 - with: - distribution: temurin - java-version: 17 - - name: login to ${{ env.registry }} - uses: docker/login-action@343f7c4344506bcbf9b4de18042ae17996df046d - with: - registry: ${{ env.registry }} - username: ${{ secrets.DOCKER_USER_RW }} - password: ${{ secrets.DOCKER_PASSWORD_RW }} - - name: build image - uses: gradle/actions/setup-gradle@9e899d11ad247ec76be7a60bc1cf9d3abbb9e7f1 - with: - arguments: distDocker -PdockerOrgName=${{ env.registry }}/${{ github.repository_owner }} -Pbranch=main - - name: install goss - run: | - mkdir -p docker/reports - curl -L https://github.com/aelsabbahy/goss/releases/download/v0.4.4/goss-${{ steps.prep.outputs.PLATFORM_PAIR }} -o ./docker/tests/goss-${{ steps.prep.outputs.PLATFORM_PAIR }} - - name: test docker - uses: gradle/actions/setup-gradle@9e899d11ad247ec76be7a60bc1cf9d3abbb9e7f1 - env: - architecture: ${{ steps.prep.outputs.ARCH }} - with: - arguments: testDocker -PdockerOrgName=${{ env.registry }}/${{ github.repository_owner }} -Pbranch=main - - name: login to $ {{ env.registry }} - uses: docker/login-action@343f7c4344506bcbf9b4de18042ae17996df046d - with: - registry: ${{ env.registry }} - username: ${{ secrets.DOCKER_USER_RW }} - password: ${{ secrets.DOCKER_PASSWORD_RW }} - - name: publish - env: - architecture: ${{ steps.prep.outputs.ARCH }} - run: ./gradlew --no-daemon dockerUpload -PdockerOrgName=${{ env.registry }}/${{ secrets.DOCKER_ORG }} -Pbranch=main - multiArch: - permissions: - contents: read - packages: write - needs: buildDocker - runs-on: ubuntu-22.04 - steps: - - name: Checkout Repo - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 - - name: Set up Java - uses: actions/setup-java@387ac29b308b003ca37ba93a6cab5eb57c8f5f93 - with: - distribution: temurin - java-version: 17 - - name: setup gradle - uses: gradle/actions/setup-gradle@9e899d11ad247ec76be7a60bc1cf9d3abbb9e7f1 - - name: Login to ${{ env.registry }} - uses: docker/login-action@343f7c4344506bcbf9b4de18042ae17996df046d - with: - registry: ${{ env.registry }} - username: ${{ secrets.DOCKER_USER_RW }} - password: ${{ secrets.DOCKER_PASSWORD_RW }} - - name: multi-arch docker - run: ./gradlew manifestDocker -PdockerOrgName=${{ env.registry }}/${{ secrets.DOCKER_ORG }} -Pbranch=main - diff --git a/.github/workflows/pre-review.yml b/.github/workflows/pre-review.yml index 0d3a3e6c900..b3adc7c42cd 100644 --- a/.github/workflows/pre-review.yml +++ b/.github/workflows/pre-review.yml @@ -6,6 +6,13 @@ on: - main - release-* +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +env: + GRADLE_OPTS: "-Xmx6g -Dorg.gradle.daemon=false -Dorg.gradle.parallel=true" + jobs: repolint: name: "Repository Linting" @@ -41,8 +48,10 @@ jobs: java-version: 17 - name: Setup Gradle uses: gradle/actions/setup-gradle@9e899d11ad247ec76be7a60bc1cf9d3abbb9e7f1 + with: + cache-disabled: true - name: run spotless - run: ./gradlew spotlessCheck -Dorg.gradle.parallel=true -Dorg.gradle.caching=true + run: ./gradlew spotlessCheck compile: runs-on: ubuntu-22.04 timeout-minutes: 30 @@ -59,8 +68,10 @@ jobs: java-version: 17 - name: Setup Gradle uses: gradle/actions/setup-gradle@9e899d11ad247ec76be7a60bc1cf9d3abbb9e7f1 + with: + cache-disabled: true - name: Gradle Compile - run: ./gradlew build -x test -x spotlessCheck -Dorg.gradle.parallel=true -Dorg.gradle.caching=true + run: ./gradlew build -x test -x spotlessCheck unitTests: env: GRADLEW_UNIT_TEST_ARGS: ${{matrix.gradle_args}} @@ -92,9 +103,11 @@ jobs: java-version: 17 - name: Setup Gradle uses: gradle/actions/setup-gradle@9e899d11ad247ec76be7a60bc1cf9d3abbb9e7f1 + with: + cache-disabled: true - name: run unit tests id: unitTest - run: ./gradlew $GRADLEW_UNIT_TEST_ARGS -Dorg.gradle.parallel=true -Dorg.gradle.caching=true + run: ./gradlew $GRADLEW_UNIT_TEST_ARGS - name: Publish Test Report uses: mikepenz/action-junit-report@5f47764eec0e1c1f19f40c8e60a5ba47e47015c5 if: success() || failure() # always run even if the build step fails @@ -102,6 +115,7 @@ jobs: report_paths: '**/test-results/**/TEST-*.xml' annotate_only: true unittests-passed: + name: "unittests-passed" runs-on: ubuntu-22.04 needs: [unitTests] permissions: diff --git a/.github/workflows/reference-tests.yml b/.github/workflows/reference-tests.yml index ccb03632db3..f9dd87c4589 100644 --- a/.github/workflows/reference-tests.yml +++ b/.github/workflows/reference-tests.yml @@ -5,63 +5,22 @@ on: branches: - main - release-* - pull_request_review: - types: [ submitted ] env: - GRADLE_OPTS: "-Xmx6g -Dorg.gradle.daemon=false" + GRADLE_OPTS: "-Xmx6g -Dorg.gradle.daemon=false -Dorg.gradle.parallel=true -Dorg.gradle.caching=true" total-runners: 10 -jobs: - shouldRun: - name: checks to ensure we should run - # necessary because there is no single PR approved event, need to check all comments/approvals/denials - # might also be a job running, and additional approvals - runs-on: ubuntu-22.04 - outputs: - shouldRun: ${{steps.shouldRun.outputs.result}} - steps: - - name: required check - id: shouldRun - uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea - env: - # fun fact, this changes based on incoming event, it will be different when we run this on pushes to main - RELEVANT_SHA: ${{ github.event.pull_request.head.sha || github.sha }} - with: - script: | - const { RELEVANT_SHA } = process.env; - const { data: { statuses } } = await github.rest.repos.getCombinedStatusForRef({ - owner: context.repo.owner, - repo: context.repo.repo, - ref: RELEVANT_SHA, - }); - - - const refTested = statuses && statuses.filter(({ context }) => context === 'reftests-passed'); - const alreadyRun = refTested && refTested.find(({ state }) => state === 'success') > 0; - const { data: reviews } = await github.rest.pulls.listReviews({ - owner: context.repo.owner, - repo: context.repo.repo, - pull_number: context.issue.number, - }); - const approvingReviews = reviews && reviews.filter(review => review.state === 'APPROVED'); - const shouldRun = !alreadyRun && github.actor != 'dependabot[bot]' && (approvingReviews.length > 0); - - console.log("tests should be run = %j", shouldRun); - console.log("alreadyRun = %j", alreadyRun); - console.log("approvingReviews = %j", approvingReviews.length); - - return shouldRun; +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true +jobs: referenceTestEthereum: runs-on: ubuntu-22.04 permissions: statuses: write checks: write packages: read - needs: - - shouldRun - if: ${{ needs.shouldRun.outputs.shouldRun == 'true' }} strategy: fail-fast: true matrix: @@ -79,7 +38,8 @@ jobs: java-version: 17 - name: setup gradle uses: gradle/actions/setup-gradle@9e899d11ad247ec76be7a60bc1cf9d3abbb9e7f1 - #shame the test generation isn't less redundant, we used to do this in a dependent job, but artifact downloading broke + with: + cache-disabled: true - name: execute generate reference tests run: ./gradlew ethereum:referencetests:blockchainReferenceTests ethereum:referencetests:generalstateReferenceTests ethereum:referencetests:generalstateRegressionReferenceTests -Dorg.gradle.parallel=true -Dorg.gradle.caching=true - name: list test files generated @@ -95,7 +55,7 @@ jobs: - name: refTestArgs.txt run: cat refTestArgs.txt - name: run reference tests - run: ./gradlew ethereum:referenceTests:referenceTests `cat refTestArgs.txt` -Dorg.gradle.parallel=true -Dorg.gradle.caching=true + run: ./gradlew ethereum:referenceTests:referenceTests `cat refTestArgs.txt` - name: Upload Test Report uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 if: always() # always run even if the previous step fails @@ -109,6 +69,7 @@ jobs: report_paths: '**/build/test-results/referenceTests/TEST-*.xml' annotate_only: true reftests-passed: + name: "reftests-passed" runs-on: ubuntu-22.04 needs: [ referenceTestEthereum ] permissions: diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index ceb8ba602cb..3ae4a0ddf8d 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -1,6 +1,5 @@ name: release besu on: - workflow_dispatch: release: types: [released] env: @@ -23,6 +22,8 @@ jobs: password: ${{ secrets.DOCKER_PASSWORD_RW }} - name: Setup Gradle uses: gradle/actions/setup-gradle@9e899d11ad247ec76be7a60bc1cf9d3abbb9e7f1 + with: + cache-disabled: true - name: Docker upload run: ./gradlew "-Prelease.releaseVersion=${{ github.ref_name }}" "-PdockerOrgName=${{ env.registry }}/${{ secrets.DOCKER_ORG }}" dockerUploadRelease - name: Docker manifest diff --git a/.github/workflows/sonarcloud.yml b/.github/workflows/sonarcloud.yml index 7891a6819db..3ef37c1dafa 100644 --- a/.github/workflows/sonarcloud.yml +++ b/.github/workflows/sonarcloud.yml @@ -30,6 +30,8 @@ jobs: restore-keys: ${{ runner.os }}-sonar - name: setup gradle uses: gradle/actions/setup-gradle@9e899d11ad247ec76be7a60bc1cf9d3abbb9e7f1 + with: + cache-disabled: true - name: Build and analyze env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} # Needed to get PR information, if any From c39ca6d44f40a4f122e05b186a211857ec17138e Mon Sep 17 00:00:00 2001 From: garyschulte <garyschulte@gmail.com> Date: Tue, 12 Mar 2024 18:19:37 -0700 Subject: [PATCH 2/7] Support pruned chain history in peer validators (#6698) * dao fork block presence is optional in peer validator to support peers with pruned chain history Signed-off-by: garyschulte <garyschulte@gmail.com> * Update ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/peervalidation/AbstractPeerBlockValidator.java Co-authored-by: Sally MacFarlane <macfarla.github@gmail.com> Signed-off-by: garyschulte <garyschulte@gmail.com> --------- Signed-off-by: garyschulte <garyschulte@gmail.com> Co-authored-by: Sally MacFarlane <macfarla.github@gmail.com> --- .../AbstractPeerBlockValidator.java | 24 ++++++++++++++----- .../peervalidation/DaoForkPeerValidator.java | 9 +++++++ .../DaoForkPeerValidatorTest.java | 24 +++++++++++++++++++ .../RequiredBlocksPeerValidatorTest.java | 22 +++++++++++++++++ 4 files changed, 73 insertions(+), 6 deletions(-) diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/peervalidation/AbstractPeerBlockValidator.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/peervalidation/AbstractPeerBlockValidator.java index 55c3dd89e56..4483e14cdfa 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/peervalidation/AbstractPeerBlockValidator.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/peervalidation/AbstractPeerBlockValidator.java @@ -86,12 +86,20 @@ public CompletableFuture<Boolean> validatePeer( } final List<BlockHeader> headers = res.getResult(); if (headers.size() == 0) { - // If no headers are returned, fail - LOG.debug( - "Peer {} is invalid because required block ({}) is unavailable.", - ethPeer, - blockNumber); - return false; + if (blockIsRequired()) { + // If no headers are returned, fail + LOG.debug( + "Peer {} is invalid because required block ({}) is unavailable.", + ethPeer, + blockNumber); + return false; + } else { + LOG.debug( + "Peer {} deemed valid because unavailable block ({}) is not required.", + ethPeer, + blockNumber); + return true; + } } final BlockHeader header = headers.get(0); return validateBlockHeader(ethPeer, header); @@ -105,6 +113,10 @@ public boolean canBeValidated(final EthPeer ethPeer) { return ethPeer.chainState().getEstimatedHeight() >= (blockNumber + chainHeightEstimationBuffer); } + protected boolean blockIsRequired() { + return true; + } + @Override public Duration nextValidationCheckTimeout(final EthPeer ethPeer) { if (!ethPeer.chainState().hasEstimatedHeight()) { diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/peervalidation/DaoForkPeerValidator.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/peervalidation/DaoForkPeerValidator.java index d038ad0e9a9..01ce6144ee0 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/peervalidation/DaoForkPeerValidator.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/peervalidation/DaoForkPeerValidator.java @@ -49,4 +49,13 @@ boolean validateBlockHeader(final EthPeer ethPeer, final BlockHeader header) { } return validDaoBlock; } + + /** + * In order to support chain history pruning, clients do not need to have the dao fork block to be + * deemed valid. + */ + @Override + protected boolean blockIsRequired() { + return false; + } } diff --git a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/peervalidation/DaoForkPeerValidatorTest.java b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/peervalidation/DaoForkPeerValidatorTest.java index 8586874c434..042ca8f9605 100644 --- a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/peervalidation/DaoForkPeerValidatorTest.java +++ b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/peervalidation/DaoForkPeerValidatorTest.java @@ -98,4 +98,28 @@ public void validatePeer_responsivePeerOnWrongSideOfFork() { assertThat(result).isDone(); assertThat(result).isCompletedWithValue(false); } + + @Test + public void validatePeer_responsivePeerDoesNotHaveBlockWhenPastForkHeight() { + final EthProtocolManager ethProtocolManager = EthProtocolManagerTestUtil.create(); + final long daoBlockNumber = 500; + + final PeerValidator validator = + new DaoForkPeerValidator( + ProtocolScheduleFixture.MAINNET, new NoOpMetricsSystem(), daoBlockNumber, 0); + + final RespondingEthPeer peer = + EthProtocolManagerTestUtil.createPeer(ethProtocolManager, daoBlockNumber); + + final CompletableFuture<Boolean> result = + validator.validatePeer(ethProtocolManager.ethContext(), peer.getEthPeer()); + + assertThat(result).isNotDone(); + + // Respond to block header request with empty + peer.respond(RespondingEthPeer.emptyResponder()); + + assertThat(result).isDone(); + assertThat(result).isCompletedWithValue(true); + } } diff --git a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/peervalidation/RequiredBlocksPeerValidatorTest.java b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/peervalidation/RequiredBlocksPeerValidatorTest.java index 67c80345880..87b3c7077cc 100644 --- a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/peervalidation/RequiredBlocksPeerValidatorTest.java +++ b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/peervalidation/RequiredBlocksPeerValidatorTest.java @@ -104,4 +104,26 @@ public void validatePeer_responsivePeerWithBadRequiredBlock() { assertThat(result).isDone(); assertThat(result).isCompletedWithValue(false); } + + @Test + public void validatePeer_responsivePeerDoesNotHaveBlockWhenPastForkHeight() { + final EthProtocolManager ethProtocolManager = EthProtocolManagerTestUtil.create(); + + final PeerValidator validator = + new RequiredBlocksPeerValidator( + ProtocolScheduleFixture.MAINNET, new NoOpMetricsSystem(), 1, Hash.ZERO); + + final RespondingEthPeer peer = EthProtocolManagerTestUtil.createPeer(ethProtocolManager, 1); + + final CompletableFuture<Boolean> result = + validator.validatePeer(ethProtocolManager.ethContext(), peer.getEthPeer()); + + assertThat(result).isNotDone(); + + // Respond to block header request with empty + peer.respond(RespondingEthPeer.emptyResponder()); + + assertThat(result).isDone(); + assertThat(result).isCompletedWithValue(false); + } } From 501d169db5b2dfc95339d8813d0ae4aa336c6445 Mon Sep 17 00:00:00 2001 From: Sally MacFarlane <macfarla.github@gmail.com> Date: Wed, 13 Mar 2024 12:08:24 +1000 Subject: [PATCH 3/7] edits to changelog for 24.3.0 (#6712) Signed-off-by: Sally MacFarlane <macfarla.github@gmail.com> --- CHANGELOG.md | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ee32ad05f85..6141beee13f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,7 +4,7 @@ ### Breaking Changes - RocksDB database metadata format has changed to be more expressive, the migration of an existing metadata file to the new format is automatic at startup. Before performing a downgrade to a previous version it is mandatory to revert to the original format using the subcommand `besu --data-path=/path/to/besu/datadir storage revert-metadata v2-to-v1`. -- PoA networks won't start with SNAP or CHECKPOINT sync (previously Besu would start with this config but quietly fail to sync, so it's now more obvious that it won't work) [#6625](https://github.com/hyperledger/besu/pull/6625) +- BFT networks won't start with SNAP or CHECKPOINT sync (previously Besu would start with this config but quietly fail to sync, so it's now more obvious that it won't work) [#6625](https://github.com/hyperledger/besu/pull/6625), [#6667](https://github.com/hyperledger/besu/pull/6667) ### Upcoming Breaking Changes @@ -18,8 +18,6 @@ - Make layered txpool aware of minGasPrice and minPriorityFeePerGas dynamic options [#6611](https://github.com/hyperledger/besu/pull/6611) - Update commons-compress to 1.26.0 [#6648](https://github.com/hyperledger/besu/pull/6648) - Update Vert.x to 4.5.4 [#6666](https://github.com/hyperledger/besu/pull/6666) -- Add blob transaction support to `eth_call` [#6661](https://github.com/hyperledger/besu/pull/6661) -- Add blobs to `eth_feeHistory` [#6679](https://github.com/hyperledger/besu/pull/6679) - Refactor and extend `TransactionPoolValidatorService` [#6636](https://github.com/hyperledger/besu/pull/6636) - Transaction call object to accept both `input` and `data` field simultaneously if they are set to equal values [#6702](https://github.com/hyperledger/besu/pull/6702) @@ -29,7 +27,7 @@ ### Download Links -## 24.2.0-SNAPSHOT +## 24.3.0 ### Breaking Changes - SNAP - Snap sync is now the default for named networks [#6530](https://github.com/hyperledger/besu/pull/6530) @@ -46,7 +44,7 @@ - Release docker images now provided at ghcr.io instead of dockerhub ### Deprecations -- X_SNAP and X_CHECKPOINT are marked for deprecation and will be removed in 24.4.0 in favor of SNAP and CHECKPOINT [#6405](https://github.com/hyperledger/besu/pull/6405) +- X_SNAP and X_CHECKPOINT are marked for deprecation and will be removed in 24.6.0 in favor of SNAP and CHECKPOINT [#6405](https://github.com/hyperledger/besu/pull/6405) - `--Xp2p-peer-lower-bound` is deprecated. [#6501](https://github.com/hyperledger/besu/pull/6501) ### Upcoming Breaking Changes @@ -73,6 +71,8 @@ - More accurate column size `storage rocksdb usage` subcommand [#6540](https://github.com/hyperledger/besu/pull/6540) - Adds `storage rocksdb x-stats` subcommand [#6540](https://github.com/hyperledger/besu/pull/6540) - New `eth_blobBaseFee`JSON-RPC method [#6581](https://github.com/hyperledger/besu/pull/6581) +- Add blob transaction support to `eth_call` [#6661](https://github.com/hyperledger/besu/pull/6661) +- Add blobs to `eth_feeHistory` [#6679](https://github.com/hyperledger/besu/pull/6679) - Upgrade reference tests to version 13.1 [#6574](https://github.com/hyperledger/besu/pull/6574) - Extend `BesuConfiguration` service [#6584](https://github.com/hyperledger/besu/pull/6584) - Add `ethereum_min_gas_price` and `ethereum_min_priority_fee` metrics to track runtime values of `min-gas-price` and `min-priority-fee` [#6587](https://github.com/hyperledger/besu/pull/6587) @@ -86,6 +86,9 @@ - Fix traces so that call gas costing in traces matches other clients traces [#6525](https://github.com/hyperledger/besu/pull/6525) ### Download Links +https://github.com/hyperledger/besu/releases/tag/24.3.0 +https://github.com/hyperledger/besu/releases/download/24.3.0/besu-24.3.0.tar.gz / sha256 8037ce51bb5bb396d29717a812ea7ff577b0d6aa341d67d1e5b77cbc55b15f84 +https://github.com/hyperledger/besu/releases/download/24.3.0/besu-24.3.0.zip / sha256 41ea2ca734a3b377f43ee178166b5b809827084789378dbbe4e5b52bbd8e0674 ## 24.1.2 From 602999f4cff6960ba29da41d76a932f5166d8601 Mon Sep 17 00:00:00 2001 From: Usman Saleem <usman@usmans.info> Date: Wed, 13 Mar 2024 13:01:00 +1000 Subject: [PATCH 4/7] Refactor BadBlockReason enum out of BadBlockCause (#6716) * Refactor BadBlockReason enum out of BadBlockCause Signed-off-by: Usman Saleem <usman@usmans.info> * Deleting ununsed enum value Signed-off-by: Usman Saleem <usman@usmans.info> --------- Signed-off-by: Usman Saleem <usman@usmans.info> Co-authored-by: Sally MacFarlane <macfarla.github@gmail.com> --- .../besu/ethereum/chain/BadBlockCause.java | 15 +++++-------- .../besu/ethereum/chain/BadBlockReason.java | 22 +++++++++++++++++++ 2 files changed, 27 insertions(+), 10 deletions(-) create mode 100644 ethereum/core/src/main/java/org/hyperledger/besu/ethereum/chain/BadBlockReason.java diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/chain/BadBlockCause.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/chain/BadBlockCause.java index 7a014e99cd2..601e96de25f 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/chain/BadBlockCause.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/chain/BadBlockCause.java @@ -15,19 +15,14 @@ */ package org.hyperledger.besu.ethereum.chain; +import static org.hyperledger.besu.ethereum.chain.BadBlockReason.DESCENDS_FROM_BAD_BLOCK; +import static org.hyperledger.besu.ethereum.chain.BadBlockReason.SPEC_VALIDATION_FAILURE; + import org.hyperledger.besu.ethereum.core.Block; import com.google.common.base.MoreObjects; public class BadBlockCause { - public enum BadBlockReason { - // Standard spec-related validation failures - SPEC_VALIDATION_FAILURE, - // When an unexpected exception occurs during block processing - EXCEPTIONAL_BLOCK_PROCESSING, - // This block is bad because it descends from a bad block - DESCENDS_FROM_BAD_BLOCK, - } private final BadBlockReason reason; private final String description; @@ -35,11 +30,11 @@ public enum BadBlockReason { public static BadBlockCause fromBadAncestorBlock(final Block badAncestor) { final String description = String.format("Descends from bad block %s", badAncestor.toLogString()); - return new BadBlockCause(BadBlockReason.DESCENDS_FROM_BAD_BLOCK, description); + return new BadBlockCause(DESCENDS_FROM_BAD_BLOCK, description); } public static BadBlockCause fromValidationFailure(final String failureMessage) { - return new BadBlockCause(BadBlockReason.SPEC_VALIDATION_FAILURE, failureMessage); + return new BadBlockCause(SPEC_VALIDATION_FAILURE, failureMessage); } private BadBlockCause(final BadBlockReason reason, final String description) { diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/chain/BadBlockReason.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/chain/BadBlockReason.java new file mode 100644 index 00000000000..f38655cb830 --- /dev/null +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/chain/BadBlockReason.java @@ -0,0 +1,22 @@ +/* + * Copyright Hyperledger Besu Contributors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + */ +package org.hyperledger.besu.ethereum.chain; + +public enum BadBlockReason { + // Standard spec-related validation failures + SPEC_VALIDATION_FAILURE, + // This block is bad because it descends from a bad block + DESCENDS_FROM_BAD_BLOCK, +} From e56a37da110e78a843460f6d6580521b787401cb Mon Sep 17 00:00:00 2001 From: Sally MacFarlane <macfarla.github@gmail.com> Date: Wed, 13 Mar 2024 14:26:48 +1000 Subject: [PATCH 5/7] verify syncing false (#6720) Signed-off-by: Sally MacFarlane <macfarla.github@gmail.com> --- .../jsonrpc/EthSendRawTransactionAcceptanceTest.java | 4 ++++ .../pubsub/NewPendingTransactionAcceptanceTest.java | 3 +++ 2 files changed, 7 insertions(+) diff --git a/acceptance-tests/tests/src/test/java/org/hyperledger/besu/tests/acceptance/jsonrpc/EthSendRawTransactionAcceptanceTest.java b/acceptance-tests/tests/src/test/java/org/hyperledger/besu/tests/acceptance/jsonrpc/EthSendRawTransactionAcceptanceTest.java index 4e4330da03b..c054a2a1653 100644 --- a/acceptance-tests/tests/src/test/java/org/hyperledger/besu/tests/acceptance/jsonrpc/EthSendRawTransactionAcceptanceTest.java +++ b/acceptance-tests/tests/src/test/java/org/hyperledger/besu/tests/acceptance/jsonrpc/EthSendRawTransactionAcceptanceTest.java @@ -44,6 +44,10 @@ public void setUp() throws Exception { strictNode = besu.createArchiveNode("strictNode", configureNode((true))); miningNode = besu.createMinerNode("strictMiningNode", configureNode((true))); cluster.start(lenientNode, strictNode, miningNode); + // verify all nodes are done syncing so the tx pool will be enabled + lenientNode.verify(eth.syncingStatus(false)); + strictNode.verify(eth.syncingStatus(false)); + miningNode.verify(eth.syncingStatus(false)); } @Test diff --git a/acceptance-tests/tests/src/test/java/org/hyperledger/besu/tests/acceptance/pubsub/NewPendingTransactionAcceptanceTest.java b/acceptance-tests/tests/src/test/java/org/hyperledger/besu/tests/acceptance/pubsub/NewPendingTransactionAcceptanceTest.java index 4ae94924b25..9c0368abb88 100644 --- a/acceptance-tests/tests/src/test/java/org/hyperledger/besu/tests/acceptance/pubsub/NewPendingTransactionAcceptanceTest.java +++ b/acceptance-tests/tests/src/test/java/org/hyperledger/besu/tests/acceptance/pubsub/NewPendingTransactionAcceptanceTest.java @@ -44,6 +44,9 @@ public void setUp() throws Exception { accountOne = accounts.createAccount("account-one"); minerWebSocket = new WebSocket(vertx, minerNode.getConfiguration()); archiveWebSocket = new WebSocket(vertx, archiveNode.getConfiguration()); + // verify all nodes are done syncing so the tx pool will be enabled + archiveNode.verify(eth.syncingStatus(false)); + minerNode.verify(eth.syncingStatus(false)); } @AfterEach From efd1bc7070228786f1e1b02657a5fe2fe9cca743 Mon Sep 17 00:00:00 2001 From: Fabio Di Fabio <fabio.difabio@consensys.net> Date: Wed, 13 Mar 2024 11:48:18 +0100 Subject: [PATCH 6/7] Fix txpool dump/restore race condition (#6665) Signed-off-by: Fabio Di Fabio <fabio.difabio@consensys.net> --- CHANGELOG.md | 1 + .../eth/transactions/TransactionPool.java | 34 ++++++++----------- 2 files changed, 15 insertions(+), 20 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6141beee13f..2e974eec715 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -22,6 +22,7 @@ - Transaction call object to accept both `input` and `data` field simultaneously if they are set to equal values [#6702](https://github.com/hyperledger/besu/pull/6702) ### Bug fixes +- Fix txpool dump/restore race condition [#6665](https://github.com/hyperledger/besu/pull/6665) - Make block transaction selection max time aware of PoA transitions [#6676](https://github.com/hyperledger/besu/pull/6676) - Don't enable the BFT mining coordinator when running sub commands such as `blocks export` [#6675](https://github.com/hyperledger/besu/pull/6675) diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/transactions/TransactionPool.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/transactions/TransactionPool.java index 0396d0dea4f..bb079756d57 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/transactions/TransactionPool.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/transactions/TransactionPool.java @@ -64,12 +64,11 @@ import java.util.concurrent.CompletableFuture; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ExecutionException; +import java.util.concurrent.Semaphore; import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicReference; -import java.util.concurrent.locks.Lock; -import java.util.concurrent.locks.ReentrantLock; import java.util.function.Supplier; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -664,7 +663,7 @@ private void onAdded(final Transaction transaction) { } class SaveRestoreManager { - private final Lock diskAccessLock = new ReentrantLock(); + private final Semaphore diskAccessLock = new Semaphore(1, true); private final AtomicReference<CompletableFuture<Void>> writeInProgress = new AtomicReference<>(CompletableFuture.completedFuture(null)); private final AtomicReference<CompletableFuture<Void>> readInProgress = @@ -685,25 +684,20 @@ private CompletableFuture<Void> serializeAndDedupOperation( final AtomicReference<CompletableFuture<Void>> operationInProgress) { if (configuration.getEnableSaveRestore()) { try { - if (diskAccessLock.tryLock(1, TimeUnit.MINUTES)) { - try { - if (!operationInProgress.get().isDone()) { - isCancelled.set(true); - try { - operationInProgress.get().get(); - } catch (ExecutionException ee) { - // nothing to do - } + if (diskAccessLock.tryAcquire(1, TimeUnit.MINUTES)) { + if (!operationInProgress.get().isDone()) { + isCancelled.set(true); + try { + operationInProgress.get().get(); + } catch (ExecutionException ee) { + // nothing to do } - - isCancelled.set(false); - operationInProgress.set(CompletableFuture.runAsync(operation)); - return operationInProgress.get(); - } catch (InterruptedException ie) { - isCancelled.set(false); - } finally { - diskAccessLock.unlock(); } + + isCancelled.set(false); + operationInProgress.set( + CompletableFuture.runAsync(operation).thenRun(diskAccessLock::release)); + return operationInProgress.get(); } else { CompletableFuture.failedFuture( new TimeoutException("Timeout waiting for disk access lock")); From 4cc6b744cf2cc13d0db22a8e7e47614fc2cfc5aa Mon Sep 17 00:00:00 2001 From: Fabio Di Fabio <fabio.difabio@consensys.net> Date: Wed, 13 Mar 2024 16:54:38 +0100 Subject: [PATCH 7/7] Transaction simulation service (#6686) Signed-off-by: Fabio Di Fabio <fabio.difabio@consensys.net> --- CHANGELOG.md | 1 + .../tests/acceptance/dsl/node/BesuNode.java | 1 + .../dsl/node/ThreadBesuNodeRunner.java | 55 +++++++++++- .../node/configuration/BesuNodeFactory.java | 19 ++++- .../NodeConfigurationFactory.java | 8 +- .../dsl/transaction/NodeRequests.java | 9 +- besu/build.gradle | 2 + .../org/hyperledger/besu/cli/BesuCommand.java | 17 ++++ .../TransactionSimulationServiceImpl.java | 84 +++++++++++++++++++ .../java/org/hyperledger/besu/RunnerTest.java | 11 ++- .../chainimport/JsonBlockImporterTest.java | 3 +- .../besu/cli/CommandTestAbstract.java | 2 + .../ValidatorContractController.java | 2 +- .../internal/methods/AbstractEstimateGas.java | 4 +- .../internal/methods/DebugTraceCall.java | 2 +- .../api/jsonrpc/internal/methods/EthCall.java | 2 +- .../internal/methods/EthEstimateGas.java | 2 +- .../jsonrpc/internal/methods/TraceCall.java | 2 +- .../internal/methods/TraceCallMany.java | 2 +- .../internal/methods/TraceRawTransaction.java | 2 +- .../jsonrpc/internal/methods/EthCallTest.java | 6 +- .../methods/EthCreateAccessListTest.java | 2 +- .../internal/methods/EthEstimateGasTest.java | 2 +- .../AbstractBlockTransactionSelectorTest.java | 4 + .../TransactionProcessingResult.java | 27 ++++++ .../ethereum/transaction/CallParameter.java | 51 ++++++++++- .../transaction/TransactionSimulator.java | 19 ++++- .../TransactionSimulatorResult.java | 52 +----------- .../MainnetTransactionValidatorTest.java | 4 +- .../PermissionTransactionValidatorTest.java | 3 - ...eSmartContractPermissioningController.java | 2 +- ...martContractV2PermissioningController.java | 2 +- ...nSmartContractPermissioningController.java | 2 +- plugin-api/build.gradle | 2 +- .../data/TransactionProcessingResult.java | 7 ++ .../data/TransactionSimulationResult.java | 54 ++++++++++++ .../TransactionSimulationService.java | 42 ++++++++++ .../besu/testutil/JsonTestParameters.java | 8 +- 38 files changed, 424 insertions(+), 95 deletions(-) create mode 100644 besu/src/main/java/org/hyperledger/besu/services/TransactionSimulationServiceImpl.java create mode 100644 plugin-api/src/main/java/org/hyperledger/besu/plugin/data/TransactionSimulationResult.java create mode 100644 plugin-api/src/main/java/org/hyperledger/besu/plugin/services/TransactionSimulationService.java diff --git a/CHANGELOG.md b/CHANGELOG.md index 2e974eec715..33d5d447852 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,6 +20,7 @@ - Update Vert.x to 4.5.4 [#6666](https://github.com/hyperledger/besu/pull/6666) - Refactor and extend `TransactionPoolValidatorService` [#6636](https://github.com/hyperledger/besu/pull/6636) - Transaction call object to accept both `input` and `data` field simultaneously if they are set to equal values [#6702](https://github.com/hyperledger/besu/pull/6702) +- Introduce `TransactionSimulationService` [#6686](https://github.com/hyperledger/besu/pull/6686) ### Bug fixes - Fix txpool dump/restore race condition [#6665](https://github.com/hyperledger/besu/pull/6665) diff --git a/acceptance-tests/dsl/src/main/java/org/hyperledger/besu/tests/acceptance/dsl/node/BesuNode.java b/acceptance-tests/dsl/src/main/java/org/hyperledger/besu/tests/acceptance/dsl/node/BesuNode.java index ec9d7b1173e..c822ce899ce 100644 --- a/acceptance-tests/dsl/src/main/java/org/hyperledger/besu/tests/acceptance/dsl/node/BesuNode.java +++ b/acceptance-tests/dsl/src/main/java/org/hyperledger/besu/tests/acceptance/dsl/node/BesuNode.java @@ -436,6 +436,7 @@ public NodeRequests nodeRequests() { nodeRequests = new NodeRequests( + web3jService, new JsonRpc2_0Web3j(web3jService, 2000, Async.defaultExecutorService()), new CliqueRequestFactory(web3jService), new BftRequestFactory(web3jService, bftType), diff --git a/acceptance-tests/dsl/src/main/java/org/hyperledger/besu/tests/acceptance/dsl/node/ThreadBesuNodeRunner.java b/acceptance-tests/dsl/src/main/java/org/hyperledger/besu/tests/acceptance/dsl/node/ThreadBesuNodeRunner.java index 87246069e6d..896a615dee7 100644 --- a/acceptance-tests/dsl/src/main/java/org/hyperledger/besu/tests/acceptance/dsl/node/ThreadBesuNodeRunner.java +++ b/acceptance-tests/dsl/src/main/java/org/hyperledger/besu/tests/acceptance/dsl/node/ThreadBesuNodeRunner.java @@ -28,6 +28,7 @@ import org.hyperledger.besu.cryptoservices.KeyPairSecurityModule; import org.hyperledger.besu.cryptoservices.NodeKey; import org.hyperledger.besu.ethereum.GasLimitCalculator; +import org.hyperledger.besu.ethereum.api.ApiConfiguration; import org.hyperledger.besu.ethereum.api.graphql.GraphQLConfiguration; import org.hyperledger.besu.ethereum.core.ImmutableMiningParameters; import org.hyperledger.besu.ethereum.eth.EthProtocolConfiguration; @@ -37,6 +38,7 @@ import org.hyperledger.besu.ethereum.p2p.peers.EnodeURLImpl; import org.hyperledger.besu.ethereum.storage.keyvalue.KeyValueStorageProvider; import org.hyperledger.besu.ethereum.storage.keyvalue.KeyValueStorageProviderBuilder; +import org.hyperledger.besu.ethereum.transaction.TransactionSimulator; import org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration; import org.hyperledger.besu.evm.internal.EvmConfiguration; import org.hyperledger.besu.metrics.MetricsSystemFactory; @@ -44,6 +46,7 @@ import org.hyperledger.besu.plugin.data.EnodeURL; import org.hyperledger.besu.plugin.services.BesuConfiguration; import org.hyperledger.besu.plugin.services.BesuEvents; +import org.hyperledger.besu.plugin.services.BlockchainService; import org.hyperledger.besu.plugin.services.PermissioningService; import org.hyperledger.besu.plugin.services.PicoCLIOptions; import org.hyperledger.besu.plugin.services.PrivacyPluginService; @@ -52,10 +55,12 @@ import org.hyperledger.besu.plugin.services.StorageService; import org.hyperledger.besu.plugin.services.TransactionPoolValidatorService; import org.hyperledger.besu.plugin.services.TransactionSelectionService; +import org.hyperledger.besu.plugin.services.TransactionSimulationService; import org.hyperledger.besu.plugin.services.storage.rocksdb.RocksDBPlugin; import org.hyperledger.besu.services.BesuConfigurationImpl; import org.hyperledger.besu.services.BesuEventsImpl; import org.hyperledger.besu.services.BesuPluginContextImpl; +import org.hyperledger.besu.services.BlockchainServiceImpl; import org.hyperledger.besu.services.PermissioningServiceImpl; import org.hyperledger.besu.services.PicoCLIOptionsImpl; import org.hyperledger.besu.services.PrivacyPluginServiceImpl; @@ -64,6 +69,7 @@ import org.hyperledger.besu.services.StorageServiceImpl; import org.hyperledger.besu.services.TransactionPoolValidatorServiceImpl; import org.hyperledger.besu.services.TransactionSelectionServiceImpl; +import org.hyperledger.besu.services.TransactionSimulationServiceImpl; import java.io.File; import java.nio.file.Path; @@ -95,18 +101,27 @@ private BesuPluginContextImpl buildPluginContext( final BesuNode node, final StorageServiceImpl storageService, final SecurityModuleServiceImpl securityModuleService, + final TransactionSimulationServiceImpl transactionSimulationServiceImpl, final TransactionSelectionServiceImpl transactionSelectionServiceImpl, + final TransactionPoolValidatorServiceImpl transactionPoolValidatorServiceImpl, + final BlockchainServiceImpl blockchainServiceImpl, + final RpcEndpointServiceImpl rpcEndpointServiceImpl, final BesuConfiguration commonPluginConfiguration) { final CommandLine commandLine = new CommandLine(CommandSpec.create()); final BesuPluginContextImpl besuPluginContext = new BesuPluginContextImpl(); besuPluginContext.addService(StorageService.class, storageService); besuPluginContext.addService(SecurityModuleService.class, securityModuleService); besuPluginContext.addService(PicoCLIOptions.class, new PicoCLIOptionsImpl(commandLine)); - besuPluginContext.addService(RpcEndpointService.class, new RpcEndpointServiceImpl()); + besuPluginContext.addService(RpcEndpointService.class, rpcEndpointServiceImpl); besuPluginContext.addService( TransactionSelectionService.class, transactionSelectionServiceImpl); besuPluginContext.addService( - TransactionPoolValidatorService.class, new TransactionPoolValidatorServiceImpl()); + TransactionPoolValidatorService.class, transactionPoolValidatorServiceImpl); + besuPluginContext.addService( + TransactionSimulationService.class, transactionSimulationServiceImpl); + besuPluginContext.addService(BlockchainService.class, blockchainServiceImpl); + besuPluginContext.addService(BesuConfiguration.class, commonPluginConfiguration); + final Path pluginsPath; final String pluginDir = System.getProperty("besu.plugins.dir"); if (pluginDir == null || pluginDir.isEmpty()) { @@ -147,8 +162,14 @@ public void startNode(final BesuNode node) { final StorageServiceImpl storageService = new StorageServiceImpl(); final SecurityModuleServiceImpl securityModuleService = new SecurityModuleServiceImpl(); + final TransactionSimulationServiceImpl transactionSimulationServiceImpl = + new TransactionSimulationServiceImpl(); final TransactionSelectionServiceImpl transactionSelectionServiceImpl = new TransactionSelectionServiceImpl(); + final TransactionPoolValidatorServiceImpl transactionPoolValidatorServiceImpl = + new TransactionPoolValidatorServiceImpl(); + final BlockchainServiceImpl blockchainServiceImpl = new BlockchainServiceImpl(); + final RpcEndpointServiceImpl rpcEndpointServiceImpl = new RpcEndpointServiceImpl(); final Path dataDir = node.homeDirectory(); final BesuConfigurationImpl commonPluginConfiguration = new BesuConfigurationImpl(); final var miningParameters = @@ -169,7 +190,11 @@ public void startNode(final BesuNode node) { node, storageService, securityModuleService, + transactionSimulationServiceImpl, transactionSelectionServiceImpl, + transactionPoolValidatorServiceImpl, + blockchainServiceImpl, + rpcEndpointServiceImpl, commonPluginConfiguration)); GlobalOpenTelemetry.resetForTest(); @@ -203,6 +228,7 @@ public void startNode(final BesuNode node) { ImmutableTransactionPoolConfiguration.builder() .from(node.getTransactionPoolConfiguration()) .strictTransactionReplayProtectionEnabled(node.isStrictTxReplayProtectionEnabled()) + .transactionPoolValidatorService(transactionPoolValidatorServiceImpl) .build(); final int maxPeers = 25; @@ -236,6 +262,10 @@ public void startNode(final BesuNode node) { final BesuController besuController = builder.build(); + initTransactionSimulationService( + transactionSimulationServiceImpl, besuController, node.getApiConfiguration()); + initBlockchainService(blockchainServiceImpl, besuController); + final RunnerBuilder runnerBuilder = new RunnerBuilder(); runnerBuilder.permissioningConfiguration(node.getPermissioningConfiguration()); runnerBuilder.apiConfiguration(node.getApiConfiguration()); @@ -265,7 +295,7 @@ public void startNode(final BesuNode node) { .besuPluginContext(new BesuPluginContextImpl()) .autoLogBloomCaching(false) .storageProvider(storageProvider) - .rpcEndpointService(new RpcEndpointServiceImpl()); + .rpcEndpointService(rpcEndpointServiceImpl); node.engineRpcConfiguration().ifPresent(runnerBuilder::engineJsonRpcConfiguration); final Runner runner = runnerBuilder.build(); @@ -289,6 +319,25 @@ public void startNode(final BesuNode node) { MDC.remove("node"); } + private void initBlockchainService( + final BlockchainServiceImpl blockchainServiceImpl, final BesuController besuController) { + blockchainServiceImpl.init( + besuController.getProtocolContext(), besuController.getProtocolSchedule()); + } + + private void initTransactionSimulationService( + final TransactionSimulationServiceImpl transactionSimulationService, + final BesuController besuController, + final ApiConfiguration apiConfiguration) { + transactionSimulationService.init( + besuController.getProtocolContext().getBlockchain(), + new TransactionSimulator( + besuController.getProtocolContext().getBlockchain(), + besuController.getProtocolContext().getWorldStateArchive(), + besuController.getProtocolSchedule(), + apiConfiguration.getGasCap())); + } + @Override public void stopNode(final BesuNode node) { final BesuPluginContextImpl pluginContext = besuPluginContextMap.remove(node); diff --git a/acceptance-tests/dsl/src/main/java/org/hyperledger/besu/tests/acceptance/dsl/node/configuration/BesuNodeFactory.java b/acceptance-tests/dsl/src/main/java/org/hyperledger/besu/tests/acceptance/dsl/node/configuration/BesuNodeFactory.java index 65d257fbfb7..ff6dc2ac9b1 100644 --- a/acceptance-tests/dsl/src/main/java/org/hyperledger/besu/tests/acceptance/dsl/node/configuration/BesuNodeFactory.java +++ b/acceptance-tests/dsl/src/main/java/org/hyperledger/besu/tests/acceptance/dsl/node/configuration/BesuNodeFactory.java @@ -49,6 +49,7 @@ import java.util.ArrayList; import java.util.List; import java.util.Optional; +import java.util.Set; import java.util.function.UnaryOperator; import io.vertx.core.Vertx; @@ -376,17 +377,27 @@ public BesuNode createCliqueNode(final String name) throws IOException { public BesuNode createCliqueNode(final String name, final CliqueOptions cliqueOptions) throws IOException { - return createCliqueNodeWithExtraCliOptions(name, cliqueOptions, List.of()); + return createCliqueNodeWithExtraCliOptionsAndRpcApis(name, cliqueOptions, List.of()); } - public BesuNode createCliqueNodeWithExtraCliOptions( + public BesuNode createCliqueNodeWithExtraCliOptionsAndRpcApis( final String name, final CliqueOptions cliqueOptions, final List<String> extraCliOptions) throws IOException { + return createCliqueNodeWithExtraCliOptionsAndRpcApis( + name, cliqueOptions, extraCliOptions, Set.of()); + } + + public BesuNode createCliqueNodeWithExtraCliOptionsAndRpcApis( + final String name, + final CliqueOptions cliqueOptions, + final List<String> extraCliOptions, + final Set<String> extraRpcApis) + throws IOException { return create( new BesuNodeConfigurationBuilder() .name(name) .miningEnabled() - .jsonRpcConfiguration(node.createJsonRpcWithCliqueEnabledConfig()) + .jsonRpcConfiguration(node.createJsonRpcWithCliqueEnabledConfig(extraRpcApis)) .webSocketConfiguration(node.createWebSocketEnabledConfig()) .devMode(false) .jsonRpcTxPool() @@ -584,7 +595,7 @@ public BesuNode createCliqueNodeWithValidators(final String name, final String.. new BesuNodeConfigurationBuilder() .name(name) .miningEnabled() - .jsonRpcConfiguration(node.createJsonRpcWithCliqueEnabledConfig()) + .jsonRpcConfiguration(node.createJsonRpcWithCliqueEnabledConfig(Set.of())) .webSocketConfiguration(node.createWebSocketEnabledConfig()) .jsonRpcTxPool() .devMode(false) diff --git a/acceptance-tests/dsl/src/main/java/org/hyperledger/besu/tests/acceptance/dsl/node/configuration/NodeConfigurationFactory.java b/acceptance-tests/dsl/src/main/java/org/hyperledger/besu/tests/acceptance/dsl/node/configuration/NodeConfigurationFactory.java index f2682993f8a..219d15d1adf 100644 --- a/acceptance-tests/dsl/src/main/java/org/hyperledger/besu/tests/acceptance/dsl/node/configuration/NodeConfigurationFactory.java +++ b/acceptance-tests/dsl/src/main/java/org/hyperledger/besu/tests/acceptance/dsl/node/configuration/NodeConfigurationFactory.java @@ -30,8 +30,10 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; +import java.util.HashSet; import java.util.List; import java.util.Optional; +import java.util.Set; public class NodeConfigurationFactory { @@ -44,8 +46,10 @@ public Optional<String> createGenesisConfigForValidators( return genesisConfigProvider.create(nodes); } - public JsonRpcConfiguration createJsonRpcWithCliqueEnabledConfig() { - return createJsonRpcWithRpcApiEnabledConfig(CLIQUE.name()); + public JsonRpcConfiguration createJsonRpcWithCliqueEnabledConfig(final Set<String> extraRpcApis) { + final var enabledApis = new HashSet<>(extraRpcApis); + enabledApis.add(CLIQUE.name()); + return createJsonRpcWithRpcApiEnabledConfig(enabledApis.toArray(String[]::new)); } public JsonRpcConfiguration createJsonRpcWithIbft2EnabledConfig(final boolean minerEnabled) { diff --git a/acceptance-tests/dsl/src/main/java/org/hyperledger/besu/tests/acceptance/dsl/transaction/NodeRequests.java b/acceptance-tests/dsl/src/main/java/org/hyperledger/besu/tests/acceptance/dsl/transaction/NodeRequests.java index 4741e397ddc..1151100065b 100644 --- a/acceptance-tests/dsl/src/main/java/org/hyperledger/besu/tests/acceptance/dsl/transaction/NodeRequests.java +++ b/acceptance-tests/dsl/src/main/java/org/hyperledger/besu/tests/acceptance/dsl/transaction/NodeRequests.java @@ -27,10 +27,11 @@ import java.util.Optional; import org.web3j.protocol.Web3j; +import org.web3j.protocol.Web3jService; import org.web3j.protocol.websocket.WebSocketService; public class NodeRequests { - + private final Web3jService web3jService; private final Web3j netEth; private final CliqueRequestFactory clique; private final BftRequestFactory bft; @@ -44,6 +45,7 @@ public class NodeRequests { private final TxPoolRequestFactory txPool; public NodeRequests( + final Web3jService web3jService, final Web3j netEth, final CliqueRequestFactory clique, final BftRequestFactory bft, @@ -55,6 +57,7 @@ public NodeRequests( final TxPoolRequestFactory txPool, final Optional<WebSocketService> websocketService, final LoginRequestFactory login) { + this.web3jService = web3jService; this.netEth = netEth; this.clique = clique; this.bft = bft; @@ -116,4 +119,8 @@ public void shutdown() { netEth.shutdown(); websocketService.ifPresent(WebSocketService::close); } + + public Web3jService getWeb3jService() { + return web3jService; + } } diff --git a/besu/build.gradle b/besu/build.gradle index 0b31c4a43c2..85c17e7ff35 100644 --- a/besu/build.gradle +++ b/besu/build.gradle @@ -28,6 +28,8 @@ jar { } dependencies { + api project(':datatypes') + api 'org.slf4j:slf4j-api' implementation project(':config') diff --git a/besu/src/main/java/org/hyperledger/besu/cli/BesuCommand.java b/besu/src/main/java/org/hyperledger/besu/cli/BesuCommand.java index 6709608834e..d5b6468a90c 100644 --- a/besu/src/main/java/org/hyperledger/besu/cli/BesuCommand.java +++ b/besu/src/main/java/org/hyperledger/besu/cli/BesuCommand.java @@ -139,6 +139,7 @@ import org.hyperledger.besu.ethereum.storage.keyvalue.KeyValueSegmentIdentifier; import org.hyperledger.besu.ethereum.storage.keyvalue.KeyValueStorageProvider; import org.hyperledger.besu.ethereum.storage.keyvalue.KeyValueStorageProviderBuilder; +import org.hyperledger.besu.ethereum.transaction.TransactionSimulator; import org.hyperledger.besu.ethereum.trie.forest.pruner.PrunerConfiguration; import org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration; import org.hyperledger.besu.evm.precompile.AbstractAltBnPrecompiledContract; @@ -167,6 +168,7 @@ import org.hyperledger.besu.plugin.services.TraceService; import org.hyperledger.besu.plugin.services.TransactionPoolValidatorService; import org.hyperledger.besu.plugin.services.TransactionSelectionService; +import org.hyperledger.besu.plugin.services.TransactionSimulationService; import org.hyperledger.besu.plugin.services.exception.StorageException; import org.hyperledger.besu.plugin.services.metrics.MetricCategory; import org.hyperledger.besu.plugin.services.metrics.MetricCategoryRegistry; @@ -187,6 +189,7 @@ import org.hyperledger.besu.services.TraceServiceImpl; import org.hyperledger.besu.services.TransactionPoolValidatorServiceImpl; import org.hyperledger.besu.services.TransactionSelectionServiceImpl; +import org.hyperledger.besu.services.TransactionSimulationServiceImpl; import org.hyperledger.besu.services.kvstore.InMemoryStoragePlugin; import org.hyperledger.besu.util.InvalidConfigurationException; import org.hyperledger.besu.util.LogConfigurator; @@ -370,6 +373,7 @@ public class BesuCommand implements DefaultCommandValues, Runnable { private final TransactionSelectionServiceImpl transactionSelectionServiceImpl; private final TransactionPoolValidatorServiceImpl transactionValidatorServiceImpl; + private final TransactionSimulationServiceImpl transactionSimulationServiceImpl; private final BlockchainServiceImpl blockchainServiceImpl; static class P2PDiscoveryOptionGroup { @@ -956,6 +960,7 @@ public BesuCommand( new RpcEndpointServiceImpl(), new TransactionSelectionServiceImpl(), new TransactionPoolValidatorServiceImpl(), + new TransactionSimulationServiceImpl(), new BlockchainServiceImpl()); } @@ -978,6 +983,7 @@ public BesuCommand( * @param rpcEndpointServiceImpl instance of RpcEndpointServiceImpl * @param transactionSelectionServiceImpl instance of TransactionSelectionServiceImpl * @param transactionValidatorServiceImpl instance of TransactionValidatorServiceImpl + * @param transactionSimulationServiceImpl instance of TransactionSimulationServiceImpl * @param blockchainServiceImpl instance of BlockchainServiceImpl */ @VisibleForTesting @@ -998,6 +1004,7 @@ protected BesuCommand( final RpcEndpointServiceImpl rpcEndpointServiceImpl, final TransactionSelectionServiceImpl transactionSelectionServiceImpl, final TransactionPoolValidatorServiceImpl transactionValidatorServiceImpl, + final TransactionSimulationServiceImpl transactionSimulationServiceImpl, final BlockchainServiceImpl blockchainServiceImpl) { this.besuComponent = besuComponent; this.logger = besuComponent.getBesuCommandLogger(); @@ -1018,6 +1025,7 @@ protected BesuCommand( this.rpcEndpointServiceImpl = rpcEndpointServiceImpl; this.transactionSelectionServiceImpl = transactionSelectionServiceImpl; this.transactionValidatorServiceImpl = transactionValidatorServiceImpl; + this.transactionSimulationServiceImpl = transactionSimulationServiceImpl; this.blockchainServiceImpl = blockchainServiceImpl; } @@ -1210,6 +1218,8 @@ private void preparePlugins() { TransactionSelectionService.class, transactionSelectionServiceImpl); besuPluginContext.addService( TransactionPoolValidatorService.class, transactionValidatorServiceImpl); + besuPluginContext.addService( + TransactionSimulationService.class, transactionSimulationServiceImpl); besuPluginContext.addService(BlockchainService.class, blockchainServiceImpl); // register built-in plugins @@ -1293,6 +1303,13 @@ private Runner buildRunner() { private void startPlugins() { blockchainServiceImpl.init( besuController.getProtocolContext(), besuController.getProtocolSchedule()); + transactionSimulationServiceImpl.init( + besuController.getProtocolContext().getBlockchain(), + new TransactionSimulator( + besuController.getProtocolContext().getBlockchain(), + besuController.getProtocolContext().getWorldStateArchive(), + besuController.getProtocolSchedule(), + apiConfiguration.getGasCap())); besuPluginContext.addService( BesuEvents.class, diff --git a/besu/src/main/java/org/hyperledger/besu/services/TransactionSimulationServiceImpl.java b/besu/src/main/java/org/hyperledger/besu/services/TransactionSimulationServiceImpl.java new file mode 100644 index 00000000000..5ebf48f0cef --- /dev/null +++ b/besu/src/main/java/org/hyperledger/besu/services/TransactionSimulationServiceImpl.java @@ -0,0 +1,84 @@ +/* + * Copyright Hyperledger Besu Contributors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + */ +package org.hyperledger.besu.services; + +import org.hyperledger.besu.datatypes.Hash; +import org.hyperledger.besu.datatypes.Transaction; +import org.hyperledger.besu.ethereum.chain.Blockchain; +import org.hyperledger.besu.ethereum.mainnet.ImmutableTransactionValidationParams; +import org.hyperledger.besu.ethereum.mainnet.TransactionValidationParams; +import org.hyperledger.besu.ethereum.transaction.CallParameter; +import org.hyperledger.besu.ethereum.transaction.TransactionSimulator; +import org.hyperledger.besu.evm.tracing.OperationTracer; +import org.hyperledger.besu.plugin.Unstable; +import org.hyperledger.besu.plugin.data.TransactionSimulationResult; +import org.hyperledger.besu.plugin.services.TransactionSimulationService; + +import java.util.Optional; + +/** TransactionSimulationServiceImpl */ +@Unstable +public class TransactionSimulationServiceImpl implements TransactionSimulationService { + private static final TransactionValidationParams SIMULATOR_ALLOWING_EXCEEDING_BALANCE = + ImmutableTransactionValidationParams.builder() + .from(TransactionValidationParams.transactionSimulator()) + .isAllowExceedingBalance(true) + .build(); + private Blockchain blockchain; + private TransactionSimulator transactionSimulator; + + /** Create an instance to be configured */ + public TransactionSimulationServiceImpl() {} + + /** + * Configure the service + * + * @param blockchain the blockchain + * @param transactionSimulator transaction simulator + */ + public void init(final Blockchain blockchain, final TransactionSimulator transactionSimulator) { + this.blockchain = blockchain; + this.transactionSimulator = transactionSimulator; + } + + @Override + public Optional<TransactionSimulationResult> simulate( + final Transaction transaction, + final Hash blockHash, + final OperationTracer operationTracer, + final boolean isAllowExceedingBalance) { + + final CallParameter callParameter = CallParameter.fromTransaction(transaction); + + final var blockHeader = + blockchain + .getBlockHeader(blockHash) + .or(() -> blockchain.getBlockHeaderSafe(blockHash)) + .orElseThrow( + () -> + new IllegalStateException( + "Block header not yet present for chain head hash: " + blockHash)); + + return transactionSimulator + .process( + callParameter, + isAllowExceedingBalance + ? SIMULATOR_ALLOWING_EXCEEDING_BALANCE + : TransactionValidationParams.transactionSimulator(), + operationTracer, + blockHeader) + .map(res -> new TransactionSimulationResult(transaction, res.result())); + } +} diff --git a/besu/src/test/java/org/hyperledger/besu/RunnerTest.java b/besu/src/test/java/org/hyperledger/besu/RunnerTest.java index d624064e5c1..55cdee554bd 100644 --- a/besu/src/test/java/org/hyperledger/besu/RunnerTest.java +++ b/besu/src/test/java/org/hyperledger/besu/RunnerTest.java @@ -180,7 +180,8 @@ private void syncFromGenesis(final SyncMode mode, final GenesisConfigFile genesi aheadDbNodeKey, createKeyValueStorageProvider( dataDirAhead, dbAhead, dataStorageConfiguration, miningParameters), - noOpMetricsSystem); + noOpMetricsSystem, + miningParameters); setupState( blockCount, controllerAhead.getProtocolSchedule(), controllerAhead.getProtocolContext()); @@ -235,7 +236,8 @@ private void syncFromGenesis(final SyncMode mode, final GenesisConfigFile genesi dataDirBehind, behindDbNodeKey, new InMemoryKeyValueStorageProvider(), - noOpMetricsSystem); + noOpMetricsSystem, + miningParameters); final EnodeURL aheadEnode = runnerAhead.getLocalEnode().get(); final EthNetworkConfig behindEthNetworkConfiguration = @@ -452,14 +454,15 @@ private BesuController getController( final Path dataDir, final NodeKey nodeKey, final StorageProvider storageProvider, - final ObservableMetricsSystem metricsSystem) { + final ObservableMetricsSystem metricsSystem, + final MiningParameters miningParameters) { return new MainnetBesuControllerBuilder() .genesisConfigFile(genesisConfig) .synchronizerConfiguration(syncConfig) .ethProtocolConfiguration(EthProtocolConfiguration.defaultConfig()) .dataDirectory(dataDir) .networkId(NETWORK_ID) - .miningParameters(MiningParameters.newDefault()) + .miningParameters(miningParameters) .nodeKey(nodeKey) .storageProvider(storageProvider) .metricsSystem(metricsSystem) diff --git a/besu/src/test/java/org/hyperledger/besu/chainimport/JsonBlockImporterTest.java b/besu/src/test/java/org/hyperledger/besu/chainimport/JsonBlockImporterTest.java index 4b59491dc18..dff42ef654a 100644 --- a/besu/src/test/java/org/hyperledger/besu/chainimport/JsonBlockImporterTest.java +++ b/besu/src/test/java/org/hyperledger/besu/chainimport/JsonBlockImporterTest.java @@ -432,8 +432,7 @@ protected BesuController createController() throws IOException { return createController(genesisConfigFile); } - protected BesuController createController(final GenesisConfigFile genesisConfigFile) - throws IOException { + protected BesuController createController(final GenesisConfigFile genesisConfigFile) { return new BesuController.Builder() .fromGenesisConfig(genesisConfigFile, SyncMode.FAST) .synchronizerConfiguration(SynchronizerConfiguration.builder().build()) diff --git a/besu/src/test/java/org/hyperledger/besu/cli/CommandTestAbstract.java b/besu/src/test/java/org/hyperledger/besu/cli/CommandTestAbstract.java index d784733608f..63f664d7469 100644 --- a/besu/src/test/java/org/hyperledger/besu/cli/CommandTestAbstract.java +++ b/besu/src/test/java/org/hyperledger/besu/cli/CommandTestAbstract.java @@ -92,6 +92,7 @@ import org.hyperledger.besu.services.StorageServiceImpl; import org.hyperledger.besu.services.TransactionPoolValidatorServiceImpl; import org.hyperledger.besu.services.TransactionSelectionServiceImpl; +import org.hyperledger.besu.services.TransactionSimulationServiceImpl; import org.hyperledger.besu.services.kvstore.InMemoryKeyValueStorage; import java.io.ByteArrayOutputStream; @@ -573,6 +574,7 @@ public static class TestBesuCommand extends BesuCommand { rpcEndpointServiceImpl, new TransactionSelectionServiceImpl(), new TransactionPoolValidatorServiceImpl(), + new TransactionSimulationServiceImpl(), new BlockchainServiceImpl()); } diff --git a/consensus/qbft/src/main/java/org/hyperledger/besu/consensus/qbft/validator/ValidatorContractController.java b/consensus/qbft/src/main/java/org/hyperledger/besu/consensus/qbft/validator/ValidatorContractController.java index a7b705e4a9d..e6786a92ef1 100644 --- a/consensus/qbft/src/main/java/org/hyperledger/besu/consensus/qbft/validator/ValidatorContractController.java +++ b/consensus/qbft/src/main/java/org/hyperledger/besu/consensus/qbft/validator/ValidatorContractController.java @@ -107,7 +107,7 @@ private List<Type> decodeResult( if (result.isSuccessful()) { final List<Type> decodedList = FunctionReturnDecoder.decode( - result.getResult().getOutput().toHexString(), function.getOutputParameters()); + result.result().getOutput().toHexString(), function.getOutputParameters()); if (decodedList.isEmpty()) { throw new IllegalStateException( diff --git a/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/AbstractEstimateGas.java b/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/AbstractEstimateGas.java index b3de5094c8f..0f9b1b7be14 100644 --- a/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/AbstractEstimateGas.java +++ b/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/AbstractEstimateGas.java @@ -90,7 +90,7 @@ protected long processEstimateGas( Math.pow(SUB_CALL_REMAINING_GAS_RATIO, operationTracer.getMaxDepth()); // and minimum gas remaining is necessary for some operation (additionalStipend) final long gasStipend = operationTracer.getStipendNeeded(); - final long gasUsedByTransaction = result.getResult().getEstimateGasUsedByTransaction(); + final long gasUsedByTransaction = result.result().getEstimateGasUsedByTransaction(); return ((long) ((gasUsedByTransaction + gasStipend) * subCallMultiplier)); } @@ -123,7 +123,7 @@ protected JsonRpcErrorResponse errorResponse( JsonRpcErrorConverter.convertTransactionInvalidReason( validationResult.getInvalidReason())); } else { - final TransactionProcessingResult resultTrx = result.getResult(); + final TransactionProcessingResult resultTrx = result.result(); if (resultTrx != null && resultTrx.getRevertReason().isPresent()) { return errorResponse( request, diff --git a/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/DebugTraceCall.java b/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/DebugTraceCall.java index ae5215282f8..72aa575b0cb 100644 --- a/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/DebugTraceCall.java +++ b/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/DebugTraceCall.java @@ -83,7 +83,7 @@ protected PreCloseStateHandler<Object> getSimulatorResultHandler( final TransactionTrace transactionTrace = new TransactionTrace( - result.getTransaction(), result.getResult(), tracer.getTraceFrames()); + result.transaction(), result.result(), tracer.getTraceFrames()); return new DebugTraceTransactionResult(transactionTrace); }); diff --git a/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/EthCall.java b/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/EthCall.java index 569563ca932..698685f7457 100644 --- a/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/EthCall.java +++ b/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/EthCall.java @@ -117,7 +117,7 @@ private JsonRpcErrorResponse errorResponse( JsonRpcErrorConverter.convertTransactionInvalidReason( validationResult.getInvalidReason())); } else { - final TransactionProcessingResult resultTrx = result.getResult(); + final TransactionProcessingResult resultTrx = result.result(); if (resultTrx != null && resultTrx.getRevertReason().isPresent()) { return errorResponse( request, diff --git a/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/EthEstimateGas.java b/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/EthEstimateGas.java index 9a382d6441b..c0631bf44cb 100644 --- a/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/EthEstimateGas.java +++ b/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/EthEstimateGas.java @@ -85,7 +85,7 @@ public JsonRpcResponse response(final JsonRpcRequestContext requestContext) { return errorResponse(requestContext, gasUsed.get()); } - var low = gasUsed.get().getResult().getEstimateGasUsedByTransaction(); + var low = gasUsed.get().result().getEstimateGasUsedByTransaction(); var lowResult = executeSimulation( blockHeader, diff --git a/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/TraceCall.java b/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/TraceCall.java index 439140fa315..06559af1efa 100644 --- a/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/TraceCall.java +++ b/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/TraceCall.java @@ -73,7 +73,7 @@ protected PreCloseStateHandler<Object> getSimulatorResultHandler( final TransactionTrace transactionTrace = new TransactionTrace( - result.getTransaction(), result.getResult(), tracer.getTraceFrames()); + result.transaction(), result.result(), tracer.getTraceFrames()); final Block block = blockchainQueriesSupplier.get().getBlockchain().getChainHeadBlock(); diff --git a/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/TraceCallMany.java b/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/TraceCallMany.java index a824b35f1eb..611ebee8b04 100644 --- a/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/TraceCallMany.java +++ b/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/TraceCallMany.java @@ -166,7 +166,7 @@ private JsonNode getSingleCallResult( final TransactionTrace transactionTrace = new TransactionTrace( - simulatorResult.getTransaction(), simulatorResult.getResult(), tracer.getTraceFrames()); + simulatorResult.transaction(), simulatorResult.result(), tracer.getTraceFrames()); final Block block = blockchainQueriesSupplier.get().getBlockchain().getChainHeadBlock(); diff --git a/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/TraceRawTransaction.java b/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/TraceRawTransaction.java index 26da6f2f13d..ad34d92b490 100644 --- a/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/TraceRawTransaction.java +++ b/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/TraceRawTransaction.java @@ -102,7 +102,7 @@ public JsonRpcResponse response(final JsonRpcRequestContext requestContext) { result -> { final TransactionTrace transactionTrace = new TransactionTrace( - result.getTransaction(), result.getResult(), tracer.getTraceFrames()); + result.transaction(), result.result(), tracer.getTraceFrames()); final Optional<Block> maybeBlock = blockchainQueriesSupplier .get() diff --git a/ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/EthCallTest.java b/ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/EthCallTest.java index 8aba16bfb9c..261f8de505d 100644 --- a/ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/EthCallTest.java +++ b/ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/EthCallTest.java @@ -197,7 +197,7 @@ public void shouldReturnBasicExecutionRevertErrorWithoutReason() { final TransactionSimulatorResult result = mock(TransactionSimulatorResult.class); when(result.isSuccessful()).thenReturn(false); when(result.getValidationResult()).thenReturn(ValidationResult.valid()); - when(result.getResult()).thenReturn(processingResult); + when(result.result()).thenReturn(processingResult); verify(transactionSimulator).process(any(), any(), any(), mapperCaptor.capture(), any()); assertThat(mapperCaptor.getValue().apply(mock(MutableWorldState.class), Optional.of(result))) .isEqualTo(Optional.of(expectedResponse)); @@ -236,7 +236,7 @@ public void shouldReturnExecutionRevertErrorWithABIParseError() { final TransactionSimulatorResult result = mock(TransactionSimulatorResult.class); when(result.isSuccessful()).thenReturn(false); when(result.getValidationResult()).thenReturn(ValidationResult.valid()); - when(result.getResult()).thenReturn(processingResult); + when(result.result()).thenReturn(processingResult); verify(transactionSimulator).process(any(), any(), any(), mapperCaptor.capture(), any()); assertThat(mapperCaptor.getValue().apply(mock(MutableWorldState.class), Optional.of(result))) .isEqualTo(Optional.of(expectedResponse)); @@ -277,7 +277,7 @@ public void shouldReturnExecutionRevertErrorWithParsedABI() { final TransactionSimulatorResult result = mock(TransactionSimulatorResult.class); when(result.isSuccessful()).thenReturn(false); when(result.getValidationResult()).thenReturn(ValidationResult.valid()); - when(result.getResult()).thenReturn(processingResult); + when(result.result()).thenReturn(processingResult); verify(transactionSimulator).process(any(), any(), any(), mapperCaptor.capture(), any()); System.out.println(result); diff --git a/ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/EthCreateAccessListTest.java b/ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/EthCreateAccessListTest.java index 0d129d26652..d23c10f23b4 100644 --- a/ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/EthCreateAccessListTest.java +++ b/ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/EthCreateAccessListTest.java @@ -300,7 +300,7 @@ private void mockTransactionSimulatorResult( when(mockResult.getEstimateGasUsedByTransaction()).thenReturn(estimateGas); when(mockResult.getRevertReason()) .thenReturn(isReverted ? Optional.of(Bytes.of(0)) : Optional.empty()); - when(mockTxSimResult.getResult()).thenReturn(mockResult); + when(mockTxSimResult.result()).thenReturn(mockResult); when(mockTxSimResult.isSuccessful()).thenReturn(isSuccessful); } diff --git a/ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/EthEstimateGasTest.java b/ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/EthEstimateGasTest.java index 307424ea9a1..64d015d89a0 100644 --- a/ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/EthEstimateGasTest.java +++ b/ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/EthEstimateGasTest.java @@ -451,7 +451,7 @@ private TransactionSimulatorResult getMockTransactionSimulatorResult( when(mockResult.getEstimateGasUsedByTransaction()).thenReturn(estimateGas); when(mockResult.getRevertReason()).thenReturn(revertReason); - when(mockTxSimResult.getResult()).thenReturn(mockResult); + when(mockTxSimResult.result()).thenReturn(mockResult); when(mockTxSimResult.isSuccessful()).thenReturn(isSuccessful); return mockTxSimResult; } diff --git a/ethereum/blockcreation/src/test/java/org/hyperledger/besu/ethereum/blockcreation/AbstractBlockTransactionSelectorTest.java b/ethereum/blockcreation/src/test/java/org/hyperledger/besu/ethereum/blockcreation/AbstractBlockTransactionSelectorTest.java index 4c843b2eaea..a3b657dbc3d 100644 --- a/ethereum/blockcreation/src/test/java/org/hyperledger/besu/ethereum/blockcreation/AbstractBlockTransactionSelectorTest.java +++ b/ethereum/blockcreation/src/test/java/org/hyperledger/besu/ethereum/blockcreation/AbstractBlockTransactionSelectorTest.java @@ -195,6 +195,10 @@ private Boolean isCancelled() { return false; } + protected Wei getMinGasPrice() { + return Wei.ONE; + } + protected ProcessableBlockHeader createBlock(final long gasLimit) { return createBlock(gasLimit, Wei.ONE); } diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/processing/TransactionProcessingResult.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/processing/TransactionProcessingResult.java index c053e1107b9..e77062a7ddd 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/processing/TransactionProcessingResult.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/processing/TransactionProcessingResult.java @@ -204,4 +204,31 @@ public ValidationResult<TransactionInvalidReason> getValidationResult() { public Optional<Bytes> getRevertReason() { return revertReason; } + + @Override + public Optional<String> getInvalidReason() { + return (validationResult.isValid() + ? Optional.empty() + : Optional.of(validationResult.getErrorMessage())); + } + + @Override + public String toString() { + return "TransactionProcessingResult{" + + "status=" + + status + + ", estimateGasUsedByTransaction=" + + estimateGasUsedByTransaction + + ", gasRemaining=" + + gasRemaining + + ", logs=" + + logs + + ", output=" + + output + + ", validationResult=" + + validationResult + + ", revertReason=" + + revertReason + + '}'; + } } diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/transaction/CallParameter.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/transaction/CallParameter.java index e5d37e399ef..d74f17baef8 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/transaction/CallParameter.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/transaction/CallParameter.java @@ -199,18 +199,61 @@ public int hashCode() { blobVersionedHashes); } + @Override + public String toString() { + return "CallParameter{" + + "from=" + + from + + ", to=" + + to + + ", gasLimit=" + + gasLimit + + ", maxPriorityFeePerGas=" + + maxPriorityFeePerGas.map(Wei::toHumanReadableString).orElse("N/A") + + ", maxFeePerGas=" + + maxFeePerGas.map(Wei::toHumanReadableString).orElse("N/A") + + ", maxFeePerBlobGas=" + + maxFeePerBlobGas.map(Wei::toHumanReadableString).orElse("N/A") + + ", gasPrice=" + + (gasPrice != null ? gasPrice.toHumanReadableString() : "N/A") + + ", value=" + + (value != null ? value.toHumanReadableString() : "N/A") + + ", payloadSize=" + + (payload != null ? payload.size() : "null") + + ", accessListSize=" + + accessList.map(List::size) + + ", blobVersionedHashesSize=" + + blobVersionedHashes.map(List::size) + + '}'; + } + public static CallParameter fromTransaction(final Transaction tx) { return new CallParameter( tx.getSender(), - tx.getTo().orElseGet(() -> null), + tx.getTo().orElse(null), tx.getGasLimit(), - Wei.fromQuantity(tx.getGasPrice().orElseGet(() -> Wei.ZERO)), - Optional.of(Wei.fromQuantity(tx.getMaxPriorityFeePerGas().orElseGet(() -> Wei.ZERO))), + tx.getGasPrice().orElse(Wei.ZERO), + tx.getMaxPriorityFeePerGas(), tx.getMaxFeePerGas(), - Wei.fromQuantity(tx.getValue()), + tx.getValue(), tx.getPayload(), tx.getAccessList(), tx.getMaxFeePerBlobGas(), tx.getVersionedHashes()); } + + public static CallParameter fromTransaction(final org.hyperledger.besu.datatypes.Transaction tx) { + return new CallParameter( + tx.getSender(), + tx.getTo().orElse(null), + tx.getGasLimit(), + tx.getGasPrice().map(Wei::fromQuantity).orElse(Wei.ZERO), + tx.getMaxPriorityFeePerGas().map(Wei::fromQuantity), + tx.getMaxFeePerGas().map(Wei::fromQuantity), + Wei.fromQuantity(tx.getValue()), + tx.getPayload(), + tx.getAccessList(), + tx.getMaxFeePerBlobGas().map(Wei::fromQuantity), + tx.getVersionedHashes()); + } } diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/transaction/TransactionSimulator.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/transaction/TransactionSimulator.java index 64efb974647..842dc360848 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/transaction/TransactionSimulator.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/transaction/TransactionSimulator.java @@ -106,7 +106,21 @@ public Optional<TransactionSimulatorResult> process( header); } + public Optional<TransactionSimulatorResult> process( + final CallParameter callParams, + final TransactionValidationParams transactionValidationParams, + final OperationTracer operationTracer, + final BlockHeader blockHeader) { + return process( + callParams, + transactionValidationParams, + operationTracer, + (mutableWorldState, transactionSimulatorResult) -> transactionSimulatorResult, + blockHeader); + } + public Optional<TransactionSimulatorResult> processAtHead(final CallParameter callParams) { + final var chainHeadHash = blockchain.getChainHeadHash(); return process( callParams, ImmutableTransactionValidationParams.builder() @@ -115,7 +129,10 @@ public Optional<TransactionSimulatorResult> processAtHead(final CallParameter ca .build(), OperationTracer.NO_TRACING, (mutableWorldState, transactionSimulatorResult) -> transactionSimulatorResult, - blockchain.getChainHeadHeader()); + blockchain + .getBlockHeader(chainHeadHash) + .or(() -> blockchain.getBlockHeaderSafe(chainHeadHash)) + .orElse(null)); } /** diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/transaction/TransactionSimulatorResult.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/transaction/TransactionSimulatorResult.java index 0627ca14fd2..853bc4611a3 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/transaction/TransactionSimulatorResult.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/transaction/TransactionSimulatorResult.java @@ -18,22 +18,10 @@ import org.hyperledger.besu.ethereum.mainnet.ValidationResult; import org.hyperledger.besu.ethereum.processing.TransactionProcessingResult; -import java.util.Objects; - -import com.google.common.annotations.VisibleForTesting; import org.apache.tuweni.bytes.Bytes; -public class TransactionSimulatorResult { - - private final Transaction transaction; - private final TransactionProcessingResult result; - - @VisibleForTesting - public TransactionSimulatorResult( - final Transaction transaction, final TransactionProcessingResult result) { - this.transaction = transaction; - this.result = result; - } +public record TransactionSimulatorResult( + Transaction transaction, TransactionProcessingResult result) { public boolean isSuccessful() { return result.isSuccessful(); @@ -54,40 +42,4 @@ public Bytes getOutput() { public ValidationResult<TransactionInvalidReason> getValidationResult() { return result.getValidationResult(); } - - public TransactionProcessingResult getResult() { - return result; - } - - public Transaction getTransaction() { - return transaction; - } - - @Override - public boolean equals(final Object o) { - if (this == o) { - return true; - } - if (o == null || getClass() != o.getClass()) { - return false; - } - final TransactionSimulatorResult that = (TransactionSimulatorResult) o; - return Objects.equals(transaction, that.transaction) && Objects.equals(result, that.result); - } - - @Override - public int hashCode() { - return Objects.hash(transaction, result); - } - - @Override - public String toString() { - return "TransactionSimulatorResult{" - + "transaction=" - + transaction - + ", " - + "result=" - + result - + "}"; - } } diff --git a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/mainnet/MainnetTransactionValidatorTest.java b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/mainnet/MainnetTransactionValidatorTest.java index 622d868a0ba..b0e555c9dd0 100644 --- a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/mainnet/MainnetTransactionValidatorTest.java +++ b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/mainnet/MainnetTransactionValidatorTest.java @@ -72,12 +72,12 @@ public class MainnetTransactionValidatorTest { private static final Supplier<SignatureAlgorithm> SIGNATURE_ALGORITHM = Suppliers.memoize(SignatureAlgorithmFactory::getInstance); - private static final KeyPair senderKeys = SIGNATURE_ALGORITHM.get().generateKeyPair(); + protected static final KeyPair senderKeys = SIGNATURE_ALGORITHM.get().generateKeyPair(); private static final TransactionValidationParams transactionValidationParams = processingBlockParams; - @Mock private GasCalculator gasCalculator; + @Mock protected GasCalculator gasCalculator; private final Transaction basicTransaction = new TransactionTestFixture() diff --git a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/mainnet/PermissionTransactionValidatorTest.java b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/mainnet/PermissionTransactionValidatorTest.java index 0b4c948f55e..3fdb20dfd4f 100644 --- a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/mainnet/PermissionTransactionValidatorTest.java +++ b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/mainnet/PermissionTransactionValidatorTest.java @@ -32,7 +32,6 @@ import org.hyperledger.besu.ethereum.core.TransactionTestFixture; import org.hyperledger.besu.ethereum.transaction.TransactionInvalidReason; import org.hyperledger.besu.evm.account.Account; -import org.hyperledger.besu.evm.gascalculator.GasCalculator; import java.math.BigInteger; import java.util.Optional; @@ -42,7 +41,6 @@ import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.ArgumentCaptor; -import org.mockito.Mock; import org.mockito.junit.jupiter.MockitoExtension; @ExtendWith(MockitoExtension.class) @@ -51,7 +49,6 @@ public class PermissionTransactionValidatorTest extends MainnetTransactionValida private static final Supplier<SignatureAlgorithm> SIGNATURE_ALGORITHM = Suppliers.memoize(SignatureAlgorithmFactory::getInstance); private static final KeyPair senderKeys = SIGNATURE_ALGORITHM.get().generateKeyPair(); - @Mock private GasCalculator gasCalculator; private final Transaction basicTransaction = new TransactionTestFixture() diff --git a/ethereum/permissioning/src/main/java/org/hyperledger/besu/ethereum/permissioning/NodeSmartContractPermissioningController.java b/ethereum/permissioning/src/main/java/org/hyperledger/besu/ethereum/permissioning/NodeSmartContractPermissioningController.java index 5f619ee3f80..a034f049da0 100644 --- a/ethereum/permissioning/src/main/java/org/hyperledger/besu/ethereum/permissioning/NodeSmartContractPermissioningController.java +++ b/ethereum/permissioning/src/main/java/org/hyperledger/besu/ethereum/permissioning/NodeSmartContractPermissioningController.java @@ -71,7 +71,7 @@ boolean checkSmartContractRules(final EnodeURL sourceEnode, final EnodeURL desti transactionSimulator.processAtHead(callParams); if (result.isPresent()) { - switch (result.get().getResult().getStatus()) { + switch (result.get().result().getStatus()) { case INVALID: throw new IllegalStateException("Permissioning transaction found to be Invalid"); case FAILED: diff --git a/ethereum/permissioning/src/main/java/org/hyperledger/besu/ethereum/permissioning/NodeSmartContractV2PermissioningController.java b/ethereum/permissioning/src/main/java/org/hyperledger/besu/ethereum/permissioning/NodeSmartContractV2PermissioningController.java index ca114d17955..ae78a4ab85a 100644 --- a/ethereum/permissioning/src/main/java/org/hyperledger/besu/ethereum/permissioning/NodeSmartContractV2PermissioningController.java +++ b/ethereum/permissioning/src/main/java/org/hyperledger/besu/ethereum/permissioning/NodeSmartContractV2PermissioningController.java @@ -114,7 +114,7 @@ private Bytes createPayload(final EnodeURL enodeUrl) { } private boolean parseResult(final TransactionSimulatorResult result) { - switch (result.getResult().getStatus()) { + switch (result.result().getStatus()) { case INVALID: throw new IllegalStateException("Invalid node permissioning smart contract call"); case FAILED: diff --git a/ethereum/permissioning/src/main/java/org/hyperledger/besu/ethereum/permissioning/TransactionSmartContractPermissioningController.java b/ethereum/permissioning/src/main/java/org/hyperledger/besu/ethereum/permissioning/TransactionSmartContractPermissioningController.java index f8ee921668f..dd42727af85 100644 --- a/ethereum/permissioning/src/main/java/org/hyperledger/besu/ethereum/permissioning/TransactionSmartContractPermissioningController.java +++ b/ethereum/permissioning/src/main/java/org/hyperledger/besu/ethereum/permissioning/TransactionSmartContractPermissioningController.java @@ -134,7 +134,7 @@ public boolean isPermitted(final Transaction transaction) { transactionSimulator.processAtHead(callParams); if (result.isPresent()) { - switch (result.get().getResult().getStatus()) { + switch (result.get().result().getStatus()) { case INVALID: throw new IllegalStateException( "Transaction permissioning transaction found to be Invalid"); diff --git a/plugin-api/build.gradle b/plugin-api/build.gradle index 8d051eb516d..a0129425e21 100644 --- a/plugin-api/build.gradle +++ b/plugin-api/build.gradle @@ -69,7 +69,7 @@ Calculated : ${currentHash} tasks.register('checkAPIChanges', FileStateChecker) { description = "Checks that the API for the Plugin-API project does not change without deliberate thought" files = sourceSets.main.allJava.files - knownHash = 'B/pzTaARYvd/T9WtAXtX6vFbxoK5u82GigByKD0YP6M=' + knownHash = 'ytjNiSzw9IR8YHyO4ikmqRTg1GTWkCX9QiQtwq2dRSg=' } check.dependsOn('checkAPIChanges') diff --git a/plugin-api/src/main/java/org/hyperledger/besu/plugin/data/TransactionProcessingResult.java b/plugin-api/src/main/java/org/hyperledger/besu/plugin/data/TransactionProcessingResult.java index 22f35ea37bc..ac398788e40 100644 --- a/plugin-api/src/main/java/org/hyperledger/besu/plugin/data/TransactionProcessingResult.java +++ b/plugin-api/src/main/java/org/hyperledger/besu/plugin/data/TransactionProcessingResult.java @@ -88,4 +88,11 @@ public interface TransactionProcessingResult { * @return the revert reason. */ Optional<Bytes> getRevertReason(); + + /** + * Return the reason why the transaction is invalid or empty if the transaction is successful + * + * @return the optional invalid reason as a string + */ + Optional<String> getInvalidReason(); } diff --git a/plugin-api/src/main/java/org/hyperledger/besu/plugin/data/TransactionSimulationResult.java b/plugin-api/src/main/java/org/hyperledger/besu/plugin/data/TransactionSimulationResult.java new file mode 100644 index 00000000000..1651534a9fa --- /dev/null +++ b/plugin-api/src/main/java/org/hyperledger/besu/plugin/data/TransactionSimulationResult.java @@ -0,0 +1,54 @@ +/* + * Copyright Hyperledger Besu Contributors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + */ +package org.hyperledger.besu.plugin.data; + +import org.hyperledger.besu.datatypes.Transaction; + +/** + * TransactionSimulationResult + * + * @param transaction tx + * @param result res + */ +public record TransactionSimulationResult( + Transaction transaction, TransactionProcessingResult result) { + + /** + * Was the simulation successful? + * + * @return boolean + */ + public boolean isSuccessful() { + return result.isSuccessful(); + } + + /** + * Was the transaction invalid? + * + * @return invalid + */ + public boolean isInvalid() { + return result.isInvalid(); + } + + /** + * Estimated gas used by the transaction + * + * @return estimated gas used + */ + public long getGasEstimate() { + return transaction.getGasLimit() - result.getGasRemaining(); + } +} diff --git a/plugin-api/src/main/java/org/hyperledger/besu/plugin/services/TransactionSimulationService.java b/plugin-api/src/main/java/org/hyperledger/besu/plugin/services/TransactionSimulationService.java new file mode 100644 index 00000000000..77d5822c93c --- /dev/null +++ b/plugin-api/src/main/java/org/hyperledger/besu/plugin/services/TransactionSimulationService.java @@ -0,0 +1,42 @@ +/* + * Copyright Hyperledger Besu Contributors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + */ +package org.hyperledger.besu.plugin.services; + +import org.hyperledger.besu.datatypes.Hash; +import org.hyperledger.besu.datatypes.Transaction; +import org.hyperledger.besu.evm.tracing.OperationTracer; +import org.hyperledger.besu.plugin.Unstable; +import org.hyperledger.besu.plugin.data.TransactionSimulationResult; + +import java.util.Optional; + +/** Transaction simulation service interface */ +@Unstable +public interface TransactionSimulationService extends BesuService { + /** + * Simulate transaction execution at the block identified by the hash + * + * @param transaction tx + * @param blockHash the hash of the block + * @param operationTracer the tracer + * @param isAllowExceedingBalance should ignore the sender balance during the simulation? + * @return the result of the simulation + */ + Optional<TransactionSimulationResult> simulate( + Transaction transaction, + Hash blockHash, + OperationTracer operationTracer, + boolean isAllowExceedingBalance); +} diff --git a/testutil/src/main/java/org/hyperledger/besu/testutil/JsonTestParameters.java b/testutil/src/main/java/org/hyperledger/besu/testutil/JsonTestParameters.java index 69b21b37c8b..93519d1c41d 100644 --- a/testutil/src/main/java/org/hyperledger/besu/testutil/JsonTestParameters.java +++ b/testutil/src/main/java/org/hyperledger/besu/testutil/JsonTestParameters.java @@ -242,7 +242,13 @@ public Collection<Object[]> generate(final String... paths) { return generate(getFilteredFiles(paths)); } - private Collection<Object[]> generate(final Collection<File> filteredFiles) { + /** + * Generate collection. + * + * @param filteredFiles the filtered files + * @return the collection + */ + public Collection<Object[]> generate(final Collection<File> filteredFiles) { checkState(generator != null, "Missing generator function"); final Collector<T> collector =