From 0441ba23c65c3949e54e7b5d829bdee6b0aa5c22 Mon Sep 17 00:00:00 2001 From: Matt Peterson Date: Mon, 16 Dec 2024 13:51:54 -0700 Subject: [PATCH] feat: configure gradle to build deterministic Block Node jar (#411) Signed-off-by: Matt Peterson --- .github/workflows/e2e-tests.yaml | 6 +- .github/workflows/pr-checks.yaml | 6 +- .github/workflows/release-automation.yaml | 8 +- .github/workflows/release-push-image.yaml | 4 +- .github/workflows/smoke-test.yaml | 6 +- .../generate-gradle-artifact-baseline.sh | 148 ++++++++++++++++++ .../zxc-verify-gradle-build-determinism.yaml | 62 ++++++++ .../com.hedera.block.conventions.gradle.kts | 101 ++++++++++-- 8 files changed, 316 insertions(+), 25 deletions(-) create mode 100755 .github/workflows/support/scripts/generate-gradle-artifact-baseline.sh create mode 100644 .github/workflows/zxc-verify-gradle-build-determinism.yaml diff --git a/.github/workflows/e2e-tests.yaml b/.github/workflows/e2e-tests.yaml index 07494b9d4..5c61a5801 100644 --- a/.github/workflows/e2e-tests.yaml +++ b/.github/workflows/e2e-tests.yaml @@ -56,10 +56,10 @@ jobs: fi - name: Set up JDK 21 - uses: actions/setup-java@99b8673ff64fbf99d8d325f52d9a5bdedb8483e9 # v4.2.1 + uses: actions/setup-java@8df1039502a15bceb9433410b1a100fbe190c53b # v4.5.0 with: - distribution: 'temurin' - java-version: '21' + distribution: "temurin" + java-version: "21.0.4" - name: Run Acceptance Tests id: acceptance-tests diff --git a/.github/workflows/pr-checks.yaml b/.github/workflows/pr-checks.yaml index 63d3cfe13..04a767b98 100644 --- a/.github/workflows/pr-checks.yaml +++ b/.github/workflows/pr-checks.yaml @@ -57,10 +57,10 @@ jobs: fi - name: Set up JDK 21 - uses: actions/setup-java@99b8673ff64fbf99d8d325f52d9a5bdedb8483e9 # v4.2.1 + uses: actions/setup-java@8df1039502a15bceb9433410b1a100fbe190c53b # v4.5.0 with: - distribution: 'temurin' - java-version: '21' + distribution: "temurin" + java-version: "21.0.4" - name: Cache Gradle packages uses: actions/cache@0c45773b623bea8c8e75f6c82b208c3cf94ea4f9 # v4.0.2 diff --git a/.github/workflows/release-automation.yaml b/.github/workflows/release-automation.yaml index a258dba6f..bef04163a 100644 --- a/.github/workflows/release-automation.yaml +++ b/.github/workflows/release-automation.yaml @@ -91,10 +91,10 @@ jobs: passphrase: ${{ secrets.GPG_KEY_PASSPHRASE }} - name: Install JDK - uses: actions/setup-java@99b8673ff64fbf99d8d325f52d9a5bdedb8483e9 # v4.2.1 + uses: actions/setup-java@8df1039502a15bceb9433410b1a100fbe190c53b # v4.5.0 with: distribution: "temurin" - java-version: 21 + java-version: "21.0.4" - name: Setup Gradle uses: gradle/actions/setup-gradle@d9c87d481d55275bb5441eef3fe0e46805f9ef70 # v3.5.0 @@ -192,10 +192,10 @@ jobs: passphrase: ${{ secrets.GPG_KEY_PASSPHRASE }} - name: Install JDK - uses: actions/setup-java@99b8673ff64fbf99d8d325f52d9a5bdedb8483e9 # v4.2.1 + uses: actions/setup-java@8df1039502a15bceb9433410b1a100fbe190c53b # v4.5.0 with: distribution: "temurin" - java-version: 21 + java-version: "21.0.4" - name: Setup Gradle uses: gradle/actions/setup-gradle@d9c87d481d55275bb5441eef3fe0e46805f9ef70 # v3.5.0 diff --git a/.github/workflows/release-push-image.yaml b/.github/workflows/release-push-image.yaml index abef1cfc2..85425f719 100644 --- a/.github/workflows/release-push-image.yaml +++ b/.github/workflows/release-push-image.yaml @@ -56,10 +56,10 @@ jobs: uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 - name: Install JDK - uses: actions/setup-java@99b8673ff64fbf99d8d325f52d9a5bdedb8483e9 # v4.2.1 + uses: actions/setup-java@8df1039502a15bceb9433410b1a100fbe190c53b # v4.5.0 with: distribution: "temurin" - java-version: 21 + java-version: "21.0.4" - name: Build run: ./gradlew clean build diff --git a/.github/workflows/smoke-test.yaml b/.github/workflows/smoke-test.yaml index 955d18e59..2de3ba4ca 100644 --- a/.github/workflows/smoke-test.yaml +++ b/.github/workflows/smoke-test.yaml @@ -57,10 +57,10 @@ jobs: fi - name: Set up JDK 21 - uses: actions/setup-java@99b8673ff64fbf99d8d325f52d9a5bdedb8483e9 # v4.2.1 + uses: actions/setup-java@8df1039502a15bceb9433410b1a100fbe190c53b # v4.5.0 with: - distribution: 'temurin' - java-version: '21' + distribution: "temurin" + java-version: "21.0.4" - name: Install grpcurl run: | diff --git a/.github/workflows/support/scripts/generate-gradle-artifact-baseline.sh b/.github/workflows/support/scripts/generate-gradle-artifact-baseline.sh new file mode 100755 index 000000000..b0fe6ebe2 --- /dev/null +++ b/.github/workflows/support/scripts/generate-gradle-artifact-baseline.sh @@ -0,0 +1,148 @@ +#!/usr/bin/env bash +set -o pipefail +set +e + +readonly RELEASE_LIB_PATH="hedera-node/data/lib" +readonly RELEASE_APPS_PATH="hedera-node/data/apps" + +GROUP_ACTIVE="false" + +function fail { + printf '%s\n' "$1" >&2 ## Send message to stderr. Exclude >&2 if you don't want it that way. + if [[ "${GROUP_ACTIVE}" == "true" ]]; then + end_group + fi + exit "${2-1}" ## Return a code specified by $2 or 1 by default. +} + +function start_group { + if [[ "${GROUP_ACTIVE}" == "true" ]]; then + end_group + fi + + GROUP_ACTIVE="true" + printf "::group::%s\n" "${1}" +} + +function end_group { + GROUP_ACTIVE="false" + printf "::endgroup::\n" +} + +function log { + local message="${1}" + shift + # shellcheck disable=SC2059 + printf "${message}" "${@}" +} + +function log_line { + local message="${1}" + shift + # shellcheck disable=SC2059 + printf "${message}\n" "${@}" +} + +function start_task { + local message="${1}" + shift + # shellcheck disable=SC2059 + printf "${message} .....\t" "${@}" +} + +function end_task { + printf "%s\n" "${1:-DONE}" +} + +start_group "Configuring Environment" + # Access workflow environment variables + export GITHUB_WORKSPACE GITHUB_SHA GITHUB_OUTPUT MANIFEST_PATH + + start_task "Initializing Temporary Directory" + TEMP_DIR="$(mktemp -d)" || fail "ERROR (Exit Code: ${?})" "${?}" + trap 'rm -rf "${TEMP_DIR}"' EXIT + end_task "DONE (Path: ${TEMP_DIR})" + +# start_task "Resolving the GITHUB_WORKSPACE path" +# # Ensure GITHUB_WORKSPACE is provided or default to the repository root +# if [[ -z "${GITHUB_WORKSPACE}" || ! -d "${GITHUB_WORKSPACE}" ]]; then +# GITHUB_WORKSPACE="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../../../" && pwd)" +# fi +# end_task "DONE (Path: ${GITHUB_WORKSPACE})" +# +# start_task "Resolving the GITHUB_OUTPUT path" +# # Ensure GITHUB_OUTPUT is provided or default to the repository root +# if [[ -z "${GITHUB_OUTPUT}" ]]; then +# GITHUB_OUTPUT="${TEMP_DIR}/workflow-output.txt" +# fi +# end_task "DONE (Path: ${GITHUB_OUTPUT})" +# +# start_task "Resolving the GITHUB_SHA hash" +# if [[ -z "${GITHUB_SHA}" ]]; then +# GITHUB_SHA="$(git rev-parse HEAD | tr -d '[:space:]')" || fail "ERROR (Exit Code: ${?})" "${?}" +# fi +# end_task "DONE (Commit: ${GITHUB_SHA})" +# +# start_task "Resolving the MANIFEST_PATH variable" +# if [[ -z "${MANIFEST_PATH}" ]]; then +# MANIFEST_PATH="${GITHUB_WORKSPACE}/.manifests/gradle" +# fi +# end_task "DONE (Path: ${MANIFEST_PATH})" +# +# start_task "Ensuring the MANIFEST_PATH location is present" +# if [[ ! -d "${MANIFEST_PATH}" ]]; then +# mkdir -p "${MANIFEST_PATH}" || fail "ERROR (Exit Code: ${?})" "${?}" +# fi +# end_task +# +# start_task "Checking for the sha256sum command" +# if command -v sha256sum >/dev/null 2>&1; then +# SHA256SUM="$(command -v sha256sum)" || fail "ERROR (Exit Code: ${?})" "${?}" +# else +# fail "ERROR (Exit Code: ${?})" "${?}" +# fi +# end_task "DONE (Found: ${SHA256SUM})" +# +# start_task "Checking for prebuilt libraries" +# ls -al "${GITHUB_WORKSPACE}/${RELEASE_LIB_PATH}"/*.jar >/dev/null 2>&1 || fail "ERROR (Exit Code: ${?})" "${?}" +# end_task "FOUND (Path: ${GITHUB_WORKSPACE}/${RELEASE_LIB_PATH}/*.jar)" +# +# start_task "Checking for prebuilt applications" +# ls -al "${GITHUB_WORKSPACE}/${RELEASE_APPS_PATH}"/*.jar >/dev/null 2>&1 || fail "ERROR (Exit Code: ${?})" "${?}" +# end_task "FOUND (Path: ${GITHUB_WORKSPACE}/${RELEASE_APPS_PATH}/*.jar)" +end_group + +#start_group "Generating Library Hashes (${GITHUB_WORKSPACE}/${RELEASE_LIB_PATH}/*.jar)" +# pushd "${GITHUB_WORKSPACE}/${RELEASE_LIB_PATH}" >/dev/null 2>&1 || fail "PUSHD ERROR (Exit Code: ${?})" "${?}" +# ${SHA256SUM} -b -- *.jar | sort -k 2 | tee -a "${TEMP_DIR}"/libraries.sha256 +# popd >/dev/null 2>&1 || fail "POPD ERROR (Exit Code: ${?})" "${?}" +#end_group +# +#start_group "Generating Application Hashes (${GITHUB_WORKSPACE}/${RELEASE_APPS_PATH}/*.jar)" +# pushd "${GITHUB_WORKSPACE}/${RELEASE_APPS_PATH}" >/dev/null 2>&1 || fail "PUSHD ERROR (Exit Code: ${?})" "${?}" +# ${SHA256SUM} -b -- *.jar | sort -k 2 | tee -a "${TEMP_DIR}"/applications.sha256 +# popd >/dev/null 2>&1 || fail "POPD ERROR (Exit Code: ${?})" "${?}" +#end_group +# +#start_group "Generating Final Release Manifests" +# +# start_task "Generating the manifest archive" +# tar -czf "${TEMP_DIR}/manifest.tar.gz" -C "${TEMP_DIR}" libraries.sha256 applications.sha256 >/dev/null 2>&1 || fail "TAR ERROR (Exit Code: ${?})" "${?}" +# end_task +# +# start_task "Copying the manifest files" +# cp "${TEMP_DIR}/manifest.tar.gz" "${MANIFEST_PATH}/${GITHUB_SHA}.tar.gz" || fail "COPY ERROR (Exit Code: ${?})" "${?}" +# cp "${TEMP_DIR}/libraries.sha256" "${MANIFEST_PATH}/libraries.sha256" || fail "COPY ERROR (Exit Code: ${?})" "${?}" +# cp "${TEMP_DIR}/applications.sha256" "${MANIFEST_PATH}/applications.sha256" || fail "COPY ERROR (Exit Code: ${?})" "${?}" +# end_task "DONE (Path: ${MANIFEST_PATH}/${GITHUB_SHA}.tar.gz)" +# +# start_task "Setting Step Outputs" +# { +# printf "path=%s\n" "${MANIFEST_PATH}" +# printf "file=%s\n" "${MANIFEST_PATH}/${GITHUB_SHA}.tar.gz" +# printf "name=%s\n" "${GITHUB_SHA}.tar.gz" +# printf "applications=%s\n" "${MANIFEST_PATH}/applications.sha256" +# printf "libraries=%s\n" "${MANIFEST_PATH}/libraries.sha256" +# } >> "${GITHUB_OUTPUT}" +# end_task +#end_group diff --git a/.github/workflows/zxc-verify-gradle-build-determinism.yaml b/.github/workflows/zxc-verify-gradle-build-determinism.yaml new file mode 100644 index 000000000..1ad85e98a --- /dev/null +++ b/.github/workflows/zxc-verify-gradle-build-determinism.yaml @@ -0,0 +1,62 @@ +## +# Copyright (C) 2023-2024 Hedera Hashgraph, LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +## +name: "ZXC: Verify Gradle Build Determinism" +# Here, the ZXC prefix: +# Z - Ensures sort order such that this script appears at the bottom of the UI +# X - Indicates it's not for direct user consumption +# C - Indicates this is a 'workflow_call' based reusable workflow + +on: + workflow_call: + inputs: + ref: + description: "The branch, tag, or commit to checkout:" + type: string + required: false + default: "" + java-distribution: + description: "Java JDK Distribution:" + type: string + required: false + default: "temurin" + java-version: + description: "Java JDK Version:" + type: string + required: false + default: "21.0.4" + +# workflow_dispatch: +# inputs: +# version: +# description: 'Release tag:' +# type: string +# required: false + +defaults: + run: + shell: bash + +permissions: + contents: read + packages: write + +jobs: + generate-baseline: + name: Generate Baseline + runs-on: network-node-linux-medium + steps: + - name: Print + run: echo "Hello, baseline!" diff --git a/buildSrc/src/main/kotlin/com.hedera.block.conventions.gradle.kts b/buildSrc/src/main/kotlin/com.hedera.block.conventions.gradle.kts index 7dad4eb8c..13d0fd47b 100644 --- a/buildSrc/src/main/kotlin/com.hedera.block.conventions.gradle.kts +++ b/buildSrc/src/main/kotlin/com.hedera.block.conventions.gradle.kts @@ -31,6 +31,26 @@ plugins { group = "com.hedera.block" +val javaVersionMajor = JavaVersion.VERSION_21 +val javaVersionPatch = "0.4" + +val currentJavaVersionMajor = JavaVersion.current() +val currentJavaVersion = providers.systemProperty("java.version").get() +val expectedJavaVersion = "$javaVersionMajor.$javaVersionPatch" + +if (currentJavaVersion != expectedJavaVersion) { + val message = + "Gradle runs with Java $currentJavaVersion. This project works best running with Java $expectedJavaVersion. " + + "\n - From commandline: change JAVA_HOME and/or PATH to point at Java $expectedJavaVersion installation." + + "\n - From IntelliJ: change 'Gradle JVM' in 'Gradle Settings' to point at Java $expectedJavaVersion installation." + + if (currentJavaVersionMajor.ordinal < javaVersionMajor.ordinal) { // fail if version is too old + throw (RuntimeException(message)) + } else { + logger.lifecycle("WARN: $message") + } +} + java { sourceCompatibility = JavaVersion.VERSION_21 targetCompatibility = JavaVersion.VERSION_21 @@ -51,19 +71,80 @@ testing { tasks.withType { isPreserveFileTimestamps = false isReproducibleFileOrder = true - fileMode = 436 // octal: 0664 - dirMode = 509 // octal: 0775 + filePermissions { unix("0664") } + dirPermissions { unix("0775") } } -tasks.withType { options.encoding = "UTF-8" } +val deactivatedCompileLintOptions = + listOf( + // In Gradle, a module does not see the upstream (not-yet-compiled) modules. This could + // only be solved by calling 'javac' with '--source-module-path' to make other sources + // known. But this is at odds with how Gradle's incremental compilation calls the + // compiler for a subset of Java files for each project individually. + "module", // module not found when doing 'exports to ...' + "serial", // serializable class ... has no definition of serialVersionUID + "processing", // No processor claimed any of these annotations: ... + "try", // auto-closeable resource ignore is never referenced... (AutoClosableLock) + "missing-explicit-ctor", // class ... declares no explicit constructors -tasks.withType { - options.encoding = "UTF-8" - (options as StandardJavadocDocletOptions).tags( - "apiNote:a:API Note:", - "implSpec:a:Implementation Requirements:", - "implNote:a:Implementation Note:" + // Needed because we use deprecation internally and do not fix all uses right away + "removal", + "deprecation", + + // The following checks could be activated and fixed: + "this-escape", // calling public/protected method in constructor + "overrides", // overrides equals, but neither it ... overrides hashCode method + "unchecked", + "rawtypes" ) + +tasks.withType().configureEach { + // Track the full Java version as input (e.g. 17.0.3 vs. 17.0.9). + // By default, Gradle only tracks the major version as defined in the toolchain (e.g. 17). + // Since the full version is encoded in 'module-info.class' files, it should be tracked as + // it otherwise leads to wrong build cache hits. + inputs.property("fullJavaVersion", currentJavaVersion) + + options.encoding = "UTF-8" + options.isFork = true // run compiler in separate JVM process (independent of toolchain setup) + options.compilerArgs.add("-implicit:none") + options.compilerArgs.add("-Xlint:all,-" + deactivatedCompileLintOptions.joinToString(",-")) + + doLast { + // Make sure consistent line ending are used in files generated by annotation processors by + // rewriting generated files. + // To fix this problem at the root, one of these issues needs to be addressed upstream: + // - https://github.com/google/auto/issues/1656 + // - https://github.com/gradle/gradle/issues/27385 + if (System.lineSeparator() != "\n") { + destinationDirectory + .get() + .asFileTree + .filter { it.extension != "class" } + .forEach { + val content = it.readText() + val normalizedContent = content.replace(System.lineSeparator(), "\n") + if (content != normalizedContent) { + it.writeText(normalizedContent) + } + } + } + } +} + +tasks.withType().configureEach { + options { + this as StandardJavadocDocletOptions + encoding = "UTF-8" + tags( + "apiNote:a:API Note:", + "implSpec:a:Implementation Requirements:", + "implNote:a:Implementation Note:" + ) + options.windowTitle = "Hedera Block Node" + options.memberLevel = JavadocMemberLevel.PACKAGE + addStringOption("Xdoclint:all,-missing", "-Xwerror") + } } testlogger { @@ -93,4 +174,4 @@ tasks.check { dependsOn(tasks.jacocoTestReport) // Check dependency scopes in module-info.java dependsOn(tasks.checkAllModuleInfo) -} \ No newline at end of file +}