diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 16731dc40..50782ef92 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -235,7 +235,7 @@ jobs: steps: - name: Restore cygwin packages from cache id: cygwin - uses: actions/cache@3624ceb22c1c5a301c8db4169662070a689d9ea8 # v4.1.1 + uses: actions/cache@6849a6489940f00c2f30c0fb92c6274307ccb58a # v4.1.2 with: path: C:\cygwin64 key: cygwin-packages-${{ runner.os }}-v1 @@ -279,7 +279,7 @@ jobs: - name: Restore Visual Studio 2017 from cache id: vs2017 if: matrix.version == 'jdk8u' - uses: actions/cache@3624ceb22c1c5a301c8db4169662070a689d9ea8 # v4.1.1 + uses: actions/cache@6849a6489940f00c2f30c0fb92c6274307ccb58a # v4.1.2 with: path: ~/vs2017.exe key: vs2017 @@ -287,7 +287,7 @@ jobs: - name: Restore Visual Studio 2019 from cache id: vs2019 if: matrix.version == 'jdk11u' || matrix.version == 'jdk17u' - uses: actions/cache@3624ceb22c1c5a301c8db4169662070a689d9ea8 # v4.1.1 + uses: actions/cache@6849a6489940f00c2f30c0fb92c6274307ccb58a # v4.1.2 with: path: ~/vs2019.exe key: vs2019 diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index e7be2eee0..8670bb98a 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -58,7 +58,7 @@ jobs: # Initializes the CodeQL tools for scanning. - name: Initialize CodeQL - uses: github/codeql-action/init@c36620d31ac7c881962c3d9dd939c40ec9434f2b # v3.26.12 + uses: github/codeql-action/init@662472033e021d55d94146f66f6058822b0b39fd # v3.27.0 with: languages: ${{ matrix.language }} # If you wish to specify custom queries, you can do so here or in a config file. @@ -68,7 +68,7 @@ jobs: # Autobuild attempts to build any compiled languages (C/C++, C#, or Java). # If this step fails, then you should remove it and run the build manually (see below) - name: Autobuild - uses: github/codeql-action/autobuild@c36620d31ac7c881962c3d9dd939c40ec9434f2b # v3.26.12 + uses: github/codeql-action/autobuild@662472033e021d55d94146f66f6058822b0b39fd # v3.27.0 # ℹī¸ Command-line programs to run using the OS shell. # 📚 See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun @@ -81,6 +81,6 @@ jobs: # ./location_of_script_within_repo/buildscript.sh - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@c36620d31ac7c881962c3d9dd939c40ec9434f2b # v3.26.12 + uses: github/codeql-action/analyze@662472033e021d55d94146f66f6058822b0b39fd # v3.27.0 with: category: "/language:${{matrix.language}}" diff --git a/.github/workflows/ossf-scorecard.yml b/.github/workflows/ossf-scorecard.yml index 3d46ebbf9..6fefd074e 100644 --- a/.github/workflows/ossf-scorecard.yml +++ b/.github/workflows/ossf-scorecard.yml @@ -46,6 +46,6 @@ jobs: name: SARIF file path: results.sarif retention-days: 5 - - uses: github/codeql-action/upload-sarif@c36620d31ac7c881962c3d9dd939c40ec9434f2b # v2.13.4 + - uses: github/codeql-action/upload-sarif@662472033e021d55d94146f66f6058822b0b39fd # v2.13.4 with: sarif_file: results.sarif diff --git a/docker-build.sh b/docker-build.sh index efa1a62ce..24de787f1 100755 --- a/docker-build.sh +++ b/docker-build.sh @@ -20,19 +20,19 @@ # ################################################################################ -# the ${BUILD_CONFIG[CONTAINER_AS_ROOT]} can not be quoted. It is sudo (or simialrly) or nothing. "" is not an option. -# simialrly the ${cpuset} and ${userns} +# The ${BUILD_CONFIG[CONTAINER_AS_ROOT]} can not be quoted. It is sudo (or similar) or nothing. "" is not an option. +# Similarly for ${cpuset} and ${userns}. # shellcheck disable=SC2206 # shellcheck disable=SC2046 # shellcheck disable=SC2086 set -eu -# Create a data volume called ${BUILD_CONFIG[DOCKER_SOURCE_VOLUME_NAME]}, -# this gets mounted at /openjdk/build inside the container and is persistent +# Create a data volume called ${BUILD_CONFIG[DOCKER_SOURCE_VOLUME_NAME]}. +# This gets mounted at /openjdk/build inside the container and is persistent # between builds/tests unless -c is passed to this script, in which case it is # recreated using the source in the current ./openjdk directory on the host -# machine (outside the container) +# machine (outside the container). createPersistentDockerDataVolume() { set +e @@ -57,7 +57,7 @@ createPersistentDockerDataVolume() fi } -# Build the docker container +# Build the docker container. buildDockerContainer() { echo "Building docker container" @@ -76,7 +76,7 @@ buildDockerContainer() ${BUILD_CONFIG[CONTAINER_AS_ROOT]} "${BUILD_CONFIG[CONTAINER_COMMAND]}" build -t "${BUILD_CONFIG[CONTAINER_NAME]}" -f "${dockerFile}" . --build-arg "OPENJDK_CORE_VERSION=${BUILD_CONFIG[OPENJDK_CORE_VERSION]}" --build-arg "HostUID=${UID}" } -# Execute the (Adoptium) OpenJDK build inside the Docker Container +# Execute the (Adoptium) OpenJDK build inside the Docker Container. buildOpenJDKViaDocker() { local hostDir @@ -89,12 +89,12 @@ buildOpenJDKViaDocker() local localsourcesdir= if [ "${BUILD_CONFIG[OPENJDK_LOCAL_SOURCE_ARCHIVE]}" = "true" ] ; then - # OPENJDK_LOCAL_SOURCE_ARCHIVE_ABSPATH can be file, you can nto mount file + # OPENJDK_LOCAL_SOURCE_ARCHIVE_ABSPATH can be file, you can not mount file. localsourcesdir=$(dirname "${BUILD_CONFIG[OPENJDK_LOCAL_SOURCE_ARCHIVE_ABSPATH]}") fi # TODO This could be extracted overridden by the user if we support more - # architectures going forwards + # architectures going forwards. local container_architecture container_architecture="$(uname -m)/${BUILD_CONFIG[CONTAINER_IMAGE]//:*/}" local build_variant_flag="" @@ -154,7 +154,7 @@ buildOpenJDKViaDocker() BUILD_CONFIG[STATIC_LIBS_IMAGE_PATH]=$static_libs_dir if [ -z "$(command -v "${BUILD_CONFIG[CONTAINER_COMMAND]}")" ]; then - # shellcheck disable=SC2154 + # shellcheck disable=SC2154 echo "Error, please install docker and ensure that it is in your path and running!" exit fi @@ -164,25 +164,25 @@ buildOpenJDKViaDocker() createPersistentDockerDataVolume # If keep is true then use the existing container (or build a new one if we - # can't find it) + # can't find it). if [[ "${BUILD_CONFIG[REUSE_CONTAINER]}" == "true" ]] ; then # shellcheck disable=SC2086 - # If we can't find the previous Docker container then build a new one + # If we can't find the previous Docker container then build a new one. if [ "$(${BUILD_CONFIG[CONTAINER_AS_ROOT]} ${BUILD_CONFIG[CONTAINER_COMMAND]} ps -a | grep -c \"${BUILD_CONFIG[CONTAINER_NAME]}\")" == 0 ]; then - echo "No docker container for reuse was found, so creating '${BUILD_CONFIG[CONTAINER_NAME]}' " + echo "No docker container for reuse was found, so creating '${BUILD_CONFIG[CONTAINER_NAME]}'" buildDockerContainer fi else # shellcheck disable=SC2154 echo "Since you specified --ignore-container, we are removing the existing container (if it exists) and building you a new one{$good}" - # Find the previous Docker container and remove it (if it exists) + # Find the previous Docker container and remove it (if it exists). ${BUILD_CONFIG[CONTAINER_AS_ROOT]} "${BUILD_CONFIG[CONTAINER_COMMAND]}" ps -a | awk '{ print $1,$2 }' | grep "${BUILD_CONFIG[CONTAINER_NAME]}" | awk '{print $1 }' | xargs -I {} ${BUILD_CONFIG[CONTAINER_AS_ROOT]} "${BUILD_CONFIG[CONTAINER_COMMAND]}" rm -f {} - # Build a new container + # Build a new container. buildDockerContainer fi - # Show the user all of the config before we build + # Show the user all of the config before we build. displayParams echo "Target binary directory on host machine: ${hostDir}/target" @@ -190,12 +190,12 @@ buildOpenJDKViaDocker() local cpuSet cpuSet="0-$((BUILD_CONFIG[NUM_PROCESSORS] - 1))" - + local gitSshAccess=() if [[ "${BUILD_CONFIG[USE_SSH]}" == "true" ]] ; then gitSshAccess=(-v "${HOME}/.ssh:/home/build/.ssh" -v "${SSH_AUTH_SOCK}:/build-ssh-agent" -e "SSH_AUTH_SOCK=/build-ssh-agent") fi - + local dockerMode=() local dockerEntrypoint=(--entrypoint /openjdk/sbin/build.sh "${BUILD_CONFIG[CONTAINER_NAME]}") if [[ "${BUILD_CONFIG[DEBUG_DOCKER]}" == "true" ]] ; then @@ -203,7 +203,7 @@ buildOpenJDKViaDocker() dockerEntrypoint=(--entrypoint "/bin/sh" "${BUILD_CONFIG[CONTAINER_NAME]}" -c "/bin/bash") fi - # Command without gitSshAccess or dockerMode arrays + # Command without gitSshAccess or dockerMode arrays. if [ -e "${hostDir}"/pipelines ] ; then local pipelinesdir="${hostDir}"/pipelines else @@ -219,11 +219,11 @@ buildOpenJDKViaDocker() else local userns="" fi - local mountflag=Z #rw? maybe this should be bound to root/rootles content of BUILD_CONFIG[CONTAINER_AS_ROOT] rather then just podman/docker in USE_DOCKER? - mkdir -p "${hostDir}"/workspace/build # shouldnt be already there? - local localsourcesdirmount= + local mountflag=Z #rw? Maybe this should be bound to root/rootless content of BUILD_CONFIG[CONTAINER_AS_ROOT] rather then just podman/docker in USE_DOCKER? + mkdir -p "${hostDir}"/workspace/build # Shouldn't be already there? + local localsourcesdirmount="" if [ -n "${localsourcesdir}" ] ; then - localsourcesdirmount="-v ${localsourcesdir}:${localsourcesdir}:${mountflag}" #read only? Is copied anwya + localsourcesdirmount="-v ${localsourcesdir}:${localsourcesdir}:${mountflag}" # read only? Is copied anyway. fi echo "If you get permissions denied on ${targetdir} or ${pipelinesdir} try to turn off selinux" local commandString=( @@ -234,28 +234,28 @@ buildOpenJDKViaDocker() -v "${targetdir}":/"${BUILD_CONFIG[WORKSPACE_DIR]}"/"${BUILD_CONFIG[TARGET_DIR]}":"${mountflag}" -v "${pipelinesdir}":/openjdk/pipelines:"${mountflag}" -v "${configdir}":/"${BUILD_CONFIG[WORKSPACE_DIR]}"/"config":"${mountflag}" - -e "DEBUG_DOCKER_FLAG=${BUILD_CONFIG[DEBUG_DOCKER]}" + -e "DEBUG_DOCKER_FLAG=${BUILD_CONFIG[DEBUG_DOCKER]}" -e "BUILD_VARIANT=${BUILD_CONFIG[BUILD_VARIANT]}" "${dockerEntrypoint[@]:+${dockerEntrypoint[@]}}") - # If build specifies --ssh, add array to the command string + # If build specifies --ssh, add array to the command string. if [[ "${BUILD_CONFIG[USE_SSH]}" == "true" ]] ; then commandString=("${gitSshAccess[@]:+${gitSshAccess[@]}}" "${commandString[@]}") fi - # If build specifies --debug-docker, add array to the command string + # If build specifies --debug-docker, add array to the command string. if [[ "${BUILD_CONFIG[DEBUG_DOCKER]}" == "true" ]] ; then commandString=("${dockerMode[@]:+${dockerMode[@]}}" "${commandString[@]}") echo "DEBUG DOCKER MODE. To build jdk run /openjdk/sbin/build.sh" fi - # Run the command string in Docker + # Run the command string in Docker. ${BUILD_CONFIG[CONTAINER_AS_ROOT]} "${BUILD_CONFIG[CONTAINER_COMMAND]}" run --name "${BUILD_CONFIG[OPENJDK_CORE_VERSION]}-${BUILD_CONFIG[BUILD_VARIANT]}" "${commandString[@]}" - # Tell user where the resulting binary can be found on the host system + # Tell user where the resulting binary can be found on the host system. echo "The finished image can be found in ${targetdir} on the host system" - # If we didn't specify to keep the container then remove it + # If we didn't specify to keep the container then remove it. if [[ "${BUILD_CONFIG[KEEP_CONTAINER]}" == "false" ]] ; then echo "Removing container ${BUILD_CONFIG[OPENJDK_CORE_VERSION]}-${BUILD_CONFIG[BUILD_VARIANT]}" ${BUILD_CONFIG[CONTAINER_AS_ROOT]} "${BUILD_CONFIG[CONTAINER_COMMAND]}" ps -a | awk '{ print $1,$(NF) }' | grep "${BUILD_CONFIG[OPENJDK_CORE_VERSION]}-${BUILD_CONFIG[BUILD_VARIANT]}" | awk '{print $1 }' | xargs -I {} ${BUILD_CONFIG[CONTAINER_AS_ROOT]} ${BUILD_CONFIG[CONTAINER_COMMAND]} rm {} diff --git a/sbin/common/downloaders.sh b/sbin/common/downloaders.sh index 3424b089f..09d2daf5f 100755 --- a/sbin/common/downloaders.sh +++ b/sbin/common/downloaders.sh @@ -71,7 +71,7 @@ function downloadLinuxBootJDK() { set +e curl -L -o bootjdk.tar.gz "${apiURL}" if ! grep "No releases match the request" bootjdk.tar.gz; then - apiSigURL=$(curl -v "${apiURL}" 2>&1 | tr -d \\r | awk '/^< Location:/{print $3 ".sig"}') + apiSigURL=$(curl -v "${apiURL}" 2>&1 | tr -d \\r | awk '/^< [Ll]ocation:/{print $3 ".sig"}') curl -L -o bootjdk.tar.gz.sig "${apiSigURL}" gpg --keyserver keyserver.ubuntu.com --recv-keys 3B04D753C9050D9A5D343F39843C48A565F8F04B echo -e "5\ny\n" | gpg --batch --command-fd 0 --expert --edit-key 3B04D753C9050D9A5D343F39843C48A565F8F04B trust; diff --git a/test/system/reproducibleCompare/build.xml b/test/system/reproducibleCompare/build.xml index 015fbacad..75db0f1d9 100644 --- a/test/system/reproducibleCompare/build.xml +++ b/test/system/reproducibleCompare/build.xml @@ -65,6 +65,7 @@ + diff --git a/test/system/reproducibleCompare/playlist.xml b/test/system/reproducibleCompare/playlist.xml index c3c61ba26..9fd60ce40 100644 --- a/test/system/reproducibleCompare/playlist.xml +++ b/test/system/reproducibleCompare/playlist.xml @@ -60,4 +60,25 @@ os.win + + Rebuild_Same_JDK_Reproducibility_Test_Mac + unset SPEC;\ + ls -l $(TEST_ROOT)$(D)system$(D)reproducibleCompare$(D);\ + $(TEST_ROOT)$(D)system$(D)reproducibleCompare$(D)macos_repro_build_compare.sh $(SBOM_FILE) $(JDK_FILE) $(REPORTDIR); \ + $(TEST_STATUS) + + + dev + + + system + + + eclipse + + + 21+ + + os.osx,arch.aarch64 + \ No newline at end of file diff --git a/test/system/reproducibleCompare/reproducible.mk b/test/system/reproducibleCompare/reproducible.mk index 34b3b0342..e6fbfb2a1 100644 --- a/test/system/reproducibleCompare/reproducible.mk +++ b/test/system/reproducibleCompare/reproducible.mk @@ -16,6 +16,7 @@ ifndef SBOM_FILE SBOM_FILE := $(TEST_ROOT)/../jdkbinary/$(SBOM_FILE) endif ifndef JDK_FILE + JDK_FILE := $(shell find $(TEST_ROOT)/../jdkbinary/ -type f -name '*-jdk_*.tar.gz') ifneq (,$(findstring win,$(SPEC))) JDK_FILE := $(shell find $(TEST_ROOT)/../jdkbinary/ -type f -name '*-jdk_*.zip') endif diff --git a/tooling/reproducible/comparable_patch.sh b/tooling/reproducible/comparable_patch.sh index f342ff7ac..d04ff8f23 100755 --- a/tooling/reproducible/comparable_patch.sh +++ b/tooling/reproducible/comparable_patch.sh @@ -338,7 +338,8 @@ echo "Successfully removed all Signatures from ${JDK_DIR}" removeExcludedFiles # Needed due to vendor variation in jmod re-packing after signing, putting attributes in different order -processModuleInfo +# Comparable patch, as per read-me, requires java on path +processModuleInfo "${JDK_DIR}" "${OS}" "$(dirname "$(dirname "$(readlink -f "$(which java)")")")" # Patch Windows VS_VERSION_INFO[COMPANY_NAME] if [[ "$OS" =~ CYGWIN* ]] && [[ "$PATCH_VS_VERSION_INFO" = true ]]; then diff --git a/tooling/reproducible/macos_repro_build_compare.sh b/tooling/reproducible/macos_repro_build_compare.sh index edf0f7562..4831c76c5 100755 --- a/tooling/reproducible/macos_repro_build_compare.sh +++ b/tooling/reproducible/macos_repro_build_compare.sh @@ -46,16 +46,11 @@ REPORT_DIR="$3" # The Defaults Below Are Suitable For An Adoptium Mac OS X Build Environment # Which Has Been Created Via The Ansible Infrastructure Playbooks -WORK_DIR=~/comp-jdk-build +WORK_DIR=$(realpath "$(dirname "$0")")/comp-jdk-build MAC_COMPILER_BASE=/Applications MAC_COMPILER_APP_PREFIX=Xcode MAC_SDK_LOCATION=/Library/Developer/CommandLineTools/SDKs/MacOSX.sdk -# These 3 variables dictate which parameters passed to makejdk_any_platform.sh are config arguments, build arguments or should be ignored. -CONFIG_ARGS=("--disable-warnings-as-errors" "--openjdk-target" "--with-sysroot" "--with-extra-cxxflags='" "--enable-dtrace" "--with-version-opt") -NOTUSE_ARGS=("--assemble-exploded-image" "--configure-args") -FINAL_ARG=("--build-variant") - # These variables relate to the pre-requisite ant installation ANT_VERSION="1.10.5" ANT_CONTRIB_VERSION="1.0b3" @@ -76,17 +71,6 @@ is_url() { fi } -# Function to check if a value is in the array -containsElement () { - local e - for e in "${@:2}"; do - if [ "$e" == "$1" ]; then - return 0 # Match found - fi - done - return 1 # No match found -} - Create_WorkDir() { # Check if the folder exists & remove if it does echo "Checking If Working Directory: $WORK_DIR Exists" @@ -371,13 +355,13 @@ Check_And_Install_Ant() { echo "Ant Doesnt Exist At The Correct Version - Installing" # Ant Version Not Found... Check And Create Paths echo Downloading ant for SBOM creation: - curl https://archive.apache.org/dist/ant/binaries/apache-ant-${ANT_VERSION}-bin.zip > $WORK_DIR/apache-ant-${ANT_VERSION}-bin.zip - (cd $WORK_DIR && unzip -qn ./apache-ant-${ANT_VERSION}-bin.zip) - rm $WORK_DIR/apache-ant-${ANT_VERSION}-bin.zip + curl https://archive.apache.org/dist/ant/binaries/apache-ant-${ANT_VERSION}-bin.zip > "${WORK_DIR}"/apache-ant-${ANT_VERSION}-bin.zip + (cd "$WORK_DIR" && unzip -qn ./apache-ant-${ANT_VERSION}-bin.zip) + rm "$WORK_DIR"/apache-ant-${ANT_VERSION}-bin.zip echo Downloading ant-contrib-${ANT_CONTRIB_VERSION}: - curl -L https://sourceforge.net/projects/ant-contrib/files/ant-contrib/${ANT_CONTRIB_VERSION}/ant-contrib-${ANT_CONTRIB_VERSION}-bin.zip > $WORK_DIR/ant-contrib-${ANT_CONTRIB_VERSION}-bin.zip - (unzip -qnj $WORK_DIR/ant-contrib-${ANT_CONTRIB_VERSION}-bin.zip ant-contrib/ant-contrib-${ANT_CONTRIB_VERSION}.jar -d $WORK_DIR/apache-ant-${ANT_VERSION}/lib) - rm $WORK_DIR/ant-contrib-${ANT_CONTRIB_VERSION}-bin.zip + curl -L https://sourceforge.net/projects/ant-contrib/files/ant-contrib/${ANT_CONTRIB_VERSION}/ant-contrib-${ANT_CONTRIB_VERSION}-bin.zip > "$WORK_DIR"/ant-contrib-${ANT_CONTRIB_VERSION}-bin.zip + (unzip -qnj "$WORK_DIR"/ant-contrib-${ANT_CONTRIB_VERSION}-bin.zip ant-contrib/ant-contrib-${ANT_CONTRIB_VERSION}.jar -d "$WORK_DIR"/apache-ant-${ANT_VERSION}/lib) + rm "$WORK_DIR"/ant-contrib-${ANT_CONTRIB_VERSION}-bin.zip else echo "Ant Version: $ANT_VERSION Is Already Installed" fi @@ -441,118 +425,24 @@ Prepare_Env_For_Build() { echo "Setting Variables" export BOOTJDK_HOME=$WORK_DIR/jdk-${bootJDK}/Contents/Home - echo "Parsing Make JDK Any Platform ARGS For Build" - # Split the string into an array of words - IFS=' ' read -ra words <<< "$buildArgs" - - # Add The Build Time Stamp In Case It Wasnt In The SBOM ARGS - words+=( " --build-reproducible-date \"$buildStamp\"" ) - - # Initialize variables - param="" - value="" - params=() - - # Loop through the words - for word in "${words[@]}"; do - # Check if the word starts with '--' - if [[ $word == --* ]] || [[ $word == -b* ]]; then - # If a parameter already exists, store it in the params array - if [[ -n $param ]]; then - params+=("$param $value") - fi - # Reset variables for the new parameter - param="$word" - value="" - else - value+=" $word" - fi - done - - # Add the last parameter to the array - params+=("$param $value") - - # Loop Through The Parameters And Reformat Appropriately - export fixed_param="" - export fixed_value="" - export fixed_params=() - export new_params="" - CONFIG_ARRAY=() - BUILD_ARRAY=() - FINAL_ARRAY=() - for element in "${params[@]}"; do - IFS=' ' read -ra parts <<< "$element" - prepped_part0=${parts[0]} - prepped_part1=${parts[1]} - prepped_part2=${parts[2]} - - fixed_param="$prepped_part0" - fixed_value="$prepped_part1 $prepped_part2" - - # Handle Special Parameters ( overrides and = seperated ) - if [ "$fixed_param" == "--jdk-boot-dir" ]; then fixed_value=" $BOOTJDK_HOME " ; fi - if [[ "$fixed_param" == "--with-sysroot="* ]]; then - IFS='=' read -r split_param split_value <<< "$fixed_param" - fixed_param="$split_param" - fixed_value="=$MAC_SDK_LOCATION " ; - fi - if [[ "$fixed_param" == "--openjdk-target="* ]]; then - IFS='=' read -r split_param split_value <<< "$fixed_param" - fixed_param="$split_param" - fixed_value="=$split_value " - fi - if [[ "$fixed_param" == "--with-version-opt="* ]]; then - IFS='=' read -r split_param split_value <<< "$fixed_param" - fixed_param="$split_param" - fixed_value="=$split_value" - fi - if [ "$fixed_param" == "--tag" ]; then fixed_param="-b" fixed_value=" $fixed_value" ; fi - if [ "$fixed_param" == "--target-file-name" ]; then - target_file="$(echo -e "${fixed_value}" | sed -e 's/^[[:space:]]*//' -e 's/[[:space:]]*$//')" - fixed_value=" $fixed_value" - fi - if [ "$fixed_param" == "--freetype-dir" ]; then fixed_value=" $fixed_value" ; fi - if [ "$fixed_param" == "--user-openjdk-build-root-directory" ]; then fixed_value=" $WORK_DIR/temurin-build/workspace/build/openjdkbuild/ " ; fi - if [ "$fixed_param" == "--build-variant" ]; then fixed_value=" $fixed_value " ; fi - if [ "$fixed_param" == "--build-reproducible-date" ]; then fixed_value=" $fixed_value" ; fi - - if containsElement "$fixed_param" "${CONFIG_ARGS[@]}"; then - # Check if fixed_param is in CONFIG_ARGS - STRINGTOADD="$fixed_param$fixed_value" - CONFIG_ARRAY+=("$STRINGTOADD") - elif containsElement "$fixed_param" "${NOTUSE_ARGS[@]}"; then - # Check if fixed_param is in NOTUSE_ARGS - STRINGTOADD="$fixed_param$fixed_value" - IGNORED_ARRAY+=("$STRINGTOADD") - elif containsElement "$fixed_param" "${FINAL_ARG[@]}"; then - # Check if parameter should be at the end, usually just version & variant - STRINGTOADD="$fixed_param$fixed_value" - FINAL_ARRAY+=("$STRINGTOADD") - else - # Neither Config Or Ignore, so Build - STRINGTOADD="$fixed_param$fixed_value" - BUILD_ARRAY+=("$STRINGTOADD") - fi - done - - # Construct Final Parameter String - for element in "${BUILD_ARRAY[@]}"; do - build_string+="$element" - done - - for element in "${CONFIG_ARRAY[@]}"; do - config_string+="$element" - done - - for element in "${FINAL_ARRAY[@]}"; do - final_string+="$element" - done - - final_params="$build_string --configure-args \"$config_string\" $final_string" + # set --build-reproducible-date if not yet + if [[ "${buildArgs}" != *"--build-reproducible-date"* ]]; then + buildArgs="--build-reproducible-date \"${buildStamp}\" ${buildArgs}" + fi + # reset --jdk-boot-dir + # shellcheck disable=SC2001 + buildArgs="$(echo "$buildArgs" | sed -e "s|--jdk-boot-dir [^ ]*|--jdk-boot-dir ${BOOTJDK_HOME}|")" + # shellcheck disable=SC2001 + buildArgs="$(echo "$buildArgs" | sed -e "s|--with-sysroot=[^ ]*|--with-sysroot=${MAC_SDK_LOCATION}|")" + # shellcheck disable=SC2001 + buildArgs="$(echo "$buildArgs" | sed -e "s|--user-openjdk-build-root-directory [^ ]*|--user-openjdk-build-root-directory ${WORK_DIR}/temurin-build/workspace/build/openjdkbuild/|")" + # remove ingored options + buildArgs=${buildArgs/--assemble-exploded-image /} + buildArgs=${buildArgs/--enable-sbom-strace /} -echo "" + echo "" echo "Make JDK Any Platform Argument List = " - echo "$final_params" + echo "$buildArgs" echo "" echo "Parameters Parsed Successfully" } @@ -562,79 +452,57 @@ Build_JDK() { # Trigger Build cd "$WORK_DIR" - echo "cd temurin-build && ./makejdk-any-platform.sh $final_params 2>&1 | tee build.$$.log" | sh + echo "cd temurin-build && ./makejdk-any-platform.sh $buildArgs 2>&1 | tee build.$$.log" | sh # Copy The Built JDK To The Working Directory - cp $WORK_DIR/temurin-build/workspace/target/"$target_file" $WORK_DIR/reproJDK.tar.gz + cp "$WORK_DIR"/temurin-build/workspace/target/OpenJDK*-jdk_*tar.gz "$WORK_DIR"/reproJDK.tar.gz } Compare_JDK() { echo "Comparing JDKs" echo "" mkdir "$WORK_DIR/compare" - cp $WORK_DIR/src_jdk_dist.tar.gz $WORK_DIR/compare - cp $WORK_DIR/reproJDK.tar.gz $WORK_DIR/compare - - - # Check The Comparison Scripts Are Present And Copy To The Working Directory + cp "$WORK_DIR"/src_jdk_dist.tar.gz "$WORK_DIR"/compare + cp "$WORK_DIR"/reproJDK.tar.gz "$WORK_DIR"/compare + cp "$(dirname "$0")"/repro_*.sh "$WORK_DIR"/compare/ - if [ -f ./repro_common.sh ]; then - cp ./repro_common.sh $WORK_DIR/compare/repro_common.sh - else - wget -O "$WORK_DIR/compare/repro_common.sh" "https://raw.githubusercontent.com/adoptium/temurin-build/master/tooling/reproducible/repro_common.sh" - fi - if [ -f ./repro_compare.sh ]; then - cp ./repro_compare.sh $WORK_DIR/compare/repro_compare.sh - else - wget -O "$WORK_DIR/compare/repro_compare.sh" "https://raw.githubusercontent.com/adoptium/temurin-build/master/tooling/reproducible/repro_compare.sh" - fi - if [ -f ./repro_process.sh ]; then - cp ./repro_process.sh $WORK_DIR/compare/repro_process.sh - else - wget -O "$WORK_DIR/compare/repro_process.sh" "https://raw.githubusercontent.com/adoptium/temurin-build/master/tooling/reproducible/repro_process.sh" - fi - # Set Permissions chmod +x "$WORK_DIR/compare/"*sh cd "$WORK_DIR/compare" # Unzip And Rename The Source JDK echo "Unzip Source" - tar xvfz src_jdk_dist.tar.gz + tar xfz src_jdk_dist.tar.gz original_directory_name=$(find . -maxdepth 1 -type d | tail -1) mv "$original_directory_name" src_jdk #Unzip And Rename The Target JDK echo "Unzip Target" - tar xvfz reproJDK.tar.gz + tar xfz reproJDK.tar.gz original_directory_name=$(find . -maxdepth 1 -type d | grep -v src_jdk | tail -1) mv "$original_directory_name" tar_jdk # Ensure Java Home Is Set export JAVA_HOME=$BOOTJDK_HOME export PATH=$JAVA_HOME/bin:$PATH - echo "cd $WORK_DIR/compare && ./repro_compare.sh temurin src_jdk/Contents/Home temurin tar_jdk/Contents/Home Darwin 2>&1" | sh & - wait - rc=$? - set -e + rc=0 + ./repro_compare.sh temurin src_jdk/Contents/Home temurin tar_jdk/Contents/Home Darwin 2>&1 || rc=$? cd "$WORK_DIR" - # Display The Content Of reprotest.diff - echo "" - echo "---------------------------------------------" - echo "Output From JDK Comparison Script" - echo "---------------------------------------------" - cat $WORK_DIR/compare/reprotest.diff - echo "" - echo "---------------------------------------------" - echo "Copying Output To $(dirname "$0")" - cp $WORK_DIR/compare/reprotest.diff $WORK_DIR/reprotest.diff - + + if [ $rc -eq 0 ]; then + echo "Compare identical !" + else + echo "Differences found..., logged in: reprotest.diff" + fi + if [ -n "$REPORT_DIR" ]; then echo "Copying Output To $REPORT_DIR" - cp $WORK_DIR/compare/reprotest.diff "$REPORT_DIR" - cp $WORK_DIR/reproJDK.tar.gz "$REPORT_DIR" + cp "$WORK_DIR"/compare/reprotest.diff "$REPORT_DIR" + cp "$WORK_DIR"/reproJDK.tar.gz "$REPORT_DIR" fi - + Clean_Up_Everything + exit $rc } + # Clean_Up_Everything() { # Remove Working Directorys @@ -674,6 +542,3 @@ echo "---------------------------------------------" Build_JDK echo "---------------------------------------------" Compare_JDK -echo "---------------------------------------------" -Clean_Up_Everything -exit $rc diff --git a/tooling/reproducible/repro_common.sh b/tooling/reproducible/repro_common.sh index 8f7ca2942..5648fef81 100755 --- a/tooling/reproducible/repro_common.sh +++ b/tooling/reproducible/repro_common.sh @@ -89,23 +89,23 @@ function removeSystemModulesHashBuilderParams() { moduleHashesFunction="// Method jdk/internal/module/ModuleHashes\$Builder.hashForModule:(Ljava/lang/String;[B)Ljdk/internal/module/ModuleHashes\$Builder;" moduleString="// String " virtualFunction="invokevirtual" - local JDK_DIR="$1" systemModules="SystemModules\$0.class SystemModules\$all.class SystemModules\$default.class" + local JDK_DIR="$1" + local OS="$2" + local work_JDK="$3" for systemModule in $systemModules do FILES=$(find "${JDK_DIR}" -type f -name "$systemModule") for f in $FILES do + ff=$f if [[ "$OS" =~ CYGWIN* ]]; then ff=$(cygpath -w $f) - else - ff=$f fi - javap -v -sysinfo -l -p -c -s -constants "$ff" > "$f.javap.tmp" - rm "$f" - + "${work_JDK}"/bin/javap -v -sysinfo -l -p -c -s -constants "$ff" > "$f.javap.tmp" + # Remove "instruction number:" prefix, so we can just match code - if [[ $(uname) =~ Darwin* ]]; then + if [[ "$OS" =~ Darwin* ]]; then sed -i "" -E 's/^[[:space:]]+[0-9]+:(.*)/\1/' "$f.javap.tmp" else sed -i -E 's/^[[:space:]]+[0-9]+:(.*)/\1/' "$f.javap.tmp" @@ -137,6 +137,7 @@ function removeSystemModulesHashBuilderParams() { rm "$f.javap.tmp" grep -v "Last modified\|Classfile\|SHA-256 checksum" "$f.javap.tmp2" > "$f.javap" rm "$f.javap.tmp2" + rm "$f" done done @@ -153,7 +154,6 @@ function removeSystemModulesHashBuilderParams() { # reprohex - A hex UUID to identify the binary version, again generated from binary content function removeWindowsNonComparableData() { echo "Removing EXE/DLL timestamps, CRC and debug repro hex from ${JDK_DIR}" - # We need to do this for all executables if patching VS_VERSION_INFO if [[ "$PATCH_VS_VERSION_INFO" = true ]]; then FILES=$(find "${JDK_DIR}" -type f -path '*.exe' && find "${JDK_DIR}" -type f -path '*.dll') @@ -223,7 +223,7 @@ function removeWindowsNonComparableData() { # See https://github.com/adoptium/temurin-build/issues/2899#issuecomment-1153757419 function removeMacOSNonComparableData() { echo "Removing MacOS dylib non-comparable UUID from ${JDK_DIR}" - + MAC_JDK_ROOT="${JDK_DIR}/../../Contents" FILES=$(find "${MAC_JDK_ROOT}" \( -type f -and -path '*.dylib' -or -path '*/bin/*' -or -path '*/lib/jspawnhelper' -not -path '*/modules_extracted/*' -or -path '*/jpackageapplauncher*' \)) for f in $FILES do @@ -251,80 +251,77 @@ function removeMacOSNonComparableData() { # java.base also requires the dependent module "hash:" values to be excluded # as they differ due to the Signatures function processModuleInfo() { - if [[ "$OS" =~ CYGWIN* ]] || [[ "$OS" =~ Darwin* ]]; then - echo "Normalizing ModuleAttributes order in module-info.class, converting to javap" - - moduleAttr="ModuleResolution ModuleTarget" + local JDK_DIR="$1" + local OS="$2" + local work_JDK="$3" + echo "process Module Info from ${JDK_DIR}" + echo "Normalizing ModuleAttributes order in module-info.class, converting to javap" + moduleAttr="ModuleResolution ModuleTarget" + FILES=$(find "${JDK_DIR}" -type f -name "module-info.class") + for f in $FILES + do + ff=$f + if [[ "$OS" =~ CYGWIN* ]]; then + ff=$(cygpath -w $f) + fi + "${work_JDK}"/bin/javap -v -sysinfo -l -p -c -s -constants "$ff" > "$f.javap.tmp" + cc=99 + foundAttr=false + attrName="" + # Clear any attr tmp files + for attr in $moduleAttr + do + rm -f "$f.javap.$attr" + done - FILES=$(find "${JDK_DIR}" -type f -name "module-info.class") - for f in $FILES + while IFS= read -r line do - if [[ "$OS" =~ CYGWIN* ]]; then - ff=$(cygpath -w $f) - else - ff=$f + cc=$((cc+1)) + + # Module attr have only 1 line definition + if [[ "$foundAttr" = true ]] && [[ "$cc" -gt 1 ]]; then + foundAttr=false + attrName="" fi - javap -v -sysinfo -l -p -c -s -constants "$ff" > "$f.javap.tmp" - rm "$f" - cc=99 - foundAttr=false - attrName="" - # Clear any attr tmp files - for attr in $moduleAttr - do - rm -f "$f.javap.$attr" - done - - while IFS= read -r line - do - cc=$((cc+1)) - - # Module attr have only 1 line definition - if [[ "$foundAttr" = true ]] && [[ "$cc" -gt 1 ]]; then - foundAttr=false - attrName="" - fi - - # If not processing an attr then check for attr - if [[ "$foundAttr" = false ]]; then - for attr in $moduleAttr - do - if [[ "$line" =~ .*"$attr:".* ]]; then - cc=0 - foundAttr=true - attrName="$attr" - fi - done - fi - - # Echo attr to attr tmp file, otherwise to tmp2 - if [[ "$foundAttr" = true ]]; then - echo "$line" >> "$f.javap.$attrName" - else - echo "$line" >> "$f.javap.tmp2" - fi - done < "$f.javap.tmp" - rm "$f.javap.tmp" - - # Remove javap Classfile and timestamp and SHA-256 hash - if [[ "$f" =~ .*"java.base".* ]]; then - grep -v "Last modified\|Classfile\|SHA-256 checksum\|hash:" "$f.javap.tmp2" > "$f.javap" + # If not processing an attr then check for attr + if [[ "$foundAttr" = false ]]; then + for attr in $moduleAttr + do + if [[ "$line" =~ .*"$attr:".* ]]; then + cc=0 + foundAttr=true + attrName="$attr" + fi + done + fi + + # Echo attr to attr tmp file, otherwise to tmp2 + if [[ "$foundAttr" = true ]]; then + echo "$line" >> "$f.javap.$attrName" else - grep -v "Last modified\|Classfile\|SHA-256 checksum" "$f.javap.tmp2" > "$f.javap" + echo "$line" >> "$f.javap.tmp2" fi - rm "$f.javap.tmp2" - - # Append any ModuleAttr tmp files - for attr in $moduleAttr - do - if [[ -f "$f.javap.$attr" ]]; then - cat "$f.javap.$attr" >> "$f.javap" - fi - rm -f "$f.javap.$attr" - done + done < "$f.javap.tmp" + rm "$f.javap.tmp" + rm "$f" + # Remove javap Classfile and timestamp and SHA-256 hash + if [[ "$f" =~ .*"java.base".* ]]; then + grep -v "Last modified\|Classfile\|SHA-256 checksum\|hash:" "$f.javap.tmp2" > "$f.javap" + else + grep -v "Last modified\|Classfile\|SHA-256 checksum" "$f.javap.tmp2" > "$f.javap" + fi + rm "$f.javap.tmp2" + + # Append any ModuleAttr tmp files + for attr in $moduleAttr + do + if [[ -f "$f.javap.$attr" ]]; then + cat "$f.javap.$attr" >> "$f.javap" + fi + rm -f "$f.javap.$attr" done - fi + done } # Remove windowns generate classes jdk/bin/server/classes.jsa & jdk/bin/server/classes_nocoops.jsa diff --git a/tooling/reproducible/repro_compare.sh b/tooling/reproducible/repro_compare.sh index dd707fa87..2ee8616fb 100755 --- a/tooling/reproducible/repro_compare.sh +++ b/tooling/reproducible/repro_compare.sh @@ -13,7 +13,7 @@ # ******************************************************************************** # shellcheck disable=SC1091 -source repro_common.sh +source "$(dirname "$0")"/repro_common.sh BLD_TYPE1="$1" JDK_DIR1="$2" @@ -21,6 +21,10 @@ BLD_TYPE2="$3" JDK_DIR2="$4" OS="$5" +mkdir "${JDK_DIR}_BK" +cp -R "${JDK_DIR1}"/* "${JDK_DIR}"_BK +BK_JDK_DIR=$(realpath "${JDK_DIR}"_BK/) + JDK_DIR_Arr=("${JDK_DIR1}" "${JDK_DIR2}") for JDK_DIR in "${JDK_DIR_Arr[@]}" do @@ -32,7 +36,7 @@ do echo "Pre-processing ${JDK_DIR}" rc=0 - source ./repro_process.sh "${JDK_DIR}" "${OS}" || rc=$? + source "$(dirname "$0")"/repro_process.sh "${JDK_DIR}" "${OS}" || rc=$? if [ $rc != 0 ]; then echo "Pre-process of ${JDK_DIR} ${OS} failed" exit 1 @@ -46,12 +50,11 @@ do cleanTemurinBuildInfo "${JDK_DIR}" if [[ "$OS" =~ CYGWIN* ]] || [[ "$OS" =~ Darwin* ]]; then - removeSystemModulesHashBuilderParams "${JDK_DIR}" + removeSystemModulesHashBuilderParams "${JDK_DIR}" "${OS}" "${BK_JDK_DIR}" + processModuleInfo "${JDK_DIR}" "${OS}" "${BK_JDK_DIR}" fi - processModuleInfo done - files1=$(find "${JDK_DIR1}" -type f | wc -l) echo "Number of files: ${files1}" rc=0 diff --git a/tooling/reproducible/repro_process.sh b/tooling/reproducible/repro_process.sh index 92b246f2a..d44f9ac8b 100755 --- a/tooling/reproducible/repro_process.sh +++ b/tooling/reproducible/repro_process.sh @@ -13,7 +13,7 @@ # ******************************************************************************** # shellcheck disable=SC1091 -source repro_common.sh +source "$(dirname "$0")"/repro_common.sh JDK_DIR="$1" OS="$2"