Merge branch 'master' into docDisableAadoptBranchSafety

build-farm/platform-specific-configurations/alpine-linux.sh

@@ -49,7 +49,7 @@ export GNUPGHOME
 
 BOOT_JDK_VARIABLE="JDK${JDK_BOOT_VERSION}_BOOT_DIR"
 if [ ! -d "$(eval echo "\$$BOOT_JDK_VARIABLE")" ]; then
-  bootDir="$PWD/jdk-$JDK_BOOT_VERSION"
+  bootDir="$PWD/jdk$JDK_BOOT_VERSION"
   # Note we export $BOOT_JDK_VARIABLE (i.e. JDKXX_BOOT_DIR) here
   # instead of BOOT_JDK_VARIABLE (no '$').
   export "${BOOT_JDK_VARIABLE}"="$bootDir"

build-farm/platform-specific-configurations/linux.sh

@@ -51,11 +51,11 @@ function locateDragonwell8BootJDK()
     export "${BOOT_JDK_VARIABLE}"=/usr/lib/jvm/dragonwell8
   else
     echo Dragonwell 8 requires a Dragonwell boot JDK - downloading one ...
-    mkdir -p "$PWD/jdk-8"
+    mkdir -p "$PWD/jdk8"
     # if [ "$(uname -m)" = "x86_64" ]; then
-    #   curl -L "https://github.com/alibaba/dragonwell8/releases/download/dragonwell-8.11.12_jdk8u332-ga/Alibaba_Dragonwell_8.11.12_x64_linux.tar.gz" | tar xpzf - --strip-components=1 -C "$PWD/jdk-8"
+    #   curl -L "https://github.com/alibaba/dragonwell8/releases/download/dragonwell-8.11.12_jdk8u332-ga/Alibaba_Dragonwell_8.11.12_x64_linux.tar.gz" | tar xpzf - --strip-components=1 -C "$PWD/jdk8"
     # elif [ "$(uname -m)" = "aarch64" ]; then
-    #   curl -L "https://github.com/alibaba/dragonwell8/releases/download/dragonwell-8.8.9_jdk8u302-ga/Alibaba_Dragonwell_8.8.9_aarch64_linux.tar.gz" | tar xpzf - --strip-components=1 -C "$PWD/jdk-8"
+    #   curl -L "https://github.com/alibaba/dragonwell8/releases/download/dragonwell-8.8.9_jdk8u302-ga/Alibaba_Dragonwell_8.8.9_aarch64_linux.tar.gz" | tar xpzf - --strip-components=1 -C "$PWD/jdk8"
     # else
     #   echo "Unknown architecture $(uname -m) for building Dragonwell - cannot download boot JDK"
     #   exit 1
@@ -86,11 +86,11 @@ function locateDragonwell8BootJDK()
     fi
 
     # Extract the downloaded file
-    tar xpzf "$TMP_FILE" --strip-components=1 -C "$PWD/jdk-8"
+    tar xpzf "$TMP_FILE" --strip-components=1 -C "$PWD/jdk8"
 
     # Clean up the temporary file
     rm "$TMP_FILE"
-    export "${BOOT_JDK_VARIABLE}"="$PWD/jdk-8"
+    export "${BOOT_JDK_VARIABLE}"="$PWD/jdk8"
   fi
 }
 
@@ -273,16 +273,16 @@ if [ "${VARIANT}" == "${BUILD_VARIANT_DRAGONWELL}" ] && [ "$JAVA_FEATURE_VERSION
 fi
 
 if [ ! -d "$(eval echo "\$$BOOT_JDK_VARIABLE")" ]; then
-  bootDir="$PWD/jdk-$JDK_BOOT_VERSION"
+  bootDir="$PWD/jdk$JDK_BOOT_VERSION"
   # Note we export $BOOT_JDK_VARIABLE (i.e. JDKXX_BOOT_DIR) here
   # instead of BOOT_JDK_VARIABLE (no '$').
   export "${BOOT_JDK_VARIABLE}"="$bootDir"
   if [ ! -x "$bootDir/bin/javac" ]; then
     # Set to a default location as linked in the ansible playbooks
-    if [ -x "/usr/lib/jvm/jdk-${JDK_BOOT_VERSION}/bin/javac" ]; then
-      echo "Could not use ${BOOT_JDK_VARIABLE} - using /usr/lib/jvm/jdk-${JDK_BOOT_VERSION}"
+    if [ -x "/usr/lib/jvm/jdk${JDK_BOOT_VERSION}/bin/javac" ]; then
+      echo "Could not use ${BOOT_JDK_VARIABLE} - using /usr/lib/jvm/jdk${JDK_BOOT_VERSION}"
       # shellcheck disable=SC2140
-      export "${BOOT_JDK_VARIABLE}"="/usr/lib/jvm/jdk-${JDK_BOOT_VERSION}"
+      export "${BOOT_JDK_VARIABLE}"="/usr/lib/jvm/jdk${JDK_BOOT_VERSION}"
     elif [ "$JDK_BOOT_VERSION" -ge 8 ]; then # Adoptium has no build pre-8
       downloadLinuxBootJDK "${ARCHITECTURE}" "${JDK_BOOT_VERSION}" "$bootDir"
     fi

build-farm/platform-specific-configurations/mac.sh

@@ -105,15 +105,15 @@ if [ "${JDK_BOOT_VERSION}" == "7" ]; then
 fi
 BOOT_JDK_VARIABLE="JDK${JDK_BOOT_VERSION}_BOOT_DIR"
 if [ ! -d "$(eval echo "\$$BOOT_JDK_VARIABLE")" ]; then
-  bootDir="$PWD/jdk-$JDK_BOOT_VERSION"
+  bootDir="$PWD/jdk$JDK_BOOT_VERSION"
   # Note we export $BOOT_JDK_VARIABLE (i.e. JDKXX_BOOT_DIR) here
   # instead of BOOT_JDK_VARIABLE (no '$').
   export "${BOOT_JDK_VARIABLE}"="$bootDir/Contents/Home"
   if [ ! -x "$bootDir/Contents/Home/bin/javac" ]; then
-    # To support multiple vendor names we set a jdk-* symlink pointing to the actual boot JDK
-    if [ -x "/Library/Java/JavaVirtualMachines/jdk-${JDK_BOOT_VERSION}/Contents/Home/bin/javac" ]; then
-      echo "Could not use ${BOOT_JDK_VARIABLE} - using /Library/Java/JavaVirtualMachines/jdk-${JDK_BOOT_VERSION}/Contents/Home"
-      export "${BOOT_JDK_VARIABLE}"="/Library/Java/JavaVirtualMachines/jdk-${JDK_BOOT_VERSION}/Contents/Home"
+    # To support multiple vendor names we set a jdk* symlink pointing to the actual boot JDK
+    if [ -x "/Library/Java/JavaVirtualMachines/jdk${JDK_BOOT_VERSION}/Contents/Home/bin/javac" ]; then
+      echo "Could not use ${BOOT_JDK_VARIABLE} - using /Library/Java/JavaVirtualMachines/jdk${JDK_BOOT_VERSION}/Contents/Home"
+      export "${BOOT_JDK_VARIABLE}"="/Library/Java/JavaVirtualMachines/jdk${JDK_BOOT_VERSION}/Contents/Home"
     elif [ "$JDK_BOOT_VERSION" -ge 8 ]; then # Adoptium has no build pre-8
       mkdir -p "$bootDir"
       for releaseType in "ga" "ea"

sbin/build.sh

@@ -79,12 +79,18 @@ configureDevKitConfigureParameter() {
     if [[ "$OSTYPE" == "cygwin" ]] || [[ "$OSTYPE" == "msys" ]]; then
       # Windows DevKit, currently only Redist DLLs
 
+      # Default to build architecture unless target ARCHITECTURE variable is set
+      local target_arch="${BUILD_CONFIG[OS_ARCHITECTURE]}"
+      if [ ${ARCHITECTURE+x} ] && [ -n "${ARCHITECTURE}" ]; then
+        target_arch="${ARCHITECTURE}"
+      fi
+      echo "Target architecture for Windows devkit: ${target_arch}"
+
       # This is TARGET Architecture for the Redist DLLs to use
-      # ARCHITECTURE is set to the "target" architecture by caller, or defaults to build architecture if not set
       local dll_arch
-      if [[ "${ARCHITECTURE}" == "x86-32" ]]; then
+      if [[ "${target_arch}" == "x86-32" ]]; then
         dll_arch="x86"
-      elif [[ "${ARCHITECTURE}" == "aarch64" ]]; then
+      elif [[ "${target_arch}" == "aarch64" ]]; then
         dll_arch="arm64"
       else
         dll_arch="x64"

test/system/reproducibleCompare/playlist.xml

@@ -19,7 +19,7 @@
 	<test>
 		<testCaseName>Rebuild_Same_JDK_Reproducibility_Test</testCaseName>
 		<command>docker container rm -f reproducibleCompare; \
-	docker run -v "$(TEST_RESROOT):/home/jenkins/test" -w "/home/jenkins/" -v "$(TEST_JDK_HOME):/home/jenkins/jdkbinary/" --name reproducibleCompare centos:7 /bin/bash /home/jenkins/test/linux_repro_build_compare.sh $(SBOM_FILE) /home/jenkins/jdkbinary; \
+	docker run --sysctl net.ipv6.conf.all.disable_ipv6=0 --sysctl net.ipv6.conf.default.disable_ipv6=0 -v "$(TEST_RESROOT):/home/jenkins/test" -w "/home/jenkins/" -v "$(TEST_JDK_HOME):/home/jenkins/jdkbinary/" --name reproducibleCompare centos:7 /bin/bash /home/jenkins/test/linux_repro_build_compare.sh $(SBOM_FILE) /home/jenkins/jdkbinary; \
 	$(TEST_STATUS); \
 	docker cp reproducibleCompare:/home/jenkins/reprotest.diff ./; \
 	docker cp reproducibleCompare:/home/jenkins/reproJDK.tar.gz ./; \

tooling/reproducible/repro_common.sh

@@ -25,10 +25,11 @@ function expandJDK() {
     JDK_ROOT=$(realpath ${JDK_DIR}/../../)
     JDK_BIN_DIR="${JDK_ROOT}_CP/Contents/Home/bin"
   fi
-
+  echo "$(date +%T) : Expanding various components to enable comparisons ${JDK_DIR} (original files will be removed):"
   mkdir "${JDK_ROOT}_CP"
   cp -R ${JDK_ROOT}/* ${JDK_ROOT}_CP
-  echo "Expanding the 'modules' Image to compare extracted files"
+
+  echo "$(date +%T) :   Using 'jimage extract' to expand lib/modules image into lib/modules_extracted"
   modulesFile="${JDK_DIR}/lib/modules"
   mkdir "${JDK_DIR}/lib/modules_extracted"
   extractedDir="${JDK_DIR}/lib/modules_extracted"
@@ -38,11 +39,11 @@ function expandJDK() {
   fi
   "${JDK_BIN_DIR}/jimage" extract --dir "${extractedDir}" "${modulesFile}"
   rm "${JDK_DIR}/lib/modules"
-  echo "Expanding the 'src.zip' to normalize file permissions"
-  unzip "${JDK_DIR}/lib/src.zip" -d "${JDK_DIR}/lib/src_zip_expanded" 1> /dev/null
+  echo "$(date +%T) :   Unzipping lib/src.zip to normalize file permissions, then removing src.zip"
+  unzip -q "${JDK_DIR}/lib/src.zip" -d "${JDK_DIR}/lib/src_zip_expanded"
   rm "${JDK_DIR}/lib/src.zip"
 
-  echo "Expanding jmods to process binaries within"
+  echo "$(date +%T) :   Using 'jmod extract' to expand all jmods to jmods/expanded_ directories"
   FILES=$(find "${JDK_DIR}" -type f -path '*.jmod')
   for f in $FILES
     do
@@ -58,21 +59,21 @@ function expandJDK() {
       rm "$f"
     done
 
-  echo "Expanding the 'jrt-fs.jar' to remove signatures from within.."
+  echo "$(date +%T) :   Expanding lib/jrt-fs.jar to lib/jrt-fs-exanded to remove signatures from within.."
   mkdir "${JDK_DIR}/lib/jrt-fs-expanded"
-  unzip -d "${JDK_DIR}/lib/jrt-fs-expanded" "${JDK_DIR}/lib/jrt-fs.jar" 1> /dev/null
+  unzip -qd "${JDK_DIR}/lib/jrt-fs-expanded" "${JDK_DIR}/lib/jrt-fs.jar"
   rm "${JDK_DIR}/lib/jrt-fs.jar"
 
   mkdir -p "${JDK_DIR}/jmods/expanded_java.base.jmod/lib/jrt-fs-expanded"
-  unzip -d "${JDK_DIR}/jmods/expanded_java.base.jmod/lib/jrt-fs-expanded" "${JDK_DIR}/jmods/expanded_java.base.jmod/lib/jrt-fs.jar" 1> /dev/null
+  unzip -qd "${JDK_DIR}/jmods/expanded_java.base.jmod/lib/jrt-fs-expanded" "${JDK_DIR}/jmods/expanded_java.base.jmod/lib/jrt-fs.jar"
   rm "${JDK_DIR}/jmods/expanded_java.base.jmod/lib/jrt-fs.jar"
 
-  echo "Expanding 'ct.sym' to workaround zip timestamp differences (https://bugs.openjdk.org/browse/JDK-8327466)"
+  echo "$(date +%T) :   Expanding lib/ct.sym to workaround zip timestamp differences (https://bugs.openjdk.org/browse/JDK-8327466)"
   mkdir "${JDK_DIR}/lib/ct-sym-expanded"
-  unzip -d "${JDK_DIR}/lib/ct-sym-expanded" "${JDK_DIR}/lib/ct.sym" 1> /dev/null
+  unzip -qd "${JDK_DIR}/lib/ct-sym-expanded" "${JDK_DIR}/lib/ct.sym"
   rm "${JDK_DIR}/lib/ct.sym"
   mkdir -p "${JDK_DIR}/jmods/expanded_jdk.compiler.jmod/lib/ct-sym-expanded"
-  unzip -d "${JDK_DIR}/jmods/expanded_jdk.compiler.jmod/lib/ct-sym-expanded" "${JDK_DIR}/jmods/expanded_jdk.compiler.jmod/lib/ct.sym" 1> /dev/null
+  unzip -qd "${JDK_DIR}/jmods/expanded_jdk.compiler.jmod/lib/ct-sym-expanded" "${JDK_DIR}/jmods/expanded_jdk.compiler.jmod/lib/ct.sym"
   rm "${JDK_DIR}/jmods/expanded_jdk.compiler.jmod/lib/ct.sym"
 
   rm -rf "${JDK_ROOT}_CP"
@@ -141,7 +142,7 @@ function removeSystemModulesHashBuilderParams() {
         done
     done
 
-  echo "Successfully removed all SystemModules jdk.jpackage hash differences from ${JDK_DIR}"
+  echo "$(date +%T) : Successfully removed all SystemModules jdk.jpackage hash differences from ${JDK_DIR}"
 }
 
 # Required for Vendor "Comparable Builds"
@@ -153,7 +154,7 @@ function removeSystemModulesHashBuilderParams() {
 #   checksum  - A checksum value of the binary
 #   reprohex  - A hex UUID to identify the binary version, again generated from binary content
 function removeWindowsNonComparableData() {
- echo "Removing EXE/DLL timestamps, CRC and debug repro hex from ${JDK_DIR}"
+ echo "$(date +%T) : Removing EXE/DLL timestamps, CRC and debug repro hex from ${JDK_DIR}"
  # We need to do this for all executables if patching VS_VERSION_INFO
  if [[ "$PATCH_VS_VERSION_INFO" = true ]]; then
     FILES=$(find "${JDK_DIR}" -type f -path '*.exe' && find "${JDK_DIR}" -type f -path '*.dll')
@@ -163,13 +164,13 @@ function removeWindowsNonComparableData() {
  for ff in $FILES
   do
     f=$(cygpath -w $ff)
-    echo "Removing EXE/DLL non-comparable timestamp, CRC, debug repro hex from $f"
+    echo "$(date +%T) : Removing EXE/DLL non-comparable timestamp, CRC, debug repro hex from $f"
 
     # Determine non-comparable data using dumpbin
     dmpfile="$ff.dumpbin.tmp"
     rm -f "$dmpfile"
     if ! dumpbin "$f" /HEADERS > "$dmpfile"; then
-        echo "  FAILED == > dumpbin \"$f\" /ALL > $dmpfile"
+        echo "$(date +%T) :  FAILED == > dumpbin \"$f\" /ALL > $dmpfile"
         exit 1
     fi
 
@@ -254,8 +255,8 @@ function processModuleInfo() {
   local JDK_DIR="$1"
   local OS="$2"
   local work_JDK="$3"
-  echo "process Module Info from ${JDK_DIR}" 
-  echo "Normalizing ModuleAttributes order in module-info.class, converting to javap"
+  echo "$(date +%T) : Process Module Info from ${JDK_DIR}" 
+  echo "$(date +%T) : Normalizing ModuleAttributes order in module-info.class, converting to javap"
   moduleAttr="ModuleResolution ModuleTarget"
   FILES=$(find "${JDK_DIR}" -type f -name "module-info.class")
   for f in $FILES
@@ -343,7 +344,7 @@ function removeSignatures() {
   if [[ "$OS" =~ CYGWIN* ]]; then
     # signtool should be on PATH
     signToolPath="signtool"
-    echo "Removing all Signatures from ${JDK_DIR}"
+    echo "$(date +%T) : Removing all signatures from exe and dll files in ${JDK_DIR}"
     FILES=$(find "${JDK_DIR}" -type f -name '*.exe' -o -name '*.dll')
     for f in $FILES
      do
@@ -384,10 +385,16 @@ function tempSign() {
   if [[ "$OS" =~ CYGWIN* ]]; then
     # signtool should be on PATH
     signToolPath="signtool"
-    echo "Adding temp Signatures for ${JDK_DIR}"
+    echo "Generating temp signatures with openssl and adding them to exe/dll files in ${JDK_DIR}"
     selfCert="test"
-    openssl req -x509 -newkey rsa:4096 -sha256 -days 3650 -nodes -keyout $selfCert.key -out $selfCert.crt -subj "/CN=example.com" -addext "subjectAltName=DNS:example.com,DNS:*.example.com,IP:10.0.0.1"
-    openssl pkcs12 -export -passout pass:test -out $selfCert.pfx -inkey $selfCert.key -in $selfCert.crt
+
+    # semgrep needs to ignore this as it objects to the password, but that
+    # is only used for generating a temporary dummy signature required for
+    # the comparison and not used for validating anything
+    # nosemgrep
+    openssl req -x509 -quiet -newkey rsa:4096 -sha256 -days 3650 -passout pass:test -keyout $selfCert.key -out $selfCert.crt -subj "/CN=example.com" -addext "subjectAltName=DNS:example.com,DNS:*.example.com,IP:10.0.0.1"
+    # nosemgrep
+    openssl pkcs12 -export -passout pass:test -passin pass:test -out $selfCert.pfx -inkey $selfCert.key -in $selfCert.crt
     FILES=$(find "${JDK_DIR}" -type f -name '*.exe' -o -name '*.dll')
     for f in $FILES
      do
@@ -416,8 +423,7 @@ function tempSign() {
 function cleanTemurinFiles() {
   local DIR="$1"
 
-  echo "Cleaning Temurin build-scripts specific files and metadata from ${DIR}"
-  echo "Removing Temurin NOTICE file from $DIR"
+  echo "$(date +%T): Cleaning Temurin NOTICE file and build-scripts specific files and metadata from ${DIR}"
   rm "${DIR}"/NOTICE
 
   if [[ $(uname) =~ Darwin* ]]; then

tooling/reproducible/repro_compare.sh

@@ -34,11 +34,11 @@ do
     exit 1
   fi
 
-  echo "Pre-processing ${JDK_DIR}"
+  echo "$(date +%T) : Pre-processing ${JDK_DIR}"
   rc=0
   source "$(dirname "$0")"/repro_process.sh "${JDK_DIR}" "${OS}" || rc=$?
   if [ $rc != 0 ]; then
-    echo "Pre-process of ${JDK_DIR} ${OS} failed"
+    echo "$(date +%T): Pre-processing of ${JDK_DIR} ${OS} failed"
     exit 1
   fi
 
@@ -59,11 +59,19 @@ files1=$(find "${JDK_DIR1}" -type f | wc -l)
 echo "Number of files: ${files1}"
 rc=0
 output="reprotest.diff"
-echo "Comparing ${JDK_DIR1} with ${JDK_DIR2} ... output to file: ${output}"
+echo "$(date +%T) : Comparing expanded JDKs from ${JDK_DIR1} with ${JDK_DIR2} ..."
 diff -r -q "${JDK_DIR1}" "${JDK_DIR2}" > "${output}" || rc=$?
+echo "$(date +%T) : diff complete - rc=$rc. Output written to file: ${output}"
 
 cat "${output}"
 
+grep "Files .*" "${output}" | while read -r line; do
+  FILE1=$(echo "$line" | awk '{print $2}')
+  FILE2=$(echo "$line" | awk '{print $4}')
+  echo "diff -c on $FILE1 and $FILE2"
+  diff -c "$FILE1" "$FILE2"
+done
+
 num_differences=$(wc -l < "${output}")
 echo "Number of differences: ${num_differences}"
 repro_pc100=$(( (files1-num_differences)*100*100/files1 ))

tooling/reproducible/repro_process.sh

@@ -38,7 +38,5 @@ fi
 
 patchManifests "${JDK_DIR}"
 
-echo "***********"
-echo " Preprocess ${JDK_DIR} SUCCESS :-)"
-echo "***********"
-
+echo "$(date +%T) : Pre-processing of ${JDK_DIR} SUCCESSFUL :-)"
+echo "" # blank line separator in log file

tooling/reproducible/windows_repro_build_compare.sh

@@ -684,10 +684,12 @@ Check_Architecture
 echo "---------------------------------------------"
 Check_VS_Versions
 echo "---------------------------------------------"
-Get_SRC_UCRT_Version
-echo "---------------------------------------------"
-Check_UCRT_Location
-echo "---------------------------------------------"
+if [[ "${buildArgs}" != *"--use-adoptium-devkit"* ]]; then
+  Get_SRC_UCRT_Version
+  echo "---------------------------------------------"
+  Check_UCRT_Location
+  echo "---------------------------------------------"
+fi
 echo "All Validation Checks Passed - Proceeding To Build"
 echo "---------------------------------------------"
 Check_And_Install_Ant