Merge branch 'adoptium:master' into modes

adoptium · Nov 29, 2024 · e912e20 · e912e20
2 parents e6dfdfc + 616d91a
commit e912e20
Show file tree

Hide file tree

Showing 15 changed files with 239 additions and 101 deletions.
diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml
@@ -42,11 +42,11 @@ jobs:
 
     steps:
     - name: Checkout repository
-      uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6
+      uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
 
     # Initializes the CodeQL tools for scanning.
     - name: Initialize CodeQL
-      uses: github/codeql-action/init@9fdb3e49720b44c48891d036bb502feb25684276 # v3.25.6
+      uses: github/codeql-action/init@f09c1c0a94de965c15400f5634aa42fac8fb8f88 # v3.27.5
       with:
         languages: ${{ matrix.language }}
         # If you wish to specify custom queries, you can do so here or in a config file.
@@ -57,7 +57,7 @@ jobs:
     # Autobuild attempts to build any compiled languages  (C/C++, C#, or Java).
     # If this step fails, then you should remove it and run the build manually (see below)
     - name: Autobuild
-      uses: github/codeql-action/autobuild@9fdb3e49720b44c48891d036bb502feb25684276 # v3.25.6
+      uses: github/codeql-action/autobuild@f09c1c0a94de965c15400f5634aa42fac8fb8f88 # v3.27.5
 
     # ℹ️ Command-line programs to run using the OS shell.
     # 📚 https://git.io/JvXDl
@@ -71,4 +71,4 @@ jobs:
     #   make release
 
     - name: Perform CodeQL Analysis
-      uses: github/codeql-action/analyze@9fdb3e49720b44c48891d036bb502feb25684276 # v3.25.6
+      uses: github/codeql-action/analyze@f09c1c0a94de965c15400f5634aa42fac8fb8f88 # v3.27.5
diff --git a/.github/workflows/runAqa.yml b/.github/workflows/runAqa.yml
@@ -19,7 +19,7 @@ jobs:
       id: workflow_run_info
     # Checkout current (TKG) repo to access the repo-specific config file `.github/workflows/runAqaConfig.json` and the shared script `scripts/testRepo/runAqaArgParse.py`
     - name: Checkout current repo
-      uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6
+      uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
       with:
         path: 'main'
     - name: Parse parameters
@@ -107,7 +107,7 @@ jobs:
         sourceType: 'buildType'
         impl: ${{ matrix.jdk_impl }}
     - name: Checkout PR Ref
-      uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6
+      uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
       if: matrix.sdk_resource == 'build-jdk'
       with:
         repository: ${{ matrix.build_repo_branch.repo }}
@@ -168,7 +168,7 @@ jobs:
          openjdk_testRepo: ${{ matrix.openjdk_testrepo }}
          openj9_repo: ${{ matrix.openj9_repo }}
          tkg_Repo: '${{ fromJSON(steps.get-pr.outputs.result).head.repo.full_name }}:${{ fromJSON(steps.get-pr.outputs.result).head.ref }}'
-    - uses: actions/upload-artifact@65462800fd760344b1a7b4382951275a0abb4808 # v4.3.3
+    - uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4.4.3
       if: failure()
       with:
         name: test_output

diff --git a/.github/workflows/testTKG.yml b/.github/workflows/testTKG.yml
@@ -15,12 +15,12 @@ jobs:
     runs-on: ubuntu-latest
     if: startsWith(github.event.comment.body, 'run tkg-test')
     steps:
-      - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
         with:
           ref: refs/pull/${{ github.event.issue.number }}/head
 
       - name: Set up JDK
-        uses: actions/setup-java@99b8673ff64fbf99d8d325f52d9a5bdedb8483e9 # v4.2.1
+        uses: actions/setup-java@8df1039502a15bceb9433410b1a100fbe190c53b # v4.5.0
         with:
           distribution: 'temurin'
           java-version: 11

diff --git a/clean.mk b/clean.mk
@@ -26,8 +26,8 @@ cleanBuild:
 	$(RM) -r $(BUILD_ROOT)
 
 clean: cleanBuild
-	$(RM) -r $(TEST_ROOT)$(D)TKG$(D)output_*
+	$(RM) -r $(Q)$(TEST_ROOT)$(Q)$(D)TKG$(D)output_*
 	$(RM) $(FAILEDTARGETS)
-	ant -f $(TEST_ROOT)$(D)TKG$(D)scripts/build_tools.xml clean
+	ant -f $(Q)$(TEST_ROOT)$(Q)$(D)TKG$(D)scripts/build_tools.xml clean
 
 .PHONY: cleanBuild clean
diff --git a/compile.mk b/compile.mk
@@ -38,7 +38,7 @@ TEST_FLAG_PARAM := -DTEST_FLAG=$(TEST_FLAG)
 else
 TEST_FLAG_PARAM :=
 endif
-COMPILE_CMD=ant -f scripts$(D)build_test.xml -DTEST_ROOT=$(TEST_ROOT) -DBUILD_ROOT=$(BUILD_ROOT) -DJDK_VERSION=$(JDK_VERSION) -DJDK_IMPL=$(JDK_IMPL) -DJDK_VENDOR=$(JDK_VENDOR) -DJCL_VERSION=$(JCL_VERSION) -DBUILD_LIST=${COMPILE_BUILD_LIST} -DRESOURCES_DIR=${RESOURCES_DIR} -DSPEC=${SPEC} -DTEST_JDK_HOME=${TEST_JDK_HOME} -DJVM_VERSION=$(JVM_VERSION) -DLIB_DIR=$(LIB_DIR) ${TEST_FLAG_PARAM}
+COMPILE_CMD=ant -f scripts$(D)build_test.xml $(Q)-DTEST_ROOT=$(TEST_ROOT)$(Q) $(Q)-DBUILD_ROOT=$(BUILD_ROOT)$(Q) $(Q)-DJDK_VERSION=$(JDK_VERSION)$(Q) $(Q)-DJDK_IMPL=$(JDK_IMPL)$(Q) $(Q)-DJDK_VENDOR=$(JDK_VENDOR)$(Q) $(Q)-DJCL_VERSION=$(JCL_VERSION)$(Q) $(Q)-DBUILD_LIST=${COMPILE_BUILD_LIST}$(Q) $(Q)-DRESOURCES_DIR=${RESOURCES_DIR}$(Q) $(Q)-DSPEC=${SPEC}$(Q) $(Q)-DTEST_JDK_HOME=${TEST_JDK_HOME}$(Q) $(Q)-DJVM_VERSION=$(JVM_VERSION)$(Q) $(Q)-DLIB_DIR=$(LIB_DIR)$(Q) ${TEST_FLAG_PARAM}
 
 
 compile:

diff --git a/makefile b/makefile
@@ -108,7 +108,7 @@ endif
 # compile tools
 #######################################
 include moveDmp.mk
-COMPILE_TOOLS_CMD=ant -f .$(D)scripts$(D)build_tools.xml -DTEST_JDK_HOME=$(TEST_JDK_HOME) -DTEST_ROOT=$(TEST_ROOT) -DLIB_DIR=$(LIB_DIR)
+COMPILE_TOOLS_CMD=ant -f .$(D)scripts$(D)build_tools.xml $(Q)-DTEST_JDK_HOME=$(TEST_JDK_HOME)$(Q) $(Q)-DTEST_ROOT=$(TEST_ROOT)$(Q) $(Q)-DLIB_DIR=$(LIB_DIR)$(Q)
 
 compileTools:
 	$(RM) -r $(COMPILATION_OUTPUT); \

diff --git a/scripts/getDependencies.pl b/scripts/getDependencies.pl
@@ -75,14 +75,14 @@
 		sha1 => 'a0f58cad836a410f6ba133aaa209aea7e54aaf8a'
 	},
 	byte_buddy => {
-		url => 'https://repo1.maven.org/maven2/net/bytebuddy/byte-buddy/1.14.12/byte-buddy-1.14.12.jar',
+		url => 'https://repo1.maven.org/maven2/net/bytebuddy/byte-buddy/1.15.4/byte-buddy-1.15.4.jar',
 		fname => 'byte-buddy.jar',
-		sha1 => '6e37f743dc15a8d7a4feb3eb0025cbc612d5b9e1'
+		sha1 => 'e8bd42992701720649765383ff570f415190b83f'
 	},
 	byte_buddy_agent => {
-		url => 'https://repo1.maven.org/maven2/net/bytebuddy/byte-buddy-agent/1.14.12/byte-buddy-agent-1.14.12.jar',
+		url => 'https://repo1.maven.org/maven2/net/bytebuddy/byte-buddy-agent/1.15.4/byte-buddy-agent-1.15.4.jar',
 		fname => 'byte-buddy-agent.jar',
-		sha1 => 'be4984cb6fd1ef1d11f218a648889dfda44b8a15'
+		sha1 => '58e850dde88f3cf20f41f659440bef33f6c4fe02'
 	 },
 	objenesis => {
 		url => 'https://repo1.maven.org/maven2/org/objenesis/objenesis/3.3/objenesis-3.3.jar',
@@ -167,6 +167,13 @@
 		shafn => 'jtreg_7_3_1_1.tar.gz.sha256sum.txt',
 		shaalg => '256'
 	},
+	jtreg_7_4_1 => {
+		url => 'https://ci.adoptium.net/job/dependency_pipeline/lastSuccessfulBuild/artifact/jtreg/jtreg-7.4+1.tar.gz',
+		fname => 'jtreg_7_4_1.tar.gz',
+		shaurl => 'https://ci.adoptium.net/job/dependency_pipeline/lastSuccessfulBuild/artifact/jtreg/jtreg-7.4+1.tar.gz.sha256sum.txt',
+		shafn => 'jtreg_7_4_1.tar.gz.sha256sum.txt',
+		shaalg => '256'
+	},
 	jython => {
 		url => 'https://repo1.maven.org/maven2/org/python/jython-standalone/2.7.2/jython-standalone-2.7.2.jar',
 		fname => 'jython-standalone.jar',
@@ -177,55 +184,91 @@
 		fname => 'jcstress-tests-all-20240222.jar',
 		sha1 => '200da75e67689e8a604ec6fe9a6f55b2c000b6ce'
 	},
+	maven => {
+		url => 'https://dlcdn.apache.org/maven/maven-3/3.9.8/binaries/apache-maven-3.9.8-bin.tar.gz',
+		fname => 'apache-maven-bin.tar.gz',
+		sha1 => '7f15c63c129f036dd5c96b1a591ed8d888f75617'
+	});
+
+my %system_jars = (
+	json_simple => {
+		url => 'https://repo1.maven.org/maven2/com/googlecode/json-simple/json-simple/1.1.1/json-simple-1.1.1.jar',
+		fname => 'json-simple.jar',
+		sha1 => 'c9ad4a0850ab676c5c64461a05ca524cdfff59f1',
+	},
+	jcstress => {
+		url => 'https://builds.shipilev.net/jcstress/jcstress-tests-all-20240222.jar',
+		fname => 'jcstress-tests-all-20240222.jar',
+		sha1 => '200da75e67689e8a604ec6fe9a6f55b2c000b6ce',
+	},
+	ant_launcher => {
+		url => 'https://repo1.maven.org/maven2/org/apache/ant/ant-launcher/1.8.1/ant-launcher-1.8.1.jar',
+		dir => 'apache-ant/lib',
+		fname => 'ant-launcher.jar',
+		is_system_test => 1
+	},
+	asm => {
+		url => 'https://repository.ow2.org/nexus/content/repositories/releases/org/ow2/asm/asm/9.0/asm-9.0.jar',
+		dir => 'asm',
+		fname => 'asm.jar',
+		is_system_test => 1
+	},
+	cvsclient => {
+		url => 'https://repo1.maven.org/maven2/org/netbeans/lib/cvsclient/20060125/cvsclient-20060125.jar',
+		dir => 'cvsclient',
+		fname => 'org-netbeans-lib-cvsclient.jar',
+		is_system_test => 1
+	},
 	hamcrest_core => {
 		url => 'https://repo1.maven.org/maven2/org/hamcrest/hamcrest-core/1.3/hamcrest-core-1.3.jar',
+		dir => 'junit',
 		fname => 'hamcrest-core.jar',
-		sha1 => '42a25dc3219429f0e5d060061f71acb49bf010a0'
+		is_system_test => 1
 	},
-	bcprov_jdk18on => {
-		url => 'https://repo1.maven.org/maven2/org/bouncycastle/bcprov-jdk18on/1.78.1/bcprov-jdk18on-1.78.1.jar',
-		fname => 'bcprov-jdk18on.jar',
-		sha1 => '39e9e45359e20998eb79c1828751f94a818d25f8'
+	junit => {
+		url => 'https://repo1.maven.org/maven2/junit/junit/4.12/junit-4.12.jar',
+		dir => 'junit',
+		fname => 'junit.jar',
+		is_system_test => 1
 	},
-	junit_vintage_engine => {
-		url => 'https://repo1.maven.org/maven2/org/junit/vintage/junit-vintage-engine/5.10.2/junit-vintage-engine-5.10.2.jar',
-		fname => 'junit-vintage-engine.jar',
-		sha1 => '2905387f99f86a6618d1f7c005e7a5946224f317'
+	log4j_api => {
+		url => 'https://repo1.maven.org/maven2/org/apache/logging/log4j/log4j-api/2.15.0/log4j-api-2.15.0.jar',
+		dir => 'log4j',
+		fname => 'log4j-api.jar',
+		is_system_test => 1
 	},
-	junit_platform_suite => {
-		url => 'https://repo1.maven.org/maven2/org/junit/platform/junit-platform-suite/1.10.1/junit-platform-suite-1.10.1.jar',
-		fname => 'junit-platform-suite.jar',
-		sha1 => 'a219dbd79ec2b1fc61b806554fcf4eb5c17a6d1d'
+	log4j_core => {
+		url => 'https://repo1.maven.org/maven2/org/apache/logging/log4j/log4j-core/2.15.0/log4j-core-2.15.0.jar',
+		dir => 'log4j',
+		fname => 'log4j-core.jar',
+		is_system_test => 1
 	},
-	junit_jupiter_api => {
-		url => 'https://repo1.maven.org/maven2/org/junit/jupiter/junit-jupiter-api/5.10.2/junit-jupiter-api-5.10.2.jar',
-		fname => 'junit-jupiter-api.jar',
-		sha1 => 'fb55d6e2bce173f35fd28422e7975539621055ef'
+	mauve => {
+		url => 'https://ci.adoptium.net/job/systemtest.getDependency/lastSuccessfulBuild/artifact/systemtest_prereqs/mauve/mauve.jar',
+		dir => 'mauve',
+		fname => 'mauve.jar',
+		is_system_test => 1
 	},
-	junit_jupiter_engine => {
-		url => 'https://repo1.maven.org/maven2/org/junit/jupiter/junit-jupiter-engine/5.10.2/junit-jupiter-engine-5.10.2.jar',
-		fname => 'junit-jupiter-engine.jar',
-		sha1 => 'f1f8fe97bd58e85569205f071274d459c2c4f8cd'
-	},
-	junit_jupiter_params => {
-		url => 'https://repo1.maven.org/maven2/org/junit/jupiter/junit-jupiter-params/5.10.2/junit-jupiter-params-5.10.2.jar',
-		fname => 'junit-jupiter-params.jar',
-		sha1 => '359132c82a9d3fa87a325db6edd33b5fdc67a3d8'
-	},
-	junit_platform_suite_api => {
-		url => 'https://repo1.maven.org/maven2/org/junit/platform/junit-platform-suite-api/1.10.2/junit-platform-suite-api-1.10.2.jar',
-		fname => 'junit-platform-suite-api.jar',
-		sha1 => '174bba1574c37352b0eb2c06e02b6403738ad57c'
+	tools => {
+		url => 'https://ci.adoptium.net/job/systemtest.getDependency/lastSuccessfulBuild/artifact/systemtest_prereqs/tools/tools.jar',
+		dir => 'tools',
+		fname => 'tools.jar',
+		is_system_test => 1
 	});
 
+my %jars_to_use;
+if ($path =~ /system_lib/) {
+	%jars_to_use = %system_jars;
+} else {
+	%jars_to_use = %base;
+}
 my @dependencies = split(',', $dependencyList);
-
 # Put all dependent jars hash to array to prepare downloading
 my @jars_info;
-foreach my $dependency (keys %base) {
+foreach my $dependency (keys %jars_to_use) {
 	foreach my $i (@dependencies) {
 		if ($i eq "all" || $dependency eq $i) {
-			push(@jars_info, $base{$dependency});
+			push(@jars_info, $jars_to_use{$dependency});
 		}
 	}
 }
@@ -242,15 +285,33 @@
 	for my $i (0 .. $#jars_info) {
 		my $url = $jars_info[$i]{url};
 		my $fn = $jars_info[$i]{fname};
-		my $filename = $path . $sep . $fn;
+		my $sha1 = $jars_info[$i]{sha1};
+		my $dir = $jars_info[$i]{dir} // "";
+		my $full_dir_path = File::Spec->catdir($path, $dir);
+		my $url_custom = $customUrl;
+
+		if (!-d $full_dir_path) {
+			make_path($full_dir_path, {chmod => 0755, verbose => 1}) or die "Failed to create directory: $full_dir_path: $!";
+			print "Directory created: $full_dir_path\n";
+		}
+
+		my $filename = File::Spec->catfile($full_dir_path, $fn);
 		my $shaurl = $jars_info[$i]{shaurl};
 		my $shafn = $jars_info[$i]{shafn};
 
-		# if customUrl is provided, use customUrl and reset $url and $shaurl
-		if ($customUrl ne "") {
-			$url = "$customUrl/$fn";
+		# if url_custom is provided, use url_custom and reset $url and $shaurl
+		if ($url_custom ne "") {
+			if (defined $jars_info[$i]{is_system_test} && $jars_info[$i]{is_system_test} == 1) {
+				$url_custom =~ s/test.getDependency/systemtest.getDependency/;
+				$url_custom .= "systemtest_prereqs/";
+				$url_custom .= $jars_info[$i]{dir};
+				$url_custom .= '/' unless $url_custom =~ /\/$/;
+			}
+
+			$url = "$url_custom/$jars_info[$i]{fname}";
+
 			if (defined $shaurl && $shaurl ne '') {
-				$shaurl = "$customUrl/$shafn";
+				$shaurl = "$url_custom/$shafn";
 			}
 		}
 
@@ -268,10 +329,12 @@
 
 		my $expectedsha = $jars_info[$i]{sha1};
 		if (!$expectedsha) {
-			$shafn = $path . $sep . $shafn;
-			# if the sha file exists, parse the file and get the expected sha
-			if (-e $shafn) {
-				$expectedsha = getShaFromFile($shafn, $fn);
+			if (defined $shafn && $shafn ne '') {
+				$shafn = $path . $sep . $shafn;
+				# if the sha file exists, parse the file and get the expected sha
+				if (-e $shafn) {
+					$expectedsha = getShaFromFile($shafn, $fn);
+				}
 			}
 
 			# if expectedsha is not set above and shaurl is provided, download the sha file
@@ -287,30 +350,40 @@
 			next;
 		}
 
+		my $ignoreChecksum = (!defined $sha1 || $sha1 eq '') && (!defined $shaurl || $shaurl eq '');
 		# download the dependent third party jar
-		downloadFile($url, $filename);
 
-		# if shaurl is provided, re-download the sha file and reset the expectedsha value
-		# as the dependent third party jar is newly downloadeded
-		if ($shaurl) {
-			downloadFile($shaurl, $shafn);
-			$expectedsha = getShaFromFile($shafn, $fn);
-		}
+		if ($ignoreChecksum && -e $filename) {
+			print "$filename exists, not downloading.\n";
+		} else {
+			downloadFile($url, $filename);
 
-		if (!$expectedsha) {
-			die "ERROR: cannot get the expected sha for file $fn.\n";
-		}
+			# if shaurl is provided, re-download the sha file and reset the expectedsha value
+			# as the dependent third party jar is newly downloadeded
+			if (!$ignoreChecksum) {
+				if ($shaurl) {
+					downloadFile($shaurl, $shafn);
+					$expectedsha = getShaFromFile($shafn, $fn);
+				}
 
-		# validate dependencies sha sum
-		$sha = Digest::SHA->new($shaalg);
-		$sha->addfile($filename);
-		$digest = $sha->hexdigest;
+				if (!$expectedsha) {
+					die "ERROR: cannot get the expected sha for file $fn.\n";
+				}
 
-		if ($digest ne $expectedsha) {
-			print "Expected sha is: $expectedsha,\n";
-			print "Actual sha is  : $digest.\n";
-			print "Please delete $filename and rerun the program!";
-			die "ERROR: sha checksum error.\n";
+				# validate dependencies sha sum
+				$sha = Digest::SHA->new($shaalg);
+				$sha->addfile($filename);
+				$digest = $sha->hexdigest;
+
+				if ($digest ne $expectedsha) {
+					print "Expected sha is: $expectedsha,\n";
+					print "Actual sha is  : $digest.\n";
+					print "Please delete $filename and rerun the program!";
+					die "ERROR: sha checksum error.\n";
+				}
+			} else {
+				print "Checksum verification skipped for $filename\n";
+			}
 		}
 	}
 	print "downloaded dependent third party jars successfully\n";
@@ -346,6 +419,8 @@ sub downloadFile {
 	# note _ENCODE_FILE_NEW flag is set for zos
 	if ('.txt' eq substr $filename, -length('.txt')) {
 		$output = qx{_ENCODE_FILE_NEW=ISO8859-1 curl $curlOpts -k -o $filename $url 2>&1};
+	} elsif ('.jar' eq substr $filename, -length('.jar')) {
+		$output = qx{_ENCODE_FILE_NEW=BINARY curl $curlOpts -k -o $filename $url 2>&1};
 	} else {
 		$output = qx{_ENCODE_FILE_NEW=UNTAGGED curl $curlOpts -k -o $filename $url 2>&1};
 	}