From 1103c703eeadffd50de8101c8f34e7cea268b51e Mon Sep 17 00:00:00 2001 From: Anna Babu Palathingal <148897727+annaibm@users.noreply.github.com> Date: Fri, 24 May 2024 16:36:30 -0400 Subject: [PATCH 1/3] Added system test prestage jars - Enables to avoid system test prerequisite zip file being empty. related: adoptium/aqa-tests#4912 Signed-off-by: Anna Babu Palathingal --- scripts/getDependencies.pl | 139 +++++++++++++++++++++++++++++-------- 1 file changed, 111 insertions(+), 28 deletions(-) diff --git a/scripts/getDependencies.pl b/scripts/getDependencies.pl index 3edaa248..557acd77 100644 --- a/scripts/getDependencies.pl +++ b/scripts/getDependencies.pl @@ -218,14 +218,75 @@ sha1 => '174bba1574c37352b0eb2c06e02b6403738ad57c' }); -my @dependencies = split(',', $dependencyList); +my %system_jars = ( + ant_launcher => { + url => 'https://repo1.maven.org/maven2/org/apache/ant/ant-launcher/1.8.1/ant-launcher-1.8.1.jar', + dir => 'apache-ant/lib', + fname => 'ant-launcher.jar', + is_system_test => 1 + }, + asm => { + url => 'https://repository.ow2.org/nexus/content/repositories/releases/org/ow2/asm/asm/9.0/asm-9.0.jar', + dir => 'asm', + fname => 'asm.jar', + is_system_test => 1 + }, + cvsclient => { + url => 'https://repo1.maven.org/maven2/org/netbeans/lib/cvsclient/20060125/cvsclient-20060125.jar', + dir => 'cvsclient', + fname => 'org-netbeans-lib-cvsclient.jar', + is_system_test => 1 + }, + hamcrest_core => { + url => 'https://repo1.maven.org/maven2/org/hamcrest/hamcrest-core/1.3/hamcrest-core-1.3.jar', + dir => 'junit', + fname => 'hamcrest-core.jar', + is_system_test => 1 + }, + junit => { + url => 'https://repo1.maven.org/maven2/junit/junit/4.12/junit-4.12.jar', + dir => 'junit', + fname => 'junit.jar', + is_system_test => 1 + }, + log4j_api => { + url => 'https://repo1.maven.org/maven2/org/apache/logging/log4j/log4j-api/2.15.0/log4j-api-2.15.0.jar', + dir => 'log4j', + fname => 'log4j-api.jar', + is_system_test => 1 + }, + log4j_core => { + url => 'https://repo1.maven.org/maven2/org/apache/logging/log4j/log4j-core/2.15.0/log4j-core-2.15.0.jar', + dir => 'log4j', + fname => 'log4j-core.jar', + is_system_test => 1 + }, + mauve => { + url => 'https://ci.adoptium.net/job/systemtest.getDependency/lastSuccessfulBuild/artifact/systemtest_prereqs/mauve/mauve.jar', + dir => 'mauve', + fname => 'mauve.jar', + is_system_test => 1 + }, + tools => { + url => 'https://ci.adoptium.net/job/systemtest.getDependency/lastSuccessfulBuild/artifact/systemtest_prereqs/tools/tools.jar', + dir => 'tools', + fname => 'tools.jar', + is_system_test => 1 + }); +my %jars_to_use; +if ($path =~ /system_lib/) { + %jars_to_use = (%base, %system_jars); +} else { + %jars_to_use = %base; +} +my @dependencies = split(',', $dependencyList); # Put all dependent jars hash to array to prepare downloading my @jars_info; -foreach my $dependency (keys %base) { +foreach my $dependency (keys %jars_to_use) { foreach my $i (@dependencies) { if ($i eq "all" || $dependency eq $i) { - push(@jars_info, $base{$dependency}); + push(@jars_info, $jars_to_use{$dependency}); } } } @@ -242,15 +303,30 @@ for my $i (0 .. $#jars_info) { my $url = $jars_info[$i]{url}; my $fn = $jars_info[$i]{fname}; - my $filename = $path . $sep . $fn; + my $sha1 = $jars_info[$i]{sha1}; + my $dir = $jars_info[$i]{dir} // ""; + my $full_dir_path = File::Spec->catdir($path, $dir); + my $url_custom = $customUrl; + + if (!-d $full_dir_path) { + make_path($full_dir_path, {chmod => 0755, verbose => 1}) or die "Failed to create directory: $full_dir_path: $!"; + print "Directory created: $full_dir_path\n"; + } + + my $filename = File::Spec->catfile($full_dir_path, $fn); my $shaurl = $jars_info[$i]{shaurl}; my $shafn = $jars_info[$i]{shafn}; - # if customUrl is provided, use customUrl and reset $url and $shaurl - if ($customUrl ne "") { - $url = "$customUrl/$fn"; + # if url_custom is provided, use url_custom and reset $url and $shaurl + if ($url_custom ne "") { + if ($jars_info[$i]{is_system_test}) { + $url_custom =~ s/test.getDependency/systemtest.getDependency/; + $url_custom .= "systemtest_prereqs/"; + $url_custom .= $jars_info[$i]{dir}; + } + $url = "$url_custom/$fn"; if (defined $shaurl && $shaurl ne '') { - $shaurl = "$customUrl/$shafn"; + $shaurl = "$url_custom/$shafn"; } } @@ -268,10 +344,12 @@ my $expectedsha = $jars_info[$i]{sha1}; if (!$expectedsha) { - $shafn = $path . $sep . $shafn; - # if the sha file exists, parse the file and get the expected sha - if (-e $shafn) { - $expectedsha = getShaFromFile($shafn, $fn); + if (defined $shafn && $shafn ne '') { + $shafn = $path . $sep . $shafn; + # if the sha file exists, parse the file and get the expected sha + if (-e $shafn) { + $expectedsha = getShaFromFile($shafn, $fn); + } } # if expectedsha is not set above and shaurl is provided, download the sha file @@ -287,30 +365,35 @@ next; } + my $ignoreChecksum = (!defined $sha1 || $sha1 eq '') && (!defined $shaurl || $shaurl eq ''); # download the dependent third party jar downloadFile($url, $filename); # if shaurl is provided, re-download the sha file and reset the expectedsha value # as the dependent third party jar is newly downloadeded - if ($shaurl) { - downloadFile($shaurl, $shafn); - $expectedsha = getShaFromFile($shafn, $fn); - } + if (!$ignoreChecksum) { + if ($shaurl) { + downloadFile($shaurl, $shafn); + $expectedsha = getShaFromFile($shafn, $fn); + } - if (!$expectedsha) { - die "ERROR: cannot get the expected sha for file $fn.\n"; - } + if (!$expectedsha) { + die "ERROR: cannot get the expected sha for file $fn.\n"; + } - # validate dependencies sha sum - $sha = Digest::SHA->new($shaalg); - $sha->addfile($filename); - $digest = $sha->hexdigest; + # validate dependencies sha sum + $sha = Digest::SHA->new($shaalg); + $sha->addfile($filename); + $digest = $sha->hexdigest; - if ($digest ne $expectedsha) { - print "Expected sha is: $expectedsha,\n"; - print "Actual sha is : $digest.\n"; - print "Please delete $filename and rerun the program!"; - die "ERROR: sha checksum error.\n"; + if ($digest ne $expectedsha) { + print "Expected sha is: $expectedsha,\n"; + print "Actual sha is : $digest.\n"; + print "Please delete $filename and rerun the program!"; + die "ERROR: sha checksum error.\n"; + } + } else { + print "Checksum verification skipped for $filename\n"; } } print "downloaded dependent third party jars successfully\n"; From 2faba9115e88cb3ab89b31c2a881bc746ba32b8a Mon Sep 17 00:00:00 2001 From: Anna Babu Palathingal <148897727+annaibm@users.noreply.github.com> Date: Tue, 28 May 2024 16:16:41 -0400 Subject: [PATCH 2/3] Add follow redirect in querying - Added `-L` to support http TRSS URL. resolves: https://github.com/adoptium/TKG/issues/555 Signed-off-by: Anna Babu Palathingal --- src/org/testKitGen/TestDivider.java | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/src/org/testKitGen/TestDivider.java b/src/org/testKitGen/TestDivider.java index 40b32bbc..8d503d7f 100644 --- a/src/org/testKitGen/TestDivider.java +++ b/src/org/testKitGen/TestDivider.java @@ -73,13 +73,13 @@ private void divideOnTestTime(List> parallelLists, List te if (testDuration < limitFactor) { Map.Entry entry = new AbstractMap.SimpleEntry<>(index, testDuration); machineQueue.offer(entry); - } else { + } else { /* If the test time is greater than the limiting factor, set it as the new limiting factor. */ limitFactor = testDuration; System.out.println("Warning: Test " + testName + " has duration " + formatTime(testDuration) + ", which is greater than the specified test list execution time " + testTime + "m. So this value is used to limit the overall execution time."); } index++; - + } } } @@ -107,19 +107,19 @@ private void divideOnMachineNum(List> parallelLists, List } private String constructURL(String impl, String plat, String group, String level) { - int limit = 10; // limit the number of builds used to calculate the average duration + int limit = 10; // limit the number of builds used to calculate the average duration String URL = (arg.getTRSSURL().isEmpty() ? Constants.TRSS_URL : arg.getTRSSURL()) + "/api/getTestAvgDuration?limit=" + limit + "&jdkVersion=" + arg.getJdkVersion() + "&impl=" + impl + "&platform=" + plat; if (tt.isSingleTest()) { URL += "&testName=" + tt.getTestTargetName(); } else if (tt.isCategory()) { if (!group.equals("")) { - URL += "&group=" + group; + URL += "&group=" + group; } if (!level.equals("")) { - URL += "&level=" + level; + URL += "&level=" + level; } - } + } return URL; } @@ -226,7 +226,7 @@ private Map getDataFromTRSS() { String level = getLevel(); Map map = new HashMap(); String URL = constructURL(impl, plat, group, level); - String command = "curl --silent --max-time 120 " + URL; + String command = "curl --silent --max-time 120 -L " + URL; System.out.println("Attempting to get test duration data from TRSS."); System.out.println(command); Process process; @@ -241,6 +241,7 @@ private Map getDataFromTRSS() { parseDuration(responseReader, map); } catch (IOException | ParseException e) { System.out.println("Warning: cannot parse data from TRSS."); + e.printStackTrace(); } return map; } @@ -294,7 +295,7 @@ private Queue> createDurationQueue() { } } - System.out.println("\nTEST DURATION"); + System.out.println("\nTEST DURATION"); System.out.println("===================================================================================="); System.out.println("Total number of tests searched: " + numOfTests); int foundNum = numOfTests - testsNotFound.size() - testsInvalid.size(); From b57b5183fb640180c4dcc28d9947605970298db8 Mon Sep 17 00:00:00 2001 From: Lan Xia Date: Thu, 30 May 2024 20:41:59 -0400 Subject: [PATCH 3/3] Revert "Added system test pre-stage jars" --- scripts/getDependencies.pl | 139 ++++++++----------------------------- 1 file changed, 28 insertions(+), 111 deletions(-) diff --git a/scripts/getDependencies.pl b/scripts/getDependencies.pl index 557acd77..3edaa248 100644 --- a/scripts/getDependencies.pl +++ b/scripts/getDependencies.pl @@ -218,75 +218,14 @@ sha1 => '174bba1574c37352b0eb2c06e02b6403738ad57c' }); -my %system_jars = ( - ant_launcher => { - url => 'https://repo1.maven.org/maven2/org/apache/ant/ant-launcher/1.8.1/ant-launcher-1.8.1.jar', - dir => 'apache-ant/lib', - fname => 'ant-launcher.jar', - is_system_test => 1 - }, - asm => { - url => 'https://repository.ow2.org/nexus/content/repositories/releases/org/ow2/asm/asm/9.0/asm-9.0.jar', - dir => 'asm', - fname => 'asm.jar', - is_system_test => 1 - }, - cvsclient => { - url => 'https://repo1.maven.org/maven2/org/netbeans/lib/cvsclient/20060125/cvsclient-20060125.jar', - dir => 'cvsclient', - fname => 'org-netbeans-lib-cvsclient.jar', - is_system_test => 1 - }, - hamcrest_core => { - url => 'https://repo1.maven.org/maven2/org/hamcrest/hamcrest-core/1.3/hamcrest-core-1.3.jar', - dir => 'junit', - fname => 'hamcrest-core.jar', - is_system_test => 1 - }, - junit => { - url => 'https://repo1.maven.org/maven2/junit/junit/4.12/junit-4.12.jar', - dir => 'junit', - fname => 'junit.jar', - is_system_test => 1 - }, - log4j_api => { - url => 'https://repo1.maven.org/maven2/org/apache/logging/log4j/log4j-api/2.15.0/log4j-api-2.15.0.jar', - dir => 'log4j', - fname => 'log4j-api.jar', - is_system_test => 1 - }, - log4j_core => { - url => 'https://repo1.maven.org/maven2/org/apache/logging/log4j/log4j-core/2.15.0/log4j-core-2.15.0.jar', - dir => 'log4j', - fname => 'log4j-core.jar', - is_system_test => 1 - }, - mauve => { - url => 'https://ci.adoptium.net/job/systemtest.getDependency/lastSuccessfulBuild/artifact/systemtest_prereqs/mauve/mauve.jar', - dir => 'mauve', - fname => 'mauve.jar', - is_system_test => 1 - }, - tools => { - url => 'https://ci.adoptium.net/job/systemtest.getDependency/lastSuccessfulBuild/artifact/systemtest_prereqs/tools/tools.jar', - dir => 'tools', - fname => 'tools.jar', - is_system_test => 1 - }); - -my %jars_to_use; -if ($path =~ /system_lib/) { - %jars_to_use = (%base, %system_jars); -} else { - %jars_to_use = %base; -} my @dependencies = split(',', $dependencyList); + # Put all dependent jars hash to array to prepare downloading my @jars_info; -foreach my $dependency (keys %jars_to_use) { +foreach my $dependency (keys %base) { foreach my $i (@dependencies) { if ($i eq "all" || $dependency eq $i) { - push(@jars_info, $jars_to_use{$dependency}); + push(@jars_info, $base{$dependency}); } } } @@ -303,30 +242,15 @@ for my $i (0 .. $#jars_info) { my $url = $jars_info[$i]{url}; my $fn = $jars_info[$i]{fname}; - my $sha1 = $jars_info[$i]{sha1}; - my $dir = $jars_info[$i]{dir} // ""; - my $full_dir_path = File::Spec->catdir($path, $dir); - my $url_custom = $customUrl; - - if (!-d $full_dir_path) { - make_path($full_dir_path, {chmod => 0755, verbose => 1}) or die "Failed to create directory: $full_dir_path: $!"; - print "Directory created: $full_dir_path\n"; - } - - my $filename = File::Spec->catfile($full_dir_path, $fn); + my $filename = $path . $sep . $fn; my $shaurl = $jars_info[$i]{shaurl}; my $shafn = $jars_info[$i]{shafn}; - # if url_custom is provided, use url_custom and reset $url and $shaurl - if ($url_custom ne "") { - if ($jars_info[$i]{is_system_test}) { - $url_custom =~ s/test.getDependency/systemtest.getDependency/; - $url_custom .= "systemtest_prereqs/"; - $url_custom .= $jars_info[$i]{dir}; - } - $url = "$url_custom/$fn"; + # if customUrl is provided, use customUrl and reset $url and $shaurl + if ($customUrl ne "") { + $url = "$customUrl/$fn"; if (defined $shaurl && $shaurl ne '') { - $shaurl = "$url_custom/$shafn"; + $shaurl = "$customUrl/$shafn"; } } @@ -344,12 +268,10 @@ my $expectedsha = $jars_info[$i]{sha1}; if (!$expectedsha) { - if (defined $shafn && $shafn ne '') { - $shafn = $path . $sep . $shafn; - # if the sha file exists, parse the file and get the expected sha - if (-e $shafn) { - $expectedsha = getShaFromFile($shafn, $fn); - } + $shafn = $path . $sep . $shafn; + # if the sha file exists, parse the file and get the expected sha + if (-e $shafn) { + $expectedsha = getShaFromFile($shafn, $fn); } # if expectedsha is not set above and shaurl is provided, download the sha file @@ -365,35 +287,30 @@ next; } - my $ignoreChecksum = (!defined $sha1 || $sha1 eq '') && (!defined $shaurl || $shaurl eq ''); # download the dependent third party jar downloadFile($url, $filename); # if shaurl is provided, re-download the sha file and reset the expectedsha value # as the dependent third party jar is newly downloadeded - if (!$ignoreChecksum) { - if ($shaurl) { - downloadFile($shaurl, $shafn); - $expectedsha = getShaFromFile($shafn, $fn); - } + if ($shaurl) { + downloadFile($shaurl, $shafn); + $expectedsha = getShaFromFile($shafn, $fn); + } - if (!$expectedsha) { - die "ERROR: cannot get the expected sha for file $fn.\n"; - } + if (!$expectedsha) { + die "ERROR: cannot get the expected sha for file $fn.\n"; + } - # validate dependencies sha sum - $sha = Digest::SHA->new($shaalg); - $sha->addfile($filename); - $digest = $sha->hexdigest; + # validate dependencies sha sum + $sha = Digest::SHA->new($shaalg); + $sha->addfile($filename); + $digest = $sha->hexdigest; - if ($digest ne $expectedsha) { - print "Expected sha is: $expectedsha,\n"; - print "Actual sha is : $digest.\n"; - print "Please delete $filename and rerun the program!"; - die "ERROR: sha checksum error.\n"; - } - } else { - print "Checksum verification skipped for $filename\n"; + if ($digest ne $expectedsha) { + print "Expected sha is: $expectedsha,\n"; + print "Actual sha is : $digest.\n"; + print "Please delete $filename and rerun the program!"; + die "ERROR: sha checksum error.\n"; } } print "downloaded dependent third party jars successfully\n";