diff --git a/api-generation/build.gradle b/api-generation/build.gradle index 713c24c289..9219e00463 100644 --- a/api-generation/build.gradle +++ b/api-generation/build.gradle @@ -15,6 +15,9 @@ dependencies { shadowJar { mergeServiceFiles() + manifest { + attributes 'Add-Opens': openJavaModules.join(" ") + } } build.dependsOn shadowJar diff --git a/assembly-extensions/build.gradle b/assembly-extensions/build.gradle index 2fbb9e51b0..a657fcd824 100644 --- a/assembly-extensions/build.gradle +++ b/assembly-extensions/build.gradle @@ -18,6 +18,9 @@ shadowJar { mergeServiceFiles() archiveBaseName = "${archiveBaseName.get()}_${scalaBaseVersion}" + manifest { + attributes 'Add-Opens': openJavaModules.join(" ") + } } artifacts { diff --git a/assembly-scoring/build.gradle b/assembly-scoring/build.gradle index fd967cf84a..f6477c1f66 100644 --- a/assembly-scoring/build.gradle +++ b/assembly-scoring/build.gradle @@ -17,6 +17,9 @@ jar { shadowJar { mergeServiceFiles() archiveBaseName = "${archiveBaseName.get()}_${scalaBaseVersion}" + manifest { + attributes 'Add-Opens': openJavaModules.join(" ") + } } artifacts { diff --git a/assembly/build.gradle b/assembly/build.gradle index 235bd86bb0..cda1995c39 100644 --- a/assembly/build.gradle +++ b/assembly/build.gradle @@ -66,6 +66,10 @@ shadowJar { from "$project.buildDir/reports/" include '**/*' exclude 'www/flow/packs/test-*/**' + manifest { + attributes 'Add-Opens': openJavaModules.join(" ") + } + archiveBaseName = "${archiveBaseName.get()}_${scalaBaseVersion}" } diff --git a/build.gradle b/build.gradle index daecd2b003..a2bea56724 100644 --- a/build.gradle +++ b/build.gradle @@ -199,6 +199,12 @@ configure(subprojects) { project -> // All subprojects needs Spark support apply from: "$rootDir/gradle/spark.gradle" + // All tests need Java module system run options + apply from: "$rootDir/gradle/javaModuleSystem.gradle" + tasks.withType(Test).configureEach { + jvmArgs += project.defaultJavaModuleRunOptions + } + if (project in docProjects) { apply plugin: 'java' apply plugin: 'kr.motd.sphinx' diff --git a/ci/Jenkinsfile-kubernetes b/ci/Jenkinsfile-kubernetes index 56dcd1f4ad..431823995d 100644 --- a/ci/Jenkinsfile-kubernetes +++ b/ci/Jenkinsfile-kubernetes @@ -119,7 +119,7 @@ def getBuildAndTestStagesForSparkVersion(commons, props, sparkMajorVersion) { def sparklingVersion = getSparklingVersion(props, sparkMajorVersion) def master = "k8s://" + getMinikubeMaster() - // Spark home within sparkling_water_tests image the code runs currently in, for a current sparkVersion + // Spark home within sparkling_water_tests_jdk17 image the code runs currently in, for a current sparkVersion def currentSparkHome = "/home/jenkins/spark-${sparkVersion}-bin" // Spark home within a built sparkling-water:scala/python/r/external-backend-... image @@ -652,7 +652,7 @@ def testPythonExternalBackendAutoClientMode(master, version, swImageSparkHome) { def withSparklingWaterMinikubeImage(commons, code) { def repoUrl = commons.getAWSDockerRepo() commons.withAWSDocker { - def image = "${repoUrl}/opsh2oai/sparkling_water_tests:" + commons.getDockerImageVersion() + def image = "${repoUrl}/opsh2oai/sparkling_water_tests_jdk17:" + commons.getDockerImageVersion() def dockerOptions = "--init --privileged --network=host -v /var/run/docker.sock:/var/run/docker.sock" groovy.lang.Closure initCode = { sh "activate_java_8" diff --git a/ci/build.gradle b/ci/build.gradle index 4d2b2eb12a..7f42ef167f 100644 --- a/ci/build.gradle +++ b/ci/build.gradle @@ -21,6 +21,12 @@ task createDockerfile(type: Dockerfile, dependsOn: copyFiles) { destFile = outputFile from testingBaseImage + // switching to a newer Java version as the base image uses ancient 8u171 + // we're copying JDK from another docker image + copyFile(new Dockerfile.CopyFile(testingJdkImageJavaPath, testingJdkImageJavaPath).withStage(testingJdkImage)) + runCommand("rm -rf /usr/lib/jvm/java-8-oracle") + environmentVariable('JAVA_HOME', testingJdkImageJavaPath) + runCommand "rm /etc/startup/70_start_slapd" // Install Terraform @@ -29,8 +35,7 @@ task createDockerfile(type: Dockerfile, dependsOn: copyFiles) { | unzip terraform.zip -d /usr/local/bin/ && \\ | rm -f terraform.zip """.stripMargin() - - environmentVariable("PATH","/usr/lib/jvm/java-8-oracle/bin:/usr/local/nvidia/bin:/usr/local/cuda/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin") + environmentVariable("PATH","${testingJdkImageJavaPath}/bin:/usr/local/nvidia/bin:/usr/local/cuda/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin") // Update to R 4.2 runCommand """\\ @@ -99,8 +104,7 @@ task createDockerfile(type: Dockerfile, dependsOn: copyFiles) { rm /home/jenkins/miniconda.sh """ - environmentVariable("PATH", "/home/jenkins/miniconda/bin:/usr/lib/jvm/java-8-oracle/bin:/usr/local/nvidia/bin:/usr/local/cuda/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin") - environmentVariable("JAVA_HOME", "/usr/lib/jvm/java-8-oracle") + environmentVariable("PATH", "/home/jenkins/miniconda/bin:\$PATH") runCommand """\\ conda install anaconda-client conda-build conda-verify -y && \\ diff --git a/ci/commons.groovy b/ci/commons.groovy index ca2171e04b..6e4bf7e1f8 100644 --- a/ci/commons.groovy +++ b/ci/commons.groovy @@ -63,7 +63,7 @@ def withDocker(image, groovy.lang.Closure code, String dockerOptions = "", groov def withSparklingWaterDockerImage(code) { def repoUrl = getAWSDockerRepo() withAWSDocker { - def image = "${repoUrl}/opsh2oai/sparkling_water_tests:" + getDockerImageVersion() + def image = "${repoUrl}/opsh2oai/sparkling_water_tests_jdk17:" + getDockerImageVersion() def dockerOptions = "--init --privileged" groovy.lang.Closure initCode = { sh "activate_java_8" diff --git a/ci/docker/Jenkinsfile-build-docker b/ci/docker/Jenkinsfile-build-docker index fe0e529ec3..1757069299 100644 --- a/ci/docker/Jenkinsfile-build-docker +++ b/ci/docker/Jenkinsfile-build-docker @@ -35,7 +35,7 @@ node('mr-0xc10') { dir("ci/build/docker") { sh """ docker pull $testingBaseImage - docker build --no-cache -t harbor.h2o.ai/opsh2oai/sparkling_water_tests:${nextVersion} -f Dockerfile . + docker build --no-cache -t harbor.h2o.ai/opsh2oai/sparkling_water_tests_jdk17:${nextVersion} -f Dockerfile . """ } } @@ -44,8 +44,8 @@ node('mr-0xc10') { def repoUrl = commons.getAWSDockerRepo() commons.withAWSDocker { sh """ - docker tag harbor.h2o.ai/opsh2oai/sparkling_water_tests:${nextVersion} $repoUrl/opsh2oai/sparkling_water_tests:${nextVersion} - docker push $repoUrl/opsh2oai/sparkling_water_tests:${nextVersion} + docker tag harbor.h2o.ai/opsh2oai/sparkling_water_tests_jdk17:${nextVersion} $repoUrl/opsh2oai/sparkling_water_tests_jdk17:${nextVersion} + docker push $repoUrl/opsh2oai/sparkling_water_tests_jdk17:${nextVersion} """ } } diff --git a/gradle.properties b/gradle.properties index 60ea292169..4811660909 100644 --- a/gradle.properties +++ b/gradle.properties @@ -37,3 +37,5 @@ spotlessModern=true testH2OBranch=rel-3.42.0 makeBooklet=false testingBaseImage=harbor.h2o.ai/opsh2oai/h2o-3-hadoop-cdh-6.3:84 +testingJdkImage=eclipse-temurin:17.0.8.1_1-jdk-alpine +testingJdkImageJavaPath=/opt/java/openjdk diff --git a/gradle/javaModuleSystem.gradle b/gradle/javaModuleSystem.gradle new file mode 100644 index 0000000000..4bf928c633 --- /dev/null +++ b/gradle/javaModuleSystem.gradle @@ -0,0 +1,23 @@ +String jdk8CompatibilityOption = "-XX:+IgnoreUnrecognizedVMOptions" +List openJavaModules = [ + "java.base/java.lang", + "java.base/java.lang.invoke", + "java.base/java.lang.reflect", + "java.base/java.io", + "java.base/java.net", + "java.base/java.nio", + "java.base/java.util", + "java.base/java.util.concurrent", + "java.base/java.util.concurrent.atomic", + "java.base/sun.nio.ch", + "java.base/sun.nio.cs", + "java.base/sun.security.action", + "java.base/sun.util.calendar", + "java.security.jgss/sun.security.krb5" +] + +List defaultJavaModuleRunOptions = + [jdk8CompatibilityOption] + openJavaModules.collect { module -> "--add-opens=${module}=ALL-UNNAMED " } + +ext.defaultJavaModuleRunOptions = defaultJavaModuleRunOptions +ext.openJavaModules = openJavaModules diff --git a/package/build.gradle b/package/build.gradle index b1d0e79f80..06934cf0de 100644 --- a/package/build.gradle +++ b/package/build.gradle @@ -12,6 +12,9 @@ jar { shadowJar { archiveClassifier = null + manifest { + attributes 'Add-Opens': openJavaModules.join(" ") + } } defineShadowPublication().call() diff --git a/scoring-package/build.gradle b/scoring-package/build.gradle index 0c8479ace4..874046fa42 100644 --- a/scoring-package/build.gradle +++ b/scoring-package/build.gradle @@ -12,6 +12,9 @@ jar { shadowJar { archiveClassifier = null + manifest { + attributes 'Add-Opens': openJavaModules.join(" ") + } } defineShadowPublication().call()