update dependencies and fix various issues due to incompatibility wit…

…h jdk8
h2oai · Jan 24, 2024 · 0ff7d39 · 0ff7d39
1 parent 696dc9e
commit 0ff7d39
Show file tree

Hide file tree

Showing 60 changed files with 2,560 additions and 845 deletions.
diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml
@@ -35,7 +35,7 @@ jobs:
 
     steps:
     - name: Checkout repository
-      uses: actions/checkout@v2
+      uses: actions/checkout@v3
 
     # Initializes the CodeQL tools for scanning.
     - name: Initialize CodeQL
@@ -63,8 +63,7 @@ jobs:
     - if: matrix.language == 'java'
       name: Build Java
       uses: gradle/gradle-build-action@v2
-      with:
-        arguments: assemble
+      run: ./gradlew --init-script init.gradle assemble
 
     - name: Perform CodeQL Analysis
       uses: github/codeql-action/analyze@v2
diff --git a/.github/workflows/snyk-scan.yml b/.github/workflows/snyk-scan.yml
@@ -28,7 +28,7 @@ jobs:
       - uses: snyk/actions/setup@master
       - uses: actions/setup-java@v3
         with:
-          java-version: "8"
+          java-version: "17"
           distribution: 'adopt'
 
       - name: Snyk scan for Java dependencies
@@ -76,7 +76,7 @@ jobs:
       - uses: snyk/actions/setup@master
       - uses: actions/setup-java@v3
         with:
-          java-version: "8"
+          java-version: "17"
           distribution: 'adopt'
 
       - name: Snyk scan for Java dependencies - ${{ matrix.depsfiles }}

diff --git a/Jenkinsfile b/Jenkinsfile
@@ -4,7 +4,7 @@
 
 import ai.h2o.ci.Utils
 
-JAVA_IMAGE = 'harbor.h2o.ai/dockerhub-proxy/library/openjdk:8u222-jdk-slim'
+JAVA_IMAGE = 'eclipse-temurin@sha256:24643c2dd329ef482ecd042b59cbfb7fe13716342e22674a0abd763559c8a1dd'
 NODE_LABEL = 'docker'
 DOCKERHUB_CREDS = 'dockerhub'
 HARBOR_URL = "http://harbor.h2o.ai/"
@@ -27,6 +27,11 @@ pipeline {
     }
 
     parameters {
+        booleanParam(
+            name: 'ONLY_PUSH_MLOPS_SCORER',
+            defaultValue: true,
+            description: "Whether to only push MLOps Scorer to remote container registry."
+        )
         booleanParam(
             name: 'PUSH_TO_HARBOR',
             defaultValue: false,
@@ -43,9 +48,9 @@ pipeline {
             description: 'Whether to also push distribution ZIP archive to S3.',
         )
         booleanParam(
-                name: 'PUSH_TO_VORVAN',
-                defaultValue: false,
-                description: 'Whether to also push Docker images to h2o.ai maintained gcr.io repo Vorvan.',
+            name: 'PUSH_TO_VORVAN',
+            defaultValue: false,
+            description: 'Whether to also push Docker images to h2o.ai maintained gcr.io repo Vorvan.',
         )
     }
 
@@ -65,8 +70,8 @@ pipeline {
             // Run inside JAVA_IMAGE container on NODE_LABEL host.
             agent {
                 docker {
-                    registryCredentialsId HARBOR_CREDS
-                    registryUrl HARBOR_URL
+                    //registryCredentialsId HARBOR_CREDS
+                    //registryUrl HARBOR_URL
                     image JAVA_IMAGE
                     label NODE_LABEL
                 }
@@ -76,7 +81,7 @@ pipeline {
                     script {
                         versionText = getVersion()
                         echo "Version: ${versionText}"
-                        sh "./gradlew check"
+                        sh "./gradlew --init-script init.gradle check"
                     }
                 }
             }
@@ -100,7 +105,7 @@ pipeline {
             steps {
                 timeout(time: 60, unit: 'MINUTES') {
                     script {
-                        sh "./gradlew distributionZip"
+                        sh "./gradlew --init-script init.gradle distributionZip"
                         if (isReleaseVersion(versionText)) {
                             utilsLib.appendBuildDescription("Release ${versionText}")
                         }
@@ -156,8 +161,8 @@ pipeline {
             }
             agent {
                 docker {
-                    registryCredentialsId HARBOR_CREDS
-                    registryUrl HARBOR_URL
+                    //registryCredentialsId HARBOR_CREDS
+                    //registryUrl HARBOR_URL
                     image JAVA_IMAGE
                     label NODE_LABEL
                 }
@@ -169,14 +174,20 @@ pipeline {
                         def imageTags = isMasterBranch() || isReleaseBranch() ? "${versionText},${gitCommitHash}" : "${gitCommitHash}"
                         withDockerCredentials(DOCKERHUB_CREDS, "FROM_") {
                             withDockerCredentials("harbor.h2o.ai", "TO_") {
-                                sh "./gradlew jib \
-                                -Djib.to.auth.username=${TO_DOCKER_USERNAME} \
+                                def cmd = "./gradlew --init-script init.gradle "
+                                if (params.ONLY_PUSH_MLOPS_SCORER) {
+                                    cmd += ":local-rest-scorer:jib "
+                                } else {
+                                    cmd += "jib "
+                                }
+                                cmd += "-Djib.to.auth.username=${TO_DOCKER_USERNAME} \
                                 -Djib.to.auth.password=${TO_DOCKER_PASSWORD} \
                                 -Djib.from.auth.username=${FROM_DOCKER_USERNAME} \
                                 -Djib.from.auth.password=${FROM_DOCKER_PASSWORD} \
                                 -Djib.to.tags=${imageTags} \
                                 -Djib.allowInsecureRegistries=true \
                                 -DsendCredentialsOverHttp=true"
+                                sh cmd
                             }
                         }
                     }
@@ -192,26 +203,32 @@ pipeline {
             }
             agent {
                 docker {
-                    registryCredentialsId HARBOR_CREDS
-                    registryUrl HARBOR_URL
+                    //registryCredentialsId HARBOR_CREDS
+                    //registryUrl HARBOR_URL
                     image JAVA_IMAGE
                     label NODE_LABEL
                 }
             }
             steps {
                 timeout(time: 30, unit: 'MINUTES') {
                     script {
+                        def cmd = "./gradlew --init-script init.gradle "
+                        if (params.ONLY_PUSH_MLOPS_SCORER) {
+                            cmd += ":local-rest-scorer:jib "
+                        } else {
+                            cmd += "jib "
+                        }
                         def gitCommitHash = env.GIT_COMMIT
                         def imageTags = isMasterBranch() || isReleaseBranch() ? "${versionText},${gitCommitHash}" : "${gitCommitHash}"
                         withDockerCredentials(DOCKERHUB_CREDS, "FROM_") {
                             withDockerCredentials(DOCKERHUB_CREDS, "TO_") {
-                                sh "./gradlew jib \
-                                -Djib.to.auth.username=${TO_DOCKER_USERNAME} \
+                                cmd += "-Djib.to.auth.username=${TO_DOCKER_USERNAME} \
                                 -Djib.to.auth.password=${TO_DOCKER_PASSWORD} \
                                 -Djib.from.auth.username=${FROM_DOCKER_USERNAME} \
                                 -Djib.from.auth.password=${FROM_DOCKER_PASSWORD} \
                                 -Djib.to.tags=${imageTags} \
                                 -PdockerRepositoryPrefix=h2oai/"
+                                sh cmd
                             }
                         }
                     }
@@ -227,26 +244,32 @@ pipeline {
             }
             agent {
                 docker {
-                    registryCredentialsId HARBOR_CREDS
-                    registryUrl HARBOR_URL
+                    //registryCredentialsId HARBOR_CREDS
+                    //registryUrl HARBOR_URL
                     image JAVA_IMAGE
                     label NODE_LABEL
                 }
             }
             steps {
                 timeout(time: 30, unit: 'MINUTES') {
                     script {
+                        def cmd = "./gradlew --init-script init.gradle "
+                        if (params.ONLY_PUSH_MLOPS_SCORER) {
+                            cmd += ":local-rest-scorer:jib "
+                        } else {
+                            cmd += "jib "
+                        }
                         def gitCommitHash = env.GIT_COMMIT
                         def imageTags = isMasterBranch() || isReleaseBranch() ? "${versionText},${gitCommitHash}" : "${gitCommitHash}"
                         withDockerCredentials(DOCKERHUB_CREDS, "FROM_") {
                             withGCRCredentials(VORVAN_CRED) {
                                 def gcrCreds = readFile("${GCR_JSON_KEY}")
                                 withEnv(['TO_DOCKER_USERNAME=_json_key', "TO_DOCKER_PASSWORD=${gcrCreds}"]) {
-                                    sh "./gradlew jib \
-                                    -Djib.from.auth.username=${FROM_DOCKER_USERNAME} \
+                                    cmd += "-Djib.from.auth.username=${FROM_DOCKER_USERNAME} \
                                     -Djib.from.auth.password=${FROM_DOCKER_PASSWORD} \
                                     -Djib.to.tags=${imageTags} \
                                     -PdockerRepositoryPrefix=gcr.io/vorvan/h2oai/"
+                                    sh cmd
                                 }
                             }
                         }
@@ -264,8 +287,8 @@ pipeline {
  */
 def getVersion() {
     def version = sh(
-            script: "./gradlew -q -Dorg.gradle.internal.launcher.welcomeMessageEnabled=false printVersion",
-            returnStdout: true).trim()
+            script: "./gradlew --init-script init.gradle -q -Dorg.gradle.internal.launcher.welcomeMessageEnabled=false printVersion",
+            returnStdout: true).trim().tokenize("\n").last()
     if (!version) {
         error "Version must be set"
     }

diff --git a/aws-lambda-scorer/lambda-template/build.gradle b/aws-lambda-scorer/lambda-template/build.gradle
@@ -1,3 +1,7 @@
+plugins {
+    id 'org.springframework.boot'
+}
+
 apply from: project(":").file('gradle/java.gradle')
 
 dependencies {
@@ -11,11 +15,14 @@ dependencies {
     implementation group: 'com.google.code.gson', name: 'gson'
     implementation group: 'io.swagger.core.v3', name: 'swagger-annotations'
 
-    testImplementation group: 'com.google.truth.extensions', name: 'truth-java8-extension'
     testImplementation group: 'org.junit.jupiter', name: 'junit-jupiter-api'
     testRuntimeOnly group: 'org.junit.jupiter', name: 'junit-jupiter-engine'
 }
 
+bootJar {
+    enabled=false
+}
+
 test {
     useJUnitPlatform()
 }

diff --git a/...ambda-scorer/lambda-template/src/main/java/ai/h2o/mojos/deploy/aws/lambda/MojoScorer.java b/...ambda-scorer/lambda-template/src/main/java/ai/h2o/mojos/deploy/aws/lambda/MojoScorer.java
@@ -35,10 +35,10 @@ public final class MojoScorer {
   private static final Object pipelineLock = new Object();
   private static MojoPipeline pipeline;
 
-  private final ScoreRequestToMojoFrameConverter requestConverter
-          = new ScoreRequestToMojoFrameConverter();
-  private final MojoFrameToScoreResponseConverter responseConverter
-          = new MojoFrameToScoreResponseConverter();
+  private final ScoreRequestToMojoFrameConverter requestConverter =
+      new ScoreRequestToMojoFrameConverter();
+  private final MojoFrameToScoreResponseConverter responseConverter =
+      new MojoFrameToScoreResponseConverter();
   private final RequestChecker requestChecker = new RequestChecker(new SampleRequestBuilder());
 
   /** Processes a single {@link ScoreRequest} in the given AWS Lambda {@link Context}. */

diff --git a/aws-sagemaker-hosted-scorer/build.gradle b/aws-sagemaker-hosted-scorer/build.gradle
@@ -12,8 +12,8 @@ dependencies {
     implementation group: 'io.springfox', name: 'springfox-boot-starter', version: springFoxVersion
     implementation group: 'com.google.guava', name: 'guava', version: guavaVersion
     implementation group: 'org.springframework.boot', name: 'spring-boot-starter-web'
-    implementation group: 'org.apache.tomcat.embed', name: 'tomcat-embed-core', version: '9.0.63'
-    implementation group: 'org.apache.tomcat.embed', name: 'tomcat-embed-websocket', version: '9.0.63'
+    implementation group: 'org.apache.tomcat.embed', name: 'tomcat-embed-core', version: '9.0.75'
+    implementation group: 'org.apache.tomcat.embed', name: 'tomcat-embed-websocket', version: '9.0.75'
 }
 
 test {
@@ -49,7 +49,7 @@ jib {
         }
     }
     container {
-        jvmFlags = defaultJibContainerJvmFlags.split(" ").each { it.trim() }
+        jvmFlags = ["-Dmojo.path=/mojos/pipeline.mojo", "--add-opens", "java.base/java.lang=ALL-UNNAMED", "--add-opens", "java.base/java.util=ALL-UNNAMED", "--add-opens", "java.base/java.lang.reflect=ALL-UNNAMED"]
         ports = ['8080']
         volumes = [
                 // mojo pipeline and license file will live here

diff --git a/...scorer/src/main/java/ai/h2o/mojos/deploy/sagemaker/hosted/config/ScorerConfiguration.java b/...scorer/src/main/java/ai/h2o/mojos/deploy/sagemaker/hosted/config/ScorerConfiguration.java
@@ -64,12 +64,12 @@ public MojoScorer mojoScorer(
       ScoreRequestTransformer scoreRequestTransformer,
       CsvToMojoFrameConverter csvConverter) {
     return new MojoScorer(
-            requestConverter,
-            responseConverter,
-            contributionRequestConverter,
-            contributionResponseConverter,
-            modelInfoConverter,
-            scoreRequestTransformer,
-            csvConverter);
+        requestConverter,
+        responseConverter,
+        contributionRequestConverter,
+        contributionResponseConverter,
+        modelInfoConverter,
+        scoreRequestTransformer,
+        csvConverter);
   }
 }
diff --git a/...er/src/main/java/ai/h2o/mojos/deploy/sagemaker/hosted/controller/ModelsApiController.java b/...er/src/main/java/ai/h2o/mojos/deploy/sagemaker/hosted/controller/ModelsApiController.java
@@ -13,7 +13,6 @@
 import java.io.IOException;
 import java.util.Arrays;
 import java.util.List;
-
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.springframework.beans.factory.annotation.Autowired;
@@ -25,10 +24,9 @@
 @Controller
 public class ModelsApiController implements ModelApi {
 
-  private static final String UNIMPLEMENTED_MESSAGE
-          = "Shapley values are not implemented yet";
-  private static final List<CapabilityType> SUPPORTED_CAPABILITIES
-      = Arrays.asList(CapabilityType.SCORE);
+  private static final String UNIMPLEMENTED_MESSAGE = "Shapley values are not implemented yet";
+  private static final List<CapabilityType> SUPPORTED_CAPABILITIES =
+      Arrays.asList(CapabilityType.SCORE);
   private static final Logger log = LoggerFactory.getLogger(ModelsApiController.class);
 
   private final MojoScorer scorer;
@@ -101,8 +99,7 @@ public ResponseEntity<ScoreResponse> getScoreByFile(String file) {
   }
 
   @Override
-  public ResponseEntity<ContributionResponse> getContribution(
-          ContributionRequest request) {
+  public ResponseEntity<ContributionResponse> getContribution(ContributionRequest request) {
     // TODO: to be implemented in the future
     log.info(" Unsupported operation: " + UNIMPLEMENTED_MESSAGE);
     return ResponseEntity.status(HttpStatus.NOT_IMPLEMENTED).build();

diff --git a/build.gradle b/build.gradle
@@ -8,15 +8,14 @@ buildscript {
     dependencies {
         classpath group: 'org.springframework.boot', name: 'spring-boot-gradle-plugin',
                 version: springBootPluginVersion
-        classpath group: 'gradle.plugin.org.hidetake', name: 'gradle-swagger-generator-plugin',
-                version: swaggerGradlePluginVersion
-        classpath group: 'com.github.jengelman.gradle.plugins', name: 'shadow',
+        classpath group: 'gradle.plugin.com.github.johnrengelman', name: 'shadow',
                 version: shadowJarVersion
-//        classpath group: 'com.diffplug.spotless', name: 'spotless-plugin-gradle', version: spotlessPluginVersion
         classpath group: 'net.ltgt.gradle', name: 'gradle-errorprone-plugin',
                 version: errorpronePluginVersion
         classpath group: 'com.google.cloud.tools.jib', name: 'com.google.cloud.tools.jib.gradle.plugin',
                 version: jibPluginVersion
+        classpath group: 'org.openapitools', name: 'openapi-generator-gradle-plugin',
+                version: openApiGeneratorGradlePluginVersion
     }
 }
 

diff --git a/common/jdbc/build.gradle b/common/jdbc/build.gradle
@@ -15,6 +15,9 @@ dependencies {
     implementation group: 'org.apache.spark', name: 'spark-sql_2.12'
     implementation group: 'org.apache.spark', name: 'spark-mllib_2.12'
     implementation group: 'com.typesafe', name:'config'
+    implementation group: 'org.slf4j', name: 'slf4j-api', version: '1.7.36'
+    implementation group: 'ch.qos.logback', name: 'logback-classic', version: '1.0.13'
+    implementation group: 'ch.qos.logback', name: 'logback-core', version: '1.0.13'
 
     testImplementation group: 'org.scalatest', name: 'scalatest_2.12', version: '3.0.5'
     testRuntimeOnly group:'org.scala-lang.modules', name: 'scala-xml_2.12', version: '1.1.1'