Skip to content
This repository has been archived by the owner on Nov 29, 2024. It is now read-only.

debug #384

Closed
wants to merge 5 commits into from
Closed

debug #384

Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions build.gradle
Original file line number Diff line number Diff line change
Expand Up @@ -47,8 +47,8 @@ subprojects {
// }
//
task distributionZip(type: Zip) {
archiveName "${project.name}-${project.version}.zip"
destinationDir(file(buildDir))
getArchiveFileName().set("${project.name}-${project.version}.zip")
getDestinationDirectory().set(file(buildDir))
}

task printVersion {
Expand Down
8 changes: 7 additions & 1 deletion common/rest-java-model/build.gradle
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@ openApiGenerate {
invokerPackage = "ai.h2o.mojos.deploy.common.rest"
inputSpec = "$rootDir/common/swagger/v1openapi3/swagger.json"
outputDir = "$buildDir/gen"
generateAliasAsModel = true
globalProperties.set([
"skipFormModel": "false",
])
Expand All @@ -31,18 +32,23 @@ openApiGenerate {
"interfaceOnly": "true",
"basePackage": "ai.h2o.mojos.deploy.common.rest",
"modelPackage": "ai.h2o.mojos.deploy.common.rest.model",
"apiPackage": "ai.h2o.mojos.deploy.common.rest.api",
])
}

bootJar {
enabled=false
}

jar {
enabled=true
}

compileJava.dependsOn tasks.openApiValidate, tasks.openApiGenerate
sourceSets {
main {
java {
srcDir("$buildDir/gen/src/main/java")
srcDir("${buildDir}/gen/src/main/java")
}
}
}
3 changes: 2 additions & 1 deletion common/rest-spring-api/build.gradle
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@ openApiGenerate {
generatorName = 'spring'
inputSpec = "$rootDir/common/swagger/v1openapi3/swagger.json"
outputDir = "$buildDir/gen"
generateAliasAsModel = true
globalProperties.set([
"skipFormModel": "false",
])
Expand All @@ -46,7 +47,7 @@ compileJava.dependsOn tasks.openApiValidate, tasks.openApiGenerate
sourceSets {
main {
java {
srcDir("$buildDir/gen/src/main/java")
srcDir("${buildDir}/gen/src/main/java")
}
}
}
8 changes: 6 additions & 2 deletions common/transform/build.gradle
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@ plugins {
apply from: project(":").file('gradle/java.gradle')

dependencies {
implementation project(':common:rest-java-model')
implementation project(':common:rest-spring-api')
implementation group: 'io.swagger.core.v3', name: 'swagger-annotations'
implementation group: 'ai.h2o', name: 'mojo2-runtime-api'
implementation group: 'ai.h2o', name: 'mojo2-runtime-impl'
Expand All @@ -29,7 +29,11 @@ dependencies {
}

bootJar {
enabled=false
enabled = false
}

jar {
enabled = true
}

test {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -19,8 +19,8 @@ public class MojoFrameToContributionResponseConverter {
* ContributionResponse}.
*/
public ContributionResponse contributionResponseWithNoOutputGroup(MojoFrame shapleyMojoFrame) {
List<List<String>> outputRows =
Stream.generate(ArrayList<String>::new)
List<Row> outputRows =
Stream.generate(Row::new)
.limit(shapleyMojoFrame.getNrows())
.collect(Collectors.toList());
Utils.copyResultFields(shapleyMojoFrame, outputRows);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,7 @@ public class MojoFrameToScoreResponseConverter
// Note: assumption is that pipeline supports Prediction interval.
// However for some h2o3 model, even classification model may still set
// this to be true.
private Boolean supportPredictionInterval;
private final Boolean supportPredictionInterval;

public MojoFrameToScoreResponseConverter(boolean supportPredictionInterval) {
this.supportPredictionInterval = supportPredictionInterval;
Expand All @@ -57,8 +57,8 @@ public MojoFrameToScoreResponseConverter() {
@Override
public ScoreResponse apply(MojoFrame mojoFrame, ScoreRequest scoreRequest) {
Set<String> includedFields = getSetOfIncludedFields(scoreRequest);
List<List<String>> outputRows =
Stream.generate(ArrayList<String>::new)
List<Row> outputRows =
Stream.generate(Row::new)
.limit(mojoFrame.getNrows())
.collect(Collectors.toList());
copyFilteredInputFields(scoreRequest, includedFields, outputRows);
Expand All @@ -81,7 +81,7 @@ public ScoreResponse apply(MojoFrame mojoFrame, ScoreRequest scoreRequest) {
* response frame, only one column rows will be populated into the outputRows to ensure backward
* compatible.
*/
private void fillOutputRows(MojoFrame mojoFrame, List<List<String>> outputRows) {
private void fillOutputRows(MojoFrame mojoFrame, List<Row> outputRows) {
List<List<String>> targetRows = getTargetRows(mojoFrame);
for (int rowIdx = 0; rowIdx < mojoFrame.getNrows(); rowIdx++) {
outputRows.get(rowIdx).addAll(targetRows.get(rowIdx));
Expand Down Expand Up @@ -182,9 +182,9 @@ private List<Integer> getTargetFieldIndices(MojoFrame mojoFrame) {
* Extract prediction interval columns rows from MOJO response frame. Note: Assumption is
* prediction interval should already be enabled and response frame has expected structure.
*/
private List<List<String>> getPredictionIntervalRows(MojoFrame mojoFrame, int targetIdx) {
List<List<String>> predictionIntervalRows =
Stream.generate(ArrayList<String>::new)
private List<Row> getPredictionIntervalRows(MojoFrame mojoFrame, int targetIdx) {
List<Row> predictionIntervalRows =
Stream.generate(Row::new)
.limit(mojoFrame.getNrows())
.collect(Collectors.toList());
for (int row = 0; row < mojoFrame.getNrows(); row++) {
Expand Down Expand Up @@ -234,15 +234,15 @@ private int getTargetColIdx(List<String> mojoColumns) {
}

private static void copyFilteredInputFields(
ScoreRequest scoreRequest, Set<String> includedFields, List<List<String>> outputRows) {
ScoreRequest scoreRequest, Set<String> includedFields, List<Row> outputRows) {
if (includedFields.isEmpty()) {
return;
}
boolean generateRowIds = shouldGenerateRowIds(scoreRequest, includedFields);
List<List<String>> inputRows = scoreRequest.getRows();
List<Row> inputRows = scoreRequest.getRows();
for (int row = 0; row < outputRows.size(); row++) {
List<String> inputRow = inputRows.get(row);
List<String> outputRow = outputRows.get(row);
Row inputRow = inputRows.get(row);
Row outputRow = outputRows.get(row);
List<String> inputFields = scoreRequest.getFields();
for (int col = 0; col < inputFields.size(); col++) {
if (includedFields.contains(inputFields.get(col))) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -2,11 +2,14 @@

import static java.util.Arrays.asList;

import ai.h2o.mojos.deploy.common.rest.model.Row;
import ai.h2o.mojos.deploy.common.rest.model.ScoreRequest;
import ai.h2o.mojos.runtime.frame.MojoFrameMeta;
import java.util.List;

/** Checks that the request is of the correct form matching the corresponding mojo pipeline. */
/**
* Checks that the request is of the correct form matching the corresponding mojo pipeline.
*/
public class RequestChecker {
private final SampleRequestBuilder sampleRequestBuilder;

Expand Down Expand Up @@ -36,15 +39,15 @@ private String getProblemMessageOrNull(ScoreRequest scoreRequest, MojoFrameMeta
if (fields == null || fields.isEmpty()) {
return "List of input fields cannot be empty";
}
List<List<String>> rows = scoreRequest.getRows();
List<Row> rows = scoreRequest.getRows();
if (rows == null || rows.isEmpty()) {
return "List of input data rows cannot be empty";
}
List<String> expectedFields = asList(expectedMeta.getColumnNames());
if (!fields.containsAll(expectedFields)) {
return String.format(
"Input fields don't contain all the Mojo fields, expected %s actual %s",
expectedFields.toString(), fields.toString());
expectedFields, fields);
}
int i = 0;
for (List<String> row : scoreRequest.getRows()) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,9 @@
* request to further play with and fill with actual meaningful data.
*/
public class SampleRequestBuilder {
/** Builds a valid {@link ScoreRequest} based on the given mojo input {@link MojoFrameMeta}. */
/**
* Builds a valid {@link ScoreRequest} based on the given mojo input {@link MojoFrameMeta}.
*/
public ScoreRequest build(MojoFrameMeta inputMeta) {
ScoreRequest request = new ScoreRequest();
final List<String> fields =
Expand Down
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
package ai.h2o.mojos.deploy.common.transform;

import ai.h2o.mojos.deploy.common.rest.model.Row;
import ai.h2o.mojos.deploy.common.rest.model.ScoreRequest;
import ai.h2o.mojos.runtime.frame.MojoFrame;
import ai.h2o.mojos.runtime.frame.MojoFrameBuilder;
Expand All @@ -17,7 +18,7 @@ public MojoFrame apply(ScoreRequest scoreRequest, MojoFrameBuilder frameBuilder)
List<String> fields = scoreRequest.getFields();

if (scoreRequest.getRows() != null) {
for (List<String> row : scoreRequest.getRows()) {
for (Row row : scoreRequest.getRows()) {
MojoRowBuilder rowBuilder = frameBuilder.getMojoRowBuilder();
for (int i = 0; i < row.size(); i++) {
rowBuilder.setValue(fields.get(i), row.get(i));
Expand Down
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
package ai.h2o.mojos.deploy.common.transform;

import ai.h2o.mojos.deploy.common.rest.model.DataField;
import ai.h2o.mojos.deploy.common.rest.model.Row;
import ai.h2o.mojos.deploy.common.rest.model.ScoreRequest;
import com.google.common.collect.ImmutableMap;
import java.util.ArrayList;
Expand Down Expand Up @@ -30,35 +31,25 @@ public void accept(ScoreRequest scoreRequest, List<DataField> dataFields) {
transformRow(scoreRequest.getFields(), scoreRequest.getRows(), dataFieldMap));
}

private List<List<String>> transformRow(
List<String> fields, List<List<String>> rows, Map<String, DataField> dataFields) {
return rows.stream()
.map(
row -> {
List<String> transformData =
IntStream.range(0, row.size())
.mapToObj(
fieldIdx -> {
String colName = fields.get(fieldIdx);
String origin = row.get(fieldIdx);
if (dataFields.containsKey(colName)) {
String sanitizeValue =
Utils.sanitizeBoolean(
origin, dataFields.get(colName).getDataType());
if (!sanitizeValue.equals(origin)) {
logger.debug("Value '{}' parsed as '{}'", origin, sanitizeValue);
}
return sanitizeValue;
} else {
logger.debug("Column '{}' can not be found in Input schema", colName);
return origin;
}
})
.collect(Collectors.toList());
List<String> transformedRow = new ArrayList<>();
transformedRow.addAll(transformData);
return transformedRow;
})
.collect(Collectors.toList());
private List<Row> transformRow(
List<String> fields, List<Row> rows, Map<String, DataField> dataFields) {
return rows.stream().map(row -> IntStream.range(0, row.size()).mapToObj(
fieldIdx -> {
String colName = fields.get(fieldIdx);
String origin = row.get(fieldIdx);
if (dataFields.containsKey(colName)) {
String sanitizeValue = Utils.sanitizeBoolean(
origin, dataFields.get(colName).getDataType()
);
if (!sanitizeValue.equals(origin)) {
logger.debug("Value '{}' parsed as '{}'", origin, sanitizeValue);
}
return sanitizeValue;
} else {
logger.warn("Column '{}' can not be found in Input schema", colName);
return origin;
}
}
).collect(Collectors.toCollection(Row::new))).collect(Collectors.toList());
}
}
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
package ai.h2o.mojos.deploy.common.transform;

import ai.h2o.mojos.deploy.common.rest.model.DataField;
import ai.h2o.mojos.deploy.common.rest.model.Row;
import ai.h2o.mojos.runtime.frame.MojoFrame;
import java.util.List;

Expand All @@ -10,7 +11,7 @@ public class Utils {
*
* @param mojoFrame {@link MojoFrame}
*/
public static void copyResultFields(MojoFrame mojoFrame, List<List<String>> outputRows) {
public static void copyResultFields(MojoFrame mojoFrame, List<Row> outputRows) {
String[][] outputColumns = new String[mojoFrame.getNcols()][];
for (int col = 0; col < mojoFrame.getNcols(); col++) {
outputColumns[col] = mojoFrame.getColumn(col).getDataAsStrings();
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -474,8 +474,8 @@ private MojoFrame generateDummyTransformedMojoFrame() {

private ScoreResponse generateDummyResponse() {
ScoreResponse response = new ScoreResponse();
List<List<String>> outputRows =
Stream.generate(ArrayList<String>::new).limit(4).collect(Collectors.toList());
List<Row> outputRows =
Stream.generate(Row::new).limit(4).collect(Collectors.toList());
response.setScore(outputRows);
response.setFields(Arrays.asList("field1"));
return response;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -38,16 +38,16 @@ void transform_BooleanLiteral_Transformed() {
// Given
ScoreRequest scoreRequest = new ScoreRequest();
scoreRequest.setFields(Collections.singletonList("test"));
List<List<String>> rows =
List<Row> rows =
new ArrayList<>(
Arrays.asList(
new ArrayList<>(),
new ArrayList<>(),
new ArrayList<>(),
new ArrayList<>(),
new ArrayList<>(),
new ArrayList<>(),
new ArrayList<>()));
new Row(),
new Row(),
new Row(),
new Row(),
new Row(),
new Row(),
new Row()));
rows.get(0).addAll(Collections.singletonList("true"));
rows.get(1).addAll(Collections.singletonList("False"));
rows.get(2).addAll(Collections.singletonList("TrUE"));
Expand Down Expand Up @@ -85,7 +85,7 @@ void transform_NonBooleanLiteral_Unchanged() {
// Given
ScoreRequest scoreRequest = new ScoreRequest();
scoreRequest.setFields(Collections.singletonList("test"));
List<List<String>> rows = new ArrayList<>(Arrays.asList(new Row(), new Row()));
List<Row> rows = new ArrayList<>(Arrays.asList(new Row(), new Row()));
rows.get(0).addAll(Collections.singletonList("unchangedFeature1"));
rows.get(1).addAll(Collections.singletonList("unchangedFeature2"));
scoreRequest.setRows(rows);
Expand Down
8 changes: 4 additions & 4 deletions gradle.properties
Original file line number Diff line number Diff line change
Expand Up @@ -37,11 +37,11 @@ jakartaServletVersion = 6.0.0
tomcatVersion = 9.0.75

# External plugins:
springBootPluginVersion = 3.2.0
springBootPluginVersion = 3.2.3
swaggerGradlePluginVersion = 2.19.2
errorpronePluginVersion = 3.1.0
jibPluginVersion = 3.4.0
openApiGeneratorGradlePluginVersion = 7.0.1
openApiGeneratorGradlePluginVersion = 7.2.0

# External tools:
checkStyleVersion = 8.21
Expand All @@ -52,8 +52,8 @@ errorproneVersion = 2.23.0
# Docker settings
dockerRepositoryPrefix = harbor.h2o.ai/opsh2oai/h2oai/
dockerIncludePython = true
# Digest of eclipse-temurin:17.0.9_9-jdk-alpine
javaBaseImage = eclipse-temurin@sha256:24643c2dd329ef482ecd042b59cbfb7fe13716342e22674a0abd763559c8a1dd
# Digest of eclipse-temurin:21.0.2_13-jdk-alpine
javaBaseImage = eclipse-temurin@sha256:82698e23d15ada036bc176f6fb210401e0679cd0a4b1e71d05e7329982d6062c

# Increase timeouts to avoid read error from OSS Nexus
# See:
Expand Down
4 changes: 2 additions & 2 deletions gradle/java_no_style.gradle
Original file line number Diff line number Diff line change
Expand Up @@ -5,5 +5,5 @@
apply plugin: 'java'
apply from: project(":").file('gradle/mixins/dependencies.gradle')

sourceCompatibility = JavaVersion.VERSION_17
targetCompatibility = JavaVersion.VERSION_17
sourceCompatibility = "21"
targetCompatibility = "21"
1 change: 0 additions & 1 deletion gradle/mixins/checkstyle.gradle
Original file line number Diff line number Diff line change
@@ -1,5 +1,4 @@
// Defines shared Gradle project Checkstyle analyzer configuration.

apply plugin: 'checkstyle'

checkstyle {
Expand Down
2 changes: 1 addition & 1 deletion gradle/wrapper/gradle-wrapper.properties
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
distributionBase=GRADLE_USER_HOME
distributionPath=wrapper/dists
distributionUrl=https\://services.gradle.org/distributions/gradle-7.4.2-bin.zip
distributionUrl=https\://services.gradle.org/distributions/gradle-8.6-bin.zip
zipStoreBase=GRADLE_USER_HOME
zipStorePath=wrapper/dists
1 change: 0 additions & 1 deletion local-rest-scorer/build.gradle
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,6 @@ plugins {
apply from: project(":").file('gradle/java.gradle')

dependencies {
runtimeOnly group: 'org.springframework.boot', name: 'spring-boot-properties-migrator'
implementation project(':common:rest-spring-api')
implementation project(':common:transform')
implementation group: 'ai.h2o', name: 'h2o-genmodel'
Expand Down
Loading
Loading