From 25b614744db063f6c1e355b6095fdc9178694b49 Mon Sep 17 00:00:00 2001 From: yantian Date: Fri, 20 Dec 2024 12:34:33 +0800 Subject: [PATCH] add test for CreateTableRequest and DataField json parse --- .../org/apache/paimon/catalog/Catalog.java | 6 +- .../apache/paimon/rest/RESTObjectMapper.java | 21 +++ .../rest/requests/CreateTableRequest.java | 52 ++++++-- .../paimon/rest/requests/TableSchema.java | 123 ++++++++++++++++++ .../apache/paimon/utils/JsonSerdeUtil.java | 30 ++--- .../apache/paimon/rest/MockRESTMessage.java | 21 +++ .../paimon/rest/RESTObjectMapperTest.java | 33 +++++ 7 files changed, 258 insertions(+), 28 deletions(-) create mode 100644 paimon-core/src/main/java/org/apache/paimon/rest/requests/TableSchema.java diff --git a/paimon-core/src/main/java/org/apache/paimon/catalog/Catalog.java b/paimon-core/src/main/java/org/apache/paimon/catalog/Catalog.java index 3a18d2be8c8e..e1d044e95b3d 100644 --- a/paimon-core/src/main/java/org/apache/paimon/catalog/Catalog.java +++ b/paimon-core/src/main/java/org/apache/paimon/catalog/Catalog.java @@ -371,15 +371,17 @@ default void repairTable(Identifier identifier) throws TableNotExistException { */ class NoPermissionException extends RuntimeException { private static final String MSG = "No permission for %s %s."; + private static final String DATABASE_TYPE_NAME = "database"; + private static final String TABLE_TYPE_NAME = "table"; public static NoPermissionException createDatabaseNoPermissionException( String databaseName, Throwable cause) { - return new NoPermissionException("database", databaseName, cause); + return new NoPermissionException(DATABASE_TYPE_NAME, databaseName, cause); } public static NoPermissionException createTableNoPermissionException( Identifier identifier, Throwable cause) { - return new NoPermissionException("table", identifier.getFullName(), cause); + return new NoPermissionException(TABLE_TYPE_NAME, identifier.getFullName(), cause); } public NoPermissionException(String resourceType, String resourceName, Throwable cause) { diff --git a/paimon-core/src/main/java/org/apache/paimon/rest/RESTObjectMapper.java b/paimon-core/src/main/java/org/apache/paimon/rest/RESTObjectMapper.java index b1c83e90224a..fa66e754ac85 100644 --- a/paimon-core/src/main/java/org/apache/paimon/rest/RESTObjectMapper.java +++ b/paimon-core/src/main/java/org/apache/paimon/rest/RESTObjectMapper.java @@ -18,18 +18,39 @@ package org.apache.paimon.rest; +import org.apache.paimon.types.DataField; +import org.apache.paimon.types.DataType; +import org.apache.paimon.types.DataTypeJsonParser; + import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.databind.DeserializationFeature; +import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.databind.Module; import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.databind.ObjectMapper; import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.databind.SerializationFeature; +import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.databind.module.SimpleModule; import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.datatype.jsr310.JavaTimeModule; +import static org.apache.paimon.utils.JsonSerdeUtil.registerJsonObjects; + /** Object mapper for REST request and response. */ public class RESTObjectMapper { public static ObjectMapper create() { ObjectMapper mapper = new ObjectMapper(); mapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false); mapper.configure(SerializationFeature.FAIL_ON_EMPTY_BEANS, false); + mapper.registerModule(createPaimonRestJacksonModule()); mapper.registerModule(new JavaTimeModule()); return mapper; } + + public static Module createPaimonRestJacksonModule() { + SimpleModule module = new SimpleModule("Paimon_REST"); + registerJsonObjects( + module, + DataField.class, + DataField::serializeJson, + DataTypeJsonParser::parseDataField); + registerJsonObjects( + module, DataType.class, DataType::serializeJson, DataTypeJsonParser::parseDataType); + return module; + } } diff --git a/paimon-core/src/main/java/org/apache/paimon/rest/requests/CreateTableRequest.java b/paimon-core/src/main/java/org/apache/paimon/rest/requests/CreateTableRequest.java index 1e152d7f1f56..3a3262932522 100644 --- a/paimon-core/src/main/java/org/apache/paimon/rest/requests/CreateTableRequest.java +++ b/paimon-core/src/main/java/org/apache/paimon/rest/requests/CreateTableRequest.java @@ -29,30 +29,60 @@ /** Request for creating table. */ public class CreateTableRequest implements RESTRequest { - private static final String FIELD_IDENTIFIER = "identifier"; + private static final String FIELD_DATABASE_NAME = "database"; + private static final String FIELD_TABLE_NAME = "table"; + private static final String FIELD_BRANCH_NAME = "branch"; private static final String FIELD_SCHEMA = "schema"; - @JsonProperty(FIELD_IDENTIFIER) - private Identifier identifier; + @JsonProperty(FIELD_DATABASE_NAME) + private String databaseName; + + @JsonProperty(FIELD_TABLE_NAME) + private String tableName; + + @JsonProperty(FIELD_BRANCH_NAME) + private String branchName; @JsonProperty(FIELD_SCHEMA) - private Schema schema; + private TableSchema schema; @JsonCreator public CreateTableRequest( - @JsonProperty(FIELD_IDENTIFIER) Identifier identifier, - @JsonProperty(FIELD_SCHEMA) Schema schema) { - this.identifier = identifier; + @JsonProperty(FIELD_DATABASE_NAME) String databaseName, + @JsonProperty(FIELD_TABLE_NAME) String tableName, + @JsonProperty(FIELD_BRANCH_NAME) String branchName, + @JsonProperty(FIELD_SCHEMA) TableSchema schema) { + this.databaseName = databaseName; + this.tableName = tableName; + this.branchName = branchName; this.schema = schema; } - @JsonGetter(FIELD_IDENTIFIER) - public Identifier getIdentifier() { - return identifier; + public CreateTableRequest(Identifier identifier, Schema schema) { + this( + identifier.getDatabaseName(), + identifier.getTableName(), + identifier.getBranchName(), + new TableSchema(schema)); + } + + @JsonGetter(FIELD_DATABASE_NAME) + public String getDatabaseName() { + return databaseName; + } + + @JsonGetter(FIELD_TABLE_NAME) + public String getTableName() { + return tableName; + } + + @JsonGetter(FIELD_BRANCH_NAME) + public String getBranchName() { + return branchName; } @JsonGetter(FIELD_SCHEMA) - public Schema getSchema() { + public TableSchema getSchema() { return schema; } } diff --git a/paimon-core/src/main/java/org/apache/paimon/rest/requests/TableSchema.java b/paimon-core/src/main/java/org/apache/paimon/rest/requests/TableSchema.java new file mode 100644 index 000000000000..d49e4062eea5 --- /dev/null +++ b/paimon-core/src/main/java/org/apache/paimon/rest/requests/TableSchema.java @@ -0,0 +1,123 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.rest.requests; + +import org.apache.paimon.schema.Schema; +import org.apache.paimon.types.DataField; + +import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonCreator; +import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonGetter; +import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonProperty; + +import java.util.List; +import java.util.Map; +import java.util.Objects; + +/** + * Wrap the {@link Schema} class to support RESTCatalog. Define a class as: 1. This class in rest + * catalog is easy to maintain. 2. It's easy to manage rest API fields. + */ +public class TableSchema { + private static final String FIELD_FILED_NAME = "fields"; + private static final String FIELD_PARTITION_KEYS_NAME = "partitionKeys"; + private static final String FIELD_PRIMARY_KEYS_NAME = "primaryKeys"; + private static final String FIELD_OPTIONS_NAME = "options"; + private static final String FIELD_COMMENT_NAME = "comment"; + + @JsonProperty(FIELD_FILED_NAME) + private final List fields; + + @JsonProperty(FIELD_PARTITION_KEYS_NAME) + private final List partitionKeys; + + @JsonProperty(FIELD_PRIMARY_KEYS_NAME) + private final List primaryKeys; + + @JsonProperty(FIELD_OPTIONS_NAME) + private final Map options; + + @JsonProperty(FIELD_COMMENT_NAME) + private final String comment; + + @JsonCreator + public TableSchema( + @JsonProperty(FIELD_FILED_NAME) List fields, + @JsonProperty(FIELD_PARTITION_KEYS_NAME) List partitionKeys, + @JsonProperty(FIELD_PRIMARY_KEYS_NAME) List primaryKeys, + @JsonProperty(FIELD_OPTIONS_NAME) Map options, + @JsonProperty(FIELD_COMMENT_NAME) String comment) { + this.fields = fields; + this.partitionKeys = partitionKeys; + this.primaryKeys = primaryKeys; + this.options = options; + this.comment = comment; + } + + public TableSchema(Schema schema) { + this.fields = schema.fields(); + this.partitionKeys = schema.partitionKeys(); + this.primaryKeys = schema.primaryKeys(); + this.options = schema.options(); + this.comment = schema.comment(); + } + + @JsonGetter(FIELD_FILED_NAME) + public List getFields() { + return fields; + } + + @JsonGetter(FIELD_PARTITION_KEYS_NAME) + public List getPartitionKeys() { + return partitionKeys; + } + + @JsonGetter(FIELD_PRIMARY_KEYS_NAME) + public List getPrimaryKeys() { + return primaryKeys; + } + + @JsonGetter(FIELD_OPTIONS_NAME) + public Map getOptions() { + return options; + } + + @JsonGetter(FIELD_COMMENT_NAME) + public String getComment() { + return comment; + } + + @Override + public boolean equals(Object o) { + if (o == null || getClass() != o.getClass()) { + return false; + } else { + TableSchema that = (TableSchema) o; + return Objects.equals(fields, that.fields) + && Objects.equals(partitionKeys, that.partitionKeys) + && Objects.equals(primaryKeys, that.primaryKeys) + && Objects.equals(options, that.options) + && Objects.equals(comment, that.comment); + } + } + + @Override + public int hashCode() { + return Objects.hash(fields, partitionKeys, primaryKeys, options, comment); + } +} diff --git a/paimon-core/src/main/java/org/apache/paimon/utils/JsonSerdeUtil.java b/paimon-core/src/main/java/org/apache/paimon/utils/JsonSerdeUtil.java index a919d83c8741..edc6dac5f992 100644 --- a/paimon-core/src/main/java/org/apache/paimon/utils/JsonSerdeUtil.java +++ b/paimon-core/src/main/java/org/apache/paimon/utils/JsonSerdeUtil.java @@ -154,21 +154,7 @@ public static String toFlatJson(T t) { } } - private static Module createPaimonJacksonModule() { - SimpleModule module = new SimpleModule("Paimon"); - registerJsonObjects( - module, TableSchema.class, SchemaSerializer.INSTANCE, SchemaSerializer.INSTANCE); - registerJsonObjects( - module, - DataField.class, - DataField::serializeJson, - DataTypeJsonParser::parseDataField); - registerJsonObjects( - module, DataType.class, DataType::serializeJson, DataTypeJsonParser::parseDataType); - return module; - } - - private static void registerJsonObjects( + public static void registerJsonObjects( SimpleModule module, Class clazz, JsonSerializer serializer, @@ -192,6 +178,20 @@ public T deserialize(JsonParser parser, DeserializationContext context) }); } + private static Module createPaimonJacksonModule() { + SimpleModule module = new SimpleModule("Paimon"); + registerJsonObjects( + module, TableSchema.class, SchemaSerializer.INSTANCE, SchemaSerializer.INSTANCE); + registerJsonObjects( + module, + DataField.class, + DataField::serializeJson, + DataTypeJsonParser::parseDataField); + registerJsonObjects( + module, DataType.class, DataType::serializeJson, DataTypeJsonParser::parseDataType); + return module; + } + /** * Parses the provided JSON string and casts it to the specified type of {@link JsonNode}. * diff --git a/paimon-core/src/test/java/org/apache/paimon/rest/MockRESTMessage.java b/paimon-core/src/test/java/org/apache/paimon/rest/MockRESTMessage.java index 5f40b15f4f9d..a701b54c9dbd 100644 --- a/paimon-core/src/test/java/org/apache/paimon/rest/MockRESTMessage.java +++ b/paimon-core/src/test/java/org/apache/paimon/rest/MockRESTMessage.java @@ -18,14 +18,18 @@ package org.apache.paimon.rest; +import org.apache.paimon.catalog.Identifier; import org.apache.paimon.rest.requests.AlterDatabaseRequest; import org.apache.paimon.rest.requests.CreateDatabaseRequest; +import org.apache.paimon.rest.requests.CreateTableRequest; import org.apache.paimon.rest.responses.AlterDatabaseResponse; import org.apache.paimon.rest.responses.CreateDatabaseResponse; import org.apache.paimon.rest.responses.ErrorResponse; import org.apache.paimon.rest.responses.GetDatabaseResponse; import org.apache.paimon.rest.responses.ListDatabasesResponse; import org.apache.paimon.rest.responses.ListTablesResponse; +import org.apache.paimon.schema.Schema; +import org.apache.paimon.types.DataTypes; import org.apache.paimon.shade.guava30.com.google.common.collect.Lists; @@ -90,4 +94,21 @@ public static ListTablesResponse listTablesResponse() { public static ListTablesResponse listTablesEmptyResponse() { return new ListTablesResponse(Lists.newArrayList()); } + + public static CreateTableRequest createTableRequest(String name) { + Identifier identifier = Identifier.create(databaseName(), name); + Map options = new HashMap<>(); + options.put("k1", "v1"); + Schema schema = + Schema.newBuilder() + .column("pt", DataTypes.INT()) + .column("pk", DataTypes.INT()) + .column("col1", DataTypes.INT()) + .column("col2", DataTypes.STRING()) + .partitionKeys("pt") + .primaryKey("pk", "pt") + .options(options) + .build(); + return new CreateTableRequest(identifier, schema); + } } diff --git a/paimon-core/src/test/java/org/apache/paimon/rest/RESTObjectMapperTest.java b/paimon-core/src/test/java/org/apache/paimon/rest/RESTObjectMapperTest.java index 1cf382a9f438..6da61932a6e7 100644 --- a/paimon-core/src/test/java/org/apache/paimon/rest/RESTObjectMapperTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/rest/RESTObjectMapperTest.java @@ -20,12 +20,16 @@ import org.apache.paimon.rest.requests.AlterDatabaseRequest; import org.apache.paimon.rest.requests.CreateDatabaseRequest; +import org.apache.paimon.rest.requests.CreateTableRequest; import org.apache.paimon.rest.responses.AlterDatabaseResponse; import org.apache.paimon.rest.responses.ConfigResponse; import org.apache.paimon.rest.responses.CreateDatabaseResponse; import org.apache.paimon.rest.responses.ErrorResponse; import org.apache.paimon.rest.responses.GetDatabaseResponse; import org.apache.paimon.rest.responses.ListDatabasesResponse; +import org.apache.paimon.types.DataField; +import org.apache.paimon.types.DataTypes; +import org.apache.paimon.types.IntType; import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.databind.ObjectMapper; @@ -125,4 +129,33 @@ public void alterDatabaseResponseParseTest() throws Exception { assertEquals(response.getUpdated().size(), parseData.getUpdated().size()); assertEquals(response.getMissing().size(), parseData.getMissing().size()); } + + @Test + public void createTableRequestParseTest() throws Exception { + CreateTableRequest request = MockRESTMessage.createTableRequest("t1"); + String requestStr = mapper.writeValueAsString(request); + CreateTableRequest parseData = mapper.readValue(requestStr, CreateTableRequest.class); + assertEquals(request.getDatabaseName(), parseData.getDatabaseName()); + assertEquals(request.getTableName(), parseData.getTableName()); + assertEquals(request.getBranchName(), parseData.getBranchName()); + assertEquals(request.getSchema(), parseData.getSchema()); + } + + // This test is to guarantee the compatibility of field name in RESTCatalog. + @Test + public void dataFieldParseTest() throws Exception { + int id = 1; + String name = "col1"; + IntType type = DataTypes.INT(); + String descStr = "desc"; + String dataFieldStr = + String.format( + "{\"id\": %d,\"name\":\"%s\",\"type\":\"%s\", \"description\":\"%s\"}", + id, name, type, descStr); + DataField parseData = mapper.readValue(dataFieldStr, DataField.class); + assertEquals(id, parseData.id()); + assertEquals(name, parseData.name()); + assertEquals(type, parseData.type()); + assertEquals(descStr, parseData.description()); + } }