add ut

jerry-024 · Dec 27, 2024 · 35aa8ec · 35aa8ec
1 parent d0cae62
commit 35aa8ec
Show file tree

Hide file tree

Showing 6 changed files with 172 additions and 43 deletions.
diff --git a/paimon-common/src/main/java/org/apache/paimon/types/RowType.java b/paimon-common/src/main/java/org/apache/paimon/types/RowType.java
@@ -24,6 +24,9 @@
 import org.apache.paimon.utils.Preconditions;
 import org.apache.paimon.utils.StringUtils;
 
+import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonCreator;
+import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonGetter;
+import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonProperty;
 import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.core.JsonGenerator;
 
 import java.io.IOException;
@@ -52,9 +55,13 @@ public final class RowType extends DataType {
 
     private static final long serialVersionUID = 1L;
 
+    public static final String FILED_FIELDS = "fields";
+
     public static final String FORMAT = "ROW<%s>";
 
+    @JsonProperty(FILED_FIELDS)
     private final List<DataField> fields;
+
     private InternalRow.FieldGetter[] fieldGetters;
 
     public RowType(boolean isNullable, List<DataField> fields) {
@@ -67,14 +74,16 @@ public RowType(boolean isNullable, List<DataField> fields) {
         validateFields(fields);
     }
 
-    public RowType(List<DataField> fields) {
+    @JsonCreator
+    public RowType(@JsonProperty(FILED_FIELDS) List<DataField> fields) {
         this(true, fields);
     }
 
     public RowType copy(List<DataField> newFields) {
         return new RowType(isNullable(), newFields);
     }
 
+    @JsonGetter(FILED_FIELDS)
     public List<DataField> getFields() {
         return fields;
     }

diff --git a/paimon-common/src/main/java/org/apache/paimon/utils/InternalRowPartitionComputer.java b/paimon-common/src/main/java/org/apache/paimon/utils/InternalRowPartitionComputer.java
@@ -111,7 +111,7 @@ public static GenericRow convertSpecToInternalRow(
         List<String> fieldNames = partType.getFieldNames();
         for (Map.Entry<String, String> entry : spec.entrySet()) {
             Object value =
-                    defaultPartValue.equals(entry.getValue())
+                    defaultPartValue != null && defaultPartValue.equals(entry.getValue())
                             ? null
                             : castFromString(
                                     entry.getValue(), partType.getField(entry.getKey()).type());
@@ -120,18 +120,6 @@ public static GenericRow convertSpecToInternalRow(
         return partRow;
     }
 
-    public static GenericRow convertSpecToInternalRow(Map<String, String> spec, RowType partType) {
-        checkArgument(spec.size() == partType.getFieldCount());
-        GenericRow partRow = new GenericRow(spec.size());
-        List<String> fieldNames = partType.getFieldNames();
-        for (Map.Entry<String, String> entry : spec.entrySet()) {
-            Object value =
-                    castFromString(entry.getValue(), partType.getField(entry.getKey()).type());
-            partRow.setField(fieldNames.indexOf(entry.getKey()), value);
-        }
-        return partRow;
-    }
-
     public static String partToSimpleString(
             RowType partitionType, BinaryRow partition, String delimiter, int maxLength) {
         FieldGetter[] getters = partitionType.fieldGetters();

diff --git a/paimon-core/src/main/java/org/apache/paimon/rest/RESTCatalog.java b/paimon-core/src/main/java/org/apache/paimon/rest/RESTCatalog.java
@@ -145,7 +145,8 @@ public RESTCatalog(CatalogContext catalogContext) {
         Map<String, String> initHeaders =
                 RESTUtil.merge(
                         configHeaders(catalogOptions.toMap()), this.catalogAuth.getHeaders());
-        Options options = new Options(fetchOptionsFromServer(initHeaders, initHeaders));
+        Options options =
+                new Options(fetchOptionsFromServer(initHeaders, catalogContext.options().toMap()));
         this.context =
                 CatalogContext.create(
                         options, catalogContext.preferIO(), catalogContext.fallbackIO());
@@ -482,7 +483,8 @@ public List<PartitionEntry> listPartitionsFromServer(Identifier identifier)
                 InternalRowSerializer serializer =
                         new InternalRowSerializer(partition.getPartitionType());
                 GenericRow row =
-                        convertSpecToInternalRow(partition.getSpec(), partition.getPartitionType());
+                        convertSpecToInternalRow(
+                                partition.getSpec(), partition.getPartitionType(), null);
                 PartitionEntry partitionEntry =
                         new PartitionEntry(
                                 serializer.toBinaryRow(row).copy(),

diff --git a/paimon-core/src/test/java/org/apache/paimon/rest/MockRESTMessage.java b/paimon-core/src/test/java/org/apache/paimon/rest/MockRESTMessage.java
@@ -22,14 +22,17 @@
 import org.apache.paimon.rest.requests.AlterDatabaseRequest;
 import org.apache.paimon.rest.requests.AlterTableRequest;
 import org.apache.paimon.rest.requests.CreateDatabaseRequest;
+import org.apache.paimon.rest.requests.CreatePartitionRequest;
 import org.apache.paimon.rest.requests.CreateTableRequest;
+import org.apache.paimon.rest.requests.DropPartitionRequest;
 import org.apache.paimon.rest.requests.RenameTableRequest;
 import org.apache.paimon.rest.responses.AlterDatabaseResponse;
 import org.apache.paimon.rest.responses.CreateDatabaseResponse;
 import org.apache.paimon.rest.responses.ErrorResponse;
 import org.apache.paimon.rest.responses.GetDatabaseResponse;
 import org.apache.paimon.rest.responses.GetTableResponse;
 import org.apache.paimon.rest.responses.ListDatabasesResponse;
+import org.apache.paimon.rest.responses.ListPartitionsResponse;
 import org.apache.paimon.rest.responses.ListTablesResponse;
 import org.apache.paimon.schema.Schema;
 import org.apache.paimon.schema.SchemaChange;
@@ -39,6 +42,7 @@
 import org.apache.paimon.types.IntType;
 import org.apache.paimon.types.RowType;
 
+import org.apache.paimon.shade.guava30.com.google.common.collect.ImmutableList;
 import org.apache.paimon.shade.guava30.com.google.common.collect.Lists;
 
 import java.util.ArrayList;
@@ -131,6 +135,28 @@ public static AlterTableRequest alterTableRequest() {
         return new AlterTableRequest(getChanges());
     }
 
+    public static CreatePartitionRequest createPartitionRequest(String tableName) {
+        Identifier identifier = Identifier.create(databaseName(), tableName);
+        return new CreatePartitionRequest(identifier, Collections.singletonMap("pt", "1"));
+    }
+
+    public static DropPartitionRequest dropPartitionRequest() {
+        return new DropPartitionRequest(Collections.singletonMap("pt", "1"));
+    }
+
+    public static ListPartitionsResponse listPartitionsResponse() {
+        Map<String, String> spec = new HashMap<>();
+        spec.put("a", "1");
+        spec.put("b", "2");
+        List<DataField> fields = new ArrayList<>();
+        fields.add(new DataField(0, "a", DataTypes.INT()));
+        fields.add(new DataField(1, "b", DataTypes.STRING()));
+        RowType partitionType = new RowType(false, fields);
+        ListPartitionsResponse.Partition partition =
+                new ListPartitionsResponse.Partition(spec, partitionType, 1, 1, 1, 1);
+        return new ListPartitionsResponse(ImmutableList.of(partition));
+    }
+
     public static List<SchemaChange> getChanges() {
         // add option
         SchemaChange addOption = SchemaChange.setOption("snapshot.time-retained", "2h");

diff --git a/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogTest.java b/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogTest.java
@@ -22,6 +22,7 @@
 import org.apache.paimon.catalog.CatalogContext;
 import org.apache.paimon.catalog.Database;
 import org.apache.paimon.catalog.Identifier;
+import org.apache.paimon.manifest.PartitionEntry;
 import org.apache.paimon.options.CatalogOptions;
 import org.apache.paimon.options.Options;
 import org.apache.paimon.rest.requests.CreateTableRequest;
@@ -31,7 +32,9 @@
 import org.apache.paimon.rest.responses.GetDatabaseResponse;
 import org.apache.paimon.rest.responses.GetTableResponse;
 import org.apache.paimon.rest.responses.ListDatabasesResponse;
+import org.apache.paimon.rest.responses.ListPartitionsResponse;
 import org.apache.paimon.rest.responses.ListTablesResponse;
+import org.apache.paimon.rest.responses.SuccessResponse;
 import org.apache.paimon.schema.SchemaChange;
 import org.apache.paimon.table.Table;
 
@@ -71,27 +74,17 @@ public class RESTCatalogTest {
     private RESTCatalog restCatalog;
     private RESTCatalog mockRestCatalog;
     private String warehouseStr;
+    private String serverUrl;
     @Rule public TemporaryFolder folder = new TemporaryFolder();
 
     @Before
     public void setUp() throws IOException {
         mockWebServer = new MockWebServer();
         mockWebServer.start();
-        String baseUrl = mockWebServer.url("").toString();
-        Options options = new Options();
-        options.set(RESTCatalogOptions.URI, baseUrl);
-        String initToken = "init_token";
-        options.set(RESTCatalogOptions.TOKEN, initToken);
-        options.set(RESTCatalogOptions.THREAD_POOL_SIZE, 1);
+        serverUrl = mockWebServer.url("").toString();
+        Options options = mockInitOptions();
         warehouseStr = folder.getRoot().getPath();
-        String mockResponse =
-                String.format(
-                        "{\"defaults\": {\"%s\": \"%s\", \"%s\": \"%s\"}}",
-                        RESTCatalogInternalOptions.PREFIX.key(),
-                        "prefix",
-                        CatalogOptions.WAREHOUSE.key(),
-                        warehouseStr);
-        mockResponse(mockResponse, 200);
+        mockConfig(warehouseStr);
         restCatalog = new RESTCatalog(CatalogContext.create(options));
         mockRestCatalog = spy(restCatalog);
     }
@@ -359,6 +352,70 @@ public void testDropTableWhenTableNotExistAndIgnoreIfNotExistsIsFalse() throws E
                 () -> restCatalog.dropTable(Identifier.create(databaseName, tableName), false));
     }
 
+    @Test
+    public void testCreatePartition() throws Exception {
+        String databaseName = MockRESTMessage.databaseName();
+        Map<String, String> partitionSpec = new HashMap<>();
+        partitionSpec.put("p1", "v1");
+        mockResponse(mapper.writeValueAsString(new SuccessResponse()), 200);
+        assertDoesNotThrow(
+                () ->
+                        mockRestCatalog.createPartition(
+                                Identifier.create(databaseName, "table"), partitionSpec));
+    }
+
+    @Test
+    public void testCreatePartitionWhenTableNotExist() throws Exception {
+        String databaseName = MockRESTMessage.databaseName();
+        Map<String, String> partitionSpec = new HashMap<>();
+        partitionSpec.put("p1", "v1");
+        mockResponse("", 404);
+        assertThrows(
+                Catalog.TableNotExistException.class,
+                () ->
+                        mockRestCatalog.createPartition(
+                                Identifier.create(databaseName, "table"), partitionSpec));
+    }
+
+    @Test
+    public void testCreatePartitionWhenTableNoPermissionException() throws Exception {
+        String databaseName = MockRESTMessage.databaseName();
+        Map<String, String> partitionSpec = new HashMap<>();
+        partitionSpec.put("p1", "v1");
+        mockResponse("", 403);
+        assertThrows(
+                Catalog.TableNoPermissionException.class,
+                () ->
+                        mockRestCatalog.createPartition(
+                                Identifier.create(databaseName, "table"), partitionSpec));
+    }
+
+    @Test
+    public void testListPartitionsWhenMetastorePartitionedIsTrue() throws Exception {
+        Options options = mockInitOptions();
+        options = options.set(RESTCatalogOptions.METASTORE_PARTITIONED, true);
+        mockConfig(warehouseStr);
+        RESTCatalog restCatalog = new RESTCatalog(CatalogContext.create(options));
+        RESTCatalog mockRestCatalog = spy(restCatalog);
+        String databaseName = MockRESTMessage.databaseName();
+        ListPartitionsResponse response = MockRESTMessage.listPartitionsResponse();
+        mockResponse(mapper.writeValueAsString(response), 200);
+        List<PartitionEntry> result =
+                mockRestCatalog.listPartitions(Identifier.create(databaseName, "table"));
+        verify(mockRestCatalog, times(1)).listPartitionsFromServer(any());
+        assertEquals(response.getPartitions().size(), result.size());
+    }
+
+    @Test
+    public void testListPartitionsFromFile() throws Exception {
+        String databaseName = MockRESTMessage.databaseName();
+        GetTableResponse response = MockRESTMessage.getTableResponse();
+        mockResponse(mapper.writeValueAsString(response), 200);
+        mockRestCatalog.listPartitions(Identifier.create(databaseName, "table"));
+        verify(mockRestCatalog, times(1)).getTable(any());
+        verify(mockRestCatalog, times(0)).listPartitionsFromServer(any());
+    }
+
     private void mockResponse(String mockResponse, int httpCode) {
         MockResponse mockResponseObj =
                 new MockResponse()
@@ -367,4 +424,24 @@ private void mockResponse(String mockResponse, int httpCode) {
                         .addHeader("Content-Type", "application/json");
         mockWebServer.enqueue(mockResponseObj);
     }
+
+    private void mockConfig(String warehouseStr) {
+        String mockResponse =
+                String.format(
+                        "{\"defaults\": {\"%s\": \"%s\", \"%s\": \"%s\"}}",
+                        RESTCatalogInternalOptions.PREFIX.key(),
+                        "prefix",
+                        CatalogOptions.WAREHOUSE.key(),
+                        warehouseStr);
+        mockResponse(mockResponse, 200);
+    }
+
+    public Options mockInitOptions() {
+        Options options = new Options();
+        options.set(RESTCatalogOptions.URI, serverUrl);
+        String initToken = "init_token";
+        options.set(RESTCatalogOptions.TOKEN, initToken);
+        options.set(RESTCatalogOptions.THREAD_POOL_SIZE, 1);
+        return options;
+    }
 }
diff --git a/paimon-core/src/test/java/org/apache/paimon/rest/RESTObjectMapperTest.java b/paimon-core/src/test/java/org/apache/paimon/rest/RESTObjectMapperTest.java
@@ -18,11 +18,12 @@
 
 package org.apache.paimon.rest;
 
-import org.apache.paimon.data.GenericRow;
 import org.apache.paimon.rest.requests.AlterDatabaseRequest;
 import org.apache.paimon.rest.requests.AlterTableRequest;
 import org.apache.paimon.rest.requests.CreateDatabaseRequest;
+import org.apache.paimon.rest.requests.CreatePartitionRequest;
 import org.apache.paimon.rest.requests.CreateTableRequest;
+import org.apache.paimon.rest.requests.DropPartitionRequest;
 import org.apache.paimon.rest.requests.RenameTableRequest;
 import org.apache.paimon.rest.responses.AlterDatabaseResponse;
 import org.apache.paimon.rest.responses.ConfigResponse;
@@ -31,23 +32,24 @@
 import org.apache.paimon.rest.responses.GetDatabaseResponse;
 import org.apache.paimon.rest.responses.GetTableResponse;
 import org.apache.paimon.rest.responses.ListDatabasesResponse;
+import org.apache.paimon.rest.responses.ListPartitionsResponse;
 import org.apache.paimon.rest.responses.ListTablesResponse;
+import org.apache.paimon.rest.responses.SuccessResponse;
 import org.apache.paimon.types.DataField;
 import org.apache.paimon.types.DataTypes;
 import org.apache.paimon.types.IntType;
-import org.apache.paimon.types.RowType;
-import org.apache.paimon.utils.InternalRowPartitionComputer;
 
+import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.core.JsonProcessingException;
 import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.databind.ObjectMapper;
 
 import org.junit.Test;
 
 import java.util.ArrayList;
 import java.util.HashMap;
-import java.util.List;
 import java.util.Map;
 
 import static org.junit.Assert.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertDoesNotThrow;
 
 /** Test for {@link RESTObjectMapper}. */
 public class RESTObjectMapperTest {
@@ -199,14 +201,39 @@ public void alterTableRequestParseTest() throws Exception {
     }
 
     @Test
-    public void testPartitionSpecToInternalRow() {
-        Map<String, String> spec = new HashMap<>();
-        spec.put("a", "1");
-        spec.put("b", "2");
-        List<DataField> fields = new ArrayList<>();
-        fields.add(new DataField(0, "a", DataTypes.INT()));
-        fields.add(new DataField(1, "b", DataTypes.STRING()));
-        RowType partitionType = new RowType(false, fields);
-        GenericRow row = InternalRowPartitionComputer.convertSpecToInternalRow(spec, partitionType);
+    public void createPartitionRequestParseTest() throws JsonProcessingException {
+        CreatePartitionRequest request = MockRESTMessage.createPartitionRequest("t1");
+        String requestStr = mapper.writeValueAsString(request);
+        CreatePartitionRequest parseData =
+                mapper.readValue(requestStr, CreatePartitionRequest.class);
+        assertEquals(parseData.getIdentifier(), parseData.getIdentifier());
+        assertEquals(parseData.getPartitionSpec().size(), parseData.getPartitionSpec().size());
+    }
+
+    @Test
+    public void dropPartitionRequestParseTest() throws JsonProcessingException {
+        DropPartitionRequest request = MockRESTMessage.dropPartitionRequest();
+        String requestStr = mapper.writeValueAsString(request);
+        DropPartitionRequest parseData = mapper.readValue(requestStr, DropPartitionRequest.class);
+        assertEquals(parseData.getPartitionSpec().size(), parseData.getPartitionSpec().size());
+    }
+
+    @Test
+    public void listPartitionsResponseParseTest() throws Exception {
+        ListPartitionsResponse response = MockRESTMessage.listPartitionsResponse();
+        String responseStr = mapper.writeValueAsString(response);
+        ListPartitionsResponse parseData =
+                mapper.readValue(responseStr, ListPartitionsResponse.class);
+        assertEquals(
+                response.getPartitions().get(0).getFileCount(),
+                parseData.getPartitions().get(0).getFileCount());
+    }
+
+    @Test
+    public void successResponseParseTest() throws Exception {
+        SuccessResponse response = new SuccessResponse();
+        assertDoesNotThrow(() -> mapper.writeValueAsString(response));
+        assertDoesNotThrow(
+                () -> mapper.readValue(mapper.writeValueAsString(response), SuccessResponse.class));
     }
 }