Skip to content

Commit

Permalink
create partition return PartitionResponse and drop partition return b…
Browse files Browse the repository at this point in the history
…oolean
  • Loading branch information
jerry-024 committed Dec 30, 2024
1 parent 4ec183c commit 1d8faff
Show file tree
Hide file tree
Showing 8 changed files with 127 additions and 118 deletions.
25 changes: 10 additions & 15 deletions paimon-core/src/main/java/org/apache/paimon/rest/RESTCatalog.java
Original file line number Diff line number Diff line change
Expand Up @@ -57,7 +57,7 @@
import org.apache.paimon.rest.responses.ListDatabasesResponse;
import org.apache.paimon.rest.responses.ListPartitionsResponse;
import org.apache.paimon.rest.responses.ListTablesResponse;
import org.apache.paimon.rest.responses.SuccessResponse;
import org.apache.paimon.rest.responses.PartitionResponse;
import org.apache.paimon.schema.Schema;
import org.apache.paimon.schema.SchemaChange;
import org.apache.paimon.schema.TableSchema;
Expand Down Expand Up @@ -93,7 +93,6 @@
import static org.apache.paimon.catalog.CatalogUtils.checkNotSystemTable;
import static org.apache.paimon.catalog.CatalogUtils.isSystemDatabase;
import static org.apache.paimon.options.CatalogOptions.CASE_SENSITIVE;
import static org.apache.paimon.rest.RESTCatalogOptions.METASTORE_PARTITIONED;
import static org.apache.paimon.utils.InternalRowPartitionComputer.convertSpecToInternalRow;
import static org.apache.paimon.utils.Preconditions.checkNotNull;
import static org.apache.paimon.utils.ThreadPoolUtils.createScheduledThreadPool;
Expand Down Expand Up @@ -382,7 +381,7 @@ public void createPartition(Identifier identifier, Map<String, String> partition
resourcePaths.partitions(
identifier.getDatabaseName(), identifier.getTableName()),
request,
SuccessResponse.class,
PartitionResponse.class,
headers());
} catch (NoSuchResourceException e) {
throw new TableNotExistException(identifier);
Expand All @@ -397,17 +396,14 @@ public void dropPartition(Identifier identifier, Map<String, String> partitions)
checkNotSystemTable(identifier, "dropPartition");
dropPartitionMetadata(identifier, partitions);
Table table = getTable(identifier);
if (table != null) {
cleanPartitionsInFileSystem(table, partitions);
} else {
throw new TableNotExistException(identifier);
}
cleanPartitionsInFileSystem(table, partitions);
}

@Override
public List<PartitionEntry> listPartitions(Identifier identifier)
throws TableNotExistException {
boolean whetherSupportListPartitions = context.options().get(METASTORE_PARTITIONED);
boolean whetherSupportListPartitions =
context.options().get(CoreOptions.METASTORE_PARTITIONED_TABLE);
if (whetherSupportListPartitions) {
FileStoreTable table = (FileStoreTable) getTable(identifier);
RowType rowType = table.schema().logicalPartitionType();
Expand Down Expand Up @@ -489,8 +485,7 @@ public List<PartitionEntry> listPartitionsFromServer(Identifier identifier, RowT
}

@VisibleForTesting
PartitionEntry convertToPartitionEntry(
ListPartitionsResponse.Partition partition, RowType rowType) {
PartitionEntry convertToPartitionEntry(PartitionResponse partition, RowType rowType) {
InternalRowSerializer serializer = new InternalRowSerializer(rowType);
GenericRow row = convertSpecToInternalRow(partition.getSpec(), rowType, null);
return new PartitionEntry(
Expand Down Expand Up @@ -528,18 +523,18 @@ protected GetTableResponse getTableResponse(Identifier identifier)
}
}

protected SuccessResponse dropPartitionMetadata(
Identifier identifier, Map<String, String> partitions)
protected boolean dropPartitionMetadata(Identifier identifier, Map<String, String> partitions)
throws TableNoPermissionException {
try {
DropPartitionRequest request = new DropPartitionRequest(partitions);
return client.delete(
client.delete(
resourcePaths.partitions(
identifier.getDatabaseName(), identifier.getTableName()),
request,
headers());
return true;
} catch (NoSuchResourceException ignore) {
return new SuccessResponse();
return true;
} catch (ForbiddenException e) {
throw new TableNoPermissionException(identifier, e);
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -71,10 +71,4 @@ public class RESTCatalogOptions {
.stringType()
.noDefaultValue()
.withDescription("REST Catalog auth token provider path.");

public static final ConfigOption<Boolean> METASTORE_PARTITIONED =
ConfigOptions.key("metastore-partitioned")
.booleanType()
.defaultValue(false)
.withDescription("REST Catalog Server whether support list partitions.");
}
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,6 @@
import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonProperty;

import java.util.List;
import java.util.Map;

/** Response for listing partitions. */
@JsonIgnoreProperties(ignoreUnknown = true)
Expand All @@ -35,80 +34,16 @@ public class ListPartitionsResponse implements RESTResponse {
public static final String FIELD_PARTITIONS = "partitions";

@JsonProperty(FIELD_PARTITIONS)
private final List<Partition> partitions;
private final List<PartitionResponse> partitions;

@JsonCreator
public ListPartitionsResponse(@JsonProperty(FIELD_PARTITIONS) List<Partition> partitions) {
public ListPartitionsResponse(
@JsonProperty(FIELD_PARTITIONS) List<PartitionResponse> partitions) {
this.partitions = partitions;
}

@JsonGetter(FIELD_PARTITIONS)
public List<Partition> getPartitions() {
public List<PartitionResponse> getPartitions() {
return partitions;
}

/** Partition for rest api. */
@JsonIgnoreProperties(ignoreUnknown = true)
public static class Partition implements RESTResponse {

private static final String FIELD_SPEC = "spec";
public static final String FIELD_RECORD_COUNT = "recordCount";
public static final String FIELD_FILE_SIZE_IN_BYTES = "fileSizeInBytes";
public static final String FIELD_FILE_COUNT = "fileCount";
public static final String FIELD_LAST_FILE_CREATION_TIME = "lastFileCreationTime";

@JsonProperty(FIELD_SPEC)
private final Map<String, String> spec;

@JsonProperty(FIELD_RECORD_COUNT)
private final long recordCount;

@JsonProperty(FIELD_FILE_SIZE_IN_BYTES)
private final long fileSizeInBytes;

@JsonProperty(FIELD_FILE_COUNT)
private final long fileCount;

@JsonProperty(FIELD_LAST_FILE_CREATION_TIME)
private final long lastFileCreationTime;

@JsonCreator
public Partition(
@JsonProperty(FIELD_SPEC) Map<String, String> spec,
@JsonProperty(FIELD_RECORD_COUNT) long recordCount,
@JsonProperty(FIELD_FILE_SIZE_IN_BYTES) long fileSizeInBytes,
@JsonProperty(FIELD_FILE_COUNT) long fileCount,
@JsonProperty(FIELD_LAST_FILE_CREATION_TIME) long lastFileCreationTime) {
this.spec = spec;
this.recordCount = recordCount;
this.fileSizeInBytes = fileSizeInBytes;
this.fileCount = fileCount;
this.lastFileCreationTime = lastFileCreationTime;
}

@JsonGetter(FIELD_SPEC)
public Map<String, String> getSpec() {
return spec;
}

@JsonGetter(FIELD_RECORD_COUNT)
public long getRecordCount() {
return recordCount;
}

@JsonGetter(FIELD_FILE_SIZE_IN_BYTES)
public long getFileSizeInBytes() {
return fileSizeInBytes;
}

@JsonGetter(FIELD_FILE_COUNT)
public long getFileCount() {
return fileCount;
}

@JsonGetter(FIELD_LAST_FILE_CREATION_TIME)
public long getLastFileCreationTime() {
return lastFileCreationTime;
}
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,92 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

package org.apache.paimon.rest.responses;

import org.apache.paimon.rest.RESTResponse;

import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonCreator;
import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonGetter;
import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonIgnoreProperties;
import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonProperty;

import java.util.Map;

/** Partition for rest api. */
@JsonIgnoreProperties(ignoreUnknown = true)
public class PartitionResponse implements RESTResponse {
public static final String FIELD_SPEC = "spec";
public static final String FIELD_RECORD_COUNT = "recordCount";
public static final String FIELD_FILE_SIZE_IN_BYTES = "fileSizeInBytes";
public static final String FIELD_FILE_COUNT = "fileCount";
public static final String FIELD_LAST_FILE_CREATION_TIME = "lastFileCreationTime";

@JsonProperty(FIELD_SPEC)
private final Map<String, String> spec;

@JsonProperty(FIELD_RECORD_COUNT)
private final long recordCount;

@JsonProperty(FIELD_FILE_SIZE_IN_BYTES)
private final long fileSizeInBytes;

@JsonProperty(FIELD_FILE_COUNT)
private final long fileCount;

@JsonProperty(FIELD_LAST_FILE_CREATION_TIME)
private final long lastFileCreationTime;

@JsonCreator
public PartitionResponse(
@JsonProperty(FIELD_SPEC) Map<String, String> spec,
@JsonProperty(FIELD_RECORD_COUNT) long recordCount,
@JsonProperty(FIELD_FILE_SIZE_IN_BYTES) long fileSizeInBytes,
@JsonProperty(FIELD_FILE_COUNT) long fileCount,
@JsonProperty(FIELD_LAST_FILE_CREATION_TIME) long lastFileCreationTime) {
this.spec = spec;
this.recordCount = recordCount;
this.fileSizeInBytes = fileSizeInBytes;
this.fileCount = fileCount;
this.lastFileCreationTime = lastFileCreationTime;
}

@JsonGetter(FIELD_SPEC)
public Map<String, String> getSpec() {
return spec;
}

@JsonGetter(FIELD_RECORD_COUNT)
public long getRecordCount() {
return recordCount;
}

@JsonGetter(FIELD_FILE_SIZE_IN_BYTES)
public long getFileSizeInBytes() {
return fileSizeInBytes;
}

@JsonGetter(FIELD_FILE_COUNT)
public long getFileCount() {
return fileCount;
}

@JsonGetter(FIELD_LAST_FILE_CREATION_TIME)
public long getLastFileCreationTime() {
return lastFileCreationTime;
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,7 @@
import org.apache.paimon.rest.responses.ListDatabasesResponse;
import org.apache.paimon.rest.responses.ListPartitionsResponse;
import org.apache.paimon.rest.responses.ListTablesResponse;
import org.apache.paimon.rest.responses.PartitionResponse;
import org.apache.paimon.schema.Schema;
import org.apache.paimon.schema.SchemaChange;
import org.apache.paimon.types.DataField;
Expand Down Expand Up @@ -147,8 +148,7 @@ public static DropPartitionRequest dropPartitionRequest() {
public static ListPartitionsResponse listPartitionsResponse() {
Map<String, String> spec = new HashMap<>();
spec.put("f0", "1");
ListPartitionsResponse.Partition partition =
new ListPartitionsResponse.Partition(spec, 1, 1, 1, 1);
PartitionResponse partition = new PartitionResponse(spec, 1, 1, 1, 1);
return new ListPartitionsResponse(ImmutableList.of(partition));
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@

package org.apache.paimon.rest;

import org.apache.paimon.CoreOptions;
import org.apache.paimon.catalog.Catalog;
import org.apache.paimon.catalog.CatalogContext;
import org.apache.paimon.catalog.Database;
Expand All @@ -34,6 +35,7 @@
import org.apache.paimon.rest.responses.ListDatabasesResponse;
import org.apache.paimon.rest.responses.ListPartitionsResponse;
import org.apache.paimon.rest.responses.ListTablesResponse;
import org.apache.paimon.rest.responses.PartitionResponse;
import org.apache.paimon.rest.responses.SuccessResponse;
import org.apache.paimon.schema.SchemaChange;
import org.apache.paimon.table.Table;
Expand Down Expand Up @@ -472,7 +474,7 @@ public void testDropPartitionWhenTableNoExist() throws Exception {
@Test
public void testListPartitionsWhenMetastorePartitionedIsTrue() throws Exception {
Options options = mockInitOptions();
options = options.set(RESTCatalogOptions.METASTORE_PARTITIONED, true);
options = options.set(CoreOptions.METASTORE_PARTITIONED_TABLE, true);
mockConfig(warehouseStr);
RESTCatalog restCatalog = new RESTCatalog(CatalogContext.create(options));
RESTCatalog mockRestCatalog = spy(restCatalog);
Expand Down Expand Up @@ -506,8 +508,7 @@ public void convertToPartitionEntryTest() {
fields.add(new DataField(0, "a", DataTypes.INT()));
fields.add(new DataField(1, "b", DataTypes.STRING()));
RowType partitionRowType = new RowType(false, fields);
ListPartitionsResponse.Partition partition =
new ListPartitionsResponse.Partition(spec, 1, 1, 1, 1);
PartitionResponse partition = new PartitionResponse(spec, 1, 1, 1, 1);
PartitionEntry partitionEntry =
mockRestCatalog.convertToPartitionEntry(partition, partitionRowType);
InternalRowPartitionComputer partitionComputer =
Expand Down
12 changes: 4 additions & 8 deletions paimon-open-api/rest-catalog-open-api.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -471,7 +471,7 @@ paths:
content:
application/json:
schema:
$ref: '#/components/schemas/SuccessResponse'
$ref: '#/components/schemas/PartitionResponse'
"404":
description: Resource not found
content:
Expand Down Expand Up @@ -508,11 +508,7 @@ paths:
$ref: '#/components/schemas/DropPartitionRequest'
responses:
"200":
description: OK
content:
application/json:
schema:
$ref: '#/components/schemas/SuccessResponse'
description: Success, no content
"404":
description: Resource not found
content:
Expand Down Expand Up @@ -880,8 +876,8 @@ components:
partitions:
type: array
items:
$ref: '#/components/schemas/Partition'
Partition:
$ref: '#/components/schemas/PartitionResponse'
PartitionResponse:
type: object
properties:
spec:
Expand Down
Loading

0 comments on commit 1d8faff

Please sign in to comment.