Skip to content

Commit

Permalink
[core] Support alter table API (apache#4775)
Browse files Browse the repository at this point in the history
  • Loading branch information
@jerry-024
jerry-024 authored Dec 26, 2024
1 parent 49a5bd2 commit 8be4b1f
Show file tree
Hide file tree
Showing 9 changed files with 510 additions and 35 deletions.
34 changes: 22 additions & 12 deletions paimon-core/src/main/java/org/apache/paimon/rest/RESTCatalog.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,7 @@
import org.apache.paimon.rest.exceptions.ForbiddenException;
import org.apache.paimon.rest.exceptions.NoSuchResourceException;
import org.apache.paimon.rest.requests.AlterDatabaseRequest;
import org.apache.paimon.rest.requests.AlterTableRequest;
import org.apache.paimon.rest.requests.CreateDatabaseRequest;
import org.apache.paimon.rest.requests.CreateTableRequest;
import org.apache.paimon.rest.requests.RenameTableRequest;
Expand Down Expand Up @@ -302,7 +303,13 @@ public void createTable(Identifier identifier, Schema schema, boolean ignoreIfEx
public void renameTable(Identifier fromTable, Identifier toTable, boolean ignoreIfNotExists)
throws TableNotExistException, TableAlreadyExistException {
try {
renameTable(fromTable, toTable);
RenameTableRequest request = new RenameTableRequest(toTable);
client.post(
resourcePaths.renameTable(
fromTable.getDatabaseName(), fromTable.getTableName()),
request,
GetTableResponse.class,
headers());
} catch (NoSuchResourceException e) {
if (!ignoreIfNotExists) {
throw new TableNotExistException(fromTable);
Expand All @@ -318,7 +325,20 @@ public void renameTable(Identifier fromTable, Identifier toTable, boolean ignore
public void alterTable(
Identifier identifier, List<SchemaChange> changes, boolean ignoreIfNotExists)
throws TableNotExistException, ColumnAlreadyExistException, ColumnNotExistException {
throw new UnsupportedOperationException("TODO");
try {
AlterTableRequest request = new AlterTableRequest(changes);
client.post(
resourcePaths.table(identifier.getDatabaseName(), identifier.getTableName()),
request,
GetTableResponse.class,
headers());
} catch (NoSuchResourceException e) {
if (!ignoreIfNotExists) {
throw new TableNotExistException(identifier);
}
} catch (ForbiddenException e) {
throw new TableNoPermissionException(identifier, e);
}
}

@Override
Expand Down Expand Up @@ -376,16 +396,6 @@ Map<String, String> fetchOptionsFromServer(
return response.merge(clientProperties);
}

@VisibleForTesting
void renameTable(Identifier fromTable, Identifier newIdentifier) {
RenameTableRequest request = new RenameTableRequest(newIdentifier);
client.post(
resourcePaths.table(fromTable.getDatabaseName(), fromTable.getTableName()),
request,
GetTableResponse.class,
headers());
}

@VisibleForTesting
Table getDataOrFormatTable(Identifier identifier) throws TableNotExistException {
Preconditions.checkArgument(identifier.getSystemTableName() == null);
Expand Down
11 changes: 11 additions & 0 deletions paimon-core/src/main/java/org/apache/paimon/rest/ResourcePaths.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -71,4 +71,15 @@ public String table(String databaseName, String tableName) {
.add(tableName)
.toString();
}

public String renameTable(String databaseName, String tableName) {
return SLASH.add("v1")
.add(prefix)
.add("databases")
.add(databaseName)
.add("tables")
.add(tableName)
.add("rename")
.toString();
}
}
49 changes: 49 additions & 0 deletions paimon-core/src/main/java/org/apache/paimon/rest/requests/AlterTableRequest.java
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,49 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

package org.apache.paimon.rest.requests;

import org.apache.paimon.rest.RESTRequest;
import org.apache.paimon.schema.SchemaChange;

import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonCreator;
import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonGetter;
import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonIgnoreProperties;
import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonProperty;

import java.util.List;

/** Request for altering table. */
@JsonIgnoreProperties(ignoreUnknown = true)
public class AlterTableRequest implements RESTRequest {

private static final String FIELD_NEW_UPDATE = "changes";

@JsonProperty(FIELD_NEW_UPDATE)
private final List<SchemaChange> changes;

@JsonCreator
public AlterTableRequest(@JsonProperty(FIELD_NEW_UPDATE) List<SchemaChange> changes) {
this.changes = changes;
}

@JsonGetter(FIELD_NEW_UPDATE)
public List<SchemaChange> getChanges() {
return changes;
}
}
93 changes: 85 additions & 8 deletions paimon-core/src/main/java/org/apache/paimon/schema/SchemaChange.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,8 @@
import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonGetter;
import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonIgnoreProperties;
import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonProperty;
import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonSubTypes;
import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonTypeInfo;

import javax.annotation.Nullable;

Expand All @@ -38,6 +40,42 @@
* @since 0.4.0
*/
@Public
@JsonTypeInfo(
use = JsonTypeInfo.Id.NAME,
include = JsonTypeInfo.As.PROPERTY,
property = SchemaChange.Actions.FIELD_ACTION)
@JsonSubTypes({
@JsonSubTypes.Type(
value = SchemaChange.SetOption.class,
name = SchemaChange.Actions.SET_OPTION_ACTION),
@JsonSubTypes.Type(
value = SchemaChange.RemoveOption.class,
name = SchemaChange.Actions.REMOVE_OPTION_ACTION),
@JsonSubTypes.Type(
value = SchemaChange.UpdateComment.class,
name = SchemaChange.Actions.UPDATE_COMMENT_ACTION),
@JsonSubTypes.Type(
value = SchemaChange.AddColumn.class,
name = SchemaChange.Actions.ADD_COLUMN_ACTION),
@JsonSubTypes.Type(
value = SchemaChange.RenameColumn.class,
name = SchemaChange.Actions.RENAME_COLUMN_ACTION),
@JsonSubTypes.Type(
value = SchemaChange.DropColumn.class,
name = SchemaChange.Actions.DROP_COLUMN_ACTION),
@JsonSubTypes.Type(
value = SchemaChange.UpdateColumnType.class,
name = SchemaChange.Actions.UPDATE_COLUMN_TYPE_ACTION),
@JsonSubTypes.Type(
value = SchemaChange.UpdateColumnNullability.class,
name = SchemaChange.Actions.UPDATE_COLUMN_NULLABILITY_ACTION),
@JsonSubTypes.Type(
value = SchemaChange.UpdateColumnComment.class,
name = SchemaChange.Actions.UPDATE_COLUMN_COMMENT_ACTION),
@JsonSubTypes.Type(
value = SchemaChange.UpdateColumnPosition.class,
name = SchemaChange.Actions.UPDATE_COLUMN_POSITION_ACTION),
})
public interface SchemaChange extends Serializable {

static SchemaChange setOption(String key, String value) {
Expand Down Expand Up @@ -124,18 +162,28 @@ final class SetOption implements SchemaChange {

private static final long serialVersionUID = 1L;

private static final String FIELD_KEY = "key";
private static final String FIELD_VALUE = "value";

@JsonProperty(FIELD_KEY)
private final String key;

@JsonProperty(FIELD_VALUE)
private final String value;

private SetOption(String key, String value) {
@JsonCreator
private SetOption(
@JsonProperty(FIELD_KEY) String key, @JsonProperty(FIELD_VALUE) String value) {
this.key = key;
this.value = value;
}

@JsonGetter(FIELD_KEY)
public String key() {
return key;
}

@JsonGetter(FIELD_VALUE)
public String value() {
return value;
}
Expand Down Expand Up @@ -163,12 +211,16 @@ final class RemoveOption implements SchemaChange {

private static final long serialVersionUID = 1L;

private static final String FIELD_KEY = "key";

@JsonProperty(FIELD_KEY)
private final String key;

private RemoveOption(String key) {
private RemoveOption(@JsonProperty(FIELD_KEY) String key) {
this.key = key;
}

@JsonGetter(FIELD_KEY)
public String key() {
return key;
}
Expand Down Expand Up @@ -196,13 +248,17 @@ final class UpdateComment implements SchemaChange {

private static final long serialVersionUID = 1L;

private static final String FIELD_COMMENT = "comment";

// If comment is null, means to remove comment
@JsonProperty(FIELD_COMMENT)
private final @Nullable String comment;

private UpdateComment(@Nullable String comment) {
private UpdateComment(@JsonProperty(FIELD_COMMENT) @Nullable String comment) {
this.comment = comment;
}

@JsonGetter(FIELD_COMMENT)
public @Nullable String comment() {
return comment;
}
Expand Down Expand Up @@ -467,13 +523,17 @@ public int hashCode() {
final class UpdateColumnPosition implements SchemaChange {

private static final long serialVersionUID = 1L;
private static final String FIELD_MOVE = "move";

@JsonProperty(FIELD_MOVE)
private final Move move;

private UpdateColumnPosition(Move move) {
@JsonCreator
private UpdateColumnPosition(@JsonProperty(FIELD_MOVE) Move move) {
this.move = move;
}

@JsonGetter(FIELD_MOVE)
public Move move() {
return move;
}
Expand Down Expand Up @@ -525,11 +585,11 @@ public static Move last(String fieldName) {

private static final long serialVersionUID = 1L;

private static final String FIELD_FILED_NAMES = "fieldName";
private static final String FIELD_FILED_NAME = "fieldName";
private static final String FIELD_REFERENCE_FIELD_NAME = "referenceFieldName";
private static final String FIELD_TYPE = "type";

@JsonProperty(FIELD_FILED_NAMES)
@JsonProperty(FIELD_FILED_NAME)
private final String fieldName;

@JsonProperty(FIELD_REFERENCE_FIELD_NAME)
Expand All @@ -540,15 +600,15 @@ public static Move last(String fieldName) {

@JsonCreator
public Move(
@JsonProperty(FIELD_FILED_NAMES) String fieldName,
@JsonProperty(FIELD_FILED_NAME) String fieldName,
@JsonProperty(FIELD_REFERENCE_FIELD_NAME) String referenceFieldName,
@JsonProperty(FIELD_TYPE) MoveType type) {
this.fieldName = fieldName;
this.referenceFieldName = referenceFieldName;
this.type = type;
}

@JsonGetter(FIELD_FILED_NAMES)
@JsonGetter(FIELD_FILED_NAME)
public String fieldName() {
return fieldName;
}
Expand Down Expand Up @@ -690,4 +750,21 @@ public int hashCode() {
return result;
}
}

/** Actions for schema changes： identify for schema change. */
class Actions {
public static final String FIELD_ACTION = "action";
public static final String SET_OPTION_ACTION = "setOption";
public static final String REMOVE_OPTION_ACTION = "removeOption";
public static final String UPDATE_COMMENT_ACTION = "updateComment";
public static final String ADD_COLUMN_ACTION = "addColumn";
public static final String RENAME_COLUMN_ACTION = "renameColumn";
public static final String DROP_COLUMN_ACTION = "dropColumn";
public static final String UPDATE_COLUMN_TYPE_ACTION = "updateColumnType";
public static final String UPDATE_COLUMN_NULLABILITY_ACTION = "updateColumnNullability";
public static final String UPDATE_COLUMN_COMMENT_ACTION = "updateColumnComment";
public static final String UPDATE_COLUMN_POSITION_ACTION = "updateColumnPosition";

private Actions() {}
}
}
8 changes: 8 additions & 0 deletions paimon-core/src/test/java/org/apache/paimon/rest/MockRESTMessage.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@

import org.apache.paimon.catalog.Identifier;
import org.apache.paimon.rest.requests.AlterDatabaseRequest;
import org.apache.paimon.rest.requests.AlterTableRequest;
import org.apache.paimon.rest.requests.CreateDatabaseRequest;
import org.apache.paimon.rest.requests.CreateTableRequest;
import org.apache.paimon.rest.requests.RenameTableRequest;
Expand Down Expand Up @@ -126,11 +127,17 @@ public static RenameTableRequest renameRequest(String toTableName) {
return new RenameTableRequest(newIdentifier);
}

public static AlterTableRequest alterTableRequest() {
return new AlterTableRequest(getChanges());
}

public static List<SchemaChange> getChanges() {
// add option
SchemaChange addOption = SchemaChange.setOption("snapshot.time-retained", "2h");
// remove option
SchemaChange removeOption = SchemaChange.removeOption("compaction.max.file-num");
// update comment
SchemaChange updateComment = SchemaChange.updateComment(null);
// add column
SchemaChange addColumn =
SchemaChange.addColumn("col1_after", DataTypes.ARRAY(DataTypes.STRING()));
Expand Down Expand Up @@ -179,6 +186,7 @@ public static List<SchemaChange> getChanges() {
List<SchemaChange> schemaChanges = new ArrayList<>();
schemaChanges.add(addOption);
schemaChanges.add(removeOption);
schemaChanges.add(updateComment);
schemaChanges.add(addColumn);
schemaChanges.add(addColumnMap);
schemaChanges.add(addColumnRowType);
Expand Down
Loading

0 comments on commit 8be4b1f

Please sign in to comment.