Skip to content

Commit

Permalink
delete no need code and fix ut fail
Browse files Browse the repository at this point in the history
  • Loading branch information
@jerry-024
jerry-024 committed Dec 24, 2024
1 parent 29f96b8 commit 1706b6e
Show file tree
Hide file tree
Showing 4 changed files with 64 additions and 34 deletions.
46 changes: 24 additions & 22 deletions paimon-core/src/main/java/org/apache/paimon/rest/RESTCatalog.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -66,8 +66,9 @@
import org.apache.paimon.shade.guava30.com.google.common.collect.ImmutableList;
import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.databind.ObjectMapper;

import java.io.IOException;
import java.io.UncheckedIOException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.time.Duration;
import java.util.ArrayList;
import java.util.HashMap;
Expand All @@ -90,14 +91,15 @@
/** A catalog implementation for REST. */
public class RESTCatalog implements Catalog {

private static final Logger LOG = LoggerFactory.getLogger(RESTCatalog.class);
private static final ObjectMapper OBJECT_MAPPER = RESTObjectMapper.create();

private final RESTClient client;
private final ResourcePaths resourcePaths;
private final Map<String, String> baseHeader;
private final AuthSession catalogAuth;
private final CatalogContext context;
private final FileIO fileIO;
private final Optional<FileIO> fileIOOptional;

private volatile ScheduledExecutorService refreshExecutor = null;

Expand Down Expand Up @@ -142,23 +144,21 @@ public RESTCatalog(CatalogContext catalogContext) {
options, catalogContext.preferIO(), catalogContext.fallbackIO());
this.resourcePaths =
ResourcePaths.forCatalogProperties(options.get(RESTCatalogInternalOptions.PREFIX));
this.fileIO = getFileIOFromOptions(context);
this.fileIOOptional = getFileIOFromOptions(context);
}

// todo: whether it's ok
private static FileIO getFileIOFromOptions(CatalogContext context) {
Options options = context.options();
String warehouseStr = options.get(CatalogOptions.WAREHOUSE);
Path warehousePath = new Path(warehouseStr);
FileIO fileIO;
CatalogContext contextWithNewOptions =
CatalogContext.create(options, context.preferIO(), context.fallbackIO());
private static Optional<FileIO> getFileIOFromOptions(CatalogContext context) {
try {
fileIO = FileIO.get(warehousePath, contextWithNewOptions);
} catch (IOException e) {
throw new UncheckedIOException(e);
Options options = context.options();
String warehouseStr = options.get(CatalogOptions.WAREHOUSE);
Path warehousePath = new Path(warehouseStr);
CatalogContext contextWithNewOptions =
CatalogContext.create(options, context.preferIO(), context.fallbackIO());
return Optional.of(FileIO.get(warehousePath, contextWithNewOptions));
} catch (Exception ignore) {
LOG.warn("Can not get FileIO from options.");
}
return fileIO;
return Optional.empty();
}

@Override
Expand All @@ -173,7 +173,10 @@ public Map<String, String> options() {

@Override
public FileIO fileIO() {
return this.fileIO;
if (this.fileIOOptional.isPresent()) {
return this.fileIOOptional.get();
}
throw new RuntimeException("FileIO is not configured.");
}

@Override
Expand Down Expand Up @@ -391,7 +394,6 @@ Map<String, String> fetchOptionsFromServer(
return response.merge(clientProperties);
}

// todo: how know which exception to throw
@VisibleForTesting
void updateTable(Identifier fromTable, Identifier toTable, List<SchemaChange> changes) {
UpdateTableRequest request =
Expand All @@ -410,14 +412,14 @@ Table getDataOrFormatTable(Identifier identifier) throws TableNotExistException
String uuid = null;
FileStoreTable table =
FileStoreTableFactory.create(
fileIO,
fileIO(),
newTableLocation(warehouse(), identifier),
tableSchema,
new CatalogEnvironment(
identifier,
uuid,
Lock.factory(
lockFactory(context.options(), fileIO, Optional.empty())
lockFactory(context.options(), fileIO(), Optional.empty())
.orElse(null),
lockContext(context.options()).orElse(null),
identifier),
Expand All @@ -430,7 +432,7 @@ Table getDataOrFormatTable(Identifier identifier) throws TableNotExistException
ObjectTable.builder()
.underlyingTable(table)
.objectLocation(objectLocation)
.objectFileIO(this.fileIO)
.objectFileIO(this.fileIO())
.build();
}
return table;
Expand Down Expand Up @@ -483,7 +485,7 @@ private Table getAllInSystemDatabase(Identifier identifier) throws TableNotExist
};
Table table =
SystemTableLoader.loadGlobal(
tableName, fileIO, getAllTablePathsFunction, context.options());
tableName, fileIO(), getAllTablePathsFunction, context.options());
if (table == null) {
throw new TableNotExistException(identifier);
}
Expand Down
6 changes: 0 additions & 6 deletions paimon-core/src/main/java/org/apache/paimon/rest/RESTCatalogFactory.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,8 +21,6 @@
import org.apache.paimon.catalog.Catalog;
import org.apache.paimon.catalog.CatalogContext;
import org.apache.paimon.catalog.CatalogFactory;
import org.apache.paimon.options.CatalogOptions;
import org.apache.paimon.options.Options;

/** Factory to create {@link RESTCatalog}. */
public class RESTCatalogFactory implements CatalogFactory {
Expand All @@ -35,10 +33,6 @@ public String identifier() {

@Override
public Catalog create(CatalogContext context) {
Options options = context.options();
if (options.getOptional(CatalogOptions.WAREHOUSE).isPresent()) {
throw new IllegalArgumentException("Can not config warehouse in RESTCatalog.");
}
return new RESTCatalog(context);
}
}
34 changes: 34 additions & 0 deletions paimon-core/src/main/java/org/apache/paimon/rest/requests/SchemaChanges.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,7 @@
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Objects;

/** Schema changes to serialize List of SchemaChange . */
@JsonIgnoreProperties(ignoreUnknown = true)
Expand Down Expand Up @@ -201,4 +202,37 @@ public List<SchemaChange.UpdateColumnComment> getUpdateColumnComments() {
public List<SchemaChange.Move> getUpdateColumnPositions() {
return updateColumnPositions;
}

@Override
public boolean equals(Object o) {
if (o == null || getClass() != o.getClass()) {
return false;
}
SchemaChanges that = (SchemaChanges) o;
return Objects.equals(setOptions, that.setOptions)
&& Objects.equals(removeOptions, that.removeOptions)
&& Objects.equals(comment, that.comment)
&& Objects.equals(addColumns, that.addColumns)
&& Objects.equals(renameColumns, that.renameColumns)
&& Objects.equals(dropColumns, that.dropColumns)
&& Objects.equals(updateColumnTypes, that.updateColumnTypes)
&& Objects.equals(updateColumnNullabilities, that.updateColumnNullabilities)
&& Objects.equals(updateColumnComments, that.updateColumnComments)
&& Objects.equals(updateColumnPositions, that.updateColumnPositions);
}

@Override
public int hashCode() {
return Objects.hash(
setOptions,
removeOptions,
comment,
addColumns,
renameColumns,
dropColumns,
updateColumnTypes,
updateColumnNullabilities,
updateColumnComments,
updateColumnPositions);
}
}
12 changes: 6 additions & 6 deletions paimon-core/src/main/java/org/apache/paimon/schema/SchemaChange.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -232,14 +232,14 @@ final class AddColumn implements SchemaChange {
private static final long serialVersionUID = 1L;

private static final String FIELD_FILED_NAMES = "field-names";
private static final String FIELD_DATA_TYPES = "data-types";
private static final String FIELD_DATA_TYPE = "data-type";
private static final String FIELD_COMMENT = "comment";
private static final String FIELD_MOVE = "move";

@JsonProperty(FIELD_FILED_NAMES)
private final String[] fieldNames;

@JsonProperty(FIELD_DATA_TYPES)
@JsonProperty(FIELD_DATA_TYPE)
private final DataType dataType;

@JsonProperty(FIELD_COMMENT)
Expand All @@ -251,7 +251,7 @@ final class AddColumn implements SchemaChange {
@JsonCreator
private AddColumn(
@JsonProperty(FIELD_FILED_NAMES) String[] fieldNames,
@JsonProperty(FIELD_DATA_TYPES) DataType dataType,
@JsonProperty(FIELD_DATA_TYPE) DataType dataType,
@JsonProperty(FIELD_COMMENT) String description,
@JsonProperty(FIELD_MOVE) Move move) {
this.fieldNames = fieldNames;
Expand Down Expand Up @@ -283,7 +283,7 @@ public String[] getFieldNames() {
return fieldNames;
}

@JsonGetter(FIELD_DATA_TYPES)
@JsonGetter(FIELD_DATA_TYPE)
public DataType getDataType() {
return dataType;
}
Expand Down Expand Up @@ -312,7 +312,7 @@ public boolean equals(Object o) {
return Arrays.equals(fieldNames, addColumn.fieldNames)
&& dataType.equals(addColumn.dataType)
&& Objects.equals(description, addColumn.description)
&& move.equals(addColumn.move);
&& Objects.equals(move, addColumn.move);
}

@Override
Expand Down Expand Up @@ -435,7 +435,7 @@ final class UpdateColumnType implements SchemaChange {

private static final long serialVersionUID = 1L;
private static final String FIELD_FILED_NAMES = "field-names";
private static final String FIELD_NEW_DATA_TYPE = "new-data-types";
private static final String FIELD_NEW_DATA_TYPE = "new-data-type";
private static final String FIELD_KEEP_NULLABILITY = "keep-nullability";

@JsonProperty(FIELD_FILED_NAMES)
Expand Down

0 comments on commit 1706b6e

Please sign in to comment.