Skip to content

Commit

Permalink
RESTCatalogTest remove inner api dependencies
Browse files Browse the repository at this point in the history
  • Loading branch information
@jerry-024
jerry-024 committed Dec 30, 2024
1 parent d84198d commit 67da134
Show file tree
Hide file tree
Showing 2 changed files with 46 additions and 86 deletions.
39 changes: 19 additions & 20 deletions paimon-core/src/main/java/org/apache/paimon/rest/RESTCatalog.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -155,20 +155,6 @@ public RESTCatalog(CatalogContext catalogContext) {
this.fileIO = getFileIOFromOptions(context);
}

private static FileIO getFileIOFromOptions(CatalogContext context) {
try {
Options options = context.options();
String warehouseStr = options.get(CatalogOptions.WAREHOUSE);
Path warehousePath = new Path(warehouseStr);
CatalogContext contextWithNewOptions =
CatalogContext.create(options, context.preferIO(), context.fallbackIO());
return FileIO.get(warehousePath, contextWithNewOptions);
} catch (IOException e) {
LOG.warn("Can not get FileIO from options.");
throw new RuntimeException(e);
}
}

@Override
public String warehouse() {
return context.options().get(CatalogOptions.WAREHOUSE);
Expand Down Expand Up @@ -435,7 +421,7 @@ protected Map<String, String> fetchOptionsFromServer(
return response.merge(clientProperties);
}

protected Table getDataOrFormatTable(Identifier identifier) throws TableNotExistException {
private Table getDataOrFormatTable(Identifier identifier) throws TableNotExistException {
Preconditions.checkArgument(identifier.getSystemTableName() == null);
GetTableResponse response = getTableResponse(identifier);
FileStoreTable table =
Expand All @@ -458,7 +444,7 @@ protected Table getDataOrFormatTable(Identifier identifier) throws TableNotExist
return table;
}

protected List<PartitionEntry> listPartitionsFromServer(Identifier identifier, RowType rowType)
private List<PartitionEntry> listPartitionsFromServer(Identifier identifier, RowType rowType)
throws TableNotExistException {
try {
ListPartitionsResponse response =
Expand All @@ -481,7 +467,7 @@ protected List<PartitionEntry> listPartitionsFromServer(Identifier identifier, R
}
}

protected void cleanPartitionsInFileSystem(Table table, Map<String, String> partitions) {
private void cleanPartitionsInFileSystem(Table table, Map<String, String> partitions) {
FileStoreTable fileStoreTable = (FileStoreTable) table;
try (FileStoreCommit commit =
fileStoreTable
Expand All @@ -493,8 +479,7 @@ protected void cleanPartitionsInFileSystem(Table table, Map<String, String> part
}
}

protected GetTableResponse getTableResponse(Identifier identifier)
throws TableNotExistException {
private GetTableResponse getTableResponse(Identifier identifier) throws TableNotExistException {
try {
return client.get(
resourcePaths.table(identifier.getDatabaseName(), identifier.getTableName()),
Expand All @@ -507,7 +492,7 @@ protected GetTableResponse getTableResponse(Identifier identifier)
}
}

protected boolean dropPartitionMetadata(Identifier identifier, Map<String, String> partitions)
private boolean dropPartitionMetadata(Identifier identifier, Map<String, String> partitions)
throws TableNoPermissionException, PartitionNotExistException {
try {
DropPartitionRequest request = new DropPartitionRequest(partitions);
Expand Down Expand Up @@ -565,4 +550,18 @@ private PartitionEntry convertToPartitionEntry(PartitionResponse partition, RowT
partition.getFileCount(),
partition.getLastFileCreationTime());
}

private static FileIO getFileIOFromOptions(CatalogContext context) {
try {
Options options = context.options();
String warehouseStr = options.get(CatalogOptions.WAREHOUSE);
Path warehousePath = new Path(warehouseStr);
CatalogContext contextWithNewOptions =
CatalogContext.create(options, context.preferIO(), context.fallbackIO());
return FileIO.get(warehousePath, contextWithNewOptions);
} catch (IOException e) {
LOG.warn("Can not get FileIO from options.");
throw new RuntimeException(e);
}
}
}
93 changes: 27 additions & 66 deletions paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -58,22 +58,13 @@
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertThrows;
import static org.junit.jupiter.api.Assertions.assertDoesNotThrow;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.anyBoolean;
import static org.mockito.ArgumentMatchers.anyList;
import static org.mockito.ArgumentMatchers.eq;
import static org.mockito.Mockito.doNothing;
import static org.mockito.Mockito.spy;
import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;

/** Test for REST Catalog. */
public class RESTCatalogTest {

private final ObjectMapper mapper = RESTObjectMapper.create();
private MockWebServer mockWebServer;
private RESTCatalog restCatalog;
private RESTCatalog mockRestCatalog;
private String warehouseStr;
private String serverUrl;
@Rule public TemporaryFolder folder = new TemporaryFolder();
Expand All @@ -87,7 +78,6 @@ public void setUp() throws IOException {
warehouseStr = folder.getRoot().getPath();
mockConfig(warehouseStr);
restCatalog = new RESTCatalog(CatalogContext.create(options));
mockRestCatalog = spy(restCatalog);
}

@After
Expand Down Expand Up @@ -148,9 +138,7 @@ public void testGetDatabase() throws Exception {
public void testDropDatabase() throws Exception {
String name = MockRESTMessage.databaseName();
mockResponse("", 200);
assertDoesNotThrow(() -> mockRestCatalog.dropDatabase(name, false, true));
verify(mockRestCatalog, times(1)).dropDatabase(eq(name), eq(false), eq(true));
verify(mockRestCatalog, times(0)).listTables(eq(name));
assertDoesNotThrow(() -> restCatalog.dropDatabase(name, false, true));
}

@Test
Expand All @@ -160,17 +148,15 @@ public void testDropDatabaseWhenNoExistAndIgnoreIfNotExistsIsFalse() throws Exce
mockResponse(mapper.writeValueAsString(response), 404);
assertThrows(
Catalog.DatabaseNotExistException.class,
() -> mockRestCatalog.dropDatabase(name, false, true));
() -> restCatalog.dropDatabase(name, false, true));
}

@Test
public void testDropDatabaseWhenNoExistAndIgnoreIfNotExistsIsTrue() throws Exception {
String name = MockRESTMessage.databaseName();
ErrorResponse response = MockRESTMessage.noSuchResourceExceptionErrorResponse();
mockResponse(mapper.writeValueAsString(response), 404);
assertDoesNotThrow(() -> mockRestCatalog.dropDatabase(name, true, true));
verify(mockRestCatalog, times(1)).dropDatabase(eq(name), eq(true), eq(true));
verify(mockRestCatalog, times(0)).listTables(eq(name));
assertDoesNotThrow(() -> restCatalog.dropDatabase(name, true, true));
}

@Test
Expand All @@ -180,9 +166,7 @@ public void testDropDatabaseWhenCascadeIsFalseAndNoTables() throws Exception {
ListTablesResponse response = MockRESTMessage.listTablesEmptyResponse();
mockResponse(mapper.writeValueAsString(response), 200);
mockResponse("", 200);
assertDoesNotThrow(() -> mockRestCatalog.dropDatabase(name, false, cascade));
verify(mockRestCatalog, times(1)).dropDatabase(eq(name), eq(false), eq(cascade));
verify(mockRestCatalog, times(1)).listTables(eq(name));
assertDoesNotThrow(() -> restCatalog.dropDatabase(name, false, cascade));
}

@Test
Expand All @@ -193,17 +177,15 @@ public void testDropDatabaseWhenCascadeIsFalseAndTablesExist() throws Exception
mockResponse(mapper.writeValueAsString(response), 200);
assertThrows(
Catalog.DatabaseNotEmptyException.class,
() -> mockRestCatalog.dropDatabase(name, false, cascade));
verify(mockRestCatalog, times(1)).dropDatabase(eq(name), eq(false), eq(cascade));
verify(mockRestCatalog, times(1)).listTables(eq(name));
() -> restCatalog.dropDatabase(name, false, cascade));
}

@Test
public void testAlterDatabase() throws Exception {
String name = MockRESTMessage.databaseName();
AlterDatabaseResponse response = MockRESTMessage.alterDatabaseResponse();
mockResponse(mapper.writeValueAsString(response), 200);
assertDoesNotThrow(() -> mockRestCatalog.alterDatabase(name, new ArrayList<>(), true));
assertDoesNotThrow(() -> restCatalog.alterDatabase(name, new ArrayList<>(), true));
}

@Test
Expand All @@ -214,15 +196,15 @@ public void testAlterDatabaseWhenDatabaseNotExistAndIgnoreIfNotExistsIsFalse()
mockResponse(mapper.writeValueAsString(response), 404);
assertThrows(
Catalog.DatabaseNotExistException.class,
() -> mockRestCatalog.alterDatabase(name, new ArrayList<>(), false));
() -> restCatalog.alterDatabase(name, new ArrayList<>(), false));
}

@Test
public void testAlterDatabaseWhenDatabaseNotExistAndIgnoreIfNotExistsIsTrue() throws Exception {
String name = MockRESTMessage.databaseName();
ErrorResponse response = MockRESTMessage.noSuchResourceExceptionErrorResponse();
mockResponse(mapper.writeValueAsString(response), 404);
assertDoesNotThrow(() -> mockRestCatalog.alterDatabase(name, new ArrayList<>(), true));
assertDoesNotThrow(() -> restCatalog.alterDatabase(name, new ArrayList<>(), true));
}

@Test
Expand All @@ -239,10 +221,8 @@ public void testGetTable() throws Exception {
String databaseName = MockRESTMessage.databaseName();
GetTableResponse response = MockRESTMessage.getTableResponse();
mockResponse(mapper.writeValueAsString(response), 200);
Table result = mockRestCatalog.getTable(Identifier.create(databaseName, "table"));
// catalog will add path option
Table result = restCatalog.getTable(Identifier.create(databaseName, "table"));
assertEquals(response.getSchema().options().size() + 1, result.options().size());
verify(mockRestCatalog, times(1)).getDataOrFormatTable(any());
}

@Test
Expand Down Expand Up @@ -272,11 +252,10 @@ public void testRenameTable() throws Exception {
mockResponse(mapper.writeValueAsString(response), 200);
assertDoesNotThrow(
() ->
mockRestCatalog.renameTable(
restCatalog.renameTable(
Identifier.create(databaseName, fromTableName),
Identifier.create(databaseName, toTableName),
true));
verify(mockRestCatalog, times(1)).renameTable(any(), any(), anyBoolean());
}

@Test
Expand All @@ -288,7 +267,7 @@ public void testRenameTableWhenTableNotExistAndIgnoreIfNotExistsIsFalse() throws
assertThrows(
Catalog.TableNotExistException.class,
() ->
mockRestCatalog.renameTable(
restCatalog.renameTable(
Identifier.create(databaseName, fromTableName),
Identifier.create(databaseName, toTableName),
false));
Expand All @@ -303,7 +282,7 @@ public void testRenameTableWhenToTableAlreadyExist() throws Exception {
assertThrows(
Catalog.TableAlreadyExistException.class,
() ->
mockRestCatalog.renameTable(
restCatalog.renameTable(
Identifier.create(databaseName, fromTableName),
Identifier.create(databaseName, toTableName),
false));
Expand All @@ -316,10 +295,7 @@ public void testAlterTable() throws Exception {
GetTableResponse response = MockRESTMessage.getTableResponse();
mockResponse(mapper.writeValueAsString(response), 200);
assertDoesNotThrow(
() ->
mockRestCatalog.alterTable(
Identifier.create(databaseName, "t1"), changes, true));
verify(mockRestCatalog, times(1)).alterTable(any(), anyList(), anyBoolean());
() -> restCatalog.alterTable(Identifier.create(databaseName, "t1"), changes, true));
}

@Test
Expand All @@ -330,7 +306,7 @@ public void testAlterTableWhenTableNotExistAndIgnoreIfNotExistsIsFalse() throws
assertThrows(
Catalog.TableNotExistException.class,
() ->
mockRestCatalog.alterTable(
restCatalog.alterTable(
Identifier.create(databaseName, "t1"), changes, false));
}

Expand Down Expand Up @@ -362,7 +338,7 @@ public void testCreatePartition() throws Exception {
mockResponse(mapper.writeValueAsString(response), 200);
assertDoesNotThrow(
() ->
mockRestCatalog.createPartition(
restCatalog.createPartition(
Identifier.create(databaseName, "table"), partitionSpec));
}

Expand All @@ -375,7 +351,7 @@ public void testCreatePartitionWhenTableNotExist() throws Exception {
assertThrows(
Catalog.TableNotExistException.class,
() ->
mockRestCatalog.createPartition(
restCatalog.createPartition(
Identifier.create(databaseName, "table"), partitionSpec));
}

Expand All @@ -388,7 +364,7 @@ public void testCreatePartitionWhenTableNoPermissionException() throws Exception
assertThrows(
Catalog.TableNoPermissionException.class,
() ->
mockRestCatalog.createPartition(
restCatalog.createPartition(
Identifier.create(databaseName, "table"), partitionSpec));
}

Expand All @@ -400,14 +376,11 @@ public void testDropPartition() throws Exception {
partitionSpec.put(response.getSchema().primaryKeys().get(0), "1");
mockResponse(mapper.writeValueAsString(""), 200);
mockResponse(mapper.writeValueAsString(response), 200);
doNothing().when(mockRestCatalog).cleanPartitionsInFileSystem(any(), any());
assertDoesNotThrow(
assertThrows(
RuntimeException.class,
() ->
mockRestCatalog.dropPartition(
restCatalog.dropPartition(
Identifier.create(databaseName, "table"), partitionSpec));
verify(mockRestCatalog, times(1)).dropPartitionMetadata(any(), any());
verify(mockRestCatalog, times(1)).getTable(any());
verify(mockRestCatalog, times(1)).cleanPartitionsInFileSystem(any(), any());
}

@Test
Expand All @@ -418,14 +391,11 @@ public void testDropPartitionWhenPartitionNoExist() throws Exception {
partitionSpec.put(response.getSchema().primaryKeys().get(0), "1");
mockResponse(mapper.writeValueAsString(""), 404);
mockResponse(mapper.writeValueAsString(response), 200);
doNothing().when(mockRestCatalog).cleanPartitionsInFileSystem(any(), any());
assertThrows(
Catalog.PartitionNotExistException.class,
() ->
mockRestCatalog.dropPartition(
restCatalog.dropPartition(
Identifier.create(databaseName, "table"), partitionSpec));
verify(mockRestCatalog, times(1)).dropPartitionMetadata(any(), any());
verify(mockRestCatalog, times(0)).cleanPartitionsInFileSystem(any(), any());
}

@Test
Expand All @@ -435,15 +405,11 @@ public void testDropPartitionWhenTableNoPermission() throws Exception {
GetTableResponse response = MockRESTMessage.getTableResponse();
partitionSpec.put(response.getSchema().primaryKeys().get(0), "1");
mockResponse(mapper.writeValueAsString(""), 403);
doNothing().when(mockRestCatalog).cleanPartitionsInFileSystem(any(), any());
assertThrows(
Catalog.TableNoPermissionException.class,
() ->
mockRestCatalog.dropPartition(
restCatalog.dropPartition(
Identifier.create(databaseName, "table"), partitionSpec));
verify(mockRestCatalog, times(1)).dropPartitionMetadata(any(), any());
verify(mockRestCatalog, times(0)).getTable(any());
verify(mockRestCatalog, times(0)).cleanPartitionsInFileSystem(any(), any());
}

@Test
Expand All @@ -454,15 +420,11 @@ public void testDropPartitionWhenTableNoExist() throws Exception {
partitionSpec.put(response.getSchema().primaryKeys().get(0), "1");
mockResponse(mapper.writeValueAsString(""), 200);
mockResponse("", 404);
doNothing().when(mockRestCatalog).cleanPartitionsInFileSystem(any(), any());
assertThrows(
Catalog.TableNotExistException.class,
() ->
mockRestCatalog.dropPartition(
restCatalog.dropPartition(
Identifier.create(databaseName, "table"), partitionSpec));
verify(mockRestCatalog, times(1)).dropPartitionMetadata(any(), any());
verify(mockRestCatalog, times(1)).getTable(any());
verify(mockRestCatalog, times(0)).cleanPartitionsInFileSystem(any(), any());
}

@Test
Expand All @@ -473,8 +435,7 @@ public void testListPartitionsWhenMetastorePartitionedIsTrue() throws Exception
ListPartitionsResponse response = MockRESTMessage.listPartitionsResponse();
mockResponse(mapper.writeValueAsString(response), 200);
List<PartitionEntry> result =
mockRestCatalog.listPartitions(Identifier.create(databaseName, "table"));
verify(mockRestCatalog, times(1)).listPartitionsFromServer(any(), any());
restCatalog.listPartitions(Identifier.create(databaseName, "table"));
assertEquals(response.getPartitions().size(), result.size());
}

Expand All @@ -484,9 +445,9 @@ public void testListPartitionsFromFile() throws Exception {
GetTableResponse response = MockRESTMessage.getTableResponse();
mockResponse(mapper.writeValueAsString(response), 200);
mockResponse(mapper.writeValueAsString(response), 200);
mockRestCatalog.listPartitions(Identifier.create(databaseName, "table"));
verify(mockRestCatalog, times(2)).getTable(any());
verify(mockRestCatalog, times(0)).listPartitionsFromServer(any(), any());
List<PartitionEntry> partitionEntries =
restCatalog.listPartitions(Identifier.create(databaseName, "table"));
assertEquals(partitionEntries.size(), 0);
}

private void mockResponse(String mockResponse, int httpCode) {
Expand Down

0 comments on commit 67da134

Please sign in to comment.