Skip to content

Commit

Permalink
Hive: Refactor TestHiveCatalog tests to use the core CatalogTests
Browse files Browse the repository at this point in the history
  • Loading branch information
@nk1506
nk1506 committed Dec 12, 2023
1 parent 09b44bb commit 768d7c6
Show file tree
Hide file tree
Showing 10 changed files with 87 additions and 286 deletions.
1 change: 1 addition & 0 deletions build.gradle
View file
Original file line number Diff line number Diff line change
Expand Up @@ -723,6 +723,7 @@ project(':iceberg-hive-metastore') {
}

testImplementation project(path: ':iceberg-api', configuration: 'testArtifacts')
testImplementation project(path: ':iceberg-core', configuration: 'testArtifacts')
testImplementation libs.awaitility
}
}
Expand Down
8 changes: 8 additions & 0 deletions core/src/test/java/org/apache/iceberg/catalog/CatalogTests.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -164,6 +164,10 @@ protected boolean supportsNamesWithSlashes() {
return true;
}

protected boolean supportsNamesWithDot() {
return true;
}

@Test
public void testCreateNamespace() {
C catalog = catalog();
Expand Down Expand Up @@ -470,6 +474,8 @@ public void testNamespaceWithSlash() {

@Test
public void testNamespaceWithDot() {
Assumptions.assumeTrue(supportsNamesWithDot());

C catalog = catalog();

Namespace withDot = Namespace.of("new.db");
Expand Down Expand Up @@ -547,6 +553,8 @@ public void testTableNameWithSlash() {

@Test
public void testTableNameWithDot() {
Assumptions.assumeTrue(supportsNamesWithDot());

C catalog = catalog();

TableIdentifier ident = TableIdentifier.of("ns", "ta.ble");
Expand Down
16 changes: 15 additions & 1 deletion hive-metastore/src/main/java/org/apache/iceberg/hive/HiveCatalog.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -261,6 +261,17 @@ public void renameTable(TableIdentifier from, TableIdentifier originalTo) {
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
throw new RuntimeException("Interrupted in call to rename", e);
} catch (RuntimeException e) {
// in case of table already exists,
// Hive rename operation throws exception as
// java.lang.RuntimeException:InvalidOperationException(message:new table <> already exists)
if (e.getCause() instanceof InvalidOperationException
&& e.getCause().getMessage() != null
&& e.getCause().getMessage().contains(String.format("new table %s already exists", to))) {
throw new org.apache.iceberg.exceptions.AlreadyExistsException(
"Table already exists: %s", to);
}
throw new RuntimeException("Failed to rename " + from + " to " + to, e);
}
}

Expand Down Expand Up @@ -288,7 +299,7 @@ public void createNamespace(Namespace namespace, Map<String, String> meta) {

} catch (AlreadyExistsException e) {
throw new org.apache.iceberg.exceptions.AlreadyExistsException(
e, "Namespace '%s' already exists!", namespace);
e, "Namespace already exists: %s", namespace);

} catch (TException e) {
throw new RuntimeException(
Expand Down Expand Up @@ -500,6 +511,9 @@ protected String defaultWarehouseLocation(TableIdentifier tableIdentifier) {
return String.format("%s/%s", databaseData.getLocationUri(), tableIdentifier.name());
}

} catch (NoSuchObjectException e) {
throw new NoSuchNamespaceException(
e, "Namespace does not exist: %s", tableIdentifier.namespace().levels()[0]);
} catch (TException e) {
throw new RuntimeException(
String.format("Metastore operation failed for %s", tableIdentifier), e);
Expand Down
2 changes: 1 addition & 1 deletion hive-metastore/src/main/java/org/apache/iceberg/hive/HiveTableOperations.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -209,7 +209,7 @@ protected void doCommit(TableMetadata base, TableMetadata metadata) {
String baseMetadataLocation = base != null ? base.metadataFileLocation() : null;
if (!Objects.equals(baseMetadataLocation, metadataLocation)) {
throw new CommitFailedException(
"Base metadata location '%s' is not same as the current table metadata location '%s' for %s.%s",
"Cannot commit: Base metadata location '%s' is not same as the current table metadata location '%s' for %s.%s",
baseMetadataLocation, metadataLocation, database, tableName);
}

Expand Down
9 changes: 7 additions & 2 deletions hive-metastore/src/test/java/org/apache/iceberg/hive/HiveCreateReplaceTableTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@
import java.nio.file.Path;
import java.util.Collections;
import java.util.concurrent.TimeUnit;
import org.apache.hadoop.hive.metastore.api.Database;
import org.apache.iceberg.AppendFiles;
import org.apache.iceberg.CatalogProperties;
import org.apache.iceberg.CatalogUtil;
Expand All @@ -43,6 +44,7 @@
import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap;
import org.apache.iceberg.relocated.com.google.common.collect.Maps;
import org.apache.iceberg.types.Types;
import org.apache.thrift.TException;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.BeforeEach;
Expand All @@ -68,12 +70,12 @@ public class HiveCreateReplaceTableTest {

@RegisterExtension
private static final HiveMetastoreExtension HIVE_METASTORE_EXTENSION =
new HiveMetastoreExtension(DB_NAME, Collections.emptyMap());
new HiveMetastoreExtension(Collections.emptyMap());

private static HiveCatalog catalog;

@BeforeAll
public static void initCatalog() {
public static void initCatalog() throws TException {
catalog =
(HiveCatalog)
CatalogUtil.loadCatalog(
Expand All @@ -83,6 +85,9 @@ public static void initCatalog() {
CatalogProperties.CLIENT_POOL_CACHE_EVICTION_INTERVAL_MS,
String.valueOf(TimeUnit.SECONDS.toMillis(10))),
HIVE_METASTORE_EXTENSION.hiveConf());
String dbPath = HIVE_METASTORE_EXTENSION.metastore().getDatabasePath(DB_NAME);
Database db = new Database(DB_NAME, "description", dbPath, Maps.newHashMap());
HIVE_METASTORE_EXTENSION.metastoreClient().createDatabase(db);
}

@BeforeEach
Expand Down
9 changes: 1 addition & 8 deletions hive-metastore/src/test/java/org/apache/iceberg/hive/HiveMetastoreExtension.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,8 +21,6 @@
import java.util.Map;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.metastore.HiveMetaStoreClient;
import org.apache.hadoop.hive.metastore.api.Database;
import org.apache.iceberg.relocated.com.google.common.collect.Maps;
import org.junit.jupiter.api.extension.AfterAllCallback;
import org.junit.jupiter.api.extension.BeforeAllCallback;
import org.junit.jupiter.api.extension.ExtensionContext;
Expand All @@ -31,10 +29,8 @@ public class HiveMetastoreExtension implements BeforeAllCallback, AfterAllCallba
private HiveMetaStoreClient metastoreClient;
private TestHiveMetastore metastore;
private final Map<String, String> hiveConfOverride;
private final String databaseName;

public HiveMetastoreExtension(String databaseName, Map<String, String> hiveConfOverride) {
this.databaseName = databaseName;
public HiveMetastoreExtension(Map<String, String> hiveConfOverride) {
this.hiveConfOverride = hiveConfOverride;
}

Expand All @@ -50,9 +46,6 @@ public void beforeAll(ExtensionContext extensionContext) throws Exception {

metastore.start(hiveConfWithOverrides);
metastoreClient = new HiveMetaStoreClient(hiveConfWithOverrides);
String dbPath = metastore.getDatabasePath(databaseName);
Database db = new Database(databaseName, "description", dbPath, Maps.newHashMap());
metastoreClient.createDatabase(db);
}

@Override
Expand Down
10 changes: 8 additions & 2 deletions hive-metastore/src/test/java/org/apache/iceberg/hive/HiveTableBaseTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,14 +31,17 @@
import java.util.concurrent.TimeUnit;
import java.util.stream.Collectors;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hive.metastore.api.Database;
import org.apache.iceberg.CatalogProperties;
import org.apache.iceberg.CatalogUtil;
import org.apache.iceberg.PartitionSpec;
import org.apache.iceberg.Schema;
import org.apache.iceberg.TableMetadataParser;
import org.apache.iceberg.catalog.TableIdentifier;
import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap;
import org.apache.iceberg.relocated.com.google.common.collect.Maps;
import org.apache.iceberg.types.Types;
import org.apache.thrift.TException;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.BeforeEach;
Expand All @@ -52,7 +55,7 @@ public class HiveTableBaseTest {

@RegisterExtension
protected static final HiveMetastoreExtension HIVE_METASTORE_EXTENSION =
new HiveMetastoreExtension(DB_NAME, Collections.emptyMap());
new HiveMetastoreExtension(Collections.emptyMap());

protected static HiveCatalog catalog;

Expand All @@ -71,7 +74,7 @@ public class HiveTableBaseTest {
private Path tableLocation;

@BeforeAll
public static void initCatalog() {
public static void initCatalog() throws TException {
catalog =
(HiveCatalog)
CatalogUtil.loadCatalog(
Expand All @@ -81,6 +84,9 @@ public static void initCatalog() {
CatalogProperties.CLIENT_POOL_CACHE_EVICTION_INTERVAL_MS,
String.valueOf(TimeUnit.SECONDS.toMillis(10))),
HIVE_METASTORE_EXTENSION.hiveConf());
String dbPath = HIVE_METASTORE_EXTENSION.metastore().getDatabasePath(DB_NAME);
Database db = new Database(DB_NAME, "description", dbPath, Maps.newHashMap());
HIVE_METASTORE_EXTENSION.metastoreClient().createDatabase(db);
}

@BeforeEach
Expand Down
3 changes: 1 addition & 2 deletions hive-metastore/src/test/java/org/apache/iceberg/hive/TestCachedClientPool.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -40,11 +40,10 @@

public class TestCachedClientPool {
private static final long EVICTION_INTERVAL = TimeUnit.SECONDS.toMillis(10);
private static final String DB_NAME = "hivedb";

@RegisterExtension
private static final HiveMetastoreExtension HIVE_METASTORE_EXTENSION =
new HiveMetastoreExtension(DB_NAME, Collections.emptyMap());
new HiveMetastoreExtension(Collections.emptyMap());

@Test
public void testClientPoolCleaner() throws InterruptedException {
Expand Down
Loading

0 comments on commit 768d7c6

Please sign in to comment.