Skip to content

Commit

Permalink
[hive] HiveCatalog.listViews method reduces the number of queries for…
Browse files Browse the repository at this point in the history 
… hms metadata. (#4501)
  • Loading branch information
@zhuangchong
zhuangchong authored Nov 12, 2024
1 parent 4ccda6f commit 9b281bc
Show file tree
Hide file tree
Showing 2 changed files with 29 additions and 41 deletions.
66 changes: 27 additions & 39 deletions paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/HiveCatalog.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -530,25 +530,13 @@ public List<String> listViews(String databaseName) throws DatabaseNotExistExcept
getDatabase(databaseName);

try {
List<String> tables = clients.run(client -> client.getAllTables(databaseName));
List<String> views = new ArrayList<>();
for (String tableName : tables) {
Table table;
try {
table = getHmsTable(Identifier.create(databaseName, tableName));
} catch (TableNotExistException e) {
continue;
}
if (isView(table)) {
views.add(tableName);
}
}
return views;
return clients.run(
client -> client.getTables(databaseName, "*", TableType.VIRTUAL_VIEW));
} catch (TException e) {
throw new RuntimeException("Failed to list all tables in database " + databaseName, e);
throw new RuntimeException("Failed to list views in database " + databaseName, e);
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
throw new RuntimeException("Interrupted in call to listTables " + databaseName, e);
throw new RuntimeException("Interrupted in call to getTables " + databaseName, e);
}
}

Expand All @@ -570,20 +558,7 @@ public void renameView(Identifier fromView, Identifier toView, boolean ignoreIfN
} catch (ViewNotExistException ignored) {
}

try {
String fromDB = fromView.getDatabaseName();
String fromViewName = fromView.getTableName();
Table table = clients.run(client -> client.getTable(fromDB, fromViewName));
table.setDbName(toView.getDatabaseName());
table.setTableName(toView.getTableName());
clients.execute(client -> client.alter_table(fromDB, fromViewName, table));
} catch (TException e) {
throw new RuntimeException("Failed to rename view " + fromView.getFullName(), e);
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
throw new RuntimeException(
"Interrupted in call to rename view " + fromView.getFullName(), e);
}
renameHiveTable(fromView, toView);
}

@Override
Expand Down Expand Up @@ -779,12 +754,7 @@ private Table createHiveFormatTable(
@Override
protected void renameTableImpl(Identifier fromTable, Identifier toTable) {
try {
String fromDB = fromTable.getDatabaseName();
String fromTableName = fromTable.getTableName();
Table table = clients.run(client -> client.getTable(fromDB, fromTableName));
table.setDbName(toTable.getDatabaseName());
table.setTableName(toTable.getTableName());
clients.execute(client -> client.alter_table(fromDB, fromTableName, table));
Table table = renameHiveTable(fromTable, toTable);

Path fromPath = getTableLocation(fromTable);
if (!new SchemaManager(fileIO, fromPath).listAllIds().isEmpty()) {
Expand Down Expand Up @@ -816,6 +786,24 @@ protected void renameTableImpl(Identifier fromTable, Identifier toTable) {
}
}

private Table renameHiveTable(Identifier fromTable, Identifier toTable) {
try {
String fromDB = fromTable.getDatabaseName();
String fromTableName = fromTable.getTableName();
Table table = clients.run(client -> client.getTable(fromDB, fromTableName));
table.setDbName(toTable.getDatabaseName());
table.setTableName(toTable.getTableName());
clients.execute(client -> client.alter_table(fromDB, fromTableName, table));

return table;
} catch (TException e) {
throw new RuntimeException("Failed to rename table " + fromTable.getFullName(), e);
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
throw new RuntimeException("Interrupted in call to renameTable", e);
}
}

@Override
protected void alterTableImpl(Identifier identifier, List<SchemaChange> changes)
throws TableNotExistException, ColumnAlreadyExistException, ColumnNotExistException {
Expand Down Expand Up @@ -865,7 +853,7 @@ public boolean syncAllProperties() {

@Override
public void repairCatalog() {
List<String> databases = null;
List<String> databases;
try {
databases = listDatabasesInFileSystem(new Path(warehouse));
} catch (IOException e) {
Expand Down Expand Up @@ -1002,8 +990,8 @@ private boolean isFormatTable(Table table) {
}
}

private static boolean isView(Table table) {
return TableType.valueOf(table.getTableType()) == TableType.VIRTUAL_VIEW;
public static boolean isView(Table table) {
return table != null && TableType.VIRTUAL_VIEW.name().equals(table.getTableType());
}

private Table newHmsTable(
Expand Down
4 changes: 2 additions & 2 deletions paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/HiveTableUtils.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,6 @@
import org.apache.paimon.types.RowType;
import org.apache.paimon.utils.Pair;

import org.apache.hadoop.hive.metastore.TableType;
import org.apache.hadoop.hive.metastore.api.FieldSchema;
import org.apache.hadoop.hive.metastore.api.SerDeInfo;
import org.apache.hadoop.hive.metastore.api.Table;
Expand All @@ -38,12 +37,13 @@

import static org.apache.hadoop.hive.serde.serdeConstants.FIELD_DELIM;
import static org.apache.paimon.catalog.Catalog.COMMENT_PROP;
import static org.apache.paimon.hive.HiveCatalog.isView;
import static org.apache.paimon.table.FormatTableOptions.FIELD_DELIMITER;

class HiveTableUtils {

public static FormatTable convertToFormatTable(Table hiveTable) {
if (TableType.valueOf(hiveTable.getTableType()) == TableType.VIRTUAL_VIEW) {
if (isView(hiveTable)) {
throw new UnsupportedOperationException("Hive view is not supported.");
}

Expand Down

0 comments on commit 9b281bc

Please sign in to comment.