Skip to content

Commit

Permalink
[Enhancement](tvf)catalog tvf implements user permission checks and h…
Browse files Browse the repository at this point in the history 
…ides sensitive information (apache#41497) (apache#41604)

bp apache#41497 

before apache#21790
## Proposed changes
This PR unifies the duplicate parts of `catalog tvf` and `show
catalogs`, adds permission check when querying `catalog tvf`, and hides
sensitive information.
  • Loading branch information
@hubgeter
hubgeter authored Oct 10, 2024
1 parent 1db0aef commit 0fb42d3
Show file tree
Hide file tree
Showing 5 changed files with 162 additions and 41 deletions.
1 change: 1 addition & 0 deletions be/src/vec/exec/scan/vmeta_scanner.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -411,6 +411,7 @@ Status VMetaScanner::_build_catalogs_metadata_request(const TMetaScanRange& meta
// create TMetadataTableRequestParams
TMetadataTableRequestParams metadata_table_params;
metadata_table_params.__set_metadata_type(TMetadataType::CATALOGS);
metadata_table_params.__set_current_user_ident(_user_identity);

request->__set_metada_table_params(metadata_table_params);
return Status::OK();
Expand Down
85 changes: 50 additions & 35 deletions fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogMgr.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@
import org.apache.doris.analysis.DropCatalogStmt;
import org.apache.doris.analysis.ShowCatalogStmt;
import org.apache.doris.analysis.ShowCreateCatalogStmt;
import org.apache.doris.analysis.UserIdentity;
import org.apache.doris.catalog.DatabaseIf;
import org.apache.doris.catalog.Env;
import org.apache.doris.catalog.EnvFactory;
Expand Down Expand Up @@ -370,30 +371,27 @@ public ShowResultSet showCatalogs(ShowCatalogStmt showStmt, String currentCtlg)
matcher = PatternMatcherWrapper.createMysqlPattern(showStmt.getPattern(),
CaseSensibility.CATALOG.getCaseSensibility());
}
for (CatalogIf catalog : nameToCatalog.values()) {
if (Env.getCurrentEnv().getAccessManager()
.checkCtlPriv(ConnectContext.get(), catalog.getName(), PrivPredicate.SHOW)) {
String name = catalog.getName();
// Filter catalog name
if (matcher != null && !matcher.match(name)) {
continue;
}
List<String> row = Lists.newArrayList();
row.add(String.valueOf(catalog.getId()));
row.add(name);
row.add(catalog.getType());
if (name.equals(currentCtlg)) {
row.add("Yes");
} else {
row.add("No");
}
Map<String, String> props = catalog.getProperties();
String createTime = props.getOrDefault(ExternalCatalog.CREATE_TIME, FeConstants.null_string);
row.add(createTime);
row.add(TimeUtils.longToTimeString(catalog.getLastUpdateTime()));
row.add(catalog.getComment());
rows.add(row);
for (CatalogIf catalog : listCatalogsWithCheckPriv(ConnectContext.get().getCurrentUserIdentity())) {
String name = catalog.getName();
// Filter catalog name
if (matcher != null && !matcher.match(name)) {
continue;
}
List<String> row = Lists.newArrayList();
row.add(String.valueOf(catalog.getId()));
row.add(name);
row.add(catalog.getType());
if (name.equals(currentCtlg)) {
row.add("Yes");
} else {
row.add("No");
}
Map<String, String> props = catalog.getProperties();
String createTime = props.getOrDefault(ExternalCatalog.CREATE_TIME, FeConstants.null_string);
row.add(createTime);
row.add(TimeUtils.longToTimeString(catalog.getLastUpdateTime()));
row.add(catalog.getComment());
rows.add(row);

// sort by catalog name
rows.sort((x, y) -> {
Expand All @@ -413,18 +411,8 @@ public ShowResultSet showCatalogs(ShowCatalogStmt showStmt, String currentCtlg)
if (!Strings.isNullOrEmpty(catalog.getResource())) {
rows.add(Arrays.asList("resource", catalog.getResource()));
}
// use tree map to maintain display order, making it easier to view properties
Map<String, String> sortedMap = new TreeMap<>(catalog.getProperties()).descendingMap();
for (Map.Entry<String, String> elem : sortedMap.entrySet()) {
if (PrintableMap.HIDDEN_KEY.contains(elem.getKey())) {
continue;
}
if (PrintableMap.SENSITIVE_KEY.contains(elem.getKey())) {
rows.add(Arrays.asList(elem.getKey(), PrintableMap.PASSWORD_MASK));
} else {
rows.add(Arrays.asList(elem.getKey(), elem.getValue()));
}
}
Map<String, String> sortedMap = getCatalogPropertiesWithPrintable(catalog);
sortedMap.forEach((k, v) -> rows.add(Arrays.asList(k, v)));
}
} finally {
readUnlock();
Expand All @@ -433,6 +421,25 @@ public ShowResultSet showCatalogs(ShowCatalogStmt showStmt, String currentCtlg)
return new ShowResultSet(showStmt.getMetaData(), rows);
}

public static Map<String, String> getCatalogPropertiesWithPrintable(CatalogIf<?> catalog) {
// use tree map to maintain display order, making it easier to view properties
Map<String, String> sortedMap = new TreeMap<>();
catalog.getProperties().forEach(
(key, value) -> {
if (PrintableMap.HIDDEN_KEY.contains(key)) {
return;
}
if (PrintableMap.SENSITIVE_KEY.contains(key)) {
sortedMap.put(key, PrintableMap.PASSWORD_MASK);
} else {
sortedMap.put(key, value);
}
}
);
return sortedMap;
}


public ShowResultSet showCreateCatalog(ShowCreateCatalogStmt showStmt) throws AnalysisException {
List<List<String>> rows = Lists.newArrayList();
readLock();
Expand Down Expand Up @@ -537,6 +544,14 @@ public List<CatalogIf> listCatalogs() {
return nameToCatalog.values().stream().collect(Collectors.toList());
}

public List<CatalogIf> listCatalogsWithCheckPriv(UserIdentity userIdentity) {
return nameToCatalog.values().stream().filter(
catalog -> Env.getCurrentEnv().getAccessManager()
.checkCtlPriv(userIdentity, catalog.getName(), PrivPredicate.SHOW)
).collect(Collectors.toList());
}


/**
* Reply for alter catalog props event.
*/
Expand Down
9 changes: 5 additions & 4 deletions fe/fe-core/src/main/java/org/apache/doris/tablefunction/MetadataGenerator.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,7 @@
import org.apache.doris.common.util.TimeUtils;
import org.apache.doris.common.util.Util;
import org.apache.doris.datasource.CatalogIf;
import org.apache.doris.datasource.CatalogMgr;
import org.apache.doris.datasource.ExternalCatalog;
import org.apache.doris.datasource.ExternalMetaCacheMgr;
import org.apache.doris.datasource.InternalCatalog;
Expand Down Expand Up @@ -513,17 +514,17 @@ private static TFetchSchemaTableDataResult frontendsDisksMetadataResult(TMetadat

private static TFetchSchemaTableDataResult catalogsMetadataResult(TMetadataTableRequestParams params) {
TFetchSchemaTableDataResult result = new TFetchSchemaTableDataResult();
List<CatalogIf> info = Env.getCurrentEnv().getCatalogMgr().listCatalogs();
List<TRow> dataBatch = Lists.newArrayList();

UserIdentity currentUserIdentity = UserIdentity.fromThrift(params.getCurrentUserIdent());
List<CatalogIf> info = Env.getCurrentEnv().getCatalogMgr().listCatalogsWithCheckPriv(currentUserIdentity);
List<TRow> dataBatch = Lists.newArrayList();
for (CatalogIf catalog : info) {
TRow trow = new TRow();
trow.addToColumnValue(new TCell().setLongVal(catalog.getId()));
trow.addToColumnValue(new TCell().setStringVal(catalog.getName()));
trow.addToColumnValue(new TCell().setStringVal(catalog.getType()));

Map<String, String> properties = catalog.getProperties();

Map<String, String> properties = CatalogMgr.getCatalogPropertiesWithPrintable(catalog);
for (Map.Entry<String, String> entry : properties.entrySet()) {
TRow subTrow = new TRow(trow);
subTrow.addToColumnValue(new TCell().setStringVal(entry.getKey()));
Expand Down
40 changes: 40 additions & 0 deletions regression-test/data/external_table_p0/tvf/test_catalogs_tvf.out
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,3 +8,43 @@ catalog_test_hive00 hms type hms
-- !create --
catalog_test_es00 es type es

-- !test_10 --
catalog_tvf_test_dlf hms dlf.catalog.id 987654321

-- !test_11 --
catalog_tvf_test_dlf hms dlf.secret_key *XXX

-- !test_12 --
catalog_tvf_test_dlf hms dlf.access_key AAAAAAAAAAAAAAAAAAAAAA

-- !test_13 --
catalog_tvf_test_dlf hms dlf.uid 123456789

-- !test_14 --
catalog_tvf_test_dlf hms type hms

-- !test_15 --

-- !test_16 --
internal internal NULL NULL

-- !test_17 --
catalog_tvf_test_dlf hms dlf.secret_key *XXX

-- !test_18 --
catalog_tvf_test_dlf hms dlf.access_key AAAAAAAAAAAAAAAAAAAAAA

-- !test_19 --
catalog_tvf_test_dlf hms dlf.uid 123456789

-- !test_20 --
catalog_tvf_test_dlf hms type hms

-- !test_21 --

-- !test_22 --

-- !test_23 --

-- !test_24 --

68 changes: 66 additions & 2 deletions regression-test/suites/external_table_p0/tvf/test_catalogs_tvf.groovy
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,12 +18,12 @@
suite("test_catalogs_tvf","p0,external,tvf,external_docker") {
List<List<Object>> table = sql """ select * from catalogs(); """
assertTrue(table.size() > 0)
assertEquals(5, table[0].size)
assertEquals(5, table[0].size())


table = sql """ select CatalogId,CatalogName from catalogs();"""
assertTrue(table.size() > 0)
assertTrue(table[0].size == 2)
assertTrue(table[0].size() == 2)


table = sql """ select * from catalogs() where CatalogId=0;"""
Expand Down Expand Up @@ -76,4 +76,68 @@ suite("test_catalogs_tvf","p0,external,tvf,external_docker") {
// check exception
exception "catalogs table-valued-function does not support any params"
}

sql """ drop catalog if exists catalog_tvf_test_dlf """

sql """
CREATE CATALOG catalog_tvf_test_dlf PROPERTIES (
"type"="hms",
"hive.metastore.type" = "dlf",
"dlf.proxy.mode" = "DLF_ONLY",
"dlf.endpoint" = "dlf-vpc.cn-beijing.aliyuncs.com",
"dlf.region" = "cn-beijing",
"dlf.uid" = "123456789",
"dlf.catalog.id" = "987654321",
"dlf.access_key" = "AAAAAAAAAAAAAAAAAAAAAA",
"dlf.secret_key" = "BBBBBBBBBBBBBBBBBBBBBB"
);"""

order_qt_test_10 """ select CatalogName,CatalogType,Property,Value from catalogs() where CatalogName = "catalog_tvf_test_dlf" and Property= "dlf.catalog.id" """
order_qt_test_11 """ select CatalogName,CatalogType,Property,Value from catalogs() where CatalogName = "catalog_tvf_test_dlf" and Property= "dlf.secret_key" """
order_qt_test_12 """ select CatalogName,CatalogType,Property,Value from catalogs() where CatalogName = "catalog_tvf_test_dlf" and Property= "dlf.access_key" """
order_qt_test_13 """ select CatalogName,CatalogType,Property,Value from catalogs() where CatalogName = "catalog_tvf_test_dlf" and Property= "dlf.uid" """
order_qt_test_14 """ select CatalogName,CatalogType,Property,Value from catalogs() where CatalogName = "catalog_tvf_test_dlf" and Property= "type" """


def user = 'catalog_user_test'
def pwd = 'C123_567p'
try_sql("DROP USER ${user}")

sql """CREATE USER '${user}' IDENTIFIED BY '${pwd}'"""
sql """GRANT SELECT_PRIV on `internal`.``.`` to '${user}'"""



connect(user=user, password="${pwd}", url=context.config.jdbcUrl) {
sql """ switch internal """
order_qt_test_15 """ select CatalogName,CatalogType,Property,Value from catalogs() where CatalogName = "catalog_tvf_test_dlf" and Property= "type" """
order_qt_test_16 """ select CatalogName,CatalogType,Property,Value from catalogs() """
}

sql """GRANT SELECT_PRIV on `catalog_tvf_test_dlf`.``.`` to '${user}'"""


connect(user=user, password="${pwd}", url=context.config.jdbcUrl) {
sql """ switch internal """

order_qt_test_17 """ select CatalogName,CatalogType,Property,Value from catalogs() where CatalogName = "catalog_tvf_test_dlf" and Property= "dlf.secret_key" """
order_qt_test_18 """ select CatalogName,CatalogType,Property,Value from catalogs() where CatalogName = "catalog_tvf_test_dlf" and Property= "dlf.access_key" """
order_qt_test_19 """ select CatalogName,CatalogType,Property,Value from catalogs() where CatalogName = "catalog_tvf_test_dlf" and Property= "dlf.uid" """
order_qt_test_20 """ select CatalogName,CatalogType,Property,Value from catalogs() where CatalogName = "catalog_tvf_test_dlf" and Property= "type" """
}

sql """REVOKE SELECT_PRIV on `catalog_tvf_test_dlf`.``.`` FROM '${user}'"""


connect(user=user, password="${pwd}", url=context.config.jdbcUrl) {
sql """ switch internal """

order_qt_test_21 """ select CatalogName,CatalogType,Property,Value from catalogs() where CatalogName = "catalog_tvf_test_dlf" and Property= "dlf.secret_key" """
order_qt_test_22 """ select CatalogName,CatalogType,Property,Value from catalogs() where CatalogName = "catalog_tvf_test_dlf" and Property= "dlf.access_key" """
order_qt_test_23 """ select CatalogName,CatalogType,Property,Value from catalogs() where CatalogName = "catalog_tvf_test_dlf" and Property= "dlf.uid" """
order_qt_test_24 """ select CatalogName,CatalogType,Property,Value from catalogs() where CatalogName = "catalog_tvf_test_dlf" and Property= "type" """
}



}

0 comments on commit 0fb42d3

Please sign in to comment.