From 2b1264c5799cc11396800796e7d6bce8555b161b Mon Sep 17 00:00:00 2001 From: Rijesh Kunhi Parambattu <147430310+rijeshkp@users.noreply.github.com> Date: Mon, 9 Dec 2024 07:48:43 +0530 Subject: [PATCH] =?UTF-8?q?=20[Enhancement]=20(nereids)implement=20adminCh?= =?UTF-8?q?eckTabletsCommand=20in=20nereids=E2=80=A6=20(#44701)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Issue Number: close #42843 Support ADMIN CHECK tabletlist command in nereids --- .../org/apache/doris/nereids/DorisParser.g4 | 2 +- .../nereids/parser/LogicalPlanBuilder.java | 17 +++ .../doris/nereids/trees/plans/PlanType.java | 1 + .../commands/AdminCheckTabletsCommand.java | 109 ++++++++++++++++++ .../trees/plans/visitor/CommandVisitor.java | 5 + .../test_database_management_auth.groovy | 8 +- .../test_nereids_admin_check_tablet.groovy | 57 +++++++++ 7 files changed, 194 insertions(+), 5 deletions(-) create mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/AdminCheckTabletsCommand.java create mode 100644 regression-test/suites/nereids_p0/admin/test_nereids_admin_check_tablet.groovy diff --git a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 index f229a1b2f4a7b1..c3fe7bb145ba72 100644 --- a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 +++ b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 @@ -491,6 +491,7 @@ supportedAdminStatement | ADMIN DIAGNOSE TABLET tabletId=INTEGER_VALUE #adminDiagnoseTablet | ADMIN SHOW REPLICA STATUS FROM baseTableRef (WHERE STATUS EQ|NEQ STRING_LITERAL)? #adminShowReplicaStatus | ADMIN COMPACT TABLE baseTableRef (WHERE TYPE EQ STRING_LITERAL)? #adminCompactTable + | ADMIN CHECK tabletList properties=propertyClause? #adminCheckTablets ; supportedRecoverStatement @@ -508,7 +509,6 @@ unsupportedAdminStatement | ADMIN CANCEL REPAIR TABLE baseTableRef #adminCancelRepairTable | ADMIN SET (FRONTEND | (ALL FRONTENDS)) CONFIG (LEFT_PAREN propertyItemList RIGHT_PAREN)? ALL? #adminSetFrontendConfig - | ADMIN CHECK tabletList properties=propertyClause? #adminCheckTablets | ADMIN REBALANCE DISK (ON LEFT_PAREN backends+=STRING_LITERAL (COMMA backends+=STRING_LITERAL) RIGHT_PAREN)? #adminRebalanceDisk | ADMIN CANCEL REBALANCE DISK (ON LEFT_PAREN backends+=STRING_LITERAL diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java index 71ea972c4ab2a2..15aeca952a2556 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java @@ -50,6 +50,7 @@ import org.apache.doris.mtmv.MTMVRefreshTriggerInfo; import org.apache.doris.nereids.DorisParser; import org.apache.doris.nereids.DorisParser.AddConstraintContext; +import org.apache.doris.nereids.DorisParser.AdminCheckTabletsContext; import org.apache.doris.nereids.DorisParser.AdminCompactTableContext; import org.apache.doris.nereids.DorisParser.AdminDiagnoseTabletContext; import org.apache.doris.nereids.DorisParser.AdminShowReplicaDistributionContext; @@ -470,6 +471,7 @@ import org.apache.doris.nereids.trees.plans.algebra.Aggregate; import org.apache.doris.nereids.trees.plans.algebra.SetOperation.Qualifier; import org.apache.doris.nereids.trees.plans.commands.AddConstraintCommand; +import org.apache.doris.nereids.trees.plans.commands.AdminCheckTabletsCommand; import org.apache.doris.nereids.trees.plans.commands.AdminCompactTableCommand; import org.apache.doris.nereids.trees.plans.commands.AdminShowReplicaStatusCommand; import org.apache.doris.nereids.trees.plans.commands.AlterMTMVCommand; @@ -4950,4 +4952,19 @@ public LogicalPlan visitShowCollation(ShowCollationContext ctx) { } return new ShowCollationCommand(wild); } + + @Override + public LogicalPlan visitAdminCheckTablets(AdminCheckTabletsContext ctx) { + List tabletIdLists = new ArrayList<>(); + if (ctx.tabletList() != null) { + ctx.tabletList().tabletIdList.stream().forEach(tabletToken -> { + tabletIdLists.add(Long.parseLong(tabletToken.getText())); + }); + } + Map properties = ctx.properties != null + ? Maps.newHashMap(visitPropertyClause(ctx.properties)) + : Maps.newHashMap(); + return new AdminCheckTabletsCommand(tabletIdLists, properties); + } } + diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlanType.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlanType.java index bc8ccf83093b25..8198a8600a60c9 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlanType.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlanType.java @@ -132,6 +132,7 @@ public enum PlanType { PHYSICAL_WINDOW, // commands + ADMIN_CHECK_TABLETS_COMMAND, CREATE_POLICY_COMMAND, CREATE_TABLE_COMMAND, CREATE_SQL_BLOCK_RULE_COMMAND, diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/AdminCheckTabletsCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/AdminCheckTabletsCommand.java new file mode 100644 index 00000000000000..bba555d77a2e92 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/AdminCheckTabletsCommand.java @@ -0,0 +1,109 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.trees.plans.commands; + +import org.apache.doris.catalog.Env; +import org.apache.doris.common.AnalysisException; +import org.apache.doris.common.DdlException; +import org.apache.doris.common.ErrorCode; +import org.apache.doris.common.ErrorReport; +import org.apache.doris.common.util.PropertyAnalyzer; +import org.apache.doris.mysql.privilege.PrivPredicate; +import org.apache.doris.nereids.trees.plans.PlanType; +import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor; +import org.apache.doris.qe.ConnectContext; +import org.apache.doris.qe.StmtExecutor; + +import org.apache.commons.collections.CollectionUtils; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import java.util.List; +import java.util.Map; +import java.util.Objects; + +/** + * drop user command + */ +public class AdminCheckTabletsCommand extends Command implements ForwardNoSync { + private static final Logger LOG = LogManager.getLogger(AdminCheckTabletsCommand.class); + private final List tabletIds; + private final Map properties; + + /** + * Check Type in Properties. + * At present only CONSISTENCY is supported + */ + public enum CheckType { + CONSISTENCY; // check the consistency of replicas of tablet + public static CheckType getTypeFromString(String str) throws AnalysisException { + try { + return CheckType.valueOf(str.toUpperCase()); + } catch (Exception e) { + throw new AnalysisException("Unknown check type: " + str); + } + } + } + + /** + * constructor + */ + // ADMIN CHECK TABLET (id1, id2, ...) PROPERTIES ("type" = "check_consistency"); + public AdminCheckTabletsCommand(List tabletIds, Map properties) { + super(PlanType.ADMIN_CHECK_TABLETS_COMMAND); + this.tabletIds = tabletIds; + this.properties = properties; + } + + @Override + public void run(ConnectContext ctx, StmtExecutor executor) throws Exception { + // check auth + if (!Env.getCurrentEnv().getAccessManager().checkGlobalPriv(ConnectContext.get(), PrivPredicate.ADMIN)) { + ErrorReport.reportAnalysisException(ErrorCode.ERR_SPECIFIC_ACCESS_DENIED_ERROR, "ADMIN"); + } + + if (CollectionUtils.isEmpty(tabletIds)) { + throw new AnalysisException("Tablet id list is empty"); + } + + String typeStr = PropertyAnalyzer.analyzeType(properties); + if (typeStr == null) { + throw new AnalysisException("Should specify 'type' property"); + } + CheckType checkType = CheckType.getTypeFromString(typeStr); + + if (properties != null && !properties.isEmpty()) { + throw new AnalysisException("Unknown properties: " + properties.keySet()); + } + + if (Objects.requireNonNull(checkType) == CheckType.CONSISTENCY) { + Env.getCurrentEnv().getConsistencyChecker().addTabletsToCheck(tabletIds); + } + } + + @Override + protected void checkSupportedInCloudMode(ConnectContext ctx) throws DdlException { + LOG.info("AdminCheckTabletsCommand not supported in cloud mode"); + throw new DdlException("Unsupported operation"); + } + + @Override + public R accept(PlanVisitor visitor, C context) { + return visitor.visitAdminCheckTabletsCommand(this, context); + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/CommandVisitor.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/CommandVisitor.java index 77846ff3ad5328..dea3a004e816be 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/CommandVisitor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/CommandVisitor.java @@ -18,6 +18,7 @@ package org.apache.doris.nereids.trees.plans.visitor; import org.apache.doris.nereids.trees.plans.commands.AddConstraintCommand; +import org.apache.doris.nereids.trees.plans.commands.AdminCheckTabletsCommand; import org.apache.doris.nereids.trees.plans.commands.AdminCompactTableCommand; import org.apache.doris.nereids.trees.plans.commands.AdminShowReplicaStatusCommand; import org.apache.doris.nereids.trees.plans.commands.AlterJobStatusCommand; @@ -595,4 +596,8 @@ default R visitCreateRoutineLoadCommand(CreateRoutineLoadCommand createRoutineLo default R visitShowProcessListCommand(ShowProcessListCommand showProcessListCommand, C context) { return visitCommand(showProcessListCommand, context); } + + default R visitAdminCheckTabletsCommand(AdminCheckTabletsCommand adminCheckTabletsCommand, C context) { + return visitCommand(adminCheckTabletsCommand, context); + } } diff --git a/regression-test/suites/auth_call/test_database_management_auth.groovy b/regression-test/suites/auth_call/test_database_management_auth.groovy index 0e1805e20d6840..79114c18e789b5 100644 --- a/regression-test/suites/auth_call/test_database_management_auth.groovy +++ b/regression-test/suites/auth_call/test_database_management_auth.groovy @@ -23,14 +23,14 @@ suite("test_database_management_auth","p0,auth_call") { String user = 'test_database_management_auth_user' String pwd = 'C123_567p' String dbName = 'test_database_management_auth_db' - def String show_dis_error_msg = "denied" + def String error_in_cloud = "denied" //cloud-mode if (isCloudMode()) { def clusters = sql " SHOW CLUSTERS; " assertTrue(!clusters.isEmpty()) def validCluster = clusters[0][0] sql """GRANT USAGE_PRIV ON CLUSTER ${validCluster} TO ${user}"""; - show_dis_error_msg = "Unsupported" + error_in_cloud = "Unsupported" } try_sql("DROP USER ${user}") @@ -79,7 +79,7 @@ suite("test_database_management_auth","p0,auth_call") { } test { sql """SHOW REPLICA DISTRIBUTION FROM tbl;""" - exception "${show_dis_error_msg}" + exception "${error_in_cloud}" } test { sql """SHOW REPLICA STATUS FROM db1.tbl1;""" @@ -95,7 +95,7 @@ suite("test_database_management_auth","p0,auth_call") { } test { sql """ADMIN CHECK TABLET (10000, 10001) PROPERTIES("type" = "consistency");""" - exception "denied" + exception "${error_in_cloud}" } test { sql """SHOW TABLET DIAGNOSIS 0;""" diff --git a/regression-test/suites/nereids_p0/admin/test_nereids_admin_check_tablet.groovy b/regression-test/suites/nereids_p0/admin/test_nereids_admin_check_tablet.groovy new file mode 100644 index 00000000000000..8812e09ea9f6f3 --- /dev/null +++ b/regression-test/suites/nereids_p0/admin/test_nereids_admin_check_tablet.groovy @@ -0,0 +1,57 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + + +suite("test_nereids_admin_check_tablet") { + def table = "test_nereids_admin_check_tablet" + // create table and insert data + sql """ drop table if exists ${table} force""" + sql """ + create table ${table} ( + `id` int(11), + `name` varchar(128), + `da` date + ) + engine=olap + duplicate key(id) + partition by range(da)( + PARTITION p3 VALUES LESS THAN ('2023-01-01'), + PARTITION p4 VALUES LESS THAN ('2024-01-01'), + PARTITION p5 VALUES LESS THAN ('2025-01-01') + ) + distributed by hash(id) buckets 2 + properties( + "replication_num"="1", + "light_schema_change"="true" + ); + """ + + def tabletId + def tablets = sql_return_maparray "SHOW TABLETS FROM ${table}" + + for (def tablet : tablets) { + //get any partition ID. + tabletId = tablet.TabletId; + break; + } + + logger.info("${tabletId}"); + + checkNereidsExecute("ADMIN CHECK TABLET (${tabletId}) PROPERTIES('type' = 'consistency')"); + +} +