From ed112527f9ba41cefbf5593aa94dc0bd9fbfa2ea Mon Sep 17 00:00:00 2001 From: Rijesh Kunhi Parambattu <147430310+rijeshkp@users.noreply.github.com> Date: Tue, 10 Dec 2024 07:17:25 +0530 Subject: [PATCH] [Feat](nereids) support ShowTabletStorageFormat command in nereids (#43295) Issue Number: close #42783 --- .../org/apache/doris/nereids/DorisParser.g4 | 4 +- .../nereids/parser/LogicalPlanBuilder.java | 13 ++ .../doris/nereids/trees/plans/PlanType.java | 1 + .../ShowTabletStorageFormatCommand.java | 129 ++++++++++++++++++ .../trees/plans/visitor/CommandVisitor.java | 6 + .../test_database_management_auth.groovy | 2 +- ..._nereids_show_tablet_storage_format.groovy | 25 ++++ 7 files changed, 177 insertions(+), 3 deletions(-) create mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ShowTabletStorageFormatCommand.java create mode 100644 regression-test/suites/nereids_p0/show/test_nereids_show_tablet_storage_format.groovy diff --git a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 index 3bc884408c55ce..1b01cad4293569 100644 --- a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 +++ b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 @@ -270,6 +270,7 @@ supportedShowStatement | SHOW DATA SKEW FROM baseTableRef #showDataSkew | SHOW TABLE CREATION ((FROM | IN) database=multipartIdentifier)? (LIKE STRING_LITERAL)? #showTableCreation + | SHOW TABLET STORAGE FORMAT VERBOSE? #showTabletStorageFormat ; supportedLoadStatement @@ -369,7 +370,6 @@ unsupportedShowStatement | SHOW (CLUSTERS | (COMPUTE GROUPS)) #showClusters | SHOW CONVERT_LSC ((FROM | IN) database=multipartIdentifier)? #showConvertLsc | SHOW REPLICA STATUS FROM baseTableRef wildWhere? #showReplicaStatus - | SHOW TABLET STORAGE FORMAT VERBOSE? #showTabletStorageFormat | SHOW COPY ((FROM | IN) database=multipartIdentifier)? whereClause? sortClause? limitClause? #showCopy | SHOW WARM UP JOB wildWhere? #showWarmUpJob @@ -495,6 +495,7 @@ supportedAdminStatement | ADMIN SHOW REPLICA STATUS FROM baseTableRef (WHERE STATUS EQ|NEQ STRING_LITERAL)? #adminShowReplicaStatus | ADMIN COMPACT TABLE baseTableRef (WHERE TYPE EQ STRING_LITERAL)? #adminCompactTable | ADMIN CHECK tabletList properties=propertyClause? #adminCheckTablets + | ADMIN SHOW TABLET STORAGE FORMAT VERBOSE? #adminShowTabletStorageFormat ; supportedRecoverStatement @@ -520,7 +521,6 @@ unsupportedAdminStatement (COMMA backends+=STRING_LITERAL) RIGHT_PAREN)? #adminCleanTrash | ADMIN SET TABLE name=multipartIdentifier PARTITION VERSION properties=propertyClause? #adminSetPartitionVersion - | ADMIN SHOW TABLET STORAGE FORMAT VERBOSE? #adminShowTabletStorageFormat | ADMIN COPY TABLET tabletId=INTEGER_VALUE properties=propertyClause? #adminCopyTablet | ADMIN SET TABLE name=multipartIdentifier STATUS properties=propertyClause? #adminSetTableStatus ; diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java index 8c964e997069a5..95dbbb504712f8 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java @@ -55,6 +55,7 @@ import org.apache.doris.nereids.DorisParser.AdminDiagnoseTabletContext; import org.apache.doris.nereids.DorisParser.AdminShowReplicaDistributionContext; import org.apache.doris.nereids.DorisParser.AdminShowReplicaStatusContext; +import org.apache.doris.nereids.DorisParser.AdminShowTabletStorageFormatContext; import org.apache.doris.nereids.DorisParser.AggClauseContext; import org.apache.doris.nereids.DorisParser.AggStateDataTypeContext; import org.apache.doris.nereids.DorisParser.AliasQueryContext; @@ -273,6 +274,7 @@ import org.apache.doris.nereids.DorisParser.ShowStorageEnginesContext; import org.apache.doris.nereids.DorisParser.ShowTableCreationContext; import org.apache.doris.nereids.DorisParser.ShowTableIdContext; +import org.apache.doris.nereids.DorisParser.ShowTabletStorageFormatContext; import org.apache.doris.nereids.DorisParser.ShowTabletsBelongContext; import org.apache.doris.nereids.DorisParser.ShowTrashContext; import org.apache.doris.nereids.DorisParser.ShowTriggersContext; @@ -577,6 +579,7 @@ import org.apache.doris.nereids.trees.plans.commands.ShowStorageEnginesCommand; import org.apache.doris.nereids.trees.plans.commands.ShowTableCreationCommand; import org.apache.doris.nereids.trees.plans.commands.ShowTableIdCommand; +import org.apache.doris.nereids.trees.plans.commands.ShowTabletStorageFormatCommand; import org.apache.doris.nereids.trees.plans.commands.ShowTabletsBelongCommand; import org.apache.doris.nereids.trees.plans.commands.ShowTrashCommand; import org.apache.doris.nereids.trees.plans.commands.ShowTriggersCommand; @@ -5001,5 +5004,15 @@ public LogicalPlan visitShowTableCreation(ShowTableCreationContext ctx) { } return new ShowTableCreationCommand(dbName, wild); } + + @Override + public LogicalPlan visitAdminShowTabletStorageFormat(AdminShowTabletStorageFormatContext ctx) { + return new ShowTabletStorageFormatCommand(ctx.VERBOSE() != null); + } + + @Override + public LogicalPlan visitShowTabletStorageFormat(ShowTabletStorageFormatContext ctx) { + return new ShowTabletStorageFormatCommand(ctx.VERBOSE() != null); + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlanType.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlanType.java index e98d9f6afd5ae2..91ec3591003500 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlanType.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlanType.java @@ -235,6 +235,7 @@ public enum PlanType { SHOW_STORAGE_ENGINES_COMMAND, SHOW_TABLE_ID_COMMAND, SHOW_TRASH_COMMAND, + SHOW_TABLET_STORAGE_FORMAT_COMMAND, SHOW_TRIGGERS_COMMAND, SHOW_VARIABLES_COMMAND, SHOW_AUTHORS_COMMAND, diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ShowTabletStorageFormatCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ShowTabletStorageFormatCommand.java new file mode 100644 index 00000000000000..aa535e0f39dd15 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ShowTabletStorageFormatCommand.java @@ -0,0 +1,129 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.trees.plans.commands; + +import org.apache.doris.catalog.Column; +import org.apache.doris.catalog.Env; +import org.apache.doris.catalog.ScalarType; +import org.apache.doris.common.AnalysisException; +import org.apache.doris.common.DdlException; +import org.apache.doris.common.ErrorCode; +import org.apache.doris.common.ErrorReport; +import org.apache.doris.mysql.privilege.PrivPredicate; +import org.apache.doris.nereids.trees.plans.PlanType; +import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor; +import org.apache.doris.qe.ConnectContext; +import org.apache.doris.qe.ShowResultSet; +import org.apache.doris.qe.ShowResultSetMetaData; +import org.apache.doris.qe.StmtExecutor; +import org.apache.doris.system.Backend; +import org.apache.doris.task.AgentClient; +import org.apache.doris.thrift.TCheckStorageFormatResult; + +import com.google.common.collect.Lists; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import java.util.ArrayList; +import java.util.List; + +/** + * show tablet storage format command + */ +public class ShowTabletStorageFormatCommand extends ShowCommand { + public static final Logger LOG = LogManager.getLogger(ShowTabletStorageFormatCommand.class); + private final boolean verbose; + + /** + * constructor + */ + public ShowTabletStorageFormatCommand(boolean verbose) { + super(PlanType.SHOW_TABLET_STORAGE_FORMAT_COMMAND); + this.verbose = verbose; + } + + /** + * get Meta for show + */ + public ShowResultSetMetaData getMetaData() { + ShowResultSetMetaData.Builder builder = ShowResultSetMetaData.builder(); + if (verbose) { + builder.addColumn(new Column("BackendId", ScalarType.createVarchar(30))) + .addColumn(new Column("TabletId", ScalarType.createVarchar(30))) + .addColumn(new Column("StorageFormat", ScalarType.createVarchar(30))); + } else { + builder.addColumn(new Column("BackendId", ScalarType.createVarchar(30))) + .addColumn(new Column("V1Count", ScalarType.createVarchar(30))) + .addColumn(new Column("V2Count", ScalarType.createVarchar(30))); + } + return builder.build(); + } + + @Override + public ShowResultSet doRun(ConnectContext ctx, StmtExecutor executor) throws Exception { + if (!Env.getCurrentEnv().getAccessManager().checkGlobalPriv(ConnectContext.get(), PrivPredicate.ADMIN)) { + ErrorReport.reportAnalysisException(ErrorCode.ERR_SPECIFIC_ACCESS_DENIED_ERROR, + PrivPredicate.ADMIN.getPrivs().toString()); + } + + List> resultRowSet = Lists.newArrayList(); + for (Backend be : Env.getCurrentSystemInfo().getAllBackendsByAllCluster().values()) { + if (be.isQueryAvailable() && be.isLoadAvailable()) { + AgentClient client = new AgentClient(be.getHost(), be.getBePort()); + TCheckStorageFormatResult result = client.checkStorageFormat(); + if (result == null) { + throw new AnalysisException("get tablet data from backend: " + be.getId() + "error."); + } + if (verbose) { + for (long tabletId : result.getV1Tablets()) { + List row = new ArrayList<>(); + row.add(String.valueOf(be.getId())); + row.add(String.valueOf(tabletId)); + row.add("V1"); + resultRowSet.add(row); + } + for (long tabletId : result.getV2Tablets()) { + List row = new ArrayList<>(); + row.add(String.valueOf(be.getId())); + row.add(String.valueOf(tabletId)); + row.add("V2"); + resultRowSet.add(row); + } + } else { + List row = new ArrayList<>(); + row.add(String.valueOf(be.getId())); + row.add(String.valueOf(result.getV1Tablets().size())); + row.add(String.valueOf(result.getV2Tablets().size())); + resultRowSet.add(row); + } + } + } + return new ShowResultSet(getMetaData(), resultRowSet); + } + + @Override + public R accept(PlanVisitor visitor, C context) { + return visitor.visitShowTabletStorageFormatCommand(this, context); + } + + @Override + protected void checkSupportedInCloudMode(ConnectContext ctx) throws DdlException { + LOG.info("show tablet storage format not supported in cloud mode"); + throw new DdlException("Unsupported operation"); + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/CommandVisitor.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/CommandVisitor.java index 9131671f98abfd..5da741f303fed4 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/CommandVisitor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/CommandVisitor.java @@ -118,6 +118,7 @@ import org.apache.doris.nereids.trees.plans.commands.ShowStorageEnginesCommand; import org.apache.doris.nereids.trees.plans.commands.ShowTableCreationCommand; import org.apache.doris.nereids.trees.plans.commands.ShowTableIdCommand; +import org.apache.doris.nereids.trees.plans.commands.ShowTabletStorageFormatCommand; import org.apache.doris.nereids.trees.plans.commands.ShowTabletsBelongCommand; import org.apache.doris.nereids.trees.plans.commands.ShowTrashCommand; import org.apache.doris.nereids.trees.plans.commands.ShowTriggersCommand; @@ -615,4 +616,9 @@ default R visitShowDataSkewCommand(ShowDataSkewCommand showDataSkewCommand, C co default R visitShowTableCreationCommand(ShowTableCreationCommand showTableCreationCommand, C context) { return visitCommand(showTableCreationCommand, context); } + + default R visitShowTabletStorageFormatCommand(ShowTabletStorageFormatCommand showTabletStorageFormatCommand, + C context) { + return visitCommand(showTabletStorageFormatCommand, context); + } } diff --git a/regression-test/suites/auth_call/test_database_management_auth.groovy b/regression-test/suites/auth_call/test_database_management_auth.groovy index 79114c18e789b5..b481cf8de75c00 100644 --- a/regression-test/suites/auth_call/test_database_management_auth.groovy +++ b/regression-test/suites/auth_call/test_database_management_auth.groovy @@ -107,7 +107,7 @@ suite("test_database_management_auth","p0,auth_call") { } test { sql """show tablet storage format verbose;""" - exception "denied" + exception "${error_in_cloud}" } test { sql """ADMIN CLEAN TRASH;""" diff --git a/regression-test/suites/nereids_p0/show/test_nereids_show_tablet_storage_format.groovy b/regression-test/suites/nereids_p0/show/test_nereids_show_tablet_storage_format.groovy new file mode 100644 index 00000000000000..b4b78bd1c28729 --- /dev/null +++ b/regression-test/suites/nereids_p0/show/test_nereids_show_tablet_storage_format.groovy @@ -0,0 +1,25 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + + +suite("test_nereids_show_tablet_storage_format") { + checkNereidsExecute("""show tablet storage format;""") + checkNereidsExecute("""show tablet storage format verbose;""") + checkNereidsExecute("""admin show tablet storage format;""") + checkNereidsExecute("""admin show tablet storage format verbose;""") +} +