diff --git a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 index 044ed7771a4aa1..bc91a667eda44a 100644 --- a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 +++ b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 @@ -185,7 +185,10 @@ supportedCreateStatement (WITH ROLLUP (rollupNames=identifierList)?)? #createTableLike | CREATE ROLE (IF NOT EXISTS)? name=identifier (COMMENT STRING_LITERAL)? #createRole | CREATE WORKLOAD GROUP (IF NOT EXISTS)? - name=identifierOrText properties=propertyClause? #createWorkloadGroup + name=identifierOrText properties=propertyClause? #createWorkloadGroup + | CREATE CATALOG (IF NOT EXISTS)? catalogName=identifier + (WITH RESOURCE resourceName=identifier)? + (COMMENT STRING_LITERAL)? properties=propertyClause? #createCatalog | CREATE ROW POLICY (IF NOT EXISTS)? name=identifier ON table=multipartIdentifier AS type=(RESTRICTIVE | PERMISSIVE) @@ -744,9 +747,6 @@ analyzeProperties unsupportedCreateStatement : CREATE (DATABASE | SCHEMA) (IF NOT EXISTS)? name=multipartIdentifier properties=propertyClause? #createDatabase - | CREATE CATALOG (IF NOT EXISTS)? catalogName=identifier - (WITH RESOURCE resourceName=identifier)? - (COMMENT STRING_LITERAL)? properties=propertyClause? #createCatalog | CREATE (GLOBAL | SESSION | LOCAL)? (TABLES | AGGREGATE)? FUNCTION (IF NOT EXISTS)? functionIdentifier LEFT_PAREN functionArguments? RIGHT_PAREN diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogFactory.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogFactory.java index 43ac17839c9606..984d29cae4923a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogFactory.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogFactory.java @@ -37,6 +37,7 @@ import org.apache.doris.datasource.paimon.PaimonExternalCatalogFactory; import org.apache.doris.datasource.test.TestExternalCatalog; import org.apache.doris.datasource.trinoconnector.TrinoConnectorExternalCatalogFactory; +import org.apache.doris.nereids.trees.plans.commands.CreateCatalogCommand; import com.google.common.base.Strings; import org.apache.logging.log4j.LogManager; @@ -83,6 +84,15 @@ public static CatalogIf createFromLog(CatalogLog log) throws DdlException { log.getComment(), log.getProps(), true); } + /** + * create the catalog instance from CreateCatalogCommand. + */ + public static CatalogIf createFromCommand(long catalogId, CreateCatalogCommand cmd) + throws DdlException { + return createCatalog(catalogId, cmd.getCatalogName(), cmd.getResource(), + cmd.getComment(), cmd.getProperties(), false); + } + /** * create the catalog instance from creating statement. */ diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogMgr.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogMgr.java index 070d31c071e930..0203aa7020b090 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogMgr.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogMgr.java @@ -50,6 +50,7 @@ import org.apache.doris.datasource.hive.HMSExternalCatalog; import org.apache.doris.datasource.hive.HMSExternalTable; import org.apache.doris.mysql.privilege.PrivPredicate; +import org.apache.doris.nereids.trees.plans.commands.CreateCatalogCommand; import org.apache.doris.persist.OperationType; import org.apache.doris.persist.gson.GsonPostProcessable; import org.apache.doris.persist.gson.GsonUtils; @@ -235,20 +236,16 @@ private void readUnlock() { lock.readLock().unlock(); } - /** - * Create and hold the catalog instance and write the meta log. - */ - public void createCatalog(CreateCatalogStmt stmt) throws UserException { - long id = Env.getCurrentEnv().getNextId(); - CatalogIf catalog = CatalogFactory.createFromStmt(id, stmt); + private void createCatalogImpl(CatalogIf catalog, String catalogName, + boolean ifNotExists) throws UserException { writeLock(); try { if (nameToCatalog.containsKey(catalog.getName())) { - if (stmt.isSetIfNotExists()) { - LOG.warn("Catalog {} is already exist.", stmt.getCatalogName()); + if (ifNotExists) { + LOG.warn("Catalog {} is already exist.", catalogName); return; } - throw new DdlException("Catalog had already exist with name: " + stmt.getCatalogName()); + throw new DdlException("Catalog had already exist with name: " + catalogName); } createCatalogInternal(catalog, false); Env.getCurrentEnv().getEditLog().logCatalogLog(OperationType.OP_CREATE_CATALOG, catalog.constructEditLog()); @@ -257,6 +254,24 @@ public void createCatalog(CreateCatalogStmt stmt) throws UserException { } } + /** + * Create and hold the catalog instance and write the meta log. + */ + public void createCatalog(CreateCatalogCommand cmd) throws UserException { + long id = Env.getCurrentEnv().getNextId(); + CatalogIf catalog = CatalogFactory.createFromCommand(id, cmd); + createCatalogImpl(catalog, cmd.getCatalogName(), cmd.isSetIfNotExists()); + } + + /** + * Create and hold the catalog instance and write the meta log. + */ + public void createCatalog(CreateCatalogStmt stmt) throws UserException { + long id = Env.getCurrentEnv().getNextId(); + CatalogIf catalog = CatalogFactory.createFromStmt(id, stmt); + createCatalogImpl(catalog, stmt.getCatalogName(), stmt.isSetIfNotExists()); + } + /** * Remove the catalog instance by name and write the meta log. */ diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java index e6fee894b41fa9..94c2757e560b24 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java @@ -95,6 +95,7 @@ import org.apache.doris.nereids.DorisParser.ComplexColTypeListContext; import org.apache.doris.nereids.DorisParser.ComplexDataTypeContext; import org.apache.doris.nereids.DorisParser.ConstantContext; +import org.apache.doris.nereids.DorisParser.CreateCatalogContext; import org.apache.doris.nereids.DorisParser.CreateEncryptkeyContext; import org.apache.doris.nereids.DorisParser.CreateFileContext; import org.apache.doris.nereids.DorisParser.CreateMTMVContext; @@ -500,6 +501,7 @@ import org.apache.doris.nereids.trees.plans.commands.CleanAllProfileCommand; import org.apache.doris.nereids.trees.plans.commands.Command; import org.apache.doris.nereids.trees.plans.commands.Constraint; +import org.apache.doris.nereids.trees.plans.commands.CreateCatalogCommand; import org.apache.doris.nereids.trees.plans.commands.CreateEncryptkeyCommand; import org.apache.doris.nereids.trees.plans.commands.CreateFileCommand; import org.apache.doris.nereids.trees.plans.commands.CreateJobCommand; @@ -4843,6 +4845,18 @@ public LogicalPlan visitShowDynamicPartition(ShowDynamicPartitionContext ctx) { return new ShowDynamicPartitionCommand(dbName); } + @Override + public LogicalPlan visitCreateCatalog(CreateCatalogContext ctx) { + String catalogName = ctx.catalogName.getText(); + boolean ifNotExists = ctx.IF() != null; + String resourceName = ctx.resourceName == null ? null : (ctx.resourceName.getText()); + String comment = ctx.STRING_LITERAL() == null ? null : stripQuotes(ctx.STRING_LITERAL().getText()); + Map properties = ctx.propertyClause() != null + ? Maps.newHashMap(visitPropertyClause(ctx.propertyClause())) : Maps.newHashMap(); + + return new CreateCatalogCommand(catalogName, ifNotExists, resourceName, comment, properties); + } + @Override public LogicalPlan visitRecoverDatabase(RecoverDatabaseContext ctx) { String dbName = ctx.name.getText(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlanType.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlanType.java index e8ce4b3e2d88db..26b8ed4c0c9365 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlanType.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlanType.java @@ -252,6 +252,7 @@ public enum PlanType { REPLAY_COMMAND, CREATE_ENCRYPTKEY_COMMAND, CREATE_WORKLOAD_GROUP_COMMAND, + CREATE_CATALOG_COMMAND, CREATE_FILE_COMMAND, CREATE_ROUTINE_LOAD_COMMAND, SHOW_TABLE_CREATION_COMMAND, diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/CreateCatalogCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/CreateCatalogCommand.java new file mode 100644 index 00000000000000..22c1285a0556fd --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/CreateCatalogCommand.java @@ -0,0 +1,144 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.trees.plans.commands; + +import org.apache.doris.catalog.Env; +import org.apache.doris.common.AnalysisException; +import org.apache.doris.common.Config; +import org.apache.doris.common.ErrorCode; +import org.apache.doris.common.ErrorReport; +import org.apache.doris.common.util.PrintableMap; +import org.apache.doris.common.util.PropertyAnalyzer; +import org.apache.doris.common.util.Util; +import org.apache.doris.datasource.ExternalCatalog; +import org.apache.doris.datasource.InternalCatalog; +import org.apache.doris.mysql.privilege.PrivPredicate; +import org.apache.doris.nereids.trees.plans.PlanType; +import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor; +import org.apache.doris.qe.ConnectContext; +import org.apache.doris.qe.StmtExecutor; + +import com.google.common.base.Strings; +import com.google.common.collect.Maps; + +import java.time.LocalDateTime; +import java.time.ZoneId; +import java.util.Map; +import java.util.Objects; + +/** + * Command to create a catalog in the Nereids planner. + */ +public class CreateCatalogCommand extends Command implements ForwardWithSync, NeedAuditEncryption { + private final String catalogName; + private final boolean ifNotExists; + private final String resourceName; + private final String comment; + private final Map properties; + + /** + * Constructor + */ + public CreateCatalogCommand(String catalogName, boolean ifNotExists, String resourceName, String comment, + Map properties) { + super(PlanType.CREATE_CATALOG_COMMAND); + this.catalogName = Objects.requireNonNull(catalogName, "Catalog name cannot be null"); + this.ifNotExists = ifNotExists; + this.resourceName = resourceName; + this.comment = comment; + this.properties = properties == null ? Maps.newHashMap() : properties; + } + + private void validate(ConnectContext ctx) throws AnalysisException { + Util.checkCatalogAllRules(catalogName); + if (catalogName.equals(InternalCatalog.INTERNAL_CATALOG_NAME)) { + throw new AnalysisException("Internal catalog name can't be create."); + } + + if (!Env.getCurrentEnv().getAccessManager().checkCtlPriv( + ConnectContext.get(), catalogName, PrivPredicate.CREATE)) { + ErrorReport.reportAnalysisException(ErrorCode.ERR_CATALOG_ACCESS_DENIED, + ctx.getQualifiedUser(), catalogName); + } + + if (Config.disallow_create_catalog_with_resource && !Strings.isNullOrEmpty(resourceName)) { + throw new AnalysisException("Create catalog with resource is deprecated and is not allowed." + + " You can set `disallow_create_catalog_with_resource=false` in fe.conf" + + " to enable it temporarily."); + } + + String currentDateTime = LocalDateTime.now(ZoneId.systemDefault()).toString().replace("T", " "); + properties.put(ExternalCatalog.CREATE_TIME, currentDateTime); + PropertyAnalyzer.checkCatalogProperties(properties, false); + } + + @Override + public void run(ConnectContext ctx, StmtExecutor executor) throws Exception { + validate(ctx); + Env.getCurrentEnv().getCatalogMgr().createCatalog(this); + } + + @Override + public R accept(PlanVisitor visitor, C context) { + return visitor.visitCreateCatalogCommand(this, context); + } + + public String getCatalogName() { + return catalogName; + } + + public boolean isSetIfNotExists() { + return ifNotExists; + } + + public String getResource() { + return resourceName; + } + + public String getComment() { + return comment; + } + + public Map getProperties() { + return properties; + } + + @Override + public boolean needAuditEncryption() { + return true; + } + + @Override + public String toSql() { + StringBuilder stringBuilder = new StringBuilder(); + stringBuilder.append("CREATE CATALOG ").append("`").append(catalogName).append("`"); + if (!Strings.isNullOrEmpty(resourceName)) { + stringBuilder.append(" WITH RESOURCE `").append(resourceName).append("`"); + } + if (!Strings.isNullOrEmpty(comment)) { + stringBuilder.append("\nCOMMENT \"").append(comment).append("\""); + } + if (properties.size() > 0) { + stringBuilder.append("\nPROPERTIES (\n"); + stringBuilder.append(new PrintableMap<>(properties, "=", true, true, true)); + stringBuilder.append("\n)"); + } + return stringBuilder.toString(); + } +} + diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/CommandVisitor.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/CommandVisitor.java index 0341ed2f57d7ec..efb81ae7ee037f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/CommandVisitor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/CommandVisitor.java @@ -38,6 +38,7 @@ import org.apache.doris.nereids.trees.plans.commands.CancelWarmUpJobCommand; import org.apache.doris.nereids.trees.plans.commands.CleanAllProfileCommand; import org.apache.doris.nereids.trees.plans.commands.Command; +import org.apache.doris.nereids.trees.plans.commands.CreateCatalogCommand; import org.apache.doris.nereids.trees.plans.commands.CreateEncryptkeyCommand; import org.apache.doris.nereids.trees.plans.commands.CreateFileCommand; import org.apache.doris.nereids.trees.plans.commands.CreateJobCommand; @@ -305,6 +306,10 @@ default R visitShowProcedureStatusCommand(ShowProcedureStatusCommand showProcedu return visitCommand(showProcedureStatusCommand, context); } + default R visitCreateCatalogCommand(CreateCatalogCommand createCatalogCommand, C context) { + return visitCommand(createCatalogCommand, context); + } + default R visitShowWarningErrorsCommand(ShowWarningErrorsCommand showWarningErrorsCommand, C context) { return visitCommand(showWarningErrorsCommand, context); } diff --git a/regression-test/data/nereids_p0/test_create_catalog_command.out b/regression-test/data/nereids_p0/test_create_catalog_command.out new file mode 100644 index 00000000000000..b3ff20b67db426 --- /dev/null +++ b/regression-test/data/nereids_p0/test_create_catalog_command.out @@ -0,0 +1,4 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !cmd -- +test_create_catalog \nCREATE CATALOG `test_create_catalog`\nCOMMENT "Test catalog for regression"\n PROPERTIES (\n"type" = "es",\n"hosts" = "http://127.0.0.1:9200"\n); + diff --git a/regression-test/suites/nereids_p0/test_create_catalog_command.groovy b/regression-test/suites/nereids_p0/test_create_catalog_command.groovy new file mode 100644 index 00000000000000..be28de0a246dd3 --- /dev/null +++ b/regression-test/suites/nereids_p0/test_create_catalog_command.groovy @@ -0,0 +1,43 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_create_catalog_command", "nereids_p0") { + def catalogName = "test_create_catalog" + def resourceName = "test_resource" + def catalogProperties = "\"type\"=\"es\", \"hosts\"=\"http://127.0.0.1:9200\"" + + try { + // Drop catalog if it already exists + sql("DROP CATALOG IF EXISTS ${catalogName}") + + // Create a new catalog + checkNereidsExecute( + """ + CREATE CATALOG ${catalogName} + COMMENT 'Test catalog for regression' + PROPERTIES (${catalogProperties}) + """ + ) + + // Verify the catalog was created + checkNereidsExecute("""show create catalog ${catalogName}""") + qt_cmd("""show create catalog ${catalogName}""") + } finally { + // Clean up + sql("DROP CATALOG IF EXISTS ${catalogName}") + } +}