Skip to content

Commit

Permalink
[Enhancement] (nereids)implement CreateCatalogCommand in nereids (#45150
Browse files Browse the repository at this point in the history
)

Issue Number: close #42594
  • Loading branch information
msridhar78 authored Dec 16, 2024
1 parent 79eddb7 commit cf6dba7
Show file tree
Hide file tree
Showing 9 changed files with 249 additions and 13 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -185,7 +185,10 @@ supportedCreateStatement
(WITH ROLLUP (rollupNames=identifierList)?)? #createTableLike
| CREATE ROLE (IF NOT EXISTS)? name=identifier (COMMENT STRING_LITERAL)? #createRole
| CREATE WORKLOAD GROUP (IF NOT EXISTS)?
name=identifierOrText properties=propertyClause? #createWorkloadGroup
name=identifierOrText properties=propertyClause? #createWorkloadGroup
| CREATE CATALOG (IF NOT EXISTS)? catalogName=identifier
(WITH RESOURCE resourceName=identifier)?
(COMMENT STRING_LITERAL)? properties=propertyClause? #createCatalog
| CREATE ROW POLICY (IF NOT EXISTS)? name=identifier
ON table=multipartIdentifier
AS type=(RESTRICTIVE | PERMISSIVE)
Expand Down Expand Up @@ -744,9 +747,6 @@ analyzeProperties
unsupportedCreateStatement
: CREATE (DATABASE | SCHEMA) (IF NOT EXISTS)? name=multipartIdentifier
properties=propertyClause? #createDatabase
| CREATE CATALOG (IF NOT EXISTS)? catalogName=identifier
(WITH RESOURCE resourceName=identifier)?
(COMMENT STRING_LITERAL)? properties=propertyClause? #createCatalog
| CREATE (GLOBAL | SESSION | LOCAL)?
(TABLES | AGGREGATE)? FUNCTION (IF NOT EXISTS)?
functionIdentifier LEFT_PAREN functionArguments? RIGHT_PAREN
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,7 @@
import org.apache.doris.datasource.paimon.PaimonExternalCatalogFactory;
import org.apache.doris.datasource.test.TestExternalCatalog;
import org.apache.doris.datasource.trinoconnector.TrinoConnectorExternalCatalogFactory;
import org.apache.doris.nereids.trees.plans.commands.CreateCatalogCommand;

import com.google.common.base.Strings;
import org.apache.logging.log4j.LogManager;
Expand Down Expand Up @@ -83,6 +84,15 @@ public static CatalogIf createFromLog(CatalogLog log) throws DdlException {
log.getComment(), log.getProps(), true);
}

/**
* create the catalog instance from CreateCatalogCommand.
*/
public static CatalogIf createFromCommand(long catalogId, CreateCatalogCommand cmd)
throws DdlException {
return createCatalog(catalogId, cmd.getCatalogName(), cmd.getResource(),
cmd.getComment(), cmd.getProperties(), false);
}

/**
* create the catalog instance from creating statement.
*/
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -50,6 +50,7 @@
import org.apache.doris.datasource.hive.HMSExternalCatalog;
import org.apache.doris.datasource.hive.HMSExternalTable;
import org.apache.doris.mysql.privilege.PrivPredicate;
import org.apache.doris.nereids.trees.plans.commands.CreateCatalogCommand;
import org.apache.doris.persist.OperationType;
import org.apache.doris.persist.gson.GsonPostProcessable;
import org.apache.doris.persist.gson.GsonUtils;
Expand Down Expand Up @@ -235,20 +236,16 @@ private void readUnlock() {
lock.readLock().unlock();
}

/**
* Create and hold the catalog instance and write the meta log.
*/
public void createCatalog(CreateCatalogStmt stmt) throws UserException {
long id = Env.getCurrentEnv().getNextId();
CatalogIf catalog = CatalogFactory.createFromStmt(id, stmt);
private void createCatalogImpl(CatalogIf catalog, String catalogName,
boolean ifNotExists) throws UserException {
writeLock();
try {
if (nameToCatalog.containsKey(catalog.getName())) {
if (stmt.isSetIfNotExists()) {
LOG.warn("Catalog {} is already exist.", stmt.getCatalogName());
if (ifNotExists) {
LOG.warn("Catalog {} is already exist.", catalogName);
return;
}
throw new DdlException("Catalog had already exist with name: " + stmt.getCatalogName());
throw new DdlException("Catalog had already exist with name: " + catalogName);
}
createCatalogInternal(catalog, false);
Env.getCurrentEnv().getEditLog().logCatalogLog(OperationType.OP_CREATE_CATALOG, catalog.constructEditLog());
Expand All @@ -257,6 +254,24 @@ public void createCatalog(CreateCatalogStmt stmt) throws UserException {
}
}

/**
* Create and hold the catalog instance and write the meta log.
*/
public void createCatalog(CreateCatalogCommand cmd) throws UserException {
long id = Env.getCurrentEnv().getNextId();
CatalogIf catalog = CatalogFactory.createFromCommand(id, cmd);
createCatalogImpl(catalog, cmd.getCatalogName(), cmd.isSetIfNotExists());
}

/**
* Create and hold the catalog instance and write the meta log.
*/
public void createCatalog(CreateCatalogStmt stmt) throws UserException {
long id = Env.getCurrentEnv().getNextId();
CatalogIf catalog = CatalogFactory.createFromStmt(id, stmt);
createCatalogImpl(catalog, stmt.getCatalogName(), stmt.isSetIfNotExists());
}

/**
* Remove the catalog instance by name and write the meta log.
*/
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -95,6 +95,7 @@
import org.apache.doris.nereids.DorisParser.ComplexColTypeListContext;
import org.apache.doris.nereids.DorisParser.ComplexDataTypeContext;
import org.apache.doris.nereids.DorisParser.ConstantContext;
import org.apache.doris.nereids.DorisParser.CreateCatalogContext;
import org.apache.doris.nereids.DorisParser.CreateEncryptkeyContext;
import org.apache.doris.nereids.DorisParser.CreateFileContext;
import org.apache.doris.nereids.DorisParser.CreateMTMVContext;
Expand Down Expand Up @@ -500,6 +501,7 @@
import org.apache.doris.nereids.trees.plans.commands.CleanAllProfileCommand;
import org.apache.doris.nereids.trees.plans.commands.Command;
import org.apache.doris.nereids.trees.plans.commands.Constraint;
import org.apache.doris.nereids.trees.plans.commands.CreateCatalogCommand;
import org.apache.doris.nereids.trees.plans.commands.CreateEncryptkeyCommand;
import org.apache.doris.nereids.trees.plans.commands.CreateFileCommand;
import org.apache.doris.nereids.trees.plans.commands.CreateJobCommand;
Expand Down Expand Up @@ -4843,6 +4845,18 @@ public LogicalPlan visitShowDynamicPartition(ShowDynamicPartitionContext ctx) {
return new ShowDynamicPartitionCommand(dbName);
}

@Override
public LogicalPlan visitCreateCatalog(CreateCatalogContext ctx) {
String catalogName = ctx.catalogName.getText();
boolean ifNotExists = ctx.IF() != null;
String resourceName = ctx.resourceName == null ? null : (ctx.resourceName.getText());
String comment = ctx.STRING_LITERAL() == null ? null : stripQuotes(ctx.STRING_LITERAL().getText());
Map<String, String> properties = ctx.propertyClause() != null
? Maps.newHashMap(visitPropertyClause(ctx.propertyClause())) : Maps.newHashMap();

return new CreateCatalogCommand(catalogName, ifNotExists, resourceName, comment, properties);
}

@Override
public LogicalPlan visitRecoverDatabase(RecoverDatabaseContext ctx) {
String dbName = ctx.name.getText();
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -252,6 +252,7 @@ public enum PlanType {
REPLAY_COMMAND,
CREATE_ENCRYPTKEY_COMMAND,
CREATE_WORKLOAD_GROUP_COMMAND,
CREATE_CATALOG_COMMAND,
CREATE_FILE_COMMAND,
CREATE_ROUTINE_LOAD_COMMAND,
SHOW_TABLE_CREATION_COMMAND,
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,144 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.

package org.apache.doris.nereids.trees.plans.commands;

import org.apache.doris.catalog.Env;
import org.apache.doris.common.AnalysisException;
import org.apache.doris.common.Config;
import org.apache.doris.common.ErrorCode;
import org.apache.doris.common.ErrorReport;
import org.apache.doris.common.util.PrintableMap;
import org.apache.doris.common.util.PropertyAnalyzer;
import org.apache.doris.common.util.Util;
import org.apache.doris.datasource.ExternalCatalog;
import org.apache.doris.datasource.InternalCatalog;
import org.apache.doris.mysql.privilege.PrivPredicate;
import org.apache.doris.nereids.trees.plans.PlanType;
import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor;
import org.apache.doris.qe.ConnectContext;
import org.apache.doris.qe.StmtExecutor;

import com.google.common.base.Strings;
import com.google.common.collect.Maps;

import java.time.LocalDateTime;
import java.time.ZoneId;
import java.util.Map;
import java.util.Objects;

/**
* Command to create a catalog in the Nereids planner.
*/
public class CreateCatalogCommand extends Command implements ForwardWithSync, NeedAuditEncryption {
private final String catalogName;
private final boolean ifNotExists;
private final String resourceName;
private final String comment;
private final Map<String, String> properties;

/**
* Constructor
*/
public CreateCatalogCommand(String catalogName, boolean ifNotExists, String resourceName, String comment,
Map<String, String> properties) {
super(PlanType.CREATE_CATALOG_COMMAND);
this.catalogName = Objects.requireNonNull(catalogName, "Catalog name cannot be null");
this.ifNotExists = ifNotExists;
this.resourceName = resourceName;
this.comment = comment;
this.properties = properties == null ? Maps.newHashMap() : properties;
}

private void validate(ConnectContext ctx) throws AnalysisException {
Util.checkCatalogAllRules(catalogName);
if (catalogName.equals(InternalCatalog.INTERNAL_CATALOG_NAME)) {
throw new AnalysisException("Internal catalog name can't be create.");
}

if (!Env.getCurrentEnv().getAccessManager().checkCtlPriv(
ConnectContext.get(), catalogName, PrivPredicate.CREATE)) {
ErrorReport.reportAnalysisException(ErrorCode.ERR_CATALOG_ACCESS_DENIED,
ctx.getQualifiedUser(), catalogName);
}

if (Config.disallow_create_catalog_with_resource && !Strings.isNullOrEmpty(resourceName)) {
throw new AnalysisException("Create catalog with resource is deprecated and is not allowed."
+ " You can set `disallow_create_catalog_with_resource=false` in fe.conf"
+ " to enable it temporarily.");
}

String currentDateTime = LocalDateTime.now(ZoneId.systemDefault()).toString().replace("T", " ");
properties.put(ExternalCatalog.CREATE_TIME, currentDateTime);
PropertyAnalyzer.checkCatalogProperties(properties, false);
}

@Override
public void run(ConnectContext ctx, StmtExecutor executor) throws Exception {
validate(ctx);
Env.getCurrentEnv().getCatalogMgr().createCatalog(this);
}

@Override
public <R, C> R accept(PlanVisitor<R, C> visitor, C context) {
return visitor.visitCreateCatalogCommand(this, context);
}

public String getCatalogName() {
return catalogName;
}

public boolean isSetIfNotExists() {
return ifNotExists;
}

public String getResource() {
return resourceName;
}

public String getComment() {
return comment;
}

public Map<String, String> getProperties() {
return properties;
}

@Override
public boolean needAuditEncryption() {
return true;
}

@Override
public String toSql() {
StringBuilder stringBuilder = new StringBuilder();
stringBuilder.append("CREATE CATALOG ").append("`").append(catalogName).append("`");
if (!Strings.isNullOrEmpty(resourceName)) {
stringBuilder.append(" WITH RESOURCE `").append(resourceName).append("`");
}
if (!Strings.isNullOrEmpty(comment)) {
stringBuilder.append("\nCOMMENT \"").append(comment).append("\"");
}
if (properties.size() > 0) {
stringBuilder.append("\nPROPERTIES (\n");
stringBuilder.append(new PrintableMap<>(properties, "=", true, true, true));
stringBuilder.append("\n)");
}
return stringBuilder.toString();
}
}

Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,7 @@
import org.apache.doris.nereids.trees.plans.commands.CancelWarmUpJobCommand;
import org.apache.doris.nereids.trees.plans.commands.CleanAllProfileCommand;
import org.apache.doris.nereids.trees.plans.commands.Command;
import org.apache.doris.nereids.trees.plans.commands.CreateCatalogCommand;
import org.apache.doris.nereids.trees.plans.commands.CreateEncryptkeyCommand;
import org.apache.doris.nereids.trees.plans.commands.CreateFileCommand;
import org.apache.doris.nereids.trees.plans.commands.CreateJobCommand;
Expand Down Expand Up @@ -305,6 +306,10 @@ default R visitShowProcedureStatusCommand(ShowProcedureStatusCommand showProcedu
return visitCommand(showProcedureStatusCommand, context);
}

default R visitCreateCatalogCommand(CreateCatalogCommand createCatalogCommand, C context) {
return visitCommand(createCatalogCommand, context);
}

default R visitShowWarningErrorsCommand(ShowWarningErrorsCommand showWarningErrorsCommand, C context) {
return visitCommand(showWarningErrorsCommand, context);
}
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
-- This file is automatically generated. You should know what you did if you want to edit this
-- !cmd --
test_create_catalog \nCREATE CATALOG `test_create_catalog`\nCOMMENT "Test catalog for regression"\n PROPERTIES (\n"type" = "es",\n"hosts" = "http://127.0.0.1:9200"\n);

Original file line number Diff line number Diff line change
@@ -0,0 +1,43 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.

suite("test_create_catalog_command", "nereids_p0") {
def catalogName = "test_create_catalog"
def resourceName = "test_resource"
def catalogProperties = "\"type\"=\"es\", \"hosts\"=\"http://127.0.0.1:9200\""

try {
// Drop catalog if it already exists
sql("DROP CATALOG IF EXISTS ${catalogName}")

// Create a new catalog
checkNereidsExecute(
"""
CREATE CATALOG ${catalogName}
COMMENT 'Test catalog for regression'
PROPERTIES (${catalogProperties})
"""
)

// Verify the catalog was created
checkNereidsExecute("""show create catalog ${catalogName}""")
qt_cmd("""show create catalog ${catalogName}""")
} finally {
// Clean up
sql("DROP CATALOG IF EXISTS ${catalogName}")
}
}

0 comments on commit cf6dba7

Please sign in to comment.