Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[feat](Nereids) support refresh database command #44298

Merged
merged 34 commits into from
Dec 2, 2024
Merged
Show file tree
Hide file tree
Changes from 23 commits
Commits
Show all changes
34 commits
Select commit Hold shift + click to select a range
c7ab850
support refresh database
vinlee19 Nov 8, 2024
0874499
support refresh database command
vinlee19 Nov 8, 2024
3972f9e
Merge remote-tracking branch 'origin/refresh_database_nereids' into r…
vinlee19 Nov 14, 2024
f024ebc
Merge remote-tracking branch 'origin/refresh_database_nereids' into r…
vinlee19 Nov 14, 2024
70796f8
Merge remote-tracking branch 'origin/refresh_database_nereids' into r…
vinlee19 Nov 14, 2024
737cab4
support refresh database command
vinlee19 Nov 8, 2024
1314634
support refresh database
vinlee19 Nov 8, 2024
3e117f4
support refresh database catalog
vinlee19 Nov 19, 2024
53b68fe
Merge remote-tracking branch 'origin/refresh_database_nereids' into r…
vinlee19 Nov 19, 2024
5f1e1c2
support refresh database command
vinlee19 Nov 19, 2024
5002e5e
Merge remote-tracking branch 'origin/refresh_database_nereids' into r…
vinlee19 Nov 19, 2024
5cae955
Merge remote-tracking branch 'origin/refresh_database_nereids' into r…
vinlee19 Nov 19, 2024
99f1faa
Merge remote-tracking branch 'origin/refresh_database_nereids' into r…
vinlee19 Nov 19, 2024
007ae95
support refresh database command
vinlee19 Nov 19, 2024
71a5fed
Merge remote-tracking branch 'origin/refresh_database_nereids' into r…
vinlee19 Nov 19, 2024
ff1aa84
add regression test and delete unuseless code
vinlee19 Nov 19, 2024
94cbdae
Merge remote-tracking branch 'origin/refresh_database_nereids' into r…
vinlee19 Nov 19, 2024
ffb28d3
rollback regression conf
vinlee19 Nov 19, 2024
ebf2adb
Merge branch 'master' into refresh_database_nereids
vinlee19 Nov 19, 2024
4dad95a
Update DorisParser.g4
vinlee19 Nov 19, 2024
5c21a5b
Update DorisParser.g4
vinlee19 Nov 19, 2024
c9842d5
Update DorisParser.g4
vinlee19 Nov 19, 2024
635a417
Merge remote-tracking branch 'origin/refresh_database_nereids' into r…
vinlee19 Nov 19, 2024
52d1fe4
Merge branch 'master' into refresh_database_nereids
vinlee19 Nov 20, 2024
ce3c1ec
remove empty line
vinlee19 Nov 8, 2024
a9aedae
Merge remote-tracking branch 'origin/refresh_database_nereids' into r…
vinlee19 Nov 20, 2024
3b3c286
Revert the formatting of uninvolved codes.
vinlee19 Nov 20, 2024
7e8e493
Revert the formatting of uninvolved codes.
vinlee19 Nov 20, 2024
c395bca
Merge branch 'master' into refresh_database_nereids
vinlee19 Nov 22, 2024
afbb13d
Merge branch 'master' into refresh_database_nereids
vinlee19 Nov 27, 2024
c6304a6
Refine exception descriptions
vinlee19 Nov 29, 2024
0c252c0
remove empty line
vinlee19 Nov 29, 2024
85d6c87
Merge remote-tracking branch 'upstream/master' into refresh_database_…
vinlee19 Nov 29, 2024
e828bb9
remove uselss import
vinlee19 Nov 29, 2024
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -198,25 +198,25 @@ supportedDropStatement

supportedShowStatement
: SHOW (GLOBAL | SESSION | LOCAL)? VARIABLES wildWhere? #showVariables
| SHOW AUTHORS #showAuthors
| SHOW AUTHORS #showAuthors
| SHOW LAST INSERT #showLastInsert
| SHOW ALL? GRANTS #showGrants
| SHOW GRANTS FOR userIdentify #showGrantsForUser
| SHOW VIEW
(FROM |IN) tableName=multipartIdentifier
((FROM | IN) database=identifier)? #showView
| SHOW PLUGINS #showPlugins
| SHOW PLUGINS #showPlugins
| SHOW REPOSITORIES #showRepositories
| SHOW ROLES #showRoles
| SHOW ROLES #showRoles
| SHOW PARTITION partitionId=INTEGER_VALUE #showPartitionId
| SHOW PRIVILEGES #showPrivileges
| SHOW PROC path=STRING_LITERAL #showProc
| SHOW PROC path=STRING_LITERAL #showProc
| SHOW STORAGE? ENGINES #showStorageEngines
| SHOW SQL_BLOCK_RULE (FOR ruleName=identifier)? #showSqlBlockRule
| SHOW CREATE MATERIALIZED VIEW mvName=identifier
ON tableName=multipartIdentifier #showCreateMaterializedView
ON tableName=multipartIdentifier #showCreateMaterializedView
| SHOW BACKENDS #showBackends
| SHOW FRONTENDS name=identifier? #showFrontends
vinlee19 marked this conversation as resolved.
Show resolved Hide resolved
| SHOW FRONTENDS name=identifier? #showFrontends
| SHOW TABLE tableId=INTEGER_VALUE #showTableId
| SHOW WHITELIST #showWhitelist
;
Expand Down Expand Up @@ -421,11 +421,11 @@ channelDescription

supportedRefreshStatement
: REFRESH CATALOG name=identifier propertyClause? #refreshCatalog
| REFRESH DATABASE name=multipartIdentifier propertyClause? #refreshDatabase
;

unsupportedRefreshStatement
: REFRESH TABLE name=multipartIdentifier #refreshTable
| REFRESH DATABASE name=multipartIdentifier propertyClause? #refreshDatabase
| REFRESH LDAP (ALL | (FOR user=identifierOrText)) #refreshLdap
;

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -164,6 +164,7 @@
import org.apache.doris.nereids.DorisParser.QueryTermContext;
import org.apache.doris.nereids.DorisParser.RecoverDatabaseContext;
import org.apache.doris.nereids.DorisParser.RefreshCatalogContext;
import org.apache.doris.nereids.DorisParser.RefreshDatabaseContext;
import org.apache.doris.nereids.DorisParser.RefreshMTMVContext;
import org.apache.doris.nereids.DorisParser.RefreshMethodContext;
import org.apache.doris.nereids.DorisParser.RefreshScheduleContext;
Expand Down Expand Up @@ -522,6 +523,7 @@
import org.apache.doris.nereids.trees.plans.commands.insert.InsertIntoTableCommand;
import org.apache.doris.nereids.trees.plans.commands.insert.InsertOverwriteTableCommand;
import org.apache.doris.nereids.trees.plans.commands.refresh.RefreshCatalogCommand;
import org.apache.doris.nereids.trees.plans.commands.refresh.RefreshDatabaseCommand;
import org.apache.doris.nereids.trees.plans.logical.LogicalAggregate;
import org.apache.doris.nereids.trees.plans.logical.LogicalCTE;
import org.apache.doris.nereids.trees.plans.logical.LogicalExcept;
Expand Down Expand Up @@ -563,6 +565,7 @@
import org.apache.doris.qe.SqlModeHelper;
import org.apache.doris.system.NodeType;

import com.google.common.base.Preconditions;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.ImmutableMap.Builder;
Expand Down Expand Up @@ -4059,7 +4062,6 @@ public SetDefaultStorageVaultCommand visitSetDefaultStorageVault(SetDefaultStora
}

@Override

public Object visitRefreshCatalog(RefreshCatalogContext ctx) {
if (ctx.name != null) {
String catalogName = ctx.name.getText();
Expand All @@ -4070,6 +4072,24 @@ public Object visitRefreshCatalog(RefreshCatalogContext ctx) {
throw new AnalysisException("catalog name can not be null");
}

@Override
public RefreshDatabaseCommand visitRefreshDatabase(RefreshDatabaseContext ctx) {
Map<String, String> properties = visitPropertyClause(ctx.propertyClause()) == null ? Maps.newHashMap()
: visitPropertyClause(ctx.propertyClause());
List<String> parts = visitMultipartIdentifier(ctx.name);
int size = parts.size();
Preconditions.checkArgument(size > 0, "database name can't be empty");
vinlee19 marked this conversation as resolved.
Show resolved Hide resolved
String dbName = parts.get(size - 1);

// [db].
if (size == 1) {
return new RefreshDatabaseCommand(dbName, properties);
} else if (parts.size() == 2) { // [ctl,db].
return new RefreshDatabaseCommand(parts.get(0), dbName, properties);
}
throw new IllegalArgumentException("Only one dot can be in the name: " + String.join(".", parts));
vinlee19 marked this conversation as resolved.
Show resolved Hide resolved
}

public LogicalPlan visitShowLastInsert(ShowLastInsertContext ctx) {
return new ShowLastInsertCommand();
}
Expand All @@ -4093,7 +4113,6 @@ public LogicalPlan visitShowPartitionId(ShowPartitionIdContext ctx) {
partitionId = Long.parseLong(ctx.partitionId.getText());
}
return new ShowPartitionIdCommand(partitionId);

}

@Override
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -175,6 +175,8 @@ public enum PlanType {
SET_USER_PROPERTIES_COMMAND,
SET_DEFAULT_STORAGE_VAULT_COMMAND,
REFRESH_CATALOG_COMMAND,
REFRESH_DATABASE_COMMAND,

vinlee19 marked this conversation as resolved.
Show resolved Hide resolved
PREPARED_COMMAND,
EXECUTE_COMMAND,
SHOW_BACKENDS_COMMAND,
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,149 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.

package org.apache.doris.nereids.trees.plans.commands.refresh;

import org.apache.doris.analysis.StmtType;
import org.apache.doris.catalog.DatabaseIf;
import org.apache.doris.catalog.Env;
import org.apache.doris.catalog.InfoSchemaDb;
import org.apache.doris.catalog.MysqlDb;
import org.apache.doris.common.AnalysisException;
import org.apache.doris.common.DdlException;
import org.apache.doris.common.ErrorCode;
import org.apache.doris.common.ErrorReport;
import org.apache.doris.datasource.CatalogIf;
import org.apache.doris.datasource.ExternalCatalog;
import org.apache.doris.datasource.ExternalDatabase;
import org.apache.doris.datasource.ExternalObjectLog;
import org.apache.doris.mysql.privilege.PrivPredicate;
import org.apache.doris.nereids.trees.plans.PlanType;
import org.apache.doris.nereids.trees.plans.commands.Command;
import org.apache.doris.nereids.trees.plans.commands.ForwardWithSync;
import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor;
import org.apache.doris.qe.ConnectContext;
import org.apache.doris.qe.StmtExecutor;

import com.google.common.base.Strings;

import java.util.Map;

/**
* Refresh database.
*/
public class RefreshDatabaseCommand extends Command implements ForwardWithSync {
private static final String INVALID_CACHE = "invalid_cache";

private String catalogName;
private String dbName;
private Map<String, String> properties;
private boolean invalidCache = false;

public RefreshDatabaseCommand(String dbName, Map<String, String> properties) {
super(PlanType.REFRESH_DATABASE_COMMAND);
this.dbName = dbName;
this.properties = properties;
}

public RefreshDatabaseCommand(String catalogName, String dbName, Map<String, String> properties) {
super(PlanType.REFRESH_DATABASE_COMMAND);
this.catalogName = catalogName;
this.dbName = dbName;
this.properties = properties;
}

private void validate(ConnectContext ctx) throws AnalysisException {
if (Strings.isNullOrEmpty(catalogName)) {
catalogName = ConnectContext.get().getCurrentCatalog().getName();
}
if (Strings.isNullOrEmpty(dbName)) {
ErrorReport.reportAnalysisException(ErrorCode.ERR_WRONG_DB_NAME, dbName);
}

// Don't allow dropping 'information_schema' database
if (dbName.equalsIgnoreCase(InfoSchemaDb.DATABASE_NAME)) {

ErrorReport.reportAnalysisException(
ErrorCode.ERR_DBACCESS_DENIED_ERROR, ctx.getQualifiedUser(), dbName);
}
// Don't allow dropping 'mysql' database
if (dbName.equalsIgnoreCase(MysqlDb.DATABASE_NAME)) {
ErrorReport.reportAnalysisException(
ErrorCode.ERR_DBACCESS_DENIED_ERROR, ctx.getQualifiedUser(), dbName);
}
// check access
if (!Env.getCurrentEnv().getAccessManager().checkDbPriv(ConnectContext.get(), catalogName,
dbName, PrivPredicate.SHOW)) {
ErrorReport.reportAnalysisException(ErrorCode.ERR_DB_ACCESS_DENIED_ERROR,
PrivPredicate.SHOW.getPrivs().toString(), dbName);
}
String invalidConfig = properties == null ? null : properties.get(INVALID_CACHE);
// Default is to invalid cache.
invalidCache = invalidConfig == null || invalidConfig.equalsIgnoreCase("true");
}

/**
* Refresh database
*/
public void handleRefreshDb() throws DdlException {
Env env = Env.getCurrentEnv();
CatalogIf catalog = catalogName != null ? env.getCatalogMgr().getCatalog(catalogName) : env.getCurrentCatalog();
if (catalog == null) {
throw new DdlException("Catalog " + catalogName + " doesn't exist.");
}
if (!(catalog instanceof ExternalCatalog)) {
throw new DdlException("Only support refresh database in external catalog");
}
DatabaseIf db = catalog.getDbOrDdlException(dbName);
((ExternalDatabase<?>) db).setUnInitialized(invalidCache);

ExternalObjectLog log = new ExternalObjectLog();
log.setCatalogId(catalog.getId());
log.setDbId(db.getId());
log.setInvalidCache(invalidCache);
Env.getCurrentEnv().getEditLog().logRefreshExternalDb(log);
}

@Override
public void run(ConnectContext ctx, StmtExecutor executor) throws Exception {
validate(ctx);
handleRefreshDb();
}

@Override
public <R, C> R accept(PlanVisitor<R, C> visitor, C context) {
return visitor.visitRefreshDatabaseCommand(this, context);
}

/**
* refresh database statement.
*/
public String toSql() {
StringBuilder sb = new StringBuilder();
sb.append("REFRESH DATABASE ");
if (catalogName != null) {
sb.append("`").append(catalogName).append("`.");
}
sb.append("`").append(dbName).append("`");
return sb.toString();
}

@Override
public StmtType stmtType() {
return StmtType.REFRESH;
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -81,6 +81,7 @@
import org.apache.doris.nereids.trees.plans.commands.insert.InsertIntoTableCommand;
import org.apache.doris.nereids.trees.plans.commands.insert.InsertOverwriteTableCommand;
import org.apache.doris.nereids.trees.plans.commands.refresh.RefreshCatalogCommand;
import org.apache.doris.nereids.trees.plans.commands.refresh.RefreshDatabaseCommand;

/** CommandVisitor. */
public interface CommandVisitor<R, C> {
Expand Down Expand Up @@ -283,6 +284,10 @@ default R visitShowViewCommand(ShowViewCommand showViewCommand, C context) {
return visitCommand(showViewCommand, context);
}

default R visitRefreshDatabaseCommand(RefreshDatabaseCommand refreshDatabaseCommand, C context) {
return visitCommand(refreshDatabaseCommand, context);
}

default R visitShowBackendsCommand(ShowBackendsCommand showBackendsCommand, C context) {
return visitCommand(showBackendsCommand, context);
}
Expand Down
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
-- This file is automatically generated. You should know what you did if you want to edit this
-- !sql --
-- !database --
DORIS
Doris
doris
Expand All @@ -19,7 +19,7 @@ show_test_do_not_modify
114 abf
115 abg

-- !sql --
-- !preceding_create_external_database --
DORIS
Doris
doris
Expand All @@ -29,7 +29,7 @@ init_db
mysql
show_test_do_not_modify

-- !sql --
-- !subsequent_create_external_database --
DORIS
Doris
doris
Expand All @@ -40,7 +40,21 @@ mysql
new_mysql_db
show_test_do_not_modify

-- !sql --
-- !sql_show_tables --

-- !preceding_refresh_database --

-- !subsequent_refresh_database --
new_mysql_table1

-- !preceding_refresh_database --
new_mysql_table1

-- !subsequent_refresh_database --
new_mysql_table1
new_mysql_table2

-- !preceding_drop_external_database --
DORIS
Doris
doris
Expand All @@ -51,7 +65,7 @@ mysql
new_mysql_db
show_test_do_not_modify

-- !sql --
-- !subsequent_drop_external_database --
DORIS
Doris
doris
Expand Down
Loading
Loading