From 8c88e356519c95e64b624c2669b5ffb9749d4c71 Mon Sep 17 00:00:00 2001 From: morrySnow <101034200+morrySnow@users.noreply.github.com> Date: Thu, 7 Nov 2024 14:10:30 +0800 Subject: [PATCH] [opt](variable) force update some variable by variable version (#41607) variable version: 000-100: doris-2.0.x 100-200: doris-2.1.x 200-300: doris-3.0.x update variables 000: nereids_timeout_second = 30 if original value is 5 100: enable_nereids_dml = true enable_nereids_dml_with_pipeline = true enable_nereids_planner = true enable_fallback_to_original_planner = true enable_pipeline_x_engine = true 200: enable_fallback_to_original_planner = false --- .../java/org/apache/doris/catalog/Env.java | 1 + .../doris/common/util/DocGenerator.java | 6 +- .../org/apache/doris/qe/GlobalVariable.java | 10 ++++ .../org/apache/doris/qe/SessionVariable.java | 3 +- .../java/org/apache/doris/qe/VariableMgr.java | 55 ++++++++++++++++++- 5 files changed, 70 insertions(+), 5 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java index 822c1954540138..e5ef8e0dc721ab 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java @@ -1596,6 +1596,7 @@ private void transferToMaster() { // Set initial root password if master FE first time launch. auth.setInitialRootPassword(Config.initial_root_password); } else { + VariableMgr.forceUpdateVariables(); if (journalVersion <= FeMetaVersion.VERSION_114) { // if journal version is less than 114, which means it is upgraded from version before 2.0. // When upgrading from 1.2 to 2.0, diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/util/DocGenerator.java b/fe/fe-core/src/main/java/org/apache/doris/common/util/DocGenerator.java index c0a544d825dcce..003dcdcf59867e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/util/DocGenerator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/common/util/DocGenerator.java @@ -256,8 +256,10 @@ private String genSingleSessionVariableDoc(SessionVariable sv, Field field, Lang } sb.append("\n\n"); } - sb.append(VAR_READ_ONLY[lang.idx]).append("`").append(varAttr.flag() == VariableMgr.READ_ONLY).append("`\n\n"); - sb.append(VAR_GLOBAL_ONLY[lang.idx]).append("`").append(varAttr.flag() == VariableMgr.GLOBAL).append("`\n\n"); + sb.append(VAR_READ_ONLY[lang.idx]).append("`") + .append((varAttr.flag() & VariableMgr.READ_ONLY) != 0).append("`\n\n"); + sb.append(VAR_GLOBAL_ONLY[lang.idx]).append("`") + .append((varAttr.flag() & VariableMgr.GLOBAL) != 0).append("`\n\n"); return sb.toString(); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/GlobalVariable.java b/fe/fe-core/src/main/java/org/apache/doris/qe/GlobalVariable.java index b449a4bc8e9b4e..45a6acba6dc38f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/GlobalVariable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/GlobalVariable.java @@ -33,6 +33,13 @@ // NOTE: If you want access your variable safe, please hold VariableMgr's lock before access. public final class GlobalVariable { + public static final int VARIABLE_VERSION_0 = 0; + public static final int VARIABLE_VERSION_100 = 100; + public static final int VARIABLE_VERSION_200 = 200; + public static final int VARIABLE_VERSION_300 = 300; + public static final int CURRENT_VARIABLE_VERSION = VARIABLE_VERSION_300; + public static final String VARIABLE_VERSION = "variable_version"; + public static final String VERSION_COMMENT = "version_comment"; public static final String VERSION = "version"; public static final String LOWER_CASE_TABLE_NAMES = "lower_case_table_names"; @@ -69,6 +76,9 @@ public final class GlobalVariable { public static final String ENABLE_FETCH_ICEBERG_STATS = "enable_fetch_iceberg_stats"; + @VariableMgr.VarAttr(name = VARIABLE_VERSION, flag = VariableMgr.INVISIBLE + | VariableMgr.READ_ONLY | VariableMgr.GLOBAL) + public static int variableVersion = CURRENT_VARIABLE_VERSION; @VariableMgr.VarAttr(name = VERSION_COMMENT, flag = VariableMgr.READ_ONLY) public static String versionComment = "Doris version " diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java index 88919c290d8407..45df14a6e6f2ee 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java @@ -1518,7 +1518,7 @@ public void setEnableLeftZigZag(boolean enableLeftZigZag) { @VariableMgr.VarAttr(name = ENABLE_NEREIDS_TIMEOUT, needForward = true) public boolean enableNereidsTimeout = true; - @VariableMgr.VarAttr(name = "nereids_timeout_second", needForward = true) + @VariableMgr.VarAttr(name = NEREIDS_TIMEOUT_SECOND, needForward = true) public int nereidsTimeoutSecond = 30; @VariableMgr.VarAttr(name = ENABLE_PUSH_DOWN_NO_GROUP_AGG) @@ -3928,7 +3928,6 @@ public void readFromJson(String json) throws IOException { if (attr == null) { continue; } - if (!root.containsKey(attr.name())) { continue; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/VariableMgr.java b/fe/fe-core/src/main/java/org/apache/doris/qe/VariableMgr.java index a69ca0776e24c2..b7a4f8bb710074 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/VariableMgr.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/VariableMgr.java @@ -469,6 +469,11 @@ public static void replayGlobalVariableV2(GlobalVarPersistInfo info) throws DdlE try { String json = info.getPersistJsonString(); JSONObject root = (JSONObject) JSONValue.parse(json); + // if not variable version, we set it to 0 to ensure we could force set global variable. + boolean hasVariableVersion = root.containsKey(GlobalVariable.VARIABLE_VERSION); + if (!hasVariableVersion) { + GlobalVariable.variableVersion = GlobalVariable.VARIABLE_VERSION_0; + } for (Object varName : root.keySet()) { VarContext varContext = ctxByVarName.get((String) varName); if (Env.isCheckpointThread()) { @@ -728,11 +733,15 @@ public static List> dump(SetType type, SessionVariable sessionVar, rlock.lock(); try { for (Map.Entry entry : ctxByDisplayVarName.entrySet()) { - // not show removed variables VarAttr varAttr = entry.getValue().getField().getAnnotation(VarAttr.class); + // not show removed variables if (VariableAnnotation.REMOVED.equals(varAttr.varType())) { continue; } + // not show invisible variables + if ((VariableMgr.INVISIBLE & varAttr.flag()) != 0) { + continue; + } // Filter variable not match to the regex. if (matcher != null && !matcher.match(entry.getKey())) { continue; @@ -947,4 +956,48 @@ private static ImmutableSortedMap.Builder getStringVarContex } return builder; } + + public static void forceUpdateVariables() { + int currentVariableVersion = GlobalVariable.variableVersion; + if (currentVariableVersion == GlobalVariable.VARIABLE_VERSION_0) { + // update from 2.0.15 or below to 2.0.16 or higher + if (VariableMgr.newSessionVariable().nereidsTimeoutSecond == 5) { + VariableMgr.refreshDefaultSessionVariables("update variable version", + SessionVariable.NEREIDS_TIMEOUT_SECOND, "30"); + } + } + if (currentVariableVersion < GlobalVariable.VARIABLE_VERSION_100) { + // update from 2.1.6 or below to 2.1.7 or higher + VariableMgr.refreshDefaultSessionVariables("update variable version", + SessionVariable.ENABLE_NEREIDS_DML, + String.valueOf(true)); + VariableMgr.refreshDefaultSessionVariables("update variable version", + SessionVariable.ENABLE_NEREIDS_DML_WITH_PIPELINE, + String.valueOf(true)); + VariableMgr.refreshDefaultSessionVariables("update variable version", + SessionVariable.ENABLE_NEREIDS_PLANNER, + String.valueOf(true)); + VariableMgr.refreshDefaultSessionVariables("update variable version", + SessionVariable.ENABLE_FALLBACK_TO_ORIGINAL_PLANNER, + String.valueOf(true)); + VariableMgr.refreshDefaultSessionVariables("update variable version", + SessionVariable.ENABLE_PIPELINE_X_ENGINE, + String.valueOf(true)); + } + if (currentVariableVersion < GlobalVariable.VARIABLE_VERSION_200) { + // update from 3.0.2 or below to 3.0.3 or higher + VariableMgr.refreshDefaultSessionVariables("update variable version", + SessionVariable.ENABLE_FALLBACK_TO_ORIGINAL_PLANNER, + String.valueOf(false)); + } + if (currentVariableVersion < GlobalVariable.VARIABLE_VERSION_300) { + // update to master + // do nothing + } + if (currentVariableVersion < GlobalVariable.CURRENT_VARIABLE_VERSION) { + VariableMgr.refreshDefaultSessionVariables("update variable version", + GlobalVariable.VARIABLE_VERSION, + String.valueOf(GlobalVariable.CURRENT_VARIABLE_VERSION)); + } + } }