Skip to content

Commit

Permalink
[opt](variable) force update some variable by variable version (apach…
Browse files Browse the repository at this point in the history 
…e#41607)

variable version:
000-100: doris-2.0.x
100-200: doris-2.1.x
200-300: doris-3.0.x

update variables

000:
nereids_timeout_second = 30 if original value is 5

100:
enable_nereids_dml = true
enable_nereids_dml_with_pipeline = true
enable_nereids_planner = true
enable_fallback_to_original_planner = true
enable_pipeline_x_engine = true

200:
enable_fallback_to_original_planner = false
  • Loading branch information
@morrySnow
morrySnow authored Nov 7, 2024
1 parent b494574 commit 8c88e35
Show file tree
Hide file tree
Showing 5 changed files with 70 additions and 5 deletions.
1 change: 1 addition & 0 deletions fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1596,6 +1596,7 @@ private void transferToMaster() {
// Set initial root password if master FE first time launch.
auth.setInitialRootPassword(Config.initial_root_password);
} else {
VariableMgr.forceUpdateVariables();
if (journalVersion <= FeMetaVersion.VERSION_114) {
// if journal version is less than 114, which means it is upgraded from version before 2.0.
// When upgrading from 1.2 to 2.0,
Expand Down
6 changes: 4 additions & 2 deletions fe/fe-core/src/main/java/org/apache/doris/common/util/DocGenerator.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -256,8 +256,10 @@ private String genSingleSessionVariableDoc(SessionVariable sv, Field field, Lang
}
sb.append("\n\n");
}
sb.append(VAR_READ_ONLY[lang.idx]).append("`").append(varAttr.flag() == VariableMgr.READ_ONLY).append("`\n\n");
sb.append(VAR_GLOBAL_ONLY[lang.idx]).append("`").append(varAttr.flag() == VariableMgr.GLOBAL).append("`\n\n");
sb.append(VAR_READ_ONLY[lang.idx]).append("`")
.append((varAttr.flag() & VariableMgr.READ_ONLY) != 0).append("`\n\n");
sb.append(VAR_GLOBAL_ONLY[lang.idx]).append("`")
.append((varAttr.flag() & VariableMgr.GLOBAL) != 0).append("`\n\n");
return sb.toString();
}

Expand Down
10 changes: 10 additions & 0 deletions fe/fe-core/src/main/java/org/apache/doris/qe/GlobalVariable.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,13 @@
// NOTE: If you want access your variable safe, please hold VariableMgr's lock before access.
public final class GlobalVariable {

public static final int VARIABLE_VERSION_0 = 0;
public static final int VARIABLE_VERSION_100 = 100;
public static final int VARIABLE_VERSION_200 = 200;
public static final int VARIABLE_VERSION_300 = 300;
public static final int CURRENT_VARIABLE_VERSION = VARIABLE_VERSION_300;
public static final String VARIABLE_VERSION = "variable_version";

public static final String VERSION_COMMENT = "version_comment";
public static final String VERSION = "version";
public static final String LOWER_CASE_TABLE_NAMES = "lower_case_table_names";
Expand Down Expand Up @@ -69,6 +76,9 @@ public final class GlobalVariable {

public static final String ENABLE_FETCH_ICEBERG_STATS = "enable_fetch_iceberg_stats";

@VariableMgr.VarAttr(name = VARIABLE_VERSION, flag = VariableMgr.INVISIBLE
| VariableMgr.READ_ONLY | VariableMgr.GLOBAL)
public static int variableVersion = CURRENT_VARIABLE_VERSION;

@VariableMgr.VarAttr(name = VERSION_COMMENT, flag = VariableMgr.READ_ONLY)
public static String versionComment = "Doris version "
Expand Down
3 changes: 1 addition & 2 deletions fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1518,7 +1518,7 @@ public void setEnableLeftZigZag(boolean enableLeftZigZag) {
@VariableMgr.VarAttr(name = ENABLE_NEREIDS_TIMEOUT, needForward = true)
public boolean enableNereidsTimeout = true;

@VariableMgr.VarAttr(name = "nereids_timeout_second", needForward = true)
@VariableMgr.VarAttr(name = NEREIDS_TIMEOUT_SECOND, needForward = true)
public int nereidsTimeoutSecond = 30;

@VariableMgr.VarAttr(name = ENABLE_PUSH_DOWN_NO_GROUP_AGG)
Expand Down Expand Up @@ -3928,7 +3928,6 @@ public void readFromJson(String json) throws IOException {
if (attr == null) {
continue;
}

if (!root.containsKey(attr.name())) {
continue;
}
Expand Down
55 changes: 54 additions & 1 deletion fe/fe-core/src/main/java/org/apache/doris/qe/VariableMgr.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -469,6 +469,11 @@ public static void replayGlobalVariableV2(GlobalVarPersistInfo info) throws DdlE
try {
String json = info.getPersistJsonString();
JSONObject root = (JSONObject) JSONValue.parse(json);
// if not variable version, we set it to 0 to ensure we could force set global variable.
boolean hasVariableVersion = root.containsKey(GlobalVariable.VARIABLE_VERSION);
if (!hasVariableVersion) {
GlobalVariable.variableVersion = GlobalVariable.VARIABLE_VERSION_0;
}
for (Object varName : root.keySet()) {
VarContext varContext = ctxByVarName.get((String) varName);
if (Env.isCheckpointThread()) {
Expand Down Expand Up @@ -728,11 +733,15 @@ public static List<List<String>> dump(SetType type, SessionVariable sessionVar,
rlock.lock();
try {
for (Map.Entry<String, VarContext> entry : ctxByDisplayVarName.entrySet()) {
// not show removed variables
VarAttr varAttr = entry.getValue().getField().getAnnotation(VarAttr.class);
// not show removed variables
if (VariableAnnotation.REMOVED.equals(varAttr.varType())) {
continue;
}
// not show invisible variables
if ((VariableMgr.INVISIBLE & varAttr.flag()) != 0) {
continue;
}
// Filter variable not match to the regex.
if (matcher != null && !matcher.match(entry.getKey())) {
continue;
Expand Down Expand Up @@ -947,4 +956,48 @@ private static ImmutableSortedMap.Builder<String, VarContext> getStringVarContex
}
return builder;
}

public static void forceUpdateVariables() {
int currentVariableVersion = GlobalVariable.variableVersion;
if (currentVariableVersion == GlobalVariable.VARIABLE_VERSION_0) {
// update from 2.0.15 or below to 2.0.16 or higher
if (VariableMgr.newSessionVariable().nereidsTimeoutSecond == 5) {
VariableMgr.refreshDefaultSessionVariables("update variable version",
SessionVariable.NEREIDS_TIMEOUT_SECOND, "30");
}
}
if (currentVariableVersion < GlobalVariable.VARIABLE_VERSION_100) {
// update from 2.1.6 or below to 2.1.7 or higher
VariableMgr.refreshDefaultSessionVariables("update variable version",
SessionVariable.ENABLE_NEREIDS_DML,
String.valueOf(true));
VariableMgr.refreshDefaultSessionVariables("update variable version",
SessionVariable.ENABLE_NEREIDS_DML_WITH_PIPELINE,
String.valueOf(true));
VariableMgr.refreshDefaultSessionVariables("update variable version",
SessionVariable.ENABLE_NEREIDS_PLANNER,
String.valueOf(true));
VariableMgr.refreshDefaultSessionVariables("update variable version",
SessionVariable.ENABLE_FALLBACK_TO_ORIGINAL_PLANNER,
String.valueOf(true));
VariableMgr.refreshDefaultSessionVariables("update variable version",
SessionVariable.ENABLE_PIPELINE_X_ENGINE,
String.valueOf(true));
}
if (currentVariableVersion < GlobalVariable.VARIABLE_VERSION_200) {
// update from 3.0.2 or below to 3.0.3 or higher
VariableMgr.refreshDefaultSessionVariables("update variable version",
SessionVariable.ENABLE_FALLBACK_TO_ORIGINAL_PLANNER,
String.valueOf(false));
}
if (currentVariableVersion < GlobalVariable.VARIABLE_VERSION_300) {
// update to master
// do nothing
}
if (currentVariableVersion < GlobalVariable.CURRENT_VARIABLE_VERSION) {
VariableMgr.refreshDefaultSessionVariables("update variable version",
GlobalVariable.VARIABLE_VERSION,
String.valueOf(GlobalVariable.CURRENT_VARIABLE_VERSION));
}
}
}

0 comments on commit 8c88e35

Please sign in to comment.