From bca50e9fdd5a07eb1290c4d9f60d55f78c12dacf Mon Sep 17 00:00:00 2001 From: Kaijie Chen Date: Tue, 10 Dec 2024 16:28:13 +0800 Subject: [PATCH] [performance](load) increase max_broker_concurrency to 100 (#44929) Increase default `max_broker_concurrency` to 100 to improve broker load performance. This option will affect the max number of scan / sink instances allowed in a broker load. ``` parallel instance = min(max_broker_concurrency, source file size / min_bytes_per_broker_scanner, num backends * load_parallelism) ``` S3 load time of tpcds_1000g catalog_sales: * before: 438s * after: 225s --- fe/fe-common/src/main/java/org/apache/doris/common/Config.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java index fa98487d19f745..92bf9835906cd9 100644 --- a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java +++ b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java @@ -776,7 +776,7 @@ public class Config extends ConfigBase { @ConfField(mutable = true, masterOnly = true, description = { "单个 broker scanner 的最大并发数。", "Maximal concurrency of broker scanners."}) - public static int max_broker_concurrency = 10; + public static int max_broker_concurrency = 100; // TODO(cmy): Disable by default because current checksum logic has some bugs. @ConfField(mutable = true, masterOnly = true, description = {