From 7fdd17dbb4e8c79cda0b5f1d5ed6927eac2644eb Mon Sep 17 00:00:00 2001 From: Roc Marshal Date: Thu, 5 Dec 2024 19:50:37 +0800 Subject: [PATCH 1/3] [conf][server] Make the size of the threads pool in tablet server scheduler configurable. --- .../main/java/com/alibaba/fluss/config/ConfigOptions.java | 7 +++++++ .../java/com/alibaba/fluss/server/tablet/TabletServer.java | 5 +++-- website/docs/maintenance/configuration.md | 1 + 3 files changed, 11 insertions(+), 2 deletions(-) diff --git a/fluss-common/src/main/java/com/alibaba/fluss/config/ConfigOptions.java b/fluss-common/src/main/java/com/alibaba/fluss/config/ConfigOptions.java index 3de4ff07..8a0463a6 100644 --- a/fluss-common/src/main/java/com/alibaba/fluss/config/ConfigOptions.java +++ b/fluss-common/src/main/java/com/alibaba/fluss/config/ConfigOptions.java @@ -210,6 +210,13 @@ public class ConfigOptions { "This configuration controls the directory where fluss will store its data. " + "The default value is /tmp/fluss-data"); + public static final ConfigOption SCHEDULER_THREADS = + key("scheduler.threads") + .intType() + .defaultValue(10) + .withDescription( + "This configuration item to set the core threads for the fluss scheduler in tablet servers."); + public static final ConfigOption WRITER_ID_EXPIRATION_TIME = key("server.writer-id.expiration-time") .durationType() diff --git a/fluss-server/src/main/java/com/alibaba/fluss/server/tablet/TabletServer.java b/fluss-server/src/main/java/com/alibaba/fluss/server/tablet/TabletServer.java index e6c6911e..d823234b 100644 --- a/fluss-server/src/main/java/com/alibaba/fluss/server/tablet/TabletServer.java +++ b/fluss-server/src/main/java/com/alibaba/fluss/server/tablet/TabletServer.java @@ -59,6 +59,8 @@ import java.util.concurrent.CompletableFuture; import java.util.concurrent.atomic.AtomicBoolean; +import static com.alibaba.fluss.config.ConfigOptions.SCHEDULER_THREADS; + /** * Tablet server implementation. The tablet server is responsible to manage the log tablet and kv * tablet. @@ -151,8 +153,7 @@ protected void startServices() throws Exception { this.metadataCache = new ServerMetadataCacheImpl(); - // TODO set scheduler thread number. - this.scheduler = new FlussScheduler(10); + this.scheduler = new FlussScheduler(conf.get(SCHEDULER_THREADS)); scheduler.startup(); this.logManager = diff --git a/website/docs/maintenance/configuration.md b/website/docs/maintenance/configuration.md index 0b456b58..49c4cfcf 100644 --- a/website/docs/maintenance/configuration.md +++ b/website/docs/maintenance/configuration.md @@ -58,6 +58,7 @@ during the Fluss cluster working. | data.dir | String | /tmp/fluss-data | This configuration controls the directory where fluss will store its data. The default value is /tmp/fluss-data | | server.writer-id.expiration-time | Duration | 7d | The time that the tablet server will wait without receiving any write request from a client before expiring the related status. The default value is 7 days. | | server.writer-id.expiration-check-interval | Duration | 10min | The interval at which to remove writer ids that have expired due to 'server.writer-id.expiration-time passing. The default value is 10 minutes. | +| scheduler.threads | Integer | 10 | This configuration item to set the core threads for the fluss scheduler in tablet servers. The default value is 10. | ### Zookeeper From 77f5cea54ac4d7b2eb39d9c89a9f4cbdf7e8e994 Mon Sep 17 00:00:00 2001 From: Roc Marshal Date: Mon, 9 Dec 2024 09:43:06 +0800 Subject: [PATCH 2/3] Address comments --- .../com/alibaba/fluss/config/ConfigOptions.java | 14 +++++++------- .../alibaba/fluss/server/tablet/TabletServer.java | 4 ++-- website/docs/maintenance/configuration.md | 2 +- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/fluss-common/src/main/java/com/alibaba/fluss/config/ConfigOptions.java b/fluss-common/src/main/java/com/alibaba/fluss/config/ConfigOptions.java index 8a0463a6..3ca929be 100644 --- a/fluss-common/src/main/java/com/alibaba/fluss/config/ConfigOptions.java +++ b/fluss-common/src/main/java/com/alibaba/fluss/config/ConfigOptions.java @@ -202,6 +202,13 @@ public class ConfigOptions { .noDefaultValue() .withDescription("The id for the tablet server."); + public static final ConfigOption TABLET_SERVER_SCHEDULER_THREADS = + key("tablet-server.scheduler.threads") + .intType() + .defaultValue(10) + .withDescription( + "This configuration item to set the core threads for the fluss scheduler in tablet servers."); + public static final ConfigOption DATA_DIR = key("data.dir") .stringType() @@ -210,13 +217,6 @@ public class ConfigOptions { "This configuration controls the directory where fluss will store its data. " + "The default value is /tmp/fluss-data"); - public static final ConfigOption SCHEDULER_THREADS = - key("scheduler.threads") - .intType() - .defaultValue(10) - .withDescription( - "This configuration item to set the core threads for the fluss scheduler in tablet servers."); - public static final ConfigOption WRITER_ID_EXPIRATION_TIME = key("server.writer-id.expiration-time") .durationType() diff --git a/fluss-server/src/main/java/com/alibaba/fluss/server/tablet/TabletServer.java b/fluss-server/src/main/java/com/alibaba/fluss/server/tablet/TabletServer.java index d823234b..6414f19a 100644 --- a/fluss-server/src/main/java/com/alibaba/fluss/server/tablet/TabletServer.java +++ b/fluss-server/src/main/java/com/alibaba/fluss/server/tablet/TabletServer.java @@ -59,7 +59,7 @@ import java.util.concurrent.CompletableFuture; import java.util.concurrent.atomic.AtomicBoolean; -import static com.alibaba.fluss.config.ConfigOptions.SCHEDULER_THREADS; +import static com.alibaba.fluss.config.ConfigOptions.TABLET_SERVER_SCHEDULER_THREADS; /** * Tablet server implementation. The tablet server is responsible to manage the log tablet and kv @@ -153,7 +153,7 @@ protected void startServices() throws Exception { this.metadataCache = new ServerMetadataCacheImpl(); - this.scheduler = new FlussScheduler(conf.get(SCHEDULER_THREADS)); + this.scheduler = new FlussScheduler(conf.get(TABLET_SERVER_SCHEDULER_THREADS)); scheduler.startup(); this.logManager = diff --git a/website/docs/maintenance/configuration.md b/website/docs/maintenance/configuration.md index 49c4cfcf..09730c2d 100644 --- a/website/docs/maintenance/configuration.md +++ b/website/docs/maintenance/configuration.md @@ -55,10 +55,10 @@ during the Fluss cluster working. | tablet-server.host | String | (None) | The external address of the network interface where the TabletServer is exposed. Because different TabletServer need different values for this option, usually it is specified in an additional non-shared TabletServer-specific config file. | | tablet-server.port | String | 0 | The external RPC port where the TabletServer is exposed. | | tablet-server.id | Integer | (None) | The id for the tablet server. | +| tablet-server.scheduler.threads | Integer | 10 | This configuration item to set the core threads for the fluss scheduler in tablet servers. The default value is 10. | | data.dir | String | /tmp/fluss-data | This configuration controls the directory where fluss will store its data. The default value is /tmp/fluss-data | | server.writer-id.expiration-time | Duration | 7d | The time that the tablet server will wait without receiving any write request from a client before expiring the related status. The default value is 7 days. | | server.writer-id.expiration-check-interval | Duration | 10min | The interval at which to remove writer ids that have expired due to 'server.writer-id.expiration-time passing. The default value is 10 minutes. | -| scheduler.threads | Integer | 10 | This configuration item to set the core threads for the fluss scheduler in tablet servers. The default value is 10. | ### Zookeeper From b2cc0bec64cfd08bf170d0559f0344c959e2c9db Mon Sep 17 00:00:00 2001 From: Roc Marshal Date: Tue, 17 Dec 2024 14:50:43 +0800 Subject: [PATCH 3/3] Address comments. --- .../com/alibaba/fluss/config/ConfigOptions.java | 14 +++++++------- .../alibaba/fluss/server/tablet/TabletServer.java | 4 ++-- website/docs/maintenance/configuration.md | 2 +- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/fluss-common/src/main/java/com/alibaba/fluss/config/ConfigOptions.java b/fluss-common/src/main/java/com/alibaba/fluss/config/ConfigOptions.java index 3ca929be..12f842a5 100644 --- a/fluss-common/src/main/java/com/alibaba/fluss/config/ConfigOptions.java +++ b/fluss-common/src/main/java/com/alibaba/fluss/config/ConfigOptions.java @@ -202,13 +202,6 @@ public class ConfigOptions { .noDefaultValue() .withDescription("The id for the tablet server."); - public static final ConfigOption TABLET_SERVER_SCHEDULER_THREADS = - key("tablet-server.scheduler.threads") - .intType() - .defaultValue(10) - .withDescription( - "This configuration item to set the core threads for the fluss scheduler in tablet servers."); - public static final ConfigOption DATA_DIR = key("data.dir") .stringType() @@ -234,6 +227,13 @@ public class ConfigOptions { + WRITER_ID_EXPIRATION_TIME.key() + " passing. The default value is 10 minutes."); + public static final ConfigOption BACKGROUND_THREADS = + key("server.background.threads") + .intType() + .defaultValue(10) + .withDescription( + "The number of threads to use for various background processing tasks."); + // ------------------------------------------------------------------ // ZooKeeper Settings // ------------------------------------------------------------------ diff --git a/fluss-server/src/main/java/com/alibaba/fluss/server/tablet/TabletServer.java b/fluss-server/src/main/java/com/alibaba/fluss/server/tablet/TabletServer.java index 6414f19a..b36f5eba 100644 --- a/fluss-server/src/main/java/com/alibaba/fluss/server/tablet/TabletServer.java +++ b/fluss-server/src/main/java/com/alibaba/fluss/server/tablet/TabletServer.java @@ -59,7 +59,7 @@ import java.util.concurrent.CompletableFuture; import java.util.concurrent.atomic.AtomicBoolean; -import static com.alibaba.fluss.config.ConfigOptions.TABLET_SERVER_SCHEDULER_THREADS; +import static com.alibaba.fluss.config.ConfigOptions.BACKGROUND_THREADS; /** * Tablet server implementation. The tablet server is responsible to manage the log tablet and kv @@ -153,7 +153,7 @@ protected void startServices() throws Exception { this.metadataCache = new ServerMetadataCacheImpl(); - this.scheduler = new FlussScheduler(conf.get(TABLET_SERVER_SCHEDULER_THREADS)); + this.scheduler = new FlussScheduler(conf.get(BACKGROUND_THREADS)); scheduler.startup(); this.logManager = diff --git a/website/docs/maintenance/configuration.md b/website/docs/maintenance/configuration.md index 09730c2d..7f3ad146 100644 --- a/website/docs/maintenance/configuration.md +++ b/website/docs/maintenance/configuration.md @@ -55,10 +55,10 @@ during the Fluss cluster working. | tablet-server.host | String | (None) | The external address of the network interface where the TabletServer is exposed. Because different TabletServer need different values for this option, usually it is specified in an additional non-shared TabletServer-specific config file. | | tablet-server.port | String | 0 | The external RPC port where the TabletServer is exposed. | | tablet-server.id | Integer | (None) | The id for the tablet server. | -| tablet-server.scheduler.threads | Integer | 10 | This configuration item to set the core threads for the fluss scheduler in tablet servers. The default value is 10. | | data.dir | String | /tmp/fluss-data | This configuration controls the directory where fluss will store its data. The default value is /tmp/fluss-data | | server.writer-id.expiration-time | Duration | 7d | The time that the tablet server will wait without receiving any write request from a client before expiring the related status. The default value is 7 days. | | server.writer-id.expiration-check-interval | Duration | 10min | The interval at which to remove writer ids that have expired due to 'server.writer-id.expiration-time passing. The default value is 10 minutes. | +| server.background.threads | Integer | 10 | The number of threads to use for various background processing tasks. The default value is 10. | ### Zookeeper