HDFS-17694.RBF:Make get datanodes reports configurable.

apache · Dec 19, 2024 · 2f12684 · 2f12684
1 parent 6cb2e86
commit 2f12684
Show file tree

Hide file tree

Showing 3 changed files with 34 additions and 13 deletions.
diff --git a/...f/src/main/java/org/apache/hadoop/hdfs/server/federation/metrics/NamenodeBeanMetrics.java b/...f/src/main/java/org/apache/hadoop/hdfs/server/federation/metrics/NamenodeBeanMetrics.java
@@ -93,7 +93,8 @@ public class NamenodeBeanMetrics
   /** Timeout to get the DN report. */
   private final long dnReportTimeOut;
   /** DN type -> full DN report in JSON. */
-  private final LoadingCache<DatanodeReportType, String> dnCache;
+  private LoadingCache<DatanodeReportType, String> dnCache = null;
+  private final boolean dnReportEnable;
 
 
   public NamenodeBeanMetrics(Router router) {
@@ -139,18 +140,23 @@ public NamenodeBeanMetrics(Router router) {
     this.dnReportTimeOut = conf.getTimeDuration(
         RBFConfigKeys.DN_REPORT_TIME_OUT,
         RBFConfigKeys.DN_REPORT_TIME_OUT_MS_DEFAULT, TimeUnit.MILLISECONDS);
-    long dnCacheExpire = conf.getTimeDuration(
-        RBFConfigKeys.DN_REPORT_CACHE_EXPIRE,
-        RBFConfigKeys.DN_REPORT_CACHE_EXPIRE_MS_DEFAULT, TimeUnit.MILLISECONDS);
-    this.dnCache = CacheBuilder.newBuilder()
-        .expireAfterWrite(dnCacheExpire, TimeUnit.MILLISECONDS)
-        .build(
-            new CacheLoader<DatanodeReportType, String>() {
-              @Override
-              public String load(DatanodeReportType type) throws Exception {
-                return getNodesImpl(type);
-              }
-            });
+    this.dnReportEnable = conf.getBoolean(RBFConfigKeys.DFS_ROUTER_DN_REPORT_ENABLE_KEY,
+            RBFConfigKeys.DFS_ROUTER_DN_REPORT_ENABLE_DEFAULT);
+    if (dnReportEnable) {
+      long dnCacheExpire = conf.getTimeDuration(
+              RBFConfigKeys.DN_REPORT_CACHE_EXPIRE,
+              RBFConfigKeys.DN_REPORT_CACHE_EXPIRE_MS_DEFAULT, TimeUnit.MILLISECONDS);
+      this.dnCache = CacheBuilder.newBuilder()
+              .expireAfterWrite(dnCacheExpire, TimeUnit.MILLISECONDS)
+              .build(
+                      new CacheLoader<DatanodeReportType, String>() {
+                        @Override
+                        public String load(DatanodeReportType type) throws Exception {
+                          return getNodesImpl(type);
+                        }
+                      });
+    }
+
   }
 
   /**
@@ -458,6 +464,9 @@ public String getDecomNodes() {
    * @return JSON with the nodes.
    */
   private String getNodes(final DatanodeReportType type) {
+    if (!dnReportEnable) {
+      return "N/A";
+    }
     try {
       return this.dnCache.get(type);
     } catch (ExecutionException e) {

diff --git a/...hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RBFConfigKeys.java b/...hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RBFConfigKeys.java
@@ -375,6 +375,9 @@ public class RBFConfigKeys extends CommonConfigurationKeysPublic {
   public static final String DFS_ROUTER_ENABLE_GET_DN_USAGE_KEY =
       FEDERATION_ROUTER_PREFIX + "enable.get.dn.usage";
   public static final boolean DFS_ROUTER_ENABLE_GET_DN_USAGE_DEFAULT = true;
+  public static final String DFS_ROUTER_DN_REPORT_ENABLE_KEY =
+          FEDERATION_ROUTER_PREFIX + "dn.report.enable";
+  public static final boolean DFS_ROUTER_DN_REPORT_ENABLE_DEFAULT = true;
 
   // HDFS Router-based federation quota
   public static final String DFS_ROUTER_QUOTA_ENABLE =

diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/resources/hdfs-rbf-default.xml b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/resources/hdfs-rbf-default.xml
@@ -205,6 +205,15 @@
     </description>
   </property>
 
+  <property>
+    <name>dfs.federation.router.dn.report.enable</name>
+    <value>true</value>
+    <description>
+      If false, the getLiveNodes|getDeadNodes|getDecomNodes methods in NamenodeBeanMetrics will
+      return "N/A", else will return the cached result collecting from downstream nameservices.
+    </description>
+  </property>
+
   <property>
     <name>dfs.federation.router.metrics.class</name>
     <value>org.apache.hadoop.hdfs.server.federation.metrics.FederationRPCPerformanceMonitor</value>