HDFS-17367. Add PercentUsed for Different StorageTypes in JMX (apache…

…#6735) Contributed by Hualong Zhang. Signed-off-by: Shilun Fan <[email protected]>
ABFSDriver · Apr 27, 2024 · daafc8a · daafc8a
1 parent 88ad7db
commit daafc8a
Show file tree

Hide file tree

Showing 3 changed files with 54 additions and 0 deletions.
diff --git a/hadoop-common-project/hadoop-common/src/site/markdown/Metrics.md b/hadoop-common-project/hadoop-common/src/site/markdown/Metrics.md
@@ -326,6 +326,15 @@ Each metrics record contains tags such as HAState and Hostname as additional inf
 | `FSN(Read/Write)LockOverallNanosAvgTime` | Average time of holding the lock by all operations in nanoseconds |
 | `PendingSPSPaths` | The number of paths to be processed by storage policy satisfier |
 
+BlockManager
+-------------
+
+The metrics present statistics from the BlockManager's perspective.
+
+| Name | Description                                                                                                                     |
+|:---- |:--------------------------------------------------------------------------------------------------------------------------------|
+| `StorageTypeStats` | key represents different StorageTypes, and value represents the detailed storage information corresponding to each StorageType. |
+
 JournalNode
 -----------
 

diff --git a/...op-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/StorageTypeStats.java b/...op-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/StorageTypeStats.java
@@ -24,6 +24,7 @@
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceStability;
 import org.apache.hadoop.fs.StorageType;
+import org.apache.hadoop.hdfs.DFSUtilClient;
 
 /**
  * Statistics per StorageType.
@@ -107,6 +108,24 @@ public long getBlockPoolUsed() {
     return blockPoolUsed;
   }
 
+  public float getPercentUsed() {
+    long used = getCapacityUsed();
+    long total = getCapacityTotal();
+    return DFSUtilClient.getPercentUsed(used, total);
+  }
+
+  public float getPercentBlockPoolUsed() {
+    long poolUsed = getBlockPoolUsed();
+    long total = getCapacityTotal();
+    return DFSUtilClient.getPercentUsed(poolUsed, total);
+  }
+
+  public float getPercentRemaining() {
+    long remaining = getCapacityRemaining();
+    long total = getCapacityTotal();
+    return DFSUtilClient.getPercentUsed(remaining, total);
+  }
+
   public int getNodesInService() {
     return nodesInService;
   }

diff --git a/...dfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockStatsMXBean.java b/...dfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockStatsMXBean.java
@@ -291,4 +291,30 @@ public void testStorageTypeLoad() throws Exception {
             5000);
     IOUtils.closeStreams(hotSpFileStream, coldSpFileStream);
   }
+
+  @Test
+  public void testStorageTypePercentJMX() throws Exception {
+    URL baseUrl = new URL(cluster.getHttpUri(0));
+    String result = readOutput(new URL(baseUrl, "/jmx"));
+
+    Map<String, Object> stat = (Map<String, Object>) JSON.parse(result);
+    Object[] beans = (Object[]) stat.get("beans");
+    Map<String, Object> blockStats = null;
+    for (Object bean : beans) {
+      Map<String, Object> map = (Map<String, Object>) bean;
+      if (map.get("name").equals("Hadoop:service=NameNode,name=BlockStats")) {
+        blockStats = map;
+      }
+    }
+    assertNotNull(blockStats);
+    Object[] storageTypeStatsList =
+        (Object[]) blockStats.get("StorageTypeStats");
+    assertNotNull(storageTypeStatsList);
+    Map<String, Object> entry = (Map<String, Object>) storageTypeStatsList[0];
+    Map<String, Object> storageTypeStats = (Map<String, Object>) entry.get("value");
+
+    assertTrue(storageTypeStats.containsKey("percentUsed"));
+    assertTrue(storageTypeStats.containsKey("percentBlockPoolUsed"));
+    assertTrue(storageTypeStats.containsKey("percentRemaining"));
+  }
 }