HDFS-17641 .Psane/backport hdfs 17641 to 3.3

hadoop-common-project/hadoop-common/src/site/markdown/Metrics.md

@@ -285,6 +285,7 @@ Each metrics record contains tags such as HAState and Hostname as additional inf
 | `StaleDataNodes` | Current number of DataNodes marked stale due to delayed heartbeat |
 | `NumStaleStorages` | Number of storages marked as content stale (after NameNode restart/failover before first block report is received) |
 | `MissingReplOneBlocks` | Current number of missing blocks with replication factor 1 |
+| `BadlyDistributedBlocks` | Current number of blocks that are badly distributed across racks. |
 | `HighestPriorityLowRedundancyReplicatedBlocks` | Current number of non-corrupt, low redundancy replicated blocks with the highest risk of loss (have 0 or 1 replica). Will be recovered with the highest priority. |
 | `HighestPriorityLowRedundancyECBlocks` | Current number of non-corrupt, low redundancy EC blocks with the highest risk of loss. Will be recovered with the highest priority. |
 | `NumFilesUnderConstruction` | Current number of files under construction |

hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ECBlockGroupStats.java

@@ -38,24 +38,28 @@ public final class ECBlockGroupStats {
   private final long missingBlockGroups;
   private final long bytesInFutureBlockGroups;
   private final long pendingDeletionBlocks;
+  private final long badlyDistributedBlocks;
   private final Long highestPriorityLowRedundancyBlocks;
 
   public ECBlockGroupStats(long lowRedundancyBlockGroups,
       long corruptBlockGroups, long missingBlockGroups,
-      long bytesInFutureBlockGroups, long pendingDeletionBlocks) {
+      long bytesInFutureBlockGroups, long pendingDeletionBlocks,
+      long badlyDistributedBlocks) {
     this(lowRedundancyBlockGroups, corruptBlockGroups, missingBlockGroups,
-        bytesInFutureBlockGroups, pendingDeletionBlocks, null);
+        bytesInFutureBlockGroups, pendingDeletionBlocks,
+        badlyDistributedBlocks, null);
   }
 
   public ECBlockGroupStats(long lowRedundancyBlockGroups,
       long corruptBlockGroups, long missingBlockGroups,
       long bytesInFutureBlockGroups, long pendingDeletionBlocks,
-      Long highestPriorityLowRedundancyBlocks) {
+      long badlyDistributedBlocks, Long highestPriorityLowRedundancyBlocks) {
     this.lowRedundancyBlockGroups = lowRedundancyBlockGroups;
     this.corruptBlockGroups = corruptBlockGroups;
     this.missingBlockGroups = missingBlockGroups;
     this.bytesInFutureBlockGroups = bytesInFutureBlockGroups;
     this.pendingDeletionBlocks = pendingDeletionBlocks;
+    this.badlyDistributedBlocks = badlyDistributedBlocks;
     this.highestPriorityLowRedundancyBlocks
         = highestPriorityLowRedundancyBlocks;
   }
@@ -80,6 +84,10 @@ public long getPendingDeletionBlocks() {
     return pendingDeletionBlocks;
   }
 
+  public long getBadlyDistributedBlocks() {
+    return badlyDistributedBlocks;
+  }
+
   public boolean hasHighestPriorityLowRedundancyBlocks() {
     return getHighestPriorityLowRedundancyBlocks() != null;
   }
@@ -99,7 +107,8 @@ public String toString() {
         .append(", BytesInFutureBlockGroups=").append(
             getBytesInFutureBlockGroups())
         .append(", PendingDeletionBlocks=").append(
-            getPendingDeletionBlocks());
+            getPendingDeletionBlocks())
+        .append(" , BadlyDistributedBlocks=").append(getBadlyDistributedBlocks());
     if (hasHighestPriorityLowRedundancyBlocks()) {
       statsBuilder.append(", HighestPriorityLowRedundancyBlocks=")
           .append(getHighestPriorityLowRedundancyBlocks());
@@ -116,6 +125,7 @@ public int hashCode() {
         .append(missingBlockGroups)
         .append(bytesInFutureBlockGroups)
         .append(pendingDeletionBlocks)
+        .append(badlyDistributedBlocks)
         .append(highestPriorityLowRedundancyBlocks)
         .toHashCode();
   }
@@ -135,6 +145,7 @@ public boolean equals(Object o) {
         .append(missingBlockGroups, other.missingBlockGroups)
         .append(bytesInFutureBlockGroups, other.bytesInFutureBlockGroups)
         .append(pendingDeletionBlocks, other.pendingDeletionBlocks)
+        .append(badlyDistributedBlocks, other.badlyDistributedBlocks)
         .append(highestPriorityLowRedundancyBlocks,
             other.highestPriorityLowRedundancyBlocks)
         .isEquals();
@@ -151,6 +162,7 @@ public static ECBlockGroupStats merge(Collection<ECBlockGroupStats> stats) {
     long missingBlockGroups = 0;
     long bytesInFutureBlockGroups = 0;
     long pendingDeletionBlocks = 0;
+    long badlyDistributedBlocks = 0;
     long highestPriorityLowRedundancyBlocks = 0;
     boolean hasHighestPriorityLowRedundancyBlocks = false;
 
@@ -160,6 +172,7 @@ public static ECBlockGroupStats merge(Collection<ECBlockGroupStats> stats) {
       missingBlockGroups += stat.getMissingBlockGroups();
       bytesInFutureBlockGroups += stat.getBytesInFutureBlockGroups();
       pendingDeletionBlocks += stat.getPendingDeletionBlocks();
+      badlyDistributedBlocks += stat.getBadlyDistributedBlocks();
       if (stat.hasHighestPriorityLowRedundancyBlocks()) {
         hasHighestPriorityLowRedundancyBlocks = true;
         highestPriorityLowRedundancyBlocks +=
@@ -169,9 +182,10 @@ public static ECBlockGroupStats merge(Collection<ECBlockGroupStats> stats) {
     if (hasHighestPriorityLowRedundancyBlocks) {
       return new ECBlockGroupStats(lowRedundancyBlockGroups, corruptBlockGroups,
           missingBlockGroups, bytesInFutureBlockGroups, pendingDeletionBlocks,
-          highestPriorityLowRedundancyBlocks);
+          badlyDistributedBlocks, highestPriorityLowRedundancyBlocks);
     }
     return new ECBlockGroupStats(lowRedundancyBlockGroups, corruptBlockGroups,
-        missingBlockGroups, bytesInFutureBlockGroups, pendingDeletionBlocks);
+        missingBlockGroups, bytesInFutureBlockGroups, pendingDeletionBlocks,
+        badlyDistributedBlocks);
   }
 }

hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ReplicatedBlockStats.java

@@ -37,27 +37,30 @@ public final class ReplicatedBlockStats {
   private final long missingReplicationOneBlocks;
   private final long bytesInFutureBlocks;
   private final long pendingDeletionBlocks;
+  private final long badlyDistributedBlocks;
   private final Long highestPriorityLowRedundancyBlocks;
 
   public ReplicatedBlockStats(long lowRedundancyBlocks,
       long corruptBlocks, long missingBlocks,
       long missingReplicationOneBlocks, long bytesInFutureBlocks,
-      long pendingDeletionBlocks) {
+      long pendingDeletionBlocks, long badlyDistributedBlocks) {
     this(lowRedundancyBlocks, corruptBlocks, missingBlocks,
         missingReplicationOneBlocks, bytesInFutureBlocks, pendingDeletionBlocks,
-        null);
+        badlyDistributedBlocks, null);
   }
 
   public ReplicatedBlockStats(long lowRedundancyBlocks,
       long corruptBlocks, long missingBlocks,
       long missingReplicationOneBlocks, long bytesInFutureBlocks,
-      long pendingDeletionBlocks, Long highestPriorityLowRedundancyBlocks) {
+      long pendingDeletionBlocks, long badlyDistributedBlocks,
+      Long highestPriorityLowRedundancyBlocks) {
     this.lowRedundancyBlocks = lowRedundancyBlocks;
     this.corruptBlocks = corruptBlocks;
     this.missingBlocks = missingBlocks;
     this.missingReplicationOneBlocks = missingReplicationOneBlocks;
     this.bytesInFutureBlocks = bytesInFutureBlocks;
     this.pendingDeletionBlocks = pendingDeletionBlocks;
+    this.badlyDistributedBlocks = badlyDistributedBlocks;
     this.highestPriorityLowRedundancyBlocks
         = highestPriorityLowRedundancyBlocks;
   }
@@ -86,6 +89,10 @@ public long getPendingDeletionBlocks() {
     return pendingDeletionBlocks;
   }
 
+  public long getBadlyDistributedBlocks() {
+    return badlyDistributedBlocks;
+  }
+
   public boolean hasHighestPriorityLowRedundancyBlocks() {
     return getHighestPriorityLowRedundancyBlocks() != null;
   }
@@ -94,6 +101,7 @@ public Long getHighestPriorityLowRedundancyBlocks(){
     return highestPriorityLowRedundancyBlocks;
   }
 
+
   @Override
   public String toString() {
     StringBuilder statsBuilder = new StringBuilder();
@@ -105,7 +113,8 @@ public String toString() {
             getMissingReplicationOneBlocks())
         .append(", BytesInFutureBlocks=").append(getBytesInFutureBlocks())
         .append(", PendingDeletionBlocks=").append(
-            getPendingDeletionBlocks());
+            getPendingDeletionBlocks())
+        .append(" , badlyDistributedBlocks=").append(getBadlyDistributedBlocks());
     if (hasHighestPriorityLowRedundancyBlocks()) {
         statsBuilder.append(", HighestPriorityLowRedundancyBlocks=").append(
             getHighestPriorityLowRedundancyBlocks());
@@ -127,6 +136,7 @@ public static ReplicatedBlockStats merge(
     long missingReplicationOneBlocks = 0;
     long bytesInFutureBlocks = 0;
     long pendingDeletionBlocks = 0;
+    long badlyDistributedBlocks = 0;
     long highestPriorityLowRedundancyBlocks = 0;
     boolean hasHighestPriorityLowRedundancyBlocks = false;
 
@@ -138,6 +148,7 @@ public static ReplicatedBlockStats merge(
       missingReplicationOneBlocks += stat.getMissingReplicationOneBlocks();
       bytesInFutureBlocks += stat.getBytesInFutureBlocks();
       pendingDeletionBlocks += stat.getPendingDeletionBlocks();
+      badlyDistributedBlocks += stat.getBadlyDistributedBlocks();
       if (stat.hasHighestPriorityLowRedundancyBlocks()) {
         hasHighestPriorityLowRedundancyBlocks = true;
         highestPriorityLowRedundancyBlocks +=
@@ -147,10 +158,10 @@ public static ReplicatedBlockStats merge(
     if (hasHighestPriorityLowRedundancyBlocks) {
       return new ReplicatedBlockStats(lowRedundancyBlocks, corruptBlocks,
           missingBlocks, missingReplicationOneBlocks, bytesInFutureBlocks,
-          pendingDeletionBlocks, highestPriorityLowRedundancyBlocks);
+          pendingDeletionBlocks, badlyDistributedBlocks, highestPriorityLowRedundancyBlocks);
     }
     return new ReplicatedBlockStats(lowRedundancyBlocks, corruptBlocks,
         missingBlocks, missingReplicationOneBlocks, bytesInFutureBlocks,
-        pendingDeletionBlocks);
+        pendingDeletionBlocks, badlyDistributedBlocks);
   }
 }

hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelperClient.java

@@ -1975,13 +1975,13 @@ public static ReplicatedBlockStats convert(
       return new ReplicatedBlockStats(res.getLowRedundancy(),
           res.getCorruptBlocks(), res.getMissingBlocks(),
           res.getMissingReplOneBlocks(), res.getBlocksInFuture(),
-          res.getPendingDeletionBlocks(),
+          res.getPendingDeletionBlocks(), res.getBadlyDistributedBlocks(),
           res.getHighestPrioLowRedundancyBlocks());
     }
     return new ReplicatedBlockStats(res.getLowRedundancy(),
         res.getCorruptBlocks(), res.getMissingBlocks(),
         res.getMissingReplOneBlocks(), res.getBlocksInFuture(),
-        res.getPendingDeletionBlocks());
+        res.getBadlyDistributedBlocks(), res.getPendingDeletionBlocks());
   }
 
   public static ECBlockGroupStats convert(
@@ -1990,11 +1990,12 @@ public static ECBlockGroupStats convert(
       return new ECBlockGroupStats(res.getLowRedundancy(),
           res.getCorruptBlocks(), res.getMissingBlocks(),
           res.getBlocksInFuture(), res.getPendingDeletionBlocks(),
-          res.getHighestPrioLowRedundancyBlocks());
+          res.getBadlyDistributedBlocks(), res.getHighestPrioLowRedundancyBlocks());
     }
     return new ECBlockGroupStats(res.getLowRedundancy(),
         res.getCorruptBlocks(), res.getMissingBlocks(),
-        res.getBlocksInFuture(), res.getPendingDeletionBlocks());
+        res.getBlocksInFuture(), res.getPendingDeletionBlocks(),
+        res.getBadlyDistributedBlocks());
   }
 
   public static DatanodeReportTypeProto convert(DatanodeReportType t) {
@@ -2440,6 +2441,8 @@ public static GetFsReplicatedBlockStatsResponseProto convert(
         replicatedBlockStats.getBytesInFutureBlocks());
     result.setPendingDeletionBlocks(
         replicatedBlockStats.getPendingDeletionBlocks());
+    result.setBadlyDistributedBlocks(
+        replicatedBlockStats.getBadlyDistributedBlocks());
     if (replicatedBlockStats.hasHighestPriorityLowRedundancyBlocks()) {
       result.setHighestPrioLowRedundancyBlocks(
           replicatedBlockStats.getHighestPriorityLowRedundancyBlocks());
@@ -2459,6 +2462,8 @@ public static GetFsECBlockGroupStatsResponseProto convert(
         ecBlockGroupStats.getBytesInFutureBlockGroups());
     result.setPendingDeletionBlocks(
         ecBlockGroupStats.getPendingDeletionBlocks());
+    result.setBadlyDistributedBlocks(
+        ecBlockGroupStats.getBadlyDistributedBlocks());
     if (ecBlockGroupStats.hasHighestPriorityLowRedundancyBlocks()) {
       result.setHighestPrioLowRedundancyBlocks(
           ecBlockGroupStats.getHighestPriorityLowRedundancyBlocks());

hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/ClientNamenodeProtocol.proto

@@ -363,6 +363,7 @@ message GetFsReplicatedBlockStatsResponseProto {
   required uint64 blocks_in_future = 5;
   required uint64 pending_deletion_blocks = 6;
   optional uint64 highest_prio_low_redundancy_blocks = 7;
+  required uint64 badly_distributed_blocks = 8;
 
 }
 
@@ -376,6 +377,7 @@ message GetFsECBlockGroupStatsResponseProto {
   required uint64 blocks_in_future = 4;
   required uint64 pending_deletion_blocks = 5;
   optional uint64 highest_prio_low_redundancy_blocks = 6;
+  required uint64 badly_distributed_blocks = 7;
 }
 
 enum DatanodeReportTypeProto {  // type of the datanode report

hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/metrics/FederationMBean.java

@@ -302,4 +302,52 @@ public interface FederationMBean {
    */
   @Deprecated
   boolean isSecurityEnabled();
+
+  /**
+   * Get the number of corrupts files.
+   *
+   * @return the total number of corrupt files.
+   */
+  int getCorruptFilesCount();
+
+  /**
+   * Blocks scheduled for replication.
+   *
+   * @return num of blocks scheduled for replication.
+   */
+  long getScheduledReplicationBlocks();
+
+  /**
+   * Gets the total number of missing blocks on the cluster with
+   * replication factor 1.
+   *
+   * @return the total number of missing blocks on the cluster with
+   * replication factor 1.
+   */
+  long getNumberOfMissingBlocksWithReplicationFactorOne();
+
+  /**
+   * Gets the total number of badly distributed blocks.
+   *
+   * @return the total number of badly distrubted blocks.
+   */
+  long getNumberOfBadlyDistributedBlocks();
+
+  /**
+   * Gets the total number of replicated low redundancy blocks on the cluster
+   * with the highest risk of loss.
+   *
+   * @return the total number of low redundancy blocks on the cluster
+   * with the highest risk of loss.
+   */
+  long getHighestPriorityLowRedundancyReplicatedBlocks();
+
+  /**
+   * Gets the total number of erasure coded low redundancy blocks on the cluster
+   * with the highest risk of loss.
+   *
+   * @return the total number of low redundancy blocks on the cluster
+   * with the highest risk of loss.
+   */
+  long getHighestPriorityLowRedundancyECBlocks();
 }

hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/metrics/NamenodeBeanMetrics.java

@@ -366,21 +366,56 @@ public long getPendingDeletionBlocks() {
 
   @Override
   public long getScheduledReplicationBlocks() {
-    return -1;
+    try {
+      return getRBFMetrics().getScheduledReplicationBlocks();
+    } catch (IOException e) {
+      LOG.debug("Failed to get number of scheduled replication blocks.",
+          e.getMessage());
+    }
+    return 0;
   }
 
   @Override
   public long getNumberOfMissingBlocksWithReplicationFactorOne() {
+    try {
+      return getRBFMetrics().getNumberOfMissingBlocksWithReplicationFactorOne();
+    } catch (IOException e) {
+      LOG.debug("Failed to get number of missing blocks with replication "
+          + "factor one.", e.getMessage());
+    }
+    return 0;
+  }
+
+  @Override
+  public long getNumberOfBadlyDistributedBlocks() {
+    try {
+      return getRBFMetrics().getNumberOfBadlyDistributedBlocks();
+    } catch (IOException e) {
+      LOG.debug("Failed to get number of badly distributed blocks", e);
+    }
     return 0;
   }
 
   @Override
   public long getHighestPriorityLowRedundancyReplicatedBlocks() {
+    try {
+      return getRBFMetrics().getHighestPriorityLowRedundancyReplicatedBlocks();
+    } catch (IOException e) {
+      LOG.debug("Failed to get number of highest priority low redundancy "
+          + "replicated blocks.", e.getMessage());
+    }
     return 0;
   }
 
   @Override
   public long getHighestPriorityLowRedundancyECBlocks() {
+    try {
+      return getRBFMetrics().getHighestPriorityLowRedundancyECBlocks();
+    } catch (IOException e) {
+      LOG.debug("Failed to get number of highest priority low redundancy EC "
+              + "blocks.",
+          e.getMessage());
+    }
     return 0;
   }
 
@@ -391,6 +426,11 @@ public String getCorruptFiles() {
 
   @Override
   public int getCorruptFilesCount() {
+    try {
+      return getRBFMetrics().getCorruptFilesCount();
+    } catch (IOException e) {
+      LOG.debug("Failed to get number of corrupt files.", e.getMessage());
+    }
     return 0;
   }