transmit local block id (in the stripe) to the datanode during repair

jiajunmao · Aug 28, 2024 · de95491 · de95491
1 parent 2ee80c8
commit de95491
Show file tree

Hide file tree

Showing 4 changed files with 7 additions and 7 deletions.
diff --git a/...-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReconstructStripedFile.java b/...-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReconstructStripedFile.java
@@ -476,7 +476,7 @@ public void testProcessErasureCodingTasksSubmitionShouldSucceed()
 
     BlockECReconstructionInfo invalidECInfo = new BlockECReconstructionInfo(
         new ExtendedBlock("bp-id", 123456), dataDNs, dnStorageInfo, liveIndices,
-        new byte[0], ecPolicy, null);
+        new byte[0], ecPolicy, null, 0);
     List<BlockECReconstructionInfo> ecTasks = new ArrayList<>();
     ecTasks.add(invalidECInfo);
     dataNode.getErasureCodingWorker().processErasureCodingTasks(ecTasks);

diff --git a/...dfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/protocolPB/TestPBHelper.java b/...dfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/protocolPB/TestPBHelper.java
@@ -752,7 +752,7 @@ public void testBlockECRecoveryCommand() {
     byte[] excludeReconstructedIndices0=new byte[2];
     BlockECReconstructionInfo blkECRecoveryInfo0 = new BlockECReconstructionInfo(
         new ExtendedBlock("bp1", 1234), dnInfos0, targetDnInfos0,
-        liveBlkIndices0, excludeReconstructedIndices0, StripedFileTestUtil.getDefaultECPolicy(), null);
+        liveBlkIndices0, excludeReconstructedIndices0, StripedFileTestUtil.getDefaultECPolicy(), null, 0);
     DatanodeInfo[] dnInfos1 = new DatanodeInfo[] {
         DFSTestUtil.getLocalDatanodeInfo(), DFSTestUtil.getLocalDatanodeInfo() };
     DatanodeStorageInfo targetDnInfos_2 = BlockManagerTestUtil
@@ -767,7 +767,7 @@ public void testBlockECRecoveryCommand() {
     byte[] excludeReconstructedIndices = new byte[2];
     BlockECReconstructionInfo blkECRecoveryInfo1 = new BlockECReconstructionInfo(
         new ExtendedBlock("bp2", 3256), dnInfos1, targetDnInfos1,
-        liveBlkIndices1, excludeReconstructedIndices, StripedFileTestUtil.getDefaultECPolicy(), null);
+        liveBlkIndices1, excludeReconstructedIndices, StripedFileTestUtil.getDefaultECPolicy(), null, 0);
     List<BlockECReconstructionInfo> blkRecoveryInfosList = new ArrayList<BlockECReconstructionInfo>();
     blkRecoveryInfosList.add(blkECRecoveryInfo0);
     blkRecoveryInfosList.add(blkECRecoveryInfo1);

diff --git a/...op-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockManager.java b/...op-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockManager.java
@@ -1005,9 +1005,9 @@ public void testSkipReconstructionWithManyBusyNodes3() {
     // Simulate the 2 nodes reach maxReplicationStreams.
     for(int i = 0; i < bm.getMaxReplicationStreams(); i++){ //Add some dummy EC reconstruction task.
       ds3.getDatanodeDescriptor().addBlockToBeErasureCoded(dummyBlock, dummyDDArray,
-              dummyDSArray, new byte[0], new byte[0], ecPolicy, null);
+              dummyDSArray, new byte[0], new byte[0], ecPolicy, null, 0);
       ds4.getDatanodeDescriptor().addBlockToBeErasureCoded(dummyBlock, dummyDDArray,
-              dummyDSArray, new byte[0], new byte[0], ecPolicy, null);
+              dummyDSArray, new byte[0], new byte[0], ecPolicy, null, 0);
     }
 
     // Reconstruction should be skipped since the number of non-busy nodes are not enough.

diff --git a/rsync-java.sh b/rsync-java.sh
@@ -6,13 +6,13 @@ DEST_DIR=aaronmao@mlec-mac:/home/aaronmao/hadoop-MLEC/
 
 # Function to sync .java files
 sync_java_files() {
-    rsync -avzL --progress --include='*/' --include='*.java' --exclude='*' "$SOURCE_DIR" "$DEST_DIR"
+    rsync -avzL --progress --include='*/' --include='*.java' --include='*.proto' --exclude='*' "$SOURCE_DIR" "$DEST_DIR"
     echo "Sync complete."
 }
 
 echo "Syncing .java files..."
 # Monitor the source directory for changes in .java files
-fswatch -o --exclude='.*' --include='.*\.java$' "$SOURCE_DIR" | while read f ; do
+fswatch -o --exclude='.*' --include='.*\.java$' --include='.*\.proto$' "$SOURCE_DIR" | while read f ; do
     echo "Change detected, syncing..."
     sync_java_files
 done