Skip to content

Commit

Permalink
[HUDI-1792] flink-client query error when processing files larger tha…
Browse files Browse the repository at this point in the history
…n 128mb (apache#2814)

Co-authored-by: huangjing <[email protected]>
  • Loading branch information
hj2016 and huangjing authored Apr 16, 2021
1 parent b6d949b commit 62b8a34
Showing 1 changed file with 8 additions and 1 deletion.
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,7 @@
import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Comparator;
import java.util.HashSet;
import java.util.LinkedHashMap;
import java.util.List;
Expand Down Expand Up @@ -209,7 +210,13 @@ public FileInputSplit[] createInputSplits(int minNumSplits) throws IOException {

// get the block locations and make sure they are in order with respect to their offset
final BlockLocation[] blocks = fs.getFileBlockLocations(file, 0, len);
Arrays.sort(blocks);
Arrays.sort(blocks, new Comparator<BlockLocation>() {
@Override
public int compare(BlockLocation o1, BlockLocation o2) {
long diff = o1.getLength() - o2.getOffset();
return diff < 0L ? -1 : (diff > 0L ? 1 : 0);
}
});

long bytesUnassigned = len;
long position = 0;
Expand Down

0 comments on commit 62b8a34

Please sign in to comment.