diff --git a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/grouper/TezSplitGrouper.java b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/grouper/TezSplitGrouper.java index 176eb1a10e..067acca9e6 100644 --- a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/grouper/TezSplitGrouper.java +++ b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/grouper/TezSplitGrouper.java @@ -253,36 +253,28 @@ public List getGroupedSplits(Configuration conf, "Invalid max/min group lengths. Required min>0, max>=min. " + " max: " + maxLengthPerGroup + " min: " + minLengthPerGroup); } + int newDesiredNumSplits = -1; if (lengthPerGroup > maxLengthPerGroup) { // splits too big to work. Need to override with max size. - int newDesiredNumSplits = (int)(totalLength/maxLengthPerGroup) + 1; - LOG.info("Desired splits: " + desiredNumSplits + " too small. " + - " Desired splitLength: " + lengthPerGroup + - " Max splitLength: " + maxLengthPerGroup + - " New desired splits: " + newDesiredNumSplits + - " Total length: " + totalLength + - " Original splits: " + originalSplits.size()); - - desiredNumSplits = newDesiredNumSplits; + newDesiredNumSplits = (int)(totalLength/maxLengthPerGroup) + 1; } else if (lengthPerGroup < minLengthPerGroup) { // splits too small to work. Need to override with size. - int newDesiredNumSplits = (int)(totalLength/minLengthPerGroup) + 1; - /** - * This is a workaround for systems like S3 that pass the same - * fake hostname for all splits. - */ - if (!allSplitsHaveLocalhost) { - desiredNumSplits = newDesiredNumSplits; + newDesiredNumSplits = (int)(totalLength/minLengthPerGroup) + 1; + if (allSplitsHaveLocalhost) { + // Workaround for systems like S3 that pass the same fake hostname for all splits. + LOG.info("Ignore {} configuration cause all splits seem to be on localhost.", TEZ_GROUPING_SPLIT_MIN_SIZE); + newDesiredNumSplits = desiredNumSplits; } - - LOG.info("Desired splits: " + desiredNumSplits + " too large. " + - " Desired splitLength: " + lengthPerGroup + + } + if (newDesiredNumSplits != -1) { + LOG.info("Desired splitLength " + lengthPerGroup + " exceeds min/max bounds. " + " Min splitLength: " + minLengthPerGroup + - " New desired splits: " + newDesiredNumSplits + - " Final desired splits: " + desiredNumSplits + - " All splits have localhost: " + allSplitsHaveLocalhost + + " Max splitLength: " + maxLengthPerGroup + + " Desired splits: " + desiredNumSplits + + " New Desired splits: " + newDesiredNumSplits + " Total length: " + totalLength + " Original splits: " + originalSplits.size()); + desiredNumSplits = newDesiredNumSplits; } }