diff --git a/Jenkinsfile b/Jenkinsfile index 4ce236598d..64b5a6be9d 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -98,6 +98,7 @@ pipeline { # our project defaults come from a personality file YETUS_ARGS+=("--project=tez") + YETUS_ARGS+=("--personality=${WORKSPACE}/${SOURCEDIR}/build-tools/tez-personality.sh") # lots of different output formats YETUS_ARGS+=("--brief-report-file=${WORKSPACE}/${PATCHDIR}/brief.txt") diff --git a/build-tools/smart-apply-patch.sh b/build-tools/smart-apply-patch.sh deleted file mode 100755 index 2a9dbd24dd..0000000000 --- a/build-tools/smart-apply-patch.sh +++ /dev/null @@ -1,145 +0,0 @@ -#!/usr/bin/env bash -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -set -e - -# -# Determine if the patch file is a git diff file with prefixes. -# These files are generated via "git diff" *without* the --no-prefix option. -# -# We can apply these patches more easily because we know that the a/ and b/ -# prefixes in the "diff" lines stands for the project root directory. -# So we don't have to hunt for the project root. -# And of course, we know that the patch file was generated using git, so we -# know git apply can handle it properly. -# -# Arguments: file name. -# Return: 0 if it is a git diff; 1 otherwise. -# -is_git_diff_with_prefix() { - DIFF_TYPE="unknown" - while read -r line; do - if [[ "$line" =~ ^diff\ ]]; then - if [[ "$line" =~ ^diff\ \-\-git ]]; then - DIFF_TYPE="git" - else - return 1 # All diff lines must be diff --git lines. - fi - fi - if [[ "$line" =~ ^\+\+\+\ ]] || - [[ "$line" =~ ^\-\-\-\ ]]; then - if ! [[ "$line" =~ ^....[ab]/ ]]; then - return 1 # All +++ and --- lines must start with a/ or b/. - fi - fi - done < $1 - [ x$DIFF_TYPE == x"git" ] || return 1 - return 0 # return true (= 0 in bash) -} - -PATCH_FILE=$1 -DRY_RUN=$2 -if [ -z "$PATCH_FILE" ]; then - echo usage: $0 patch-file - exit 1 -fi - -PATCH=${PATCH:-patch} # allow overriding patch binary - -# Cleanup handler for temporary files -TOCLEAN="" -cleanup() { - rm $TOCLEAN - exit $1 -} -trap "cleanup 1" HUP INT QUIT TERM - -# Allow passing "-" for stdin patches -if [ "$PATCH_FILE" == "-" ]; then - PATCH_FILE=/tmp/tmp.in.$$ - cat /dev/fd/0 > $PATCH_FILE - TOCLEAN="$TOCLEAN $PATCH_FILE" -fi - -# Special case for git-diff patches without --no-prefix -if is_git_diff_with_prefix "$PATCH_FILE"; then - GIT_FLAGS="--binary -p1 -v" - if [[ -z $DRY_RUN ]]; then - GIT_FLAGS="$GIT_FLAGS --stat --apply " - echo Going to apply git patch with: git apply "${GIT_FLAGS}" - else - GIT_FLAGS="$GIT_FLAGS --check " - fi - git apply ${GIT_FLAGS} "${PATCH_FILE}" - exit $? -fi - -# Come up with a list of changed files into $TMP -TMP=/tmp/tmp.paths.$$ -TOCLEAN="$TOCLEAN $TMP" - -if $PATCH -p0 -E --dry-run < $PATCH_FILE 2>&1 > $TMP; then - PLEVEL=0 - #if the patch applied at P0 there is the possability that all we are doing - # is adding new files and they would apply anywhere. So try to guess the - # correct place to put those files. - - TMP2=/tmp/tmp.paths.2.$$ - TOCLEAN="$TOCLEAN $TMP2" - - egrep '^patching file |^checking file ' $TMP | awk '{print $3}' | grep -v /dev/null | sort | uniq > $TMP2 - - if [ ! -s $TMP2 ]; then - echo "Error: Patch dryrun couldn't detect changes the patch would make. Exiting." - cleanup 1 - fi - - #first off check that all of the files do not exist - FOUND_ANY=0 - for CHECK_FILE in $(cat $TMP2) - do - if [[ -f $CHECK_FILE ]]; then - FOUND_ANY=1 - fi - done - - if [[ "$FOUND_ANY" = "0" ]]; then - #all of the files are new files so we have to guess where the correct place to put it is. - - # if all of the lines start with a/ or b/, then this is a git patch that - # was generated without --no-prefix - if ! grep -qv '^a/\|^b/' $TMP2 ; then - echo Looks like this is a git patch. Stripping a/ and b/ prefixes - echo and incrementing PLEVEL - PLEVEL=$[$PLEVEL + 1] - sed -i -e 's,^[ab]/,,' $TMP2 - fi - fi -elif $PATCH -p1 -E --dry-run < $PATCH_FILE 2>&1 > /dev/null; then - PLEVEL=1 -elif $PATCH -p2 -E --dry-run < $PATCH_FILE 2>&1 > /dev/null; then - PLEVEL=2 -else - echo "The patch does not appear to apply with p0 to p2"; - cleanup 1; -fi - -# If this is a dry run then exit instead of applying the patch -if [[ -n $DRY_RUN ]]; then - cleanup 0; -fi - -echo Going to apply patch with: $PATCH -p$PLEVEL -$PATCH -p$PLEVEL -E < $PATCH_FILE - -cleanup $? diff --git a/build-tools/test-patch.sh b/build-tools/test-patch.sh deleted file mode 100755 index 9b457af221..0000000000 --- a/build-tools/test-patch.sh +++ /dev/null @@ -1,821 +0,0 @@ -#!/usr/bin/env bash -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -#set -x - -### Setup some variables. -### BUILD_URL is set by Hudson if it is run by patch process -### Read variables from properties file -bindir=$(dirname $0) - -# Defaults -if [ -z "$MAVEN_HOME" ]; then - MVN=mvn -else - MVN=$MAVEN_HOME/bin/mvn -fi - -PROJECT_NAME=Tez -JENKINS=false -PATCH_DIR=/tmp -BASEDIR=$(pwd) -PS=${PS:-ps} -AWK=${AWK:-awk} -WGET=${WGET:-wget} -GIT=${GIT:-git} -GREP=${GREP:-grep} -PATCH=${PATCH:-patch} -DIFF=${DIFF:-diff} -JIRACLI=${JIRA:-jira} -SED=${SED:-sed} -CURL=${CURL:-curl} -FINDBUGS_HOME=${FINDBUGS_HOME} - -############################################################################### -printUsage() { - echo "Usage: $0 [options] patch-file | defect-number" - echo - echo "Where:" - echo " patch-file is a local patch file containing the changes to test" - echo " defect-number is a JIRA defect number (e.g. 'TEZ-1234') to test (Jenkins only)" - echo - echo "Options:" - echo "--patch-dir= The directory for working and output files (default '/tmp')" - echo "--basedir= The directory to apply the patch to (default current directory)" - echo "--mvn-cmd= The 'mvn' command to use (default \$MAVEN_HOME/bin/mvn, or 'mvn')" - echo "--ps-cmd= The 'ps' command to use (default 'ps')" - echo "--awk-cmd= The 'awk' command to use (default 'awk')" - echo "--git-cmd= The 'git' command to use (default 'git')" - echo "--grep-cmd= The 'grep' command to use (default 'grep')" - echo "--patch-cmd= The 'patch' command to use (default 'patch')" - echo "--diff-cmd= The 'diff' command to use (default 'diff')" - echo "--findbugs-home= Findbugs home directory (default FINDBUGS_HOME environment variable)" - echo "--dirty-workspace Allow the local git workspace to have uncommitted changes" - echo "--run-tests Run all tests below the base directory" - echo - echo "Jenkins-only options:" - echo "--jenkins Run by Jenkins (runs tests and posts results to JIRA)" - echo "--wget-cmd= The 'wget' command to use (default 'wget')" - echo "--jira-cmd= The 'jira' command to use (default 'jira')" - echo "--jira-password= The password for the 'jira' command" -} - -############################################################################### -parseArgs() { - for i in $* - do - case $i in - --jenkins) - JENKINS=true - ;; - --patch-dir=*) - PATCH_DIR=${i#*=} - ;; - --basedir=*) - BASEDIR=${i#*=} - ;; - --mvn-cmd=*) - MVN=${i#*=} - ;; - --ps-cmd=*) - PS=${i#*=} - ;; - --awk-cmd=*) - AWK=${i#*=} - ;; - --wget-cmd=*) - WGET=${i#*=} - ;; - --git-cmd=*) - GIT=${i#*=} - ;; - --grep-cmd=*) - GREP=${i#*=} - ;; - --patch-cmd=*) - PATCH=${i#*=} - ;; - --diff-cmd=*) - DIFF=${i#*=} - ;; - --jira-cmd=*) - JIRACLI=${i#*=} - ;; - --jira-password=*) - JIRA_PASSWD=${i#*=} - ;; - --findbugs-home=*) - FINDBUGS_HOME=${i#*=} - ;; - --dirty-workspace) - DIRTY_WORKSPACE=true - ;; - --run-tests) - RUN_TESTS=true - ;; - *) - PATCH_OR_DEFECT=$i - ;; - esac - done - if [ -z "$PATCH_OR_DEFECT" ]; then - printUsage - exit 1 - fi - if [[ $JENKINS == "true" ]] ; then - echo "Running in Jenkins mode" - defect=$PATCH_OR_DEFECT - else - echo "Running in developer mode" - JENKINS=false - ### PATCH_FILE contains the location of the patchfile - PATCH_FILE=$PATCH_OR_DEFECT - if [[ ! -e "$PATCH_FILE" ]] ; then - echo "Unable to locate the patch file $PATCH_FILE" - cleanupAndExit 0 - fi - ### Check if $PATCH_DIR exists. If it does not exist, create a new directory - if [[ ! -e "$PATCH_DIR" ]] ; then - mkdir "$PATCH_DIR" - if [[ $? == 0 ]] ; then - echo "$PATCH_DIR has been created" - else - echo "Unable to create $PATCH_DIR" - cleanupAndExit 0 - fi - fi - ### Obtain the patch filename to append it to the version number - defect=`basename $PATCH_FILE` - fi -} - -############################################################################### -checkout () { - echo "" - echo "" - echo "======================================================================" - echo "======================================================================" - echo " Testing patch for ${defect}." - echo "======================================================================" - echo "======================================================================" - echo "" - echo "" - ### When run by a developer, if the workspace contains modifications, do not continue - ### unless the --dirty-workspace option was set - status=`$GIT status --porcelain` - if [[ $JENKINS == "false" ]] ; then - if [[ "$status" != "" && -z $DIRTY_WORKSPACE ]] ; then - echo "ERROR: can't run in a workspace that contains the following modifications" - echo "$status" - cleanupAndExit 1 - fi - echo - else - cd $BASEDIR - $GIT reset --hard - $GIT clean -xdf - $GIT checkout master - $GIT pull --rebase - fi - GIT_REVISION=`git rev-parse --verify --short HEAD` - return $? -} - -############################################################################### -downloadPatch () { - ### Download latest patch file (ignoring .htm and .html) when run from patch process - if [[ $JENKINS == "true" ]] ; then - $WGET -q -O $PATCH_DIR/jira http://issues.apache.org/jira/browse/$defect - if [[ `$GREP -c 'Patch Available' $PATCH_DIR/jira` == 0 ]] ; then - echo "$defect is not \"Patch Available\". Exiting." - cleanupAndExit 0 - fi - relativePatchURL=`$GREP -o '"/jira/secure/attachment/[0-9]*/[^"]*' $PATCH_DIR/jira | $GREP -v -e 'htm[l]*$' | sort | tail -1 | $GREP -o '/jira/secure/attachment/[0-9]*/[^"]*'` - patchURL="http://issues.apache.org${relativePatchURL}" - patchNum=`echo $patchURL | $GREP -o '[0-9]*/' | $GREP -o '[0-9]*'` - echo "$defect patch is being downloaded at `date` from" - echo "$patchURL" - $WGET -q -O $PATCH_DIR/patch $patchURL - VERSION=${GIT_REVISION}_${defect}_PATCH-${patchNum} - JIRA_COMMENT="Here are the results of testing the latest attachment - $patchURL - against master revision ${GIT_REVISION}." - - ### Copy the patch file to $PATCH_DIR - else - VERSION=PATCH-${defect} - cp $PATCH_FILE $PATCH_DIR/patch - if [[ $? == 0 ]] ; then - echo "Patch file $PATCH_FILE copied to $PATCH_DIR" - else - echo "Could not copy $PATCH_FILE to $PATCH_DIR" - cleanupAndExit 0 - fi - fi -} - -############################################################################### -verifyPatch () { - # Before building, check to make sure that the patch is valid - $bindir/smart-apply-patch.sh $PATCH_DIR/patch dryrun - if [[ $? != 0 ]] ; then - echo "PATCH APPLICATION FAILED" - JIRA_COMMENT="$JIRA_COMMENT - - {color:red}-1 patch{color}. The patch command could not apply the patch." - return 1 - else - return 0 - fi -} - -############################################################################### -prebuildWithoutPatch () { - echo "" - echo "" - echo "======================================================================" - echo "======================================================================" - echo " Pre-build master to verify master stability and javac warnings" - echo "======================================================================" - echo "======================================================================" - echo "" - echo "" - echo "Compiling $(pwd)" - echo "$MVN clean test -DskipTests -Ptest-patch > $PATCH_DIR/masterJavacWarnings.txt 2>&1" - $MVN clean test -DskipTests -Ptest-patch > $PATCH_DIR/masterJavacWarnings.txt 2>&1 - if [[ $? != 0 ]] ; then - echo "master compilation is broken?" - JIRA_COMMENT="$JIRA_COMMENT - - {color:red}-1 patch{color}. master compilation may be broken." - return 1 - fi - - echo "$MVN clean test javadoc:javadoc -DskipTests > $PATCH_DIR/masterJavadocWarnings.txt 2>&1" - $MVN clean test javadoc:javadoc -DskipTests > $PATCH_DIR/masterJavadocWarnings.txt 2>&1 - if [[ $? != 0 ]] ; then - echo "master javadoc compilation is broken?" - JIRA_COMMENT="$JIRA_COMMENT - - {color:red}-1 patch{color}. master compilation may be broken." - return 1 - fi - - return 0 -} - -############################################################################### -### Check for @author tags in the patch -checkAuthor () { - echo "" - echo "" - echo "======================================================================" - echo "======================================================================" - echo " Checking there are no @author tags in the patch." - echo "======================================================================" - echo "======================================================================" - echo "" - echo "" - authorTags=`$GREP -c -i '@author' $PATCH_DIR/patch` - echo "There appear to be $authorTags @author tags in the patch." - if [[ $authorTags != 0 ]] ; then - JIRA_COMMENT="$JIRA_COMMENT - - {color:red}-1 @author{color}. The patch appears to contain $authorTags @author tags which the Tez community has agreed to not allow in code contributions." - return 1 - fi - JIRA_COMMENT="$JIRA_COMMENT - - {color:green}+1 @author{color}. The patch does not contain any @author tags." - return 0 -} - -############################################################################### -### Check for tests in the patch -checkTests () { - echo "" - echo "" - echo "======================================================================" - echo "======================================================================" - echo " Checking there are new or changed tests in the patch." - echo "======================================================================" - echo "======================================================================" - echo "" - echo "" - testReferences=`$GREP -c -i -e '^+++.*/test' $PATCH_DIR/patch` - echo "There appear to be $testReferences test files referenced in the patch." - if [[ $testReferences == 0 ]] ; then - if [[ $JENKINS == "true" ]] ; then - patchIsDoc=`$GREP -c -i 'title="documentation' $PATCH_DIR/jira` - if [[ $patchIsDoc != 0 ]] ; then - echo "The patch appears to be a documentation patch that doesn't require tests." - JIRA_COMMENT="$JIRA_COMMENT - - {color:green}+0 tests included{color}. The patch appears to be a documentation patch that doesn't require tests." - return 0 - fi - fi - JIRA_COMMENT="$JIRA_COMMENT - - {color:red}-1 tests included{color}. The patch doesn't appear to include any new or modified tests. - Please justify why no new tests are needed for this patch. - Also please list what manual steps were performed to verify this patch." - return 1 - fi - JIRA_COMMENT="$JIRA_COMMENT - - {color:green}+1 tests included{color}. The patch appears to include $testReferences new or modified test files." - return 0 -} - -cleanUpXml () { - cd $BASEDIR/conf - for file in `ls *.xml.template` - do - rm -f `basename $file .template` - done - cd $BASEDIR -} - -############################################################################### -### Attempt to apply the patch -applyPatch () { - echo "" - echo "" - echo "======================================================================" - echo "======================================================================" - echo " Applying patch." - echo "======================================================================" - echo "======================================================================" - echo "" - echo "" - export PATCH - $bindir/smart-apply-patch.sh $PATCH_DIR/patch - if [[ $? != 0 ]] ; then - echo "PATCH APPLICATION FAILED" - JIRA_COMMENT="$JIRA_COMMENT - - {color:red}-1 patch{color}. The patch command could not apply the patch." - return 1 - fi - return 0 -} - -############################################################################### -calculateJavadocWarnings() { - WARNING_FILE="$1" - RET=$(egrep "^[0-9]+ warnings?$" "$WARNING_FILE" | awk '{sum+=$1} END {print sum}') -} - -### Check there are no javadoc warnings -checkJavadocWarnings () { - echo "" - echo "" - echo "======================================================================" - echo "======================================================================" - echo " Determining number of patched javadoc warnings." - echo "======================================================================" - echo "======================================================================" - echo "" - echo "" - echo "$MVN clean test javadoc:javadoc -DskipTests > $PATCH_DIR/patchJavadocWarnings.txt 2>&1" - $MVN clean test javadoc:javadoc -DskipTests > $PATCH_DIR/patchJavadocWarnings.txt 2>&1 - calculateJavadocWarnings "$PATCH_DIR/masterJavadocWarnings.txt" - nummasterJavadocWarnings=$RET - calculateJavadocWarnings "$PATCH_DIR/patchJavadocWarnings.txt" - numPatchJavadocWarnings=$RET - grep -i warning "$PATCH_DIR/masterJavadocWarnings.txt" > "$PATCH_DIR/masterJavadocWarningsFiltered.txt" - grep -i warning "$PATCH_DIR/patchJavadocWarnings.txt" > "$PATCH_DIR/patchJavadocWarningsFiltered.txt" - diff -u "$PATCH_DIR/masterJavadocWarningsFiltered.txt" \ - "$PATCH_DIR/patchJavadocWarningsFiltered.txt" > \ - "$PATCH_DIR/diffJavadocWarnings.txt" - rm -f "$PATCH_DIR/masterJavadocWarningsFiltered.txt" "$PATCH_DIR/patchJavadocWarningsFiltered.txt" - echo "There appear to be $nummasterJavadocWarnings javadoc warnings before the patch and $numPatchJavadocWarnings javadoc warnings after applying the patch." - if [[ $nummasterJavadocWarnings != "" && $numPatchJavadocWarnings != "" ]] ; then - if [[ $numPatchJavadocWarnings -gt $nummasterJavadocWarnings ]] ; then - JIRA_COMMENT="$JIRA_COMMENT - - {color:red}-1 javadoc{color}. The javadoc tool appears to have generated `expr $(($numPatchJavadocWarnings-$nummasterJavadocWarnings))` warning messages. - See $BUILD_URL/artifact/patchprocess/diffJavadocWarnings.txt for details." - return 1 - fi - fi - JIRA_COMMENT="$JIRA_COMMENT - - {color:green}+1 javadoc{color}. There were no new javadoc warning messages." - return 0 -} - -############################################################################### -### Check there are no changes in the number of Javac warnings -checkJavacWarnings () { - echo "" - echo "" - echo "======================================================================" - echo "======================================================================" - echo " Determining number of patched javac warnings." - echo "======================================================================" - echo "======================================================================" - echo "" - echo "" - echo "$MVN clean test -DskipTests -Ptest-patch > $PATCH_DIR/patchJavacWarnings.txt 2>&1" - $MVN clean test -DskipTests -Ptest-patch > $PATCH_DIR/patchJavacWarnings.txt 2>&1 - if [[ $? != 0 ]] ; then - JIRA_COMMENT="$JIRA_COMMENT - - {color:red}-1 javac{color:red}. The patch appears to cause the build to fail." - return 2 - fi - ### Compare master and patch javac warning numbers - if [[ -f $PATCH_DIR/patchJavacWarnings.txt ]] ; then - $GREP '\[WARNING\]' $PATCH_DIR/masterJavacWarnings.txt > $PATCH_DIR/filteredmasterJavacWarnings.txt - $GREP '\[WARNING\]' $PATCH_DIR/patchJavacWarnings.txt > $PATCH_DIR/filteredPatchJavacWarnings.txt - masterJavacWarnings=`cat $PATCH_DIR/filteredmasterJavacWarnings.txt | $AWK 'BEGIN {total = 0} {total += 1} END {print total}'` - patchJavacWarnings=`cat $PATCH_DIR/filteredPatchJavacWarnings.txt | $AWK 'BEGIN {total = 0} {total += 1} END {print total}'` - echo "There appear to be $masterJavacWarnings javac compiler warnings before the patch and $patchJavacWarnings javac compiler warnings after applying the patch." - if [[ $patchJavacWarnings != "" && $masterJavacWarnings != "" ]] ; then - if [[ $patchJavacWarnings -gt $masterJavacWarnings ]] ; then - JIRA_COMMENT="$JIRA_COMMENT - - {color:red}-1 javac{color}. The applied patch generated $patchJavacWarnings javac compiler warnings (more than the master's current $masterJavacWarnings warnings)." - - $DIFF $PATCH_DIR/filteredmasterJavacWarnings.txt $PATCH_DIR/filteredPatchJavacWarnings.txt > $PATCH_DIR/diffJavacWarnings.txt - JIRA_COMMENT_FOOTER="Javac warnings: $BUILD_URL/artifact/patchprocess/diffJavacWarnings.txt -$JIRA_COMMENT_FOOTER" - - return 1 - fi - fi - fi - JIRA_COMMENT="$JIRA_COMMENT - - {color:green}+1 javac{color}. The applied patch does not increase the total number of javac compiler warnings." - return 0 -} - -############################################################################### -### Check there are no changes in the number of release audit (RAT) warnings -checkReleaseAuditWarnings () { - echo "" - echo "" - echo "======================================================================" - echo "======================================================================" - echo " Determining number of patched release audit warnings." - echo "======================================================================" - echo "======================================================================" - echo "" - echo "" - echo "$MVN apache-rat:check > $PATCH_DIR/patchReleaseAuditOutput.txt 2>&1" - $MVN apache-rat:check > $PATCH_DIR/patchReleaseAuditOutput.txt 2>&1 - find $BASEDIR -name rat.txt | xargs cat > $PATCH_DIR/patchReleaseAuditWarnings.txt - - ### Compare master and patch release audit warning numbers - if [[ -f $PATCH_DIR/patchReleaseAuditWarnings.txt ]] ; then - patchReleaseAuditWarnings=`$GREP -c '\!?????' $PATCH_DIR/patchReleaseAuditWarnings.txt` - echo "" - echo "" - echo "There appear to be $patchReleaseAuditWarnings release audit warnings after applying the patch." - if [[ $patchReleaseAuditWarnings != "" ]] ; then - if [[ $patchReleaseAuditWarnings -gt 0 ]] ; then - JIRA_COMMENT="$JIRA_COMMENT - - {color:red}-1 release audit{color}. The applied patch generated $patchReleaseAuditWarnings release audit warnings." - $GREP '\!?????' $PATCH_DIR/patchReleaseAuditWarnings.txt > $PATCH_DIR/patchReleaseAuditProblems.txt - echo "Lines that start with ????? in the release audit report indicate files that do not have an Apache license header." >> $PATCH_DIR/patchReleaseAuditProblems.txt - JIRA_COMMENT_FOOTER="Release audit warnings: $BUILD_URL/artifact/patchprocess/patchReleaseAuditProblems.txt -$JIRA_COMMENT_FOOTER" - return 1 - fi - fi - fi - JIRA_COMMENT="$JIRA_COMMENT - - {color:green}+1 release audit{color}. The applied patch does not increase the total number of release audit warnings." - return 0 -} - - -############################################################################### -### Install the new jars so tests and findbugs can find all of the updated jars -buildAndInstall () { - echo "" - echo "" - echo "======================================================================" - echo "======================================================================" - echo " Installing all of the jars" - echo "======================================================================" - echo "======================================================================" - echo "" - echo "" - echo "$MVN install -Dmaven.javadoc.skip=true -DskipTests -D${PROJECT_NAME}PatchProcess" - $MVN install -Dmaven.javadoc.skip=true -DskipTests -D${PROJECT_NAME}PatchProcess - return $? -} - - -############################################################################### -### Check there are no changes in the number of Findbugs warnings -checkFindbugsWarnings () { - echo "" - echo "" - echo "======================================================================" - echo "======================================================================" - echo " Determining number of patched Findbugs warnings." - echo "======================================================================" - echo "======================================================================" - echo "" - echo "" - - rc=0 - echo " Running findbugs " - echo "$MVN clean test findbugs:findbugs -DskipTests < /dev/null > $PATCH_DIR/patchFindBugsOutput.txt 2>&1" - $MVN clean test findbugs:findbugs -DskipTests < /dev/null > $PATCH_DIR/patchFindBugsOutput.txt 2>&1 - rc=$? - findbugs_version=$(${AWK} 'match($0, /findbugs-maven-plugin:[^:]*:findbugs/) { print substr($0, RSTART + 22, RLENGTH - 31); exit }' "${PATCH_DIR}/patchFindBugsOutput.txt") - - if [ $rc != 0 ] ; then - JIRA_COMMENT="$JIRA_COMMENT - - {color:red}-1 findbugs{color}. The patch appears to cause Findbugs (version ${findbugs_version}) to fail." - return 1 - fi - - findbugsWarnings=0 - for file in $(find $BASEDIR -name findbugsXml.xml) - do - relative_file=${file#$BASEDIR/} # strip leading $BASEDIR prefix - if [ ! $relative_file == "target/findbugsXml.xml" ]; then - module_suffix=${relative_file%/target/findbugsXml.xml} # strip trailing path - module_suffix=`basename ${module_suffix}` - fi - - cp $file $PATCH_DIR/patchFindbugsWarnings${module_suffix}.xml - $FINDBUGS_HOME/bin/setBugDatabaseInfo -timestamp "01/01/2000" \ - $PATCH_DIR/patchFindbugsWarnings${module_suffix}.xml \ - $PATCH_DIR/patchFindbugsWarnings${module_suffix}.xml - newFindbugsWarnings=`$FINDBUGS_HOME/bin/filterBugs -first "01/01/2000" $PATCH_DIR/patchFindbugsWarnings${module_suffix}.xml \ - $PATCH_DIR/newPatchFindbugsWarnings${module_suffix}.xml | $AWK '{print $1}'` - echo "Found $newFindbugsWarnings Findbugs warnings ($file)" - findbugsWarnings=$((findbugsWarnings+newFindbugsWarnings)) - $FINDBUGS_HOME/bin/convertXmlToText -html \ - $PATCH_DIR/newPatchFindbugsWarnings${module_suffix}.xml \ - $PATCH_DIR/newPatchFindbugsWarnings${module_suffix}.html - if [[ $newFindbugsWarnings > 0 ]] ; then - JIRA_COMMENT_FOOTER="Findbugs warnings: $BUILD_URL/artifact/patchprocess/newPatchFindbugsWarnings${module_suffix}.html -$JIRA_COMMENT_FOOTER" - fi - done - - if [[ $findbugsWarnings -gt 0 ]] ; then - JIRA_COMMENT="$JIRA_COMMENT - - {color:red}-1 findbugs{color}. The patch appears to introduce $findbugsWarnings new Findbugs (version ${findbugs_version}) warnings." - return 1 - fi - JIRA_COMMENT="$JIRA_COMMENT - - {color:green}+1 findbugs{color}. The patch does not introduce any new Findbugs (version ${findbugs_version}) warnings." - return 0 -} - -############################################################################### -### Run the tests -runTests () { - echo "" - echo "" - echo "======================================================================" - echo "======================================================================" - echo " Running tests." - echo "======================================================================" - echo "======================================================================" - echo "" - echo "" - - failed_tests="" - failed_test_builds="" - test_timeouts="" - test_logfile=$PATCH_DIR/testrun.txt - echo " Running tests " - echo " $MVN clean install -fn -D${PROJECT_NAME}PatchProcess" - $MVN clean install -fn > $test_logfile 2>&1 - test_build_result=$? - cat $test_logfile - module_test_timeouts=`$AWK '/^Running / { if (last) { print last } last=$2 } /^Tests run: / { last="" }' $test_logfile` - if [[ -n "$module_test_timeouts" ]] ; then - test_timeouts="$test_timeouts $module_test_timeouts" - fi - module_failed_tests=`find . -name 'TEST*.xml' | xargs $GREP -l -E "/dev/null - fi -} - -############################################################################### -### Cleanup files -cleanupAndExit () { - local result=$1 - if [[ $JENKINS == "true" ]] ; then - if [ -e "$PATCH_DIR" ] ; then - mv $PATCH_DIR $BASEDIR - fi - fi - echo "" - echo "" - echo "======================================================================" - echo "======================================================================" - echo " Finished build." - echo "======================================================================" - echo "======================================================================" - echo "" - echo "" - exit $result -} - -############################################################################### -############################################################################### -############################################################################### - -JIRA_COMMENT="" -JIRA_COMMENT_FOOTER="Console output: $BUILD_URL/console - -This message is automatically generated." - -### Check if arguments to the script have been specified properly or not -parseArgs $@ -cd $BASEDIR - -checkout -RESULT=$? -if [[ $JENKINS == "true" ]] ; then - if [[ $RESULT != 0 ]] ; then - exit 100 - fi -fi -downloadPatch -verifyPatch -(( RESULT = RESULT + $? )) -if [[ $RESULT != 0 ]] ; then - submitJiraComment 1 - cleanupAndExit 1 -fi -prebuildWithoutPatch -(( RESULT = RESULT + $? )) -if [[ $RESULT != 0 ]] ; then - submitJiraComment 1 - cleanupAndExit 1 -fi -checkAuthor -(( RESULT = RESULT + $? )) - -if [[ $JENKINS == "true" ]] ; then - cleanUpXml -fi -checkTests -(( RESULT = RESULT + $? )) -applyPatch -APPLY_PATCH_RET=$? -(( RESULT = RESULT + $APPLY_PATCH_RET )) -if [[ $APPLY_PATCH_RET != 0 ]] ; then - submitJiraComment 1 - cleanupAndExit 1 -fi -checkJavacWarnings -JAVAC_RET=$? -#2 is returned if the code could not compile -if [[ $JAVAC_RET == 2 ]] ; then - submitJiraComment 1 - cleanupAndExit 1 -fi -(( RESULT = RESULT + $JAVAC_RET )) -checkJavadocWarnings -(( RESULT = RESULT + $? )) -buildAndInstall -checkFindbugsWarnings -(( RESULT = RESULT + $? )) -checkReleaseAuditWarnings -(( RESULT = RESULT + $? )) -### Run tests for Jenkins or if explictly asked for by a developer -if [[ $JENKINS == "true" || $RUN_TESTS == "true" ]] ; then - runTests - (( RESULT = RESULT + $? )) -fi -JIRA_COMMENT_FOOTER="Test results: $BUILD_URL/testReport/ -$JIRA_COMMENT_FOOTER" - -submitJiraComment $RESULT -cleanupAndExit $RESULT diff --git a/build-tools/tez-personality.sh b/build-tools/tez-personality.sh new file mode 100755 index 0000000000..830dc1fb19 --- /dev/null +++ b/build-tools/tez-personality.sh @@ -0,0 +1,70 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +personality_plugins "all" + +## @description Globals specific to this personality +## @audience private +## @stability evolving +function personality_globals +{ + # shellcheck disable=SC2034 + BUILDTOOL=maven + #shellcheck disable=SC2034 + PATCH_BRANCH_DEFAULT=master + #shellcheck disable=SC2034 + JIRA_ISSUE_RE='^TEZ-[0-9]+$' + #shellcheck disable=SC2034 + GITHUB_REPO_DEFAULT="apache/tez" + #shellcheck disable=SC2034 + PATCH_NAMING_RULE="https://cwiki.apache.org/confluence/display/TEZ/How+to+Contribute+to+Tez" +} + +## @description maven personality handler +## @audience private +## @stability evolving +## @replaceable yes +function maven_builtin_personality_modules +{ + declare repostatus=$1 + declare testtype=$2 + declare module + + yetus_debug "Using custom Tez personality_modules" + yetus_debug "Personality: ${repostatus} ${testtype}" + + clear_personality_queue + + # this always makes sure the local repo has a fresh + # copy of everything per pom rules. + if [[ ${repostatus} == branch + && ${testtype} == mvninstall ]] || + [[ "${BUILDMODE}" = full ]];then + personality_enqueue_module . + return + fi + + # force running all unit tests if we are doing a Tez precommit + if [[ ${repostatus} == patch + && ${testtype} == unit ]];then + personality_enqueue_module . + return + fi + + for module in "${CHANGED_MODULES[@]}"; do + personality_enqueue_module "${module}" + done +}