From 3c9a3b1772d0569d747f382a9ed55d7c80194778 Mon Sep 17 00:00:00 2001 From: John Oliver Date: Wed, 26 Sep 2018 12:03:47 +0100 Subject: [PATCH 1/3] Make some freetype downloading improvements. Add release table generation script --- .../windows.sh | 8 +-- sbin/build.sh | 32 ++++++++---- sbin/common/config_init.sh | 3 ++ sbin/prepareWorkspace.sh | 51 +++++++++++-------- 4 files changed, 58 insertions(+), 36 deletions(-) diff --git a/build-farm/platform-specific-configurations/windows.sh b/build-farm/platform-specific-configurations/windows.sh index 4f74bacb3..0524afe60 100755 --- a/build-farm/platform-specific-configurations/windows.sh +++ b/build-farm/platform-specific-configurations/windows.sh @@ -65,6 +65,7 @@ if [ "${ARCHITECTURE}" == "x64" ] then if [ "${VARIANT}" == "hotspot" ] then + export CONFIGURE_ARGS_FOR_ANY_PLATFORM="${CONFIGURE_ARGS_FOR_ANY_PLATFORM} --with-toolchain-version=2013" if [ "${JAVA_TO_BUILD}" == "${JDK8_VERSION}" ] then export PATH="/cygdrive/c/Program Files (x86)/Microsoft Visual Studio 10.0/VC/bin/amd64/:/cygdrive/c/openjdk/make-3.82/:$PATH" @@ -88,12 +89,13 @@ then then export PATH="/cygdrive/c/mingw-w64\x86_64-8.1.0-win32-seh-rt_v6-rev0/mingw64/bin:/usr/bin:$PATH" export HAS_AUTOCONF=1 + export BUILD_ARGS="${BUILD_ARGS} --freetype-version 2.5.3" if [ "${JAVA_TO_BUILD}" == "${JDK8_VERSION}" ] then - export INCLUDE="C:\Program Files\Debugging Tools for Windows (x64)\sdk\inc;%INCLUDE%" + export INCLUDE="C:\Program Files\Debugging Tools for Windows (x64)\sdk\inc;$INCLUDE" export PATH="$PATH:/c/cygwin64/bin" - export CONFIGURE_ARGS_FOR_ANY_PLATFORM="${CONFIGURE_ARGS_FOR_ANY_PLATFORM} --with-freetype-src=/cygdrive/c/openjdk/freetype-2.5.3 --with-freemarker-jar=/cygdrive/c/openjdk/freemarker.jar" + export CONFIGURE_ARGS_FOR_ANY_PLATFORM="${CONFIGURE_ARGS_FOR_ANY_PLATFORM} --with-freetype-include=/cygdrive/c/openjdk/freetype-2.5.3/include --with-freetype-lib=/cygdrive/c/openjdk/freetype-2.5.3/lib64 --with-freemarker-jar=/cygdrive/c/openjdk/freemarker.jar --disable-ccache" elif [ "${JAVA_TO_BUILD}" == "${JDK9_VERSION}" ] then export CONFIGURE_ARGS_FOR_ANY_PLATFORM="${CONFIGURE_ARGS_FOR_ANY_PLATFORM} --with-freetype-src=/cygdrive/c/openjdk/freetype-2.5.3 --with-toolchain-version=2013 --with-freemarker-jar=/cygdrive/c/openjdk/freemarker.jar" @@ -104,7 +106,5 @@ then then export CONFIGURE_ARGS_FOR_ANY_PLATFORM="${CONFIGURE_ARGS_FOR_ANY_PLATFORM} --with-freemarker-jar=/cygdrive/c/openjdk/freemarker.jar --with-toolchain-version=2017" fi - else - export CONFIGURE_ARGS_FOR_ANY_PLATFORM="${CONFIGURE_ARGS_FOR_ANY_PLATFORM} --with-toolchain-version=2013" fi fi diff --git a/sbin/build.sh b/sbin/build.sh index 88f642161..0f80b2845 100755 --- a/sbin/build.sh +++ b/sbin/build.sh @@ -257,15 +257,6 @@ buildingTheRestOfTheConfigParameters() addConfigureArg "--with-freemarker-jar=" "${BUILD_CONFIG[WORKSPACE_DIR]}/${BUILD_CONFIG[WORKING_DIR]}/freemarker-${FREEMARKER_LIB_VERSION}/freemarker.jar" fi - if [[ "${BUILD_CONFIG[FREETYPE]}" == "true" ]] ; then - local freetypeDir=BUILD_CONFIG[FREETYPE_DIRECTORY] - case "${BUILD_CONFIG[OPENJDK_CORE_VERSION]}" in - jdk8*|jdk9*|jdk10*) freetypeDir=${BUILD_CONFIG[FREETYPE_DIRECTORY]:-"${BUILD_CONFIG[WORKSPACE_DIR]}/${BUILD_CONFIG[WORKING_DIR]}/installedfreetype"} ;; - *) freetypeDir=${BUILD_CONFIG[FREETYPE_DIRECTORY]:-bundled} ;; - esac - addConfigureArg "--with-freetype=" "${freetypeDir}" - fi - addConfigureArg "--with-x=" "/usr/include/X11" if [ "${BUILD_CONFIG[OPENJDK_CORE_VERSION]}" == "${JDK8_CORE_VERSION}" ] ; then @@ -280,14 +271,33 @@ buildingTheRestOfTheConfigParameters() fi } +configureFreetypeLocation() { + if [[ ! "${CONFIGURE_ARGS}" =~ "--with-freetype" ]]; then + if [[ "${BUILD_CONFIG[FREETYPE]}" == "true" ]] ; then + if [[ "$OSTYPE" == "cygwin" ]] || [[ "$OSTYPE" == "msys" ]] ; then + addConfigureArg "--with-freetype-src=" "${BUILD_CONFIG[WORKSPACE_DIR]}/libs/freetype" + else + local freetypeDir=BUILD_CONFIG[FREETYPE_DIRECTORY] + case "${BUILD_CONFIG[OPENJDK_CORE_VERSION]}" in + jdk8*|jdk9*|jdk10*) freetypeDir=${BUILD_CONFIG[FREETYPE_DIRECTORY]:-"${BUILD_CONFIG[WORKSPACE_DIR]}/${BUILD_CONFIG[WORKING_DIR]}/installedfreetype"} ;; + *) freetypeDir=${BUILD_CONFIG[FREETYPE_DIRECTORY]:-bundled} ;; + esac + + echo "setting freetype dir to ${freetypeDir}" + addConfigureArg "--with-freetype=" "${freetypeDir}" + fi + fi + fi +} + # Configure the command parameters configureCommandParameters() { configuringVersionStringParameter configuringBootJDKConfigureParameter + if [[ "$OSTYPE" == "cygwin" ]] || [[ "$OSTYPE" == "msys" ]] ; then echo "Windows or Windows-like environment detected, skipping configuring environment for custom Boot JDK and other 'configure' settings." - else echo "Building up the configure command..." buildingTheRestOfTheConfigParameters @@ -296,6 +306,8 @@ configureCommandParameters() # Now we add any configure arguments the user has specified on the command line. CONFIGURE_ARGS="${CONFIGURE_ARGS} ${BUILD_CONFIG[USER_SUPPLIED_CONFIGURE_ARGS]}" + configureFreetypeLocation + echo "Completed configuring the version string parameter, config args are now: ${CONFIGURE_ARGS}" } diff --git a/sbin/common/config_init.sh b/sbin/common/config_init.sh index 486873520..d3ac96ba9 100755 --- a/sbin/common/config_init.sh +++ b/sbin/common/config_init.sh @@ -189,6 +189,9 @@ function parseConfigurationArguments() { "--freetype-build-param" ) BUILD_CONFIG[FREETYPE_FONT_BUILD_TYPE_PARAM]="$1"; shift;; + "--freetype-version" ) + BUILD_CONFIG[FREETYPE_FONT_VERSION]="$1"; shift;; + "--skip-freetype" | "-F" ) BUILD_CONFIG[FREETYPE]=false;; diff --git a/sbin/prepareWorkspace.sh b/sbin/prepareWorkspace.sh index 323fcebf8..cb184add9 100755 --- a/sbin/prepareWorkspace.sh +++ b/sbin/prepareWorkspace.sh @@ -180,7 +180,6 @@ checkingAndDownloadingFreemarker() # Get Freetype checkingAndDownloadingFreeType() { - cd "${BUILD_CONFIG[WORKSPACE_DIR]}/libs/" || exit echo "Checking for freetype at ${BUILD_CONFIG[WORKSPACE_DIR]}/${BUILD_CONFIG[WORKING_DIR]}" @@ -189,17 +188,23 @@ checkingAndDownloadingFreeType() if [[ ! -z "$FOUND_FREETYPE" ]] ; then echo "Skipping FreeType download" else - # Then FreeType for fonts: make it and use - wget -nc --no-check-certificate https://ftp.acc.umu.se/mirror/gnu.org/savannah/freetype/freetype-"${BUILD_CONFIG[FREETYPE_FONT_VERSION]}".tar.gz + curl -L -o "freetype.tar.gz" "https://download.savannah.gnu.org/releases/freetype/freetype-${BUILD_CONFIG[FREETYPE_FONT_VERSION]}.tar.gz" + + rm -rf "./freetype" || true + mkdir -p "freetype" || true + tar xpzf freetype.tar.gz --strip-components=1 -C "freetype" + rm freetype.tar.gz + + if [[ "$OSTYPE" == "cygwin" ]] || [[ "$OSTYPE" == "msys" ]] ; then + return; + fi - gunzip -dc freetype-"${BUILD_CONFIG[FREETYPE_FONT_VERSION]}".tar.gz | tar xf - -C . - rm freetype-"${BUILD_CONFIG[FREETYPE_FONT_VERSION]}".tar.gz + cd freetype || exit - cd freetype-"${BUILD_CONFIG[FREETYPE_FONT_VERSION]}" || exit # We get the files we need at $WORKING_DIR/installedfreetype # shellcheck disable=SC2046 - if ! (bash ./configure --prefix="${BUILD_CONFIG[WORKSPACE_DIR]}/${BUILD_CONFIG[WORKING_DIR]}"/installedfreetype "${BUILD_CONFIG[FREETYPE_FONT_BUILD_TYPE_PARAM]}" && ${BUILD_CONFIG[MAKE_COMMAND_NAME]} all && ${BUILD_CONFIG[MAKE_COMMAND_NAME]} install); then + if ! (bash ./configure "${BUILD_CONFIG[FREETYPE_FONT_BUILD_TYPE_PARAM]}" && ${BUILD_CONFIG[MAKE_COMMAND_NAME]} all && ${BUILD_CONFIG[MAKE_COMMAND_NAME]} install); then # shellcheck disable=SC2154 echo "Failed to configure and build libfreetype, exiting" exit; @@ -266,30 +271,31 @@ downloadingRequiredDependencies() cd "${BUILD_CONFIG[WORKSPACE_DIR]}/libs/" || exit if [[ "$OSTYPE" == "cygwin" ]] || [[ "$OSTYPE" == "msys" ]] ; then - echo "Windows or Windows-like environment detected, skipping downloading of dependencies...: Alsa, Freetype, and CaCerts." + echo "Windows or Windows-like environment detected, skipping downloading of dependencies...: Alsa." else echo "Downloading required dependencies...: Alsa, Freetype, Freemarker, and CaCerts." echo "Checking and download Alsa dependency" checkingAndDownloadingAlsa - if [[ "${BUILD_CONFIG[FREETYPE]}" == "true" ]] ; then - if [ -z "${BUILD_CONFIG[FREETYPE_DIRECTORY]}" ]; then - echo "Checking and download FreeType Font dependency" - checkingAndDownloadingFreeType - else - echo "" - echo "---> Skipping the process of checking and downloading the FreeType Font dependency, a pre-built version provided at ${BUILD_CONFIG[FREETYPE_DIRECTORY]} <---" - echo "" - fi - else - echo "Skipping Freetype" - fi if [[ "${BUILD_CONFIG[BUILD_VARIANT]}" == "openj9" ]]; then echo "Checking and download Freemarker dependency" checkingAndDownloadingFreemarker fi fi + if [[ "${BUILD_CONFIG[FREETYPE]}" == "true" ]] ; then + if [ -z "${BUILD_CONFIG[FREETYPE_DIRECTORY]}" ]; then + echo "Checking and download FreeType Font dependency" + checkingAndDownloadingFreeType + else + echo "" + echo "---> Skipping the process of checking and downloading the FreeType Font dependency, a pre-built version provided at ${BUILD_CONFIG[FREETYPE_DIRECTORY]} <---" + echo "" + fi + else + echo "Skipping Freetype" + fi + echo "Checking and download CaCerts dependency" checkingAndDownloadCaCerts @@ -328,13 +334,14 @@ relocateToTmpIfNeeded() export TMP_WORKSPACE="${tmpdir}" export ORIGINAL_WORKSPACE="${BUILD_CONFIG[WORKSPACE_DIR]}" + trap moveTmpToWorkspaceLocation EXIT SIGINT SIGTERM + if [ -d "${ORIGINAL_WORKSPACE}" ] then cp -r "${BUILD_CONFIG[WORKSPACE_DIR]}" "${TMP_WORKSPACE}/workspace" fi BUILD_CONFIG[WORKSPACE_DIR]="${TMP_WORKSPACE}/workspace" - trap moveTmpToWorkspaceLocation EXIT SIGINT SIGTERM fi } @@ -342,8 +349,8 @@ relocateToTmpIfNeeded() function configureWorkspace() { createWorkspace + downloadingRequiredDependencies relocateToTmpIfNeeded checkoutAndCloneOpenJDKGitRepo - downloadingRequiredDependencies } From 234fd25d86217938e8463f57b3ee4f4d9f3a3517 Mon Sep 17 00:00:00 2001 From: John Oliver Date: Wed, 26 Sep 2018 12:07:56 +0100 Subject: [PATCH 2/3] Fix freetype build --- sbin/prepareWorkspace.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sbin/prepareWorkspace.sh b/sbin/prepareWorkspace.sh index cb184add9..c81828d60 100755 --- a/sbin/prepareWorkspace.sh +++ b/sbin/prepareWorkspace.sh @@ -204,7 +204,7 @@ checkingAndDownloadingFreeType() # We get the files we need at $WORKING_DIR/installedfreetype # shellcheck disable=SC2046 - if ! (bash ./configure "${BUILD_CONFIG[FREETYPE_FONT_BUILD_TYPE_PARAM]}" && ${BUILD_CONFIG[MAKE_COMMAND_NAME]} all && ${BUILD_CONFIG[MAKE_COMMAND_NAME]} install); then + if ! (bash ./configure --prefix="${BUILD_CONFIG[WORKSPACE_DIR]}/${BUILD_CONFIG[WORKING_DIR]}"/installedfreetype "${BUILD_CONFIG[FREETYPE_FONT_BUILD_TYPE_PARAM]}" && ${BUILD_CONFIG[MAKE_COMMAND_NAME]} all && ${BUILD_CONFIG[MAKE_COMMAND_NAME]} install); then # shellcheck disable=SC2154 echo "Failed to configure and build libfreetype, exiting" exit; From 8539f5fcce02a5fa7ac6c659b215befae9a7d5df Mon Sep 17 00:00:00 2001 From: John Oliver Date: Wed, 26 Sep 2018 13:47:08 +0100 Subject: [PATCH 3/3] Allow configuring tests based on release/nightly --- pipelines/build/build_base_file.groovy | 70 +++++++++++++++-------- pipelines/build/openjdk11_pipeline.groovy | 6 +- 2 files changed, 51 insertions(+), 25 deletions(-) diff --git a/pipelines/build/build_base_file.groovy b/pipelines/build/build_base_file.groovy index 9664f3264..aae1d2110 100644 --- a/pipelines/build/build_base_file.groovy +++ b/pipelines/build/build_base_file.groovy @@ -41,29 +41,7 @@ def toBuildParams(enableTests, params) { static def buildConfiguration(javaToBuild, variant, configuration, releaseTag, branch, additionalConfigureArgs) { - String buildTag = "build" - - if (configuration.os == "windows" && variant == "openj9") { - buildTag = "buildj9" - } else if (configuration.arch == "s390x" && variant == "openj9") { - buildTag = "(buildj9||build)&&openj9" - } - - def additionalNodeLabels = null - if (configuration.containsKey("additionalNodeLabels")) { - // hack as jenkins sandbox wont allow instanceof - if ("java.util.LinkedHashMap" == configuration.additionalNodeLabels.getClass().getName()) { - additionalNodeLabels = configuration.additionalNodeLabels.get(variant) - } else { - additionalNodeLabels = configuration.additionalNodeLabels - } - } - - if (additionalNodeLabels != null) { - additionalNodeLabels = "${additionalNodeLabels}&&${buildTag}" - } else { - additionalNodeLabels = "${buildTag}" - } + def additionalNodeLabels = formAdditionalNodeLabels(configuration, variant) def buildParams = [ JAVA_TO_BUILD: javaToBuild, @@ -83,20 +61,64 @@ static def buildConfiguration(javaToBuild, variant, configuration, releaseTag, b buildParams.put("BRANCH", branch) } + def isRelease = false if (releaseTag != null && releaseTag.length() > 0) { + isRelease = true buildParams.put("TAG", releaseTag) } + def testList = getTestList(configuration, isRelease) + return [ javaVersion: javaToBuild, arch : configuration.arch, os : configuration.os, variant : variant, parameters : buildParams, - test : configuration.test, + test : testList, ] } +static def getTestList(configuration, isRelease) { + if (configuration.containsKey("test")) { + def testJobType = isRelease ? "release" : "nightly" + + // hack as jenkins sandbox wont allow instanceof + if ("java.util.LinkedHashMap" == configuration.test.getClass().getName()) { + return configuration.test.get(testJobType) + } else { + return configuration.test + } + } + return [] +} + +static def formAdditionalNodeLabels(configuration, variant) { + def buildTag = "build" + + if (configuration.os == "windows" && variant == "openj9") { + buildTag = "buildj9" + } else if (configuration.arch == "s390x" && variant == "openj9") { + buildTag = "(buildj9||build)&&openj9" + } + + def labels = "${buildTag}" + + if (configuration.containsKey("additionalNodeLabels")) { + def additionalNodeLabels = null + + // hack as jenkins sandbox wont allow instanceof + if ("java.util.LinkedHashMap" == configuration.additionalNodeLabels.getClass().getName()) { + additionalNodeLabels = configuration.additionalNodeLabels.get(variant) + } else { + additionalNodeLabels = configuration.additionalNodeLabels + } + labels = "${additionalNodeLabels}&&${labels}" + } + + return labels +} + static def getConfigureArgs(configuration, additionalConfigureArgs) { def buildParams = [:] def configureArgs = ""; diff --git a/pipelines/build/openjdk11_pipeline.groovy b/pipelines/build/openjdk11_pipeline.groovy index 23b176c0f..c1cc1f21c 100644 --- a/pipelines/build/openjdk11_pipeline.groovy +++ b/pipelines/build/openjdk11_pipeline.groovy @@ -19,11 +19,15 @@ def buildConfigurations = [ additionalNodeLabels: 'build-macstadium-macos1010-1', test : ['openjdktest', 'systemtest'] ], + x64Linux : [ os : 'linux', arch : 'x64', additionalNodeLabels: 'centos6', - test : ['openjdktest', 'systemtest', 'externaltest'] + test : [ + nightly: ['openjdktest', 'systemtest', 'perftest', 'externaltest', 'externaltest_extended'], + release: ['openjdktest', 'systemtest', 'perftest', 'externaltest'] + ] ], // Currently we have to be quite specific about which windows to use as not all of them have freetype installed