diff --git a/docker-build.sh b/docker-build.sh index b7211f464..efa1a62ce 100755 --- a/docker-build.sh +++ b/docker-build.sh @@ -95,14 +95,15 @@ buildOpenJDKViaDocker() # TODO This could be extracted overridden by the user if we support more # architectures going forwards - local container_architecture="x86_64/ubuntu" + local container_architecture + container_architecture="$(uname -m)/${BUILD_CONFIG[CONTAINER_IMAGE]//:*/}" local build_variant_flag="" BUILD_CONFIG[DOCKER_FILE_PATH]="docker/${BUILD_CONFIG[OPENJDK_CORE_VERSION]}/$container_architecture" if [ "${BUILD_CONFIG[BUILD_VARIANT]}" == "openj9" ]; then build_variant_flag="--openj9" fi - docker/dockerfile-generator.sh --version "${BUILD_CONFIG[OPENJDK_FEATURE_NUMBER]}" --path "${BUILD_CONFIG[DOCKER_FILE_PATH]}" "$build_variant_flag" \ + docker/dockerfile-generator.sh --version "${BUILD_CONFIG[OPENJDK_FEATURE_NUMBER]}" --path "${BUILD_CONFIG[DOCKER_FILE_PATH]}" "$build_variant_flag" --base-image "${BUILD_CONFIG[CONTAINER_IMAGE]}" \ --dirs "${workspacedir} ${targetdir} ${targetbuilddir} ${configdir} ${localsourcesdir}" --command "${BUILD_CONFIG[CONTAINER_AS_ROOT]} ${BUILD_CONFIG[CONTAINER_COMMAND]}" # shellcheck disable=SC1090,SC1091 diff --git a/docker/dockerfile-generator.sh b/docker/dockerfile-generator.sh index 932224e05..d1d000039 100755 --- a/docker/dockerfile-generator.sh +++ b/docker/dockerfile-generator.sh @@ -24,8 +24,21 @@ DOCKERFILE_DIR= DOCKERFILE_PATH= # Default to JDK8 JDK_VERSION=8 +IMAGE="ubuntu:18.04" JDK_MAX= JDK_GA= +DNF_INSTALL=dnf + +UBUNTU_PREAMBLE="apt-get update \\ + && apt-get install -qq -u --no-install-recommends \\ + software-properties-common \\ + dirmngr \\ + gpg-agent \\ + coreutils \\ + && apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys 0x219BD9C9 \\ + && add-apt-repository 'deb http://repos.azulsystems.com/ubuntu stable main' \\ + && apt-get update \\ + && apt-get -y upgrade \\" getFile() { if [ $# -ne 2 ]; then @@ -93,6 +106,11 @@ processArgs() { shift shift ;; + --base-image) + IMAGE="${2}" + shift + shift + ;; --path) DOCKERFILE_DIR=$2 shift @@ -152,6 +170,7 @@ usage() { Options: --help | -h Print this message and exit --build Build the docker image after generation and create interactive container + --base-image set the base image if used container. Default: $IMAGE --clean Remove all dockerfiles (Dockerfile*) from '--path' --comments Prints comments into the dockerfile --dirs space separated list of dirs to be created, with proper permissions @@ -186,31 +205,71 @@ printPreamble() { # SPDX-License-Identifier: Apache-2.0 # ******************************************************************************** -FROM ubuntu:18.04 +FROM $IMAGE LABEL maintainer=\"AdoptOpenJDK \" " >> "$DOCKERFILE_PATH" } -# Put in apt packages required for building a JDK -printAptPackages() { + +printAptPackagesBase() { if [ ${COMMENTS} == true ]; then echo " -# Install required OS tools +# Install required OS tools to setup environment as .deb via apt-get # dirmngr, gpg-agent & coreutils are all required for the apt-add repository command" >> "$DOCKERFILE_PATH" fi + echo " +RUN $UBUNTU_PREAMBLE + && apt-get install -qq -y --no-install-recommends \\ + curl \\ + git \\ + unzip \\ + wget \\ + zip " >> "$DOCKERFILE_PATH" + echo " +RUN rm -rf /var/lib/apt/lists/*" >> "$DOCKERFILE_PATH" +} +printDnfPackagesBase() { + if [ ${COMMENTS} == true ]; then + echo " +# Install required OS tools to setup environment as rpms via dnf" >> "$DOCKERFILE_PATH" + fi + local skipGpg="" # it may bite from time to time + #local skipGpg="--nogpgcheck" + local erasing="--allowerasing" + if [ ${DNF_INSTALL} = yum ] ; then + erasing="" + fi + if echo "${IMAGE}" | grep -e "stream8" -e "centos:7" ; then + echo " +RUN cd /etc/yum.repos.d/ ; sed -i 's/mirrorlist/#mirrorlist/g' /etc/yum.repos.d/CentOS-* +RUN cd /etc/yum.repos.d/ ; sed -i 's|#baseurl=http://mirror.centos.org|baseurl=http://vault.centos.org|g' /etc/yum.repos.d/CentOS-* " >> "$DOCKERFILE_PATH" + fi echo " -RUN apt-get update \\ - && apt-get install -qq -u --no-install-recommends \\ - software-properties-common \\ - dirmngr \\ - gpg-agent \\ - coreutils \\ - && apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys 0x219BD9C9 \\ - && add-apt-repository 'deb http://repos.azulsystems.com/ubuntu stable main' \\ - && apt-get update \\ - && apt-get -y upgrade \\ +RUN ${DNF_INSTALL} $skipGpg -y update $erasing +RUN ${DNF_INSTALL} $skipGpg -y install $erasing \\ + bzip2-libs \\ + bzip2 \\ + curl \\ + git \\ + unzip \\ + /usr/bin/which \\ + wget \\ + zip " >> "$DOCKERFILE_PATH" + echo " +RUN ${DNF_INSTALL} clean all" >> "$DOCKERFILE_PATH" +} + +printAptPackagesJdk() { + if [ ${COMMENTS} == true ]; then + echo " +# Install required OS tools to build JDK as .deb via apt-get +# dirmngr, gpg-agent & coreutils are all required for the apt-add repository command" >> "$DOCKERFILE_PATH" + fi + + echo " +RUN $UBUNTU_PREAMBLE && apt-get install -qq -y --no-install-recommends \\ ant \\ ant-contrib \\ @@ -218,9 +277,7 @@ RUN apt-get update \\ ca-certificates \\ cmake \\ cpio \\ - curl \\ file \\ - git \\ libasound2-dev \\ libcups2-dev \\ libelf-dev \\ @@ -236,10 +293,7 @@ RUN apt-get update \\ make \\ perl \\ ssh \\ - systemtap-sdt-dev \\ - unzip \\ - wget \\ - zip \\" >> "$DOCKERFILE_PATH" + systemtap-sdt-dev \\" >> "$DOCKERFILE_PATH" if [ ${OPENJ9} = true ]; then echo " gcc-7 \\ @@ -268,6 +322,62 @@ RUN apt-get update \\ echo " && rm -rf /var/lib/apt/lists/*" >> "$DOCKERFILE_PATH" } +printDnfPackagesJdk() { + if [ ${COMMENTS} == true ]; then + echo " +# Install required OS tools to build JDK as rpms via dnf" >> "$DOCKERFILE_PATH" + fi + local skipGpg="" # it may bite from time to time + #local skipGpg="--nogpgcheck" + local erasing="--allowerasing" + if [ ${DNF_INSTALL} = yum ] ; then + erasing="" + fi + echo " +RUN ${DNF_INSTALL} $skipGpg -y install $erasing \\ + ant \\ + autoconf \\ + automake \\ + ca-certificates \\ + cmake \\ + cpio \\ + diffutils \\ + file \\ + alsa-lib-devel \\ + cups-devel \\ + gcc \\ + gcc-c++ \\ + gdb \\ + fontconfig-devel \\ + freetype-devel \\ + libtool \\ + libX11-devel \\ + libXi-devel \\ + libXinerama-devel \\ + libXrandr-devel \\ + libXrender-devel \\ + libXt-devel \\ + libXtst-devel \\ + lksctp-tools-devel \\ + lksctp-tools pcsc-lite-libs \\ + make \\ + perl \\ + procps-ng \\ + openssh-clients \\ + openssl \\ + systemtap-sdt-devel \\ + kernel-headers \\ + \"lcms*\" \\ + nss-devel \\ " >> "$DOCKERFILE_PATH" + if echo "${IMAGE}" | grep fedora ; then + echo " libstdc++-static \\ + pcsc-lite-devel \\ " >> "$DOCKERFILE_PATH" + fi + echo " tzdata-java " >> "$DOCKERFILE_PATH" + echo " +RUN ${DNF_INSTALL} clean all" >> "$DOCKERFILE_PATH" +} + printCreateFolder() { echo " RUN mkdir -p /openjdk/target @@ -285,6 +395,10 @@ ENV CC=gcc-7 CXX=g++-7" >> "$DOCKERFILE_PATH" } printCustomDirs() { + if [ ${COMMENTS} == true ]; then + echo "# In podman (in docker do not harm) shared folder is owned by root, and is read for others, unless it already exists" >> "$DOCKERFILE_PATH" + echo "# So we have to create all future-mounted dirs, with proper owner and permissions" >> "$DOCKERFILE_PATH" + fi for dir in ${DIRS} ; do echo "RUN mkdir -p $dir" >> "$DOCKERFILE_PATH" echo "RUN chmod 755 $dir" >> "$DOCKERFILE_PATH" @@ -293,6 +407,10 @@ printCustomDirs() { } printDockerJDKs() { + if [ ${COMMENTS} == true ]; then + echo " + # Linking of boot jdk must happen after the system jdk is isntalled, as it is iverwriting whatever java/javac from system" >> "$DOCKERFILE_PATH" + fi # JDK8 uses zulu-7 to as it's bootjdk if [ "${JDK_VERSION}" != 8 ] && [ "${JDK_VERSION}" != "${JDK_MAX}" ]; then if [ "${JDK_VERSION}" == 11 ]; then @@ -341,7 +459,7 @@ printDockerJDKs() { printJDK() { local JDKVersion=$1 echo " -RUN sh -c \"mkdir -p /usr/lib/jvm/jdk$JDKVersion && wget 'https://api.adoptium.net/v3/binary/latest/$JDKVersion/ga/linux/x64/jdk/hotspot/normal/adoptium?project=jdk' -O - | tar xzf - -C /usr/lib/jvm/jdk$JDKVersion --strip-components=1\"" >> "$DOCKERFILE_PATH" +RUN sh -c \"mkdir -p /usr/lib/jvm/jdk$JDKVersion && wget 'https://api.adoptium.net/v3/binary/latest/$JDKVersion/ga/linux/$(adoptiumArch)/jdk/hotspot/normal/adoptium?project=jdk' -O - | tar xzf - -C /usr/lib/jvm/jdk$JDKVersion --strip-components=1\"" >> "$DOCKERFILE_PATH" } printGitCloneJenkinsPipelines(){ @@ -370,19 +488,75 @@ RUN useradd -u \$HostUID -ms /bin/bash build WORKDIR /openjdk/build RUN chown -R build /openjdk/" >> "$DOCKERFILE_PATH" printCustomDirs +} + +printUserSet(){ echo " USER build" >> "$DOCKERFILE_PATH" } -printContainerVars(){ +adoptiumArch() { + local arch + arch=$(uname -m) + if [ "$arch" = "x86_64" ] ; then arch="x64" ; fi + echo "$arch" +} + +printContainerVars() { echo " ARG OPENJDK_CORE_VERSION ENV OPENJDK_CORE_VERSION=\$OPENJDK_CORE_VERSION -ENV ARCHITECTURE=x64 +ENV ARCHITECTURE=$(adoptiumArch) ENV JDK_PATH=jdk ENV JDK8_BOOT_DIR=/usr/lib/jvm/jdk8" >> "$DOCKERFILE_PATH" } +isRpm() { + echo "${IMAGE}" | grep -i -e "fedora" -e "centos" -e "rocky" -e "stream" -e "rhel" +} + +isDeb() { + echo "${IMAGE}" | grep -i -e "ubuntu" -e "debian" +} + +isYum() { + if echo "${IMAGE}" | grep -e "stream7" -e "centos:7" ; then + DNF_INSTALL=yum + else + DNF_INSTALL=dnf + fi +} + +printDepsBase() { + if isRpm ; then + isYum + printDnfPackagesBase + elif isDeb ; then + printAptPackagesBase + # OpenJ9 MUST use gcc7, HS doesn't have to + if [ ${OPENJ9} == true ]; then + printgcc + fi + else + echo "Unknown system, can not install build deps: $IMAGE" + fi +} + +printDepsJdk() { + if isRpm ; then + isYum + printDnfPackagesJdk + elif isDeb ; then + printAptPackagesJdk + # OpenJ9 MUST use gcc7, HS doesn't have to + if [ ${OPENJ9} == true ]; then + printgcc + fi + else + echo "Unknown system, can not install build deps: $IMAGE" + fi +} + generateFile() { mkdir -p "$DOCKERFILE_DIR" if [ -f "$DOCKERFILE_PATH" ]; then @@ -412,15 +586,9 @@ processArgs "$@" generateFile generateConfig printPreamble -printAptPackages -# OpenJ9 MUST use gcc7, HS doesn't have to -if [ ${OPENJ9} == true ]; then - printgcc -fi - -printDockerJDKs +printUserCreate +printDepsBase printGitCloneJenkinsPipelines - # If building the image straight away, it can't be assumed the folders to be copied are in place # Therefore create an image that instead git clones openjdk-build and a build can be started there if [ ${BUILD} == false ]; then @@ -428,8 +596,9 @@ if [ ${BUILD} == false ]; then else printGitClone fi - -printUserCreate +printDepsJdk +printDockerJDKs +printUserSet printContainerVars echo "Dockerfile created at $DOCKERFILE_PATH" diff --git a/makejdk-any-platform.1 b/makejdk-any-platform.1 index 80cb4158c..ddb1658dd 100755 --- a/makejdk-any-platform.1 +++ b/makejdk-any-platform.1 @@ -22,7 +22,7 @@ that you are building for further details. \fBExample:\fR The simplest use case to run is: -"./makejdk-any-platform.sh -D jdk8u" +"./makejdk-any-platform.sh -D fedora:latest jdk8u" This will start a Docker container and build you the latest Java 8 Temurin binary from the source at https://github.com/adoptium/openjdk-jdk8u @@ -97,8 +97,9 @@ specify the location for the built binary, e.g. /path/. This is typically used in conjunction with \fB<-T>\fR to create a custom path / file name for the resulting binary. .TP -.BR \-D ", " \-\-docker ", " \-\-podman +.BR \-D ", " \-\-docker ", " \-\-podman " " \fI[base_to_use]\fR build OpenJDK in a docker/podman container. -D will autodetect, using podman if found, docker otherwise. +You can optionally set base image to use. Default is ubuntu:18.04. Dnf/apt is determined by hardcoded list. When podman is used, there is bug requiring \fI--custom-cacerts\fR to be used under some circumstances. It is known issue: https://github.com/adoptium/temurin-build/issues/3862 .TP diff --git a/sbin/common/common.sh b/sbin/common/common.sh index 6cb632802..b4d4be965 100755 --- a/sbin/common/common.sh +++ b/sbin/common/common.sh @@ -22,19 +22,12 @@ function setOpenJdkVersion() { # # Format: jdkNN[u] # - local forest_name=$1 + local forest_name="${1}" echo "Setting version based on forest_name=${forest_name}" - - # The argument passed here have actually very strict format of jdk8, jdk8u..., jdk - # the build may fail later if this is not honoured. - # If your repository has a different name, you can use --version or build from dir/snapshot - local forest_name_check1=0 - local forest_name_check2=0 - # This two returns condition is there to make grep on solaris happy. -e, -q and \( and \| do not work on that platform - echo "$forest_name" | grep "^jdk[0-9]\\{1,3\\}[u]\\{0,1\\}$" >/dev/null || forest_name_check1=$? - echo "$forest_name" | grep "^jdk$" >/dev/null || forest_name_check2=$? - if [ ${forest_name_check1} -ne 0 ] && [ ${forest_name_check2} -ne 0 ]; then + forest_name_check=0 + checkOpenJdkVersion "${forest_name}" || forest_name_check=$? + if [ ${forest_name_check} -ne 0 ] ; then echo "The mandatory repo argument has a very strict format 'jdk[0-9]{1,3}[u]{0,1}' or just plain 'jdk' for tip. '$forest_name' does not match." echo "This can be worked around by using '--version jdkXYu'. If set (and matching) then the main argument can have any value." exit 1 @@ -94,6 +87,23 @@ function setOpenJdkVersion() { setBranch } +function checkOpenJdkVersion() { + local forest_name="${1}" + # The argument passed here has a very strict format of jdk8, jdk8u..., jdk + # the build may fail later if this is not honoured. + # If your repository has a different name, you can use --version or build from a dir/snapshot + local forest_name_check1=0 + local forest_name_check2=0 + # This two returns condition is there to make grep on solaris happy. -e, -q and \( and \| do not work on that platform + echo "$forest_name" | grep "^jdk[0-9]\\{1,3\\}[u]\\{0,1\\}$" >/dev/null || forest_name_check1=$? + echo "$forest_name" | grep "^jdk$" >/dev/null || forest_name_check2=$? + if [ ${forest_name_check1} -ne 0 ] && [ ${forest_name_check2} -ne 0 ]; then + return 1 + else + return 0 + fi +} + # Set the default BUILD_CONFIG[BRANCH] for the jdk version being built # For "hotspot" and "Temurin" builds of non-"u" jdk-23+ the branch is dev_ function setBranch() { diff --git a/sbin/common/config_init.sh b/sbin/common/config_init.sh index 97ea90d12..d2bccb4b8 100755 --- a/sbin/common/config_init.sh +++ b/sbin/common/config_init.sh @@ -47,6 +47,7 @@ CLEAN_GIT_REPO CLEAN_LIBS CONTAINER_COMMAND CONTAINER_NAME +CONTAINER_IMAGE COPY_MACOSX_FREE_FONT_LIB_FOR_JDK_FLAG COPY_MACOSX_FREE_FONT_LIB_FOR_JRE_FLAG COPY_TO_HOST @@ -274,13 +275,19 @@ function parseConfigurationArguments() { BUILD_CONFIG[TARGET_DIR]="$1"; shift;; "-D" ) - if which podman > /dev/null ; then BUILD_CONFIG[CONTAINER_COMMAND]="podman" ; else BUILD_CONFIG[CONTAINER_COMMAND]="docker" ; fi;; + if which podman > /dev/null ; then BUILD_CONFIG[CONTAINER_COMMAND]="podman" ; else BUILD_CONFIG[CONTAINER_COMMAND]="docker" ; fi; + if setCustomImage "${1-}"; then shift ; fi + ;; "--docker" ) - BUILD_CONFIG[CONTAINER_COMMAND]="docker";; + BUILD_CONFIG[CONTAINER_COMMAND]="docker"; + if setCustomImage "${1-}"; then shift ; fi + ;; "--podman" ) - BUILD_CONFIG[CONTAINER_COMMAND]="podman";; + BUILD_CONFIG[CONTAINER_COMMAND]="podman"; + if setCustomImage "${1-}"; then shift ; fi + ;; "--debug-docker" ) BUILD_CONFIG[DEBUG_DOCKER]="true";; @@ -418,6 +425,30 @@ function setOpenjdkSourceDir() { fi } +# Set custom image base if set +function setCustomImage() { + local imageCandidate="${1}" + # is next param empty? + if [[ -z ${imageCandidate} ]] ; then + echo "default image will be used: ${BUILD_CONFIG[CONTAINER_IMAGE]}" + return 1 + fi + # is the next parameter a switch? + if [[ ${imageCandidate} == -* ]] ; then + echo "default image will be used: ${BUILD_CONFIG[CONTAINER_IMAGE]}" + return 1 + fi + # is the next param a main arg? + if checkOpenJdkVersion "${imageCandidate}" ; then + echo "default image will be used: ${BUILD_CONFIG[CONTAINER_IMAGE]}" + return 1 + fi + # not empty, not switch, not main arg - therefore it must be an image, use it + BUILD_CONFIG[CONTAINER_IMAGE]="${imageCandidate}" + echo "base image will be set to: ${BUILD_CONFIG[CONTAINER_IMAGE]}" + return 0 +} + # Set the config defaults function configDefaults() { @@ -544,6 +575,7 @@ function configDefaults() { BUILD_CONFIG[DOCKER_SOURCE_VOLUME_NAME]=${BUILD_CONFIG[DOCKER_SOURCE_VOLUME_NAME]:-"openjdk-source-volume"} BUILD_CONFIG[CONTAINER_NAME]=${BUILD_CONFIG[CONTAINER_NAME]:-openjdk_container} + BUILD_CONFIG[CONTAINER_IMAGE]=${BUILD_CONFIG[CONTAINER_IMAGE]:-"ubuntu:18.04"} BUILD_CONFIG[TMP_CONTAINER_NAME]=${BUILD_CONFIG[TMP_CONTAINER_NAME]:-openjdk-copy-src} BUILD_CONFIG[CLEAN_DOCKER_BUILD]=${BUILD_CONFIG[CLEAN_DOCKER_BUILD]:-false}