From 3c02510fb6fc9ff750b8c847fcd3b52a4154b958 Mon Sep 17 00:00:00 2001 From: Scott J Dickerson Date: Mon, 2 May 2022 15:32:41 -0400 Subject: [PATCH] build: Add conditional builds for rpm build customization Instead of sed swapping in a 0 or 1 for a `%define` variable in the `spec.in` file, or running rpmbuild with a `--define`, use conditional builds. See [1] for how they work. With a conditional build, source rpms retain the ability to build with different options depending on how `rpmbuild` is called. This allows a minimal (en_US, Firefox, Safari) or full (all locales, all browsers) build from srpm to be selected at build time using `--with` or `--without` build options. Since at least copr builds chroots directly from a srpm, conditional builds can be used to change build options. With this change, a "full build" can be forced simply by configuring the target chroot setting "without" to include "ovirt_build_full". Notes: - A `ovirt_build_full` is the is the default - For GitHub CI, a minimal build with unit tests is used - For copr builds, use `build-srpm.sh -c` to keep CI and copr srpm builds in the same place - Refactored `build-srpm.sh` and `build-rpm.sh`: - each script has only the envvars setup that the work needs to run - use `getopts` to parse options so they don't positionally clash - add a snapshot build release suffix if the Makefile var `MILESTONE` is not empty [1] - https://rpm-software-management.github.io/rpm/manual/conditionalbuilds.html --- .automation/build-rpm.sh | 55 ++++++++++----- .automation/build-srpm.sh | 70 ++++++++++--------- .copr/Makefile | 11 +-- .github/workflows/build.yml | 2 +- Makefile | 2 +- README.adoc | 56 ++++++++++----- bump_release.sh | 20 +----- ovirt-engine.spec.in | 134 ++++++++++++++++++++++++------------ version.mak | 4 ++ 9 files changed, 212 insertions(+), 142 deletions(-) diff --git a/.automation/build-rpm.sh b/.automation/build-rpm.sh index a860f088637f..f7f3bab7607e 100755 --- a/.automation/build-rpm.sh +++ b/.automation/build-rpm.sh @@ -1,26 +1,49 @@ #!/bin/bash -xe +# Directory, where build artifacts will be stored, should be passed as '-a' parameter +ARTIFACTS_DIR=exported-artifacts + +# Process options +while getopts ":a:" options; do + case $options in + a) ARTIFACTS_DIR=$OPTARG;; + \?) + echo "Error: Invalid option" + exit;; + esac +done + +# Build source packages source $(dirname "$(readlink -f "$0")")/build-srpm.sh -# Install build dependencies -dnf builddep -y rpmbuild/SRPMS/*src.rpm +# GWT build memory needs to be limited +EXTRA_BUILD_FLAGS="--no-transfer-progress" +EXTRA_BUILD_FLAGS="${EXTRA_BUILD_FLAGS} -Dgwt.compiler.localWorkers=1" +EXTRA_BUILD_FLAGS="${EXTRA_BUILD_FLAGS} -Dgwt.jvmArgs='-Xms1G -Xmx3G'" -# Perform reasonable quick build with unit tests execution -BUILD_UT=1 -BUILD_ALL_USER_AGENTS=0 -BUILD_LOCALES=0 +# Maven memory needs to be limited +export MAVEN_OPTS="-Xms1G -Xmx2G" + +# Set the location of the JDK that will be used for compilation +export JAVA_HOME="${JAVA_HOME:=/usr/lib/jvm/java-11}" + +# Install build dependencies +if [[ $(id -u) -ne 0 ]]; then + sudo dnf builddep -y ${top_dir}/SRPMS/*src.rpm +else + dnf builddep -y ${top_dir}/SRPMS/*src.rpm +fi -# Build binary package +# Build binary package with the minimal build. GH RPM builds +# will be used only for OST so Firefox and Chrome are enough. rpmbuild \ - -D "_topmdir rpmbuild" \ - -D "_rpmdir rpmbuild" \ - ${RELEASE:+-D "release_suffix ${RELEASE}"} \ - -D "ovirt_build_ut ${BUILD_UT}" \ - -D "ovirt_build_all_user_agents ${BUILD_ALL_USER_AGENTS}" \ - -D "ovirt_build_locales ${BUILD_LOCALES}" \ + -D "_topdir ${top_dir}" \ + -D "release_suffix ${SNAPSHOT_SUFFIX}" \ -D "ovirt_build_extra_flags ${EXTRA_BUILD_FLAGS}" \ - --rebuild rpmbuild/SRPMS/*src.rpm + --with ovirt_build_minimal \ + --with ovirt_build_ut \ + --rebuild ${top_dir}/SRPMS/*src.rpm # Move RPMs to exported artifacts -[[ -d $ARTIFACTS_DIR ]] || mkdir -p $ARTIFACTS_DIR -find rpmbuild -iname \*rpm | xargs mv -t $ARTIFACTS_DIR +[[ -d ${ARTIFACTS_DIR} ]] || mkdir -p ${ARTIFACTS_DIR} +find ${top_dir} -iname \*rpm -print0 | xargs -0 mv -t ${ARTIFACTS_DIR} diff --git a/.automation/build-srpm.sh b/.automation/build-srpm.sh index 5578c7e152ab..13f08b133f54 100755 --- a/.automation/build-srpm.sh +++ b/.automation/build-srpm.sh @@ -1,45 +1,47 @@ #!/bin/bash -xe -# git hash of current commit should be passed as the 1st paraameter -if [ "${GITHUB_SHA}" == "" ]; then - GIT_HASH=$(git rev-list HEAD | wc -l) -else - GIT_HASH=$(git rev-parse --short $GITHUB_SHA) -fi - -# Directory, where build artifacts will be stored, should be passed as the 1st parameter -ARTIFACTS_DIR=${1:-exported-artifacts} - -# Prepare the version string (with support for SNAPSHOT versioning) -VERSION=$(mvn help:evaluate -q -DforceStdout -Dexpression=project.version) -VERSION=${VERSION/-SNAPSHOT/-0.${GIT_HASH}.$(date +%04Y%02m%02d%02H%02M)} -IFS='-' read -ra VERSION <<< "$VERSION" -RELEASE=${VERSION[1]-1} -MILESTONE=master - -# GH RPM builds will be used only for OST so Firefox and Chrome are enough -# GWT build memory needs to be limited -EXTRA_BUILD_FLAGS="" -EXTRA_BUILD_FLAGS="${EXTRA_BUILD_FLAGS} --no-transfer-progress" -EXTRA_BUILD_FLAGS="${EXTRA_BUILD_FLAGS} -Dgwt.userAgent=gecko1_8,safari" -EXTRA_BUILD_FLAGS="${EXTRA_BUILD_FLAGS} -Dgwt.compiler.localWorkers=1" -EXTRA_BUILD_FLAGS="${EXTRA_BUILD_FLAGS} -Dgwt.jvmArgs='-Xms1G -Xmx3G'" +# Building srpm for copr? +copr_build=0 + +# Process options +while getopts ":c" options; do + case $options in + c) copr_build=1;; + \?) + echo "Error: Invalid option" + exit;; + esac +done + +# Ensure the build validation passes +make validations -export MAVEN_OPTS="-Xms1G -Xmx2G" +# git hash of the HEAD commit (GH action may be the PR merge commit) +GIT_HASH=$(git rev-parse --short HEAD) -# Set the location of the JDK that will be used for compilation: -export JAVA_HOME="${JAVA_HOME:=/usr/lib/jvm/java-11}" +# Prepare the SNAPSHOT release suffix if a MILESTONE is available +SNAPSHOT_SUFFIX= +if [[ "$(make -f version.mak print-MILESTONE)" != "MILESTONE=" ]]; then + SNAPSHOT_SUFFIX=".git${GIT_HASH}" +fi +export SNAPSHOT_SUFFIX -[ -d ${ARTIFACTS_DIR} ] || mkdir -p ${ARTIFACTS_DIR} -[ -d rpmbuild/SOURCES ] || mkdir -p rpmbuild/SOURCES +# For a copr snapshot build, burn in the release_suffix as -D options are not preserved when rebuilding from srpm +if [[ ${copr_build} -eq 1 && "${SNAPSHOT_SUFFIX}" != "" ]]; then + sed "s:%{?release_suffix}:${SNAPSHOT_SUFFIX}:" -i ovirt-engine.spec.in +fi -make validations +# Create RPM build directories +export top_dir="${PWD}/rpmbuild" +test -d "${top_dir}" && rm -rf "${top_dir}" || : +mkdir -p "${top_dir}/SOURCES" # Get the tarball make dist -mv *.tar.gz rpmbuild/SOURCES +mv *.tar.gz ${top_dir}/SOURCES -# create the src.rpm +# Create the src.rpm rpmbuild \ - -D "_topdir rpmbuild" \ - -ts rpmbuild/SOURCES/*.gz + -D "_topdir ${top_dir}" \ + ${SNAPSHOT_SUFFIX:+-D "release_suffix ${SNAPSHOT_SUFFIX}"} \ + -ts ${top_dir}/SOURCES/*.gz diff --git a/.copr/Makefile b/.copr/Makefile index d5395725b1a6..6f2f92fd00ed 100644 --- a/.copr/Makefile +++ b/.copr/Makefile @@ -9,12 +9,5 @@ git_cfg_safe: git config --global --add safe.directory "$(shell pwd)" srpm: installdeps git_cfg_safe - $(eval SUFFIX=.git$(shell git rev-parse --short HEAD)) - # changing the spec file as passing -D won't preserve the suffix when rebuilding in mock - sed "s:%{?release_suffix}:${SUFFIX}:" -i ovirt-engine.spec.in - mkdir -p tmp.repos/SOURCES - make dist - rpmbuild \ - -D "_topdir tmp.repos" \ - -ts ./*.tar.gz - cp tmp.repos/SRPMS/*.src.rpm $(outdir) + ./.automation/build-srpm.sh -c + cp rpmbuild/SRPMS/*.src.rpm $(outdir) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index a7f5bb9dc363..503cf0bec220 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -92,7 +92,7 @@ jobs: - name: Perform build run: | - .automation/build-rpm.sh $ARTIFACTS_DIR + .automation/build-rpm.sh -a $ARTIFACTS_DIR - name: Create DNF repository run: | diff --git a/Makefile b/Makefile index 864969635fc1..30639d09c648 100644 --- a/Makefile +++ b/Makefile @@ -371,7 +371,7 @@ validations: generated-files fi; \ fi if [ "$(BUILD_VALIDATION)" != 0 ]; then \ - echo "Performing build validation." &&\ + echo "Performing build validation." &&\ build/shell-check.sh && \ build/python-check.sh && \ build/ansible-check.sh && \ diff --git a/README.adoc b/README.adoc index 7b6122c23944..cf30ae310422 100644 --- a/README.adoc +++ b/README.adoc @@ -580,34 +580,56 @@ And install a drop-in configuration file to override engine developement setup: # yum-builddep @srpm@ # rpmbuild -tb @tarball@ -The following spec file variables are available for package customization: +The following spec file options / conditional builds (set with `--with` or `--without`) +are available for package customization: -ovirt_build_quick:: Quick build, best for syntax checks. Default is `0`. +ovirt_build_quick:: + Quick build, best for syntax checks. Default is `without`. -ovirt_build_minimal:: Build minimal Firefox only package. Default is -`0`. +ovirt_build_minimal:: + Build minimal user agent (Gecko/Firefox and WebKit/Chrome/Safari) and locale (English) + package. For snapshot builds, the default is `with`, otherwise default is `without`. -ovirt_build_user_agent:: When using quick or minimal build, build only -for this user agent. Default is `gecko1_8` (Firefox). To build for -Chrome use `safari`. +ovirt_build_full:: + Build all GWT components with all user agents and locales. Selected by default when + `ovirt_build_quick` or `ovirt_build_minimal` is not explicitly selected. -ovirt_build_gwt:: Build GWT components. Default is `1`. +ovirt_build_gwt:: + Build GWT components. Default is `with`, but is `without` when the `ovirt_build_quick` + option is enalbed. -ovirt_build_all_user_agents:: Build GWT components for all supported -browsers. Default is `1`. +ovirt_build_all_user_agents:: + Build GWT components for all supported browsers. Default is `with`, but is `without` + when the `ovirt_build_minimal` option is enabled. -ovirt_build_locales:: Build GWT components for all supported locales. -Default is `1`. +ovirt_build_all_locales:: + Build GWT components for all supported locales. Default is `with`, but is `without` + when the `ovirt_build_minimal` option is enabled. + +ovirt_build_ut:: + Run unit tests with the build. Default is `with` when `ovirt_build_full` is selected, + otherwise defaults to `without`. + +The following spec file variables (set with `--define`) are available for package customization: + +ovirt_build_user_agent:: + When using quick or minimal build, build only for this user agent. Default is + `gecko1_8,safari` (Firefox and Chrome/Safari). + +ovirt_build_locales:: + When using quick or minimal build, build only for this set of locales. Default is + `en_US`. To build for additional locales, use a comma separated list. For example, + setting to `en_US,ja_JP` will build for English and Japanese. -Examples: +==== Examples -Build minimal rpm package for Firefox:: +Build minimal rpm package for Firefox and Chrome/Safari:: - $ rpmbuild -D"ovirt_build_minimal 1" -tb @tarball@ + $ rpmbuild --with ovirt_build_minimal -tb @tarball@ -Build minimal rpm package for Chrome or Safari:: +Build minimal rpm package for Chrome/Safari only:: - $ rpmbuild -D"ovirt_build_minimal 1" -D"ovirt_build_user_agent safari" -tb @tarball@ + $ rpmbuild --with ovirt_build_minimal --define "ovirt_build_user_agent safari" -tb @tarball@ === Ansible Lint diff --git a/bump_release.sh b/bump_release.sh index 6181d27a7565..e817fb7fabb6 100755 --- a/bump_release.sh +++ b/bump_release.sh @@ -18,25 +18,7 @@ export CHANGELOG # Add changelog to the spec file sed -i "/^%changelog/a ${CHANGELOG}" ovirt-engine.spec.in -# Adjust copr build config for releasing -patch -p0 --ignore-whitespace .copr/Makefile <<'__EOF__' -diff --git a/.copr/Makefile b/.copr/Makefile -index 51e3299e3e5..b2d4a195740 100644 ---- a/.copr/Makefile -+++ b/.copr/Makefile -@@ -9,9 +9,9 @@ git_cfg_safe: - git config --global --add safe.directory "$(shell pwd)" - - srpm: installdeps git_cfg_safe -- $(eval SUFFIX=.git$(shell git rev-parse --short HEAD)) -+ # $(eval SUFFIX=.git$(shell git rev-parse --short HEAD)) - # changing the spec file as passing -D won't preserve the suffix when rebuilding in mock -- sed "s:%{?release_suffix}:${SUFFIX}:" -i ovirt-engine.spec.in -+ # sed "s:%{?release_suffix}:${SUFFIX}:" -i ovirt-engine.spec.in - mkdir -p tmp.repos/SOURCES - make dist - rpmbuild \ -__EOF__ +# copr/build-srpm.sh will skip the snapshot suffix if `version.mak` does not select a MILESTONE # commit git add -u diff --git a/ovirt-engine.spec.in b/ovirt-engine.spec.in index 02f333602003..af1778258a77 100644 --- a/ovirt-engine.spec.in +++ b/ovirt-engine.spec.in @@ -1,47 +1,100 @@ # CUSTOMIZATION-BEGIN # -# ovirt_build_extra_flags -# ovirt_build_ut -%if "@MILESTONE@"!="" && 0%{!?ovirt_build_minimal:1} -%global ovirt_build_minimal 1 -%endif -%if 0%{?ovirt_build_user_agent:1} -%global _ovirt_build_user_agent %{ovirt_build_user_agent} +# Note: rpm 4.18 has a much better `bcond()` macro that makes the if/else blocks a single line +# `bcond_with abc` means "require `rpmbuild --with abc` to enable this option" (off by default) +# `bcond_without abc` means "require `rpmbuild --without abc` to disable this option" (on by default) +# +# == build types (group of flags) +# conditional build "quick" consists of the following build options: +# --without ovirt_build_gwt +# --without ovirt_build_ut +%bcond_with ovirt_build_quick + +# conditional build "draft" consists of the "minimal" options plus extra build flag '-Pgwtdraft': +# --with ovirt_build_gwt +# --without ovirt_build_all_user_agents +# --without ovirt_build_all_locales +# --without ovirt_build_ut +# --define ovirt_build_user_agent gecko1_8,safari +# --define ovirt_build_locales en_US +# EXTRA_BUILD_FLAGS=..." -P gwtdraft " +%bcond_with ovirt_build_draft + +# conditional build "minimal" consists of the following build options: +# --with ovirt_build_gwt +# --without ovirt_build_all_user_agents +# --without ovirt_build_all_locales +# --without ovirt_build_ut +# --define ovirt_build_user_agent gecko1_8,safari +# --define ovirt_build_locales en_US +# +# *** This is the default build if a MILESTONE (snapshot build) is set +%global MILESTONE "@MILESTONE@" +%if %{defined _with_ovirt_build_quick} || %{defined _with_ovirt_build_draft} || %{defined _with_ovirt_build_full} || %{MILESTONE}=="" +%bcond_with ovirt_build_minimal %else -%global _ovirt_build_user_agent gecko1_8,safari +%bcond_without ovirt_build_minimal %endif -%if 0%{?ovirt_build_quick} -%global _ovirt_build_ut 0 -%global ovirt_build_gwt 0 + +%global ovirt_build_minimal_user_agents gecko1_8,safari +%global ovirt_build_minimal_user_locales en_US + +# conditional build "full" performs unit tests, and builds GWT for all user agents and locales +# *** This is the default build if no other build type has been selected +%if %{with ovirt_build_quick} || %{with ovirt_build_draft} || %{with ovirt_build_minimal} +%bcond_with ovirt_build_full +%else +%bcond_without ovirt_build_full %endif -%if 0%{?ovirt_build_minimal} -%global _ovirt_build_ut 0 -%global ovirt_build_all_user_agents 0 -%global ovirt_build_locales 0 -%global _ovirt_build_extra_flags -D gwt.userAgent=%{_ovirt_build_user_agent} + +%{echo:build type: %{?with_ovirt_build_quick:quick}%{?with_ovirt_build_minimal:minimal}%{?with_ovirt_build_draft:draft}%{?with_ovirt_build_full:full}} + +# == build flags +# build GWT (webadmin), but NOT when "quick" +%if %{without ovirt_build_quick} +%bcond_without ovirt_build_gwt +%else +%bcond_with ovirt_build_gwt %endif -%if 0%{?ovirt_build_draft} -%global _ovirt_build_ut 0 -%global ovirt_build_all_user_agents 0 -%global ovirt_build_locales 0 -%global _ovirt_build_extra_flags -D gwt.userAgent=%{_ovirt_build_user_agent} -P gwtdraft + +# build all user agents, but only by default when "full" +%if %{with ovirt_build_full} +%bcond_without ovirt_build_all_user_agents +%else +%bcond_with ovirt_build_all_user_agents %endif -%if 0%{!?ovirt_build_ut:1} -%if 0%{?_ovirt_build_ut:1} -%global ovirt_build_ut %{_ovirt_build_ut} + +# build all locales, but only by default when "full" +%if %{with ovirt_build_full} +%bcond_without ovirt_build_all_locales %else -%global ovirt_build_ut 1 +%bcond_with ovirt_build_all_locales %endif + +# build unit tests, but only by default when "full" +%if %{with ovirt_build_full} +%bcond_without ovirt_build_ut +%else +%bcond_with ovirt_build_ut %endif -%if 0%{!?ovirt_build_gwt:1} -%global ovirt_build_gwt 1 + +# == convert the build flags to `EXTRA_BUILD_FLAGS` as needed +%if %{defined ovirt_build_user_agent} || %{with ovirt_build_minimal} || %{with ovirt_build_draft} +%global extra_build_flags_user_agents -Dgwt.userAgent=%{?ovirt_build_user_agent}%{!?ovirt_build_user_agent:%{?ovirt_build_minimal_user_agents}} %endif -%if 0%{!?ovirt_build_locales:1} -%global ovirt_build_locales 1 + +%if %{defined ovirt_build_locales} || %{with ovirt_build_minimal} || %{with ovirt_build_draft} +%global extra_build_flags_locales -Dgwt.locale=%{?ovirt_build_locales}%{!?ovirt_build_locales:%{?ovirt_build_minimal_user_locales}} %endif -%if 0%{!?ovirt_build_all_user_agents:1} -%global ovirt_build_all_user_agents 1 + +%if %{defined extra_build_flags_user_agents} || %{defined extra_build_flags_locales} || %{with ovirt_build_draft} || %{defined ovirt_build_extra_flags} +%global EXTRA_BUILD_FLAGS \ + %{?extra_build_flags_user_agents} \ + %{?extra_build_flags_locales} \ + %{?with_ovirt_build_draft:-P gwtdraft} \ + %{?ovirt_build_extra_flags} %endif + # define "rhv_build 1" to perform RHV Manager build #if True %global rhv_build 1 @@ -50,7 +103,6 @@ #end if # # CUSTOMIZATION-END -# #raw @@ -66,14 +118,6 @@ %global _jarsign_opts --unsign=/usr/share/ovirt-engine %endif -# -# rpm does not support override -# nor modify of variables -# -%if %{?_ovirt_build_extra_flags:1}%{?ovirt_build_extra_flags:1}0 -%global EXTRA_BUILD_FLAGS %{?_ovirt_build_extra_flags:%{_ovirt_build_extra_flags}}%{?ovirt_build_extra_flags: %{ovirt_build_extra_flags}} -%endif - # # Supported distributions: # Fedora >= 30 @@ -160,10 +204,10 @@ getent passwd %1 >/dev/null || useradd -r -u %2 -g %3 -c %5 -s /sbin/nologin -d %global make_common_opts \\\ -j1 \\\ - BUILD_GWT=%{ovirt_build_gwt} \\\ - BUILD_ALL_USER_AGENTS=%{ovirt_build_all_user_agents} \\\ - BUILD_LOCALES=%{ovirt_build_locales} \\\ - BUILD_UT=%{ovirt_build_ut} \\\ + %{!?with_ovirt_build_gwt:BUILD_GWT=0} \\\ + %{?with_ovirt_build_all_user_agents:BUILD_ALL_USER_AGENTS=1} \\\ + %{?with_ovirt_build_all_locales:BUILD_LOCALES=1} \\\ + %{!?with_ovirt_build_ut:BUILD_UT=0} \\\ BUILD_VALIDATION=0 \\\ PACKAGE_NAME=%{name} \\\ RPM_VERSION=%{version} \\\ @@ -198,7 +242,7 @@ getent passwd %1 >/dev/null || useradd -r -u %2 -g %3 -c %5 -s /sbin/nologin -d PKG_GROUP=%{engine_group} \\\ POSTGRESQL_SYSTEMD_SERVICE=postgresql.service \\\ WILDFLY_OVERLAY_MODULES=%{wildfly_overlay_modules} \\\ - %{?EXTRA_BUILD_FLAGS:EXTRA_BUILD_FLAGS="%{EXTRA_BUILD_FLAGS}"} \\\ + %{?EXTRA_BUILD_FLAGS:EXTRA_BUILD_FLAGS="%{shrink:%{EXTRA_BUILD_FLAGS}}"} \\\ %{?extra_common_opts} %if %{rhv_build} diff --git a/version.mak b/version.mak index efc95808a142..607c056b4778 100644 --- a/version.mak +++ b/version.mak @@ -26,6 +26,7 @@ AUTO_MILESTONE=$(shell cat pom.xml | head -n 20 | grep '' | head -n 1 | # AUTO_RPM_RELEASE is set to 0.0.something on SNAPSHOT builds, and to RPM_RELEASE_ON_RELEASE otherwise. AUTO_RPM_RELEASE=$(shell if cat pom.xml | head -n 20 | grep '' | head -n 1 | sed -e 's/.*>\(.*\)<.*/\1/' | grep -q 'SNAPSHOT$$'; then echo 0.2.$(MILESTONE).$$(date -u +%Y%m%d%H%M%S); else echo $(RPM_RELEASE_ON_RELEASE); fi) +# MILESTONE should be automatic. MILESTONE=$(AUTO_MILESTONE) # RPM release should be automatic. If needed to be set manually: @@ -48,3 +49,6 @@ RPM_RELEASE=$(AUTO_RPM_RELEASE) # Or leave empty to have only mead numbering. # DOWNSTREAM_RPM_RELEASE_PREFIX=0. + +# Helper target to print the value of a Makefile variable (`make print-MILESTONE`) +print-%: ; @echo $*=$($*)