From 89e6381bd5c330bb2899fba1edf58216105810a7 Mon Sep 17 00:00:00 2001 From: Rich Persaud Date: Fri, 24 Jan 2020 18:29:11 -0500 Subject: [PATCH 1/9] do_build.sh: test for bash as system shell If /bin/sh is not symlinked to bash, some packages in the OpenXT build will fail with non-obvious error messages. Before starting the build, confirm that the default non-interactive shell is bash. In Debian/Devuan, the default is dash. Future improvement: identify and patch each offending package to explicitly require bash in shell scripts. This would avoid imposing the overhead of bash on all build scripts, at the cost of OpenXT carrying patches until they could be upstreamed. OXT-1736 Signed-off-by: Rich Persaud --- do_build.sh | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/do_build.sh b/do_build.sh index 1ba09c55b..769628954 100755 --- a/do_build.sh +++ b/do_build.sh @@ -16,6 +16,11 @@ STEPS="setupoe,initramfs,stubinitramfs,dom0,uivm,ndvm,syncvm,installer,installer # Packages tree can use hardlinks to save disk space, if $SYNC_CACHE_OE/oe-archives is populated # Requires a valid NETBOOT_HTTP_URL in .config +# OpenXT build requires system shell to be bash +SYSTEM_SHELL=$(basename $(readlink -f /bin/sh)) +[[ ${SYSTEM_SHELL} == "bash" ]] || \ + { echo "OpenXT build requires bash as system shell. Please symlink /bin/sh to bash"; exit 1; } + TOPDIR=`pwd` OUTPUT_DIR="$TOPDIR/build-output" CMD="$0" From 365a728932f5a005936dc5d62cdebcf69e36906e Mon Sep 17 00:00:00 2001 From: Rich Persaud Date: Fri, 24 Jan 2020 18:29:11 -0500 Subject: [PATCH 2/9] do_build.sh: keep bitbake memory-resident Since bitbake is called multiple times by do_build, keeping it memory resident for 30 seconds will reduce recipe parsing. OXT-1736 Signed-off-by: Rich Persaud --- do_build.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/do_build.sh b/do_build.sh index 769628954..8f4fa0777 100755 --- a/do_build.sh +++ b/do_build.sh @@ -21,6 +21,9 @@ SYSTEM_SHELL=$(basename $(readlink -f /bin/sh)) [[ ${SYSTEM_SHELL} == "bash" ]] || \ { echo "OpenXT build requires bash as system shell. Please symlink /bin/sh to bash"; exit 1; } +# Keep bitbake memory-resident between invocations +export BB_SERVER_TIMEOUT=30 + TOPDIR=`pwd` OUTPUT_DIR="$TOPDIR/build-output" CMD="$0" From 57712d6933f145ef353b60c50852b407417ae3c9 Mon Sep 17 00:00:00 2001 From: Rich Persaud Date: Fri, 24 Jan 2020 18:29:11 -0500 Subject: [PATCH 3/9] do_build.sh: enable optional interactive build For faster interactive use, retain bitbake interactive status display instead of piping output to cumulative build log. Skip release-oriented steps such as source ISO generation generation and log compression. OXT-1736 Signed-off-by: Rich Persaud --- do_build.sh | 24 +++++++++++++++++------- 1 file changed, 17 insertions(+), 7 deletions(-) diff --git a/do_build.sh b/do_build.sh index 8f4fa0777..63c001fe0 100755 --- a/do_build.sh +++ b/do_build.sh @@ -24,6 +24,9 @@ SYSTEM_SHELL=$(basename $(readlink -f /bin/sh)) # Keep bitbake memory-resident between invocations export BB_SERVER_TIMEOUT=30 +# Optional interactive use: set to 1 to disable logs +INTERACTIVE=0 + TOPDIR=`pwd` OUTPUT_DIR="$TOPDIR/build-output" CMD="$0" @@ -235,7 +238,11 @@ do_oe() fi echo "STARTING OE BUILD $image $machine, started at" `date -u +'%H:%M:%S UTC'` - < /dev/null ./bb $BBFLAGS "$image" | do_oe_log + if [ $INTERACTIVE -eq 1 ]; then + ./bb $BBFLAGS "$image" + else + < /dev/null ./bb $BBFLAGS "$image" | do_oe_log + fi popd } @@ -1289,12 +1296,15 @@ do_ship() do_updates do_netboots do_installer_isos - do_source_iso - do_source_info - do_licences - do_syncui - do_info - do_logs + + if [ $INTERACTIVE -eq 0 ]; then + do_source_iso + do_source_info + do_licences + do_syncui + do_info + do_logs + fi } do_copy() From 55a5221f0706a710a8e81b4e499f45f74101d122 Mon Sep 17 00:00:00 2001 From: Rich Persaud Date: Fri, 24 Jan 2020 18:29:11 -0500 Subject: [PATCH 4/9] do_build.sh: remove obsolete steps: syncui, nilfvm Remove logic for syncui and nilf/vpn vm, which are no longer built. OXT-1736 Signed-off-by: Rich Persaud --- do_build.sh | 75 +---------------------------------------------------- 1 file changed, 1 insertion(+), 74 deletions(-) diff --git a/do_build.sh b/do_build.sh index 63c001fe0..4d09e1032 100755 --- a/do_build.sh +++ b/do_build.sh @@ -1,7 +1,7 @@ #! /bin/bash -e set -o pipefail -STEPS="setupoe,initramfs,stubinitramfs,dom0,uivm,ndvm,syncvm,installer,installer2,syncui,source,sdk,license,sourceinfo,ship" +STEPS="setupoe,initramfs,stubinitramfs,dom0,uivm,ndvm,syncvm,installer,installer2,source,sdk,license,sourceinfo,ship" # Additional steps: @@ -315,28 +315,6 @@ do_oe_ndvm() do_oe_ndvm_copy $path } -do_oe_nilfvm_copy() -{ - local path="$1" - do_oe_copy "$path" "nilfvm" "xenclient-nilfvm" "xenclient-nilfvm" - - local binaries="tmp-glibc/deploy/images" - pushd "$path" - cp "$binaries/service-nilfvm" "$OUTPUT_DIR/$NAME/raw/service-nilfvm" - popd -} - -do_oe_nilfvm() -{ - local path="$1" - - echo This step is now useless, everything we need should be built as part of the tools. - return - - do_oe "$path" "xenclient-nilfvm" "xenclient-nilfvm-image" - do_oe_nilfvm_copy $path -} - do_oe_syncvm_copy() { local path="$1" @@ -350,23 +328,6 @@ do_oe_syncvm() do_oe_syncvm_copy $path } -do_oe_syncui_copy() -{ - local path="$1" - pushd "$path" - mkdir -p "$OUTPUT_DIR/$NAME/raw" - cp tmp-glibc/deploy/tar/sync-wui-0+git*.tar.gz "$OUTPUT_DIR/$NAME/raw/sync-wui-${RELEASE}.tar.gz" - cp tmp-glibc/deploy/tar/sync-wui-sources-0+git*.tar.gz "$OUTPUT_DIR/$NAME/raw/sync-wui-sources-${RELEASE}.tar.gz" - popd -} - -do_oe_syncui() -{ - local path="$1" - do_oe "$path" "xenclient-syncui" "sync-wui" - do_oe_syncui_copy "$path" -} - do_oe_dom0_copy() { local path="$1" @@ -1184,10 +1145,6 @@ do_xctools_debian_repo() local dest_dir="${path}/tmp-glibc/deb-xctools-image/" local d_output_dir="${OUTPUT_DIR}/${NAME}/xctools-debian-repo/debian" - echo "Building Debian Service VM tools" - do_oe "${path}" "xenclient-nilfvm" "linux-xenclient-nilfvm" - do_oe "${path}" "xenclient-nilfvm" "deb-servicevm-tools" - echo "Building XC Tools Debian/Ubuntu repository" do_oe "${path}" "xenclient-dom0" "deb-xctools-image" @@ -1245,26 +1202,6 @@ do_xctools() { do_xctools_linux "$1" } -do_syncui() -{ - local name="sync-wui-${RELEASE}.tar.gz" - local file="$OUTPUT_DIR/$NAME/raw/$name" - local out="$OUTPUT_DIR/$NAME/sync" - - if [ ! -r "${file}" ]; then - echo "syncui: Not built, skipping" - return 0 - fi - - echo "syncui:" - echo " - copy $name" - - mkdir -p "$out" - cp "$file" "$out" - - echo -} - do_info() { local out="$OUTPUT_DIR/$NAME" @@ -1301,7 +1238,6 @@ do_ship() do_source_iso do_source_info do_licences - do_syncui do_info do_logs fi @@ -1399,15 +1335,8 @@ do_build() do_oe_uivm "$path" ;; ndvm) do_oe_ndvm "$path" ;; - nilfvm) - do_oe_nilfvm "$path" ;; - vpnvm) - # for retro-compatibility - do_oe_nilfvm "$path" ;; syncvm) do_oe_syncvm "$path" ;; - syncui) - do_oe_syncui "$path" ;; uivmcp) do_oe_uivm_copy "$path" ;; ndvmcp) @@ -1416,8 +1345,6 @@ do_build() do_oe_vpnvm_copy "$path" ;; syncvmcp) do_oe_syncvm_copy "$path" ;; - syncuicp) - do_oe_syncui_copy "$path" ;; xctools*) do_xctools "$path/xctools" ;; debian) From e9c80ebe73f7e5a94bf62c7f89894f7f5f685a6e Mon Sep 17 00:00:00 2001 From: Rich Persaud Date: Fri, 24 Jan 2020 18:29:11 -0500 Subject: [PATCH 5/9] do_build.sh: for PXE builds, symlink latest output Create a symlink (build-output/_do_build_latest) to the most recent build output directory. This can be used to specify the location of an HTTP package repository for PXE installs. OXT-1736 Signed-off-by: Rich Persaud --- do_build.sh | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/do_build.sh b/do_build.sh index 4d09e1032..91d16a033 100755 --- a/do_build.sh +++ b/do_build.sh @@ -1241,6 +1241,12 @@ do_ship() do_info do_logs fi + + pushd $OUTPUT_DIR + SYMLINK_OUTPUT="_do_build_latest" + rm -f $SYMLINK_OUTPUT + ln -s $NAME $SYMLINK_OUTPUT + popd } do_copy() From 5a617abd3f7d3fea7c9dc8e875a2f4d548575392 Mon Sep 17 00:00:00 2001 From: Rich Persaud Date: Fri, 24 Jan 2020 18:29:11 -0500 Subject: [PATCH 6/9] do_build.sh: regenerate local.conf for each build local.conf contains version information that is embedded in the OpenXT installer. It must be generated for each build. Define filename as variable. OXT-1736 Signed-off-by: Rich Persaud --- do_build.sh | 28 ++++++++++++++++------------ 1 file changed, 16 insertions(+), 12 deletions(-) diff --git a/do_build.sh b/do_build.sh index 91d16a033..fd8036c1d 100755 --- a/do_build.sh +++ b/do_build.sh @@ -41,6 +41,8 @@ BRANCH=master BUILD_UID=`id -u` export BUILD_UID +CONF_LOCAL="conf/local.conf" + # TODO: move some of the above definitions into common-config if [ -f ${CMD_DIR}/common-config ]; then @@ -100,16 +102,18 @@ do_oe_setup() mkdir -p conf fi - if [ ! -f "conf/local.conf" -o "conf/local.conf" -ot "conf/local.conf-dist" ]; then - cp conf/local.conf-dist conf/local.conf + # Force generation of local.conf with latest build version + rm -f ${CONF_LOCAL} + if [ ! -f "${CONF_LOCAL}" -o "${CONF_LOCAL}" -ot "${CONF_LOCAL}-dist" ]; then + cp "${CONF_LOCAL}-dist" "${CONF_LOCAL}" if [ ! -z "${OE_TARBALL_MIRROR}" ] ; then - cat >> conf/local.conf <> ${CONF_LOCAL} <> conf/local.conf <> ${CONF_LOCAL} <> conf/local.conf + echo "XENCLIENT_BUILD = \"$ID\"" >> ${CONF_LOCAL} else - echo "XENCLIENT_BUILD = \"$NAME\"" >> conf/local.conf + echo "XENCLIENT_BUILD = \"$NAME\"" >> ${CONF_LOCAL} fi - cat >> conf/local.conf <> ${CONF_LOCAL} <> conf/local.conf <> ${CONF_LOCAL} <> conf/local.conf <> ${CONF_LOCAL} <> conf/local.conf <> ${CONF_LOCAL} <> conf/local.conf <> ${CONF_LOCAL} < Date: Fri, 24 Jan 2020 18:29:11 -0500 Subject: [PATCH 7/9] example-config: uprev to OE zeus, bitbake 1.44 Other OpenXT repos have been moved to OE zeus, do the same here. OXT-1736 Signed-off-by: Rich Persaud --- example-config | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/example-config b/example-config index c90e45899..5eb93ce88 100644 --- a/example-config +++ b/example-config @@ -2,7 +2,7 @@ BRANCH="master" # OpenEmbedded branch to build. -OE_BRANCH="pyro" +OE_BRANCH="zeus" # OpenXT git repositories. Optionally replace with a local mirror. OPENXT_GIT_MIRROR="github.com/OpenXT" @@ -12,7 +12,7 @@ OPENXT_GIT_PROTOCOL="git" OE_GIT_MIRROR= # Bitbake default version to checkout -BB_BRANCH="1.34" +BB_BRANCH="1.44" # Bitbake tag to override checkout #BITBAKE_TAG="1.34" From c53f8c07a50b15abb654693f6a50c2a91c9c1a55 Mon Sep 17 00:00:00 2001 From: Rich Persaud Date: Fri, 24 Jan 2020 18:29:11 -0500 Subject: [PATCH 8/9] local-conf: enable bitbake parallel builds On build machines with multiple cores, bitbake can execute build tasks in parallel. These defaults should be modified to match the number of cores on the build machine. OXT-1736 Signed-off-by: Rich Persaud --- build/conf/local.conf-dist | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/build/conf/local.conf-dist b/build/conf/local.conf-dist index fbbb139ef..1f4e99c77 100644 --- a/build/conf/local.conf-dist +++ b/build/conf/local.conf-dist @@ -1,5 +1,9 @@ CONF_VERSION = "1" +# Enable parallel builds on multi-core machines +BB_NUMBER_THREADS ?= "4" +PARALLEL_MAKE ?= "-j 4" + # 1) Basic config # ## TODO: DISTRO should be in auto.conf From 6436d4e7bb870c3267d4441681d79b9938517dd2 Mon Sep 17 00:00:00 2001 From: Rich Persaud Date: Fri, 24 Jan 2020 18:29:11 -0500 Subject: [PATCH 9/9] do_build.sh: initialize variables Initialize some undefined variables identified by set -u. Future work: document original feature purpose, current relevance and whether to remove or test/fix. OXT-1736 Signed-off-by: Rich Persaud --- do_build.sh | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/do_build.sh b/do_build.sh index fd8036c1d..1c02fc9cd 100755 --- a/do_build.sh +++ b/do_build.sh @@ -41,6 +41,15 @@ BRANCH=master BUILD_UID=`id -u` export BUILD_UID +DEBUG="" +CONFIG="" +XC_TOOLS_BUILD="" +ARGNAME="" +OE_BUILD_CACHE_DL="" +OE_TARBALL_MIRROR="" +NETBOOT_HTTP_URL="" +FREEZE_URIS="" + CONF_LOCAL="conf/local.conf" # TODO: move some of the above definitions into common-config