Merge pull request vmware#506 from sshedi/dockerfile-fixes

CI/CD fixes

.github/workflows/ci.yml

@@ -16,7 +16,7 @@ jobs:
             - name: run container with tests
               working-directory: ${{ github.workspace }}
               run: |
-                docker run --security-opt seccomp:unconfined --rm -e DIST -v$(pwd):/build -w/build ${DIST}/tdnf-build ./ci/docker-entrypoint.sh
+                docker run --privileged --security-opt seccomp:unconfined --rm -e DIST -v$(pwd):/build -w/build ${DIST}/tdnf-build ./ci/docker-entrypoint.sh
     fedora:
         env:
             DIST: fedora

ci/Dockerfile.fedora

@@ -1,15 +1,7 @@
-FROM fedora:34
+FROM fedora:latest
 
-RUN dnf -y upgrade
-RUN dnf -y install gcc make cmake libcurl-devel rpm-devel rpm-build \
-        libsolv-devel popt-devel sed createrepo_c glib2-devel expat \
-        findutils python3-pytest python3-requests python3-urllib3 \
-        python3-pyOpenSSL python3 python3-devel valgrind gpgme-devel \
-        expat-devel openssl-devel sqlite-devel rpm-sign which python3-pip \
-        shadow-utils sudo e2fsprogs util-linux
-
-RUN pip3 install flake8
-
-RUN mkdir -p /var/lib/tdnf
+COPY ci/prep.sh /prep.sh
+RUN chmod +x /prep.sh
+RUN /prep.sh
 
 CMD ["/bin/bash"]

ci/Dockerfile.photon

@@ -1,19 +1,7 @@
 FROM photon:latest
 
-RUN tdnf update  -y
-RUN tdnf remove  -y toybox
-RUN tdnf install -y --enablerepo=photon-debuginfo \
-               build-essential cmake curl-devel rpm-build \
-               libsolv-devel popt-devel sed createrepo_c glib expat-libs \
-               findutils python3 python3-pip python3-setuptools \
-               python3-devel valgrind gpgme-devel glibc-debuginfo \
-               expat-devel openssl-devel zlib-devel sqlite-devel \
-               python3-requests python3-urllib3 python3-pyOpenSSL \
-               sudo shadow which e2fsprogs util-linux
-
-# this can to be removed once latest pytest gets published in photon
-RUN pip3 install pytest flake8
-
-RUN mkdir -p /var/lib/tdnf
+COPY ci/prep.sh /prep.sh
+RUN chmod +x /prep.sh
+RUN /prep.sh
 
 CMD ["/bin/bash"]

ci/docker-entrypoint.sh

@@ -1,14 +1,23 @@
 #!/bin/bash
+
 set -e
 
 rm -rf build
 mkdir -p build
 cd build || exit 1
 
-mkdir -p /usr/lib/sysimage/tdnf
-cmake -DHISTORY_DB_DIR=/usr/lib/sysimage/tdnf .. && make -j32 && make python -j32 && make check -j32 || exit
+JOBS=$(( ($(nproc)+1) / 2 ))
+HIST_DB_DIR="/usr/lib/sysimage/tdnf"
+
+{
+  mkdir -p ${HIST_DB_DIR}
+  cmake -DHISTORY_DB_DIR=${HIST_DB_DIR} ..
+  make -j${JOBS}
+  make python -j${JOBS}
+  make check -j${JOBS}
+} || exit 1
 
 if ! flake8 ../pytests ; then
-  echo "flake8 tests failed"
+  echo "ERROR: flake8 tests failed" >&2
   exit 1
 fi

ci/prep.sh

@@ -0,0 +1,63 @@
+#!/bin/bash
+
+set -ex
+
+rel_file="/etc/os-release"
+
+common_pkgs=(
+  cmake
+  createrepo_c
+  e2fsprogs
+  expat-devel
+  findutils
+  gpgme-devel
+  libsolv-devel
+  openssl-devel
+  popt-devel
+  python3-devel
+  python3-pip
+  python3-pyOpenSSL
+  python3-pytest
+  python3-requests
+  python3-setuptools
+  python3-urllib3
+  rpm-build
+  sed
+  sqlite-devel
+  sudo
+  util-linux
+  valgrind
+  which
+)
+
+if grep -qw "Fedora" ${rel_file}; then
+  fedora_packages=(
+    ${common_pkgs[@]}
+    gcc
+    glib2-devel
+    libcurl-devel
+    make
+    python3-flake8
+    rpm-devel
+    rpm-sign
+    shadow-utils
+  )
+  dnf -y upgrade --refresh
+  dnf -y install ${fedora_packages[@]}
+elif grep -qw "Photon" ${rel_file}; then
+  photon_packages=(
+    ${common_pkgs[@]}
+    build-essential
+    curl-devel
+    glib
+    glibc-debuginfo
+    shadow
+    zlib-devel
+  )
+  tdnf -y upgrade --refresh
+  tdnf remove -y toybox
+  tdnf -y install --enablerepo=photon-debuginfo ${photon_packages[@]}
+  pip3 install flake8
+fi
+
+mkdir -p /var/lib/tdnf

pytests/conftest.py

@@ -193,6 +193,14 @@ def assert_file_exists(self, file_path):
             raise FileNotFoundError(errno.ENOENT, os.strerror(errno.ENOENT),
                                     file_path)
 
+    def clear_directory(self, dir_path):
+        for item in os.listdir(dir_path):
+            item_path = os.path.join(dir_path, item)
+            if os.path.isfile(item_path) or os.path.islink(item_path):
+                os.unlink(item_path)
+            elif os.path.isdir(item_path):
+                shutil.rmtree(item_path)
+
     def check_package(self, package, version=None):
         ''' Check if a package exists '''
         ret = self.run(["tdnf", "--disablerepo=*", "list", "-j", "--installed", package])

pytests/mount-small-cache.in

@@ -1,8 +1,7 @@
-#!/usr/bin/env bash
+#!/bin/bash
 
 ## file:    mount-small-cache.sh
 ## brief:   mounts an intentionally small directory for out-of-diskspace tests.
-## author:  Preston Narchetti <[email protected]>
 
 if [ ${EUID} -ne 0 ]; then
   echo "Script must be run as root." 1>&2
@@ -13,29 +12,34 @@ fi
 small_cache_path="@CMAKE_CURRENT_BINARY_DIR@/small_cache/"
 quota_size="1M"
 
-## used to check return code for each command.
-function check_err {
-  rc=$?
+check_err() {
+  local rc=$?
   if [ $rc -ne 0 ]; then
-      echo $1
-      exit $rc
+    echo "$1" 1>&2
+    exit $rc
   fi
 }
 
-echo "Creating mount point."
-rm -rf ./virtual_disks
-mkdir -p ./virtual_disks
-rm -rf ${small_cache_path}
-mkdir -p ${small_cache_path}
-check_err "Failed to mkdir ${small_cache_path}."
-
-echo "Creating tmpfs."
-touch ./virtual_disks/fs.ext4
-dd if=/dev/zero of=./virtual_disks/fs.ext4 bs=${quota_size} count=1
-check_err "Failed to make exf4 tmpfs: dd"
-mkfs.ext4 ./virtual_disks/fs.ext4
-check_err "Failed to make exf4 tmpfs: mkfs.ext4"
-
-echo "Mounting tmpfs at ${small_cache_path}."
-mount -o loop,rw,usrquota,grpquota ./virtual_disks/fs.ext4 ${small_cache_path}
-check_err "Failed to mount exf4 tmpfs."
+vdisk="virtual_disks"
+ext4_fn="${vdisk}/fs.ext4"
+
+echo "Creating mount point ..."
+if mountpoint ${small_cache_path} &> /dev/null; then
+  umount ${small_cache_path}
+  check_err "ERROR: umount ${small_cache_path}"
+  sync
+fi
+rm -rf ${vdisk} ${small_cache_path}
+mkdir -p ${vdisk} ${small_cache_path}
+check_err "ERROR: failed to mkdir ${vdisk} ${small_cache_path}"
+
+echo "Creating tmpfs ..."
+dd if=/dev/zero of=${ext4_fn} bs=${quota_size} count=1
+check_err "ERROR: dd failed"
+
+mkfs.ext4 ${ext4_fn}
+check_err "ERROR: mkfs.ext4 failed"
+
+echo "Mounting tmpfs at ${small_cache_path} ..."
+mount -o loop,rw,usrquota,grpquota ${ext4_fn} ${small_cache_path}
+check_err "ERROR: failed to mount exf4 tmpfs"

pytests/repo/setup-repo.sh

@@ -8,22 +8,21 @@
 #
 
 if [ $# -ne 2 ]; then
-    echo "Usage: $0 <repo_path> <specs_dir>"
+    echo "Usage: $0 <repo_path> <specs_dir>" >&2
     exit 1
 fi
 
-function fix_dir_perms()
-{
+function fix_dir_perms() {
   chmod 755 ${TEST_REPO_DIR}
   find ${TEST_REPO_DIR} -type d -exec chmod 0755 {} \;
   find ${TEST_REPO_DIR} -type f -exec chmod 0644 {} \;
 }
 
 ## used to check return code for each command.
-function check_err {
-  rc=$?
+function check_err() {
+  local rc=$?
   if [ $rc -ne 0 ]; then
-      echo $1
+      echo "$1" >&2
       exit $rc
   fi
 }
@@ -37,7 +36,7 @@ fi
 
 REPO_SRC_DIR=$2
 if [ ! -d ${REPO_SRC_DIR} ]; then
-    echo "specs dir does not exist"
+    echo "ERROR: specs dir does not exist" >&2
     exit 1
 fi
 
@@ -58,21 +57,21 @@ mkdir -p -m 755 ${BUILD_PATH}/BUILD \
     ${TEST_REPO_DIR}/yum.repos.d \
     ${PUBLISH_PATH} \
     ${PUBLISH_SRC_PATH} \
-   ${PUBLISH_SHA512_PATH} \
+    ${PUBLISH_SHA512_PATH} \
     ${GNUPGHOME}
 
 #gpgkey data for unattended key generation
 cat << EOF > ${TEST_REPO_DIR}/gpgkeydata
-     %echo Generating a key for repogpgcheck signatures
-     %no-protection
-     Key-Type: default
-     Subkey-Type: default
-     Name-Real: tdnf test
-     Name-Comment: tdnf test key
-     Name-Email: [email protected]
-     Expire-Date: 0
-     %commit
-     %echo done
+%echo Generating a key for repogpgcheck signatures
+%no-protection
+Key-Type: RSA
+Subkey-Type: RSA
+Name-Real: tdnf test
+Name-Comment: tdnf test key
+Name-Email: [email protected]
+Expire-Date: 0
+%commit
+%echo done
 EOF
 
 #generate a key non interactively. this is used in testing
@@ -90,12 +89,11 @@ for d in conflicts enhances obsoletes provides recommends requires suggests supp
     sed s/@@dep@@/$d/ < ${REPO_SRC_DIR}/tdnf-repoquery-deps.spec.in > ${BUILD_PATH}/SOURCES/tdnf-repoquery-$d.spec
 done
 
-echo building packages
+echo "Building packages"
 for spec in ${REPO_SRC_DIR}/*.spec ${BUILD_PATH}/SOURCES/*.spec ; do
-    echo "building ${spec}"
-    rpmbuild  --define "_topdir ${BUILD_PATH}" \
-        -r ${BUILD_PATH} -ba ${spec} 2>&1
-    check_err "failed to build ${spec}"
+    echo "Building ${spec}"
+    rpmbuild -D "_topdir ${BUILD_PATH}" -ba ${spec} 2>&1
+    check_err "ERROR: failed to build ${spec}"
 done
 rpmsign --addsign ${BUILD_PATH}/RPMS/*/*.rpm
 check_err "Failed to sign built packages."

pytests/repo/tdnf-bad-pre.spec

@@ -19,19 +19,12 @@ Part of tdnf test spec. Test bad install scripts.
 %build
 
 %install
-mkdir -p %_topdir/%buildroot/usr/bin
-cat << EOF >> %_topdir/%buildroot/usr/bin/bad-pre.sh
-#!/bin/sh
-# dummy script. Return false because we are bad.
-/bin/false
-EOF
 
 %pre
 # fail intentionally
-/bin/false
+%{_bindir}/false
 
 %files
-/usr/bin/bad-pre.sh
 
 %changelog
 *   Fri Apr 2 2021 Oliver Kurth <[email protected]> 1.0.0-1

pytests/repo/tdnf-conflict-file0.spec

@@ -16,9 +16,10 @@ Part of tdnf test spec. Two packages containing one identical file.
 %build
 
 %install
-mkdir -p %_topdir/%buildroot/usr/lib/conflict
+mkdir -p %{buildroot}/usr/lib/conflict
 # for a file conflict, cntents of the files need to differ
-echo file0 > %_topdir/%buildroot/usr/lib/conflict/conflicting-file
+echo file0 > %{buildroot}/usr/lib/conflict/conflicting-file
+
 %files
 /usr/lib/conflict/conflicting-file
 

pytests/repo/tdnf-conflict-file1.spec

@@ -16,8 +16,9 @@ Part of tdnf test spec. Two packages containing one identical file.
 %build
 
 %install
-mkdir -p %_topdir/%buildroot/usr/lib/conflict
-echo file > %_topdir/%buildroot/usr/lib/conflict/conflicting-file
+mkdir -p %{buildroot}/usr/lib/conflict
+echo file > %{buildroot}/usr/lib/conflict/conflicting-file
+
 %files
 /usr/lib/conflict/conflicting-file
 

pytests/repo/tdnf-missing-dep.spec

@@ -22,8 +22,8 @@ Part of tdnf test spec. Basic install/remove/upgrade test
 %build
 
 %install
-mkdir -p %_topdir/%buildroot/lib/systemd/system/
-cat << EOF >> %_topdir/%buildroot/lib/systemd/system/%name.service
+mkdir -p %{buildroot}/lib/systemd/system/
+cat << EOF >> %{buildroot}/lib/systemd/system/%name.service
 [Unit]
 Description=%name.service for whatprovides test.
 

pytests/repo/tdnf-multi1.spec

@@ -20,8 +20,8 @@ Part of tdnf test spec. Basic install/remove/upgrade test
 
 # files should not conflict
 %install
-mkdir -p %_topdir/%buildroot/usr/share/multiinstall
-touch %_topdir/%buildroot/usr/share/multiinstall-%{release}
+mkdir -p %{buildroot}/usr/share/multiinstall
+touch %{buildroot}/usr/share/multiinstall-%{release}
 
 %files
 /usr/share/multiinstall-%{release}