From 7bc814ddef659befe7000eca86bba23a8fe9efcd Mon Sep 17 00:00:00 2001 From: Jia Tan Date: Tue, 31 Oct 2023 16:51:07 +0800 Subject: [PATCH] CI: Temporarily add CI to v5.4 --- .github/workflows/ci.yml | 146 ++++++++++++++++ .github/workflows/windows-ci.yml | 125 ++++++++++++++ build-aux/ci_build.sh | 280 +++++++++++++++++++++++++++++++ 3 files changed, 551 insertions(+) create mode 100644 .github/workflows/ci.yml create mode 100644 .github/workflows/windows-ci.yml create mode 100755 build-aux/ci_build.sh diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 000000000..0d0b209c1 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,146 @@ +############################################################################# +# +# Author: Jia Tan +# +# This file has been put into the public domain. +# You can do whatever you want with this file. +# +############################################################################# + +name: CI + +on: + # Triggers the workflow on push or pull request events but only for the master branch + push: + branches: [ master ] + pull_request: + branches: [ master ] + + # Allows running workflow manually + workflow_dispatch: + +jobs: + POSIX: + strategy: + matrix: + os: [ubuntu-latest, macos-latest] + build_system: [autotools, cmake] + runs-on: ${{ matrix.os }} + steps: + - uses: actions/checkout@8ade135a41bc03ea155e62e844d188df1ea18608 #v4.1.0 + + ######################## + # Install Dependencies # + ######################## + + # Install Autotools on Linux + - name: Install Dependencies + if: ${{ matrix.os == 'ubuntu-latest' && matrix.build_system == 'autotools' }} + run: | + sudo apt-get update + sudo apt-get install -y autoconf automake build-essential po4a autopoint gcc-multilib doxygen + + # Install Autotools on Mac + - name: Install Dependencies + if: ${{ matrix.os == 'macos-latest' && matrix.build_system == 'autotools' }} + run: brew install autoconf automake libtool po4a doxygen + + # Install CMake on Linux + - name: Install Dependencies + if: ${{ matrix.os == 'ubuntu-latest' && matrix.build_system == 'cmake' }} + run: | + sudo apt-get update + sudo apt-get install -y build-essential cmake + + # Install CMake on Mac + - name: Install Dependencies + if: ${{ matrix.os == 'macos-latest' && matrix.build_system == 'cmake' }} + run: brew install cmake + + ################## + # Build and Test # + ################## + + # -b specifies the build system to use. + # -p specifies the phase (build or test) to help narrow down an error + # if one occurs. + # + # The first two builds/tests are only run on Autotools Linux and + # affect the CFLAGS. Resetting the CFLAGS requires clearing the + # config cache between runs, so the tests that require CFLAGS are + # done first. + - name: Build 32-bit + if: ${{ matrix.os == 'ubuntu-latest' && matrix.build_system == 'autotools' }} + run: ./build-aux/ci_build.sh -b autotools -p build -f "-m32" + - name: Test 32-bit + if: ${{ matrix.os == 'ubuntu-latest' && matrix.build_system == 'autotools' }} + run: | + ./build-aux/ci_build.sh -b autotools -p test -f "-m32" -n 32_bit + cd ../xz_build && make distclean + + # ifunc must be disabled for this test because __attribute__ ifunc is + # incompatible with -fsanitize=address. + # + # The sandbox must also be disabled because it will prevent access to + # the /proc/ filesystem on Linux, which is used by the sanitizer's + # instrumentation. + - name: Build with -fsanitize=address,undefined + if: ${{ matrix.os == 'ubuntu-latest' && matrix.build_system == 'autotools' }} + run: ./build-aux/ci_build.sh -b autotools -p build -f "-fsanitize=address,undefined" + - name: Test with -fsanitize=address,undefined + if: ${{ matrix.os == 'ubuntu-latest' && matrix.build_system == 'autotools' }} + run: | + ./build-aux/ci_build.sh -b autotools -p test -f "-fsanitize=address,undefined" + cd ../xz_build && make distclean + + - name: Build with full features + run: ./build-aux/ci_build.sh -b ${{ matrix.build_system }} -p build + - name: Test with full features + run: ./build-aux/ci_build.sh -b ${{ matrix.build_system }} -p test -n full_features + + - name: Build without encoders + run: ./build-aux/ci_build.sh -b ${{ matrix.build_system }} -d encoders,shared -p build + - name: Test without encoders + run: ./build-aux/ci_build.sh -b ${{ matrix.build_system }} -d encoders,shared -p test -n no_encoders + + - name: Build without decoders + run: ./build-aux/ci_build.sh -b ${{ matrix.build_system }} -d decoders,shared -p build + - name: Test without decoders + run: ./build-aux/ci_build.sh -b ${{ matrix.build_system }} -d decoders,shared -p test -n no_decoders + + - name: Build without threads + run: ./build-aux/ci_build.sh -b ${{ matrix.build_system }} -d threads,shared -p build + - name: Test without threads + run: ./build-aux/ci_build.sh -b ${{ matrix.build_system }} -d threads,shared -p test -n no_threads + + - name: Build without BCJ filters + run: ./build-aux/ci_build.sh -b ${{ matrix.build_system }} -d bcj,shared,nls -p build + - name: Test without BCJ filters + run: ./build-aux/ci_build.sh -b ${{ matrix.build_system }} -d bcj,shared,nls -p test -n no_bcj + + - name: Build without Delta filters + run: ./build-aux/ci_build.sh -b ${{ matrix.build_system }} -d delta,shared,nls -p build + - name: Test without Delta filters + run: ./build-aux/ci_build.sh -b ${{ matrix.build_system }} -d delta,shared,nls -p test -n no_delta + + - name: Build without sha256 check + run: ./build-aux/ci_build.sh -b ${{ matrix.build_system }} -c crc32,crc64 -d shared,nls -p build + - name: Test without sha256 check + run: ./build-aux/ci_build.sh -b ${{ matrix.build_system }} -c crc32,crc64 -d shared,nls -p test -n no_sha256 + + - name: Build without crc64 check + run: ./build-aux/ci_build.sh -b ${{ matrix.build_system }} -c crc32,sha256 -d shared,nls -p build + - name: Test without crc64 check + run: ./build-aux/ci_build.sh -b ${{ matrix.build_system }} -c crc32,sha256 -d shared,nls -p test -n no_crc64 + + - name: Build small + run: ./build-aux/ci_build.sh -b ${{ matrix.build_system }} -d small -p build + - name: Test small + run: ./build-aux/ci_build.sh -b ${{ matrix.build_system }} -d small -p test -n small + + # Attempt to upload the test logs as artifacts if any step has failed + - uses: actions/upload-artifact@a8a3f3ad30e3422c9c7b888a15615d19a852ae32 #v3.1.3 + if: ${{ failure() }} + with: + name: ${{ matrix.os }} ${{ matrix.build_system }} Test Logs + path: build-aux/artifacts diff --git a/.github/workflows/windows-ci.yml b/.github/workflows/windows-ci.yml new file mode 100644 index 000000000..85562da7e --- /dev/null +++ b/.github/workflows/windows-ci.yml @@ -0,0 +1,125 @@ +############################################################################# +# +# Author: Jia Tan +# +# This file has been put into the public domain. +# You can do whatever you want with this file. +# +############################################################################# + +name: Windows-CI + +# Only run the Windows CI manually since it takes much longer than the others. +on: workflow_dispatch + +jobs: + POSIX: + strategy: + matrix: + # Test different environments since the code may change between + # them and we want to ensure that we support all potential users. + # clang64 builds are currently broken when building static libraries + # due to a bug in ldd search path: + # https://github.com/llvm/llvm-project/issues/67779 + # TODO - re-enable clang64 when this is resolved. + msys2_env: [mingw64, mingw32, ucrt64, msys] + build_system: [autotools, cmake] + + # Set the shell to be msys2 as a default to avoid setting it for + # every individual run command. + defaults: + run: + shell: msys2 {0} + + runs-on: windows-latest + + steps: + ##################### + # Setup Environment # + ##################### + + # Rely on the msys2 GitHub Action to set up the msys2 environment. + - name: Setup MSYS2 + uses: msys2/setup-msys2@27b3aa77f672cb6b3054121cfd80c3d22ceebb1d #v2.20.1 + with: + msystem: ${{ matrix.msys2_env }} + update: true + install: pactoys make + + - name: Checkout code + # Need to explicitly set the shell here since we set the default + # shell as msys2 earlier. This avoids an extra msys2 dependency on + # git. + shell: powershell + # Avoid Windows line endings. Otherwise test_scripts.sh will fail + # because the expected output is stored in the test framework as a + # text file and will not match the output from xzgrep. + run: git config --global core.autocrlf false + + - uses: actions/checkout@8ade135a41bc03ea155e62e844d188df1ea18608 #v4.1.0 + + + ######################## + # Install Dependencies # + ######################## + + # The pacman repository has a different naming scheme for default + # msys packages than the others. The pacboy tool allows installing + # the packages possible in matrix setup without a burdensome amount + # of ifs. + - name: Install Dependencies + if: ${{ matrix.msys2_env == 'msys' && matrix.build_system == 'autotools' }} + run: pacman --noconfirm -S --needed autotools base-devel doxygen gettext-devel gcc + + - name: Install Dependencies + if: ${{ matrix.msys2_env != 'msys' && matrix.build_system == 'autotools' }} + run: pacboy --noconfirm -S --needed autotools:p toolchain:p doxygen:p + + - name: Install Dependencies + if: ${{ matrix.msys2_env == 'msys' && matrix.build_system == 'cmake' }} + run: pacman --noconfirm -S --needed cmake base-devel gcc + + - name: Install Dependencies + if: ${{ matrix.msys2_env != 'msys' && matrix.build_system == 'cmake' }} + run: pacboy --noconfirm -S --needed cmake:p toolchain:p + + ################## + # Build and Test # + ################## + + - name: Build with full features + run: ./build-aux/ci_build.sh -a "--no-po4a" -b ${{ matrix.build_system }} -p build + - name: Test with full features + run: ./build-aux/ci_build.sh -a "--no-po4a" -b ${{ matrix.build_system }} -p test -n full_features + + - name: Build without threads + run: ./build-aux/ci_build.sh -a "--no-po4a" -b ${{ matrix.build_system }} -d threads,shared -p build + - name: Test without threads + run: ./build-aux/ci_build.sh -a "--no-po4a" -b ${{ matrix.build_system }} -d threads,shared -p test -n no_threads + + - name: Build without encoders + run: ./build-aux/ci_build.sh -a "--no-po4a" -b ${{ matrix.build_system }} -d encoders,shared -p build + - name: Test without encoders + run: ./build-aux/ci_build.sh -a "--no-po4a" -b ${{ matrix.build_system }} -d encoders,shared -p test -n no_encoders + + - name: Build without decoders + run: ./build-aux/ci_build.sh -a "--no-po4a" -b ${{ matrix.build_system }} -d decoders,shared -p build + - name: Test without decoders + run: ./build-aux/ci_build.sh -a "--no-po4a" -b ${{ matrix.build_system }} -d decoders,shared -p test -n no_decoders + + - name: Build with only crc32 check + run: ./build-aux/ci_build.sh -a "--no-po4a" -b ${{ matrix.build_system }} -c crc32 -d shared,nls -p build + - name: Test with only crc32 check + run: ./build-aux/ci_build.sh -a "--no-po4a" -b ${{ matrix.build_system }} -c crc32 -d shared,nls -p test -n crc32_only + + + ############### + # Upload Logs # + ############### + + # Upload the test logs as artifacts if any step has failed. + - uses: actions/upload-artifact@a8a3f3ad30e3422c9c7b888a15615d19a852ae32 #v3.1.3 + if: ${{ failure() }} + with: + name: ${{ matrix.msys2_env }} ${{ matrix.build_system }} Test Logs + path: build-aux/artifacts diff --git a/build-aux/ci_build.sh b/build-aux/ci_build.sh new file mode 100755 index 000000000..41cc2bc53 --- /dev/null +++ b/build-aux/ci_build.sh @@ -0,0 +1,280 @@ +#!/bin/bash +# +############################################################################# +# +# Script meant to be used for Continuous Integration automation for POSIX +# systems. On GitHub, this is used by Ubuntu and MacOS builds. +# +############################################################################# +# +# Author: Jia Tan +# +# This file has been put into the public domain. +# You can do whatever you want with this file. +# +############################################################################# + +set -e + +USAGE="Usage: $0 + -a [autogen flags] + -b [autotools|cmake] + -c [crc32|crc64|sha256] + -d [encoders|decoders|bcj|delta|threads|shared|nls|small|ifunc|clmul|sandbox] + -f [CFLAGS] + -l [destdir] + -n [ARTIFACTS_DIR_NAME] + -p [all|build|test] + -s [srcdir]" + +# Absolute path of script directory +ABS_DIR=$(cd -- "$(dirname -- "$0")" && pwd) + +# Default CLI option values +AUTOGEN_FLAGS="" +BUILD_SYSTEM="autotools" +CHECK_TYPE="crc32,crc64,sha256" +BCJ="y" +DELTA="y" +ENCODERS="y" +DECODERS="y" +THREADS="y" +SHARED="y" +NATIVE_LANG_SUPPORT="y" +SMALL="n" +IFUNC="y" +CLMUL="y" +SANDBOX="y" +SRC_DIR="$ABS_DIR/../" +DEST_DIR="$SRC_DIR/../xz_build" +PHASE="all" +ARTIFACTS_DIR_NAME="output" + + +################### +# Parse arguments # +################### + +while getopts a:b:c:d:l:n:s:p:f:h opt; do + # b option can have either value "autotools" OR "cmake" + case ${opt} in + h) + echo "$USAGE" + exit 0 + ;; + a) + AUTOGEN_FLAGS="$OPTARG" + ;; + b) + case "$OPTARG" in + autotools) ;; + cmake) ;; + *) echo "Invalid build system: $OPTARG"; exit 1;; + esac + BUILD_SYSTEM="$OPTARG" + ;; + c) CHECK_TYPE="$OPTARG" + ;; + # d options can be a comma separated list of things to disable at + # configure time + d) + for disable_arg in $(echo "$OPTARG" | sed "s/,/ /g"); do + case "$disable_arg" in + encoders) ENCODERS="n" ;; + decoders) DECODERS="n" ;; + bcj) BCJ="n" ;; + delta) DELTA="n" ;; + threads) THREADS="n" ;; + shared) SHARED="n";; + nls) NATIVE_LANG_SUPPORT="n";; + small) SMALL="y";; + ifunc) IFUNC="n";; + clmul) CLMUL="n";; + sandbox) SANDBOX="n";; + *) echo "Invalid disable value: $disable_arg"; exit 1 ;; + esac + done + ;; + l) DEST_DIR="$OPTARG" + ;; + n) ARTIFACTS_DIR_NAME="$OPTARG" + ;; + s) SRC_DIR="$OPTARG" + ;; + p) PHASE="$OPTARG" + ;; + f) + CFLAGS="$OPTARG" + export CFLAGS + ;; + esac +done + + +#################### +# Helper Functions # +#################### + +# These two functions essentially implement the ternary "?" operator. +add_extra_option() { + # First argument is option value ("y" or "n") + # Second argument is option to set if "y" + # Third argument is option to set if "n" + if [ "$1" = "y" ] + then + EXTRA_OPTIONS="$EXTRA_OPTIONS $2" + else + EXTRA_OPTIONS="$EXTRA_OPTIONS $3" + fi +} + + +add_to_filter_list() { + # First argument is option value ("y" or "n") + # Second argument is option to set if "y" + if [ "$1" = "y" ] + then + FILTER_LIST="$FILTER_LIST$2" + fi +} + + +############### +# Build Phase # +############### + +if [ "$PHASE" = "all" ] || [ "$PHASE" = "build" ] +then + # Checksum options should be specified differently based on the + # build system. It must be calculated here since we won't know + # the build system used until all args have been parsed. + # Autotools - comma separated + # CMake - semi-colon separated + if [ "$BUILD_SYSTEM" = "autotools" ] + then + SEP="," + else + SEP=";" + fi + + CHECK_TYPE_TEMP="" + for crc in $(echo "$CHECK_TYPE" | sed "s/,/ /g"); do + case "$crc" in + # Remove "crc32" from cmake build, if specified. + crc32) + if [ "$BUILD_SYSTEM" = "cmake" ] + then + continue + fi + ;; + crc64) ;; + sha256) ;; + *) echo "Invalid check type: $crc"; exit 1 ;; + esac + + CHECK_TYPE_TEMP="$CHECK_TYPE_TEMP$SEP$crc" + done + + # Remove the first character from $CHECK_TYPE_TEMP since it will + # always be the delimiter. + CHECK_TYPE="${CHECK_TYPE_TEMP:1}" + + FILTER_LIST="lzma1$SEP"lzma2 + + # Build based on arguments + mkdir -p "$DEST_DIR" + + # Generate configure option values + EXTRA_OPTIONS="" + + case $BUILD_SYSTEM in + autotools) + cd "$SRC_DIR" + + # Run autogen.sh script if not already run + if [ ! -f configure ] + then + ./autogen.sh "$AUTOGEN_FLAGS" + fi + + cd "$DEST_DIR" + + add_to_filter_list "$BCJ" ",x86,powerpc,ia64,arm,armthumb,arm64,sparc" + add_to_filter_list "$DELTA" ",delta" + + add_extra_option "$ENCODERS" "--enable-encoders=$FILTER_LIST" "--disable-encoders" + add_extra_option "$DECODERS" "--enable-decoders=$FILTER_LIST" "--disable-decoders" + add_extra_option "$THREADS" "" "--disable-threads" + add_extra_option "$SHARED" "" "--disable-shared" + add_extra_option "$NATIVE_LANG_SUPPORT" "" "--disable-nls" + add_extra_option "$SMALL" "--enable-small" "" + add_extra_option "$IFUNC" "" "--disable-ifunc" + add_extra_option "$CLMUL" "" "--disable-clmul-crc" + add_extra_option "$SANDBOX" "" "--enable-sandbox=no" + + # Run configure script + "$SRC_DIR"/configure --enable-werror --enable-checks="$CHECK_TYPE" $EXTRA_OPTIONS --config-cache + + # Build the project + make + ;; + cmake) + cd "$DEST_DIR" + + add_to_filter_list "$BCJ" ";x86;powerpc;ia64;arm;armthumb;arm64;sparc" + add_to_filter_list "$DELTA" ";delta" + + add_extra_option "$THREADS" "-DENABLE_THREADS=ON" "-DENABLE_THREADS=OFF" + + # Disable MicroLZMA if encoders are not configured. + add_extra_option "$ENCODERS" "-DENCODERS=$FILTER_LIST" "-DENCODERS= -DMICROLZMA_ENCODER=OFF" + + # Disable MicroLZMA and lzip decoders if decoders are not configured. + add_extra_option "$DECODERS" "-DDECODERS=$FILTER_LIST" "-DDECODERS= -DMICROLZMA_DECODER=OFF -DLZIP_DECODER=OFF" + + # CMake disables the shared library by default. + add_extra_option "$SHARED" "-DBUILD_SHARED_LIBS=ON" "" + + add_extra_option "$SMALL" "-DHAVE_SMALL=ON" "" + + # Remove old cache file to clear previous settings. + rm -f "CMakeCache.txt" + cmake "$SRC_DIR/CMakeLists.txt" -B "$DEST_DIR" $EXTRA_OPTIONS -DADDITIONAL_CHECK_TYPES="$CHECK_TYPE" -G "Unix Makefiles" + cmake --build "$DEST_DIR" + ;; + esac +fi + + +############## +# Test Phase # +############## + +if [ "$PHASE" = "all" ] || [ "$PHASE" = "test" ] +then + case $BUILD_SYSTEM in + autotools) + cd "$DEST_DIR" + # If the tests fail, copy the test logs into the artifacts folder + if make check + then + : + else + mkdir -p "$SRC_DIR/build-aux/artifacts/$ARTIFACTS_DIR_NAME" + cp ./tests/*.log "$SRC_DIR/build-aux/artifacts/$ARTIFACTS_DIR_NAME" + exit 1 + fi + ;; + cmake) + cd "$DEST_DIR" + if make test + then + : + else + mkdir -p "$SRC_DIR/build-aux/artifacts/$ARTIFACTS_DIR_NAME" + cp ./Testing/Temporary/*.log "$SRC_DIR/build-aux/artifacts/$ARTIFACTS_DIR_NAME" + exit 1 + fi + ;; + esac +fi