From 6ad9727c64cfa30cd74521d7efad1d867dea47f9 Mon Sep 17 00:00:00 2001 From: Frankie Dintino Date: Sun, 11 Apr 2021 00:36:26 -0400 Subject: [PATCH] Add wheel build scripts for libavif --- config.sh | 10 +- config_libavif.sh | 293 +++++++++++++++++++++++++++++ dependency_licenses/AOM.txt | 26 +++ dependency_licenses/DAV1D.txt | 23 +++ dependency_licenses/LIBAVIF.txt | 170 +++++++++++++++++ dependency_licenses/LIBYUV.txt | 24 +++ dependency_licenses/PATENTS.txt | 107 +++++++++++ dependency_licenses/RAV1E.txt | 22 +++ dependency_licenses/SVT-AV1.txt | 26 +++ patches/aom-2.0.2-manylinux1.patch | 41 ++++ 10 files changed, 740 insertions(+), 2 deletions(-) create mode 100644 config_libavif.sh create mode 100644 dependency_licenses/AOM.txt create mode 100644 dependency_licenses/DAV1D.txt create mode 100644 dependency_licenses/LIBAVIF.txt create mode 100644 dependency_licenses/LIBYUV.txt create mode 100644 dependency_licenses/PATENTS.txt create mode 100644 dependency_licenses/RAV1E.txt create mode 100644 dependency_licenses/SVT-AV1.txt create mode 100644 patches/aom-2.0.2-manylinux1.patch diff --git a/config.sh b/config.sh index b135b083..10db974e 100644 --- a/config.sh +++ b/config.sh @@ -2,6 +2,9 @@ # Test for macOS with [ -n "$IS_MACOS" ] ARCHIVE_SDIR=pillow-depends-master +PILLOW_WHEELS_DIR=$(abspath $(dirname "$CONFIG_PATH")) + +source $PILLOW_WHEELS_DIR/config_libavif.sh # Package versions for fresh source builds FREETYPE_VERSION=2.10.4 @@ -36,7 +39,7 @@ function untar { function pre_build { # Any stuff that you need to do before you start building the wheels # Runs in the root directory of this repository. - curl -fsSL -o pillow-depends-master.zip https://github.com/python-pillow/pillow-depends/archive/master.zip + curl -fsSL -o pillow-depends-master.zip https://github.com/fdintino/pillow-depends/archive/master.zip untar pillow-depends-master.zip if [ -n "$IS_MACOS" ]; then # Update to latest zlib for macOS build @@ -101,6 +104,9 @@ function pre_build { export FREETYPE_CFLAGS='' fi + # defined in config_libavif.sh + build_libavif + # Append licenses for filename in dependency_licenses/*; do echo -e "\n\n----\n\n$(basename $filename | cut -f 1 -d '.')\n" | cat >> Pillow/LICENSE @@ -127,7 +133,7 @@ function run_tests_in_repo { EXP_CODECS="jpg jpg_2000" EXP_CODECS="$EXP_CODECS libtiff zlib" -EXP_MODULES="freetype2 littlecms2 pil tkinter webp" +EXP_MODULES="avif freetype2 littlecms2 pil tkinter webp" if [ -z "$IS_MACOS" ] && [[ "$MB_PYTHON_VERSION" != pypy3* ]]; then EXP_FEATURES="fribidi harfbuzz raqm transp_webp webp_anim webp_mux" else diff --git a/config_libavif.sh b/config_libavif.sh new file mode 100644 index 00000000..344fcf7f --- /dev/null +++ b/config_libavif.sh @@ -0,0 +1,293 @@ +set -e + +PILLOW_WHEELS_DIR=$(abspath $(dirname "$CONFIG_PATH")) + +LIBAVIF_VERSION=0.9.0 +CARGO_C_VERSION=0.8.0 +AOM_VERSION=2.0.2 +DAV1D_VERSION=0.8.2 +SVT_AV1_VERSION=0.8.6 +RAV1E_VERSION=0.4.0 + +function install_nasm { + if [ -e nasm-stamp ]; then return; fi + + echo "::group::Build nasm" + if [ -n "$IS_MACOS" ]; then + brew install nasm + else + build_simple nasm 2.15.05 https://www.nasm.us/pub/nasm/releasebuilds/2.15.05/ + fi + echo "::endgroup::" + + touch nasm-stamp +} + +function install_meson { + if [ -e meson-stamp ]; then return; fi + + echo "::group::Install meson" + $PYTHON_EXE -m pip install meson + echo "::endgroup::" + + touch meson-stamp +} + +function install_ninja { + if [ -e ninja-stamp ]; then return; fi + + echo "::group::Install ninja" + $PYTHON_EXE -m pip install ninja + local ninja_exe=$(dirname $PYTHON_EXE)/ninja + ln -s $ninja_exe /usr/local/bin/ninja-build + echo "::endgroup::" + + touch ninja-stamp +} + +function install_rust { + if [ -e rust-stamp ]; then return; fi + + echo "::group::Install rust" + if [ -n "$IS_MACOS" ]; then + brew install rust + else + if [[ "$MB_ML_VER" == "1" ]]; then + # Download and use old rustup-init that's compatible with glibc on el5 + curl -sLO https://static.rust-lang.org/rustup/archive/1.22.1/$PLAT-unknown-linux-gnu/rustup-init + chmod u+x rustup-init + ./rustup-init --default-toolchain nightly-2020-07-18 -y + else + curl https://sh.rustup.rs -sSf | /bin/sh -s -- -y + fi + fi + if [ -e $HOME/.cargo/env ]; then + source $HOME/.cargo/env + fi + echo "::endgroup::" + + touch rust-stamp +} + +function install_more_modern_cmake { + # install_modern_cmake installs cmake 2.8, but some libavif dependencies + # require a higher version + if [ -e cmake-more-modern ]; then return; fi + + echo "::group::Install cmake" + if [ -n "$IS_MACOS" ]; then + brew install cmake + else + $PYTHON_EXE -m pip install cmake + fi + echo "::endgroup::" + + touch cmake-more-modern +} + +function install_cargo_c { + install_rust + + if which cargo-cbuild 1>/dev/null 2>/dev/null; then return; fi + + echo "::group::Install cargo-c" + if [ -n "$IS_MACOS" ]; then + brew install cargo-c + else + mkdir -p $HOME/.cargo/bin + (cd $HOME/.cargo/bin \ + && fetch_unpack \ + https://github.com/lu-zero/cargo-c/releases/download/v$CARGO_C_VERSION/cargo-c-linux.tar.gz \ + cargo-c-$CARGO_C_VERSION-linux.tar.gz) + fi + echo "::endgroup::" +} + +function build_aom { + if [ -e aom-stamp ]; then return; fi + + if [ ! -n "$IS_MACOS" ]; then + install_more_modern_cmake + install_nasm + fi + + echo "::group::Build aom" + + if [ -n "$IS_MACOS" ]; then + brew install aom + else + (rm_mkdir aom-$AOM_VERSION \ + && cd aom-$AOM_VERSION \ + && fetch_unpack \ + https://storage.googleapis.com/aom-releases/libaom-$AOM_VERSION.tar.gz) + + if [[ "$MB_ML_VER" == "1" ]]; then + (cd aom-$AOM_VERSION \ + && patch -p1 < $PILLOW_WHEELS_DIR/patches/aom-2.0.2-manylinux1.patch) + fi + mkdir aom-$AOM_VERSION/build/linux + (cd aom-$AOM_VERSION/build/linux \ + && cmake \ + -DCMAKE_C_FLAGS=-fPIC \ + -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_INSTALL_PREFIX="${BUILD_PREFIX}" \ + -DCMAKE_INSTALL_LIBDIR=lib \ + -DBUILD_SHARED_LIBS=0 \ + -DENABLE_DOCS=0 \ + -DENABLE_EXAMPLES=0 \ + -DENABLE_TESTDATA=0 \ + -DENABLE_TESTS=0 \ + -DENABLE_TOOLS=0 \ + ../.. \ + && make install) + fi + echo "::endgroup::" + + touch aom-stamp +} + +function build_dav1d { + if [ -e dav1d-stamp ]; then return; fi + + install_meson + install_ninja + install_nasm + + echo "::group::Build dav1d" + fetch_unpack "https://code.videolan.org/videolan/dav1d/-/archive/$DAV1D_VERSION/dav1d-$DAV1D_VERSION.tar.gz" + (cd dav1d-$DAV1D_VERSION \ + && meson . build \ + "--prefix=${BUILD_PREFIX}" \ + --default-library=static \ + --buildtype=release \ + && ninja -vC build install) + echo "::endgroup::" + + touch dav1d-stamp +} + +function build_svt_av1 { + if [ -e svt-av1-stamp ]; then return; fi + + echo "::group::Build SVT-AV1" + + install_nasm + install_more_modern_cmake + + fetch_unpack \ + "https://gitlab.com/AOMediaCodec/SVT-AV1/-/archive/v$SVT_AV1_VERSION/SVT-AV1-v$SVT_AV1_VERSION.tar.gz" + + (cd SVT-AV1-v$SVT_AV1_VERSION/Build/linux \ + && cmake \ + ../.. \ + -DCMAKE_INSTALL_PREFIX="${BUILD_PREFIX}" \ + -DBUILD_SHARED_LIBS=OFF \ + -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_INSTALL_LIBDIR=lib \ + && make install \ + && cp SvtAv1Enc.pc $BUILD_PREFIX/lib/pkgconfig) + + echo "::endgroup::" + + touch svt-av1-stamp +} + +function build_rav1e { + if [ -e rav1e-stamp ]; then return; fi + + install_cargo_c + install_nasm + + echo "::group::Build rav1e" + + CARGO_VENDOR_TGZ=$ARCHIVE_SDIR/rav1e-vendor-$RAV1E_VERSION.tar.gz + if [ -e $CARGO_VENDOR_TGZ ]; then + mkdir -p "$HOME/.cargo" + tar -C $ARCHIVE_SDIR -zxf $CARGO_VENDOR_TGZ + VENDOR_DIR=$(pwd -P)/$ARCHIVE_SDIR/vendor + cat > ~/.cargo/config < +Date: Wed, 7 Apr 2021 17:12:57 -0400 +Subject: [PATCH] Define _mm256_bsrli_epi128 for old glibc; remove version.pl version constraint + +--- + aom_dsp/x86/intrapred_x86.h | 4 ++++ + build/cmake/version.pl | 1 - + 2 files changed, 4 insertions(+), 1 deletion(-) + +diff --git a/aom_dsp/x86/intrapred_x86.h b/aom_dsp/x86/intrapred_x86.h +index b13f575..ddba5f0 100644 +--- a/aom_dsp/x86/intrapred_x86.h ++++ b/aom_dsp/x86/intrapred_x86.h +@@ -35,4 +35,8 @@ static INLINE __m128i dc_sum_32_sse2(const uint8_t *ref) { + return _mm_add_epi16(x0, high); + } + ++#if defined(__GNUC__) && __GNUC__ < 6 ++ #define _mm256_bsrli_epi128(a, count) _mm256_srli_si256((a), (count)) ++#endif ++ + #endif // AOM_AOM_DSP_X86_INTRAPRED_X86_H_ +-- +2.30.0 + +diff --git a/build/cmake/version.pl b/build/cmake/version.pl +index 7d23f2b..73c5f81 100755 +--- a/build/cmake/version.pl ++++ b/build/cmake/version.pl +@@ -11,7 +11,6 @@ + ## + use strict; + use warnings; +-use 5.010; + use Getopt::Long; + + my $git_desc = ''; +-- +2.30.0 +