opencv_python (#3)

* override path

* path shenanigans

* split into multiple stages

* use gcc 10

* set CHOST to arm-linux-gnueabihf

* split out qt into its own stage

* add stage 3

* patch libvpx

* fix cache

* try latest libvpx release

* use older libvpx

* include qt in libs

* add docker artifact compression

* Revert "add docker artifact compression"

This reverts commit da49b5d.

* update pkgconfig + ldflags to find Qt

* CI_BUILD

* override compilers

* set LD_LIBRARY_PATH

.github/workflows/build_wheel_dockerfile.yml

@@ -166,4 +166,4 @@ jobs:
         if: ${{ matrix.build_params['key'] == fromJSON(needs.calculate_matrix.outputs.build_matrix_last) }}
         with:
           name: ${{ inputs.package }}-${{ inputs.version }}-cpython${{ inputs.python_version }}
-          path: /tmp/output/*.whl
+          path: /tmp/output/*.whl

packages/opencv_python/Dockerfile.stage1

@@ -0,0 +1,52 @@
+# adapted from https://github.com/opencv/opencv-python/blob/e26e6e4611b40ea5344e4e4b6e2467539a07a11f/docker/manylinux2014/Dockerfile_aarch64
+
+FROM ghcr.io/bjia56/armv7l-wheel-builder:main
+
+ENV CCACHE_VERSION=3.7.9
+ENV FFMPEG_VERSION=5.1.3
+ENV FREETYPE_VERSION=2.13.1
+ENV LIBPNG_VERSION=1.6.40
+ENV NASM_VERSION=2.15.04
+ENV OPENSSL_VERSION=1_1_1u
+ENV QT_VERSION=5.15.0
+ENV YASM_VERSION=1.3.0
+
+ENV LD_LIBRARY_PATH /usr/local/lib:$LD_LIBRARY_PATH
+
+RUN yum install zlib-devel curl-devel xcb-util-renderutil-devel xcb-util-devel xcb-util-image-devel xcb-util-keysyms-devel xcb-util-wm-devel mesa-libGL-devel libxkbcommon-devel libxkbcommon-x11-devel libXi-devel lapack-devel epel-release -y && \
+    yum install dejavu-sans-fonts -y && \
+    cp /usr/include/lapacke/lapacke*.h /usr/include/ && \
+    curl https://raw.githubusercontent.com/xianyi/OpenBLAS/v0.3.3/cblas.h -o /usr/include/cblas.h && \
+    # libpng will be built from source
+    yum remove libpng -y
+
+WORKDIR /
+RUN mkdir ~/openblas_sources && \
+    cd ~/openblas_sources && \
+    wget -q https://github.com/OpenMathLib/OpenBLAS/releases/download/v0.3.23/OpenBLAS-0.3.23.tar.gz && \
+    tar xf OpenBLAS-0.3.23.tar.gz && \
+    cd OpenBLAS-0.3.23 && \
+    make PREFIX=/usr/local TARGET=ARMV7 -j4 && \
+    make PREFIX=/usr/local TARGET=ARMV7 install
+
+RUN mkdir ~/libpng_sources && \
+    cd ~/libpng_sources && \
+    curl -O -L https://download.sourceforge.net/libpng/libpng-${LIBPNG_VERSION}.tar.gz && \
+    tar -xf libpng-${LIBPNG_VERSION}.tar.gz && \
+    cd libpng-${LIBPNG_VERSION} && \
+    ./configure --prefix=/usr/local && \
+    make && \
+    make install && \
+    cd .. && \
+    rm -rf ~/libpng_sources
+
+RUN mkdir ~/freetype_sources && \
+    cd ~/freetype_sources && \
+    curl -O -L https://github.com/bjia56/armv7l-wheels/releases/download/build-dependency-archives/freetype-${FREETYPE_VERSION}.tar.gz && \
+    tar -xf freetype-${FREETYPE_VERSION}.tar.gz && \
+    cd freetype-${FREETYPE_VERSION} && \
+    ./configure --prefix="/ffmpeg_build" --enable-freetype-config && \
+    make && \
+    make install && \
+    cd .. && \
+    rm -rf ~/freetype_sources

packages/opencv_python/Dockerfile.stage2

@@ -0,0 +1,19 @@
+# adapted from https://github.com/opencv/opencv-python/blob/e26e6e4611b40ea5344e4e4b6e2467539a07a11f/docker/manylinux2014/Dockerfile_aarch64
+
+FROM base
+
+RUN curl -O -L https://download.qt.io/official_releases/qt/5.15/${QT_VERSION}/single/qt-everywhere-src-${QT_VERSION}.tar.xz && \
+    tar -xf qt-everywhere-src-${QT_VERSION}.tar.xz && \
+    cd qt-everywhere-src-${QT_VERSION} && \
+    export MAKEFLAGS=-j$(nproc) && \
+    export PREV_PATH=$PATH && \
+    export PATH=/opt/rh/devtoolset-10/root/usr/bin:$PATH && \
+    ./configure -prefix /opt/Qt${QT_VERSION} -release -opensource -confirm-license -qtnamespace QtOpenCVPython -xcb -xcb-xlib -bundled-xcb-xinput -no-openssl -no-dbus -skip qt3d -skip qtactiveqt -skip qtcanvas3d -skip qtconnectivity -skip qtdatavis3d -skip qtdoc -skip qtgamepad -skip qtgraphicaleffects -skip qtimageformats -skip qtlocation -skip qtmultimedia -skip qtpurchasing -skip qtqa -skip qtremoteobjects -skip qtrepotools -skip qtscript -skip qtscxml -skip qtsensors -skip qtserialbus -skip qtserialport -skip qtspeech -skip qttranslations -skip qtwayland -skip qtwebchannel -skip qtwebengine -skip qtwebsockets -skip qtwebview -skip xmlpatterns -skip declarative -make libs && \
+    make && \
+    make install && \
+    cd .. && \
+    rm -rf qt-everywhere* && \
+    export PATH=$PREV_PATH
+
+ENV QTDIR /opt/Qt${QT_VERSION}
+ENV PATH "$QTDIR/bin:$PATH"

packages/opencv_python/Dockerfile.stage3

@@ -0,0 +1,75 @@
+# adapted from https://github.com/opencv/opencv-python/blob/e26e6e4611b40ea5344e4e4b6e2467539a07a11f/docker/manylinux2014/Dockerfile_aarch64
+
+FROM base
+
+RUN mkdir ~/openssl_sources && \
+    cd ~/openssl_sources && \
+    curl -O -L https://github.com/openssl/openssl/archive/OpenSSL_${OPENSSL_VERSION}.tar.gz && \
+    tar -xf OpenSSL_${OPENSSL_VERSION}.tar.gz && \
+    cd openssl-OpenSSL_${OPENSSL_VERSION} && \
+    ./config --prefix="/ffmpeg_build" --openssldir="/ffmpeg_build" no-pinshared shared zlib && \
+    make -j$(getconf _NPROCESSORS_ONLN) && \
+    # skip installing documentation
+    make install_sw && \
+    cd .. && \
+    rm -rf ~/openssl_build ~/openssl_sources
+
+RUN mkdir ~/nasm_sources && \
+    cd ~/nasm_sources && \
+    curl -O -L http://www.nasm.us/pub/nasm/releasebuilds/${NASM_VERSION}/nasm-${NASM_VERSION}.tar.gz && \
+    tar -xf nasm-${NASM_VERSION}.tar.gz && cd nasm-${NASM_VERSION} && ./autogen.sh && \
+    ./configure --prefix="/ffmpeg_build" --bindir="$HOME/bin" && \
+    make -j$(getconf _NPROCESSORS_ONLN) && \
+    make install && \
+    cd .. && \
+    rm -rf ~/nasm_sources
+
+RUN mkdir ~/yasm_sources && \
+    cd ~/yasm_sources && \
+    curl -O -L http://www.tortall.net/projects/yasm/releases/yasm-${YASM_VERSION}.tar.gz && \
+    tar -xf yasm-${YASM_VERSION}.tar.gz && \
+    cd yasm-${YASM_VERSION} && \
+    ./configure --prefix="/ffmpeg_build" --bindir="$HOME/bin" && \
+    make -j$(getconf _NPROCESSORS_ONLN) && \
+    make install && \
+    cd .. && \
+    rm -rf ~/yasm_sources
+
+COPY ./libvpx.patch /tmp/libvpx.patch
+
+RUN mkdir ~/libvpx_sources && \
+    cd ~/libvpx_sources && \
+    git clone --depth 1 --branch v1.11.0 --single-branch https://github.com/webmproject/libvpx.git && \
+    cd libvpx && \
+    git apply /tmp/libvpx.patch && \
+    ./configure --prefix="/ffmpeg_build" --disable-examples --disable-unit-tests --enable-vp9-highbitdepth --as=yasm --enable-pic --enable-shared && \
+    make -j$(getconf _NPROCESSORS_ONLN) && \
+    make install && \
+    cd .. && \
+    rm -rf ~/libvpx_sources
+
+RUN mkdir ~/ffmpeg_sources && \
+    cd ~/ffmpeg_sources && \
+    curl -O -L https://ffmpeg.org/releases/ffmpeg-${FFMPEG_VERSION}.tar.gz && \
+    tar -xf ffmpeg-${FFMPEG_VERSION}.tar.gz && \
+    cd ffmpeg-${FFMPEG_VERSION} && \
+    PATH=~/bin:$PATH && \
+    PKG_CONFIG_PATH="/ffmpeg_build/lib/pkgconfig" ./configure --prefix="/ffmpeg_build" --extra-cflags="-I/ffmpeg_build/include" --extra-ldflags="-L/ffmpeg_build/lib" --enable-openssl --enable-libvpx --enable-shared --enable-pic --bindir="$HOME/bin" && \
+    make -j$(getconf _NPROCESSORS_ONLN) && \
+    make install && \
+    echo "/ffmpeg_build/lib/" >> /etc/ld.so.conf && \
+    ldconfig && \
+    rm -rf ~/ffmpeg_sources
+
+RUN curl -O -L https://github.com/ccache/ccache/releases/download/v${CCACHE_VERSION}/ccache-${CCACHE_VERSION}.tar.gz && \
+    tar -xf ccache-${CCACHE_VERSION}.tar.gz && \
+    cd ccache-${CCACHE_VERSION} && \
+    linux32 ./configure && \
+    make -j$(getconf _NPROCESSORS_ONLN) && \
+    make install && \
+    cd .. && \
+    rm -rf ccache-${CCACHE_VERSION}.tar.gz
+
+ENV PKG_CONFIG_PATH /usr/local/lib/pkgconfig:/ffmpeg_build/lib/pkgconfig
+ENV LDFLAGS -L/ffmpeg_build/lib
+ENV PATH "$HOME/bin:$PATH"

packages/opencv_python/Dockerfile.stage4

@@ -0,0 +1,24 @@
+FROM base
+
+ARG PYTHON_VERSION
+ARG VERSION
+ARG OUTPUT_DIR
+
+COPY . /repo
+
+ENV PKG_CONFIG_PATH /usr/local/lib/pkgconfig:/ffmpeg_build/lib/pkgconfig:/opt/Qt5.15.0/lib/pkgconfig
+ENV LDFLAGS -L/ffmpeg_build/lib -L/opt/Qt5.15.0/lib
+ENV LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/opt/Qt5.15.0/lib
+
+ENV CC=/opt/rh/devtoolset-10/root/usr/bin/gcc
+ENV FC=/opt/rh/devtoolset-10/root/usr/bin/gfortran5
+ENV CXX=/opt/rh/devtoolset-10/root/usr/bin/g++
+ENV LD=/opt/rh/devtoolset-10/root/usr/bin/g++
+
+RUN mkdir /opencv_python
+WORKDIR /opencv_python
+RUN select_python ${PYTHON_VERSION}
+RUN export MAKEFLAGS="-j$(nproc)" && export CI_BUILD=1 && /repo/armv7l_build_wheels.sh ${PYTHON_VERSION} ${VERSION}
+
+RUN mkdir -p ${OUTPUT_DIR} && \
+    cp build${PYTHON_VERSION}/wheelhouse/opencv_python*manylinux*armv7l.whl ${OUTPUT_DIR}

packages/opencv_python/armv7l_build_wheels.sh

@@ -10,13 +10,13 @@ build_wheel() (
     VER=$2
     mkdir build$PY_VER
     cd build$PY_VER
-    pip$PY_VER wheel --no-deps opencv_python==$VER
+    pip$PY_VER wheel --no-deps opencv_python==$VER --no-binary opencv_python --extra-index-url https://bjia56.github.io/armv7l-wheels/ --prefer-binary
 )
 
 test_wheel() (
     PY_VER=$1
     cd build$PY_VER
-    pip$PY_VER install wheelhouse/opencv_python*manylinux*armv7l.whl
+    pip$PY_VER install wheelhouse/opencv_python*manylinux*armv7l.whl --extra-index-url https://bjia56.github.io/armv7l-wheels/ --prefer-binary
     python$PY_VER -c "import cv2; print(cv2)"
 )
 

packages/opencv_python/build.yaml

@@ -1,10 +1,14 @@
 strategy: dockerfile
-source: ./Dockerfile
+source:
+- ./Dockerfile.stage1
+- ./Dockerfile.stage2
+- ./Dockerfile.stage3
+- ./Dockerfile.stage4
 disable: true
 
 python:
   versions:
   - "3.9"
 
 package:
-  versions: latest
+  versions: latest

packages/opencv_python/libvpx.patch

@@ -0,0 +1,13 @@
+diff --git a/build/make/configure.sh b/build/make/configure.sh
+index 9d3cd80cb..f64d38a7a 100644
+--- a/build/make/configure.sh
++++ b/build/make/configure.sh
+@@ -752,7 +752,7 @@ process_common_toolchain() {
+       aarch64*)
+         tgt_isa=arm64
+         ;;
+-      armv7*-hardfloat* | armv7*-gnueabihf | arm-*-gnueabihf)
++      armv7*-hardfloat* | armv7h* | armv7*-gnueabihf | arm-*-gnueabihf)
+         tgt_isa=armv7
+         float_abi=hard
+         ;;