From 258e1e4d7e99bb4fdb3127a752bfc19c58ec0ebc Mon Sep 17 00:00:00 2001 From: Jason Wallace Date: Wed, 24 May 2023 16:29:53 +0200 Subject: [PATCH] Build uStreamer Debian package with Janus plugin (#6) Resolves https://github.com/tiny-pilot/ustreamer-debian/issues/3 This PR builds a uStreamer Debian package (with Janus plugin) in CI. The reason why we couldn't just use the [official(?) uStreamer Debian package](https://salsa.debian.org/reedy/ustreamer/) is because it doesn't compile the Janus plugin (which we need for WebRTC support). Now we can avoid building uStreamer from source every time TinyPilot is installed or updated. You can test the uStreamer Debian package on a device via this [scratch TinyPilot Pro build bundle](https://app.circleci.com/pipelines/github/tiny-pilot/tinypilot-pro/2801/workflows/ba68bf47-f01c-4240-a008-6456f956dca4/jobs/20659/artifacts). Notes: * We compile the uStreamer Debian package for both [`armhf` and `amd64` architectures](https://github.com/tiny-pilot/ustreamer-debian/pull/6/files#diff-78a8a19706dbd2a4425dd72bdab0502ed7a2cef16365ab7030a5a0588927bf47R33-R38) because the `amd64` version will be used when testing the uStreamer Ansible role in CI (using molecule) and the `armhf` version will be used when installing TinyPilot on a device. * FYI, compiling for multiple architectures produces Debian packages in the following [directory structure](https://github.com/tiny-pilot/ustreamer-debian/pull/6/files#diff-78a8a19706dbd2a4425dd72bdab0502ed7a2cef16365ab7030a5a0588927bf47R46-R49): * `/build/linux_arm_v7/*.deb` * `/build/linux_amd64/*.deb` As opposed to just `/build/*.deb` when compiling for a single architecture. * Most of the [lintian code](https://github.com/tiny-pilot/ustreamer-debian/pull/6/files#diff-78a8a19706dbd2a4425dd72bdab0502ed7a2cef16365ab7030a5a0588927bf47R52-R80) is copy/pasted from [TinyPilot Community repo](https://github.com/tiny-pilot/tinypilot/blob/b984ab93a58533220ad6358831ce8e405810db8f/.circleci/continue_config.yml#L97-L122), besides the [loop to check multiple Debian packages](https://github.com/tiny-pilot/ustreamer-debian/pull/6/files#diff-78a8a19706dbd2a4425dd72bdab0502ed7a2cef16365ab7030a5a0588927bf47R71-R80). * Seeing as [uStreamer makes use of a simple `v${MAJOR}.${MINOR}` versioning schema](https://github.com/tiny-pilot/ustreamer/tags), I used that (without the `v` prefix) as the Debian package version with an added a timestamp revision number to allow for the Debian package to be updated even when the uStreamer version has stayed the same. The resulting uStreamer Debian package versioning schema being [`MAJOR.MINOR-YYYYMMDDhhmmss`](https://github.com/tiny-pilot/ustreamer-debian/pull/6/files#diff-dd2c0eb6ea5cfc6c4bd4eac30934e2d5746747af48fef6da689e85b752f39557R63) * Seeing as Docker doesn't support dynamic `WORKDIR` values based on a command's output, I stole these ([1](https://github.com/tiny-pilot/ustreamer-debian/pull/6/files#diff-dd2c0eb6ea5cfc6c4bd4eac30934e2d5746747af48fef6da689e85b752f39557R46-R70), [2](https://github.com/tiny-pilot/ustreamer-debian/pull/6/files#diff-dd2c0eb6ea5cfc6c4bd4eac30934e2d5746747af48fef6da689e85b752f39557R140-R146)) clever workarounds from Michael's [PR](https://github.com/tiny-pilot/tinypilot/pull/1352/files#diff-d3aed37eb2a4156ced425c7bcab79741d1234d58d182d91099b780c3c3136ce4R31-R53). * The [copyright notice I've used here](https://github.com/tiny-pilot/ustreamer-debian/pull/6/files#diff-eb1289be1c2cdfc905cecdba0c6810aa4d987d52e34f9dee681c1ff9cd69c34fR1-R2) was given to me my Michael via email * When we built uStreamer on the device, we technically [only ever ran `make`](https://github.com/tiny-pilot/ansible-role-ustreamer/blob/b1017f8f4436071b5d8dcf812ee351fc846e1fa6/tasks/main.yml#L153-L161) and never `make install`. This Debian package builds using `make && make install` which results in the binary being moved to `bin/ustreamer`. I thought running the `make install` is a good thing, so I just [added a symlink (`/opt/ustreamer/ustreamer -> bin/ustreamer`)](https://github.com/tiny-pilot/ustreamer-debian/pull/6/files#diff-fe915a3611d3df024aa33709affc37ff7a049adc629d69f2f0fe0dd063ad155fR1-R2) to maintain TinyPilot's path to uStreamer. Helpful resources on creating Debian packages: * https://vincent.bernat.ch/en/blog/2019-pragmatic-debian-packaging * https://github.com/vincentbernat/pragmatic-debian-packages/ * https://salsa.debian.org/reedy/ustreamer/ * https://github.com/tiny-pilot/tinypilot/pull/1352/files Review on CodeApprove --- .circleci/config.yml | 60 ++++++++++++++ .lintianignore | 4 + Dockerfile | 148 +++++++++++++++++++++++++++++++++++ debian/compat | 1 + debian/rules | 17 ++++ debian/ustreamer.copyright | 2 + debian/ustreamer.install | 3 + debian/ustreamer.links | 2 + debian/ustreamer.preinstall | 10 +++ dev-scripts/build-debian-pkg | 6 +- 10 files changed, 250 insertions(+), 3 deletions(-) create mode 100644 .lintianignore create mode 100644 Dockerfile create mode 100644 debian/compat create mode 100755 debian/rules create mode 100644 debian/ustreamer.copyright create mode 100755 debian/ustreamer.install create mode 100644 debian/ustreamer.links create mode 100755 debian/ustreamer.preinstall diff --git a/.circleci/config.yml b/.circleci/config.yml index 7deafac..ee6c6ea 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -22,8 +22,68 @@ jobs: - run: name: Run static analysis on bash scripts command: ./dev-scripts/check-bash + build_debian_package: + docker: + - image: cimg/base:stable + steps: + - checkout + - setup_remote_docker: + version: 20.10.11 + docker_layer_caching: true + - run: + name: Enable multiarch builds with QEMU + command: ./dev-scripts/enable-multiarch-docker + - run: + name: Build Debian package + command: ./dev-scripts/build-debian-pkg "linux/arm/v7,linux/amd64" + - run: + name: Print Debian package contents + command: | + set -exu + while read -r file; do + dpkg --contents "${file}" + done < <(find . -name '*.deb') + - persist_to_workspace: + root: build + paths: + - ./*/*.deb + - store_artifacts: + path: build + lint_debian_package: + docker: + - image: cimg/base:2022.11-22.04 + steps: + - checkout + - attach_workspace: + at: ./ + - run: + name: Update apt packages + command: sudo apt-get update + - run: + name: Install lintian + command: sudo apt-get install -y lintian=2.114.0ubuntu1 + - run: + name: Print lintian version + command: lintian --version + - run: + name: Run lintian + command: | + set -exu + while read -r file; do + lintian \ + --check \ + --no-tag-display-limit \ + --suppress-tags-from-file .lintianignore \ + --no-cfg \ + --fail-on warning,error \ + "${file}" + done < <(find . -name '*.deb') workflows: test: jobs: - check_whitespace - check_bash + - build_debian_package + - lint_debian_package: + requires: + - build_debian_package diff --git a/.lintianignore b/.lintianignore new file mode 100644 index 0000000..e418aa0 --- /dev/null +++ b/.lintianignore @@ -0,0 +1,4 @@ +# Debian doesn't want packages to install to /opt, but it also doesn't give +# clear guidance on where they *should* go. It's too much churn at this point to +# change, so we're going to ignore this. +dir-or-file-in-opt diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..6907ef7 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,148 @@ +# syntax=docker/dockerfile:1.4 +# Enable here-documents: +# https://github.com/moby/buildkit/blob/master/frontend/dockerfile/docs/syntax.md#here-documents + +FROM debian:bullseye-20220328-slim AS build + +RUN set -x && \ + apt-get update && \ + DEBIAN_FRONTEND=noninteractive apt-get install -y \ + debhelper \ + dpkg-dev \ + devscripts \ + git \ + build-essential \ + wget \ + gnupg + +# Add bullseye-backports apt suite to later install janus dependency. +RUN cat | bash <<'EOF' +set -ex +# Add keyring. +wget \ + --output-document - \ + https://ftp-master.debian.org/keys/archive-key-11.asc | \ + gpg \ + --dearmor > \ + /usr/share/keyrings/bullseye-archive-keyring.gpg +# Add repository. +echo 'deb [signed-by=/usr/share/keyrings/bullseye-archive-keyring.gpg] http://deb.debian.org/debian bullseye-backports main' > \ + /etc/apt/sources.list.d/bullseye-backports.list +# Update package index. +apt-get update +EOF + +# Docker populates this value from the --platform argument. See +# https://docs.docker.com/build/building/multi-platform/ +ARG TARGETPLATFORM + +ARG PKG_NAME='ustreamer' +ARG PKG_VERSION='5.38' + +# This should be a timestamp, formatted `YYYYMMDDhhmmss`. That way the package +# manager always installs the most recently built package. +ARG PKG_BUILD_NUMBER + +# Docker's platform names don't match Debian's platform names, so we translate +# the platform name from the Docker version to the Debian version and save the +# result to a file so we can re-use it in later stages. +RUN cat | bash <<'EOF' +set -exu +case "${TARGETPLATFORM}" in + 'linux/amd64') + PKG_ARCH='amd64' + ;; + 'linux/arm/v7') + PKG_ARCH='armhf' + ;; + *) + echo "Unrecognized target platform: ${TARGETPLATFORM}" >&2 + exit 1 +esac +echo "${PKG_ARCH}" > /tmp/pkg-arch +echo "${PKG_NAME}-${PKG_VERSION}-${PKG_BUILD_NUMBER}-${PKG_ARCH}" > /tmp/pkg-id +EOF + +# We ultimately need the directory name to be the package ID, but there's no +# way to specify a dynamic value in Docker's WORKDIR command, so we use a +# placeholder directory name to assemble the Debian package and then rename the +# directory to its package ID name in the final stages of packaging. +WORKDIR /build/placeholder-pkg-id + +RUN git \ + clone \ + --branch "v${PKG_VERSION}" \ + --depth 1 \ + https://github.com/tiny-pilot/ustreamer.git \ + . + +COPY debian debian + +WORKDIR debian + +RUN set -x && \ + PKG_ARCH="$(cat /tmp/pkg-arch)" && \ + set -u && \ + cat >control < +Build-Depends: debhelper (>= 11), + dh-exec, + libevent-dev, + libjpeg-dev, + uuid-dev, + libbsd-dev, + janus-dev, + libasound2-dev, + libspeex-dev, + libspeexdsp-dev, + libopus-dev + +Package: ${PKG_NAME} +Architecture: ${PKG_ARCH} +Depends: \${shlibs:Depends}, \${misc:Depends} +Homepage: https://github.com/tiny-pilot/ustreamer +Description: Lightweight and fast MJPEG-HTTP streamer + µStreamer is a lightweight and very quick server to stream MJPEG video + from any V4L2 device to the net. All new browsers have native + support of this video format, as well as most video players such as + mplayer, VLC etc. µStreamer is a part of the PiKVM project designed to + stream VGA and HDMI screencast hardware data with the highest resolution + and FPS possible. +EOF + +RUN cat >changelog < $(date '+%a, %d %b %Y %H:%M:%S %z') +EOF + +# Install build dependencies based on Debian control file. +RUN mk-build-deps \ + --tool 'apt-get --option Debug::pkgProblemResolver=yes --no-install-recommends -qqy' \ + --install \ + --remove \ + control + +# Allow Janus C header files to be included when compiling third-party plugins. +# https://github.com/tiny-pilot/ansible-role-tinypilot/issues/192 +RUN sed \ + --in-place \ + 's/^#include "refcount\.h"$/#include "\.\.\/refcount\.h"/g' \ + /usr/include/janus/plugins/plugin.h + +# Rename the placeholder build directory to the final package ID. +WORKDIR /build +RUN set -x && \ + PKG_ID="$(cat /tmp/pkg-id)" && \ + mv placeholder-pkg-id "${PKG_ID}" && \ + cd "${PKG_ID}" && \ + DH_VERBOSE=1 dpkg-buildpackage --build=binary + +FROM scratch as artifact + +COPY --from=build "/build/*.deb" ./ diff --git a/debian/compat b/debian/compat new file mode 100644 index 0000000..b4de394 --- /dev/null +++ b/debian/compat @@ -0,0 +1 @@ +11 diff --git a/debian/rules b/debian/rules new file mode 100755 index 0000000..4963524 --- /dev/null +++ b/debian/rules @@ -0,0 +1,17 @@ +#!/usr/bin/make -f + +# Set all dpkg-architecture variables (i.e., ${DEB_HOST_MULTIARCH}, etc.). +include /usr/share/dpkg/architecture.mk + +# Prevent debhelper from generating an extra package with debug symbols. +export DEB_BUILD_OPTIONS=noddebs + +export PREFIX=/opt/ustreamer + +%: + dh $@ + +override_dh_usrlocal: + +override_dh_auto_build: + dh_auto_build -- WITH_JANUS=1 diff --git a/debian/ustreamer.copyright b/debian/ustreamer.copyright new file mode 100644 index 0000000..5c82c5b --- /dev/null +++ b/debian/ustreamer.copyright @@ -0,0 +1,2 @@ +Copyright (C) 2018-2022 by Maxim Devaev mdevaev@gmail.com +Copyright (c) 2023 TinyPilot, LLC diff --git a/debian/ustreamer.install b/debian/ustreamer.install new file mode 100755 index 0000000..f5e30ba --- /dev/null +++ b/debian/ustreamer.install @@ -0,0 +1,3 @@ +#!/usr/bin/dh-exec +janus/libjanus_ustreamer.so usr/lib/${DEB_HOST_MULTIARCH}/janus/plugins/ +LICENSE opt/ustreamer/ diff --git a/debian/ustreamer.links b/debian/ustreamer.links new file mode 100644 index 0000000..1e18840 --- /dev/null +++ b/debian/ustreamer.links @@ -0,0 +1,2 @@ +opt/ustreamer/bin/ustreamer opt/ustreamer/ustreamer +opt/ustreamer/bin/ustreamer-dump opt/ustreamer/ustreamer-dump diff --git a/debian/ustreamer.preinstall b/debian/ustreamer.preinstall new file mode 100755 index 0000000..a412cf6 --- /dev/null +++ b/debian/ustreamer.preinstall @@ -0,0 +1,10 @@ +#!/bin/bash + +# Exit script on first failure. +set -e + +# If a .git directory exists, the previous version was installed with the legacy +# installer, so wipe the install location. +if [[ -d /opt/ustreamer/.git ]]; then + rm -rf /opt/ustreamer +fi diff --git a/dev-scripts/build-debian-pkg b/dev-scripts/build-debian-pkg index aafc594..55525d2 100755 --- a/dev-scripts/build-debian-pkg +++ b/dev-scripts/build-debian-pkg @@ -25,8 +25,8 @@ set -u readonly BUILD_TARGETS="${1:-linux/arm/v7,linux/amd64}" -PKG_VERSION="$(date '+%Y%m%d%H%M%S')" -readonly PKG_VERSION +PKG_BUILD_NUMBER="$(date '+%Y%m%d%H%M%S')" +readonly PKG_BUILD_NUMBER # Use plain Docker build progress output when we're running in CI. DOCKER_PROGRESS='auto' @@ -38,7 +38,7 @@ readonly DOCKER_PROGRESS DOCKER_BUILDKIT=1 docker buildx build \ --file Dockerfile \ --platform "${BUILD_TARGETS}" \ - --build-arg PKG_VERSION="${PKG_VERSION}" \ + --build-arg PKG_BUILD_NUMBER="${PKG_BUILD_NUMBER}" \ --target=artifact \ --progress="${DOCKER_PROGRESS}" \ --output "type=local,dest=$(pwd)/build/" \