chore: add pre-commit (#1731)

.github/workflows/build.yml

@@ -28,7 +28,18 @@ concurrency:
   cancel-in-progress: true
 
 jobs:
+  pre_commit:
+    name: Pre-commit checks
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.x"
+      - uses: pre-commit/[email protected]
+
   build_manylinux:
+    needs: [pre_commit]
     name: ${{ matrix.policy }}_${{ matrix.platform }}
     runs-on: ubuntu-22.04
     permissions:

.pre-commit-config.yaml

@@ -0,0 +1,44 @@
+repos:
+
+- repo: https://github.com/pre-commit/pre-commit-hooks
+  rev: v5.0.0
+  hooks:
+  - id: check-case-conflict
+  - id: check-merge-conflict
+  - id: check-yaml
+  - id: check-toml
+  - id: debug-statements
+  - id: end-of-file-fixer
+    exclude: docker/build_scripts/python_versions.json
+  - id: mixed-line-ending
+  - id: trailing-whitespace
+
+- repo: https://github.com/shellcheck-py/shellcheck-py
+  rev: v0.10.0.1
+  hooks:
+  - id: shellcheck
+
+- repo: https://github.com/astral-sh/ruff-pre-commit
+  rev: v0.8.0
+  hooks:
+  - id: ruff
+    args: ["--fix", "--show-fixes", "--target-version=py37"]
+    exclude: ^tools/.*.py|docker/build_scripts/manylinux-interpreters.py$
+  - id: ruff-format
+    args: ["--target-version=py37"]
+    exclude: ^tools/.*.py|docker/build_scripts/manylinux-interpreters.py$
+  - id: ruff
+    name: ruff on python 3.12 tools
+    files: ^tools/.*.py|docker/build_scripts/manylinux-interpreters.py$
+    args: ["--fix", "--show-fixes", "--target-version=py312"]
+  - id: ruff-format
+    name: ruff-format on python 3.12 tools
+    files: ^tools/.*.py|docker/build_scripts/manylinux-interpreters.py$
+    args: ["--target-version=py312"]
+
+- repo: https://github.com/codespell-project/codespell
+  rev: v2.3.0
+  hooks:
+  - id: codespell
+    args: ["-w"]
+    exclude: ^docker/build_scripts/.*pubkey.*.txt$

build.sh

@@ -12,22 +12,15 @@ export POLICY
 export PLATFORM
 
 # get docker default multiarch image prefix for PLATFORM
-if [ "${PLATFORM}" == "x86_64" ]; then
-	GOARCH="amd64"
-elif [ "${PLATFORM}" == "i686" ]; then
-	GOARCH="386"
-elif [ "${PLATFORM}" == "aarch64" ]; then
-	GOARCH="arm64"
-elif [ "${PLATFORM}" == "ppc64le" ]; then
-	GOARCH="ppc64le"
-elif [ "${PLATFORM}" == "s390x" ]; then
-	GOARCH="s390x"
-elif [ "${PLATFORM}" == "armv7l" ]; then
-	GOARCH="arm/v7"
-else
-	echo "Unsupported platform: '${PLATFORM}'"
-	exit 1
-fi
+case "${PLATFORM}" in
+	x86_64) GOARCH="amd64";;
+	i686) GOARCH="386";;
+	aarch64) GOARCH="arm64";;
+	ppc64le) GOARCH="ppc64le";;
+	s390x) GOARCH="s390x";;
+	armv7l) GOARCH="arm/v7";;
+	*) echo "Unsupported platform: '${PLATFORM}'"; exit 1;;
+esac
 
 # setup BASEIMAGE and its specific properties
 if [ "${POLICY}" == "manylinux2014" ]; then
@@ -63,45 +56,45 @@ export DEVTOOLSET_ROOTPATH
 export PREPEND_PATH
 export LD_LIBRARY_PATH_ARG
 
-BUILD_ARGS_COMMON="
-	--platform=linux/${GOARCH}
+BUILD_ARGS_COMMON=(
+	"--platform=linux/${GOARCH}"
 	--build-arg POLICY --build-arg PLATFORM --build-arg BASEIMAGE
 	--build-arg DEVTOOLSET_ROOTPATH --build-arg PREPEND_PATH --build-arg LD_LIBRARY_PATH_ARG
-	--rm -t quay.io/pypa/${POLICY}_${PLATFORM}:${COMMIT_SHA}
+	--rm -t "quay.io/pypa/${POLICY}_${PLATFORM}:${COMMIT_SHA}"
 	-f docker/Dockerfile docker/
-"
+)
 
 if [ "${CI:-}" == "true" ]; then
 	# Force plain output on CI
-	BUILD_ARGS_COMMON="--progress plain ${BUILD_ARGS_COMMON}"
+	BUILD_ARGS_COMMON=(--progress plain "${BUILD_ARGS_COMMON[@]}")
 	# Workaround issue on ppc64le
-	if [ ${PLATFORM} == "ppc64le" ] && [ "${MANYLINUX_BUILD_FRONTEND}" == "docker" ]; then
-		BUILD_ARGS_COMMON="--network host ${BUILD_ARGS_COMMON}"
+	if [ "${PLATFORM}" == "ppc64le" ] && [ "${MANYLINUX_BUILD_FRONTEND}" == "docker" ]; then
+		BUILD_ARGS_COMMON=(--network host "${BUILD_ARGS_COMMON[@]}")
 	fi
 fi
 
 USE_LOCAL_CACHE=0
 if [ "${MANYLINUX_BUILD_FRONTEND}" == "docker" ]; then
-	docker build ${BUILD_ARGS_COMMON}
+	docker build "${BUILD_ARGS_COMMON[@]}"
 elif [ "${MANYLINUX_BUILD_FRONTEND}" == "podman" ]; then
-	podman build ${BUILD_ARGS_COMMON}
+	podman build "${BUILD_ARGS_COMMON[@]}"
 elif [ "${MANYLINUX_BUILD_FRONTEND}" == "docker-buildx" ]; then
 	USE_LOCAL_CACHE=1
 	docker buildx build \
 		--load \
-		--cache-from=type=local,src=$(pwd)/.buildx-cache-${POLICY}_${PLATFORM} \
-		--cache-to=type=local,dest=$(pwd)/.buildx-cache-staging-${POLICY}_${PLATFORM},mode=max \
-		${BUILD_ARGS_COMMON}
+		"--cache-from=type=local,src=$(pwd)/.buildx-cache-${POLICY}_${PLATFORM}" \
+		"--cache-to=type=local,dest=$(pwd)/.buildx-cache-staging-${POLICY}_${PLATFORM},mode=max" \
+		"${BUILD_ARGS_COMMON[@]}"
 else
 	echo "Unsupported build frontend: '${MANYLINUX_BUILD_FRONTEND}'"
 	exit 1
 fi
 
-docker run --rm -v $(pwd)/tests:/tests:ro quay.io/pypa/${POLICY}_${PLATFORM}:${COMMIT_SHA} /tests/run_tests.sh
+docker run --rm -v "$(pwd)/tests:/tests:ro" "quay.io/pypa/${POLICY}_${PLATFORM}:${COMMIT_SHA}" /tests/run_tests.sh
 
 if [ ${USE_LOCAL_CACHE} -ne 0 ]; then
-	if [ -d $(pwd)/.buildx-cache-${POLICY}_${PLATFORM} ]; then
-		rm -rf $(pwd)/.buildx-cache-${POLICY}_${PLATFORM}
+	if [ -d "$(pwd)/.buildx-cache-${POLICY}_${PLATFORM}" ]; then
+		rm -rf "$(pwd)/.buildx-cache-${POLICY}_${PLATFORM}"
 	fi
-	mv $(pwd)/.buildx-cache-staging-${POLICY}_${PLATFORM} $(pwd)/.buildx-cache-${POLICY}_${PLATFORM}
+	mv "$(pwd)/.buildx-cache-staging-${POLICY}_${PLATFORM}" "$(pwd)/.buildx-cache-${POLICY}_${PLATFORM}"
 fi

deploy.sh

@@ -12,26 +12,26 @@ fi
 set -x
 
 TAG="quay.io/pypa/${POLICY}_${PLATFORM}"
-COMMIT_ABBREV_SHA=$(git show -s --format=%h ${COMMIT_SHA})
-COMMIT_DATE=$(git show -s --format=%cd --date=short ${COMMIT_SHA})
+COMMIT_ABBREV_SHA=$(git show -s --format=%h "${COMMIT_SHA}")
+COMMIT_DATE=$(git show -s --format=%cd --date=short "${COMMIT_SHA}")
 BUILD_ID=${COMMIT_DATE}-${COMMIT_ABBREV_SHA}
 # Dependabot does not work with the BUILD_ID format
 # Use a version like tag
-if $(git rev-parse --is-shallow-repository); then
+if eval "$(git rev-parse --is-shallow-repository)"; then
   git fetch --unshallow
 fi
-BUILD_NUMBER=$(git rev-list --since=${COMMIT_DATE}T00:00:00Z --first-parent --count ${COMMIT_SHA})
+BUILD_NUMBER=$(git rev-list "--since=${COMMIT_DATE}T00:00:00Z" --first-parent --count "${COMMIT_SHA}")
 BUILD_ID2=${COMMIT_DATE//-/.}-${BUILD_NUMBER}
 
-docker tag ${TAG}:${COMMIT_SHA} ${TAG}:${BUILD_ID}
-docker tag ${TAG}:${COMMIT_SHA} ${TAG}:${BUILD_ID2}
-docker tag ${TAG}:${COMMIT_SHA} ${TAG}:latest
+docker tag "${TAG}:${COMMIT_SHA}" "${TAG}:${BUILD_ID}"
+docker tag "${TAG}:${COMMIT_SHA}" "${TAG}:${BUILD_ID2}"
+docker tag "${TAG}:${COMMIT_SHA}" "${TAG}:latest"
 
 set +x
 
 if [ $DRY_RUN -eq 0 ]; then
-  docker login -u $QUAY_USERNAME -p $QUAY_PASSWORD quay.io
-  docker push ${TAG}:${BUILD_ID}
-  docker push ${TAG}:${BUILD_ID2}
-  docker push ${TAG}:latest
+  docker login -u "${QUAY_USERNAME}" -p "${QUAY_PASSWORD}" quay.io
+  docker push "${TAG}:${BUILD_ID}"
+  docker push "${TAG}:${BUILD_ID2}"
+  docker push "${TAG}:latest"
 fi

docker/build_scripts/ambv-pubkey.txt

@@ -1008,4 +1008,3 @@ xEJP1pajaM6+D8oa0fqFYgcAmXeZ6fBg531VOuNwWdavel8vSqx/jik4b8SS4ygK
 Xw8b
 =4+Ub
 -----END PGP PUBLIC KEY BLOCK-----
-

docker/build_scripts/build-cpython.sh

@@ -8,7 +8,8 @@ set -exuo pipefail
 MY_DIR=$(dirname "${BASH_SOURCE[0]}")
 
 # Get build utilities
-source $MY_DIR/build_utils.sh
+# shellcheck source-path=SCRIPTDIR
+source "${MY_DIR}/build_utils.sh"
 
 
 CPYTHON_VERSION=$1
@@ -20,24 +21,24 @@ function pyver_dist_dir {
 	# Thus:
 	# 3.2.1   -> 3.2.1
 	# 3.7.0b4 -> 3.7.0
-	echo $1 | awk -F "." '{printf "%d.%d.%d", $1, $2, $3}'
+	echo "$1" | awk -F "." '{printf "%d.%d.%d", $1, $2, $3}'
 }
 
-CPYTHON_DIST_DIR=$(pyver_dist_dir ${CPYTHON_VERSION})
-fetch_source Python-${CPYTHON_VERSION}.tar.xz ${CPYTHON_DOWNLOAD_URL}/${CPYTHON_DIST_DIR}
-fetch_source Python-${CPYTHON_VERSION}.tar.xz.asc ${CPYTHON_DOWNLOAD_URL}/${CPYTHON_DIST_DIR}
-gpg --import ${MY_DIR}/cpython-pubkeys.txt
-gpg --verify Python-${CPYTHON_VERSION}.tar.xz.asc
-tar -xJf Python-${CPYTHON_VERSION}.tar.xz
-pushd Python-${CPYTHON_VERSION}
+CPYTHON_DIST_DIR=$(pyver_dist_dir "${CPYTHON_VERSION}")
+fetch_source "Python-${CPYTHON_VERSION}.tar.xz" "${CPYTHON_DOWNLOAD_URL}/${CPYTHON_DIST_DIR}"
+fetch_source "Python-${CPYTHON_VERSION}.tar.xz.asc" "${CPYTHON_DOWNLOAD_URL}/${CPYTHON_DIST_DIR}"
+gpg --import "${MY_DIR}/cpython-pubkeys.txt"
+gpg --verify "Python-${CPYTHON_VERSION}.tar.xz.asc"
+tar -xJf "Python-${CPYTHON_VERSION}.tar.xz"
+pushd "Python-${CPYTHON_VERSION}"
 PREFIX="/opt/_internal/cpython-${CPYTHON_VERSION}"
-mkdir -p ${PREFIX}/lib
+mkdir -p "${PREFIX}/lib"
 CFLAGS_EXTRA=""
-CONFIGURE_ARGS="--disable-shared --with-ensurepip=no"
+CONFIGURE_ARGS=(--disable-shared --with-ensurepip=no)
 
 if [ "${2:-}" == "nogil" ]; then
 	PREFIX="${PREFIX}-nogil"
-	CONFIGURE_ARGS="${CONFIGURE_ARGS} --disable-gil"
+	CONFIGURE_ARGS+=(--disable-gil)
 fi
 
 if [ "${CPYTHON_VERSION}" == "3.6.15" ]; then
@@ -51,7 +52,7 @@ if [ "${AUDITWHEEL_POLICY}" == "manylinux2014" ] ; then
 fi
 
 if [ "${BASE_POLICY}_${AUDITWHEEL_ARCH}" == "musllinux_armv7l" ]; then
-	CONFIGURE_ARGS="${CONFIGURE_ARGS} --build=arm-linux-musleabihf"
+	CONFIGURE_ARGS+=(--build=arm-linux-musleabihf)
 fi
 
 SQLITE_PREFIX=$(find /opt/_internal -maxdepth 1 -name 'sqlite*')
@@ -64,10 +65,10 @@ fi
 
 OPENSSL_PREFIX=$(find /opt/_internal -maxdepth 1 -name 'openssl*')
 if [ "${OPENSSL_PREFIX}" != "" ]; then
-	CONFIGURE_ARGS="${CONFIGURE_ARGS} --with-openssl=${OPENSSL_PREFIX}"
+	CONFIGURE_ARGS+=("--with-openssl=${OPENSSL_PREFIX}")
 	case "${CPYTHON_VERSION}" in
 		3.8.*|3.9.*) export LD_RUN_PATH=${OPENSSL_PREFIX}/lib;;
-		*) CONFIGURE_ARGS="${CONFIGURE_ARGS} --with-openssl-rpath=auto";;
+		*) CONFIGURE_ARGS+=(--with-openssl-rpath=auto);;
 	esac
 fi
 
@@ -82,18 +83,18 @@ fi
 ./configure \
 	CFLAGS_NODIST="${MANYLINUX_CFLAGS} ${MANYLINUX_CPPFLAGS} ${CFLAGS_EXTRA}" \
 	LDFLAGS_NODIST="${MANYLINUX_LDFLAGS}" \
-	--prefix=${PREFIX} ${CONFIGURE_ARGS} > /dev/null
+	"--prefix=${PREFIX}" "${CONFIGURE_ARGS[@]}" > /dev/null
 make > /dev/null
 make install > /dev/null
 popd
-rm -rf Python-${CPYTHON_VERSION} Python-${CPYTHON_VERSION}.tgz Python-${CPYTHON_VERSION}.tgz.asc
+rm -rf "Python-${CPYTHON_VERSION}" "Python-${CPYTHON_VERSION}.tgz" "Python-${CPYTHON_VERSION}.tgz.asc"
 
 if [ "${OPENSSL_PREFIX}" != "" ]; then
-	rm -rf ${OPENSSL_PREFIX}/bin ${OPENSSL_PREFIX}/include ${OPENSSL_PREFIX}/lib/pkgconfig ${OPENSSL_PREFIX}/lib/*.so
+	rm -rf "${OPENSSL_PREFIX:?}/bin" "${OPENSSL_PREFIX}/include" "${OPENSSL_PREFIX}/lib/pkgconfig" "${OPENSSL_PREFIX}/lib/*.so"
 fi
 
 # We do not need precompiled .pyc and .pyo files.
-clean_pyc ${PREFIX}
+clean_pyc "${PREFIX}"
 
 # Strip ELF files found in ${PREFIX}
-strip_ ${PREFIX}
+strip_ "${PREFIX}"

docker/build_scripts/build-curl.sh

@@ -8,12 +8,13 @@ set -exuo pipefail
 MY_DIR=$(dirname "${BASH_SOURCE[0]}")
 
 # Get build utilities
-source $MY_DIR/build_utils.sh
+# shellcheck source-path=SCRIPTDIR
+source "${MY_DIR}/build_utils.sh"
 
 # Install a more recent curl
-check_var ${CURL_ROOT}
-check_var ${CURL_HASH}
-check_var ${CURL_DOWNLOAD_URL}
+check_var "${CURL_ROOT}"
+check_var "${CURL_HASH}"
+check_var "${CURL_DOWNLOAD_URL}"
 
 # Only needed on manylinux2014
 if [ "${AUDITWHEEL_POLICY}" != "manylinux2014" ]; then
@@ -30,15 +31,15 @@ fi
 SO_COMPAT=4
 PREFIX=/opt/_internal/curl-${SO_COMPAT}
 
-fetch_source ${CURL_ROOT}.tar.gz ${CURL_DOWNLOAD_URL}
-check_sha256sum ${CURL_ROOT}.tar.gz ${CURL_HASH}
-tar -xzf ${CURL_ROOT}.tar.gz
-pushd ${CURL_ROOT}
+fetch_source "${CURL_ROOT}.tar.gz" "${CURL_DOWNLOAD_URL}"
+check_sha256sum "${CURL_ROOT}.tar.gz" "${CURL_HASH}"
+tar -xzf "${CURL_ROOT}.tar.gz"
+pushd "${CURL_ROOT}"
 ./configure --prefix=${PREFIX} --disable-static --without-libpsl --with-openssl CPPFLAGS="${MANYLINUX_CPPFLAGS}" CFLAGS="${MANYLINUX_CFLAGS}" CXXFLAGS="${MANYLINUX_CXXFLAGS}" LDFLAGS="${MANYLINUX_LDFLAGS} -Wl,-rpath=\$(LIBRPATH)" > /dev/null
 make > /dev/null
 make install > /dev/null
 popd
-rm -rf ${CURL_ROOT} ${CURL_ROOT}.tar.gz ${PREFIX}/share/man
+rm -rf "${CURL_ROOT}" "${CURL_ROOT}.tar.gz" ${PREFIX}/share/man
 
 if [ ! -f ${PREFIX}/lib/libcurl.so.${SO_COMPAT} ]; then
 	echo "please update SO_COMPAT"

docker/build_scripts/build-git.sh

@@ -8,7 +8,8 @@ set -exuo pipefail
 MY_DIR=$(dirname "${BASH_SOURCE[0]}")
 
 # Get build utilities
-source $MY_DIR/build_utils.sh
+# shellcheck source-path=SCRIPTDIR
+source "${MY_DIR}/build_utils.sh"
 
 if [ "${BASE_POLICY}" == "musllinux" ]; then
 	export NO_REGEX=NeedsStartEnd
@@ -22,22 +23,23 @@ if [ -d /opt/_internal ]; then
 	CURL_PREFIX=$(find /opt/_internal -maxdepth 1 -name 'curl-*')
 	if [ "${CURL_PREFIX}" != "" ]; then
 		export CURLDIR=${CURL_PREFIX}
-		export CURL_LDFLAGS="-Wl,-rpath=${CURL_PREFIX}/lib $(${CURL_PREFIX}/bin/curl-config --libs)"
+		CURL_LDFLAGS="-Wl,-rpath=${CURL_PREFIX}/lib $("${CURL_PREFIX}/bin/curl-config" --libs)"
+		export CURL_LDFLAGS
 	fi
 fi
 
 # Install newest git
-check_var ${GIT_ROOT}
-check_var ${GIT_HASH}
-check_var ${GIT_DOWNLOAD_URL}
-
-fetch_source ${GIT_ROOT}.tar.gz ${GIT_DOWNLOAD_URL}
-check_sha256sum ${GIT_ROOT}.tar.gz ${GIT_HASH}
-tar -xzf ${GIT_ROOT}.tar.gz
-pushd ${GIT_ROOT}
+check_var "${GIT_ROOT}"
+check_var "${GIT_HASH}"
+check_var "${GIT_DOWNLOAD_URL}"
+
+fetch_source "${GIT_ROOT}.tar.gz" "${GIT_DOWNLOAD_URL}"
+check_sha256sum "${GIT_ROOT}.tar.gz" "${GIT_HASH}"
+tar -xzf "${GIT_ROOT}.tar.gz"
+pushd "${GIT_ROOT}"
 make install prefix=/usr/local NO_GETTEXT=1 NO_TCLTK=1 DESTDIR=/manylinux-rootfs CPPFLAGS="${MANYLINUX_CPPFLAGS}" CFLAGS="${MANYLINUX_CFLAGS}" CXXFLAGS="${MANYLINUX_CXXFLAGS}" LDFLAGS="${MANYLINUX_LDFLAGS}"
 popd
-rm -rf ${GIT_ROOT} ${GIT_ROOT}.tar.gz
+rm -rf "${GIT_ROOT}" "${GIT_ROOT}.tar.gz"
 
 
 # Strip what we can