From 59d3a1a44da6a1fe221579db5f122755e83cdcab Mon Sep 17 00:00:00 2001 From: Emmanuel Evbuomwan Date: Mon, 9 Dec 2024 13:33:37 +0100 Subject: [PATCH] ci: multi-python docker image for GHA matrix --- .dockerignore | 3 ++- .github/workflows/container-smoke-test.yml | 6 +++++- .github/workflows/tests.yml | 13 +++++++++++-- GNUmakefile | 12 ++++++++---- container/Dockerfile.dev | 4 +++- container/compose.yml | 1 + mypy.ini | 2 +- pyproject.toml | 5 ++--- requirements/requirements-dev.txt | 13 +++++-------- requirements/requirements-typing.txt | 9 ++++----- requirements/requirements.txt | 7 +++---- website/README.rst | 2 +- 12 files changed, 46 insertions(+), 31 deletions(-) diff --git a/.dockerignore b/.dockerignore index 4b946a334..0fc564b64 100644 --- a/.dockerignore +++ b/.dockerignore @@ -4,7 +4,8 @@ # Include source directories and files required for building. !go -!src +!src/karapace +!src/schema_registry !requirements/*.txt !README.rst !LICENSE diff --git a/.github/workflows/container-smoke-test.yml b/.github/workflows/container-smoke-test.yml index 25cdc08c9..89d6d4a2d 100644 --- a/.github/workflows/container-smoke-test.yml +++ b/.github/workflows/container-smoke-test.yml @@ -8,6 +8,9 @@ on: jobs: smoke-test-container: runs-on: ubuntu-latest + strategy: + matrix: + python-version: [ '3.10', '3.11', '3.12' ] env: BUILDKIT_PROGRESS: plain steps: @@ -18,7 +21,7 @@ jobs: fetch-depth: 0 - name: Install requirements - run: make install + run: make install-dev - name: Resolve Karapace version run: | @@ -32,6 +35,7 @@ jobs: - name: Run container run: make start-karapace-docker-resources env: + PYTHON_VERSION: ${{ matrix.python-version }} KARAPACE_VERSION: ${{ env.KARAPACE_VERSION }} RUNNER_UID: ${{ env.RUNNER_UID }} RUNNER_GID: ${{ env.RUNNER_GID }} diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 60df28286..df5b5683b 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -20,7 +20,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: [ '3.9', '3.10', '3.11', '3.12' ] + python-version: [ '3.10', '3.11', '3.12' ] env: PYTEST_ADDOPTS: >- --log-dir=/tmp/ci-logs @@ -40,14 +40,21 @@ jobs: with: go-version: '1.21.0' + - name: Install requirements + run: make install-dev + - name: Resolve Karapace version - run: echo KARAPACE_VERSION=4.1.1.dev44+gac20eeed.d20241205 >> $GITHUB_ENV + run: | + source ./venv/bin/activate + KARAPACE_VERSION=$(python -c "from karapace import version; print(version.__version__)") + echo KARAPACE_VERSION=$KARAPACE_VERSION >> $GITHUB_ENV - run: echo "RUNNER_UID=$(id -u)" >> $GITHUB_ENV - run: echo "RUNNER_GID=$(id -g)" >> $GITHUB_ENV - run: make unit-tests-in-docker env: + PYTHON_VERSION: ${{ matrix.python-version }} KARAPACE_VERSION: ${{ env.KARAPACE_VERSION }} RUNNER_UID: ${{ env.RUNNER_UID }} RUNNER_GID: ${{ env.RUNNER_GID }} @@ -56,6 +63,7 @@ jobs: - run: make e2e-tests-in-docker env: + PYTHON_VERSION: ${{ matrix.python-version }} KARAPACE_VERSION: ${{ env.KARAPACE_VERSION }} RUNNER_UID: ${{ env.RUNNER_UID }} RUNNER_GID: ${{ env.RUNNER_GID }} @@ -64,6 +72,7 @@ jobs: - run: make integration-tests-in-docker env: + PYTHON_VERSION: ${{ matrix.python-version }} KARAPACE_VERSION: ${{ env.KARAPACE_VERSION }} RUNNER_UID: ${{ env.RUNNER_UID }} RUNNER_GID: ${{ env.RUNNER_GID }} diff --git a/GNUmakefile b/GNUmakefile index 032def928..554a44a8c 100644 --- a/GNUmakefile +++ b/GNUmakefile @@ -3,7 +3,7 @@ SHELL := /usr/bin/env bash VENV_DIR ?= $(CURDIR)/venv PIP ?= pip3 --disable-pip-version-check --no-input --require-virtualenv PYTHON ?= python3 -PYTHON_VERSION ?= 3.9 +PYTHON_VERSION ?= 3.10 DOCKER_COMPOSE ?= docker compose KARAPACE-CLI ?= $(DOCKER_COMPOSE) -f container/compose.yml run --rm karapace-cli @@ -105,6 +105,10 @@ schema: pin-requirements: docker run -e CUSTOM_COMPILE_COMMAND='make pin-requirements' -it -v .:/karapace --security-opt label=disable python:$(PYTHON_VERSION)-bullseye /bin/bash -c "$(PIN_VERSIONS_COMMAND)" +.PHONY: stop-karapace-docker-resources +stop-karapace-docker-resources: + $(DOCKER_COMPOSE) -f container/compose.yml down -v --remove-orphans + .PHONY: start-karapace-docker-resources start-karapace-docker-resources: export KARAPACE_VERSION ?= 4.1.1.dev44+gac20eeed.d20241205 start-karapace-docker-resources: @@ -121,15 +125,15 @@ unit-tests-in-docker: start-karapace-docker-resources .PHONY: e2e-tests-in-docker e2e-tests-in-docker: export PYTEST_ARGS ?= -e2e-tests-in-docker: start-karapace-docker-resources +e2e-tests-in-docker: stop-karapace-docker-resources start-karapace-docker-resources rm -fr runtime/* sleep 10 - $(KARAPACE-CLI) $(PYTHON) -m pytest -s -vvv $(PYTEST_ARGS) tests/e2e/test_karapace.py + $(KARAPACE-CLI) $(PYTHON) -m pytest -s -vvv $(PYTEST_ARGS) tests/e2e/ rm -fr runtime/* .PHONY: integration-tests-in-docker integration-tests-in-docker: export PYTEST_ARGS ?= -integration-tests-in-docker: start-karapace-docker-resources +integration-tests-in-docker: stop-karapace-docker-resources start-karapace-docker-resources rm -fr runtime/* sleep 10 $(KARAPACE-CLI) $(PYTHON) -m pytest -s -vvv $(PYTEST_ARGS) tests/integration/ diff --git a/container/Dockerfile.dev b/container/Dockerfile.dev index 37a6cb5ab..c5827994c 100644 --- a/container/Dockerfile.dev +++ b/container/Dockerfile.dev @@ -1,5 +1,7 @@ +ARG PYTHON_VERSION + # Current versions of avro and zstandard don't yet have wheels for 3.11. -FROM python:3.10.11-bullseye AS builder +FROM python:${PYTHON_VERSION}-bullseye AS builder ARG KARAPACE_VERSION ARG RUNNER_UID diff --git a/container/compose.yml b/container/compose.yml index 4b7d8728b..30d823632 100644 --- a/container/compose.yml +++ b/container/compose.yml @@ -124,6 +124,7 @@ services: dockerfile: container/Dockerfile.dev args: KARAPACE_VERSION: $KARAPACE_VERSION + PYTHON_VERSION: $PYTHON_VERSION RUNNER_UID: $RUNNER_UID RUNNER_GID: $RUNNER_GID tty: true diff --git a/mypy.ini b/mypy.ini index 981e4061c..14b7bc0aa 100644 --- a/mypy.ini +++ b/mypy.ini @@ -1,6 +1,6 @@ [mypy] mypy_path = $MYPY_CONFIG_FILE_DIR/stubs -python_version = 3.9 +python_version = 3.10 packages = karapace show_error_codes = True pretty = True diff --git a/pyproject.toml b/pyproject.toml index 9df5adf5c..18e068985 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "karapace" -requires-python = ">= 3.9" +requires-python = ">= 3.10" dynamic = ["version"] readme = "README.rst" license = {file = "LICENSE"} @@ -56,7 +56,6 @@ classifiers=[ "Intended Audience :: Information Technology", "Intended Audience :: System Administrators", "License :: OSI Approved :: Apache Software License", - "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", "Programming Language :: Python :: 3.12", @@ -114,5 +113,5 @@ include-package-data = true version_file = "src/karapace/version.py" [tool.black] -target-version = ["py39"] +target-version = ["py310", "py311", "py312"] line-length = 125 diff --git a/requirements/requirements-dev.txt b/requirements/requirements-dev.txt index bb95b5fc9..00c6d4d4c 100644 --- a/requirements/requirements-dev.txt +++ b/requirements/requirements-dev.txt @@ -1,5 +1,5 @@ # -# This file is autogenerated by pip-compile with Python 3.9 +# This file is autogenerated by pip-compile with Python 3.10 # by the following command: # # make pin-requirements @@ -119,9 +119,9 @@ httpcore==1.0.7 # via httpx httptools==0.6.4 # via uvicorn -httpx==0.28.0 +httpx==0.28.1 # via fastapi -hypothesis==6.122.1 +hypothesis==6.122.3 # via karapace (/karapace/pyproject.toml) idna==3.10 # via @@ -131,9 +131,7 @@ idna==3.10 # requests # yarl importlib-metadata==8.5.0 - # via - # flask - # opentelemetry-api + # via opentelemetry-api iniconfig==2.0.0 # via pytest isodate==0.7.2 @@ -166,7 +164,7 @@ multidict==6.1.0 # via # aiohttp # yarl -networkx==3.2.1 +networkx==3.4.2 # via karapace (/karapace/pyproject.toml) opentelemetry-api==1.28.2 # via @@ -314,7 +312,6 @@ typing-extensions==4.12.2 # pydantic # pydantic-core # rich-toolkit - # starlette # typer # uvicorn ujson==5.10.0 diff --git a/requirements/requirements-typing.txt b/requirements/requirements-typing.txt index eec3f3d32..bcda10c84 100644 --- a/requirements/requirements-typing.txt +++ b/requirements/requirements-typing.txt @@ -1,5 +1,5 @@ # -# This file is autogenerated by pip-compile with Python 3.9 +# This file is autogenerated by pip-compile with Python 3.10 # by the following command: # # make pin-requirements @@ -78,7 +78,7 @@ httpcore==1.0.7 # via httpx httptools==0.6.4 # via uvicorn -httpx==0.28.0 +httpx==0.28.1 # via fastapi idna==3.10 # via @@ -112,7 +112,7 @@ mypy==1.13.0 # via karapace (/karapace/pyproject.toml) mypy-extensions==1.0.0 # via mypy -networkx==3.2.1 +networkx==3.4.2 # via karapace (/karapace/pyproject.toml) opentelemetry-api==1.28.2 # via @@ -211,7 +211,7 @@ typer==0.15.1 # via fastapi-cli types-cachetools==5.5.0.20240820 # via karapace (/karapace/pyproject.toml) -types-jsonschema==4.23.0.20240813 +types-jsonschema==4.23.0.20241208 # via karapace (/karapace/pyproject.toml) types-protobuf==3.20.4.6 # via karapace (/karapace/pyproject.toml) @@ -227,7 +227,6 @@ typing-extensions==4.12.2 # pydantic # pydantic-core # rich-toolkit - # starlette # typer # uvicorn ujson==5.10.0 diff --git a/requirements/requirements.txt b/requirements/requirements.txt index 4a32fdcd3..edf3064a4 100644 --- a/requirements/requirements.txt +++ b/requirements/requirements.txt @@ -1,5 +1,5 @@ # -# This file is autogenerated by pip-compile with Python 3.9 +# This file is autogenerated by pip-compile with Python 3.10 # by the following command: # # make pin-requirements @@ -77,7 +77,7 @@ httpcore==1.0.7 # via httpx httptools==0.6.4 # via uvicorn -httpx==0.28.0 +httpx==0.28.1 # via fastapi idna==3.10 # via @@ -107,7 +107,7 @@ multidict==6.1.0 # via # aiohttp # yarl -networkx==3.2.1 +networkx==3.4.2 # via karapace (/karapace/pyproject.toml) opentelemetry-api==1.28.2 # via @@ -210,7 +210,6 @@ typing-extensions==4.12.2 # pydantic # pydantic-core # rich-toolkit - # starlette # typer # uvicorn ujson==5.10.0 diff --git a/website/README.rst b/website/README.rst index c333ba578..9b24823fc 100644 --- a/website/README.rst +++ b/website/README.rst @@ -6,7 +6,7 @@ A static HTML site, generated with Sphinx. You can find the website source in th Dependencies ------------ -You need Python 3.9+. Install the dependencies with ``pip``:: +You need Python 3.10+. Install the dependencies with ``pip``:: pip install -r requirements.txt