feat: e2e tests within docker

- we split out e2e tests into own folder

- e2e tests will run against the service in docker compose

- we start by splitting out the prometheus and kafka e2e tests which do not need local running services

.coveragerc

@@ -1,4 +1,8 @@
 [run]
 branch = true
 relative_files = true
-source = src/karapace
+source = src
+disable_warnings = module-not-measured, no-data-collected
+
+[report]
+skip_empty = true

.dockerignore

@@ -10,7 +10,6 @@
 !LICENSE
 !pyproject.toml
 !setup.py
-!container/start.sh
 !container/healthcheck.py
 
 # Ignore some files in source directories.

.github/workflows/container-smoke-test.yml

@@ -26,14 +26,22 @@ jobs:
           KARAPACE_VERSION=$(python -c "from karapace import version; print(version.__version__)")
           echo KARAPACE_VERSION=$KARAPACE_VERSION >> $GITHUB_ENV
 
-      - name: Build container
-        run: docker build --build-arg KARAPACE_VERSION=${{ env.KARAPACE_VERSION }} --file=container/Dockerfile .
+      - run: echo "RUNNER_UID=$(id -u)" >> $GITHUB_ENV
+      - run: echo "RUNNER_GID=$(id -g)" >> $GITHUB_ENV
 
       - name: Run container
-        run: docker compose --file=container/compose.yml up --build --wait --detach
+        run: make start-karapace-docker-resources
+        env:
+          KARAPACE_VERSION: ${{ env.KARAPACE_VERSION }}
+          RUNNER_UID: ${{ env.RUNNER_UID }}
+          RUNNER_GID: ${{ env.RUNNER_GID }}
 
-      - name: Smoke test registry
-        run: bin/smoke-test-registry.sh
+      - name: Smoke test schema registry
+        run: bin/smoke-test-schema-registry.sh
+        env:
+          KARAPACE_PORT: 8081
 
       - name: Smoke test REST proxy
-        run: bin/smoke-test-rest.sh
+        run: bin/smoke-test-rest-proxy.sh
+        env:
+          KARAPACE_PORT: 8082

.github/workflows/tests.yml

@@ -14,14 +14,13 @@ env:
   FORCE_COLOR: 1
   PIP_PROGRESS_BAR: off
   PYTHONUNBUFFERED: 1
-  KARAPACE_DOTENV: ${{ github.workspace }}/karapace.config.env
 
 jobs:
-  unit-tests:
+  tests:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python-version: [ '3.10', '3.11', '3.12' ]
+        python-version: [ '3.9', '3.10', '3.11', '3.12' ]
     env:
       PYTEST_ADDOPTS: >-
         --log-dir=/tmp/ci-logs
@@ -41,53 +40,35 @@ jobs:
       with:
         go-version: '1.21.0'
 
-    - name: Install requirements
-      run: make install
-
     - name: Resolve Karapace version
-      run: |
-        source ./venv/bin/activate
-        KARAPACE_VERSION=$(python -c "from karapace import version; print(version.__version__)")
-        echo KARAPACE_VERSION=$KARAPACE_VERSION >> $GITHUB_ENV
+      run: echo KARAPACE_VERSION=4.1.1.dev44+gac20eeed.d20241205 >> $GITHUB_ENV
 
-    - name: Run containers
-      run: KARAPACE_VERSION=${{ env.KARAPACE_VERSION }} docker compose --file=container/compose.yml up --build --wait --detach
+    - run: echo "RUNNER_UID=$(id -u)" >> $GITHUB_ENV
+    - run: echo "RUNNER_GID=$(id -g)" >> $GITHUB_ENV
 
-    - run: make install-dev
     - run: make unit-tests-in-docker
       env:
-        COVERAGE_FILE: ".coverage.${{ matrix.python-version }}"
-        PYTEST_ARGS: "--cov=src --cov-append --numprocesses 4"
-        KARAPACE_VERSION=: ${{ env.KARAPACE_VERSION }}
-
-  integration-tests:
-    runs-on: ubuntu-latest
-    strategy:
-      matrix:
-        python-version: [ '3.10', '3.11', '3.12' ]
-    env:
-      PYTEST_ADDOPTS: >-
-        --log-dir=/tmp/ci-logs
-        --log-file=/tmp/ci-logs/pytest.log
-        --showlocals
-    steps:
-    - uses: actions/checkout@v4
-
-    - name: Set up Python ${{ matrix.python-version }}
-      uses: actions/setup-python@v5
-      with:
-        cache: pip
-        python-version: ${{ matrix.python-version }}
+        KARAPACE_VERSION: ${{ env.KARAPACE_VERSION }}
+        RUNNER_UID: ${{ env.RUNNER_UID }}
+        RUNNER_GID: ${{ env.RUNNER_GID }}
+        COVERAGE_FILE: "/opt/karapace/coverage/.coverage.${{ matrix.python-version }}"
+        PYTEST_ARGS: "--cov=karapace --cov=schema_registry --cov-append --numprocesses 4"
 
-    - name: Set up Go
-      uses: actions/setup-go@v5
-      with:
-        go-version: '1.21.0'
+    - run: make e2e-tests-in-docker
+      env:
+        KARAPACE_VERSION: ${{ env.KARAPACE_VERSION }}
+        RUNNER_UID: ${{ env.RUNNER_UID }}
+        RUNNER_GID: ${{ env.RUNNER_GID }}
+        COVERAGE_FILE: "/opt/karapace/coverage/.coverage.${{ matrix.python-version }}"
+        PYTEST_ARGS: "--cov=karapace --cov=schema_registry --cov-append --numprocesses 4"
 
-    - run: make integration-tests
+    - run: make integration-tests-in-docker
       env:
-        COVERAGE_FILE: ".coverage.${{ matrix.python-version }}"
-        PYTEST_ARGS: "--cov=src --cov-append --random-order --numprocesses 4"
+        KARAPACE_VERSION: ${{ env.KARAPACE_VERSION }}
+        RUNNER_UID: ${{ env.RUNNER_UID }}
+        RUNNER_GID: ${{ env.RUNNER_GID }}
+        COVERAGE_FILE: "/opt/karapace/coverage/.coverage.${{ matrix.python-version }}"
+        PYTEST_ARGS: "--cov=karapace --cov=schema_registry --cov-append --random-order --numprocesses 4"
 
     - name: Archive logs
       uses: actions/upload-artifact@v4

.gitignore

@@ -10,12 +10,13 @@
 __pycache__/
 /build/
 /dist/
-/karapace.egg-info/
+src/karapace.egg-info/
 /karapace-rpm-src.tar
 /kafka_*.tgz
 /kafka_*/
 venv
-/karapace/version.py
+*.so
+src/karapace/version.py
 .run
 .python-version
 .hypothesis/

GNUmakefile

@@ -3,9 +3,9 @@ SHELL := /usr/bin/env bash
 VENV_DIR ?= $(CURDIR)/venv
 PIP      ?= pip3 --disable-pip-version-check --no-input --require-virtualenv
 PYTHON   ?= python3
-PYTHON_VERSION ?= 3.10
+PYTHON_VERSION ?= 3.9
 DOCKER_COMPOSE ?= docker compose
-KARAPACE-CLI   ?= $(DOCKER_COMPOSE) -f container/compose.yml run karapace-cli
+KARAPACE-CLI   ?= $(DOCKER_COMPOSE) -f container/compose.yml run --rm karapace-cli
 
 define PIN_VERSIONS_COMMAND
 pip install pip-tools && \
@@ -105,9 +105,40 @@ schema:
 pin-requirements:
 	docker run -e CUSTOM_COMPILE_COMMAND='make pin-requirements' -it -v .:/karapace --security-opt label=disable python:$(PYTHON_VERSION)-bullseye /bin/bash -c "$(PIN_VERSIONS_COMMAND)"
 
+.PHONY: start-karapace-docker-resources
+start-karapace-docker-resources: export KARAPACE_VERSION ?= 4.1.1.dev44+gac20eeed.d20241205
+start-karapace-docker-resources:
+	sudo touch .coverage.3.9 .coverage.3.10 .coverage.3.11 .coverage.3.12
+	sudo chown ${RUNNER_UID}:${RUNNER_GID} .coverage.3.9 .coverage.3.10 .coverage.3.11 .coverage.3.12
+	$(DOCKER_COMPOSE) -f container/compose.yml up -d --build --wait --detach
+
 .PHONY: unit-tests-in-docker
 unit-tests-in-docker: export PYTEST_ARGS ?=
-unit-tests-in-docker:
+unit-tests-in-docker: start-karapace-docker-resources
 	rm -fr runtime/*
 	$(KARAPACE-CLI) $(PYTHON) -m pytest -s -vvv $(PYTEST_ARGS) tests/unit/
 	rm -fr runtime/*
+
+.PHONY: e2e-tests-in-docker
+e2e-tests-in-docker: export PYTEST_ARGS ?=
+e2e-tests-in-docker: start-karapace-docker-resources
+	rm -fr runtime/*
+	sleep 10
+	$(KARAPACE-CLI) $(PYTHON) -m pytest -s -vvv $(PYTEST_ARGS) tests/e2e/test_karapace.py
+	rm -fr runtime/*
+
+.PHONY: integration-tests-in-docker
+integration-tests-in-docker: export PYTEST_ARGS ?=
+integration-tests-in-docker: start-karapace-docker-resources
+	rm -fr runtime/*
+	sleep 10
+	$(KARAPACE-CLI) $(PYTHON) -m pytest -s -vvv $(PYTEST_ARGS) tests/integration/
+	rm -fr runtime/*
+
+.PHONY: type-check-mypy-in-docker
+type-check-mypy-in-docker: start-karapace-docker-resources
+	$(KARAPACE-CLI) $(PYTHON) -m mypy src
+
+.PHONY: cli
+cli: start-karapace-docker-resources
+	$(KARAPACE-CLI) bash

bin/smoke-test-rest.sh → bin/smoke-test-rest-proxy.sh

@@ -3,7 +3,7 @@
 retries=5
 
 for ((i = 0; i <= retries; i++)); do
-    response=$(curl --silent --verbose --fail http://localhost:8082/topics)
+    response=$(curl --silent --verbose --fail "http://localhost:$KARAPACE_PORT/topics")
 
     if [[ $response == '["_schemas","__consumer_offsets"]' ]]; then
         echo "Ok!"

bin/smoke-test-registry.sh → bin/smoke-test-schema-registry.sh

@@ -6,9 +6,8 @@ for ((i = 0; i <= retries; i++)); do
     response=$(
         curl --silent --verbose --fail --request POST \
             --header 'Content-Type: application/vnd.schemaregistry.v1+json' \
-            --header 'Authorization: Basic Og==' \
             --data '{"schema": "{\"type\": \"record\", \"name\": \"Obj\", \"fields\":[{\"name\": \"age\", \"type\": \"int\"}]}"}' \
-            http://localhost:8081/subjects/test-key/versions
+            "http://localhost:$KARAPACE_PORT/subjects/test-key/versions"
     )
 
     if [[ $response == '{"id":1}' ]]; then

container/Dockerfile

@@ -55,10 +55,6 @@ RUN apt-get update \
 COPY --from=builder /venv /venv
 ENV PATH="/venv/bin:$PATH"
 
-COPY ./container/start.sh /opt/karapace
-RUN chmod 500 /opt/karapace/start.sh \
- && chown karapace:karapace /opt/karapace/start.sh
-
 COPY ./container/healthcheck.py /opt/karapace
 
 WORKDIR /opt/karapace

container/Dockerfile.dev

@@ -2,10 +2,17 @@
 FROM python:3.10.11-bullseye AS builder
 
 ARG KARAPACE_VERSION
+ARG RUNNER_UID
+ARG RUNNER_GID
+
+# Setup files and directories.
+RUN mkdir /opt/karapace /opt/karapace/runtime /var/log/karapace /opt/karapace/coverage \
+&& touch /opt/karapace/coverage/.coverage.3.9 /opt/karapace/coverage/.coverage.3.10 /opt/karapace/coverage/.coverage.3.11 /opt/karapace/coverage/.coverage.3.12 \
+&& chown --recursive "$RUNNER_UID:$RUNNER_GID" /opt/karapace /opt/karapace/coverage /var/log/karapace
 
 # Create, activate, and enforce usage of virtualenv.
-RUN python3 -m venv /venv
-ENV PATH="/venv/bin:$PATH"
+RUN python3 -m venv /opt/karapace/venv
+ENV PATH="/opt/karapace/venv/bin:$PATH"
 ENV PIP_REQUIRE_VIRTUALENV=true
 
 # Install golang needed by extensions
@@ -15,48 +22,24 @@ RUN wget --progress=dot:giga "https://go.dev/dl/go${GO_VERSION}.linux-$(dpkg --p
  && tar -C /usr/local -xzf "go${GO_VERSION}.linux-$(dpkg --print-architecture).tar.gz" \
  && rm "go${GO_VERSION}.linux-$(dpkg --print-architecture).tar.gz"
 
-# Copy the requirements.txt and install dependencies in venv. Using a separate
-# command to use layer caching.
-#
-# Note: the requirements.txt is pinned, if any of the dependencies is updated
-# the cache will be invalidated and the image regenerated, which is the
-# intended behavior.
-COPY ./requirements/requirements.txt /build/
-COPY ./requirements/requirements-dev.txt /build/
-RUN --mount=type=cache,target=/root/.cache/pip \
-    python3 -m pip install -r /build/requirements.txt -r /build/requirements-dev.txt
-
-COPY . /build/karapace-repo
-WORKDIR /build/karapace-repo
-RUN --mount=type=cache,target=/root/.cache/pip \
-    if [ -z "${KARAPACE_VERSION}" ]; then \
-        PRETEND_VERSION="$(python -c 'from src.karapace import version; print(version.__version__)')"; \
-    else \
-        PRETEND_VERSION=$KARAPACE_VERSION; \
-    fi; \
-    SETUPTOOLS_SCM_PRETEND_VERSION=$PRETEND_VERSION python3 -m pip install --no-deps .
-
-# Karapace image, i.e. production.
-FROM python:3.10.11-slim-bullseye AS karapace
-
-# Setup user and directories.
-RUN groupadd --system karapace \
- && useradd --system --gid karapace karapace \
- && mkdir /opt/karapace /opt/karapace/runtime /var/log/karapace \
- && chown --recursive karapace:karapace /opt/karapace /var/log/karapace
-
 # Install protobuf compiler.
 ARG PROTOBUF_COMPILER_VERSION="3.12.4-1+deb11u1"
 RUN apt-get update \
  && apt-get install --assume-yes --no-install-recommends \
     protobuf-compiler=$PROTOBUF_COMPILER_VERSION \
  && rm -rf /var/lib/apt/lists/*
 
-# Copy virtualenv from builder and activate it.
-COPY --from=builder /venv /venv
-ENV PATH="/venv/bin:$PATH"
-
-COPY ./container/healthcheck.py /opt/karapace
+# Install Java via openjdk-11
+COPY --from=openjdk:11 /usr/local/openjdk-11 /usr/local/openjdk-11
+ENV JAVA_HOME /usr/local/openjdk-11
+RUN update-alternatives --install /usr/bin/java java /usr/local/openjdk-11/bin/java 1
 
 WORKDIR /opt/karapace
-USER karapace
+
+COPY ./requirements /opt/karapace/requirements
+RUN python3 -m pip install -r /opt/karapace/requirements/requirements.txt -r /opt/karapace/requirements/requirements-dev.txt -r /opt/karapace/requirements/requirements-typing.txt
+
+COPY . .
+RUN SETUPTOOLS_SCM_PRETEND_VERSION=$KARAPACE_VERSION python3 -m pip install .
+
+ENV PYTHONPATH="/opt/karapace/src:$PYTHONPATH"